char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Daniel Beveniusmodel-conversion : add causal-convert-mmproj target [no ci] (#22969)89730c8d2

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
8.8 KiB239 linesraw
1MAKEFLAGS += --no-print-directory
2
3define validate_model_path
4	@if [ -z "$(MODEL_PATH)" ]; then \
5		echo "Error: MODEL_PATH must be provided either as:"; \
6		echo "  1. Environment variable: export MODEL_PATH=/path/to/model"; \
7		echo "  2. Command line argument: make $(1) MODEL_PATH=/path/to/model"; \
8		exit 1; \
9	fi
10endef
11
12define validate_embedding_model_path
13	@if [ -z "$(EMBEDDING_MODEL_PATH)" ]; then \
14		echo "Error: EMBEDDING_MODEL_PATH must be provided either as:"; \
15		echo "  1. Environment variable: export EMBEDDING_MODEL_PATH=/path/to/model"; \
16		echo "  2. Command line argument: make $(1) EMBEDDING_MODEL_PATH=/path/to/model"; \
17		exit 1; \
18	fi
19endef
20
21define quantize_model
22	@CONVERTED_MODEL="$(1)" QUANTIZED_TYPE="$(QUANTIZED_TYPE)" \
23	TOKEN_EMBD_TYPE="$(TOKEN_EMBD_TYPE)" OUTPUT_TYPE="$(OUTPUT_TYPE)" \
24	./scripts/utils/quantize.sh "$(1)" "$(QUANTIZED_TYPE)" "$(TOKEN_EMBD_TYPE)" "$(OUTPUT_TYPE)"
25	@echo "Export the quantized model path to $(2) variable in your environment"
26endef
27
28DEVICE ?= auto
29
30###
31### Casual Model targets/recipes
32###
33causal-convert-model-bf16: OUTTYPE=bf16
34causal-convert-model-bf16: causal-convert-model
35
36causal-convert-model-debug: DEBUG=--debug
37causal-convert-model-debug: causal-convert-model
38
39causal-convert-model:
40	$(call validate_model_path,causal-convert-model)
41	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \
42	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
43	./scripts/causal/convert-model.sh $(DEBUG)
44
45causal-convert-mm-model-bf16: OUTTYPE=bf16
46causal-convert-mm-model-bf16: MM_OUTTYPE=f16
47causal-convert-mm-model-bf16: causal-convert-mm-model
48
49causal-convert-mm-model:
50	$(call validate_model_path,causal-convert-mm-model)
51	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \
52	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
53	./scripts/causal/convert-model.sh
54
55	$(MAKE) causal-convert-mmproj MM_OUTTYPE="$(MM_OUTTYPE)"
56
57causal-convert-mmproj:
58	$(call validate_model_path,causal-convert-mmproj)
59	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(MM_OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \
60	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
61	./scripts/causal/convert-model.sh --mmproj
62
63causal-run-original-model:
64	$(call validate_model_path,causal-run-original-model)
65	@MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/run-org-model.py --device "$(DEVICE)"
66
67causal-run-converted-model:
68	@CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/causal/run-converted-model.sh
69
70causal-verify-logits: causal-run-original-model causal-run-converted-model
71	@MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/compare-logits.py
72	@MODEL_PATH="$(MODEL_PATH)" ./scripts/utils/check-nmse.py -m ${MODEL_PATH}
73
74causal-run-original-embeddings:
75	@./scripts/causal/run-casual-gen-embeddings-org.py
76
77causal-run-converted-embeddings:
78	@./scripts/causal/run-converted-model-embeddings-logits.sh
79
80causal-verify-embeddings: causal-run-original-embeddings causal-run-converted-embeddings
81	@./scripts/causal/compare-embeddings-logits.sh
82
83causal-inspect-original-model:
84	@./scripts/utils/inspect-org-model.py --list-all -s
85
86causal-list-original-model-tensors:
87	@./scripts/utils/inspect-org-model.py --list-all-short -s
88
89causal-inspect-converted-model:
90	@./scripts/utils/inspect-converted-model.sh
91
92causal-start-embedding-server:
93	@./scripts/utils/run-embedding-server.sh ${CONVERTED_MODEL}
94
95causal-curl-embedding-endpoint: causal-run-original-embeddings
96	@./scripts/utils/curl-embedding-server.sh | ./scripts/causal/compare-embeddings-logits.sh
97
98causal-quantize-Q8_0: QUANTIZED_TYPE = Q8_0
99causal-quantize-Q8_0: causal-quantize-model
100
101causal-quantize-Q4_0: QUANTIZED_TYPE = Q4_0
102causal-quantize-Q4_0: causal-quantize-model
103
104# For Quantization Aware Trained (QAT) models in Q4_0 we explicitly set the
105# token embedding and output types to Q8_0 instead of the default Q6_K.
106causal-quantize-qat-Q4_0: QUANTIZED_TYPE = Q4_0
107causal-quantize-qat-Q4_0: TOKEN_EMBD_TYPE = Q8_0
108causal-quantize-qat-Q4_0: OUTPUT_TYPE = Q8_0
109causal-quantize-qat-Q4_0: causal-quantize-model
110
111causal-quantize-model:
112	$(call quantize_model,$(CONVERTED_MODEL),QUANTIZED_MODEL)
113
114causal-run-quantized-model:
115	@QUANTIZED_MODEL="$(QUANTIZED_MODEL)" ./scripts/causal/run-converted-model.sh ${QUANTIZED_MODEL}
116
117
118###
119### Embedding Model targets/recipes
120###
121
122embedding-convert-model-bf16: OUTTYPE=bf16
123embedding-convert-model-bf16: embedding-convert-model
124
125embedding-convert-model:
126	$(call validate_embedding_model_path,embedding-convert-model)
127	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(EMBEDDING_MODEL_PATH)" \
128	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
129	./scripts/embedding/convert-model.sh
130
131embedding-convert-model-st:
132	$(call validate_embedding_model_path,embedding-convert-model-st)
133	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(EMBEDDING_MODEL_PATH)" \
134	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
135	./scripts/embedding/convert-model.sh -st
136
137embedding-run-original-model:
138	$(call validate_embedding_model_path,embedding-run-original-model)
139	@EMBEDDING_MODEL_PATH="$(EMBEDDING_MODEL_PATH)" \
140	USE_SENTENCE_TRANSFORMERS="$(USE_SENTENCE_TRANSFORMERS)" \
141	./scripts/embedding/run-original-model.py \
142	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)") \
143	$(if $(USE_SENTENCE_TRANSFORMERS),--use-sentence-transformers)
144
145embedding-run-original-model-st: USE_SENTENCE_TRANSFORMERS=1
146embedding-run-original-model-st: embedding-run-original-model
147
148embedding-run-converted-model:
149	@./scripts/embedding/run-converted-model.sh $(CONVERTED_EMBEDDING_MODEL) \
150	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)") \
151	$(if $(EMBD_NORMALIZE),--embd-normalize "$(EMBD_NORMALIZE)")
152
153embedding-verify-logits: embedding-run-original-model embedding-run-converted-model
154	@./scripts/embedding/compare-embeddings-logits.sh \
155	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)")
156
157embedding-verify-logits-st: embedding-run-original-model-st embedding-run-converted-model
158	@./scripts/embedding/compare-embeddings-logits.sh \
159	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)")
160
161embedding-inspect-original-model:
162	$(call validate_embedding_model_path,embedding-inspect-original-model)
163	@EMBEDDING_MODEL_PATH="$(EMBEDDING_MODEL_PATH)" ./scripts/utils/inspect-org-model.py -m ${EMBEDDING_MODEL_PATH} --list-all -s
164
165embedding-inspect-converted-model:
166	@CONVERTED_EMBEDDING_MODEL="$(CONVERTED_EMBEDDING_MODEL)" ./scripts/utils/inspect-converted-model.sh ${CONVERTED_EMBEDDING_MODEL}
167
168embedding-start-embedding-server:
169	@./scripts/utils/run-embedding-server.sh ${CONVERTED_EMBEDDING_MODEL}
170
171embedding-curl-embedding-endpoint:
172	@./scripts/utils/curl-embedding-server.sh | ./scripts/embedding/compare-embeddings-logits.sh
173
174embedding-quantize-Q8_0: QUANTIZED_TYPE = Q8_0
175embedding-quantize-Q8_0: embedding-quantize-model
176
177embedding-quantize-Q4_0: QUANTIZED_TYPE = Q4_0
178embedding-quantize-Q4_0: embedding-quantize-model
179
180# For Quantization Aware Trained (QAT) models in Q4_0 we explicitly set the
181# token embedding and output types to Q8_0 instead of the default Q6_K.
182embedding-quantize-qat-Q4_0: QUANTIZED_TYPE = Q4_0
183embedding-quantize-qat-Q4_0: TOKEN_EMBD_TYPE = Q8_0
184embedding-quantize-qat-Q4_0: OUTPUT_TYPE = Q8_0
185embedding-quantize-qat-Q4_0: embedding-quantize-model
186
187embedding-quantize-model:
188	$(call quantize_model,$(CONVERTED_EMBEDDING_MODEL),QUANTIZED_EMBEDDING_MODEL)
189
190embedding-run-quantized-model:
191	@./scripts/embedding/run-converted-model.sh $(QUANTIZED_EMBEDDING_MODEL) \
192	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)")
193
194###
195### Perplexity targets/recipes
196###
197perplexity-data-gen:
198	CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/utils/perplexity-gen.sh
199
200perplexity-run-full:
201	QUANTIZED_MODEL="$(QUANTIZED_MODEL)" LOOGITS_FILE="$(LOGITS_FILE)" \
202	./scripts/utils/perplexity-run.sh
203
204perplexity-run:
205	QUANTIZED_MODEL="$(QUANTIZED_MODEL)" ./scripts/utils/perplexity-run-simple.sh
206
207###
208### HuggingFace targets/recipes
209###
210
211hf-create-model:
212	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}"
213
214hf-create-model-dry-run:
215	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -d
216
217hf-create-model-embedding:
218	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -e
219
220hf-create-model-embedding-dry-run:
221	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -e -d
222
223hf-create-model-private:
224	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -p
225
226hf-upload-gguf-to-model:
227	@./scripts/utils/hf-upload-gguf-model.py -m "${MODEL_PATH}" -r "${REPO_ID}" -o "${NAME_IN_REPO}"
228
229hf-create-collection:
230	@./scripts/utils/hf-create-collection.py -n "${NAME}" -d "${DESCRIPTION}" -ns "${NAMESPACE}"
231
232hf-add-model-to-collection:
233	@./scripts/utils/hf-add-model-to-collection.py -c "${COLLECTION}" -m "${MODEL}"
234
235
236.PHONY: clean
237clean:
238	@${RM} -rf data .converted_embedding_model.txt .converted_model.txt .embedding_model_name.txt .model_name.txt
239