hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Gaurav Gargspec: Add benchmark-only synthetic speculative acceptance options (#27711)2bb9bddaf2026-08-27

1#include "arg.h"
2#include "common.h"
3#include "download.h"
4#include "llama.h"
5#include "speculative.h"
6
7#include <cmath>
8#include <limits>
9#include <string>
10#include <vector>
11#include <sstream>
12#include <unordered_set>
13
14#undef NDEBUG
15#include <cassert>
16
17static void test(void) {
18    common_params params;
19
20    auto assert_output_limits = [](int32_t n_batch, int32_t n_parallel, int32_t n_draft,
21                                   int32_t total, int32_t per_seq) {
22        const auto limits = common_speculative_get_output_limits(n_batch, n_parallel, n_draft);
23        assert(limits.total == total);
24        assert(limits.per_seq == per_seq);
25    };
26
27    assert_output_limits(16, 2,  3, 8, 4);
28    assert_output_limits(16, 2, -1, 2, 1);
29    assert_output_limits( 6, 2,  3, 6, 4);
30    assert_output_limits( 2, 1,  3, 2, 2);
31    assert_output_limits(
32            std::numeric_limits<int32_t>::max(),
33            std::numeric_limits<int32_t>::max(),
34            std::numeric_limits<int32_t>::max(),
35            std::numeric_limits<int32_t>::max(),
36            std::numeric_limits<int32_t>::max());
37
38    {
39        common_params_speculative spec;
40        spec.synth_len = 3.4;
41
42        auto assert_invalid = [](const common_params_speculative & value, int32_t n_max) {
43            try {
44                common_speculative_synth_rates_resolve(&value, n_max);
45                assert(false);
46            } catch (const std::invalid_argument &) {
47            }
48        };
49
50        const auto rates = common_speculative_synth_rates_resolve(&spec, 4);
51        assert(rates.size() == 4);
52        assert(std::abs(rates[0] - 0.80581) < 1e-5);
53        assert(std::abs(rates[1] - 0.64933) < 1e-5);
54        assert(std::abs(rates[2] - 0.52323) < 1e-5);
55        assert(std::abs(rates[3] - 0.42163) < 1e-5);
56        assert(std::abs(1.0 + rates[0] + rates[1] + rates[2] + rates[3] - 3.4) < 1e-8);
57
58        spec.synth_len = 1.0;
59        assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({0.0, 0.0, 0.0, 0.0}));
60
61        spec.synth_len = 5.0;
62        assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({1.0, 1.0, 1.0, 1.0}));
63
64        spec.synth_len = 5.1;
65        assert_invalid(spec, 4);
66
67        spec.synth_len = std::numeric_limits<double>::quiet_NaN();
68        assert_invalid(spec, 4);
69
70        spec.synth_len = 0.0;
71        assert_invalid(spec, 4);
72
73        spec.synth_len = -1.0;
74        spec.synth_rates = {0.8, 0.6, 0.4};
75        assert_invalid(spec, 4);
76
77        spec.synth_rates = {0.8, 0.6, 0.4, 0.2};
78        assert(common_speculative_synth_rates_resolve(&spec, 4) == spec.synth_rates);
79
80        spec.synth_rates = {0.8, 0.9, 0.4, 0.2};
81        assert_invalid(spec, 4);
82
83        spec.synth_rates = {0.8, std::numeric_limits<double>::quiet_NaN(), 0.4, 0.2};
84        assert_invalid(spec, 4);
85
86        spec.synth_rates = {0.8, 0.6, 0.4, -0.2};
87        assert_invalid(spec, 4);
88
89        spec.synth_rates = {0.8, 0.6, 0.4, 0.2};
90        spec.synth_len = 3.0;
91        assert_invalid(spec, 4);
92    }
93
94    {
95        common_params base;
96        base.n_parallel = 4;
97        base.n_outputs_max_per_seq = 8;
98
99        const auto draft = common_base_params_to_speculative(base);
100        assert(draft.n_outputs_max == 4);
101        assert(draft.n_outputs_max_per_seq == 1);
102    }
103
104    printf("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n");
105    for (int ex = 0; ex < LLAMA_EXAMPLE_COUNT; ex++) {
106        try {
107            auto ctx_arg = common_params_parser_init(params, (enum llama_example)ex);
108            common_params_add_preset_options(ctx_arg.options);
109            std::unordered_set<std::string> seen_args;
110            std::unordered_set<std::string> seen_env_vars;
111            for (const auto & opt : ctx_arg.options) {
112                // check for args duplications
113                for (const auto & arg : opt.get_args()) {
114                    if (seen_args.find(arg) == seen_args.end()) {
115                        seen_args.insert(arg);
116                    } else {
117                        fprintf(stderr, "test-arg-parser: found different handlers for the same argument: %s", arg.c_str());
118                        exit(1);
119                    }
120                }
121                // check for env var duplications
122                for (const auto & env : opt.get_env()) {
123                    if (seen_env_vars.find(env) == seen_env_vars.end()) {
124                        seen_env_vars.insert(env);
125                    } else {
126                        fprintf(stderr, "test-arg-parser: found different handlers for the same env var: %s", env.c_str());
127                        exit(1);
128                    }
129                }
130
131                // exclude spec args from this check
132                // ref: https://github.com/ggml-org/llama.cpp/pull/22397
133                const bool skip = opt.is_spec;
134
135                // ensure shorter argument precedes longer argument
136                if (!skip && opt.args.size() > 1) {
137                    const std::string first(opt.args.front());
138                    const std::string last(opt.args.back());
139
140                    if (first.length() > last.length()) {
141                        fprintf(stderr, "test-arg-parser: shorter argument should come before longer one: %s, %s\n",
142                                first.c_str(), last.c_str());
143                        assert(false);
144                    }
145                }
146
147                // same check for negated arguments
148                if (opt.args_neg.size() > 1) {
149                    const std::string first(opt.args_neg.front());
150                    const std::string last(opt.args_neg.back());
151
152                    if (first.length() > last.length()) {
153                        fprintf(stderr, "test-arg-parser: shorter negated argument should come before longer one: %s, %s\n",
154                                first.c_str(), last.c_str());
155                        assert(false);
156                    }
157                }
158            }
159        } catch (std::exception & e) {
160            printf("%s\n", e.what());
161            assert(false);
162        }
163    }
164
165    auto list_str_to_char = [](std::vector<std::string> & argv) -> std::vector<char *> {
166        std::vector<char *> res;
167        for (auto & arg : argv) {
168            res.push_back(const_cast<char *>(arg.data()));
169        }
170        return res;
171    };
172
173    std::vector<std::string> argv;
174
175    printf("test-arg-parser: test invalid usage\n\n");
176
177    // missing value
178    argv = {"binary_name", "-m"};
179    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
180
181    // wrong value (int)
182    argv = {"binary_name", "-ngl", "hello"};
183    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
184
185    // wrong value (enum)
186    argv = {"binary_name", "-sm", "hello"};
187    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
188
189    {
190        common_params penalty_params;
191        assert(penalty_params.sampling.penalty_last_n == 64);
192        assert(penalty_params.sampling.dry_penalty_last_n == 64);
193
194        argv = {"binary_name", "--repeat-last-n", "-1"};
195        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
196
197        argv = {"binary_name", "--dry-penalty-last-n", "-1"};
198        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
199
200        argv = {"binary_name", "--repeat-penalty", "0"};
201        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
202
203        argv = {"binary_name", "--repeat-penalty", "-1"};
204        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
205
206        argv = {"binary_name", "--repeat-penalty", "nan"};
207        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
208
209        argv = {"binary_name", "--repeat-penalty", "inf"};
210        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
211
212        argv = {"binary_name", "--repeat-penalty", "-inf"};
213        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
214
215        const char * penalty_options[] = {"--frequency-penalty", "--presence-penalty"};
216        const char * nonfinite_values[] = {"nan", "inf", "-inf"};
217        for (const char * option : penalty_options) {
218            for (const char * value : nonfinite_values) {
219                argv = {"binary_name", option, value};
220                assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
221            }
222        }
223    }
224
225    // non-existence arg in specific example (--draft cannot be used outside llama-speculative)
226    argv = {"binary_name", "--draft", "123"};
227    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING));
228
229    argv = {"binary_name", "-lm", "hello"};
230    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
231
232    printf("test-arg-parser: test valid usage\n\n");
233
234    argv = {"binary_name", "-m", "model_file.gguf"};
235    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
236    assert(params.model.path == "model_file.gguf");
237
238    argv = {"binary_name", "-t", "1234"};
239    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
240    assert(params.cpuparams.n_threads == 1234);
241
242    argv = {"binary_name", "--verbose"};
243    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
244    assert(params.verbosity > 1);
245
246    argv = {"binary_name", "-m", "abc.gguf", "--predict", "6789", "--batch-size", "9090"};
247    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
248    assert(params.model.path == "abc.gguf");
249    assert(params.n_predict == 6789);
250    assert(params.n_batch == 9090);
251
252    // --draft cannot be used outside llama-speculative
253    argv = {"binary_name", "--spec-draft-n-max", "123"};
254    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));
255    assert(params.speculative.draft.n_max == 123);
256
257    {
258        common_params synth_params;
259        argv = {"binary_name", "--spec-synth-len", "3.4"};
260        assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
261        assert(synth_params.speculative.synth_len == 3.4);
262    }
263
264    {
265        common_params synth_params;
266        argv = {"binary_name", "--spec-synth-rates", "0.8,0.6,0.2"};
267        assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
268        assert(synth_params.speculative.synth_rates == std::vector<double>({0.8, 0.6, 0.2}));
269    }
270
271    {
272        common_params synth_params;
273        argv = {"binary_name", "--spec-synth-len", "3.4x"};
274        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
275    }
276
277    argv = {"binary_name", "-lm", "none"};
278    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
279    assert(params.load_mode == LLAMA_LOAD_MODE_NONE);
280
281    argv = {"binary_name", "-lm", "mmap"};
282    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
283    assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);
284
285    argv = {"binary_name", "-lm", "mlock"};
286    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
287    assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);
288
289    argv = {"binary_name", "-lm", "mmap+mlock"};
290    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
291    assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);
292
293    argv = {"binary_name", "-lm", "dio"};
294    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
295    assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);
296
297    // multi-value args (CSV)
298    argv = {"binary_name", "--lora", "file1.gguf,\"file2,2.gguf\",\"file3\"\"3\"\".gguf\",file4\".gguf"};
299    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
300    assert(params.lora_adapters.size() == 4);
301    assert(params.lora_adapters[0].path == "file1.gguf");
302    assert(params.lora_adapters[1].path == "file2,2.gguf");
303    assert(params.lora_adapters[2].path == "file3\"3\".gguf");
304    assert(params.lora_adapters[3].path == "file4\".gguf");
305
306// skip this part on windows, because setenv is not supported
307#ifdef _WIN32
308    printf("test-arg-parser: skip on windows build\n");
309#else
310    printf("test-arg-parser: test environment variables (valid + invalid usages)\n\n");
311
312    setenv("LLAMA_ARG_THREADS", "blah", true);
313    argv = {"binary_name"};
314    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
315
316    setenv("LLAMA_ARG_MODEL", "blah.gguf", true);
317    setenv("LLAMA_ARG_THREADS", "1010", true);
318    argv = {"binary_name"};
319    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
320    assert(params.model.path == "blah.gguf");
321    assert(params.cpuparams.n_threads == 1010);
322
323    setenv("LLAMA_ARG_LOAD_MODE", "blah", true);
324    argv = {"binary_name"};
325    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
326
327    setenv("LLAMA_ARG_LOAD_MODE", "mmap", true);
328    argv = {"binary_name"};
329    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
330    assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);
331
332    setenv("LLAMA_ARG_LOAD_MODE", "mlock", true);
333    argv = {"binary_name"};
334    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
335    assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);
336
337    setenv("LLAMA_ARG_LOAD_MODE", "mmap+mlock", true);
338    argv = {"binary_name"};
339    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
340    assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);
341
342    setenv("LLAMA_ARG_LOAD_MODE", "dio", true);
343    argv = {"binary_name"};
344    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
345    assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);
346
347    printf("test-arg-parser: test negated environment variables\n\n");
348
349    setenv("LLAMA_ARG_LOAD_MODE", "none", true);
350    setenv("LLAMA_ARG_NO_PERF", "1", true); // legacy format
351    argv = {"binary_name"};
352    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
353    assert(params.load_mode == LLAMA_LOAD_MODE_NONE);
354    assert(params.no_perf == true);
355
356    printf("test-arg-parser: test environment variables being overwritten\n\n");
357
358    setenv("LLAMA_ARG_MODEL", "blah.gguf", true);
359    setenv("LLAMA_ARG_THREADS", "1010", true);
360    argv = {"binary_name", "-m", "overwritten.gguf"};
361    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
362    assert(params.model.path == "overwritten.gguf");
363    assert(params.cpuparams.n_threads == 1010);
364#endif // _WIN32
365
366    printf("test-arg-parser: test download functions\n\n");
367    const char * GOOD_URL = "http://ggml.ai/";
368    const char * BAD_URL  = "http://ggml.ai/404";
369
370    {
371        printf("test-arg-parser: test good URL\n\n");
372        auto res = common_remote_get_content(GOOD_URL, {});
373        assert(res.first == 200);
374        assert(res.second.size() > 0);
375        std::string str(res.second.data(), res.second.size());
376        assert(str.find("llama.cpp") != std::string::npos);
377    }
378
379    {
380        printf("test-arg-parser: test bad URL\n\n");
381        auto res = common_remote_get_content(BAD_URL, {});
382        assert(res.first == 404);
383    }
384
385    {
386        printf("test-arg-parser: test max size error\n");
387        common_remote_params params;
388        params.max_size = 1;
389        try {
390            common_remote_get_content(GOOD_URL, params);
391            assert(false && "it should throw an error");
392        } catch (std::exception & e) {
393            printf("  expected error: %s\n\n", e.what());
394        }
395    }
396
397    printf("test-arg-parser: all tests OK\n\n");
398}
399
400int main(void) {
401    try {
402        test();
403    } catch (std::exception & e) {
404        fprintf(stderr, "test-arg-parser: exception: %s\n", e.what());
405        return 1;
406    }
407    return 0;
408}