char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyencommon: add json.h abstraction (#27511)d9f918d2d

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
19.6 KiB506 linesraw
1// tests the HF model resolution and the model handler assembly end-to-end on
2// synthetic repo listings: a local httplib server bound to the loopback
3// serves hardcoded HF API responses, so the real client, hf_cache, resolution
4// and CLI parsing run against them without external network access
5
6#include "arg.h"
7#include "common.h"
8#include "download.h"
9#include "http.h"
10#include "log.h"
11
12#include "json.h"
13
14#include <algorithm>
15#include <cstdio>
16#include <cstdlib>
17#include <filesystem>
18#include <map>
19#include <thread>
20#include <string>
21#include <vector>
22
23// the case and reordering being checked, printed with every failure
24static std::string g_context;
25
26// independent of NDEBUG, so the checks stay alive in Release builds
27#define REQUIRE(x) do {                                                         \
28    if (!(x)) {                                                                 \
29        fprintf(stderr, "%s:%d: [%s] REQUIRE(%s) failed\n",                     \
30                __FILE__, __LINE__, g_context.c_str(), #x);                     \
31        std::abort();                                                           \
32    }                                                                           \
33} while (0)
34
35#define REQUIRE_EQ(actual, expected) do {                                       \
36    if (!((actual) == (expected))) {                                            \
37        fprintf(stderr, "%s:%d: [%s] REQUIRE_EQ(%s, %s) failed\n  actual:   '%s'\n  expected: '%s'\n", \
38                __FILE__, __LINE__, g_context.c_str(), #actual, #expected,      \
39                std::string(actual).c_str(), std::string(expected).c_str());    \
40        std::abort();                                                           \
41    }                                                                           \
42} while (0)
43
44//
45// synthetic repos keyed by repo id, served over the loopback by a real
46// httplib server, so the tested code runs its own client and transport
47//
48
49static std::map<std::string, std::vector<std::string>> g_repos;
50
51static const char * COMMIT = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
52
53// the server lives in main, so its destructor runs before the static teardown
54// tears down the winsock state httplib brings in
55static void serve_repos(httplib::Server & server) {
56    server.Get(R"(/api/models/(.+)/refs)", [](const httplib::Request & req, httplib::Response & res) {
57        if (g_repos.count(req.matches[1])) {
58            res.set_content(common_json{{"branches", common_json::array({ common_json{{"name", "main"}, {"targetCommit", COMMIT}} })}}.dump(),
59                            "application/json");
60        } else {
61            res.status = 404;
62        }
63    });
64    server.Get(R"(/api/models/(.+)/tree/.+)", [](const httplib::Request & req, httplib::Response & res) {
65        if (!g_repos.count(req.matches[1])) {
66            res.status = 404;
67            return;
68        }
69        auto files = common_json::array();
70        size_t i = 0;
71        for (const auto & p : g_repos[req.matches[1]]) {
72            char oid[41];
73            snprintf(oid, sizeof(oid), "%040lx", (unsigned long) ++i);
74            files.push_back({{"type", "file"}, {"path", p}, {"size", 1}, {"oid", oid}});
75        }
76        res.set_content(files.dump(), "application/json");
77    });
78}
79
80static common_params_model model_ref(const std::string & hf_repo, const std::string & hf_file = "") {
81    common_params_model m;
82    m.hf_repo = hf_repo;
83    m.hf_file = hf_file;
84    return m;
85}
86
87// the model cache is isolated under a temporary directory named after the
88// loopback port, so concurrent runs on a shared machine keep their own, and
89// the local path the handler wires for a file is snapshots/<commit>/<path>
90static std::filesystem::path cache_dir;
91
92static std::string cached(std::string repo_id, const std::string & path) {
93    string_replace_all(repo_id, "/", "--");
94    return (cache_dir / ("models--" + repo_id) / "snapshots" / COMMIT / path).string();
95}
96
97//
98// fixtures mimicking real repo layouts
99//
100
101// flat layout in the style of ggml-org/gemma-4-31B-it-GGUF
102static const std::vector<std::string> flat = {
103    "README.md",
104    "model-BF16.gguf",
105    "model-Q4_K_M.gguf",
106    "model-Q8_0.gguf",
107    "mmproj-model-BF16.gguf",
108    "mmproj-model-Q8_0.gguf",
109    "mtp-model-BF16.gguf",
110    "mtp-model-Q4_0.gguf",
111    "mtp-model-Q8_0.gguf",
112    "dflash-model-BF16.gguf",
113    "dflash-model-Q8_0.gguf",
114};
115
116// quants in subdirectories with sharded files and root sidecars,
117// in the style of stepfun-ai/Step-3.7-Flash-GGUF
118static const std::vector<std::string> subdir = {
119    "mmproj-model-f16.gguf",
120    "model-mtp-BF16.gguf",
121    "model-mtp-Q8_0.gguf",
122    "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",
123    "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf",
124    "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf",
125    "Q8_0/model-Q8_0-00001-of-00002.gguf",
126    "Q8_0/model-Q8_0-00002-of-00002.gguf",
127};
128
129// sidecar quants exist where the full model quant does not,
130// in the style of ggml-org/Qwen3.6-27B-GGUF
131static const std::vector<std::string> hole = {
132    "model-BF16.gguf",
133    "model-Q4_K_M.gguf",
134    "model-Q8_0.gguf",
135    "mtp-model-BF16.gguf",
136    "mtp-model-Q4_0.gguf",
137    "mtp-model-Q8_0.gguf",
138    "dflash-model-BF16.gguf",
139    "dflash-model-Q8_0.gguf",
140};
141
142// unsloth-style naming with UD quants and a suffix MTP file
143static const std::vector<std::string> unsloth = {
144    "model-UD-Q8_K_XL.gguf",
145    "mmproj-BF16.gguf",
146    "model-MTP-BF16.gguf",
147};
148
149// bartowski-style vendor prefix and mradermacher-style dot quant
150static const std::vector<std::string> vendors = {
151    "TheDrummer_Model-24B-v4.1-Q8_0.gguf",
152    "BlackSheep-24B.Q8_0.gguf",
153};
154
155// every speculative sidecar type at the same quant
156static const std::vector<std::string> quad = {
157    "model-Q8_0.gguf",
158    "mtp-model-Q8_0.gguf",
159    "dflash-model-Q8_0.gguf",
160    "eagle3-model-Q8_0.gguf",
161    "dspark-model-Q8_0.gguf",
162};
163
164static const std::vector<std::string> dflash_only = {
165    "model-Q8_0.gguf",
166    "dflash-model-Q8_0.gguf",
167};
168
169static const std::vector<std::string> eagle3_only = {
170    "model-Q8_0.gguf",
171    "eagle3-model-Q8_0.gguf",
172};
173
174// a single full quant with dspark sidecars at other quants,
175// in the style of ggml-org/DeepSeek-V4-Flash-0731-GGUF
176static const std::vector<std::string> spark = {
177    "README.md",
178    "model-MXFP4.gguf",
179    "dspark-model-BF16.gguf",
180    "dspark-model-MXFP4.gguf",
181};
182
183// dspark outranks dflash in the type auto-selection
184static const std::vector<std::string> dspark_dflash = {
185    "model-Q8_0.gguf",
186    "dflash-model-Q8_0.gguf",
187    "dspark-model-Q8_0.gguf",
188};
189
190//
191// table-driven plan resolution through the real entry point,
192// each case replayed on multiple deterministic reorderings of the listing,
193// except the cases whose pick legitimately depends on the listing order
194//
195
196struct plan_case {
197    const char * name;
198    const std::vector<std::string> files;
199    const char * hf_repo;
200    const char * hf_file;
201    bool sidecars;        // request mmproj + mtp + dflash + eagle3 + dspark
202    bool order_dependent; // the expected pick depends on the listing order
203    const char * primary;
204    std::vector<std::string> model_files;
205    const char * mmproj;
206    const char * mtp;
207    const char * dflash;
208    const char * eagle3;
209    const char * dspark;
210};
211
212static const plan_case plan_cases[] = {
213    // exact tag picks the matching primary, sidecars follow the tag
214    {"flat exact tag", flat, "test/repo:Q8_0", "", true, false,
215     "model-Q8_0.gguf", {"model-Q8_0.gguf"},
216     "mmproj-model-Q8_0.gguf", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "", ""},
217
218    // no tag falls back to the default quant preference
219    {"flat default", flat, "test/repo", "", false, false,
220     "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"},
221     "", "", "", "", ""},
222
223    // no tag and no default match falls back to the first model in the listing
224    {"unsloth fallback", unsloth, "test/repo", "", true, true,
225     "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"},
226     "mmproj-BF16.gguf", "", "", "", ""},
227
228    // explicit hf_file picks that exact file
229    {"flat hf_file", flat, "test/repo", "model-BF16.gguf", false, false,
230     "model-BF16.gguf", {"model-BF16.gguf"},
231     "", "", "", "", ""},
232
233    // missing hf_file resolves nothing
234    {"flat missing hf_file", flat, "test/repo", "nope.gguf", false, false,
235     "", {},
236     "", "", "", "", ""},
237
238    // a sharded primary brings all its parts, a subdir primary finds the root sidecar
239    {"subdir shards", subdir, "test/repo:Q3_K_M", "", true, false,
240     "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",
241     {"Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",
242      "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf",
243      "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf"},
244     "mmproj-model-f16.gguf", "model-mtp-Q8_0.gguf", "", "", ""},
245
246    // a tag with no matching full model still resolves the requested sidecars
247    {"hole tag sidecar", hole, "test/repo:Q4_0", "", true, false,
248     "", {},
249     "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""},
250
251    // the same tag without a requested sidecar resolves nothing
252    {"hole tag alone", hole, "test/repo:Q4_0", "", false, false,
253     "", {},
254     "", "", "", "", ""},
255
256    // no tag anchors the sidecars on the primary quant
257    {"hole default anchor", hole, "test/repo", "", true, false,
258     "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"},
259     "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""},
260
261    // the mtp- keyword is case sensitive, a suffix -MTP file is not discovered
262    {"unsloth suffix mtp", unsloth, "test/repo:Q8_K_XL", "", true, false,
263     "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"},
264     "mmproj-BF16.gguf", "", "", "", ""},
265
266    // vendor prefixes and the dot quant convention both match the tag,
267    // first match wins between two files at the same quant
268    {"vendor prefix", vendors, "test/repo:Q8_0", "", false, true,
269     "TheDrummer_Model-24B-v4.1-Q8_0.gguf", {"TheDrummer_Model-24B-v4.1-Q8_0.gguf"},
270     "", "", "", "", ""},
271
272    // every sidecar type resolves at the tag
273    {"quad exact tag", quad, "test/repo:Q8_0", "", true, false,
274     "model-Q8_0.gguf", {"model-Q8_0.gguf"},
275     "", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "eagle3-model-Q8_0.gguf", "dspark-model-Q8_0.gguf"},
276
277    // no tag anchors the dspark sidecar on the only full quant
278    {"spark default anchor", spark, "test/repo", "", true, false,
279     "model-MXFP4.gguf", {"model-MXFP4.gguf"},
280     "", "", "", "", "dspark-model-MXFP4.gguf"},
281
282    // a tag with no matching full model still resolves the exact dspark sidecar
283    {"spark tag sidecar", spark, "test/repo:BF16", "", true, false,
284     "", {},
285     "", "", "", "", "dspark-model-BF16.gguf"},
286};
287
288static void check_plan(const plan_case & c) {
289    common_download_opts opts;
290    opts.download_mmproj = c.sidecars;
291    opts.download_mtp    = c.sidecars;
292    opts.download_dflash = c.sidecars;
293    opts.download_eagle3 = c.sidecars;
294    opts.download_dspark = c.sidecars;
295
296    auto plan = common_download_get_hf_plan(model_ref(c.hf_repo, c.hf_file), opts);
297
298    REQUIRE_EQ(plan.primary.path, c.primary);
299    REQUIRE_EQ(plan.mmproj.path,  c.mmproj);
300    REQUIRE_EQ(plan.mtp.path,     c.mtp);
301    REQUIRE_EQ(plan.dflash.path,  c.dflash);
302    REQUIRE_EQ(plan.eagle3.path,  c.eagle3);
303    REQUIRE_EQ(plan.dspark.path,  c.dspark);
304
305    // exact shard set, order insensitive; the primary must be the first split
306    std::vector<std::string> actual;
307    for (const auto & f : plan.model_files) {
308        actual.push_back(f.path);
309    }
310    std::sort(actual.begin(), actual.end());
311    auto expected = c.model_files;
312    std::sort(expected.begin(), expected.end());
313    REQUIRE(actual == expected);
314    if (!expected.empty()) {
315        REQUIRE(plan.primary.path == expected.front());
316    }
317}
318
319static void test_plan_resolution() {
320    printf("test-model-resolution: plan resolution on %zu cases\n", sizeof(plan_cases) / sizeof(plan_cases[0]));
321
322    for (const auto & c : plan_cases) {
323        printf("  %s\n", c.name);
324        // invariant: the resolution is insensitive to the listing order
325        for (size_t rot = 0; rot < c.files.size(); ++rot) {
326            if (c.order_dependent && rot > 0) {
327                continue;
328            }
329            g_context = std::string(c.name) + ", reordering " + std::to_string(rot);
330            auto files = c.files;
331            std::rotate(files.begin(), files.begin() + rot, files.end());
332            if (rot % 2 == 1) {
333                std::reverse(files.begin(), files.end());
334            }
335            g_repos["test/repo"] = files;
336            check_plan(c);
337        }
338    }
339    g_repos.clear();
340}
341
342//
343// end-to-end assembly: real CLI parsing, real handler init resolving over the
344// loopback, downloads skipped by flipping offline before apply
345//
346
347static void assemble(std::vector<std::string> argv, common_params & params) {
348    std::vector<char *> cargv;
349    g_context.clear();
350    for (auto & a : argv) {
351        g_context += g_context.empty() ? a : " " + a;
352        cargv.push_back(a.data());
353    }
354    bool ok = common_params_parse((int) cargv.size(), cargv.data(), params, LLAMA_EXAMPLE_SERVER);
355    REQUIRE(ok);
356
357    auto handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
358
359    // skip the network execution, on_done still wires the params
360    params.offline = true;
361    common_models_handler_apply(handler, params);
362}
363
364static void test_task_assembly() {
365    printf("test-model-resolution: end-to-end assembly\n");
366
367    g_repos["test/main"]   = flat;
368    g_repos["test/hole"]   = hole;
369    g_repos["test/quad"]   = quad;
370    g_repos["test/dflash"] = dflash_only;
371    g_repos["test/eagle3"] = eagle3_only;
372    g_repos["test/spark"]  = spark;
373    g_repos["test/pair"]   = dspark_dflash;
374    g_repos["test/small"]  = {"draft-model-Q4_K_M.gguf"};
375    g_repos["test/preset"] = {"preset.ini", "model-Q8_0.gguf"};
376
377    {
378        // plain -hf wires the model and its mmproj, nothing speculative
379        common_params params;
380        assemble({"server", "-hf", "test/main:Q8_0"}, params);
381        REQUIRE_EQ(params.model.path,  cached("test/main", "model-Q8_0.gguf"));
382        REQUIRE_EQ(params.mmproj.path, cached("test/main", "mmproj-model-Q8_0.gguf"));
383        REQUIRE(params.speculative.draft.mparams.path.empty());
384    }
385    {
386        // --no-mmproj disables the mmproj discovery
387        common_params params;
388        assemble({"server", "-hf", "test/main:Q8_0", "--no-mmproj"}, params);
389        REQUIRE(params.mmproj.path.empty());
390    }
391    {
392        // an explicit --mmproj wins over the discovery
393        common_params params;
394        assemble({"server", "-hf", "test/main:Q8_0", "--mmproj", "/local/mmproj.gguf"}, params);
395        REQUIRE(params.mmproj.path == "/local/mmproj.gguf");
396    }
397    {
398        // -hf with a spec type wires the sidecar of the main repo as fallback draft
399        common_params params;
400        assemble({"server", "-hf", "test/main:Q8_0", "--spec-type", "draft-mtp"}, params);
401        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/main", "mtp-model-Q8_0.gguf"));
402    }
403    {
404        // -hfd with a spec type wires the draft repo sidecar at its tag,
405        // not its full model, and suppresses the main repo fallback
406        common_params params;
407        assemble({"server", "-hf", "test/hole:Q8_0", "-hfd", "test/hole:Q4_0", "--spec-type", "draft-mtp"}, params);
408        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/hole", "mtp-model-Q4_0.gguf"));
409    }
410    {
411        // an explicit -md file wins over the sidecar resolution
412        common_params params;
413        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/main", "-md", "mtp-model-BF16.gguf", "--spec-type", "draft-mtp"}, params);
414        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/main", "mtp-model-BF16.gguf"));
415    }
416    {
417        // -hfd without a spec type auto-selects the type, mtp first when all ship
418        common_params params;
419        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/quad:Q8_0"}, params);
420        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_MTP});
421        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/quad", "mtp-model-Q8_0.gguf"));
422    }
423    {
424        // auto-selection with only a dflash sidecar
425        common_params params;
426        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/dflash:Q8_0"}, params);
427        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH});
428        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/dflash", "dflash-model-Q8_0.gguf"));
429    }
430    {
431        // auto-selection with only an eagle3 sidecar
432        common_params params;
433        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/eagle3:Q8_0"}, params);
434        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3});
435        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/eagle3", "eagle3-model-Q8_0.gguf"));
436    }
437    {
438        // auto-selection prefers dspark over dflash when both ship
439        common_params params;
440        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/pair:Q8_0"}, params);
441        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK});
442        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/pair", "dspark-model-Q8_0.gguf"));
443    }
444    {
445        // -hf with the dspark spec type wires the sidecar of the main repo,
446        // anchored on the only full quant
447        common_params params;
448        assemble({"server", "-hf", "test/spark", "--spec-type", "draft-dspark"}, params);
449        REQUIRE_EQ(params.model.path, cached("test/spark", "model-MXFP4.gguf"));
450        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/spark", "dspark-model-MXFP4.gguf"));
451    }
452    {
453        // -hfd on a repo without sidecars keeps resolving a full model as draft
454        common_params params;
455        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/small"}, params);
456        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_NONE});
457        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/small", "draft-model-Q4_K_M.gguf"));
458    }
459    {
460        // a preset repo wires the preset and clears the model for router mode
461        common_params params;
462        assemble({"server", "-hf", "test/preset"}, params);
463        REQUIRE_EQ(params.models_preset, cached("test/preset", "preset.ini"));
464        REQUIRE(params.model.path.empty());
465        REQUIRE(params.model.hf_repo.empty());
466    }
467
468    g_repos.clear();
469}
470
471int main(void) {
472    // unbuffered, so a crash cannot swallow the reports already printed
473    setvbuf(stdout, nullptr, _IONBF, 0);
474    setvbuf(stderr, nullptr, _IONBF, 0);
475
476    // the negative cases legitimately log errors on every reordering,
477    // keep the output down to the reports
478    common_log_pause(common_log_main());
479
480    // the loopback endpoint also keeps the client init from rejecting
481    // https on the builds without TLS support
482    httplib::Server server;
483    serve_repos(server);
484    int port = server.bind_to_any_port("127.0.0.1");
485
486    // isolate the cache, its location is read once so it is set
487    // before anything else
488    cache_dir = std::filesystem::temp_directory_path() /
489                ("test-model-resolution-cache-" + std::to_string(port));
490    std::filesystem::remove_all(cache_dir);
491    common_set_env("LLAMA_CACHE", cache_dir.string());
492
493    std::thread server_thread([&server] { server.listen_after_bind(); });
494    server.wait_until_ready();
495    common_set_env("MODEL_ENDPOINT", "http://127.0.0.1:" + std::to_string(port) + "/");
496
497    test_plan_resolution();
498    test_task_assembly();
499
500    server.stop();
501    server_thread.join();
502
503    std::filesystem::remove_all(cache_dir);
504    printf("test-model-resolution: all tests OK\n");
505    return 0;
506}