char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenserver: add dedup-cache-models preset option (#27346)ee4c505a4

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.2 KiB154 linesraw
1#pragma once
2
3#include "common.h"
4#include "download.h"
5
6#include <set>
7#include <map>
8#include <string>
9#include <vector>
10#include <cstring>
11#include <memory>
12
13// pseudo-env variable to identify preset-only arguments
14#define COMMON_ARG_PRESET_LOAD_ON_STARTUP    "__PRESET_LOAD_ON_STARTUP"
15#define COMMON_ARG_PRESET_STOP_TIMEOUT       "__PRESET_STOP_TIMEOUT"
16#define COMMON_ARG_PRESET_DEDUP_CACHE_MODELS "__PRESET_DEDUP_CACHE_MODELS"
17
18//
19// CLI argument parsing
20//
21
22struct common_arg {
23    std::set<enum llama_example> examples = {LLAMA_EXAMPLE_COMMON};
24    std::set<enum llama_example> excludes = {};
25    std::vector<const char *> args;
26    std::vector<const char *> args_neg;  // for negated args like --no-xxx
27    const char * value_hint   = nullptr; // help text or example for arg value
28    const char * value_hint_2 = nullptr; // for second arg value
29    const char * env          = nullptr;
30    std::string help;
31    bool is_sampling = false; // is current arg a sampling param?
32    bool is_spec = false; // is current arg a speculative decoding param?
33    bool is_preset_only = false; // is current arg preset-only (not treated as CLI arg)
34    void (*handler_void)   (common_params & params) = nullptr;
35    void (*handler_string) (common_params & params, const std::string &) = nullptr;
36    void (*handler_str_str)(common_params & params, const std::string &, const std::string &) = nullptr;
37    void (*handler_int)    (common_params & params, int) = nullptr;
38    void (*handler_bool)   (common_params & params, bool) = nullptr;
39
40    common_arg() = default;
41
42    common_arg(
43        const std::initializer_list<const char *> & args,
44        const char * value_hint,
45        const std::string & help,
46        void (*handler)(common_params & params, const std::string &)
47    ) : args(args), value_hint(value_hint), help(help), handler_string(handler) {}
48
49    common_arg(
50        const std::initializer_list<const char *> & args,
51        const char * value_hint,
52        const std::string & help,
53        void (*handler)(common_params & params, int)
54    ) : args(args), value_hint(value_hint), help(help), handler_int(handler) {}
55
56    common_arg(
57        const std::initializer_list<const char *> & args,
58        const std::string & help,
59        void (*handler)(common_params & params)
60    ) : args(args), help(help), handler_void(handler) {}
61
62    common_arg(
63        const std::initializer_list<const char *> & args,
64        const std::initializer_list<const char *> & args_neg,
65        const std::string & help,
66        void (*handler)(common_params & params, bool)
67    ) : args(args), args_neg(args_neg), help(help), handler_bool(handler) {}
68
69    // support 2 values for arg
70    common_arg(
71        const std::initializer_list<const char *> & args,
72        const char * value_hint,
73        const char * value_hint_2,
74        const std::string & help,
75        void (*handler)(common_params & params, const std::string &, const std::string &)
76    ) : args(args), value_hint(value_hint), value_hint_2(value_hint_2), help(help), handler_str_str(handler) {}
77
78    common_arg & set_examples(std::initializer_list<enum llama_example> examples);
79    common_arg & set_excludes(std::initializer_list<enum llama_example> excludes);
80    common_arg & set_env(const char * env);
81    common_arg & set_sampling();
82    common_arg & set_spec();
83    common_arg & set_preset_only();
84    bool in_example(enum llama_example ex);
85    bool is_exclude(enum llama_example ex);
86    bool get_value_from_env(std::string & output) const;
87    bool has_value_from_env() const;
88    std::string to_string() const;
89
90    // for using as key in std::map
91    bool operator<(const common_arg& other) const {
92        if (args.empty() || other.args.empty()) {
93            return false;
94        }
95        return strcmp(args[0], other.args[0]) < 0;
96    }
97    bool operator==(const common_arg& other) const {
98        if (args.empty() || other.args.empty()) {
99            return false;
100        }
101        return strcmp(args[0], other.args[0]) == 0;
102    }
103
104    // get all args and env vars (including negated args/env)
105    std::vector<std::string> get_args() const;
106    std::vector<std::string> get_env() const;
107};
108
109namespace common_arg_utils {
110    bool is_truthy(const std::string & value);
111    bool is_falsey(const std::string & value);
112    bool is_autoy(const std::string & value);
113}
114
115struct common_params_context {
116    enum llama_example ex = LLAMA_EXAMPLE_COMMON;
117    common_params & params;
118    std::vector<common_arg> options;
119    void(*print_usage)(int, char **) = nullptr;
120    common_params_context(common_params & params) : params(params) {}
121};
122
123// parse input arguments from CLI
124// if one argument has invalid value, it will automatically display usage of the specific argument (and not the full usage message)
125bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);
126
127// load all backends and print the list of available (non-CPU) devices to stdout
128void common_print_available_devices();
129
130// parse input arguments from CLI into a map
131bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<common_arg, std::string> & out_map);
132
133// populate preset-only arguments
134// these arguments are not treated as command line arguments
135// see: https://github.com/ggml-org/llama.cpp/issues/18163
136void common_params_add_preset_options(std::vector<common_arg> & args);
137
138struct common_models_handler {
139    common_download_hf_plan plan;
140    common_download_hf_plan plan_spec;
141    common_download_opts opts;
142};
143
144// initialize downloading opts and hf_plan if needed, but does not download anything yet
145common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex);
146
147// check if the model is a preset repo (i.e. has a preset file)
148bool common_models_handler_is_preset_repo(const common_models_handler & handler);
149
150// download and update params with the downloaded model path
151void common_models_handler_apply(common_models_handler & handler, common_params & params, common_download_callback * callback = nullptr);
152
153// initialize argument parser context - used by test-arg-parser and preset
154common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);