char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskycommon: fix missing exports in llama-common (#22340)5594d1322

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.1 KiB88 linesraw
1#include "arg.h"
2#include "common.h"
3#include "debug.h"
4#include "log.h"
5#include "llama.h"
6
7#include <clocale>
8#include <string>
9#include <vector>
10
11static bool run(llama_context * ctx, const common_params & params) {
12    const llama_model * model = llama_get_model(ctx);
13    const llama_vocab * vocab = llama_model_get_vocab(model);
14
15    const bool add_bos = llama_vocab_get_add_bos(vocab);
16
17    std::vector<llama_token> tokens = common_tokenize(ctx, params.prompt, add_bos, true);
18
19    if (tokens.empty()) {
20        LOG_ERR("%s : there are not input tokens to process - (try to provide a prompt with '-p')\n", __func__);
21        return false;
22    }
23
24    LOG_INF("number of input tokens = %zu\n", tokens.size());
25    for (size_t i = 0; i < tokens.size(); ++i) {
26        LOG_INF("  %d\n", tokens[i]);
27    }
28
29    if (llama_decode(ctx, llama_batch_get_one(tokens.data(), tokens.size()))) {
30        LOG_ERR("%s : failed to eval\n", __func__);
31        return false;
32    }
33
34    return true;
35}
36
37int main(int argc, char ** argv) {
38    std::setlocale(LC_NUMERIC, "C");
39
40    common_debug_cb_user_data cb_data;
41
42    common_params params;
43
44    common_init();
45
46    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
47        return 1;
48    }
49
50    llama_backend_init();
51    llama_numa_init(params.numa);
52
53    // pass the callback to the backend scheduler
54    // it will be executed for each node during the graph computation
55    params.cb_eval = common_debug_cb_eval;
56    params.cb_eval_user_data = &cb_data;
57    params.warmup = false;
58
59    // init
60    auto llama_init = common_init_from_params(params);
61
62    auto * model = llama_init->model();
63    auto * ctx   = llama_init->context();
64
65    if (model == nullptr || ctx == nullptr) {
66        LOG_ERR("%s : failed to init\n", __func__);
67        return 1;
68    }
69
70    // print system information
71    {
72        LOG_INF("\n");
73        LOG_INF("%s\n", common_params_get_system_info(params).c_str());
74        LOG_INF("\n");
75    }
76
77    bool OK = run(ctx, params);
78    if (!OK) {
79        return 1;
80    }
81
82    LOG("\n");
83    llama_perf_context_print(ctx);
84
85    llama_backend_free();
86
87    return 0;
88}