char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovspec : refactor params (#22397)14e733e36

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.2 KiB45 linesraw
1#include "arg.h"
2#include "common.h"
3#include "ngram-cache.h"
4#include "llama.h"
5
6#include <clocale>
7#include <string>
8#include <vector>
9
10int main(int argc, char ** argv){
11    std::setlocale(LC_NUMERIC, "C");
12
13    common_params params;
14
15    common_init();
16
17    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_LOOKUP)) {
18        return 1;
19    }
20
21    // init llama.cpp
22    llama_backend_init();
23    llama_numa_init(params.numa);
24
25    // load the model
26    auto llama_init = common_init_from_params(params);
27
28    auto * model = llama_init->model();
29    auto * ctx = llama_init->context();
30
31    GGML_ASSERT(model != nullptr);
32
33    // tokenize the prompt
34    std::vector<llama_token> inp;
35    inp = common_tokenize(ctx, params.prompt, true, true);
36    fprintf(stderr, "%s: tokenization done\n", __func__);
37
38    common_ngram_cache ngram_cache;
39    common_ngram_cache_update(ngram_cache, LLAMA_NGRAM_STATIC, LLAMA_NGRAM_STATIC, inp, inp.size(), true);
40    fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.speculative.ngram_cache.lookup_cache_static.c_str());
41
42    common_ngram_cache_save(ngram_cache, params.speculative.ngram_cache.lookup_cache_static);
43
44    return 0;
45}