char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenfit: also take into account n_streams (#27496)2fb989b9e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.6 KiB78 linesraw
1#include "llama.h"
2#include "../src/llama-ext.h"
3
4#include "arg.h"
5#include "common.h"
6#include "fit.h"
7#include "log.h"
8
9#include <cinttypes>
10
11#if defined(_MSC_VER)
12#pragma warning(disable: 4244 4267) // possible loss of data
13#endif
14
15// satisfies -Wmissing-declarations
16int llama_fit_params(int argc, char ** argv);
17
18int llama_fit_params(int argc, char ** argv) {
19    common_params params;
20
21    common_init();
22
23    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_FIT_PARAMS)) {
24        return 1;
25    }
26
27    llama_backend_init();
28    llama_numa_init(params.numa);
29
30    auto mparams = common_model_params_to_llama(params);
31    auto cparams = common_context_params_to_llama(params);
32
33    if (!params.fit_params_print) {
34        const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams,
35                params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx,
36                nullptr,
37                params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
38        if (status != COMMON_PARAMS_FIT_STATUS_SUCCESS) {
39            LOG_ERR("%s: failed to fit CLI arguments to free memory, exiting...\n", __func__);
40            exit(1);
41        }
42
43        LOG_INF("%s: printing fitted CLI arguments to stdout...\n", __func__);
44        common_log_flush(common_log_main());
45        printf("-c %" PRIu32 " -ngl %" PRIi32, cparams.n_ctx, mparams.n_gpu_layers);
46
47        size_t nd = llama_max_devices();
48        while (nd > 1 && mparams.tensor_split[nd - 1] == 0.0f) {
49            nd--;
50        }
51        if (nd > 1) {
52            for (size_t id = 0; id < nd; id++) {
53                if (id == 0) {
54                    printf(" -ts ");
55                }
56                printf("%s%" PRIu32, id > 0 ? "," : "", uint32_t(mparams.tensor_split[id]));
57            }
58        }
59
60        const size_t ntbo = llama_max_tensor_buft_overrides();
61        bool any_tbo = false;
62        for (size_t itbo = 0; itbo < ntbo && mparams.tensor_buft_overrides[itbo].pattern != nullptr; itbo++) {
63            if (itbo == 0) {
64                printf(" -ot \"");
65            }
66            printf("%s%s=%s", itbo > 0 ? "," : "", mparams.tensor_buft_overrides[itbo].pattern, ggml_backend_buft_name(mparams.tensor_buft_overrides[itbo].buft));
67            any_tbo = true;
68        }
69        printf("%s\n", any_tbo ? "\"" : "");
70    } else {
71        LOG_INF("%s: printing estimated memory in MiB to stdout (device, model, context, compute) ...\n", __func__);
72        common_log_flush(common_log_main());
73
74        common_fit_print(params.model.path.c_str(), &mparams, &cparams);
75    }
76
77    return 0;
78}