char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Kevin Pougetggml-virtgpu: improve the reliability of the code (#19846)ffaafde16

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
5.0 KiB144 linesraw
1#include "backend-dispatched.h"
2#include "backend-virgl-apir.h"
3#include "shared/api_remoting.h"
4#include "shared/apir_backend.h"
5#include "shared/apir_cs.h"
6
7#include <dlfcn.h>
8#include <ggml-backend.h>
9
10#include <iostream>
11
12#define APIR_LLAMA_CPP_GGML_LIBRARY_PATH_ENV "APIR_LLAMA_CPP_GGML_LIBRARY_PATH"
13#define APIR_LLAMA_CPP_GGML_LIBRARY_REG_ENV  "APIR_LLAMA_CPP_GGML_LIBRARY_REG"
14#define APIR_LLAMA_CPP_LOG_TO_FILE_ENV       "APIR_LLAMA_CPP_LOG_TO_FILE"
15
16#define GGML_DEFAULT_BACKEND_REG "ggml_backend_init"
17
18static void * backend_library_handle = NULL;
19static FILE * apir_logfile           = NULL;
20
21static void log_to_file_callback(enum ggml_log_level level, const char * text, void * user_data) {
22    FILE * logfile = (FILE *) user_data;
23    fprintf(logfile, "[%d] %s", level, text);
24    fflush(logfile);
25}
26
27extern "C" {
28void apir_backend_deinit(uint32_t virgl_ctx_id) {
29    GGML_UNUSED(virgl_ctx_id);
30
31    auto buffers = apir_get_track_backend_buffers();
32    for (const auto & buffer : buffers) {
33        apir_untrack_backend_buffer(buffer);
34        buffer->iface.free_buffer(buffer);
35    }
36
37    if (backend_library_handle) {
38        GGML_LOG_INFO(GGML_VIRTGPU_BCK "The GGML backend library was loaded. Unloading it.\n");
39        dlclose(backend_library_handle);
40        backend_library_handle = NULL;
41    }
42
43    if (apir_logfile) {
44        fclose(apir_logfile);
45        apir_logfile = NULL;
46    }
47}
48
49#define APIR_GGML_LIBRARY_PATH_KEY "ggml.library.path"
50#define APIR_GGML_LIBRARY_REG_KEY  "ggml.library.reg"
51
52ApirLoadLibraryReturnCode apir_backend_initialize(uint32_t virgl_ctx_id, struct virgl_apir_callbacks * virgl_cbs) {
53    const char * dlsym_error;
54
55    const char * apir_log_to_file = getenv(APIR_LLAMA_CPP_LOG_TO_FILE_ENV);
56    if (apir_log_to_file) {
57        apir_logfile = fopen(apir_log_to_file, "w");
58        if (apir_logfile) {
59            ggml_log_set(log_to_file_callback, apir_logfile);
60        } else {
61            GGML_LOG_INFO(GGML_VIRTGPU_BCK "Could not open the log file at '%s'\n", apir_log_to_file);
62        }
63    }
64
65    const char * library_name      = virgl_cbs->get_config(virgl_ctx_id, APIR_GGML_LIBRARY_PATH_KEY);
66    const char * virgl_library_reg = virgl_cbs->get_config(virgl_ctx_id, APIR_GGML_LIBRARY_REG_KEY);
67    const char * library_reg       = virgl_library_reg ? virgl_library_reg : GGML_DEFAULT_BACKEND_REG;
68
69    if (!library_name) {
70        GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: cannot open the GGML library: env var '%s' not defined\n", __func__,
71                       APIR_LLAMA_CPP_GGML_LIBRARY_PATH_ENV);
72
73        return APIR_LOAD_LIBRARY_ENV_VAR_MISSING;
74    }
75
76    backend_library_handle = dlopen(library_name, RTLD_LAZY);
77
78    if (!backend_library_handle) {
79        GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: cannot open the GGML library: %s\n", __func__, dlerror());
80
81        return APIR_LOAD_LIBRARY_CANNOT_OPEN;
82    }
83
84    if (!library_reg) {
85        GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: cannot register the GGML library: env var '%s' not defined\n", __func__,
86                       APIR_LLAMA_CPP_GGML_LIBRARY_REG_ENV);
87
88        return APIR_LOAD_LIBRARY_ENV_VAR_MISSING;
89    }
90
91    void * ggml_backend_reg_fct = dlsym(backend_library_handle, library_reg);
92    dlsym_error                 = dlerror();
93    if (dlsym_error) {
94        GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: cannot find the GGML backend registration symbol '%s' (from %s): %s\n",
95                       __func__, library_reg, APIR_LLAMA_CPP_GGML_LIBRARY_REG_ENV, dlsym_error);
96
97        return APIR_LOAD_LIBRARY_SYMBOL_MISSING;
98    }
99
100    uint32_t ret = backend_dispatch_initialize(ggml_backend_reg_fct);
101
102    return (ApirLoadLibraryReturnCode) (APIR_LOAD_LIBRARY_INIT_BASE_INDEX + ret);
103}
104
105uint32_t apir_backend_dispatcher(uint32_t               virgl_ctx_id,
106                                 virgl_apir_callbacks * virgl_cbs,
107                                 uint32_t               cmd_type,
108                                 char *                 dec_cur,
109                                 const char *           dec_end,
110                                 char *                 enc_cur,
111                                 const char *           enc_end,
112                                 char **                enc_cur_after) {
113    apir_encoder enc = {
114        .cur   = enc_cur,
115        .start = enc_cur,
116        .end   = enc_end,
117        .fatal = false,
118    };
119
120    apir_decoder dec = {
121        .cur   = dec_cur,
122        .end   = dec_end,
123        .fatal = false,
124    };
125
126    virgl_apir_context ctx = {
127        .ctx_id = virgl_ctx_id,
128        .iface  = virgl_cbs,
129    };
130
131    if (cmd_type >= APIR_BACKEND_DISPATCH_TABLE_COUNT) {
132        GGML_LOG_ERROR(GGML_VIRTGPU_BCK "%s: Received an invalid dispatch index (%d >= %d)\n", __func__, cmd_type,
133                       APIR_BACKEND_DISPATCH_TABLE_COUNT);
134        return APIR_BACKEND_FORWARD_INDEX_INVALID;
135    }
136
137    backend_dispatch_t forward_fct = apir_backend_dispatch_table[cmd_type];
138    uint32_t           ret         = forward_fct(&enc, &dec, &ctx);
139
140    *enc_cur_after = enc.cur;
141
142    return ret;
143}
144}