char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Adrien Gallouëtggml : don't crash when backend search path can't be read (#28271)4cbe8b070

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
18.1 KiB605 linesraw
1#include "ggml-backend-impl.h"
2#include "ggml-backend.h"
3#include "ggml-backend-dl.h"
4#include "ggml-impl.h"
5#include <algorithm>
6#include <cstring>
7#include <filesystem>
8#include <memory>
9#include <string>
10#include <type_traits>
11#include <vector>
12#include <cctype>
13
14#ifdef _WIN32
15#    define WIN32_LEAN_AND_MEAN
16#    ifndef NOMINMAX
17#        define NOMINMAX
18#    endif
19#    include <windows.h>
20#elif defined(__APPLE__)
21#    include <mach-o/dyld.h>
22#    include <dlfcn.h>
23#else
24#    include <dlfcn.h>
25#    include <unistd.h>
26#endif
27
28// Backend registry
29#ifdef GGML_USE_CPU
30#include "ggml-cpu.h"
31#endif
32
33#ifdef GGML_USE_CUDA
34#include "ggml-cuda.h"
35#endif
36
37#ifdef GGML_USE_METAL
38#include "ggml-metal.h"
39#endif
40
41#ifdef GGML_USE_SYCL
42#include "ggml-sycl.h"
43#endif
44
45#ifdef GGML_USE_VULKAN
46#include "ggml-vulkan.h"
47#endif
48
49#ifdef GGML_USE_WEBGPU
50#include "ggml-webgpu.h"
51#endif
52
53#ifdef GGML_USE_ZDNN
54#include "ggml-zdnn.h"
55#endif
56
57#ifdef GGML_USE_OPENCL
58#include "ggml-opencl.h"
59#endif
60
61#ifdef GGML_USE_HEXAGON
62#include "ggml-hexagon.h"
63#endif
64
65#ifdef GGML_USE_BLAS
66#include "ggml-blas.h"
67#endif
68
69#ifdef GGML_USE_RPC
70#include "ggml-rpc.h"
71#endif
72
73#ifdef GGML_USE_VIRTGPU_FRONTEND
74#include "ggml-virtgpu.h"
75#endif
76
77#ifdef GGML_USE_CANN
78#include "ggml-cann.h"
79#endif
80
81#ifdef GGML_USE_ZENDNN
82#include "ggml-zendnn.h"
83#endif
84
85#ifdef GGML_USE_OPENVINO
86#include "ggml-openvino.h"
87#endif
88
89#ifdef GGML_USE_ET
90#include "ggml-et.h"
91#endif
92
93namespace fs = std::filesystem;
94
95static std::string path_str(const fs::path & path) {
96    try {
97#if defined(__cpp_lib_char8_t)
98        // C++20 and later: u8string() returns std::u8string
99        const std::u8string u8str = path.u8string();
100        return std::string(reinterpret_cast<const char *>(u8str.data()), u8str.size());
101#else
102        // C++17: u8string() returns std::string
103        return path.u8string();
104#endif
105    } catch (...) {
106        return std::string();
107    }
108}
109
110struct ggml_backend_reg_entry {
111    ggml_backend_reg_t reg;
112    dl_handle_ptr handle;
113};
114
115struct ggml_backend_registry {
116    std::vector<ggml_backend_reg_entry> backends;
117    std::vector<ggml_backend_dev_t> devices;
118
119    ggml_backend_registry() {
120#ifdef GGML_USE_CUDA
121        register_backend(ggml_backend_cuda_reg());
122#endif
123#ifdef GGML_USE_METAL
124        register_backend(ggml_backend_metal_reg());
125#endif
126#ifdef GGML_USE_SYCL
127        register_backend(ggml_backend_sycl_reg());
128#endif
129#ifdef GGML_USE_VULKAN
130    // Add runtime disable check
131    if (getenv("GGML_DISABLE_VULKAN") == nullptr) {
132        register_backend(ggml_backend_vk_reg());
133    } else {
134        GGML_LOG_DEBUG("Vulkan backend disabled by GGML_DISABLE_VULKAN environment variable\n");
135    }
136#endif
137#ifdef GGML_USE_WEBGPU
138        register_backend(ggml_backend_webgpu_reg());
139#endif
140#ifdef GGML_USE_ZDNN
141        register_backend(ggml_backend_zdnn_reg());
142#endif
143#ifdef GGML_USE_VIRTGPU_FRONTEND
144        register_backend(ggml_backend_virtgpu_reg());
145#endif
146
147#ifdef GGML_USE_OPENCL
148        register_backend(ggml_backend_opencl_reg());
149#endif
150#ifdef GGML_USE_ZENDNN
151        register_backend(ggml_backend_zendnn_reg());
152#endif
153#ifdef GGML_USE_HEXAGON
154        register_backend(ggml_backend_hexagon_reg());
155#endif
156#ifdef GGML_USE_CANN
157        register_backend(ggml_backend_cann_reg());
158#endif
159#ifdef GGML_USE_BLAS
160        register_backend(ggml_backend_blas_reg());
161#endif
162#ifdef GGML_USE_RPC
163        register_backend(ggml_backend_rpc_reg());
164#endif
165#ifdef GGML_USE_OPENVINO
166        register_backend(ggml_backend_openvino_reg());
167#endif
168#ifdef GGML_USE_ET
169        register_backend(ggml_backend_et_reg());
170#endif
171#ifdef GGML_USE_CPU
172        register_backend(ggml_backend_cpu_reg());
173#endif
174    }
175
176    ~ggml_backend_registry() {
177        // FIXME: backends cannot be safely unloaded without a function to destroy all the backend resources,
178        // since backend threads may still be running and accessing resources from the dynamic library
179        for (auto & entry : backends) {
180            if (entry.handle) {
181                entry.handle.release(); // NOLINT
182            }
183        }
184    }
185
186    void register_backend(ggml_backend_reg_t reg, dl_handle_ptr handle = nullptr) {
187        if (!reg) {
188            return;
189        }
190
191        for (auto & entry : backends) {
192            if (entry.reg == reg) {
193                return;
194            }
195        }
196
197#ifndef NDEBUG
198        GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
199            __func__, ggml_backend_reg_name(reg), ggml_backend_reg_dev_count(reg));
200#endif
201        backends.push_back({ reg, std::move(handle) });
202        for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); i++) {
203            register_device(ggml_backend_reg_dev_get(reg, i));
204        }
205    }
206
207    void register_device(ggml_backend_dev_t device) {
208        for (auto & dev : devices) {
209            if (dev == device) {
210                return;
211            }
212        }
213
214#ifndef NDEBUG
215        GGML_LOG_DEBUG("%s: registered device %s (%s)\n", __func__, ggml_backend_dev_name(device), ggml_backend_dev_description(device));
216#endif
217        devices.push_back(device);
218    }
219
220    ggml_backend_reg_t load_backend(const fs::path & path, bool silent) {
221        dl_handle_ptr handle { dl_load_library(path) };
222        if (!handle) {
223            if (!silent) {
224                GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path_str(path).c_str(), dl_error());
225            }
226            return nullptr;
227        }
228
229        auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");
230        if (score_fn && score_fn() == 0) {
231            if (!silent) {
232                GGML_LOG_INFO("%s: backend %s is not supported on this system\n", __func__, path_str(path).c_str());
233            }
234            return nullptr;
235        }
236
237        auto backend_init_fn = (ggml_backend_init_t) dl_get_sym(handle.get(), "ggml_backend_init");
238        if (!backend_init_fn) {
239            if (!silent) {
240                GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s\n", __func__, path_str(path).c_str());
241            }
242            return nullptr;
243        }
244
245        ggml_backend_reg_t reg = backend_init_fn();
246        if (!reg || reg->api_version != GGML_BACKEND_API_VERSION) {
247            if (!silent) {
248                if (!reg) {
249                    GGML_LOG_ERROR("%s: failed to initialize backend from %s: ggml_backend_init returned NULL\n",
250                        __func__, path_str(path).c_str());
251                } else {
252                    GGML_LOG_ERROR("%s: failed to initialize backend from %s: incompatible API version (backend: %d, current: %d)\n",
253                        __func__, path_str(path).c_str(), reg->api_version, GGML_BACKEND_API_VERSION);
254                }
255            }
256            return nullptr;
257        }
258
259        GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, ggml_backend_reg_name(reg), path_str(path).c_str());
260
261        register_backend(reg, std::move(handle));
262
263        return reg;
264    }
265
266    void unload_backend(ggml_backend_reg_t reg, bool silent) {
267        auto it = std::find_if(backends.begin(), backends.end(),
268                               [reg](const ggml_backend_reg_entry & entry) { return entry.reg == reg; });
269
270        if (it == backends.end()) {
271            if (!silent) {
272                GGML_LOG_ERROR("%s: backend not found\n", __func__);
273            }
274            return;
275        }
276
277        if (!silent) {
278            GGML_LOG_DEBUG("%s: unloading %s backend\n", __func__, ggml_backend_reg_name(reg));
279        }
280
281        // remove devices
282        devices.erase(
283            std::remove_if(devices.begin(), devices.end(),
284                            [reg](ggml_backend_dev_t dev) { return ggml_backend_dev_backend_reg(dev) == reg; }),
285            devices.end());
286
287        // remove backend
288        backends.erase(it);
289    }
290};
291
292static ggml_backend_registry & get_reg() {
293    static ggml_backend_registry reg;
294    return reg;
295}
296
297// Internal API
298void ggml_backend_register(ggml_backend_reg_t reg) {
299    get_reg().register_backend(reg);
300}
301
302void ggml_backend_device_register(ggml_backend_dev_t device) {
303    get_reg().register_device(device);
304}
305
306// Backend (reg) enumeration
307static bool striequals(const char * a, const char * b) {
308    for (; *a && *b; a++, b++) {
309        if (std::tolower(*a) != std::tolower(*b)) {
310            return false;
311        }
312    }
313    return *a == *b;
314}
315
316size_t ggml_backend_reg_count() {
317    return get_reg().backends.size();
318}
319
320ggml_backend_reg_t ggml_backend_reg_get(size_t index) {
321    GGML_ASSERT(index < ggml_backend_reg_count());
322    return get_reg().backends[index].reg;
323}
324
325ggml_backend_reg_t ggml_backend_reg_by_name(const char * name) {
326    for (size_t i = 0; i < ggml_backend_reg_count(); i++) {
327        ggml_backend_reg_t reg = ggml_backend_reg_get(i);
328        if (striequals(ggml_backend_reg_name(reg), name)) {
329            return reg;
330        }
331    }
332    return nullptr;
333}
334
335// Device enumeration
336size_t ggml_backend_dev_count() {
337    return get_reg().devices.size();
338}
339
340ggml_backend_dev_t ggml_backend_dev_get(size_t index) {
341    GGML_ASSERT(index < ggml_backend_dev_count());
342    return get_reg().devices[index];
343}
344
345ggml_backend_dev_t ggml_backend_dev_by_name(const char * name) {
346    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
347        ggml_backend_dev_t dev = ggml_backend_dev_get(i);
348        if (striequals(ggml_backend_dev_name(dev), name)) {
349            return dev;
350        }
351    }
352    return nullptr;
353}
354
355ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type) {
356    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
357        ggml_backend_dev_t dev = ggml_backend_dev_get(i);
358        if (ggml_backend_dev_type(dev) == type) {
359            return dev;
360        }
361    }
362    return nullptr;
363}
364
365// Convenience functions
366ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params) {
367    ggml_backend_dev_t dev = ggml_backend_dev_by_name(name);
368    if (!dev) {
369        return nullptr;
370    }
371    return ggml_backend_dev_init(dev, params);
372}
373
374ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params) {
375    ggml_backend_dev_t dev = ggml_backend_dev_by_type(type);
376    if (!dev) {
377        return nullptr;
378    }
379    return ggml_backend_dev_init(dev, params);
380}
381
382ggml_backend_t ggml_backend_init_best(void) {
383    ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
384    dev = dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU);
385    dev = dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
386    if (!dev) {
387        return nullptr;
388    }
389    return ggml_backend_dev_init(dev, nullptr);
390}
391
392// Dynamic loading
393ggml_backend_reg_t ggml_backend_load(const char * path) {
394    return get_reg().load_backend(path, false);
395}
396
397void ggml_backend_unload(ggml_backend_reg_t reg) {
398    get_reg().unload_backend(reg, true);
399}
400
401static fs::path get_executable_path() {
402#if defined(__APPLE__)
403    // get executable path
404    std::vector<char> path;
405    uint32_t size;
406    while (true) {
407        size = path.size();
408        if (_NSGetExecutablePath(path.data(), &size) == 0) {
409            break;
410        }
411        path.resize(size);
412    }
413    std::string base_path(path.data(), size);
414    // remove executable name
415    auto last_slash = base_path.find_last_of('/');
416    if (last_slash != std::string::npos) {
417        base_path = base_path.substr(0, last_slash);
418    }
419    return base_path + "/";
420#elif defined(__linux__) || defined(__FreeBSD__)
421    std::string base_path = ".";
422    std::vector<char> path(1024);
423    while (true) {
424        // get executable path
425#    if defined(__linux__)
426        ssize_t len = readlink("/proc/self/exe", path.data(), path.size());
427#    elif defined(__FreeBSD__)
428        ssize_t len = readlink("/proc/curproc/file", path.data(), path.size());
429#    endif
430        if (len == -1) {
431            break;
432        }
433        if (len < (ssize_t) path.size()) {
434            base_path = std::string(path.data(), len);
435            // remove executable name
436            auto last_slash = base_path.find_last_of('/');
437            if (last_slash != std::string::npos) {
438                base_path = base_path.substr(0, last_slash);
439            }
440            break;
441        }
442        path.resize(path.size() * 2);
443    }
444
445    return base_path + "/";
446#elif defined(_WIN32)
447    std::vector<wchar_t> path(MAX_PATH);
448    DWORD len = GetModuleFileNameW(NULL, path.data(), path.size());
449    if (len == 0) {
450        return {};
451    }
452    std::wstring base_path(path.data(), len);
453    // remove executable name
454    auto last_slash = base_path.find_last_of('\\');
455    if (last_slash != std::string::npos) {
456        base_path = base_path.substr(0, last_slash);
457    }
458    return base_path + L"\\";
459#else
460    return {};
461#endif
462}
463
464static fs::path backend_filename_prefix() {
465#ifdef _WIN32
466    return fs::u8path("ggml-");
467#else
468    return fs::u8path("libggml-");
469#endif
470}
471
472static fs::path backend_filename_extension() {
473#ifdef _WIN32
474    return fs::u8path(".dll");
475#else
476    return fs::u8path(".so");
477#endif
478}
479
480static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {
481    // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths
482    const fs::path name_path = fs::u8path(name);
483    const fs::path file_prefix = backend_filename_prefix().native() + name_path.native() + fs::u8path("-").native();
484    const fs::path file_extension = backend_filename_extension();
485
486    std::vector<fs::path> search_paths;
487    if (user_search_path == nullptr) {
488#ifdef GGML_BACKEND_DIR
489        search_paths.push_back(fs::u8path(GGML_BACKEND_DIR));
490#endif
491        // default search paths: executable directory, current directory
492        search_paths.push_back(get_executable_path());
493        std::error_code cwd_ec;
494        const fs::path cwd = fs::current_path(cwd_ec);
495        if (cwd_ec) {
496            GGML_LOG_DEBUG("%s: current_path() failure, error-message: %s\n", __func__, cwd_ec.message().c_str());
497        } else {
498            search_paths.push_back(cwd);
499        }
500    } else {
501        search_paths.push_back(fs::u8path(user_search_path));
502    }
503
504    int best_score = 0;
505    fs::path best_path;
506    std::error_code ec;
507
508    for (const auto & search_path : search_paths) {
509        if (!fs::exists(search_path, ec)) {
510            if (ec) {
511                GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str());
512            } else {
513                GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());
514            }
515            continue;
516        }
517        std::error_code dir_ec;
518        fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied, dir_ec);
519        if (dir_ec) {
520            GGML_LOG_DEBUG("%s: failed to enumerate %s: %s\n", __func__, path_str(search_path).c_str(), dir_ec.message().c_str());
521            continue;
522        }
523        for (const fs::directory_iterator end; dir_it != end; dir_it.increment(dir_ec)) {
524            const auto & entry = *dir_it;
525            if (entry.is_regular_file(ec)) {
526                auto filename = entry.path().filename();
527                auto ext = entry.path().extension();
528                if (filename.native().find(file_prefix) == 0 && ext == file_extension) {
529                    dl_handle_ptr handle { dl_load_library(entry) };
530                    if (!handle && !silent) {
531                        GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path_str(entry.path()).c_str(), dl_error());
532                    }
533                    if (handle) {
534                        auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");
535                        if (score_fn) {
536                            int s = score_fn();
537#ifndef NDEBUG
538                            GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, path_str(entry.path()).c_str(), s);
539#endif
540                            if (s > best_score) {
541                                best_score = s;
542                                best_path = entry.path();
543                            }
544                        } else {
545                            if (!silent) {
546                                GGML_LOG_INFO("%s: failed to find ggml_backend_score in %s\n", __func__, path_str(entry.path()).c_str());
547                            }
548                        }
549                    }
550                }
551            }
552        }
553    }
554
555    if (best_score == 0) {
556        // try to load the base backend
557        for (const auto & search_path : search_paths) {
558            fs::path filename = backend_filename_prefix().native() + name_path.native() + backend_filename_extension().native();
559            fs::path path = search_path / filename;
560            if (std::error_code ec; fs::exists(path, ec)) {
561                return get_reg().load_backend(path, silent);
562            } else {
563                if (ec) {
564                    GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(path).c_str(), ec.message().c_str());
565                }
566            }
567        }
568        return nullptr;
569    }
570
571    return get_reg().load_backend(best_path, silent);
572}
573
574void ggml_backend_load_all() {
575    ggml_backend_load_all_from_path(nullptr);
576}
577
578void ggml_backend_load_all_from_path(const char * dir_path) {
579#ifdef NDEBUG
580    bool silent = true;
581#else
582    bool silent = false;
583#endif
584
585    ggml_backend_load_best("blas", silent, dir_path);
586    ggml_backend_load_best("zendnn", silent, dir_path);
587    ggml_backend_load_best("cann", silent, dir_path);
588    ggml_backend_load_best("cuda", silent, dir_path);
589    ggml_backend_load_best("hip", silent, dir_path);
590    ggml_backend_load_best("metal", silent, dir_path);
591    ggml_backend_load_best("rpc", silent, dir_path);
592    ggml_backend_load_best("sycl", silent, dir_path);
593    ggml_backend_load_best("vulkan", silent, dir_path);
594    ggml_backend_load_best("virtgpu", silent, dir_path);
595    ggml_backend_load_best("opencl", silent, dir_path);
596    ggml_backend_load_best("hexagon", silent, dir_path);
597    ggml_backend_load_best("musa", silent, dir_path);
598    ggml_backend_load_best("openvino", silent, dir_path);
599    ggml_backend_load_best("cpu", silent, dir_path);
600    // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend
601    const char * backend_path = std::getenv("GGML_BACKEND_PATH");
602    if (backend_path) {
603        ggml_backend_load(backend_path);
604    }
605}