char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

lnigamggml : recurrent state rollback for ggml_ssm_scan (#26623)1692f9e50

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
24.0 KiB712 linesraw
1#include "ggml-backend.h"
2#include "ggml-backend-impl.h"
3#include "ggml-cpu.h"
4#include "repack.h"
5#include "traits.h"
6#include "ggml-impl.h"
7#include "amx/amx.h"
8
9#include <cctype>
10#include <string>
11#include <vector>
12
13#ifdef GGML_USE_CPU_HBM
14#    include "hbm.h"
15#endif
16
17#ifdef GGML_USE_CPU_KLEIDIAI
18#    include "kleidiai/kleidiai.h"
19#endif
20
21#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
22#    include "spacemit/ime.h"
23#endif
24
25#if defined(_WIN32)
26#    define WIN32_LEAN_AND_MEAN
27#    ifndef NOMINMAX
28#        define NOMINMAX
29#    endif
30#    include <windows.h>
31#else
32#    include <unistd.h>
33#endif
34
35#if defined(__APPLE__)
36#    include <sys/sysctl.h>
37#    include <sys/types.h>
38#endif
39
40// ggml-backend interface
41
42std::vector<ggml_backend_buffer_type_t> & ggml_backend_cpu_get_extra_buffer_types() {
43    static std::vector<ggml_backend_buffer_type_t> bufts = []() {
44        std::vector<ggml_backend_buffer_type_t> bufts;
45
46#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)
47        if (ggml_backend_amx_buffer_type()) {
48            bufts.push_back(ggml_backend_amx_buffer_type());
49        }
50#endif
51
52#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
53        if (ggml_backend_cpu_riscv64_spacemit_buffer_type()) {
54            bufts.push_back(ggml_backend_cpu_riscv64_spacemit_buffer_type());
55        }
56#endif
57
58#ifdef GGML_USE_CPU_KLEIDIAI
59        if (ggml_backend_cpu_kleidiai_buffer_type()) {
60            bufts.push_back(ggml_backend_cpu_kleidiai_buffer_type());
61        }
62#endif
63
64#ifdef GGML_USE_CPU_REPACK
65        if (ggml_backend_cpu_repack_buffer_type()) {
66            bufts.push_back(ggml_backend_cpu_repack_buffer_type());
67        }
68#endif
69
70        return bufts;
71    }();
72
73    return bufts;
74}
75
76static ggml_backend_buffer_type_t * ggml_backend_cpu_device_get_extra_buffers_type(ggml_backend_dev_t device) {
77    static std::vector<ggml_backend_buffer_type_t> extra_bufts = [] {
78        std::vector<ggml_backend_buffer_type_t> bufts = ggml_backend_cpu_get_extra_buffer_types();
79        bufts.push_back(nullptr);
80        return bufts;
81    }();
82
83    return extra_bufts.data();
84
85    GGML_UNUSED(device);
86}
87
88static bool ggml_backend_cpu_is_extra_buffer_type(ggml_backend_buffer_type_t buft) {
89    for (auto * extra : ggml_backend_cpu_get_extra_buffer_types()) {
90        if (extra == buft) {
91            return true;
92        }
93    }
94    return false;
95}
96
97// CPU backend - backend (stream)
98
99struct ggml_backend_cpu_context {
100    int                 n_threads;
101    ggml_threadpool_t   threadpool;
102
103    uint8_t *           work_data;
104    size_t              work_size;
105
106    ggml_abort_callback abort_callback;
107    void *              abort_callback_data;
108
109    bool                use_ref;  // use reference implementation
110};
111
112static const char * ggml_backend_cpu_get_name(ggml_backend_t backend) {
113    return "CPU";
114
115    GGML_UNUSED(backend);
116}
117
118static void ggml_backend_cpu_free(ggml_backend_t backend) {
119    struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
120    delete[] cpu_ctx->work_data;
121    delete cpu_ctx;
122    delete backend;
123}
124
125struct ggml_backend_plan_cpu {
126    struct ggml_cplan cplan;
127    struct ggml_cgraph cgraph;
128};
129
130static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, const struct ggml_cgraph * cgraph) {
131    struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
132
133    struct ggml_backend_plan_cpu * cpu_plan = new ggml_backend_plan_cpu;
134
135    cpu_plan->cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
136    cpu_plan->cgraph = *cgraph; // FIXME: deep copy
137
138    if (cpu_plan->cplan.work_size > 0) {
139        cpu_plan->cplan.work_data = new uint8_t[cpu_plan->cplan.work_size];
140        if (cpu_plan->cplan.work_data == NULL) {
141            delete cpu_plan;
142            return NULL;
143        }
144    }
145
146    cpu_plan->cplan.abort_callback      = cpu_ctx->abort_callback;
147    cpu_plan->cplan.abort_callback_data = cpu_ctx->abort_callback_data;
148    cpu_plan->cplan.use_ref             = cpu_ctx->use_ref;
149
150    return cpu_plan;
151}
152
153static void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
154    struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;
155
156    delete[] cpu_plan->cplan.work_data;
157    delete cpu_plan;
158
159    GGML_UNUSED(backend);
160}
161
162static enum ggml_status ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
163    struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;
164
165    return ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan);
166
167    GGML_UNUSED(backend);
168}
169
170static enum ggml_status ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
171    struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
172
173    struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
174
175    if (cpu_ctx->work_size < cplan.work_size) {
176        delete[] cpu_ctx->work_data;
177        cpu_ctx->work_data = new uint8_t[cplan.work_size];
178        if (cpu_ctx->work_data == NULL) {
179            cpu_ctx->work_size = 0;
180            return GGML_STATUS_ALLOC_FAILED;
181        }
182        cpu_ctx->work_size = cplan.work_size;
183    }
184    cplan.work_data = (uint8_t *)cpu_ctx->work_data;
185
186    cplan.abort_callback      = cpu_ctx->abort_callback;
187    cplan.abort_callback_data = cpu_ctx->abort_callback_data;
188    cplan.use_ref             = cpu_ctx->use_ref;
189
190    return ggml_graph_compute(cgraph, &cplan);
191}
192
193static const struct ggml_backend_i ggml_backend_cpu_i = {
194    /* .get_name                = */ ggml_backend_cpu_get_name,
195    /* .free                    = */ ggml_backend_cpu_free,
196    /* .set_tensor_async        = */ NULL,
197    /* .get_tensor_async        = */ NULL,
198    /* .set_tensor_2d_async     = */ NULL,
199    /* .get_tensor_2d_async     = */ NULL,
200    /* .cpy_tensor_async        = */ NULL,
201    /* .synchronize             = */ NULL,
202    /* .graph_plan_create       = */ ggml_backend_cpu_graph_plan_create,
203    /* .graph_plan_free         = */ ggml_backend_cpu_graph_plan_free,
204    /* .graph_plan_update       = */ NULL,
205    /* .graph_plan_compute      = */ ggml_backend_cpu_graph_plan_compute,
206    /* .graph_compute           = */ ggml_backend_cpu_graph_compute,
207    /* .event_record            = */ NULL,
208    /* .event_wait              = */ NULL,
209    /* .graph_optimize          = */ NULL,
210};
211
212static ggml_guid_t ggml_backend_cpu_guid(void) {
213    static ggml_guid guid = { 0xaa, 0x67, 0xc7, 0x43, 0x96, 0xe6, 0xa3, 0x8a, 0xe3, 0xaf, 0xea, 0x92, 0x36, 0xbc, 0xfc, 0x89 };
214    return &guid;
215}
216
217ggml_backend_t ggml_backend_cpu_init(void) {
218    // initialize CPU backend now to avoid slowing the first graph computation
219    ggml_cpu_init();
220
221    struct ggml_backend_cpu_context * ctx = new ggml_backend_cpu_context;
222    if (ctx == NULL) {
223        return NULL;
224    }
225
226    ctx->n_threads           = GGML_DEFAULT_N_THREADS;
227    ctx->threadpool          = NULL;
228    ctx->work_data           = NULL;
229    ctx->work_size           = 0;
230    ctx->abort_callback      = NULL;
231    ctx->abort_callback_data = NULL;
232    ctx->use_ref             = false;
233
234    ggml_backend_t cpu_backend = new ggml_backend {
235        /* .guid    = */ ggml_backend_cpu_guid(),
236        /* .iface   = */ ggml_backend_cpu_i,
237        /* .device  = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),
238        /* .context = */ ctx,
239    };
240
241    if (cpu_backend == NULL) {
242        delete ctx;
243        return NULL;
244    }
245
246    return cpu_backend;
247}
248
249bool ggml_backend_is_cpu(ggml_backend_t backend) {
250    return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cpu_guid());
251}
252
253void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {
254    GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
255
256    struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
257    ctx->n_threads = n_threads;
258}
259
260void ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_threadpool_t threadpool) {
261    GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
262
263    struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
264
265    if (ctx->threadpool && ctx->threadpool != threadpool) {
266        // already had a different threadpool, pause/suspend it before switching
267        ggml_threadpool_pause(ctx->threadpool);
268    }
269    ctx->threadpool = threadpool;
270}
271
272void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data) {
273    GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
274
275    struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
276    ctx->abort_callback = abort_callback;
277    ctx->abort_callback_data = abort_callback_data;
278}
279
280void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref) {
281    GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
282
283    struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
284    ctx->use_ref = use_ref;
285}
286
287// CPU backend - device
288
289struct ggml_backend_cpu_device_context {
290    std::string description = "CPU";
291
292    ggml_backend_cpu_device_context() {
293#ifdef __APPLE__
294        size_t len = 0;
295        if (!sysctlbyname("machdep.cpu.brand_string", NULL, &len, NULL, 0)) {
296            description.resize(len);
297            sysctlbyname("machdep.cpu.brand_string", &description[0], &len, NULL, 0); // NOLINT
298        }
299#elif defined(__linux__)
300        FILE * f = fopen("/proc/cpuinfo", "r");
301        if (f) {
302            char buf[1024];
303            while (fgets(buf, sizeof(buf), f)) {
304                if (strncmp(buf, "model name", 10) == 0) {
305                    char * p = strchr(buf, ':');
306                    if (p) {
307                        p++;
308                        while (std::isspace(*p)) {
309                            p++;
310                        }
311                        while (std::isspace(p[strlen(p) - 1])) {
312                            p[strlen(p) - 1] = '\0';
313                        }
314                        description = p;
315                        break;
316                    }
317                }
318            }
319            fclose(f);
320        }
321#elif defined(_WIN32)
322        HKEY hKey;
323        if (RegOpenKeyEx(HKEY_LOCAL_MACHINE,
324                        TEXT("HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"),
325                        0,
326                        KEY_READ,
327                        &hKey) == ERROR_SUCCESS) {
328            DWORD cpu_brand_size = 0;
329            if (RegQueryValueExA(hKey,
330                                "ProcessorNameString",
331                                NULL,
332                                NULL,
333                                NULL,
334                                &cpu_brand_size) == ERROR_SUCCESS) {
335                description.resize(cpu_brand_size);
336                if (RegQueryValueExA(hKey,
337                                    "ProcessorNameString",
338                                    NULL,
339                                    NULL,
340                                    (LPBYTE)&description[0], // NOLINT
341                                    &cpu_brand_size) == ERROR_SUCCESS) {
342                    if (description.find('\0') != std::string::npos) {
343                        description.resize(description.find('\0'));
344                    }
345                }
346            }
347            RegCloseKey(hKey);
348        }
349#endif
350    }
351};
352
353static const char * ggml_backend_cpu_device_get_name(ggml_backend_dev_t dev) {
354    return "CPU";
355
356    GGML_UNUSED(dev);
357}
358
359static const char * ggml_backend_cpu_device_get_description(ggml_backend_dev_t dev) {
360    struct ggml_backend_cpu_device_context * ctx = (struct ggml_backend_cpu_device_context *)dev->context;
361
362    return ctx->description.c_str();
363}
364
365static void ggml_backend_cpu_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
366#ifdef _WIN32
367    MEMORYSTATUSEX status;
368    status.dwLength = sizeof(status);
369    GlobalMemoryStatusEx(&status);
370    *total = status.ullTotalPhys;
371    *free = status.ullAvailPhys;
372#else
373    long pages = sysconf(_SC_PHYS_PAGES);
374    long page_size = sysconf(_SC_PAGE_SIZE);
375    *total = pages * page_size;
376
377    // "free" system memory is ill-defined, for practical purposes assume that all of it is free:
378    *free = *total;
379#endif // _WIN32
380
381    GGML_UNUSED(dev);
382}
383
384static enum ggml_backend_dev_type ggml_backend_cpu_device_get_type(ggml_backend_dev_t dev) {
385    return GGML_BACKEND_DEVICE_TYPE_CPU;
386
387    GGML_UNUSED(dev);
388}
389
390static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {
391    props->name        = ggml_backend_cpu_device_get_name(dev);
392    props->description = ggml_backend_cpu_device_get_description(dev);
393    props->type        = ggml_backend_cpu_device_get_type(dev);
394    ggml_backend_cpu_device_get_memory(dev, &props->memory_free, &props->memory_total);
395    props->caps = {
396        /* .async                 = */ false,
397        /* .host_buffer           = */ false,
398        /* .buffer_from_host_ptr  = */ true,
399        /* .events                = */ false,
400        /* .mmap_support          = */ true,
401    };
402}
403
404static ggml_backend_t ggml_backend_cpu_device_init_backend(ggml_backend_dev_t dev, const char * params) {
405    return ggml_backend_cpu_init();
406
407    GGML_UNUSED(dev);
408    GGML_UNUSED(params);
409}
410
411static ggml_backend_buffer_type_t ggml_backend_cpu_device_get_buffer_type(ggml_backend_dev_t dev) {
412    return ggml_backend_cpu_buffer_type();
413
414    GGML_UNUSED(dev);
415}
416
417static ggml_backend_buffer_t ggml_backend_cpu_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {
418    return ggml_backend_cpu_buffer_from_ptr(ptr, size);
419
420    GGML_UNUSED(dev);
421    GGML_UNUSED(max_tensor_size);
422}
423
424static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
425    const struct ggml_tensor * src0 = op->src[0];
426    const struct ggml_tensor * src1 = op->src[1];
427
428    if (op->op == GGML_OP_NONE || op->op == GGML_OP_RESHAPE || op->op == GGML_OP_VIEW || op->op == GGML_OP_PERMUTE || op->op == GGML_OP_TRANSPOSE) {
429        return true;
430    }
431
432    // check extra buffer types
433    // note: only the first sources are checked for extra buffer types to reduce overhead, increase if necessary
434    for (int i = 0; i < 4; i++) {
435        if (op->src[i] && op->src[i]->buffer &&
436            ggml_backend_cpu_is_extra_buffer_type(op->src[i]->buffer->buft)) {
437            auto * buf_extra = (ggml::cpu::extra_buffer_type *) op->src[i]->buffer->buft->context;
438            return buf_extra->supports_op(dev, op);
439        }
440    }
441
442    switch (op->op) {
443        case GGML_OP_CPY:
444        case GGML_OP_SET_ROWS:
445            return
446                op->type != GGML_TYPE_IQ3_XXS &&
447                op->type != GGML_TYPE_IQ3_S   &&
448                op->type != GGML_TYPE_IQ2_XXS &&
449                op->type != GGML_TYPE_IQ2_XS  &&
450                op->type != GGML_TYPE_IQ2_S   &&
451                op->type != GGML_TYPE_IQ1_S   &&
452                op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float
453        case GGML_OP_MUL_MAT:
454            return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type;
455        case GGML_OP_SOFT_MAX_BACK: {
456            if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) {
457                return false;
458            }
459            float max_bias = 0.0f;
460
461            memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float));
462
463            return max_bias == 0.0f;
464        }
465        case GGML_OP_IM2COL_BACK:
466            return src0->type == GGML_TYPE_F32 && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16);
467        case GGML_OP_GET_ROWS_BACK:
468            return src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16;
469        case GGML_OP_OUT_PROD:
470            return (src0->type == GGML_TYPE_F32 ||
471                    ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) &&
472                src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
473        case GGML_OP_CONV_2D:
474            return ggml_is_contiguous(op->src[0]);
475        case GGML_OP_SSM_SCAN:
476            return ggml_get_op_params_i32(op, 0) == 1 || op->src[3]->ne[0] == 1;
477        default:
478            return true;
479    }
480}
481
482static bool ggml_backend_cpu_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
483    return ggml_backend_buft_is_host(buft) || ggml_backend_cpu_is_extra_buffer_type(buft);
484    GGML_UNUSED(dev);
485}
486
487static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {
488    /* .get_name             = */ ggml_backend_cpu_device_get_name,
489    /* .get_description      = */ ggml_backend_cpu_device_get_description,
490    /* .get_memory           = */ ggml_backend_cpu_device_get_memory,
491    /* .get_type             = */ ggml_backend_cpu_device_get_type,
492    /* .get_props            = */ ggml_backend_cpu_device_get_props,
493    /* .init_backend         = */ ggml_backend_cpu_device_init_backend,
494    /* .get_buffer_type      = */ ggml_backend_cpu_device_get_buffer_type,
495    /* .get_host_buffer_type = */ NULL,
496    /* .buffer_from_host_ptr = */ ggml_backend_cpu_device_buffer_from_host_ptr,
497    /* .supports_op          = */ ggml_backend_cpu_device_supports_op,
498    /* .supports_buft        = */ ggml_backend_cpu_device_supports_buft,
499    /* .offload_op           = */ NULL,
500    /* .event_new            = */ NULL,
501    /* .event_free           = */ NULL,
502    /* .event_synchronize    = */ NULL,
503};
504
505// CPU backend - backend (reg)
506
507static const char * ggml_backend_cpu_reg_get_name(ggml_backend_reg_t reg) {
508    return "CPU";
509
510    GGML_UNUSED(reg);
511}
512
513static size_t ggml_backend_cpu_reg_get_device_count(ggml_backend_reg_t reg) {
514    return 1;
515
516    GGML_UNUSED(reg);
517}
518
519static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg, size_t index) {
520    GGML_ASSERT(index == 0);
521
522    static ggml_backend_cpu_device_context ctx;
523    static ggml_backend_device ggml_backend_cpu_device = {
524        /* .iface   = */ ggml_backend_cpu_device_i,
525        /* .reg     = */ reg,
526        /* .context = */ &ctx,
527    };
528
529    return &ggml_backend_cpu_device;
530}
531
532// This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,
533// and additionally to allow other backends to expose their own list of features that applications can query using the same API
534static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t reg) {
535    static std::vector<ggml_backend_feature> features = []() {
536        ggml_cpu_init();
537
538        std::vector<ggml_backend_feature> features;
539        if (ggml_cpu_has_sse3()) {
540            features.push_back({ "SSE3", "1" });
541        }
542        if (ggml_cpu_has_ssse3()) {
543            features.push_back({ "SSSE3", "1" });
544        }
545        if (ggml_cpu_has_avx()) {
546            features.push_back({ "AVX", "1" });
547        }
548        if (ggml_cpu_has_avx_vnni()) {
549            features.push_back({ "AVX_VNNI", "1" });
550        }
551        if (ggml_cpu_has_avx2()) {
552            features.push_back({ "AVX2", "1" });
553        }
554        if (ggml_cpu_has_f16c()) {
555            features.push_back({ "F16C", "1" });
556        }
557        if (ggml_cpu_has_fma()) {
558            features.push_back({ "FMA", "1" });
559        }
560        if (ggml_cpu_has_bmi2()) {
561            features.push_back({ "BMI2", "1" });
562        }
563        if (ggml_cpu_has_avx512()) {
564            features.push_back({ "AVX512", "1" });
565        }
566        if (ggml_cpu_has_avx512_vbmi()) {
567            features.push_back({ "AVX512_VBMI", "1" });
568        }
569        if (ggml_cpu_has_avx512_vnni()) {
570            features.push_back({ "AVX512_VNNI", "1" });
571        }
572        if (ggml_cpu_has_avx512_bf16()) {
573            features.push_back({ "AVX512_BF16", "1" });
574        }
575        if (ggml_cpu_has_amx_int8()) {
576            features.push_back({ "AMX_INT8", "1" });
577        }
578        if (ggml_cpu_has_neon()) {
579            features.push_back({ "NEON", "1" });
580        }
581        if (ggml_cpu_has_arm_fma()) {
582            features.push_back({ "ARM_FMA", "1" });
583        }
584        if (ggml_cpu_has_fp16_va()) {
585            features.push_back({ "FP16_VA", "1" });
586        }
587        if (ggml_cpu_has_matmul_int8()) {
588            features.push_back({ "MATMUL_INT8", "1" });
589        }
590        if (ggml_cpu_has_sve()) {
591            features.push_back({ "SVE", "1" });
592        }
593        if (ggml_cpu_has_dotprod()) {
594            features.push_back({ "DOTPROD", "1" });
595        }
596        if (ggml_cpu_get_sve_cnt() > 0) {
597            static std::string sve_cnt = std::to_string(ggml_cpu_get_sve_cnt());
598            features.push_back({ "SVE_CNT", sve_cnt.c_str() });
599        }
600        if (ggml_cpu_has_sme()) {
601            features.push_back({ "SME", "1" });
602        }
603        if (ggml_cpu_has_sme2()) {
604            features.push_back({ "SME2", "1" });
605        }
606        if (ggml_cpu_has_riscv_v()) {
607            features.push_back({ "RISCV_V", "1" });
608        }
609        if (ggml_cpu_get_rvv_vlen() > 0) {
610            static std::string rvv_vlen = std::to_string(ggml_cpu_get_rvv_vlen());
611            features.push_back({ "RVV_VLEN", rvv_vlen.c_str() });
612        }
613        if (ggml_cpu_has_vsx()) {
614            features.push_back({ "VSX", "1" });
615        }
616        if (ggml_cpu_has_vxe()) {
617            features.push_back({ "VXE", "1" });
618        }
619        if (ggml_cpu_has_wasm_simd()) {
620            features.push_back({ "WASM_SIMD", "1" });
621        }
622        if (ggml_cpu_has_llamafile()) {
623            features.push_back({ "LLAMAFILE", "1" });
624        }
625    #ifdef GGML_USE_ACCELERATE
626        features.push_back({ "ACCELERATE", "1" });
627    #endif
628    #ifdef GGML_USE_CPU_HBM
629        features.push_back({ "CPU_HBM", "1" });
630    #endif
631    #ifdef GGML_USE_OPENMP
632        features.push_back({ "OPENMP", "1" });
633    #endif
634    #ifdef GGML_USE_CPU_KLEIDIAI
635        features.push_back({ "KLEIDIAI", "1" });
636    #endif
637    #ifdef GGML_USE_CPU_REPACK
638        features.push_back({ "REPACK", "1" });
639    #endif
640
641        features.push_back({ nullptr, nullptr });
642
643        return features;
644    }();
645
646    return features.data();
647
648    GGML_UNUSED(reg);
649}
650
651static void * ggml_backend_cpu_get_proc_address(ggml_backend_reg_t reg, const char * name) {
652    if (strcmp(name, "ggml_backend_set_n_threads") == 0) {
653        ggml_backend_set_n_threads_t fct = ggml_backend_cpu_set_n_threads;
654        return (void *)fct;
655    }
656    if (strcmp(name, "ggml_backend_dev_get_extra_bufts") == 0) {
657        ggml_backend_dev_get_extra_bufts_t fct = ggml_backend_cpu_device_get_extra_buffers_type;
658        return (void *)fct;
659    }
660    if (strcmp(name, "ggml_backend_get_features") == 0) {
661        return (void *)ggml_backend_cpu_get_features;
662    }
663    if (strcmp(name, "ggml_backend_set_abort_callback") == 0) {
664        return (void *)ggml_backend_cpu_set_abort_callback;
665    }
666    if (strcmp(name, "ggml_backend_cpu_numa_init") == 0) {
667        return (void *)ggml_numa_init;
668    }
669    if (strcmp(name, "ggml_backend_cpu_is_numa") == 0) {
670        return (void *)ggml_is_numa;
671    }
672    if (strcmp(name, "ggml_backend_cpu_set_use_ref") == 0) {
673        return (void *)ggml_backend_cpu_set_use_ref;
674    }
675
676    // threadpool - TODO:  move to ggml-base
677    if (strcmp(name, "ggml_threadpool_new") == 0) {
678        return (void *)ggml_threadpool_new;
679    }
680    if (strcmp(name, "ggml_threadpool_free") == 0) {
681        return (void *)ggml_threadpool_free;
682    }
683    if (strcmp(name, "ggml_backend_cpu_set_threadpool") == 0) {
684        return (void *)ggml_backend_cpu_set_threadpool;
685    }
686
687    return NULL;
688
689    GGML_UNUSED(reg);
690}
691
692static const struct ggml_backend_reg_i ggml_backend_cpu_reg_i = {
693    /* .get_name         = */ ggml_backend_cpu_reg_get_name,
694    /* .get_device_count = */ ggml_backend_cpu_reg_get_device_count,
695    /* .get_device       = */ ggml_backend_cpu_reg_get_device,
696    /* .get_proc_address = */ ggml_backend_cpu_get_proc_address,
697};
698
699ggml_backend_reg_t ggml_backend_cpu_reg(void) {
700    // init CPU feature detection
701    ggml_cpu_init();
702
703    static struct ggml_backend_reg ggml_backend_cpu_reg = {
704        /* .api_version = */ GGML_BACKEND_API_VERSION,
705        /* .iface       = */ ggml_backend_cpu_reg_i,
706        /* .context     = */ NULL,
707    };
708
709    return &ggml_backend_cpu_reg;
710}
711
712GGML_BACKEND_DL_IMPL(ggml_backend_cpu_reg)