char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Ruben Ortlamllama: add default load-mode auto, which avoids mmap on iGPUs (#26081)153d324bc

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
20.9 KiB638 linesraw
1#include "ggml-zdnn.h"
2#include "ggml-impl.h"
3#include "ggml-backend-impl.h"
4
5#include "ggml-zdnn/common.hpp"
6#include "ggml-zdnn/mmf.hpp"
7#include "ggml-zdnn/utils.hpp"
8#include "ggml.h"
9
10#include <vector>
11#include <memory>
12#include <csignal>  // raise(SIGTRAP)
13#include <unistd.h>
14
15static void ggml_zdnn_compute_forward_mul_mat(
16    const ggml_backend_zdnn_context * ctx,
17          ggml_tensor * dst) {
18
19    const ggml_tensor * src0 = dst->src[0];  // weights
20    const ggml_tensor * src1 = dst->src[1];  // inputs
21
22    // TODO: implement support for quantized types
23    // we currently only support f32, f16, and bf16
24    ggml_zdnn_mul_mat_f(ctx, src0, src1, dst);
25}
26
27static bool ggml_zdnn_compute_forward(
28    ggml_backend_zdnn_context * ctx,
29    ggml_tensor * dst) {
30
31    switch (dst->op) {
32        case GGML_OP_MUL_MAT:
33            {
34                ggml_zdnn_compute_forward_mul_mat(ctx, dst);
35            } break;
36
37        default:
38            return false;
39    }
40
41    return true;
42}
43
44static enum ggml_status ggml_zdnn_graph_compute(ggml_backend_t backend, ggml_cgraph * gf) {
45    ggml_backend_zdnn_context        * ctx     = (       ggml_backend_zdnn_context *)backend->context;
46    ggml_backend_zdnn_device_context * ctx_dev = (ggml_backend_zdnn_device_context *)backend->device->context;
47
48    ctx->gf = gf;
49    for (int i = 0; i < gf->n_nodes; i++) {
50        ggml_tensor * node = gf->nodes[i];
51
52        if (ggml_is_empty(node)
53            || node->op == GGML_OP_NONE
54            || node->op == GGML_OP_RESHAPE
55            || node->op == GGML_OP_VIEW
56            || node->op == GGML_OP_PERMUTE
57            || node->op == GGML_OP_TRANSPOSE) {
58            continue;
59        }
60
61        if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
62            continue;
63        }
64
65        bool ok = ggml_zdnn_compute_forward(ctx, node);
66        if (!ok) {
67            GGML_LOG_ERROR("%s: unsupported op %s (%s)\n",
68                           __func__, node->name, ggml_op_name(node->op));
69        }
70
71        GGML_ASSERT(ok);
72    }
73
74    return GGML_STATUS_SUCCESS;
75
76    GGML_UNUSED(ctx_dev);
77}
78
79static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_dev, const ggml_tensor * op) {
80    switch (op->op) {
81        case GGML_OP_NONE:
82        case GGML_OP_RESHAPE:
83        case GGML_OP_VIEW:
84        case GGML_OP_TRANSPOSE:
85        case GGML_OP_PERMUTE:
86            return true;
87
88        case GGML_OP_MUL_MAT:
89            {
90                const ggml_tensor * weights = op->src[0];
91                const ggml_tensor * inputs  = op->src[1];
92
93                const int64_t ne10 = inputs->ne[0];
94                const int64_t ne0  = op->ne[0];
95                const int64_t ne1  = op->ne[1];
96
97                const int64_t max_batch = ctx_dev->max_size;
98
99                if (!ggml_is_matrix(weights) || !ggml_is_matrix(inputs) ||
100                    !ggml_is_contiguous(weights) || !ggml_is_contiguous(inputs) ||
101                    weights->view_src != nullptr || inputs->view_src != nullptr ||
102                    ne0 > max_batch || ne1 > max_batch || ne10 > max_batch) {
103                        return false;
104                }
105
106                switch (weights->type) {
107                    case GGML_TYPE_F32:
108                    case GGML_TYPE_F16:
109                    case GGML_TYPE_BF16:
110                        return true;
111                    default:
112                        return false;
113                }
114            } break;
115
116        default:
117            return false;
118    }
119}
120
121////////////////////////////////////////////////////////////////////////////////
122
123//
124// globals
125//
126
127// initialised in ggml_backend_zdnn_reg
128static ggml_backend_reg    g_ggml_backend_zdnn_reg;
129static ggml_backend_device g_ggml_backend_zdnn_device;
130
131static ggml_backend_zdnn_device_context g_ggml_ctx_dev_main = {
132    /* .zdnn_device           = */ 0,
133    /* .zdnn_device_ref_count = */ 0,
134    /* .has_parmblkformat_0   = */ false,
135    /* .has_parmblkformat_1   = */ false,
136    /* .max_size              = */ 0,
137    /* .name                  = */ "",
138};
139
140static int ggml_backend_zdnn_device_acq(ggml_backend_zdnn_device_context * ctx) {
141    assert(ctx != NULL);
142
143    if (ctx->zdnn_device == 0) {
144        ctx->zdnn_device = 1;
145    }
146
147    if (ctx->zdnn_device >= 1) {
148        ctx->has_parmblkformat_0 = zdnn_is_nnpa_parmblk_fmt_installed(1, NNPA_PARMBLKFORMAT_0);
149        ctx->has_parmblkformat_1 = zdnn_is_nnpa_parmblk_fmt_installed(1, NNPA_PARMBLKFORMAT_1);
150        ctx->max_size = zdnn_get_nnpa_max_dim_idx_size();
151        strncpy(ctx->name, GGML_ZDNN_NAME, sizeof(ctx->name) - 1);
152    }
153
154    ctx->zdnn_device_ref_count++;
155    return ctx->zdnn_device;
156}
157
158static void ggml_backend_zdnn_device_rel(ggml_backend_zdnn_device_context * ctx) {
159    assert(ctx != NULL);
160    assert(ctx->zdnn_device_ref_count > 0);
161
162    ctx->zdnn_device_ref_count--;
163    if (ctx->zdnn_device_ref_count == 0) {
164        if (ctx->zdnn_device >= 0) {
165            ctx->zdnn_device = 0;
166        }
167    }
168}
169
170static ggml_backend_zdnn_context * ggml_zdnn_init(ggml_backend_dev_t dev) {
171    GGML_LOG_INFO("%s: allocating\n", __func__);
172    GGML_LOG_INFO("%s: found 1 device\n", __func__);
173
174    #ifdef STATIC_LIB
175    zdnn_init();
176    #endif
177
178    ggml_backend_zdnn_context * ctx = new ggml_backend_zdnn_context();
179    ggml_backend_zdnn_device_context * ctx_dev = (ggml_backend_zdnn_device_context *)dev->context;
180
181    int device = 1;
182    GGML_LOG_INFO("%s: picking default device: %s\n", __func__, ctx_dev->name);
183
184    ctx->device = device;
185    GGML_LOG_INFO("%s: NNPA name: %s\n", __func__, ctx_dev->name);
186    GGML_LOG_INFO("%s: NNPA_PARMBLKFORMAT_0 = %s\n", __func__, ctx_dev->has_parmblkformat_0 ? "true" : "false");
187    GGML_LOG_INFO("%s: NNPA_PARMBLKFORMAT_1 = %s\n", __func__, ctx_dev->has_parmblkformat_1 ? "true" : "false");
188
189    ctx->gf = nullptr;
190
191    return ctx;
192}
193
194static void ggml_zdnn_free(ggml_backend_zdnn_context * ctx) {
195    GGML_LOG_INFO("%s: deallocating\n", __func__);
196    delete ctx;
197}
198
199//
200// backend interface
201//
202
203static void ggml_backend_zdnn_buffer_free_buffer(ggml_backend_buffer_t buffer) {
204    ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context;
205
206    for (const auto & buf_ptr : ctx->buffers) {
207        ggml_backend_zdnn_buffer * buf = buf_ptr.get();
208
209        // Free any extra buffer allocated for the tensor. E.g., bias for GGML_OP_MUL_MAT
210        if (buf->extra != nullptr) free(buf->extra->data);
211        if (buf->ztensor.buffer_size > 0) ZDNN_CHECK(zdnn_free_ztensor_buffer(&buf->ztensor));
212    }
213
214    delete ctx;
215}
216
217static void * ggml_backend_zdnn_buffer_get_base(ggml_backend_buffer_t buffer) {
218    ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context;
219    return ctx->all_data;
220}
221
222static enum ggml_status ggml_backend_zdnn_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
223    if (tensor->view_src != NULL) {
224        assert(tensor->view_src->buffer->buft == buffer->buft);
225        return GGML_STATUS_SUCCESS;
226    }
227
228    ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context;
229
230    const int64_t tsize = ggml_nbytes(tensor);
231    int buffer_idx = ctx->n_buffers;
232
233    std::unique_ptr<ggml_backend_zdnn_buffer> zdnn_buffer = std::make_unique<ggml_backend_zdnn_buffer>();
234    zdnn_buffer->data = tensor->data;
235    zdnn_buffer->size = tsize;
236    zdnn_buffer->extra = nullptr;
237    snprintf(zdnn_buffer->name, GGML_MAX_NAME, "%s", tensor->name);
238
239    ggml_zdnn_init_tensor(zdnn_buffer.get(), tensor);
240    tensor->extra = zdnn_buffer.get();
241
242    switch (tensor->op) {
243        case GGML_OP_MUL_MAT:
244            {
245                std::unique_ptr<ggml_backend_zdnn_buffer> zdnn_bias_buffer = std::make_unique<ggml_backend_zdnn_buffer>();
246                zdnn_bias_buffer->data = (void *)calloc(tensor->ne[0], ggml_element_size(tensor));
247                zdnn_bias_buffer->size = ggml_element_size(tensor) * tensor->ne[0];
248                snprintf(zdnn_bias_buffer->name, GGML_MAX_NAME, "%.*s (bias)",
249                         GGML_MAX_NAME - (int)sizeof(" (bias)"), tensor->name);
250
251                const int64_t bias_dim[GGML_MAX_DIMS] = { 1, 1, 1, tensor->ne[0] };
252                ggml_zdnn_create_tensor(zdnn_bias_buffer->pre_tfm_desc,
253                                        zdnn_bias_buffer->tfm_desc,
254                                        zdnn_bias_buffer->ztensor,
255                                        tensor, bias_dim, ZDNN_1D);
256
257                ggml_zdnn_load_tensor(zdnn_bias_buffer->ztensor, zdnn_bias_buffer->data);
258                zdnn_buffer->extra = zdnn_bias_buffer.get();
259
260                ctx->buffers.push_back(std::move(zdnn_bias_buffer));
261                ctx->n_buffers++;
262            } break;
263        default:
264            break;
265    }
266
267    ctx->buffers.push_back(std::move(zdnn_buffer));
268    ctx->n_buffers++;
269
270    // GGML_LOG_INFO("%s: initialised tensor '%s' in buffer %d, size = %8.2f MiB\n",
271    //               __func__, tensor->name, buffer_idx, tsize);
272
273    return GGML_STATUS_SUCCESS;
274
275    GGML_UNUSED(buffer_idx);
276}
277
278static void ggml_backend_zdnn_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
279    memset((char *)tensor->data + offset, value, size);
280
281    GGML_UNUSED(buffer);
282}
283
284static void ggml_backend_zdnn_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
285    memcpy((char *)tensor->data + offset, data, size);
286
287    ggml_backend_zdnn_buffer * extra = (ggml_backend_zdnn_buffer *)tensor->extra;
288
289    // Fixes the LLAMA_SET_ROWS bug
290    // see: https://github.com/ggml-org/llama.cpp/issues/15414
291    if (tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_COMPUTE && extra->ztensor.is_transformed) zdnn_reset_ztensor(&extra->ztensor);
292    if (extra->ztensor.is_transformed == false) ggml_zdnn_load_tensor(extra->ztensor, tensor->data);
293
294    GGML_UNUSED(buffer);
295}
296
297static void ggml_backend_zdnn_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
298    memcpy(data, (const char *)tensor->data + offset, size);
299
300    GGML_UNUSED(buffer);
301}
302
303static void ggml_backend_zdnn_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
304    ggml_backend_zdnn_buffer_context * ctx = (ggml_backend_zdnn_buffer_context *)buffer->context;
305
306    memset(ctx->all_data, value, ctx->all_size);
307}
308
309static ggml_backend_buffer_i ggml_backend_zdnn_buffer_i = {
310    /* .free_buffer   = */ ggml_backend_zdnn_buffer_free_buffer,
311    /* .get_base      = */ ggml_backend_zdnn_buffer_get_base,
312    /* .init_tensor   = */ ggml_backend_zdnn_buffer_init_tensor,
313    /* .memset_tensor = */ ggml_backend_zdnn_buffer_memset_tensor,
314    /* .set_tensor    = */ ggml_backend_zdnn_buffer_set_tensor,
315    /* .get_tensor    = */ ggml_backend_zdnn_buffer_get_tensor,
316    /* .set_tensor_2d = */ NULL,
317    /* .get_tensor_2d = */ NULL,
318    /* .cpy_tensor    = */ NULL,
319    /* .clear         = */ ggml_backend_zdnn_buffer_clear,
320    /* .reset         = */ NULL,
321};
322
323//
324// default buffer type
325//
326
327static const char * ggml_backend_zdnn_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
328    return GGML_ZDNN_NAME;
329
330    GGML_UNUSED(buft);
331}
332
333static ggml_backend_buffer_t ggml_backend_zdnn_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
334    ggml_backend_zdnn_buffer_context * ctx = new ggml_backend_zdnn_buffer_context();
335
336    const size_t size_page = sysconf(_SC_PAGESIZE);
337
338    size_t size_aligned = size;
339    if ((size_aligned % size_page) != 0) {
340        size_aligned += size_page - (size_aligned % size_page);
341    }
342
343    ggml_backend_zdnn_device_context * ctx_dev = (ggml_backend_zdnn_device_context *)buft->device->context;
344
345    GGML_ASSERT(ctx_dev->zdnn_device >= 0);
346    int device = ctx_dev->zdnn_device; GGML_UNUSED(device);
347
348    ctx->all_data  = ggml_aligned_malloc(size_aligned);
349    ctx->all_size  = size_aligned;
350    ctx->owned     = true;
351    ctx->n_buffers = 1;
352
353    if (ctx->all_data != NULL) {
354        std::unique_ptr<ggml_backend_zdnn_buffer> zdnn_buffer = std::make_unique<ggml_backend_zdnn_buffer>();
355        zdnn_buffer->data = ctx->all_data;
356        zdnn_buffer->size = size_aligned;
357        ctx->buffers.push_back(std::move(zdnn_buffer));
358    }
359
360    if (size_aligned > 0 && (ctx->all_data == NULL)) {
361        GGML_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f\n",
362                       __func__, size_aligned / 1024.0 / 1024.0);
363        delete ctx;
364        return NULL;
365    }
366
367    return ggml_backend_buffer_init(buft, ggml_backend_zdnn_buffer_i, ctx, size);
368}
369
370static size_t ggml_backend_zdnn_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
371    return 256;
372
373    GGML_UNUSED(buft);
374}
375
376static bool ggml_backend_zdnn_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
377    /* while it resides in host memory, additional transformation is needed */
378    return false;
379
380    GGML_UNUSED(buft);
381}
382
383ggml_backend_buffer_type_t ggml_backend_zdnn_buffer_type(void) {
384    static ggml_backend_buffer_type ggml_backend_buffer_type_zdnn = {
385        /* .iface   = */ {
386            /* .get_name       = */ ggml_backend_zdnn_buffer_type_get_name,
387            /* .alloc_buffer   = */ ggml_backend_zdnn_buffer_type_alloc_buffer,
388            /* .get_alignment  = */ ggml_backend_zdnn_buffer_type_get_alignment,
389            /* .get_max_size   = */ NULL,
390            /* .get_alloc_size = */ NULL,  // defaults to ggml_nbytes
391            /* .is_host        = */ ggml_backend_zdnn_buffer_type_is_host,
392        },
393        /* .device  = */ &g_ggml_backend_zdnn_device,
394        /* .context = */ NULL,
395    };
396
397    return &ggml_backend_buffer_type_zdnn;
398}
399
400//
401// backend
402//
403
404static const char * ggml_backend_zdnn_name(ggml_backend_t backend) {
405    return GGML_ZDNN_NAME;
406
407    GGML_UNUSED(backend);
408}
409
410static void ggml_backend_zdnn_free(ggml_backend_t backend) {
411    ggml_backend_zdnn_context * ctx = (ggml_backend_zdnn_context *)backend->context;
412
413    ggml_zdnn_free(ctx);
414    free(backend);
415}
416
417static enum ggml_status ggml_backend_zdnn_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
418    return ggml_zdnn_graph_compute(backend, cgraph);
419}
420
421static ggml_backend_i ggml_backend_zdnn_i = {
422    /* .get_name               = */ ggml_backend_zdnn_name,
423    /* .free                   = */ ggml_backend_zdnn_free,
424    /* .set_tensor_async       = */ NULL,
425    /* .get_tensor_async       = */ NULL,
426    /* .set_tensor_2d_async    = */ NULL,
427    /* .get_tensor_2d_async    = */ NULL,
428    /* .cpy_tensor_async       = */ NULL,
429    /* .synchronize            = */ NULL,
430    /* .graph_plan_create      = */ NULL,
431    /* .graph_plan_free        = */ NULL,
432    /* .graph_plan_update      = */ NULL,
433    /* .graph_plan_compute     = */ NULL,
434    /* .graph_compute          = */ ggml_backend_zdnn_graph_compute,
435    /* .event_record           = */ NULL,
436    /* .event_wait             = */ NULL,
437    /* .graph_optimize         = */ NULL,
438};
439
440static ggml_guid_t ggml_backend_zdnn_guid(void) {
441    static const char * guid_str = "IBM-ZDNN-ACCELER";
442    return reinterpret_cast<ggml_guid_t>((void *)guid_str);
443}
444
445bool ggml_backend_is_zdnn(ggml_backend_t backend) {
446    return backend != NULL &&
447           ggml_guid_matches(backend->guid, ggml_backend_zdnn_guid());
448
449    GGML_UNUSED(backend);
450}
451
452//
453// backend device
454//
455
456static const char * ggml_backend_zdnn_device_get_name(ggml_backend_dev_t dev) {
457    return GGML_ZDNN_NAME;
458
459    GGML_UNUSED(dev);
460}
461
462static const char * ggml_backend_zdnn_device_get_description(ggml_backend_dev_t dev) {
463    return "IBM Z Neural Network Processing Assist (NNPA)";
464
465    GGML_UNUSED(dev);
466}
467
468static void ggml_backend_zdnn_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
469    *free  = 0;
470    *total = 0;
471
472    GGML_UNUSED(dev);
473}
474
475static enum ggml_backend_dev_type ggml_backend_zdnn_device_get_type(ggml_backend_dev_t dev) {
476    return GGML_BACKEND_DEVICE_TYPE_ACCEL;
477
478    GGML_UNUSED(dev);
479}
480
481static void ggml_backend_zdnn_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {
482    props->name        = ggml_backend_zdnn_device_get_name(dev);
483    props->description = ggml_backend_zdnn_device_get_description(dev);
484    props->type        = ggml_backend_zdnn_device_get_type(dev);
485    ggml_backend_zdnn_device_get_memory(dev, &props->memory_free, &props->memory_total);
486    props->caps = (ggml_backend_dev_caps) {
487        /* .async                = */ false,
488        /* .host_buffer          = */ false,
489        /* .buffer_from_host_ptr = */ false,
490        /* .events               = */ false,
491        /* .mmap_support         = */ true,
492    };
493}
494
495static ggml_backend_t ggml_backend_zdnn_device_init(ggml_backend_dev_t dev, const char * params) {
496    ggml_backend_zdnn_context * ctx = ggml_zdnn_init(dev);
497    if (ctx == NULL) {
498        GGML_LOG_ERROR("%s: error: failed to allocate context\n", __func__);
499        return NULL;
500    }
501
502    ggml_backend_t backend = (ggml_backend *)malloc(sizeof(ggml_backend));
503    *backend = (ggml_backend) {
504        /* .guid       = */ ggml_backend_zdnn_guid(),
505        /* .iface      = */ ggml_backend_zdnn_i,
506        /* .device     = */ dev,
507        /* .context    = */ ctx
508    };
509
510    return backend;
511
512    GGML_UNUSED(params);
513}
514
515static ggml_backend_buffer_type_t ggml_backend_zdnn_device_get_buffer_type(ggml_backend_dev_t dev) {
516    return ggml_backend_zdnn_buffer_type();
517
518    GGML_UNUSED(dev);
519}
520
521static bool ggml_backend_zdnn_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
522    ggml_backend_zdnn_device_context * ctx_dev = (ggml_backend_zdnn_device_context *) dev->context;
523
524    return ggml_zdnn_supports_op(ctx_dev, op);
525}
526
527static bool ggml_backend_zdnn_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
528    return
529        buft->iface.get_name == ggml_backend_zdnn_buffer_type_get_name;
530
531    GGML_UNUSED(dev);
532}
533
534static ggml_backend_device_i ggml_backend_zdnn_device_i = {
535    /* .get_name             = */ ggml_backend_zdnn_device_get_name,
536    /* .get_description      = */ ggml_backend_zdnn_device_get_description,
537    /* .get_memory           = */ ggml_backend_zdnn_device_get_memory,
538    /* .get_type             = */ ggml_backend_zdnn_device_get_type,
539    /* .get_props            = */ ggml_backend_zdnn_device_get_props,
540    /* .init_backend         = */ ggml_backend_zdnn_device_init,
541    /* .get_buffer_type      = */ ggml_backend_zdnn_device_get_buffer_type,
542    /* .get_host_buffer_type = */ NULL,
543    /* .buffer_from_host_ptr = */ NULL,
544    /* .supports_op          = */ ggml_backend_zdnn_device_supports_op,
545    /* .supports_buft        = */ ggml_backend_zdnn_device_supports_buft,
546    /* .offload_op           = */ NULL,
547    /* .event_new            = */ NULL,
548    /* .event_free           = */ NULL,
549    /* .event_synchronize    = */ NULL,
550};
551
552//
553// backend registry
554//
555
556static const char * ggml_backend_zdnn_reg_get_name(ggml_backend_reg_t reg) {
557    return GGML_ZDNN_NAME;
558
559    GGML_UNUSED(reg);
560}
561
562static size_t ggml_backend_zdnn_reg_device_count(ggml_backend_reg_t reg) {
563    if (!zdnn_is_nnpa_installed()) {
564        return 0;
565    }
566    return 1;
567
568    GGML_UNUSED(reg);
569}
570
571static ggml_backend_dev_t ggml_backend_zdnn_reg_device_get(ggml_backend_reg_t reg, size_t index) {
572    GGML_ASSERT(index == 0);
573
574    return &g_ggml_backend_zdnn_device;
575
576    GGML_UNUSED(reg);
577    GGML_UNUSED(index);
578}
579
580static ggml_backend_feature g_ggml_backend_zdnn_features[] = {
581    { "NNPA", zdnn_is_nnpa_installed() ? "1" : "0" },
582    { "NNPA_PARMBLKFORMAT_0", zdnn_is_nnpa_parmblk_fmt_installed(1, NNPA_PARMBLKFORMAT_0) ? "1" : "0" },
583    { "NNPA_PARMBLKFORMAT_1", zdnn_is_nnpa_parmblk_fmt_installed(1, NNPA_PARMBLKFORMAT_1) ? "1" : "0" },
584    { NULL, NULL },
585};
586
587static ggml_backend_feature * ggml_backend_zdnn_get_features(ggml_backend_reg_t reg) {
588    return g_ggml_backend_zdnn_features;
589
590    GGML_UNUSED(reg);
591}
592
593static void * ggml_backend_zdnn_get_proc_address(ggml_backend_reg_t reg, const char * name) {
594    if (strcmp(name, "ggml_backend_get_features") == 0) {
595        return (void *) ggml_backend_zdnn_get_features;
596    }
597
598    return NULL;
599
600    GGML_UNUSED(reg);
601}
602
603static ggml_backend_reg_i ggml_backend_zdnn_reg_i = {
604    /* .get_name         = */ ggml_backend_zdnn_reg_get_name,
605    /* .get_device_count = */ ggml_backend_zdnn_reg_device_count,
606    /* .get_device       = */ ggml_backend_zdnn_reg_device_get,
607    /* .get_proc_address = */ ggml_backend_zdnn_get_proc_address
608};
609
610static void ggml_zdnn_cleanup(void) {
611    ggml_backend_zdnn_device_rel(&g_ggml_ctx_dev_main);
612}
613
614// TODO: make thread-safe
615ggml_backend_reg_t ggml_backend_zdnn_reg(void) {
616    ggml_backend_zdnn_device_acq(&g_ggml_ctx_dev_main);
617
618    // register cleanup callback
619    atexit(ggml_zdnn_cleanup);
620
621    {
622        g_ggml_backend_zdnn_reg = (ggml_backend_reg) {
623            /* .api_version = */ GGML_ZDNN_VERSION,
624            /* .iface       = */ ggml_backend_zdnn_reg_i,
625            /* .context     = */ NULL
626        };
627
628        g_ggml_backend_zdnn_device = (ggml_backend_device) {
629            /* .iface       = */ ggml_backend_zdnn_device_i,
630            /* .reg         = */ &g_ggml_backend_zdnn_reg,
631            /* .context     = */ &g_ggml_ctx_dev_main
632        };
633
634        return &g_ggml_backend_zdnn_reg;
635    }
636}
637
638GGML_BACKEND_DL_IMPL(ggml_backend_zdnn_reg)