char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aman Guptaggml: allow passing alloc dependencies in graph_optimize (#27301)57291f264

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.6 KiB72 linesraw
1#include "../../include/ggml-virtgpu.h"
2#include "ggml-remoting.h"
3
4static const char * ggml_backend_remoting_get_name(ggml_backend_t backend) {
5    UNUSED(backend);
6
7    return "API Remoting backend";
8}
9
10static void ggml_backend_remoting_free(ggml_backend_t backend) {
11    delete backend;
12}
13
14static ggml_status ggml_backend_remoting_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
15    virtgpu * gpu = DEV_TO_GPU(backend->device);
16
17    return apir_backend_graph_compute(gpu, cgraph);
18}
19
20static void ggml_backend_remoting_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {
21    UNUSED(params);
22    virtgpu * gpu = DEV_TO_GPU(backend->device);
23#if true
24    UNUSED(gpu);
25    UNUSED(cgraph);
26#else
27    // not working yet
28
29    apir_backend_graph_optimize(gpu, cgraph);
30#endif
31}
32
33static ggml_backend_i ggml_backend_remoting_interface = {
34    /* .get_name                = */ ggml_backend_remoting_get_name,
35    /* .free                    = */ ggml_backend_remoting_free,
36    /* .set_tensor_async        = */ NULL,  // ggml_backend_remoting_set_tensor_async,
37    /* .get_tensor_async        = */ NULL,  // ggml_backend_remoting_get_tensor_async,
38    /* .set_tensor_2d_async     = */ NULL,
39    /* .get_tensor_2d_async     = */ NULL,
40    /* .cpy_tensor_async        = */ NULL,  // ggml_backend_remoting_cpy_tensor_async,
41    /* .synchronize             = */ NULL,  // ggml_backend_remoting_synchronize,
42    /* .graph_plan_create       = */ NULL,
43    /* .graph_plan_free         = */ NULL,
44    /* .graph_plan_update       = */ NULL,
45    /* .graph_plan_compute      = */ NULL,
46    /* .graph_compute           = */ ggml_backend_remoting_graph_compute,
47    /* .event_record            = */ NULL,
48    /* .event_wait              = */ NULL,
49    /* .graph_optimize          = */ ggml_backend_remoting_graph_optimize,
50};
51
52static ggml_guid_t ggml_backend_remoting_guid() {
53    static ggml_guid guid = { 0xb8, 0xf7, 0x4f, 0x86, 0x14, 0x03, 0x86, 0x02,
54                              0x91, 0xc8, 0xdd, 0xe9, 0x02, 0x3f, 0xc0, 0x2b };
55
56    return &guid;
57}
58
59ggml_backend_t ggml_backend_remoting_device_init(ggml_backend_dev_t dev, const char * params) {
60    UNUSED(params);
61
62    ggml_backend_remoting_device_context * ctx = (ggml_backend_remoting_device_context *) dev->context;
63
64    ggml_backend_t remoting_backend = new ggml_backend{
65        /* .guid      = */ ggml_backend_remoting_guid(),
66        /* .interface = */ ggml_backend_remoting_interface,
67        /* .device    = */ ggml_backend_reg_dev_get(ggml_backend_virtgpu_reg(), ctx->device),
68        /* .context   = */ ctx,
69    };
70
71    return remoting_backend;
72}