char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

JusteLeoggml-virtgpu: fix circular dependency in headers (#22557)09294365a

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
18.9 KiB545 linesraw
1#include "virtgpu.h"
2#include "ggml-remoting.h"
3
4#include <stdio.h>
5#include <unistd.h>
6
7#include <cassert>
8#include <cerrno>
9#include <cstdlib>
10
11static virt_gpu_result_t virtgpu_open_device(virtgpu * gpu, const drmDevicePtr dev);
12static virt_gpu_result_t virtgpu_open(virtgpu * gpu);
13
14static virt_gpu_result_t virtgpu_init_capset(virtgpu * gpu);
15static virt_gpu_result_t virtgpu_init_context(virtgpu * gpu);
16
17static int      virtgpu_ioctl_context_init(virtgpu * gpu, virgl_renderer_capset capset_id);
18static int      virtgpu_ioctl_get_caps(virtgpu *             gpu,
19                                       virgl_renderer_capset id,
20                                       uint32_t              version,
21                                       void *                capset,
22                                       size_t                capset_size);
23static uint64_t virtgpu_ioctl_getparam(virtgpu * gpu, uint64_t param);
24static void     virtgpu_init_renderer_info(virtgpu * gpu);
25
26static void log_call_duration(long long call_duration_ns, const char * name);
27
28const uint64_t APIR_HANDSHAKE_MAX_WAIT_MS   = 2 * 1000;   // 2s
29const uint64_t APIR_LOADLIBRARY_MAX_WAIT_MS = 60 * 1000;  // 60s
30
31static int virtgpu_handshake(virtgpu * gpu) {
32    apir_encoder * encoder;
33    apir_decoder * decoder;
34
35    encoder = remote_call_prepare(gpu, APIR_COMMAND_TYPE_HANDSHAKE, 0);
36    if (!encoder) {
37        GGML_ABORT(GGML_VIRTGPU "%s: failed to prepare the remote call encoder", __func__);
38        return 1;
39    }
40
41    /* write handshake props */
42
43    uint32_t guest_major = APIR_PROTOCOL_MAJOR;
44    uint32_t guest_minor = APIR_PROTOCOL_MINOR;
45    apir_encode_uint32_t(encoder, &guest_major);
46    apir_encode_uint32_t(encoder, &guest_minor);
47
48    /* *** */
49
50    uint32_t  ret_magic;
51    long long call_duration_ns;
52    ret_magic = remote_call(gpu, encoder, &decoder, APIR_HANDSHAKE_MAX_WAIT_MS, &call_duration_ns);
53    log_call_duration(call_duration_ns, "API Remoting handshake");
54
55    if (!decoder) {
56        GGML_ABORT(GGML_VIRTGPU
57                   "%s: failed to initiate the communication with the virglrenderer library. "
58                   "Most likely, the wrong virglrenderer library was loaded in the hypervisor.",
59                   __func__);
60        return 1;
61    }
62
63    /* read handshake return values */
64
65    uint32_t host_major;
66    uint32_t host_minor;
67
68    if (ret_magic != APIR_HANDSHAKE_MAGIC) {
69        GGML_ABORT(GGML_VIRTGPU "%s: handshake with the virglrenderer failed (code=%d | %s)", __func__, ret_magic,
70                   apir_backend_initialize_error(ret_magic));
71    } else {
72        apir_decode_uint32_t(decoder, &host_major);
73        apir_decode_uint32_t(decoder, &host_minor);
74    }
75
76    remote_call_finish(gpu, encoder, decoder);
77
78    if (ret_magic != APIR_HANDSHAKE_MAGIC) {
79        return 1;
80    }
81
82    GGML_LOG_INFO(GGML_VIRTGPU "%s: Guest is running with %u.%u\n", __func__, guest_major, guest_minor);
83    GGML_LOG_INFO(GGML_VIRTGPU "%s: Host is running with %u.%u\n", __func__, host_major, host_minor);
84
85    if (guest_major != host_major) {
86        GGML_LOG_ERROR(GGML_VIRTGPU "Host major (%d) and guest major (%d) version differ\n", host_major, guest_major);
87    } else if (guest_minor != host_minor) {
88        GGML_LOG_WARN(GGML_VIRTGPU "Host minor (%d) and guest minor (%d) version differ\n", host_minor, guest_minor);
89    }
90
91    return 0;
92}
93
94static ApirLoadLibraryReturnCode virtgpu_load_library(virtgpu * gpu) {
95    apir_encoder *            encoder;
96    apir_decoder *            decoder;
97    ApirLoadLibraryReturnCode ret;
98
99    encoder = remote_call_prepare(gpu, APIR_COMMAND_TYPE_LOADLIBRARY, 0);
100    if (!encoder) {
101        GGML_ABORT(GGML_VIRTGPU "%s: hypercall error: failed to prepare the API Remoting command encoder", __func__);
102        return APIR_LOAD_LIBRARY_HYPERCALL_INITIALIZATION_ERROR;
103    }
104
105    long long call_duration_ns;
106
107    ret = (ApirLoadLibraryReturnCode) remote_call(gpu, encoder, &decoder, APIR_LOADLIBRARY_MAX_WAIT_MS,
108                                                  &call_duration_ns);
109    log_call_duration(call_duration_ns, "API Remoting LoadLibrary");
110
111    if (!decoder) {
112        GGML_ABORT(GGML_VIRTGPU "%s: hypercall error: failed to trigger the API Remoting hypercall.\n", __func__);
113        return APIR_LOAD_LIBRARY_HYPERCALL_INITIALIZATION_ERROR;
114    }
115
116    remote_call_finish(gpu, encoder, decoder);
117
118    if (ret == APIR_LOAD_LIBRARY_SUCCESS) {
119        GGML_LOG_INFO(GGML_VIRTGPU "The API Remoting backend was successfully loaded and initialized\n");
120
121        return ret;
122    }
123
124    // something wrong happened, find out what.
125    if (ret < APIR_LOAD_LIBRARY_INIT_BASE_INDEX) {
126        if (ret == APIR_LOAD_LIBRARY_ENV_VAR_MISSING) {
127            GGML_ABORT(GGML_VIRTGPU
128                       "%s: virglrenderer could not open the API Remoting backend library, "
129                       "some environment variables are missing. "
130                       "Make sure virglrenderer is correctly configured by the hypervisor. (%s)",
131                       __func__, apir_load_library_error(ret));
132        } else if (ret == APIR_LOAD_LIBRARY_CANNOT_OPEN) {
133            GGML_ABORT(GGML_VIRTGPU
134                       "%s: virglrenderer could not open the API Remoting backend library. "
135                       "Make sure virglrenderer is correctly configured by the hypervisor. (%s)",
136                       __func__, apir_load_library_error(ret));
137        } else if (ret == APIR_LOAD_LIBRARY_ENV_VAR_MISSING) {
138            GGML_ABORT(GGML_VIRTGPU
139                       "%s: could not load the backend library, some symbols are missing. "
140                       "Make sure virglrenderer is correctly configured by the hypervisor. (%s) ",
141                       __func__, apir_load_library_error(ret));
142        } else {
143            GGML_ABORT(GGML_VIRTGPU "%s: virglrenderer could not load the API Remoting backend library. (%s - code %d)",
144                       __func__, apir_load_library_error(ret), ret);
145        }
146        return ret;
147    }
148
149    GGML_LOG_INFO(GGML_VIRTGPU "%s: virglrenderer successfully loaded the API Remoting backend library.\n", __func__);
150
151    ApirLoadLibraryReturnCode apir_ret = (ApirLoadLibraryReturnCode) (ret - APIR_LOAD_LIBRARY_INIT_BASE_INDEX);
152
153    if (apir_ret == APIR_LOAD_LIBRARY_CANNOT_OPEN) {
154        GGML_ABORT(GGML_VIRTGPU
155                   "%s: the API Remoting backend library couldn't load the GGML backend library. "
156                   "Make sure virglrenderer is correctly configured by the hypervisor. (%s)",
157                   __func__, apir_load_library_error(apir_ret));
158    } else if (apir_ret == APIR_LOAD_LIBRARY_SYMBOL_MISSING) {
159        GGML_ABORT(
160            GGML_VIRTGPU
161            "%s: the API Remoting backend library couldn't load the GGML backend library, some symbols are missing. "
162            "Make sure virglrenderer is correctly configured by the hypervisor. (%s)",
163            __func__, apir_load_library_error(apir_ret));
164    } else if (apir_ret < APIR_LOAD_LIBRARY_INIT_BASE_INDEX) {
165        GGML_ABORT(GGML_VIRTGPU
166                   "%s: the API Remoting backend library couldn't load the GGML backend library: apir code=%d | %s)",
167                   __func__, apir_ret, apir_load_library_error(apir_ret));
168    } else {
169        uint32_t lib_ret = apir_ret - APIR_LOAD_LIBRARY_INIT_BASE_INDEX;
170        GGML_ABORT(GGML_VIRTGPU
171                   "%s: the API Remoting backend library failed to initialize its backend library: apir code=%d)",
172                   __func__, lib_ret);
173    }
174    return ret;
175}
176
177virtgpu * create_virtgpu() {
178    virtgpu * gpu = new virtgpu();
179
180    gpu->use_apir_capset = getenv("GGML_REMOTING_USE_APIR_CAPSET") != nullptr;
181    util_sparse_array_init(&gpu->shmem_array, sizeof(virtgpu_shmem), 1024);
182
183    // Initialize mutex to protect shared data_shmem buffer
184    if (mtx_init(&gpu->data_shmem_mutex, mtx_plain) != thrd_success) {
185        delete gpu;
186        GGML_ABORT(GGML_VIRTGPU "%s: failed to initialize data_shmem mutex", __func__);
187        return NULL;
188    }
189
190    if (virtgpu_open(gpu) != APIR_SUCCESS) {
191        GGML_LOG_ERROR(GGML_VIRTGPU "%s: failed to open the virtgpu device\n", __func__);
192        return NULL;
193    }
194
195    if (virtgpu_init_capset(gpu) != APIR_SUCCESS) {
196        if (gpu->use_apir_capset) {
197            GGML_ABORT(GGML_VIRTGPU
198                       "%s: failed to initialize the virtgpu APIR capset. Make sure that the virglrenderer library "
199                       "supports it.",
200                       __func__);
201        } else {
202            GGML_ABORT(GGML_VIRTGPU "%s: failed to initialize the virtgpu Venus capset", __func__);
203        }
204        return NULL;
205    }
206
207    if (virtgpu_init_context(gpu) != APIR_SUCCESS) {
208        GGML_ABORT(GGML_VIRTGPU "%s: failed to initialize the GPU context", __func__);
209        return NULL;
210    }
211
212    if (virtgpu_shmem_create(gpu, SHMEM_REPLY_SIZE, &gpu->reply_shmem)) {
213        GGML_ABORT(GGML_VIRTGPU "%s: failed to create the shared reply memory pages", __func__);
214        return NULL;
215    }
216
217    if (virtgpu_shmem_create(gpu, SHMEM_DATA_SIZE, &gpu->data_shmem)) {
218        GGML_ABORT(GGML_VIRTGPU "%s: failed to create the shared data memory pages", __func__);
219        return NULL;
220    }
221
222    if (virtgpu_handshake(gpu)) {
223        GGML_ABORT(GGML_VIRTGPU "%s: failed to handshake with the virglrenderer library", __func__);
224        return NULL;
225    }
226
227    if (virtgpu_load_library(gpu) != APIR_LOAD_LIBRARY_SUCCESS) {
228        GGML_ABORT(GGML_VIRTGPU "%s: failed to load the backend library", __func__);
229        return NULL;
230    }
231
232    return gpu;
233}
234
235static virt_gpu_result_t virtgpu_open(virtgpu * gpu) {
236    drmDevicePtr devs[8];
237    int          count = drmGetDevices2(0, devs, ARRAY_SIZE(devs));
238    if (count < 0) {
239        GGML_LOG_ERROR(GGML_VIRTGPU "%s: failed to enumerate DRM devices\n", __func__);
240        return APIR_ERROR_INITIALIZATION_FAILED;
241    }
242
243    virt_gpu_result_t result = APIR_ERROR_INITIALIZATION_FAILED;
244    for (int i = 0; i < count; i++) {
245        result = virtgpu_open_device(gpu, devs[i]);
246        if (result == APIR_SUCCESS) {
247            break;
248        }
249    }
250
251    drmFreeDevices(devs, count);
252
253    return result;
254}
255
256static virt_gpu_result_t virtgpu_open_device(virtgpu * gpu, const drmDevicePtr dev) {
257    const char * node_path = dev->nodes[DRM_NODE_RENDER];
258
259    int fd = open(node_path, O_RDWR | O_CLOEXEC);
260    if (fd < 0) {
261        GGML_ABORT(GGML_VIRTGPU "%s: failed to open %s", __func__, node_path);
262        return APIR_ERROR_INITIALIZATION_FAILED;
263    }
264
265    drmVersionPtr version = drmGetVersion(fd);
266    if (!version || strcmp(version->name, "virtio_gpu") || version->version_major != 0) {
267        if (version) {
268            GGML_LOG_ERROR(GGML_VIRTGPU "%s: unknown DRM driver %s version %d\n", __func__, version->name,
269                           version->version_major);
270        } else {
271            GGML_LOG_ERROR(GGML_VIRTGPU "%s: failed to get DRM driver version\n", __func__);
272        }
273
274        if (version) {
275            drmFreeVersion(version);
276        }
277        close(fd);
278        return APIR_ERROR_INITIALIZATION_FAILED;
279    }
280
281    gpu->fd = fd;
282
283    drmFreeVersion(version);
284
285    GGML_LOG_INFO(GGML_VIRTGPU "using DRM device %s\n", node_path);
286
287    return APIR_SUCCESS;
288}
289
290static virt_gpu_result_t virtgpu_init_context(virtgpu * gpu) {
291    assert(!gpu->capset.version);
292    const int ret = virtgpu_ioctl_context_init(gpu, gpu->capset.id);
293    if (ret) {
294        GGML_LOG_ERROR(GGML_VIRTGPU "%s: failed to initialize context: %s\n", __func__, strerror(errno));
295        return APIR_ERROR_INITIALIZATION_FAILED;
296    }
297
298    return APIR_SUCCESS;
299}
300
301static virt_gpu_result_t virtgpu_init_capset(virtgpu * gpu) {
302    if (gpu->use_apir_capset) {
303        GGML_LOG_INFO(GGML_VIRTGPU "Using the APIR capset\n");
304        gpu->capset.id = VIRTGPU_DRM_CAPSET_APIR;
305    } else {
306        GGML_LOG_INFO(GGML_VIRTGPU "Using the Venus capset\n");
307        gpu->capset.id = VIRTGPU_DRM_CAPSET_VENUS;
308    }
309    gpu->capset.version = 0;
310
311    int ret =
312        virtgpu_ioctl_get_caps(gpu, gpu->capset.id, gpu->capset.version, &gpu->capset.data, sizeof(gpu->capset.data));
313
314    if (ret) {
315        GGML_LOG_ERROR(GGML_VIRTGPU "%s: failed to get APIR v%d capset: %s\n", __func__, gpu->capset.version,
316                       strerror(errno));
317        return APIR_ERROR_INITIALIZATION_FAILED;
318    }
319
320    assert(gpu->capset.data.supports_blob_resources);
321
322    return APIR_SUCCESS;
323}
324
325static int virtgpu_ioctl_context_init(virtgpu * gpu, virgl_renderer_capset capset_id) {
326    drm_virtgpu_context_set_param ctx_set_params[3] = {
327        {
328         .param = VIRTGPU_CONTEXT_PARAM_CAPSET_ID,
329         .value = capset_id,
330         },
331        {
332         .param = VIRTGPU_CONTEXT_PARAM_NUM_RINGS,
333         .value = 1,
334         },
335        {
336         .param = VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK,
337         .value = 0, /* don't generate drm_events on fence signaling */
338        },
339    };
340
341    drm_virtgpu_context_init args = {
342        .num_params     = ARRAY_SIZE(ctx_set_params),
343        .pad            = 0,
344        .ctx_set_params = (uintptr_t) &ctx_set_params,
345    };
346
347    return virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_CONTEXT_INIT, &args);
348}
349
350static int virtgpu_ioctl_get_caps(virtgpu *             gpu,
351                                  virgl_renderer_capset id,
352                                  uint32_t              version,
353                                  void *                capset,
354                                  size_t                capset_size) {
355    drm_virtgpu_get_caps args = {
356        .cap_set_id  = id,
357        .cap_set_ver = version,
358        .addr        = (uintptr_t) capset,
359        .size        = (__u32) capset_size,
360        .pad         = 0,
361    };
362
363    return virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_GET_CAPS, &args);
364}
365
366static uint64_t virtgpu_ioctl_getparam(virtgpu * gpu, uint64_t param) {
367    /* val must be zeroed because kernel only writes the lower 32 bits */
368    uint64_t             val  = 0;
369    drm_virtgpu_getparam args = {
370        .param = param,
371        .value = (uintptr_t) &val,
372    };
373
374    const int ret = virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_GETPARAM, &args);
375    return ret ? 0 : val;
376}
377
378apir_encoder * remote_call_prepare(virtgpu * gpu, ApirCommandType apir_cmd_type, int32_t cmd_flags) {
379    /*
380     * Prepare the command encoder and its buffer
381     */
382
383    thread_local char encoder_buffer[4096];
384
385    thread_local apir_encoder enc;
386    enc = {
387        .cur   = encoder_buffer,
388        .start = encoder_buffer,
389        .end   = encoder_buffer + sizeof(encoder_buffer),
390        .fatal = false,
391    };
392
393    /*
394     * Fill the command encoder with the common args:
395     * - cmd_type (int32_t)
396     * - cmd_flags (int32_t)
397     * - reply res id (uint32_t)
398   */
399
400    int32_t cmd_type = apir_cmd_type;
401
402    // for testing during the hypervisor transition
403    if (!gpu->use_apir_capset) {
404        cmd_type += VENUS_COMMAND_TYPE_LENGTH;
405    }
406    apir_encode_int32_t(&enc, &cmd_type);
407    apir_encode_int32_t(&enc, &cmd_flags);
408
409    uint32_t reply_res_id = gpu->reply_shmem.res_id;
410    apir_encode_uint32_t(&enc, &reply_res_id);
411
412    return &enc;
413}
414
415void remote_call_finish(virtgpu * gpu, apir_encoder * enc, apir_decoder * dec) {
416    UNUSED(gpu);
417
418    if (!enc) {
419        GGML_ABORT(GGML_VIRTGPU "%s: Invalid (null) encoder", __func__);
420    }
421
422    if (!dec) {
423        GGML_ABORT(GGML_VIRTGPU "%s: Invalid (null) decoder", __func__);
424    }
425
426    if (apir_encoder_get_fatal(enc)) {
427        GGML_LOG_ERROR(GGML_VIRTGPU "%s: Failed to encode the output parameters.", __func__);
428    }
429
430    if (apir_decoder_get_fatal(dec)) {
431        GGML_LOG_ERROR(GGML_VIRTGPU "%s: Failed to decode the input parameters.", __func__);
432    }
433}
434
435uint32_t remote_call(virtgpu *       gpu,
436                     apir_encoder *  encoder,
437                     apir_decoder ** decoder,
438                     float           max_wait_ms,
439                     long long *     call_duration_ns) {
440    /*
441     * Prepare the reply notification pointer
442     */
443
444    volatile std::atomic_uint * atomic_reply_notif = (volatile std::atomic_uint *) gpu->reply_shmem.mmap_ptr;
445    *atomic_reply_notif                            = 0;
446
447    /*
448     * Trigger the execbuf ioctl
449     */
450
451    drm_virtgpu_execbuffer args = {
452        .flags   = VIRTGPU_EXECBUF_RING_IDX,
453        .size    = (uint32_t) (encoder->cur - encoder->start),
454        .command = (uintptr_t) encoder->start,
455
456        .bo_handles     = 0,
457        .num_bo_handles = 0,
458
459        .fence_fd         = 0,
460        .ring_idx         = 0,
461        .syncobj_stride   = 0,
462        .num_in_syncobjs  = 0,
463        .num_out_syncobjs = 0,
464        .in_syncobjs      = 0,
465        .out_syncobjs     = 0,
466    };
467
468    *decoder = NULL;
469
470    int ret = drmIoctl(gpu->fd, DRM_IOCTL_VIRTGPU_EXECBUFFER, &args);
471
472    if (ret != 0) {
473        GGML_ABORT(GGML_VIRTGPU "%s: the virtgpu EXECBUFFER ioctl failed (%d)", __func__, ret);
474    }
475
476    /*
477     * Wait for the response notification
478     */
479    timer_data wait_host_reply_timer = { 0, 0, 0 };
480
481    start_timer(&wait_host_reply_timer);
482
483    timespec ts_start, ts_end;
484    clock_gettime(CLOCK_MONOTONIC, &ts_start);
485    long long start_time = (long long) ts_start.tv_sec * 1000000000LL + ts_start.tv_nsec;
486
487    bool     timedout    = false;
488    uint32_t notif_value = 0;
489    while (true) {
490        notif_value = std::atomic_load_explicit(atomic_reply_notif, std::memory_order_acquire);
491
492        if (notif_value != 0) {
493            break;
494        }
495
496        int64_t base_sleep_us = 15;
497
498        os_time_sleep(base_sleep_us);
499
500        if (max_wait_ms) {
501            clock_gettime(CLOCK_MONOTONIC, &ts_end);
502            long long end_time    = (long long) ts_end.tv_sec * 1000000000LL + ts_end.tv_nsec;
503            float     duration_ms = (end_time - start_time) / 1000000;
504
505            if (duration_ms > max_wait_ms) {
506                timedout = true;
507                break;
508            }
509        }
510    }
511
512    if (call_duration_ns) {
513        *call_duration_ns = stop_timer(&wait_host_reply_timer);
514    }
515
516    if (max_wait_ms && timedout) {
517        GGML_LOG_ERROR(GGML_VIRTGPU "%s: timed out waiting for the host answer...\n", __func__);
518        return APIR_FORWARD_TIMEOUT;
519    }
520
521    /*
522     * Prepare the decoder
523     */
524    static apir_decoder response_dec;
525    response_dec.cur = (char *) gpu->reply_shmem.mmap_ptr + sizeof(*atomic_reply_notif);
526    response_dec.end = (char *) gpu->reply_shmem.mmap_ptr + gpu->reply_shmem.mmap_size;
527    *decoder         = &response_dec;
528
529    // extract the actual return value from the notif flag
530    uint32_t returned_value = notif_value - 1;
531    return returned_value;
532}
533
534static void log_call_duration(long long call_duration_ns, const char * name) {
535    double call_duration_ms = (double) call_duration_ns / 1e6;  // 1 millisecond = 1e6 nanoseconds
536    double call_duration_s  = (double) call_duration_ns / 1e9;  // 1 second = 1e9 nanoseconds
537
538    if (call_duration_s > 1) {
539        GGML_LOG_INFO(GGML_VIRTGPU "waited %.2fs for the %s host reply...\n", call_duration_s, name);
540    } else if (call_duration_ms > 1) {
541        GGML_LOG_INFO(GGML_VIRTGPU "waited %.2fms for the %s host reply...\n", call_duration_ms, name);
542    } else {
543        GGML_LOG_INFO(GGML_VIRTGPU "waited %lldns for the %s host reply...\n", call_duration_ns, name);
544    }
545}