char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Ruben Ortlamllama: add default load-mode auto, which avoids mmap on iGPUs (#26081)153d324bc

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.6 KiB54 linesraw
1#pragma once
2
3/* device */
4void                           apir_device_get_device_count(struct virtgpu * gpu);
5int                            apir_device_get_count(struct virtgpu * gpu);
6char *                         apir_device_get_name(struct virtgpu * gpu);
7char *                         apir_device_get_description(struct virtgpu * gpu);
8uint32_t                       apir_device_get_type(struct virtgpu * gpu);
9void                           apir_device_get_memory(struct virtgpu * gpu, size_t * free, size_t * total);
10bool                           apir_device_supports_op(struct virtgpu * gpu, const ggml_tensor * op);
11apir_buffer_type_host_handle_t apir_device_get_buffer_type(struct virtgpu * gpu);
12void                           apir_device_get_props(struct virtgpu * gpu,
13                                                     bool *           async,
14                                                     bool *           host_buffer,
15                                                     bool *           buffer_from_host_ptr,
16                                                     bool *           events,
17                                                     bool *           mmap_support);
18apir_buffer_context_t          apir_device_buffer_from_ptr(struct virtgpu * gpu, size_t size, size_t max_tensor_size);
19
20/* buffer-type */
21char *                apir_buffer_type_get_name(struct virtgpu * gpu, apir_buffer_type_host_handle_t host_handle);
22size_t                apir_buffer_type_get_alignment(struct virtgpu * gpu, apir_buffer_type_host_handle_t host_handle);
23size_t                apir_buffer_type_get_max_size(struct virtgpu * gpu, apir_buffer_type_host_handle_t host_handle);
24/* apir_buffer_type_is_host is deprecated. */
25apir_buffer_context_t apir_buffer_type_alloc_buffer(struct virtgpu *               gpu,
26                                                    apir_buffer_type_host_handle_t host_handle,
27                                                    size_t                         size);
28size_t                apir_buffer_type_get_alloc_size(struct virtgpu *               gpu,
29                                                      apir_buffer_type_host_handle_t host_handle,
30                                                      const ggml_tensor *            op);
31
32/* buffer */
33void * apir_buffer_get_base(struct virtgpu * gpu, apir_buffer_context_t * buffer_context);
34void   apir_buffer_set_tensor(struct virtgpu *        gpu,
35                              apir_buffer_context_t * buffer_context,
36                              ggml_tensor *           tensor,
37                              const void *            data,
38                              size_t                  offset,
39                              size_t                  size);
40void   apir_buffer_get_tensor(struct virtgpu *        gpu,
41                              apir_buffer_context_t * buffer_context,
42                              const ggml_tensor *     tensor,
43                              void *                  data,
44                              size_t                  offset,
45                              size_t                  size);
46bool   apir_buffer_cpy_tensor(struct virtgpu *        gpu,
47                              apir_buffer_context_t * buffer_context,
48                              const ggml_tensor *     src,
49                              const ggml_tensor *     dst);
50void   apir_buffer_clear(struct virtgpu * gpu, apir_buffer_context_t * buffer_context, uint8_t value);
51void   apir_buffer_free_buffer(struct virtgpu * gpu, apir_buffer_context_t * buffer_context);
52
53/* backend */
54ggml_status apir_backend_graph_compute(struct virtgpu * gpu, ggml_cgraph * cgraph);