char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Kevin Pougetggml-virtgpu: improve the reliability of the code (#19846)ffaafde16

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
10.7 KiB378 linesraw
1#pragma once
2
3#include "ggml-impl.h"
4
5#include <cassert>
6#include <cstring>
7
8#define likely(x)   __builtin_expect(!!(x), 1)
9#define unlikely(x) __builtin_expect(!!(x), 0)
10
11struct apir_encoder {
12    char *       cur;
13    const char * start;
14    const char * end;
15    bool         fatal;
16};
17
18struct apir_decoder {
19    const char * cur;
20    const char * end;
21    bool         fatal;
22};
23
24/*
25 * new encoder and decoder
26 */
27
28static apir_decoder apir_new_decoder(const char * ptr, size_t size) {
29    apir_decoder dec = {
30        .cur   = ptr,
31        .end   = ptr + size,
32        .fatal = false,
33    };
34
35    return dec;
36}
37
38static apir_encoder apir_new_encoder(char * ptr, size_t size) {
39    apir_encoder enc = {
40        .cur   = ptr,
41        .start = ptr,
42        .end   = ptr + size,
43        .fatal = false,
44    };
45
46    return enc;
47}
48
49/*
50 * fatal flag handling
51 */
52
53static inline void apir_encoder_reset_fatal(apir_encoder * enc) {
54    enc->fatal = false;
55}
56
57static inline void apir_encoder_set_fatal(apir_encoder * enc) {
58    enc->fatal = true;
59}
60
61static inline bool apir_encoder_get_fatal(const apir_encoder * enc) {
62    return enc->fatal;
63}
64
65static inline void apir_decoder_reset_fatal(apir_decoder * dec) {
66    dec->fatal = false;
67}
68
69static inline void apir_decoder_set_fatal(apir_decoder * dec) {
70    dec->fatal = true;
71}
72
73static inline bool apir_decoder_get_fatal(const apir_decoder * dec) {
74    return dec->fatal;
75}
76
77/*
78 * encode peek
79 */
80
81static inline bool apir_decoder_peek_internal(apir_decoder * dec, size_t size, void * val, size_t val_size) {
82    assert(val_size <= size);
83
84    if (unlikely(size > (size_t) (dec->end - dec->cur))) {
85        GGML_LOG_ERROR("%s: reading too much from the decoder ...\n", __func__);
86        apir_decoder_set_fatal(dec);
87        memset(val, 0, val_size);
88        return false;
89    }
90
91    /* we should not rely on the compiler to optimize away memcpy... */
92    memcpy(val, dec->cur, val_size);
93    return true;
94}
95
96static inline void apir_decoder_peek(apir_decoder * dec, size_t size, void * val, size_t val_size) {
97    apir_decoder_peek_internal(dec, size, val, val_size);
98}
99
100static inline const void * apir_decoder_use_inplace(apir_decoder * dec, size_t size) {
101    if (unlikely(size > (size_t) (dec->end - dec->cur))) {
102        GGML_LOG_ERROR("%s: reading too much from the decoder ...\n", __func__);
103        apir_decoder_set_fatal(dec);
104        return NULL;
105    }
106    const void * addr = dec->cur;
107    dec->cur += size;
108
109    return addr;
110}
111
112/*
113 * read/write
114 */
115
116static inline void apir_decoder_read(apir_decoder * dec, size_t size, void * val, size_t val_size) {
117    if (apir_decoder_peek_internal(dec, size, val, val_size)) {
118        dec->cur += size;
119    }
120}
121
122static inline char * apir_encoder_write(apir_encoder * enc, size_t size, const void * val, size_t val_size) {
123    assert(val_size <= size);
124    assert(size <= ((size_t) (enc->end - enc->cur)));
125
126    char * write_addr = enc->cur;
127    /* we should not rely on the compiler to optimize away memcpy... */
128    memcpy(write_addr, val, val_size);
129    enc->cur += size;
130
131    return write_addr;
132}
133
134/*
135 * encode/decode
136 */
137
138static inline void apir_decode(apir_decoder * dec, size_t size, void * data, size_t data_size) {
139    assert(size % 4 == 0);
140    apir_decoder_read(dec, size, data, data_size);
141}
142
143static inline void apir_encode(apir_encoder * enc, size_t size, const void * data, size_t data_size) {
144    assert(size % 4 == 0);
145    apir_encoder_write(enc, size, data, data_size);
146}
147
148/*
149 * typed encode/decode
150 */
151
152/* uint8_t */
153
154static inline void apir_encode_uint8_t(apir_encoder * enc, const uint8_t * val) {
155    apir_encode(enc, sizeof(int), val, sizeof(*val));
156}
157
158static inline void apir_decode_uint8_t(apir_decoder * dec, uint8_t * val) {
159    apir_decode(dec, sizeof(int), val, sizeof(*val));
160}
161
162/* uint64_t */
163
164static inline void apir_encode_uint64_t(apir_encoder * enc, const uint64_t * val) {
165    apir_encode(enc, 8, val, sizeof(*val));
166}
167
168static inline void apir_decode_uint64_t(apir_decoder * dec, uint64_t * val) {
169    apir_decode(dec, 8, val, sizeof(*val));
170}
171
172static inline void apir_encode_uint64_t_array(apir_encoder * enc, const uint64_t * val, uint32_t count) {
173    const size_t size = sizeof(*val) * count;
174    assert(size >= count);
175    apir_encode(enc, size, val, size);
176}
177
178static inline void apir_decode_uint64_t_array(apir_decoder * dec, uint64_t * val, uint32_t count) {
179    const size_t size = sizeof(*val) * count;
180    assert(size >= count);
181    apir_decode(dec, size, val, size);
182}
183
184static inline const uint64_t * apir_decode_uint64_t_array_inplace(apir_decoder * dec, uint32_t count) {
185    return (uint64_t *) (uintptr_t) apir_decoder_use_inplace(dec, count * sizeof(uint64_t));
186}
187
188/* int32_t */
189
190static inline void apir_encode_int32_t(apir_encoder * enc, const int32_t * val) {
191    apir_encode(enc, 4, val, sizeof(*val));
192}
193
194static inline void apir_decode_int32_t(apir_decoder * dec, int32_t * val) {
195    apir_decode(dec, 4, val, sizeof(*val));
196}
197
198static inline void apir_encode_int32_t_array(apir_encoder * enc, const int32_t * val, uint32_t count) {
199    const size_t size = sizeof(*val) * count;
200    assert(size >= count);
201    apir_encode(enc, size, val, size);
202}
203
204static inline void apir_decode_int32_t_array(apir_decoder * dec, int32_t * val, uint32_t count) {
205    const size_t size = sizeof(*val) * count;
206    assert(size >= count);
207    apir_decode(dec, size, val, size);
208}
209
210/* array size (uint64_t) */
211
212static inline void apir_encode_array_size(apir_encoder * enc, uint64_t size) {
213    apir_encode_uint64_t(enc, &size);
214}
215
216static inline uint64_t apir_decode_array_size(apir_decoder * dec, uint64_t expected_size) {
217    uint64_t size;
218    apir_decode_uint64_t(dec, &size);
219    if (size != expected_size) {
220        GGML_LOG_ERROR("%s: Couldn't decode array from the decoder\n", __func__);
221        apir_decoder_set_fatal(dec);
222        size = 0;
223    }
224    return size;
225}
226
227static inline uint64_t apir_decode_array_size_unchecked(apir_decoder * dec) {
228    uint64_t size;
229    apir_decode_uint64_t(dec, &size);
230    return size;
231}
232
233/* non-array pointer */
234
235static inline bool apir_encode_simple_pointer(apir_encoder * enc, const void * val) {
236    apir_encode_array_size(enc, val ? 1 : 0);
237    return val;
238}
239
240static inline bool apir_decode_simple_pointer(apir_decoder * dec) {
241    return apir_decode_array_size_unchecked(dec);
242}
243
244/* uint32_t */
245
246static inline void apir_encode_uint32_t(apir_encoder * enc, const uint32_t * val) {
247    apir_encode(enc, 4, val, sizeof(*val));
248}
249
250static inline void apir_decode_uint32_t(apir_decoder * dec, uint32_t * val) {
251    apir_decode(dec, 4, val, sizeof(*val));
252}
253
254static inline void apir_encode_uint32_t_array(apir_encoder * enc, const uint32_t * val, uint32_t count) {
255    const size_t size = sizeof(*val) * count;
256    assert(size >= count);
257    apir_encode(enc, size, val, size);
258}
259
260static inline void apir_decode_uint32_t_array(apir_decoder * dec, uint32_t * val, uint32_t count) {
261    const size_t size = sizeof(*val) * count;
262    assert(size >= count);
263    apir_decode(dec, size, val, size);
264}
265
266/* size_t */
267
268static inline void apir_encode_size_t(apir_encoder * enc, const size_t * val) {
269    const uint64_t tmp = *val;
270    apir_encode_uint64_t(enc, &tmp);
271}
272
273static inline void apir_decode_size_t(apir_decoder * dec, size_t * val) {
274    uint64_t tmp;
275    apir_decode_uint64_t(dec, &tmp);
276    *val = tmp;
277}
278
279static inline void apir_encode_size_t_array(apir_encoder * enc, const size_t * val, uint32_t count) {
280    if (sizeof(size_t) == sizeof(uint64_t)) {
281        apir_encode_uint64_t_array(enc, (const uint64_t *) val, count);
282    } else {
283        for (uint32_t i = 0; i < count; i++) {
284            apir_encode_size_t(enc, &val[i]);
285        }
286    }
287}
288
289static inline void apir_decode_size_t_array(apir_decoder * dec, size_t * val, uint32_t count) {
290    if (sizeof(size_t) == sizeof(uint64_t)) {
291        apir_decode_uint64_t_array(dec, (uint64_t *) val, count);
292    } else {
293        for (uint32_t i = 0; i < count; i++) {
294            apir_decode_size_t(dec, &val[i]);
295        }
296    }
297}
298
299/* opaque blob */
300
301static inline void apir_encode_blob_array(apir_encoder * enc, const void * val, size_t size) {
302    apir_encode(enc, (size + 3) & ~3, val, size);
303}
304
305static inline void apir_decode_blob_array(apir_decoder * dec, void * val, size_t size) {
306    apir_decode(dec, (size + 3) & ~3, val, size);
307}
308
309/* string */
310
311static inline void apir_encode_char_array(apir_encoder * enc, const char * val, size_t size) {
312    assert(size && strlen(val) < size);
313    apir_encode_blob_array(enc, val, size);
314}
315
316static inline void apir_decode_char_array(apir_decoder * dec, char * val, size_t size) {
317    apir_decode_blob_array(dec, val, size);
318    if (size) {
319        val[size - 1] = '\0';
320    } else {
321        GGML_LOG_ERROR("%s: Couldn't decode the blog array\n", __func__);
322        apir_decoder_set_fatal(dec);
323    }
324}
325
326/* (temp) buffer allocation */
327
328static inline void * apir_decoder_alloc_array(size_t size, size_t count) {
329    size_t alloc_size;
330    if (unlikely(__builtin_mul_overflow(size, count, &alloc_size))) {
331        GGML_LOG_ERROR("%s: overflow in array allocation of %zu * %zu bytes\n", __func__, size, count);
332        return NULL;
333    }
334
335    return malloc(alloc_size);
336}
337
338/* bool */
339
340static inline void apir_encode_bool_t(apir_encoder * enc, const bool * val) {
341    apir_encode(enc, sizeof(int), val, sizeof(bool));
342}
343
344static inline void apir_decode_bool_t(apir_decoder * dec, bool * val) {
345    apir_decode(dec, sizeof(int), val, sizeof(bool));
346}
347
348/* apir_buffer_type_host_handle_t */
349
350static inline void apir_encode_apir_buffer_type_host_handle_t(apir_encoder *                         enc,
351                                                              const apir_buffer_type_host_handle_t * val) {
352    apir_encode(enc, sizeof(apir_buffer_type_host_handle_t), val, sizeof(apir_buffer_type_host_handle_t));
353}
354
355static inline void apir_decode_apir_buffer_type_host_handle_t(apir_decoder *                   dec,
356                                                              apir_buffer_type_host_handle_t * val) {
357    apir_decode(dec, sizeof(apir_buffer_type_host_handle_t), val, sizeof(apir_buffer_type_host_handle_t));
358}
359
360/* apir_buffer_host_handle_t */
361
362static inline void apir_encode_apir_buffer_host_handle_t(apir_encoder * enc, const apir_buffer_host_handle_t * val) {
363    apir_encode(enc, sizeof(apir_buffer_host_handle_t), val, sizeof(apir_buffer_host_handle_t));
364}
365
366static inline void apir_decode_apir_buffer_host_handle_t(apir_decoder * dec, apir_buffer_host_handle_t * val) {
367    apir_decode(dec, sizeof(apir_buffer_host_handle_t), val, sizeof(apir_buffer_host_handle_t));
368}
369
370/* uintptr_t */
371
372static inline void apir_encode_uintptr_t(apir_encoder * enc, const uintptr_t * val) {
373    apir_encode(enc, sizeof(*val), val, sizeof(*val));
374}
375
376static inline void apir_decode_uintptr_t(apir_decoder * dec, uintptr_t * val) {
377    apir_decode(dec, sizeof(*val), val, sizeof(*val));
378}