char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Hongqiang Wangopencl: cache compiled cl_program binaries on disk (#26050)ed7adbfef

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
15.8 KiB453 linesraw
1// Match the version setup ggml-opencl.cpp uses, so any cl.h declarations we
2// touch are consistent across this backend's translation units.
3#define CL_TARGET_OPENCL_VERSION GGML_OPENCL_TARGET_VERSION
4#define CL_USE_DEPRECATED_OPENCL_1_2_APIS
5
6#include "cl-program-cache.h"
7
8#include "ggml-impl.h"  // GGML_LOG_INFO / WARN
9
10#include <cstdint>
11#include <cstdio>
12#include <cstdlib>
13#include <cstring>
14#include <filesystem>
15#include <fstream>
16#include <system_error>
17#include <vector>
18
19#if defined(_WIN32)
20#  ifndef WIN32_LEAN_AND_MEAN
21#    define WIN32_LEAN_AND_MEAN
22#  endif
23#  ifndef NOMINMAX
24#    define NOMINMAX
25#  endif
26#  include <windows.h>
27#  include <process.h>
28#  define ggml_getpid()    ((int) GetCurrentProcessId())
29#else
30#  include <unistd.h>
31#  define ggml_getpid()    ((int) getpid())
32#endif
33
34namespace fs = std::filesystem;
35
36// ----------------------------------------------------------------------------
37// SHA-256 (FIPS 180-4). Self-contained, ~80 lines, public-domain reference.
38// Hot path is a few KB of source per kernel ⇒ <1 ms total per process init.
39// ----------------------------------------------------------------------------
40
41namespace {
42
43struct sha256_ctx {
44    uint32_t state[8];
45    uint64_t bitlen;
46    uint8_t  buf[64];
47    size_t   buf_len;
48};
49
50const uint32_t K256[64] = {
51    0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
52    0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
53    0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
54    0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
55    0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
56    0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
57    0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
58    0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2,
59};
60
61inline uint32_t rotr32(uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); }
62
63void sha256_compress(uint32_t state[8], const uint8_t block[64]) {
64    uint32_t w[64];
65    for (int i = 0; i < 16; ++i) {
66        w[i] = ((uint32_t)block[i*4    ] << 24) |
67               ((uint32_t)block[i*4 + 1] << 16) |
68               ((uint32_t)block[i*4 + 2] <<  8) |
69               ((uint32_t)block[i*4 + 3]      );
70    }
71    for (int i = 16; i < 64; ++i) {
72        uint32_t s0 = rotr32(w[i-15], 7) ^ rotr32(w[i-15], 18) ^ (w[i-15] >> 3);
73        uint32_t s1 = rotr32(w[i-2], 17) ^ rotr32(w[i-2],  19) ^ (w[i-2]  >> 10);
74        w[i] = w[i-16] + s0 + w[i-7] + s1;
75    }
76
77    uint32_t a = state[0],b = state[1],c = state[2],d = state[3],e = state[4],f = state[5],g = state[6],h = state[7];
78
79    for (int i = 0; i < 64; ++i) {
80        uint32_t S1   = rotr32(e, 6) ^ rotr32(e, 11) ^ rotr32(e, 25);
81        uint32_t ch   = (e & f) ^ ((~e) & g);
82        uint32_t t1   = h + S1 + ch + K256[i] + w[i];
83        uint32_t S0   = rotr32(a, 2) ^ rotr32(a, 13) ^ rotr32(a, 22);
84        uint32_t maj  = (a & b) ^ (a & c) ^ (b & c);
85        uint32_t t2   = S0 + maj;
86        h = g; g = f; f = e; e = d + t1;
87        d = c; c = b; b = a; a = t1 + t2;
88    }
89    state[0]+=a; state[1]+=b; state[2]+=c; state[3]+=d;
90    state[4]+=e; state[5]+=f; state[6]+=g; state[7]+=h;
91}
92
93void sha256_init(sha256_ctx & c) {
94    c.state[0]=0x6a09e667; c.state[1]=0xbb67ae85; c.state[2]=0x3c6ef372; c.state[3]=0xa54ff53a;
95    c.state[4]=0x510e527f; c.state[5]=0x9b05688c; c.state[6]=0x1f83d9ab; c.state[7]=0x5be0cd19;
96    c.bitlen  = 0;
97    c.buf_len = 0;
98}
99
100void sha256_update(sha256_ctx & c, const void * data, size_t len) {
101    const uint8_t * p = (const uint8_t *) data;
102    c.bitlen += (uint64_t) len * 8;
103    if (c.buf_len > 0) {
104        size_t n = 64 - c.buf_len;
105        if (n > len) { n = len; }
106        memcpy(c.buf + c.buf_len, p, n);
107        c.buf_len += n;
108        p   += n;
109        len -= n;
110        if (c.buf_len == 64) {
111            sha256_compress(c.state, c.buf);
112            c.buf_len = 0;
113        }
114    }
115    while (len >= 64) {
116        sha256_compress(c.state, p);
117        p   += 64;
118        len -= 64;
119    }
120    if (len > 0) {
121        memcpy(c.buf, p, len);
122        c.buf_len = len;
123    }
124}
125
126void sha256_final(sha256_ctx & c, uint8_t out[32]) {
127    uint64_t bitlen = c.bitlen;
128    c.buf[c.buf_len++] = 0x80;
129    if (c.buf_len > 56) {
130        while (c.buf_len < 64) { c.buf[c.buf_len++] = 0; }
131        sha256_compress(c.state, c.buf);
132        c.buf_len = 0;
133    }
134    while (c.buf_len < 56) { c.buf[c.buf_len++] = 0; }
135    for (int i = 7; i >= 0; --i) { c.buf[c.buf_len++] = (uint8_t) (bitlen >> (i * 8)); }
136    sha256_compress(c.state, c.buf);
137    for (int i = 0; i < 8; ++i) {
138        out[i*4    ] = (uint8_t) (c.state[i] >> 24);
139        out[i*4 + 1] = (uint8_t) (c.state[i] >> 16);
140        out[i*4 + 2] = (uint8_t) (c.state[i] >>  8);
141        out[i*4 + 3] = (uint8_t) (c.state[i]      );
142    }
143}
144
145std::string sha256_hex(const uint8_t digest[32]) {
146    static const char hex[] = "0123456789abcdef";
147    std::string s(64, '0');
148    for (int i = 0; i < 32; ++i) {
149        s[i*2    ] = hex[digest[i] >> 4];
150        s[i*2 + 1] = hex[digest[i] & 0xf];
151    }
152    return s;
153}
154
155std::string compute_key(const std::string & key_suffix,
156                        const char *        source,
157                        const std::string & compile_opts) {
158    sha256_ctx c;
159    sha256_init(c);
160
161    static const uint8_t sep = 0;
162    sha256_update(c, source,             strlen(source));
163    sha256_update(c, &sep,               1);
164    sha256_update(c, compile_opts.data(), compile_opts.size());
165    sha256_update(c, &sep,               1);
166    sha256_update(c, key_suffix.data(),  key_suffix.size());
167
168    uint8_t digest[32];
169    sha256_final(c, digest);
170    return sha256_hex(digest);
171}
172
173bool make_dir_recursive(const std::string & path) {
174    if (path.empty()) { return false; }
175    // create_directories() already creates missing parents. It returns false
176    // (with ec clear) when the directory is already there, so re-check.
177    const fs::path p = fs::u8path(path);
178    std::error_code ec;
179    if (fs::create_directories(p, ec)) { return true; }
180    std::error_code ec_stat;
181    return fs::is_directory(p, ec_stat);
182}
183
184std::string default_cache_dir() {
185#if defined(_WIN32)
186    const char * base = std::getenv("LOCALAPPDATA");
187    if (!base || !*base) { base = std::getenv("APPDATA"); }
188    if (!base || !*base) { base = std::getenv("TEMP"); }
189    if (!base || !*base) { base = "."; }
190    return std::string(base) + "\\llama.cpp\\cl-cache";
191#elif defined(__APPLE__)
192    const char * home = std::getenv("HOME");
193    if (!home || !*home) { home = "."; }
194    return std::string(home) + "/Library/Caches/llama.cpp/cl-cache";
195#else
196    // The throwing overload aborts the process when no usable temp directory
197    // exists (e.g. Android app contexts with TMPDIR unset); an empty return
198    // here just disables the cache instead.
199    std::error_code ec;
200    const fs::path tmp_path = fs::temp_directory_path(ec);
201    if (ec || tmp_path.empty()) { return {}; }
202    return tmp_path.string() + "/llama.cpp/cl-cache";
203#endif
204}
205
206// Query a NUL-terminated string from clGetDeviceInfo / clGetPlatformInfo.
207template <typename GetInfoFn, typename Object>
208std::string query_string(GetInfoFn fn, Object obj, cl_uint name) {
209    size_t sz = 0;
210    if (fn(obj, name, 0, nullptr, &sz) != CL_SUCCESS || sz == 0) {
211        return {};
212    }
213    std::string s(sz, '\0');
214    if (fn(obj, name, sz, &s[0], nullptr) != CL_SUCCESS) {
215        return {};
216    }
217    if (!s.empty() && s.back() == '\0') {
218        s.pop_back();
219    }
220    return s;
221}
222
223std::string compute_key_suffix(cl_device_id device) {
224    cl_platform_id platform = nullptr;
225    clGetDeviceInfo(device, CL_DEVICE_PLATFORM, sizeof(platform), &platform, nullptr);
226
227    std::string s;
228    s.reserve(512);
229    s += query_string(clGetDeviceInfo,   device,   CL_DEVICE_NAME);       s.push_back('\0');
230    s += query_string(clGetDeviceInfo,   device,   CL_DRIVER_VERSION);    s.push_back('\0');
231    s += query_string(clGetDeviceInfo,   device,   CL_DEVICE_VERSION);    s.push_back('\0');
232    if (platform) {
233        s += query_string(clGetPlatformInfo, platform, CL_PLATFORM_VERSION); s.push_back('\0');
234    }
235    s += "fmt=" + std::to_string(CL_PROGRAM_CACHE_FORMAT_VERSION);
236    return s;
237}
238
239const uint8_t MAGIC[8] = { 'G','G','M','L','C','L','B','C' };
240
241bool read_all(const std::string & path, std::vector<uint8_t> & out) {
242    std::ifstream f(fs::u8path(path), std::ios::binary);
243    if (!f) { return false; }
244    f.seekg(0, std::ios::end);
245    std::streamsize sz = f.tellg();
246    if (sz < 0) { return false; }
247    f.seekg(0, std::ios::beg);
248    out.resize((size_t) sz);
249    if (sz > 0) { f.read((char *) out.data(), sz); }
250    return f.good() || f.eof();
251}
252
253bool write_atomic(const std::string & path, const uint8_t * data, size_t len) {
254    const fs::path dst = fs::u8path(path);
255    const fs::path tmp = fs::u8path(path + ".tmp." + std::to_string(ggml_getpid()));
256    {
257        std::ofstream f(tmp, std::ios::binary | std::ios::trunc);
258        if (!f) { return false; }
259        f.write((const char *) data, (std::streamsize) len);
260        if (!f.good()) {
261            std::error_code ec_rm;
262            fs::remove(tmp, ec_rm);
263            return false;
264        }
265    }
266
267    std::error_code ec;
268    fs::rename(tmp, dst, ec);
269    if (ec) {
270        std::error_code ec_rm;
271        fs::remove(tmp, ec_rm);
272        return false;
273    }
274    return true;
275}
276
277}  // namespace
278
279static bool cache_debug_enabled() {
280    static int cached = -1;
281    if (cached < 0) {
282        const char * e = std::getenv("GGML_OPENCL_KERNEL_CACHE_DEBUG");
283        cached = (e && *e) ? 1 : 0;
284    }
285    return cached != 0;
286}
287
288static std::string opts_preview(const std::string & opts, size_t n = 120) {
289    if (opts.size() <= n) { return opts; }
290    return opts.substr(0, n) + "...";
291}
292
293// Running cache tally (diagnostic; plain ints — a benign race in the rare
294// multi-threaded lazy-compile case at worst miscounts by one).
295static int g_cache_hits = 0, g_cache_misses = 0, g_cache_saves = 0;
296
297// Debug trace directly to stderr
298static void cache_debug_line(const char * kind, const std::string & key,
299                             const char * source, const std::string & opts) {
300    if (!cache_debug_enabled()) { return; }
301    fprintf(stderr, "ggml_opencl: cache %-4s [h=%d m=%d s=%d] key=%s src=%zuB opts='%s'\n",
302            kind, g_cache_hits, g_cache_misses, g_cache_saves,
303            key.substr(0, 16).c_str(), strlen(source), opts_preview(opts).c_str());
304    fflush(stderr);
305}
306
307cl_program_cache_state cl_program_cache_init(cl_device_id device) {
308    cl_program_cache_state st;
309
310    const char * env = std::getenv("GGML_OPENCL_KERNEL_CACHE_DIR");
311    if (env && (!std::strcmp(env, "0")    || !std::strcmp(env, "off")  ||
312                !std::strcmp(env, "none") || !std::strcmp(env, "disable") ||
313                !std::strcmp(env, "disabled"))) {
314        if (cache_debug_enabled()) {
315            fprintf(stderr, "ggml_opencl: kernel cache disabled by GGML_OPENCL_KERNEL_CACHE_DIR=%s\n", env);
316            fflush(stderr);
317        }
318        return st;
319    }
320
321    std::string dir;
322    if (!env || !*env || !std::strcmp(env, "1") || !std::strcmp(env, "default")) {
323        dir = default_cache_dir();
324        if (dir.empty()) {
325            GGML_LOG_INFO("ggml_opencl: kernel cache disabled (no usable default cache directory)\n");
326            return st;
327        }
328    } else {
329        dir = env;
330    }
331
332    if (!make_dir_recursive(dir)) {
333        GGML_LOG_INFO("ggml_opencl: kernel cache disabled (cannot create directory '%s')\n", dir.c_str());
334        return st;
335    }
336
337    st.dir        = dir;
338    st.key_suffix = compute_key_suffix(device);
339    GGML_LOG_INFO("ggml_opencl: kernel cache enabled at '%s'\n", st.dir.c_str());
340    if (cache_debug_enabled()) {
341        fprintf(stderr, "ggml_opencl: kernel cache enabled at '%s' "
342                        "(GGML_OPENCL_KERNEL_CACHE_DIR=off to disable)\n", st.dir.c_str());
343        fflush(stderr);
344    }
345    return st;
346}
347
348cl_program cl_program_cache_try_load(
349    const cl_program_cache_state & state,
350    cl_context                     context,
351    cl_device_id                   device,
352    const char *                   source,
353    const std::string &            compile_opts) {
354
355    if (state.dir.empty() || !source) { return nullptr; }
356
357    const std::string key  = compute_key(state.key_suffix, source, compile_opts);
358    const std::string path = state.dir + "/" + key + ".clbin";
359
360    std::vector<uint8_t> file;
361    if (!read_all(path, file)) {
362        ++g_cache_misses;
363        cache_debug_line("MISS", key, source, compile_opts);
364        return nullptr;
365    }
366    if (file.size() < 16 || std::memcmp(file.data(), MAGIC, 8) != 0) { return nullptr; }
367
368    uint32_t fmt =
369        ((uint32_t) file[ 8]) | ((uint32_t) file[ 9] <<  8) |
370        ((uint32_t) file[10] << 16) | ((uint32_t) file[11] << 24);
371    if (fmt != CL_PROGRAM_CACHE_FORMAT_VERSION) { return nullptr; }
372
373    const size_t hdr_len = 16;
374    const unsigned char * bin     = file.data() + hdr_len;
375    const size_t          bin_len = file.size() - hdr_len;
376
377    cl_int err     = CL_SUCCESS;
378    cl_int bin_err = CL_SUCCESS;
379    cl_program p = clCreateProgramWithBinary(context, 1, &device, &bin_len, &bin, &bin_err, &err);
380    if (err != CL_SUCCESS || bin_err != CL_SUCCESS || p == nullptr) {
381        if (p) { clReleaseProgram(p); }
382        return nullptr;
383    }
384
385    err = clBuildProgram(p, 0, nullptr, compile_opts.c_str(), nullptr, nullptr);
386    if (err != CL_SUCCESS) {
387        clReleaseProgram(p);
388        return nullptr;
389    }
390    ++g_cache_hits;
391    cache_debug_line("HIT", key, source, compile_opts);
392    return p;
393}
394
395void cl_program_cache_try_save(
396    const cl_program_cache_state & state,
397    cl_program                     program,
398    cl_device_id                   /*device*/,
399    const char *                   source,
400    const std::string &            compile_opts) {
401
402    if (state.dir.empty() || !program || !source) {
403        return;
404    }
405
406    cl_uint n_dev = 0;
407    if (clGetProgramInfo(program, CL_PROGRAM_NUM_DEVICES, sizeof(n_dev), &n_dev, nullptr) != CL_SUCCESS || n_dev == 0) {
408        return;
409    }
410
411    std::vector<size_t> sizes(n_dev);
412    if (clGetProgramInfo(program, CL_PROGRAM_BINARY_SIZES, sizeof(size_t) * n_dev, sizes.data(), nullptr) != CL_SUCCESS) {
413        return;
414    }
415    if (sizes.empty() || sizes[0] == 0) {
416        return;
417    }
418
419    std::vector<std::vector<uint8_t>> binaries(n_dev);
420    std::vector<unsigned char *>      bin_ptrs(n_dev);
421    for (cl_uint i = 0; i < n_dev; ++i) {
422        binaries[i].resize(sizes[i]);
423        bin_ptrs[i] = binaries[i].data();
424    }
425    if (clGetProgramInfo(program, CL_PROGRAM_BINARIES, sizeof(unsigned char *) * n_dev, bin_ptrs.data(), nullptr) != CL_SUCCESS) {
426        return;
427    }
428
429    // We only care about the first device's binary — that's the one we'd
430    // re-load with on a future cache hit. Multi-device contexts aren't a
431    // pattern this backend uses today.
432    const std::vector<uint8_t> & bin = binaries[0];
433
434    std::vector<uint8_t> file;
435    file.reserve(16 + bin.size());
436    file.insert(file.end(), MAGIC, MAGIC + 8);
437    uint32_t fmt = CL_PROGRAM_CACHE_FORMAT_VERSION;
438    file.push_back((uint8_t)  (fmt        & 0xff));
439    file.push_back((uint8_t) ((fmt >>  8) & 0xff));
440    file.push_back((uint8_t) ((fmt >> 16) & 0xff));
441    file.push_back((uint8_t) ((fmt >> 24) & 0xff));
442    file.push_back(0); file.push_back(0); file.push_back(0); file.push_back(0); // reserved
443    file.insert(file.end(), bin.begin(), bin.end());
444
445    const std::string key  = compute_key(state.key_suffix, source, compile_opts);
446    const std::string path = state.dir + "/" + key + ".clbin";
447    if (!write_atomic(path, file.data(), file.size())) {
448        GGML_LOG_INFO("ggml_opencl: kernel cache: failed to write '%s'\n", path.c_str());
449    } else {
450        ++g_cache_saves;
451        cache_debug_line("SAVE", key, source, compile_opts);
452    }
453}