char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Alessandro de Oliveira Faria (A.K.A.CABELO)vendor : update cpp-httplib to 0.56.0 (#28787)718f7b417

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
5.5 KiB132 linesraw
1#!/usr/bin/env python3
2
3import urllib.request
4import os
5import sys
6import subprocess
7
8HTTPLIB_VERSION = "refs/tags/v0.56.0"
9
10# used by examples/gguf-hash, these repos have no release tag, so we pin a commit
11XXHASH_COMMIT      = "9f465f1ea932d6ad9a26cd77496311ffa544cd68"
12SHA1_COMMIT        = "e1e2536fcf6a8f9703be8c85d58724b408552287"
13SHA256_COMMIT      = "5e637272c13f200872d55ff579f7e2ab6c3f252f"
14ROTATE_BITS_COMMIT = "27e784942f67db44abf2115c6638e735b579acd1"
15
16vendor = {
17    "https://github.com/nlohmann/json/releases/latest/download/json.hpp":     "vendor/nlohmann/json.hpp",
18    "https://github.com/nlohmann/json/releases/latest/download/json_fwd.hpp": "vendor/nlohmann/json_fwd.hpp",
19
20    "https://raw.githubusercontent.com/nothings/stb/refs/heads/master/stb_image.h": "vendor/stb/stb_image.h",
21
22    # not using latest tag to avoid this issue: https://github.com/ggml-org/llama.cpp/pull/17179#discussion_r2515877926
23    # "https://github.com/mackron/miniaudio/raw/refs/tags/0.11.24/miniaudio.h": "vendor/miniaudio/miniaudio.h",
24    "https://github.com/mackron/miniaudio/raw/9634bedb5b5a2ca38c1ee7108a9358a4e233f14d/miniaudio.h": "vendor/miniaudio/miniaudio.h",
25
26    f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/httplib.h": "httplib.h",
27    f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/split.py":  "split.py",
28    f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/LICENSE":   "vendor/cpp-httplib/LICENSE",
29
30    "https://raw.githubusercontent.com/sheredom/subprocess.h/0dccaa9aa176dd6d7ef8afeca3c18d6e80a32795/subprocess.h": "vendor/sheredom/subprocess.h",
31
32    f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.c":      "vendor/hash/xxhash/xxhash.c",
33    f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.h":      "vendor/hash/xxhash/xxhash.h",
34    f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/LICENSE":       "vendor/hash/xxhash/LICENSE",
35
36    # clibs/sha1 ships no license file, the source header says public domain
37    f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.c": "vendor/hash/sha1/sha1.c",
38    f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.h": "vendor/hash/sha1/sha1.h",
39
40    f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.c": "vendor/hash/sha256/sha256.c",
41    f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.h": "vendor/hash/sha256/sha256.h",
42    f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/LICENSE":  "vendor/hash/sha256/LICENSE",
43
44    f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/rotate-bits.h": "vendor/hash/rotate-bits/rotate-bits.h",
45    f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/LICENSE.md":   "vendor/hash/rotate-bits/LICENSE.md",
46}
47
48# local changes kept on top of the upstream sources
49patches = {
50    "vendor/hash/xxhash/xxhash.h": [(
51        '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */\n',
52        '/* Windows SDK under 10.0.22000 is missing stdalign.h so we add a check\n'
53        '   before allowing the windows compiler to use the C11 form.\n'
54        '   Reference: https://github.com/Cyan4973/xxHash/issues/955 */\n'
55        '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) \\\n'
56        '    && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n'
57    )],
58
59    # sha1 exports a bare "SHA1" symbol, which clashes with the boringssl one at link time.
60    # we compile it as C++ (see vendor/hash/CMakeLists.txt) and put it in a namespace.
61    "vendor/hash/sha1/sha1.h": [
62        (
63            '#if defined(__cplusplus)\n'
64            'extern "C" {\n'
65            '#endif\n',
66
67            'namespace vendor_hash {\n'
68        ),
69        (
70            '#if defined(__cplusplus)\n'
71            '}\n'
72            '#endif\n',
73
74            '} // namespace vendor_hash\n'
75        ),
76    ],
77
78    "vendor/hash/sha1/sha1.c": [
79        (
80            '#include "sha1.h"\n',
81
82            '#include "sha1.h"\n'
83            '\n'
84            'namespace vendor_hash {\n'
85        ),
86        (
87            '    SHA1Final((unsigned char *)hash_out, &ctx);\n'
88            '}\n',
89
90            '    SHA1Final((unsigned char *)hash_out, &ctx);\n'
91            '}\n'
92            '\n'
93            '} // namespace vendor_hash\n'
94        ),
95    ],
96
97    # silence a maybe-uninitialized warning
98    "vendor/hash/sha256/sha256.c": [(
99        "  uint32_t W[16];\n",
100        "  uint32_t W[16] = {0};\n"
101    )],
102}
103
104for url, filename in vendor.items():
105    print(f"downloading {url} to {filename}") # noqa: NP100
106    urllib.request.urlretrieve(url, filename)
107
108for filename, replacements in patches.items():
109    print(f"patching {filename}") # noqa: NP100
110    with open(filename, "r", encoding="utf-8", newline="") as f:
111        content = f.read()
112    for old, new in replacements:
113        if content.count(old) != 1:
114            print(f"Error: cannot apply patch on {filename}, upstream code has changed") # noqa: NP100
115            sys.exit(1)
116        content = content.replace(old, new)
117    with open(filename, "w", encoding="utf-8", newline="") as f:
118        f.write(content)
119
120print("Splitting httplib.h...") # noqa: NP100
121try:
122    subprocess.check_call([
123        sys.executable, "split.py",
124        "--extension", "cpp",
125        "--out", "vendor/cpp-httplib"
126    ])
127except Exception as e:
128    print(f"Error: {e}") # noqa: NP100
129    sys.exit(1)
130finally:
131    os.remove("split.py")
132    os.remove("httplib.h")