char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zhaolun Yinnix : update deprecated expressions (#28145)ccc3646c6

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
8.0 KiB275 linesraw
1{
2  lib,
3  glibc,
4  config,
5  stdenv,
6  stdenvNoCC,
7  runCommand,
8  cmake,
9  ninja,
10  pkg-config,
11  git,
12  mpi,
13  blas,
14  cudaPackages,
15  autoAddDriverRunpath,
16  darwin,
17  rocmPackages,
18  vulkan-headers,
19  vulkan-loader,
20  openssl,
21  shaderc,
22  spirv-headers,
23  nodejs,
24  importNpmLock,
25  useBlas ?
26    builtins.all (x: !x) [
27      useCuda
28      useMetalKit
29      useRocm
30      useVulkan
31    ]
32    && blas.meta.available,
33  useCuda ? config.cudaSupport,
34  useMetalKit ? stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isDarwin,
35  # Increases the runtime closure size by ~700M
36  useMpi ? false,
37  useRocm ? config.rocmSupport,
38  rocmGpuTargets ? builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets,
39  useVulkan ? false,
40  useRpc ? false,
41  llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake
42
43  # It's necessary to consistently use backendStdenv when building with CUDA support,
44  # otherwise we get libstdc++ errors downstream.
45  effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
46  enableStatic ? effectiveStdenv.hostPlatform.isStatic,
47  precompileMetalShaders ? false,
48  useWebUi ? true,
49}:
50
51let
52  inherit (lib)
53    cmakeBool
54    cmakeFeature
55    optionalAttrs
56    optionals
57    strings
58    ;
59
60  stdenv = throw "Use effectiveStdenv instead";
61
62  suffices =
63    lib.optionals useBlas [ "BLAS" ]
64    ++ lib.optionals useCuda [ "CUDA" ]
65    ++ lib.optionals useMetalKit [ "MetalKit" ]
66    ++ lib.optionals useMpi [ "MPI" ]
67    ++ lib.optionals useRocm [ "ROCm" ]
68    ++ lib.optionals useVulkan [ "Vulkan" ];
69
70  pnameSuffix =
71    strings.optionalString (suffices != [ ])
72      "-${strings.concatMapStringsSep "-" strings.toLower suffices}";
73  descriptionSuffix = strings.optionalString (
74    suffices != [ ]
75  ) ", accelerated with ${strings.concatStringsSep ", " suffices}";
76
77  xcrunHost = runCommand "xcrunHost" { } ''
78    mkdir -p $out/bin
79    ln -s /usr/bin/xcrun $out/bin
80  '';
81
82  # apple_sdk is supposed to choose sane defaults, no need to handle isAarch64
83  # separately
84  darwinBuildInputs =
85    with darwin.apple_sdk.frameworks;
86    [
87      Accelerate
88      CoreVideo
89      CoreGraphics
90    ]
91    ++ optionals useMetalKit [ MetalKit ];
92
93  cudaBuildInputs = with cudaPackages; [
94    cuda_cudart
95    cccl # <nv/target>
96    libcublas
97  ];
98
99  rocmBuildInputs = with rocmPackages; [
100    clr
101    hipblas
102    rocblas
103  ];
104
105  vulkanBuildInputs = [
106    vulkan-headers
107    vulkan-loader
108    shaderc
109    spirv-headers
110  ];
111in
112
113effectiveStdenv.mkDerivation (finalAttrs: {
114  pname = "llama-cpp${pnameSuffix}";
115  version = llamaVersion;
116
117  # Note: none of the files discarded here are visible in the sandbox or
118  # affect the output hash. This also means they can be modified without
119  # triggering a rebuild.
120  src = lib.cleanSourceWith {
121    filter =
122      name: type:
123      let
124        noneOf = builtins.all (x: !x);
125        baseName = baseNameOf name;
126      in
127      noneOf [
128        (lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths
129        (lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths
130        (lib.hasPrefix "." baseName) # Skip hidden files and directories
131        (baseName == "flake.lock")
132      ];
133    src = lib.cleanSource ../../.;
134  };
135
136  # Builds the webui locally, taking care not to require updating any sha256 hash.
137  webui = stdenvNoCC.mkDerivation {
138    pname = "webui";
139    version = llamaVersion;
140    src = lib.cleanSource ../../tools/ui;
141
142    nativeBuildInputs = [
143      nodejs
144      importNpmLock.linkNodeModulesHook
145    ];
146
147    # no sha256 required when using buildNodeModules
148    npmDeps = importNpmLock.buildNodeModules {
149      npmRoot = ../../tools/ui;
150      inherit nodejs;
151    };
152
153    installPhase = ''
154      LLAMA_UI_OUT_DIR=$out npm run build --offline
155    '';
156  };
157
158  postPatch = lib.optionalString useWebUi ''
159    cp -r ${finalAttrs.webui} tools/ui/dist
160    chmod -R u+w tools/ui/dist
161  '';
162
163  # With PR#6015 https://github.com/ggml-org/llama.cpp/pull/6015,
164  # `default.metallib` may be compiled with Metal compiler from XCode
165  # and we need to escape sandbox on MacOS to access Metal compiler.
166  # `xcrun` is used find the path of the Metal compiler, which is varible
167  # and not on $PATH
168  # see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
169  __noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
170
171  nativeBuildInputs =
172    [
173      cmake
174      ninja
175      pkg-config
176      git
177    ]
178    ++ optionals useCuda [
179      cudaPackages.cuda_nvcc
180
181      autoAddDriverRunpath
182    ]
183    ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
184    ++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
185
186  buildInputs =
187    optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
188    ++ optionals useCuda cudaBuildInputs
189    ++ optionals useMpi [ mpi ]
190    ++ optionals useRocm rocmBuildInputs
191    ++ optionals useBlas [ blas ]
192    ++ optionals useVulkan vulkanBuildInputs
193    ++ [ openssl ];
194
195  cmakeFlags =
196    [
197      (cmakeBool "LLAMA_BUILD_SERVER" true)
198      (cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
199      (cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
200      (cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
201      (cmakeBool "GGML_NATIVE" false)
202      (cmakeBool "GGML_BLAS" useBlas)
203      (cmakeBool "GGML_CUDA" useCuda)
204      (cmakeBool "GGML_HIP" useRocm)
205      (cmakeBool "GGML_METAL" useMetalKit)
206      (cmakeBool "GGML_VULKAN" useVulkan)
207      (cmakeBool "GGML_STATIC" enableStatic)
208      (cmakeBool "GGML_RPC" useRpc)
209    ]
210    ++ optionals useCuda [
211      (
212        with cudaPackages.flags;
213        cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
214          builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
215        )
216      )
217    ]
218    ++ optionals useRocm [
219      (cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
220      (cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
221    ]
222    ++ optionals useMetalKit [
223      (lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
224      (cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
225    ];
226
227  # Environment variables needed for ROCm
228  env = optionalAttrs useRocm {
229    ROCM_PATH = "${rocmPackages.clr}";
230    HIP_DEVICE_LIB_PATH = "${rocmPackages.rocm-device-libs}/amdgcn/bitcode";
231  };
232
233  # TODO(SomeoneSerge): It's better to add proper install targets at the CMake level,
234  # if they haven't been added yet.
235  postInstall = ''
236    mkdir -p $out/include
237    cp $src/include/llama.h $out/include/
238  '';
239
240  meta = {
241    # Configurations we don't want even the CI to evaluate. Results in the
242    # "unsupported platform" messages. This is mostly a no-op, because
243    # cudaPackages would've refused to evaluate anyway.
244    badPlatforms = optionals useCuda lib.platforms.darwin;
245
246    # Configurations that are known to result in build failures. Can be
247    # overridden by importing Nixpkgs with `allowBroken = true`.
248    broken = (useMetalKit && !effectiveStdenv.hostPlatform.isDarwin);
249
250    description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
251    homepage = "https://github.com/ggml-org/llama.cpp/";
252    license = lib.licenses.mit;
253
254    # Accommodates `nix run` and `lib.getExe`
255    mainProgram = "llama-cli";
256
257    # These people might respond, on the best effort basis, if you ping them
258    # in case of Nix-specific regressions or for reviewing Nix-specific PRs.
259    # Consider adding yourself to this list if you want to ensure this flake
260    # stays maintained and you're willing to invest your time. Do not add
261    # other people without their consent. Consider removing people after
262    # they've been unreachable for long periods of time.
263
264    # Note that lib.maintainers is defined in Nixpkgs, but you may just add
265    # an attrset following the same format as in
266    # https://github.com/NixOS/nixpkgs/blob/f36a80e54da29775c78d7eff0e628c2b4e34d1d7/maintainers/maintainer-list.nix
267    maintainers = with lib.maintainers; [
268      philiptaron
269      SomeoneSerge
270    ];
271
272    # Extend `badPlatforms` instead
273    platforms = lib.platforms.all;
274  };
275})