char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Fathi Boudracommon: migrate the deprecated --mmap/--no-mmap to --load-mode (#26934)0177dcc73

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.7 KiB196 linesraw
1::  MIT license
2::  Copyright (C) 2024 Intel Corporation
3::  SPDX-License-Identifier: MIT
4
5@echo off
6setlocal EnableExtensions EnableDelayedExpansion
7
8set "BIN_FILE=.\build\bin\llama-server.exe"
9set "SEED=0"
10set "GPUS_SETTING="
11
12set "MODEL_FILE=..\models\Qwen3.5-4B-Q4_0.gguf"
13set "NGL=99"
14set "CONTEXT=4096"
15set "GGML_SYCL_DEVICE=-1"
16set "SYCL_DEVICES=SYCL0"
17set "SPLIT_MODE=layer"
18set "LOG_VERBOSE=3"
19
20if "%~1"=="" goto after_args
21
22:parse_args
23if "%~1"=="" goto after_args
24
25if /I "%~1"=="-c" (
26  if "%~2"=="" goto missing_value
27  set "CONTEXT=%~2"
28  shift
29  shift
30  goto parse_args
31)
32if /I "%~1"=="--context" (
33  if "%~2"=="" goto missing_value
34  set "CONTEXT=%~2"
35  shift
36  shift
37  goto parse_args
38)
39
40if /I "%~1"=="-d" (
41  if "%~2"=="" goto missing_value
42  set "SYCL_DEVICES=%~2"
43  shift
44  shift
45  goto parse_args
46)
47if /I "%~1"=="--device" (
48  if "%~2"=="" goto missing_value
49  set "SYCL_DEVICES=%~2"
50  shift
51  shift
52  goto parse_args
53)
54
55if /I "%~1"=="-m" (
56  if "%~2"=="" goto missing_value
57  set "MODEL_FILE=%~2"
58  shift
59  shift
60  goto parse_args
61)
62if /I "%~1"=="--model" (
63  if "%~2"=="" goto missing_value
64  set "MODEL_FILE=%~2"
65  shift
66  shift
67  goto parse_args
68)
69
70if /I "%~1"=="-mg" (
71  if "%~2"=="" goto missing_value
72  set "GGML_SYCL_DEVICE=%~2"
73  set "SPLIT_MODE=none"
74  shift
75  shift
76  goto parse_args
77)
78if /I "%~1"=="--main-gpu" (
79  if "%~2"=="" goto missing_value
80  set "GGML_SYCL_DEVICE=%~2"
81  set "SPLIT_MODE=none"
82  shift
83  shift
84  goto parse_args
85)
86
87if /I "%~1"=="-sm" (
88  if "%~2"=="" goto missing_value
89  set "SPLIT_MODE=%~2"
90  shift
91  shift
92  goto parse_args
93)
94if /I "%~1"=="--split-mode" (
95  if "%~2"=="" goto missing_value
96  set "SPLIT_MODE=%~2"
97  shift
98  shift
99  goto parse_args
100)
101
102if /I "%~1"=="-ngl" (
103  if "%~2"=="" goto missing_value
104  set "NGL=%~2"
105  shift
106  shift
107  goto parse_args
108)
109if /I "%~1"=="--n-gpu-layers" (
110  if "%~2"=="" goto missing_value
111  set "NGL=%~2"
112  shift
113  shift
114  goto parse_args
115)
116
117if /I "%~1"=="-lv" (
118  if "%~2"=="" goto missing_value
119  set "LOG_VERBOSE=%~2"
120  shift
121  shift
122  goto parse_args
123)
124if /I "%~1"=="--log-verbosity" (
125  if "%~2"=="" goto missing_value
126  set "LOG_VERBOSE=%~2"
127  shift
128  shift
129  goto parse_args
130)
131
132if /I "%~1"=="-h" goto help
133if /I "%~1"=="--help" goto help
134
135echo Invalid option: %~1
136exit /b 1
137
138:missing_value
139echo Missing value for option: %~1
140exit /b 1
141
142:help
143echo Usage: %~n0 [OPTIONS]
144echo.
145echo This script processes files with specified options.
146echo.
147echo Options:
148echo   -h, --help    Display this help message and exit.
149echo   -d, --device ^<value^>    Set SYCL devices (default: SYCL0).
150echo   -c, --context ^<value^>    Set context length. Bigger need more memory.
151echo   -m, --model   ^<value^>    Full model file path.
152echo   -mg,--main-gpu ^<value^>   Set main GPU ID (0 - n) for single GPU mode.
153echo   -sm,--split-mode ^<value^> How to split the model across multiple GPUs, one of:
154echo                             - none: use one GPU only
155echo                             - layer (default): split layers and KV across GPUs
156echo                             - row: split rows across GPUs
157echo   -ngl,--n-gpu-layers ^<value^>  Max. number of layers to store in VRAM (default: -1)
158echo   -lv,--log-verbosity ^<value^>  Set the verbosity threshold. Messages with a higher verbosity will be
159echo                                ignored. Values:
160echo                                 - 0: generic output
161echo                                 - 1: error
162echo                                 - 2: warning
163echo                                 - 3: info
164echo                                 - 4: debug
165exit /b 0
166
167:after_args
168
169REM In Windows CMD, source is not available; call oneAPI setvars if present.
170if exist "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" (
171  call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" >nul
172) else (
173  echo Warning: oneAPI setvars.bat not found. Continuing without environment setup.
174)
175
176REM Support malloc device memory more than 4GB.
177set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"
178echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%
179
180echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%
181
182if not "%GGML_SYCL_DEVICE%"=="-1" (
183  echo Use %GGML_SYCL_DEVICE% as main GPU
184  REM Use single GPU only.
185  set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"
186  ) else (
187  echo Use Intel GPUs: %SYCL_DEVICES%
188  set "GPUS_SETTING=-sm %SPLIT_MODE%"
189)
190
191echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --load-mode auto --host 0.0.0.0 --port 8000
192set "ZES_ENABLE_SYSMAN=1"
193%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --load-mode auto --host 0.0.0.0 --port 8000
194
195endlocal
196