char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-device model split (aka row-split) (#28589)eafe15a5e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
19.9 KiB493 linesraw
1#!/usr/bin/env python3
2#
3# Run llama.cpp tools on Snapdragon devices (natively, via ADB, or SSH).
4#
5
6import sys
7import os
8import argparse
9import subprocess
10import platform
11import shlex
12import logging
13
14logger = logging.getLogger("run")
15
16
17MANAGED_ENV_NAMES = (
18    "GGML_HEXAGON_DEVICES",
19    "GGML_HEXAGON_VERBOSE",
20    "GGML_HEXAGON_PROFILE",
21    "GGML_HEXAGON_NHVX",
22    "GGML_HEXAGON_NHMX",
23    "GGML_HEXAGON_HOSTBUF",
24    "GGML_HEXAGON_OPBATCH",
25    "GGML_HEXAGON_OPQUEUE",
26    "GGML_HEXAGON_OPPOLL",
27    "GGML_HEXAGON_OPFILTER",
28    "GGML_HEXAGON_OPFUSION",
29    "GGML_HEXAGON_VMEM",
30    "GGML_HEXAGON_MBUF",
31    "GGML_HEXAGON_MM_SELECT",
32    "GGML_HEXAGON_FA_SELECT",
33    "GGML_HEXAGON_AR_SELECT",
34    "GGML_HEXAGON_ETM",
35    "GGML_HEXAGON_ARCH",
36    "GGML_HEXAGON_OPTRACE",
37    "GGML_OPENCL_PLATFORM",
38    "GGML_OPENCL_DEVICE",
39    "GGML_OPENCL_OPFILTER",
40    "GGML_OPENCL_KERNEL_CACHE_DIR",
41    "GGML_OPENCL_KERNEL_CACHE_DEBUG",
42    "GGML_OPENCL_FA_TUNE",
43    "GGML_OPENCL_DISABLE_FUSION",
44    "GGML_OPENCL_ADRENO_XMEM_GEMM",
45    "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER",
46    "GGML_SCHED_DEBUG",
47    "MTMD_BACKEND_DEVICE",
48    "D",
49    "DEVICE",
50)
51
52
53def parse_target(target_str):
54    if not target_str:
55        return None, None
56    if target_str.startswith("adb") or target_str.startswith("android"):
57        parts = target_str.split(":", 1)
58        serial = parts[1] if len(parts) > 1 else None
59        return "android", serial
60    elif target_str.startswith("lnx") or target_str.startswith("linux") or target_str.startswith("ubuntu"):
61        parts = target_str.split(":", 1)
62        host = parts[1] if len(parts) > 1 else None
63        return "linux", host
64    elif target_str in ("wos", "windows"):
65        return "windows", None
66    else:
67        return None, None
68
69
70def shlex_join(args_list):
71    if hasattr(shlex, 'join'):
72        return shlex.join(args_list)
73    import pipes
74    return " ".join(pipes.quote(x) for x in args_list)
75
76
77def split_device_list(devices):
78    parts = []
79    curr = []
80    bracket_depth = 0
81
82    for ch in devices:
83        if ch == '[':
84            bracket_depth += 1
85            curr.append(ch)
86        elif ch == ']':
87            if bracket_depth > 0:
88                bracket_depth -= 1
89            curr.append(ch)
90        elif ch == ',' and bracket_depth == 0:
91            part = "".join(curr).strip()
92            if part:
93                parts.append(part)
94            curr = []
95        else:
96            curr.append(ch)
97
98    part = "".join(curr).strip()
99    if part:
100        parts.append(part)
101
102    return parts
103
104
105def device_arg_from_devices(devices):
106    if devices.isdigit():
107        n = int(devices)
108        return ",".join(f"HTP{i}" for i in range(n))
109
110    names = []
111    for part in split_device_list(devices):
112        if "[" in part:
113            part = part.split("[", 1)[0].strip()
114        if part:
115            names.append(part)
116
117    return ",".join(names)
118
119
120def normalize_cmd_device_args(cmd_args):
121    for i, arg in enumerate(cmd_args):
122        if arg == "--device" and i + 1 < len(cmd_args):
123            cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1])
124        elif arg.startswith("--device="):
125            cmd_args[i] = "--device=" + device_arg_from_devices(arg.split("=", 1)[1])
126
127
128def main():
129    logging.basicConfig(level=logging.INFO, format='%(message)s')
130    # Split arguments at '--'
131    if '--' in sys.argv:
132        idx = sys.argv.index('--')
133        run_args = sys.argv[1:idx]
134        cmd_args = sys.argv[idx + 1:]
135    else:
136        run_args = sys.argv[1:]
137        cmd_args = []
138
139    parser = argparse.ArgumentParser(
140        description="Unified runner for llama.cpp tools on Snapdragon (natively, via ADB, or via SSH)."
141    )
142    parser.add_argument("--target", help="Execution target (e.g. android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: local run)")
143    parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux)")
144    parser.add_argument("--install-dir", help="Install directory name (defaults to pkg-TARGET or pkg-TARGET-dbg prefix based on target)")
145    parser.add_argument("--debug", action="store_true", help="Use debug build (defaults to pkg-TARGET-dbg folder)")
146    parser.add_argument("--devices", "--device", "-d", help="Select execution devices (split into NPU and OpenCL GPUs automatically, default: HTP0)")
147    parser.add_argument("--verbose", help="Verbose level (enables both Hexagon and OpenCL kernel cache debugging)")
148    parser.add_argument("--profile", help="Profiling flag (enables Hexagon profiling and OpenCL autotuning)")
149    parser.add_argument("--sched-debug", action="store_true", help="Enable GGML/llama.cpp scheduler debug output (GGML_SCHED_DEBUG=2)")
150    parser.add_argument("--mtmd-device", help="Specify the backend device ID for Multi-Threaded Multi-Device setup (MTMD_BACKEND_DEVICE)")
151
152    # Hexagon specific parameters
153    parser.add_argument("--hex-verbose", help="Enable verbose logging (GGML_HEXAGON_VERBOSE)")
154    parser.add_argument("--hex-profile", help="Enable NPU/Hexagon profiling and performance metrics print (GGML_HEXAGON_PROFILE)")
155    parser.add_argument("--hex-nhvx", help="Number of HVX units to use (GGML_HEXAGON_NHVX)")
156    parser.add_argument("--hex-nhmx", help="Number of HMX units to use. 0 disables HMX power-up (GGML_HEXAGON_NHMX)")
157    parser.add_argument("--hex-hostbuf", help="Enable host buffers (GGML_HEXAGON_HOSTBUF)")
158    parser.add_argument("--hex-opbatch", help="Maximum number of operations to batch into a single HTP execution (GGML_HEXAGON_OPBATCH)")
159    parser.add_argument("--hex-opqueue", help="Size of the asynchronous NPU operation queue (GGML_HEXAGON_OPQUEUE)")
160    parser.add_argument("--hex-oppoll", default="1", help="Enable (1) or Disable (0) polling for NPU opbatch completion (GGML_HEXAGON_OPPOLL) (default: 1)")
161    parser.add_argument("--hex-opfilter", help="Regex pattern to filter/select which operators are offloaded to NPU (GGML_HEXAGON_OPFILTER)")
162    parser.add_argument("--hex-opfusion", help="NPU graph node fusion optimization level (0: disabled, 1: enabled) (GGML_HEXAGON_OPFUSION)")
163    parser.add_argument("--hex-vmem", help="Maximum NPU VMEM size limit in MB to allocate (GGML_HEXAGON_VMEM)")
164    parser.add_argument("--hex-mbuf", help="Maximum host buffer size limit in MB to allocate (GGML_HEXAGON_MBUF)")
165    parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 3:HMX,2:HVX-tiled,1:HVX-flat,0:disable")
166    parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable")
167    parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable")
168    parser.add_argument("--hex-etm", help="Enable Embedded Trace Macrocell hardware tracing / trace logging (GGML_HEXAGON_ETM)")
169    parser.add_argument("--hex-arch", help="Target Hexagon NPU architecture version override (v73, v75, v79, v81, etc.) (GGML_HEXAGON_ARCH)")
170    parser.add_argument("--hex-optrace", help="Trace buffer size in number of records (GGML_HEXAGON_OPTRACE)")
171
172    # OpenCL specific parameters
173    parser.add_argument("--cl-platform", help="Select OpenCL platform name/regex (e.g. Qualified Qualcomm OpenCL platform) (GGML_OPENCL_PLATFORM)")
174    parser.add_argument("--cl-device", help="Select OpenCL device name/regex (e.g. Adreno GPU) (GGML_OPENCL_DEVICE)")
175    parser.add_argument("--cl-opfilter", help="Regex pattern to filter/select which operators are offloaded to OpenCL (GGML_OPENCL_OPFILTER)")
176    parser.add_argument("--cl-disable-fusion", action="store_true", help="Disable OpenCL kernel fusion optimizations (GGML_OPENCL_DISABLE_FUSION)")
177    parser.add_argument("--cl-cache-dir", help="Directory path to store compiled OpenCL program binaries (GGML_OPENCL_KERNEL_CACHE_DIR)")
178    parser.add_argument("--cl-cache-debug", help="Enable verbose debugging logs for the kernel caching system (GGML_OPENCL_KERNEL_CACHE_DEBUG)")
179    parser.add_argument("--cl-fa-tune", action="store_true", help="Enable automatic Flash Attention kernel autotuning (GGML_OPENCL_FA_TUNE)")
180    parser.add_argument("--cl-adreno-xmem", action="store_true", help="Enforce matmul using texture/image (xmem) memory paths on Adreno GPUs (GGML_OPENCL_ADRENO_XMEM_GEMM)")
181    parser.add_argument("--cl-adreno-large-buffer", action="store_true", help="Allow allocating larger buffer sizes on Adreno GPUs (GGML_OPENCL_ADRENO_USE_LARGE_BUFFER)")
182
183    args = parser.parse_args(run_args)
184
185    if not cmd_args:
186        parser.print_help()
187        logger.error("\nError: No command specified after '--'")
188        sys.exit(1)
189
190    target_type = None
191    target_val = None
192    target_prefix = None
193    if args.target:
194        target_type, target_val = parse_target(args.target)
195        if not target_type:
196            logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")
197            sys.exit(1)
198        target_prefix = args.target.split(":", 1)[0]
199
200    # Resolve install directory
201    install_dir = args.install_dir
202    if not install_dir:
203        if target_prefix:
204            suffix = "-dbg" if args.debug else ""
205            install_dir = f"pkg-{target_prefix}{suffix}"
206        else:
207            # Smart branch folder detection for local run if default is not set
208            prefixes = ("wos", "windows", "lnx", "linux", "ubuntu", "adb", "android")
209            suffixes = ("-dbg", "") if args.debug else ("", "-dbg")
210            found = False
211            for suffix in suffixes:
212                for prefix in prefixes:
213                    test_path = f"./pkg-{prefix}{suffix}/llama.cpp"
214                    if os.path.exists(test_path):
215                        install_dir = f"pkg-{prefix}{suffix}"
216                        found = True
217                        break
218                if found:
219                    break
220            if not install_dir:
221                install_dir = "pkg-android"  # Fallback default
222
223    # Host side package path
224    package_path = os.path.join(install_dir, "llama.cpp")
225
226    # Environment variables to map
227    env_vars = {}
228
229    def set_env(env_name, opt_val):
230        if opt_val is not None:
231            env_vars[env_name] = str(opt_val)
232
233    # Resolve and filter devices (HTP vs OpenCL)
234    device_in_cmd = None
235    for i, arg in enumerate(cmd_args):
236        if arg == "--device" and i + 1 < len(cmd_args):
237            device_in_cmd = cmd_args[i + 1]
238            break
239        elif arg.startswith("--device="):
240            device_in_cmd = arg.split("=", 1)[1]
241            break
242
243    if args.devices is not None:
244        devices_val = args.devices
245    elif device_in_cmd is not None:
246        devices_val = device_in_cmd
247    else:
248        devices_val = "HTP0"
249
250    if devices_val.isdigit():
251        hex_devices = devices_val
252        cl_device = ""
253    else:
254        parts = split_device_list(devices_val)
255        # Any device containing "htp" is Hexagon, rest is OpenCL
256        hex_parts = [p for p in parts if "htp" in p.lower()]
257        cl_parts = [
258            p for p in parts
259            if "htp" not in p.lower()
260            and p.lower() not in ("none", "cpu")
261            and not p.lower().startswith("gpuopencl")
262        ]
263        hex_devices = ",".join(hex_parts)
264        cl_device = ",".join(cl_parts)
265
266    # Set Hexagon devices
267    if hex_devices:
268        env_vars["GGML_HEXAGON_DEVICES"] = hex_devices
269
270    normalize_cmd_device_args(cmd_args)
271
272    # Set OpenCL device (unless overridden by --cl-device)
273    final_cl_device = args.cl_device if args.cl_device is not None else cl_device
274    if final_cl_device:
275        env_vars["GGML_OPENCL_DEVICE"] = final_cl_device
276
277    # Map shared & backend-specific parameters with correct overrides
278
279    # Verbose logging mapping
280    hex_verbose_val = args.hex_verbose if args.hex_verbose is not None else args.verbose
281    set_env("GGML_HEXAGON_VERBOSE", hex_verbose_val)
282
283    cl_cache_debug_val = args.cl_cache_debug if args.cl_cache_debug is not None else args.verbose
284    set_env("GGML_OPENCL_KERNEL_CACHE_DEBUG", cl_cache_debug_val)
285
286    # Profiling mapping
287    hex_profile_val = args.hex_profile if args.hex_profile is not None else args.profile
288    set_env("GGML_HEXAGON_PROFILE", hex_profile_val)
289
290    if args.cl_fa_tune or args.profile is not None:
291        env_vars["GGML_OPENCL_FA_TUNE"] = "1"
292
293    # Other Hexagon environment variables
294    set_env("GGML_HEXAGON_NHVX", args.hex_nhvx)
295    set_env("GGML_HEXAGON_NHMX", args.hex_nhmx)
296    set_env("GGML_HEXAGON_HOSTBUF", args.hex_hostbuf)
297    set_env("GGML_HEXAGON_OPBATCH", args.hex_opbatch)
298    set_env("GGML_HEXAGON_OPQUEUE", args.hex_opqueue)
299    set_env("GGML_HEXAGON_OPPOLL", args.hex_oppoll)
300    set_env("GGML_HEXAGON_OPFILTER", args.hex_opfilter)
301    set_env("GGML_HEXAGON_OPFUSION", args.hex_opfusion)
302    set_env("GGML_HEXAGON_VMEM", args.hex_vmem)
303    set_env("GGML_HEXAGON_MBUF", args.hex_mbuf)
304    set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select)
305    set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select)
306    set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select)
307    set_env("GGML_HEXAGON_ETM", args.hex_etm)
308    set_env("GGML_HEXAGON_ARCH", args.hex_arch)
309    set_env("GGML_HEXAGON_OPTRACE", args.hex_optrace)
310    set_env("MTMD_BACKEND_DEVICE", args.mtmd_device)
311
312    # OpenCL environment variables
313    set_env("GGML_OPENCL_PLATFORM", args.cl_platform)
314    set_env("GGML_OPENCL_OPFILTER", args.cl_opfilter)
315    set_env("GGML_OPENCL_KERNEL_CACHE_DIR", args.cl_cache_dir)
316
317    if args.cl_disable_fusion:
318        env_vars["GGML_OPENCL_DISABLE_FUSION"] = "1"
319
320    if args.cl_adreno_xmem:
321        env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = "1"
322
323    if args.cl_adreno_large_buffer:
324        env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = "1"
325
326    if args.sched_debug:
327        env_vars["GGML_SCHED_DEBUG"] = "2"
328
329    # Resolve executable path
330    executable = cmd_args[0]
331    known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops"]
332    if executable in known_binaries:
333        if target_type in ("android", "linux"):
334            resolved_exec = f"./bin/{executable}"
335        else:
336            if platform.system() == "Windows":
337                resolved_exec = os.path.normpath(os.path.join(package_path, "bin", f"{executable}.exe"))
338            else:
339                resolved_exec = os.path.normpath(os.path.join(package_path, "bin", executable))
340        cmd_args[0] = resolved_exec
341
342    # Infer device string to pass to the tool
343    basename = os.path.basename(executable)
344    if basename.endswith(".exe"):
345        basename = basename[:-4]
346
347    device_val = None
348    if basename == "test-backend-ops":
349        for i in range(len(cmd_args)):
350            if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args):
351                val = cmd_args[i + 1]
352                new_val = ""
353                for j, char in enumerate(val):
354                    if char in ('[', ']'):
355                        if j > 0 and val[j - 1] == '\\':
356                            new_val += char
357                        else:
358                            new_val += '\\' + char
359                    else:
360                        new_val += char
361                cmd_args[i + 1] = new_val
362
363        has_b = any(arg == "-b" for arg in cmd_args)
364        if not has_b:
365            if args.devices:
366                device_val = device_arg_from_devices(args.devices)
367            else:
368                device_val = "HTP0"
369            if device_val:
370                cmd_args += ["-b", device_val]
371    else:
372        has_device = any(arg.startswith("--device") for arg in cmd_args)
373        if not has_device:
374            if args.devices:
375                device_val = device_arg_from_devices(args.devices)
376            else:
377                device_val = "HTP0"
378
379            if device_val:
380                cmd_args += ["--device", device_val]
381
382    # Automatically add -v to known llama tools if sched-debug, verbose, or profile are set
383    verbose_trigger = (
384        args.sched_debug
385        or args.verbose is not None
386        or args.profile is not None
387        or args.hex_verbose is not None
388        or args.hex_profile is not None
389        or args.hex_optrace is not None
390    )
391    if verbose_trigger and basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"):
392        if "-v" not in cmd_args and "--verbose" not in cmd_args:
393            cmd_args.append("-v")
394
395    # Inject defaults for llama-cli, llama-completion, and llama-server if not overridden by the user
396    if basename in ("llama-cli", "llama-completion", "llama-server"):
397        if "-ngl" not in cmd_args and "--n-gpu-layers" not in cmd_args:
398            cmd_args += ["-ngl", "99"]
399        if "-fa" not in cmd_args and "--flash-attn" not in cmd_args:
400            cmd_args += ["-fa", "on"]
401
402    # Use ubatch-size 1024 for hexagon backend (HTP devices)
403    if hex_devices and basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"):
404        if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args:
405            cmd_args += ["--ubatch-size", "1024"]
406    elif basename in ("llama-cli", "llama-completion", "llama-server"):
407        if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args:
408            cmd_args += ["--ubatch-size", "1024"]
409
410    if basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"):
411        if "-t" not in cmd_args and "--threads" not in cmd_args:
412            cmd_args += ["-t", "6"]
413
414    # Resolve target directory on device
415    target_dir = args.target_dir
416    if not target_dir:
417        target_dir = "/data/local/tmp/llama.cpp" if target_type == "android" else "~/llama.cpp"
418
419    if target_type == "android":
420        # Run via ADB
421        adb_base = ["adb"]
422        if target_val: # serial
423            adb_base += ["-s", target_val]
424
425        env_parts = [
426            "LD_LIBRARY_PATH=./lib",
427            "ADSP_LIBRARY_PATH=./lib"
428        ]
429        for k, v in env_vars.items():
430            env_parts.append(f"{k}={v}")
431        env_str = " ".join(env_parts)
432
433        cmd_str = shlex_join(cmd_args)
434        adb_shell_cmd = f"cd {target_dir} && ulimit -c unlimited && {env_str} {cmd_str}"
435        full_cmd = adb_base + ["shell", adb_shell_cmd]
436
437        logger.info(f"+ {' '.join(full_cmd)}")
438        res = subprocess.run(full_cmd)
439        sys.exit(res.returncode)
440
441    elif target_type == "linux":
442        ssh_host = target_val
443        if not ssh_host:
444            logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot execute.")
445            sys.exit(1)
446
447        # Linux remote run via SSH
448        env_parts = [
449            "LD_LIBRARY_PATH=./lib",
450            "ADSP_LIBRARY_PATH=./lib"
451        ]
452        for k, v in env_vars.items():
453            env_parts.append(f"{k}={v}")
454        env_str = " ".join(env_parts)
455
456        cmd_str = shlex_join(cmd_args)
457        ssh_shell_cmd = f"cd {target_dir} && ulimit -c unlimited && {env_str} {cmd_str}"
458        full_cmd = ["ssh", ssh_host, ssh_shell_cmd]
459
460        logger.info(f"+ {' '.join(full_cmd)}")
461        res = subprocess.run(full_cmd)
462        sys.exit(res.returncode)
463
464    elif target_type == "windows":
465        logger.info("Windows target execution is currently a stub.")
466        sys.exit(0)
467
468    else:
469        # Run locally
470        local_env = os.environ.copy()
471        lib_dir = os.path.normpath(os.path.join(package_path, "lib"))
472        local_env["ADSP_LIBRARY_PATH"] = lib_dir
473        if platform.system() == "Windows":
474            local_env["PATH"] = lib_dir + os.path.pathsep + local_env.get("PATH", "")
475        else:
476            local_env["LD_LIBRARY_PATH"] = lib_dir + os.path.pathsep + local_env.get("LD_LIBRARY_PATH", "")
477
478        for k in MANAGED_ENV_NAMES:
479            local_env.pop(k, None)
480        for k, v in env_vars.items():
481            local_env[k] = v
482
483        logger.info(f"+ {shlex_join(cmd_args)}")
484        res = subprocess.run(cmd_args, env=local_env)
485        sys.exit(res.returncode)
486
487
488if __name__ == "__main__":
489    try:
490        main()
491    except KeyboardInterrupt:
492        logger.info("\nInterrupted by user.")
493        sys.exit(130)