char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovspec : update speculative-simple (#26904)f785fc9ea

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.2 KiB49 linesraw

llama.cpp/examples/speculative-simple

Demonstration of basic greedy speculative decoding

# spec-type draft-simple
./bin/llama-speculative-simple \
        -hf  ggml-org/Qwen3-8B-Base-GGUF:Q8_0 \
        -hfd ggml-org/Qwen3-0.6B-Base-GGUF \
        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
        --spec-type draft-simple --spec-draft-n-max 7 -ngld 99 --color on \
        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4

# spec-type draft-mtp
./bin/llama-speculative-simple \
        -hf ggml-org/Qwen3.6-27B-GGUF:Q8_0 \
        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
        --spec-type draft-mtp --spec-draft-n-max 3 -ngld 99 --color on \
        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4

# spec-type draft-mtp (with shared KV cache)
# note: this model needs a <s> token at the start to somewhat work without the chat template
./bin/llama-speculative-simple \
        -hf ggml-org/Gemma-4-31B-it-GGUF:Q8_0 \
        -p "<s>Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
        --spec-type draft-mtp --spec-draft-n-max 3 -ngld 99 --color on \
        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4

# spec-type draft-eagle3
./bin/llama-speculative-simple \
        -hf ggml-org/gpt-oss-20b-GGUF \
        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
        --spec-type draft-eagle3 --spec-draft-n-max 3 -ngld 99 --color on \
        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4

# spec-type draft-dflash
./bin/llama-speculative-simple \
        -hf ggml-org/Qwen3-8B-GGUF \
        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
        --spec-type draft-dflash --spec-draft-n-max 7 -ngld 99 --color on \
        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4

# spec-type draft-dspark
./bin/llama-speculative-simple \
        -hf ggml-org/Qwen3-8B-GGUF \
        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
        --spec-type draft-dspark --spec-draft-n-max 7 -ngld 99 --color on \
        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
1# llama.cpp/examples/speculative-simple
2
3Demonstration of basic greedy speculative decoding
4
5```bash
6# spec-type draft-simple
7./bin/llama-speculative-simple \
8        -hf  ggml-org/Qwen3-8B-Base-GGUF:Q8_0 \
9        -hfd ggml-org/Qwen3-0.6B-Base-GGUF \
10        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
11        --spec-type draft-simple --spec-draft-n-max 7 -ngld 99 --color on \
12        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
13
14# spec-type draft-mtp
15./bin/llama-speculative-simple \
16        -hf ggml-org/Qwen3.6-27B-GGUF:Q8_0 \
17        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
18        --spec-type draft-mtp --spec-draft-n-max 3 -ngld 99 --color on \
19        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
20
21# spec-type draft-mtp (with shared KV cache)
22# note: this model needs a <s> token at the start to somewhat work without the chat template
23./bin/llama-speculative-simple \
24        -hf ggml-org/Gemma-4-31B-it-GGUF:Q8_0 \
25        -p "<s>Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
26        --spec-type draft-mtp --spec-draft-n-max 3 -ngld 99 --color on \
27        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
28
29# spec-type draft-eagle3
30./bin/llama-speculative-simple \
31        -hf ggml-org/gpt-oss-20b-GGUF \
32        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
33        --spec-type draft-eagle3 --spec-draft-n-max 3 -ngld 99 --color on \
34        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
35
36# spec-type draft-dflash
37./bin/llama-speculative-simple \
38        -hf ggml-org/Qwen3-8B-GGUF \
39        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
40        --spec-type draft-dflash --spec-draft-n-max 7 -ngld 99 --color on \
41        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
42
43# spec-type draft-dspark
44./bin/llama-speculative-simple \
45        -hf ggml-org/Qwen3-8B-GGUF \
46        -p "Here is a quick sort implementation in C++. Just code, no comments:\n\n#include" \
47        --spec-type draft-dspark --spec-draft-n-max 7 -ngld 99 --color on \
48        -n 256 --temp 0 --top-k 1 --seed 42 -ngl 99 -lv 4
49```