char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Ruixiang Wangserver-bench : add speed-bench for speculative decoding benchmarking (#23869)689a9a470

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.1 KiB84 linesraw
1#!/usr/bin/env python3
2from __future__ import annotations
3
4import argparse
5import json
6import sys
7from typing import Any
8
9from speed_bench import fmt_value, print_rows
10
11
12def load_summary(path: str) -> list[dict[str, Any]]:
13    with open(path, "r", encoding="utf-8") as f:
14        data = json.load(f)
15    summary = data.get("summary")
16    if not isinstance(summary, list):
17        raise ValueError(f"{path} does not contain a summary list")
18    return summary
19
20
21def compare_rows(baseline: list[dict[str, Any]], speculative: list[dict[str, Any]]) -> list[dict[str, Any]]:
22    baseline_by_category = {row["category"]: row for row in baseline}
23    comparisons = []
24    for row in speculative:
25        base = baseline_by_category.get(row["category"])
26        if not base:
27            continue
28        base_speed = base.get("avg_pred_t_s")
29        spec_speed = row.get("avg_pred_t_s")
30        base_latency = base.get("avg_latency")
31        spec_latency = row.get("avg_latency")
32        comparisons.append(
33            {
34                "category": row["category"],
35                "base_avg_pred_t_s": base_speed,
36                "spec_avg_pred_t_s": spec_speed,
37                "decode_speedup": (spec_speed / base_speed) if base_speed and spec_speed else None,
38                "base_avg_latency": base_latency,
39                "spec_avg_latency": spec_latency,
40                "latency_speedup": (base_latency / spec_latency) if base_latency and spec_latency else None,
41                "accept_rate": row.get("accept_rate"),
42            }
43        )
44    return comparisons
45
46
47def print_comparison(rows: list[dict[str, Any]]) -> None:
48    if not rows:
49        print("No overlapping categories found for comparison.")
50        return
51    columns = [
52        ("category", "category", ""),
53        ("base_avg_pred_t/s", "base_avg_pred_t_s", "speed"),
54        ("spec_avg_pred_t/s", "spec_avg_pred_t_s", "speed"),
55        ("decode_speedup", "decode_speedup", "speedup"),
56        ("base_avg_latency", "base_avg_latency", "seconds"),
57        ("spec_avg_latency", "spec_avg_latency", "seconds"),
58        ("latency_speedup", "latency_speedup", "speedup"),
59        ("accept_rate", "accept_rate", "rate"),
60    ]
61    print_rows(rows, columns)
62
63
64def main(argv: list[str] | None = None) -> int:
65    parser = argparse.ArgumentParser(description="Compare two SPEED-Bench runs (baseline vs speculative).")
66    parser.add_argument("--baseline", required=True, help="Baseline results JSON produced by speed_bench.py --output")
67    parser.add_argument("--speculative", required=True, help="Speculative decoding results JSON produced by speed_bench.py --output")
68    args = parser.parse_args(argv)
69
70    try:
71        baseline = load_summary(args.baseline)
72        speculative = load_summary(args.speculative)
73    except Exception as exc:
74        print(f"speed_bench_compare: failed to load inputs: {exc}", file=sys.stderr)
75        return 2
76
77    comparisons = compare_rows(baseline, speculative)
78    print(f"Comparison: baseline={args.baseline} speculative={args.speculative}")
79    print_comparison(comparisons)
80    return 0
81
82
83if __name__ == "__main__":
84    raise SystemExit(main())