char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

YiChen Lvmetal : per-device tuned (Q, NE) for flash-attn vec (#26570)f280b2698

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.1 KiB57 linesraw
1#pragma once
2
3#include "ggml-backend.h"
4#include "ggml-cpp.h"
5#include "ggml.h"
6
7#include <cstdint>
8#include <functional>
9#include <vector>
10
11// A prebuilt graph replicated to amortize dispatch and synchronization overhead.
12struct perf_cell {
13    ggml_context_ptr        ctx;
14    ggml_backend_buffer_ptr buf;
15    ggml_cgraph *           gf     = nullptr;
16    int                     n_runs = 0;
17};
18
19using build_graph_fn  = std::function<ggml_tensor *(ggml_context *)>;
20using init_tensors_fn = std::function<void(ggml_context *)>;
21using op_flops_fn     = std::function<uint64_t(ggml_tensor *)>;
22
23perf_cell build_perf_cell(ggml_backend_t          backend,
24                          const build_graph_fn &  build,
25                          const init_tensors_fn & init,
26                          const op_flops_fn &     flops);
27
28double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps);
29
30struct cooldown_opts {
31    bool   enabled   = true;
32    double drift     = 0.10;  // anchor drift that triggers a cooldown
33    double eps       = 0.03;  // anchor tolerance to call the GPU cool again
34    int    max_wait  = 120;  // seconds of cooling per cell before giving up
35    int    max_retry = 2;    // re-measure rounds per cell before giving up
36};
37
38using set_candidate_fn   = std::function<void(int)>;
39using clear_candidate_fn = std::function<void()>;
40
41struct cell_result {
42    std::vector<double> t;
43    bool                trusted    = true;
44    double              anchor_min = 0.0;
45    double              anchor_max = 0.0;
46};
47
48// Times candidates in order while using baseline_cand as a thermal-drift anchor.
49cell_result measure_cell(ggml_backend_t             backend,
50                         const perf_cell &          cell,
51                         int                        reps,
52                         const std::vector<int> &   order,
53                         const set_candidate_fn &   set_cand,
54                         const clear_candidate_fn & clear_cand,
55                         int                        baseline_cand,
56                         const cooldown_opts &      cool,
57                         const char *               cell_label);