char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Gaurav Gargllama : support multi-output backend sampling (#25532)dd1ea5243

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
982 B46 linesraw
1#pragma once
2
3#include "llama.h"
4
5#include <vector>
6
7struct llama_vocab;
8struct llama_grammar;
9
10// sampler chain
11
12struct llama_sampler_chain {
13    llama_sampler_chain_params params;
14
15    // has .backend_init() been called?
16    bool is_init = false;
17
18    uint32_t n_nodes = 0;
19
20    struct info {
21        bool is_backend;
22
23        llama_sampler * ptr;
24    };
25
26    std::vector<info> samplers;
27
28    // pre-allocated buffer for llama_sampler_sample to avoid repeated allocations
29    std::vector<llama_token_data> cur;
30
31    // timing
32
33    mutable int64_t t_sample_us;
34
35    mutable int32_t n_sample;
36};
37
38uint32_t llama_sampler_backend_n_nodes(const llama_sampler * sampler);
39void llama_sampler_backend_begin(llama_sampler * sampler);
40
41struct llama_sampler * llama_sampler_init_dry_testing(
42        float   dry_multiplier,
43        float   dry_base,
44        int32_t dry_allowed_length,
45        int32_t dry_penalty_last_n,
46        const std::vector<std::vector<llama_token>> & seq_breakers);