char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Sigbjørn Skjæretcontext : autoscale n_ctx_train when yarn scaling specified (#28030)dfc29b64e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.1 KiB67 linesraw
1#pragma once
2
3#include "llama.h"
4
5#include <cstdint>
6#include <vector>
7
8#define LLAMA_MAX_SEQ 256
9
10struct llama_cparams {
11    uint32_t n_ctx;           // context size used during inference
12    uint32_t n_ctx_seq;       // context for a single sequence
13    uint32_t n_batch;
14    uint32_t n_ubatch;
15    uint32_t n_seq_max;
16    uint32_t n_rs_seq;        // number of recurrent-state snapshots per seq for rollback
17    uint32_t n_outputs_max;   // max outputs supported by the context
18    uint32_t n_outputs_max_per_seq;
19    int32_t  n_threads;       // number of threads to use for generation
20    int32_t  n_threads_batch; // number of threads to use for batch processing
21
22    int32_t  nextn_layer_offset = 0;
23
24    float rope_freq_base;
25    float rope_freq_scale;
26
27    uint32_t n_ctx_orig_yarn;
28    // These hyperparameters are not exposed in GGUF, because all
29    // existing YaRN models use the same values for them.
30    float yarn_ext_factor;
31    float yarn_attn_factor;
32    float yarn_beta_fast;
33    float yarn_beta_slow;
34
35    bool embeddings;
36    bool embeddings_nextn;        // also extract the hidden state before the final output norm
37    bool embeddings_nextn_masked; // extract for only rows where batch.logits != 0
38    bool causal_attn;
39    bool offload_kqv;
40    bool flash_attn;
41    bool auto_fa;
42    bool fused_gdn_ar;       // use fused gated delta net (autoregressive)
43    bool fused_gdn_ch;       // use fused gated delta net (chunked)
44    bool auto_fgdn;
45    bool fused_lid;          // use fused lightning indexer
46    bool auto_flid;
47    bool fused_dsv4_hc_pre;
48    bool fused_dsv4_hc_comb;
49    bool fused_dsv4_hc_post;
50    bool auto_fhc;
51    bool no_perf;
52    bool warmup;             // TODO: remove [TAG_LLAMA_GRAPH_NO_WARMUP]
53    bool op_offload;
54    bool kv_unified;
55    bool pipeline_parallel;
56
57    std::vector<bool> embeddings_layer_inp; // [n_layer()] extract input embeddings for layer
58
59    enum llama_context_type ctx_type;
60    enum llama_rope_scaling_type rope_scaling_type;
61    enum llama_pooling_type pooling_type;
62
63    ggml_backend_sched_eval_callback cb_eval;
64    void * cb_eval_user_data;
65
66    llama_context * ctx_other;
67};