char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
4850c7727
char/aliceai-t5-moe
1#ifndef LLAMA_H 2#define LLAMA_H 3 4#include "ggml.h" 5#include "ggml-cpu.h" 6#include "ggml-backend.h" 7#include "ggml-opt.h" 8#include "gguf.h" 9 10#include <stddef.h> 11#include <stdint.h> 12#include <stdio.h> 13#include <stdbool.h> 14 15#ifdef LLAMA_SHARED 16# if defined(_WIN32 )&& !defined(__MINGW32__ ) 17# ifdef LLAMA_BUILD 18# define LLAMA_API __declspec(dllexport) 19# else 20# define LLAMA_API __declspec(dllimport) 21# endif 22# else 23# define LLAMA_API __attribute__ ((visibility ("default"))) 24# endif 25#else 26# define LLAMA_API 27#endif 28 29#ifdef __GNUC__ 30# define DEPRECATED (func ,hint ) func __attribute__((deprecated(hint))) 31#elif defined(_MSC_VER ) 32# define DEPRECATED (func ,hint ) __declspec(deprecated(hint)) func 33#else 34# define DEPRECATED (func ,hint ) func 35#endif 36 37#define LLAMA_DEFAULT_SEED 0xFFFFFFFF 38 39#define LLAMA_TOKEN_NULL -1 40 41#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' 42#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' 43#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq' 44 45#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN 46#define LLAMA_SESSION_VERSION 10 47 48#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ 49#define LLAMA_STATE_SEQ_VERSION 3 50 51#ifdef __cplusplus 52extern "C" { 53#endif 54 55// 56// C interface 57// 58// TODO: show sample usage 59// 60 61struct llama_vocab ; 62struct llama_model ; 63struct llama_context ; 64struct llama_sampler ; 65 66typedef struct llama_memory_i * llama_memory_t ; 67 68typedef int32_t llama_pos ; 69typedef int32_t llama_token ; 70typedef int32_t llama_seq_id ; 71 72enum llama_vocab_type { 73LLAMA_VOCAB_TYPE_NONE = 0 ,// For models without vocab 74LLAMA_VOCAB_TYPE_SPM = 1 ,// LLaMA tokenizer based on byte-level BPE with byte fallback 75LLAMA_VOCAB_TYPE_BPE = 2 ,// GPT-2 tokenizer based on byte-level BPE 76LLAMA_VOCAB_TYPE_WPM = 3 ,// BERT tokenizer based on WordPiece 77LLAMA_VOCAB_TYPE_UGM = 4 ,// T5 tokenizer based on Unigram 78LLAMA_VOCAB_TYPE_RWKV = 5 ,// RWKV tokenizer based on greedy tokenization 79LLAMA_VOCAB_TYPE_PLAMO2 = 6 ,// PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming 80 }; 81 82enum llama_rope_type { 83LLAMA_ROPE_TYPE_NONE = -1 , 84LLAMA_ROPE_TYPE_NORM = 0 , 85LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX , 86LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE , 87LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE , 88LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION , 89 }; 90 91enum llama_token_type {//TODO: remove, required until per token attributes are available from GGUF file 92LLAMA_TOKEN_TYPE_UNDEFINED = 0 , 93LLAMA_TOKEN_TYPE_NORMAL = 1 , 94LLAMA_TOKEN_TYPE_UNKNOWN = 2 , 95LLAMA_TOKEN_TYPE_CONTROL = 3 , 96LLAMA_TOKEN_TYPE_USER_DEFINED = 4 , 97LLAMA_TOKEN_TYPE_UNUSED = 5 , 98LLAMA_TOKEN_TYPE_BYTE = 6 , 99 }; 100 101enum llama_token_attr { 102LLAMA_TOKEN_ATTR_UNDEFINED = 0 , 103LLAMA_TOKEN_ATTR_UNKNOWN = 1 <<0 , 104LLAMA_TOKEN_ATTR_UNUSED = 1 <<1 , 105LLAMA_TOKEN_ATTR_NORMAL = 1 <<2 , 106LLAMA_TOKEN_ATTR_CONTROL = 1 <<3 ,// SPECIAL? 107LLAMA_TOKEN_ATTR_USER_DEFINED = 1 <<4 , 108LLAMA_TOKEN_ATTR_BYTE = 1 <<5 , 109LLAMA_TOKEN_ATTR_NORMALIZED = 1 <<6 , 110LLAMA_TOKEN_ATTR_LSTRIP = 1 <<7 , 111LLAMA_TOKEN_ATTR_RSTRIP = 1 <<8 , 112LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 <<9 , 113 }; 114 115// model file types 116enum llama_ftype { 117LLAMA_FTYPE_ALL_F32 = 0 , 118LLAMA_FTYPE_MOSTLY_F16 = 1 ,// except 1d tensors 119LLAMA_FTYPE_MOSTLY_Q4_0 = 2 ,// except 1d tensors 120LLAMA_FTYPE_MOSTLY_Q4_1 = 3 ,// except 1d tensors 121// LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 122// LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed 123// LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed 124LLAMA_FTYPE_MOSTLY_Q8_0 = 7 ,// except 1d tensors 125LLAMA_FTYPE_MOSTLY_Q5_0 = 8 ,// except 1d tensors 126LLAMA_FTYPE_MOSTLY_Q5_1 = 9 ,// except 1d tensors 127LLAMA_FTYPE_MOSTLY_Q2_K = 10 ,// except 1d tensors 128LLAMA_FTYPE_MOSTLY_Q3_K_S = 11 ,// except 1d tensors 129LLAMA_FTYPE_MOSTLY_Q3_K_M = 12 ,// except 1d tensors 130LLAMA_FTYPE_MOSTLY_Q3_K_L = 13 ,// except 1d tensors 131LLAMA_FTYPE_MOSTLY_Q4_K_S = 14 ,// except 1d tensors 132LLAMA_FTYPE_MOSTLY_Q4_K_M = 15 ,// except 1d tensors 133LLAMA_FTYPE_MOSTLY_Q5_K_S = 16 ,// except 1d tensors 134LLAMA_FTYPE_MOSTLY_Q5_K_M = 17 ,// except 1d tensors 135LLAMA_FTYPE_MOSTLY_Q6_K = 18 ,// except 1d tensors 136LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19 ,// except 1d tensors 137LLAMA_FTYPE_MOSTLY_IQ2_XS = 20 ,// except 1d tensors 138LLAMA_FTYPE_MOSTLY_Q2_K_S = 21 ,// except 1d tensors 139LLAMA_FTYPE_MOSTLY_IQ3_XS = 22 ,// except 1d tensors 140LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23 ,// except 1d tensors 141LLAMA_FTYPE_MOSTLY_IQ1_S = 24 ,// except 1d tensors 142LLAMA_FTYPE_MOSTLY_IQ4_NL = 25 ,// except 1d tensors 143LLAMA_FTYPE_MOSTLY_IQ3_S = 26 ,// except 1d tensors 144LLAMA_FTYPE_MOSTLY_IQ3_M = 27 ,// except 1d tensors 145LLAMA_FTYPE_MOSTLY_IQ2_S = 28 ,// except 1d tensors 146LLAMA_FTYPE_MOSTLY_IQ2_M = 29 ,// except 1d tensors 147LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 ,// except 1d tensors 148LLAMA_FTYPE_MOSTLY_IQ1_M = 31 ,// except 1d tensors 149LLAMA_FTYPE_MOSTLY_BF16 = 32 ,// except 1d tensors 150//LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack 151//LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack 152//LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack 153LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 ,// except 1d tensors 154LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 ,// except 1d tensors 155LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 ,// except 1d tensors 156LLAMA_FTYPE_MOSTLY_NVFP4 = 39 ,// except 1d tensors 157LLAMA_FTYPE_MOSTLY_Q1_0 = 40 ,// except 1d tensors 158LLAMA_FTYPE_MOSTLY_Q2_0 = 41 ,// except 1d tensors 159 160LLAMA_FTYPE_GUESSED = 1024 ,// not specified in the model file 161 }; 162 163// Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" 164LLAMA_API const char * llama_ftype_name (enum llama_ftype ftype ); 165 166enum llama_rope_scaling_type { 167LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 , 168LLAMA_ROPE_SCALING_TYPE_NONE = 0 , 169LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 , 170LLAMA_ROPE_SCALING_TYPE_YARN = 2 , 171LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3 , 172LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE , 173 }; 174 175enum llama_pooling_type { 176LLAMA_POOLING_TYPE_UNSPECIFIED = -1 , 177LLAMA_POOLING_TYPE_NONE = 0 , 178LLAMA_POOLING_TYPE_MEAN = 1 , 179LLAMA_POOLING_TYPE_CLS = 2 , 180LLAMA_POOLING_TYPE_LAST = 3 , 181LLAMA_POOLING_TYPE_RANK = 4 ,// used by reranking models to attach the classification head to the graph 182 }; 183 184enum llama_attention_type { 185LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1 , 186LLAMA_ATTENTION_TYPE_CAUSAL = 0 , 187LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1 , 188 }; 189 190enum llama_flash_attn_type { 191LLAMA_FLASH_ATTN_TYPE_AUTO = -1 , 192LLAMA_FLASH_ATTN_TYPE_DISABLED = 0 , 193LLAMA_FLASH_ATTN_TYPE_ENABLED = 1 , 194 }; 195 196LLAMA_API const char * llama_flash_attn_type_name (enum llama_flash_attn_type flash_attn_type ); 197 198enum llama_split_mode { 199LLAMA_SPLIT_MODE_NONE = 0 ,// single GPU 200LLAMA_SPLIT_MODE_LAYER = 1 ,// split layers and KV across GPUs 201LLAMA_SPLIT_MODE_ROW = 2 ,// split layers and KV across GPUs, use tensor parallelism if supported 202LLAMA_SPLIT_MODE_TENSOR = 3 , 203 }; 204 205enum llama_load_mode { 206LLAMA_LOAD_MODE_AUTO = -1 ,// auto-detect based on device capabilities 207LLAMA_LOAD_MODE_NONE = 0 ,// no special loading mode 208LLAMA_LOAD_MODE_MMAP = 1 ,// memory map the model 209LLAMA_LOAD_MODE_MLOCK = 2 ,// force system to keep model in RAM rather than swapping or compressing 210LLAMA_LOAD_MODE_MMAP_MLOCK = 3 ,// mmap + force system to keep model in RAM rather than swapping or compressing 211LLAMA_LOAD_MODE_DIRECT_IO = 4 ,// use direct I/O if available 212 }; 213 214LLAMA_API const char * llama_load_mode_name (enum llama_load_mode load_mode ); 215LLAMA_API enum llama_load_mode llama_load_mode_from_str (const char * str ); 216 217enum llama_lazy_mode { 218LLAMA_LAZY_MODE_OFF = 0 ,// always read the whole tensor up front 219LLAMA_LAZY_MODE_AUTO = 1 ,// lazy only for marked tensors larger than 4 GiB (requires mmap) 220LLAMA_LAZY_MODE_ON = 2 ,// read the rows of tensors marked by the arch on demand (requires mmap) 221 }; 222 223enum llama_context_type { 224LLAMA_CONTEXT_TYPE_DEFAULT = 0 , 225LLAMA_CONTEXT_TYPE_MTP = 1 , 226 }; 227 228// TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) 229typedef struct llama_token_data { 230llama_token id ;// token id 231float logit ;// log-odds of the token 232float p ;// probability of the token 233 }llama_token_data ; 234 235typedef struct llama_token_data_array { 236// TODO: consider SoA 237// NOTE: this pointer can be modified by the samplers 238llama_token_data * data ; 239size_t size ; 240int64_t selected ;// this is the index in the data array (i.e. not the token id) 241bool sorted ;// note: do not assume the data is sorted - always check this flag 242 }llama_token_data_array ; 243 244typedef bool (* llama_progress_callback )(float progress ,void * user_data ); 245 246// Input data for llama_encode/llama_decode 247// A llama_batch object can contain input about one or many sequences 248// The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens 249// 250// - token : the token ids of the input (used when embd is NULL) 251// - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) 252// - pos : the positions of the respective token in the sequence 253// (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode) 254// - seq_id : the sequence to which the respective token belongs 255// (if set to NULL, the sequence ID will be assumed to be 0) 256// - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output 257// (if set to NULL: 258// - if embeddings: all tokens are output 259// - if not: only the last token is output 260// ) 261// 262typedef struct llama_batch { 263int32_t n_tokens ; 264 265llama_token * token ; 266float * embd ; 267llama_pos * pos ; 268int32_t * n_seq_id ; 269llama_seq_id ** seq_id ; 270int8_t * logits ;// TODO: rename this to "output" 271 }llama_batch ; 272 273enum llama_model_kv_override_type { 274LLAMA_KV_OVERRIDE_TYPE_INT , 275LLAMA_KV_OVERRIDE_TYPE_FLOAT , 276LLAMA_KV_OVERRIDE_TYPE_BOOL , 277LLAMA_KV_OVERRIDE_TYPE_STR , 278 }; 279 280enum llama_model_meta_key { 281LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE , 282LLAMA_MODEL_META_KEY_SAMPLING_TOP_K , 283LLAMA_MODEL_META_KEY_SAMPLING_TOP_P , 284LLAMA_MODEL_META_KEY_SAMPLING_MIN_P , 285LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY , 286LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD , 287LLAMA_MODEL_META_KEY_SAMPLING_TEMP , 288LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N , 289LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT , 290LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT , 291LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU , 292LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA , 293 }; 294 295struct llama_model_kv_override { 296enum llama_model_kv_override_type tag ; 297 298char key [128 ]; 299 300union { 301int64_t val_i64 ; 302double val_f64 ; 303bool val_bool ; 304char val_str [128 ]; 305 }; 306 }; 307 308struct llama_model_tensor_buft_override { 309const char * pattern ; 310ggml_backend_buffer_type_t buft ; 311 }; 312 313struct llama_model_params { 314// NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) 315ggml_backend_dev_t * devices ; 316 317// NULL-terminated list of buffer types to use for tensors that match a pattern 318const struct llama_model_tensor_buft_override * tensor_buft_overrides ; 319 320int32_t n_gpu_layers ;// number of layers to store in VRAM, a negative value means all layers 321enum llama_split_mode split_mode ;// how to split the model across multiple GPUs 322enum llama_load_mode load_mode ;// how to load the model 323 324enum llama_lazy_mode lazy_mode ;// on-demand reading of tensors marked by the arch 325 326// the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE 327int32_t main_gpu ; 328 329// proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() 330const float * tensor_split ; 331 332// Called with a progress value between 0.0 and 1.0. Pass NULL to disable. 333// If the provided progress_callback returns true, model loading continues. 334// If it returns false, model loading is immediately aborted. 335llama_progress_callback progress_callback ; 336 337// context pointer passed to the progress callback 338void * progress_callback_user_data ; 339 340// override key-value pairs of the model meta data 341const struct llama_model_kv_override * kv_overrides ; 342 343// Keep the booleans together to avoid misalignment during copy-by-value. 344bool vocab_only ;// only load the vocabulary, no weights 345bool check_tensors ;// validate model tensor data 346bool use_extra_bufts ;// use extra buffer types (used for weight repacking) 347bool no_host ;// bypass host buffer allowing extra buffers to be used 348bool no_alloc ;// only load metadata and simulate memory allocations 349bool load_mtp ;// whether to load MTP layers 350 }; 351 352struct llama_sampler_seq_config { 353llama_seq_id seq_id ; 354struct llama_sampler * sampler ; 355 }; 356 357// NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations 358// https://github.com/ggml-org/llama.cpp/pull/7544 359struct llama_context_params { 360uint32_t n_ctx ;// text context, 0 = from model 361uint32_t n_batch ;// logical maximum batch size that can be submitted to llama_decode 362uint32_t n_ubatch ;// physical maximum batch size 363uint32_t n_seq_max ;// max number of sequences (i.e. distinct states for recurrent models) 364uint32_t n_rs_seq ;// number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] 365uint32_t n_outputs_max ;// max outputs in a ubatch (0 = n_batch) 366uint32_t n_outputs_max_per_seq ;// max outputs per sequence (0 = n_outputs_max) 367int32_t n_threads ;// number of threads to use for generation 368int32_t n_threads_batch ;// number of threads to use for batch processing 369 370enum llama_context_type ctx_type ;// set the context type (e.g. MTP) 371enum llama_rope_scaling_type rope_scaling_type ;// RoPE scaling type, from `enum llama_rope_scaling_type` 372enum llama_pooling_type pooling_type ;// whether to pool (sum) embedding results by sequence id 373enum llama_attention_type attention_type ;// attention type to use for embeddings 374enum llama_flash_attn_type flash_attn_type ;// when to enable Flash Attention 375 376// ref: https://github.com/ggml-org/llama.cpp/pull/2054 377float rope_freq_base ;// RoPE base frequency, 0 = from model 378float rope_freq_scale ;// RoPE frequency scaling factor, 0 = from model 379float yarn_ext_factor ;// YaRN extrapolation mix factor, negative = from model 380float yarn_attn_factor ;// YaRN magnitude scaling factor 381float yarn_beta_fast ;// YaRN low correction dim 382float yarn_beta_slow ;// YaRN high correction dim 383uint32_t yarn_orig_ctx ;// YaRN original context size 384float defrag_thold ;// [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default) 385 386ggml_backend_sched_eval_callback cb_eval ; 387void * cb_eval_user_data ; 388 389enum ggml_type type_k ;// data type for K cache [EXPERIMENTAL] 390enum ggml_type type_v ;// data type for V cache [EXPERIMENTAL] 391 392// Abort callback 393// if it returns true, execution of llama_decode() will be aborted 394// currently works only with CPU execution 395ggml_abort_callback abort_callback ; 396void * abort_callback_data ; 397 398// Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. 399bool embeddings ;// if true, extract embeddings (together with logits) 400bool offload_kqv ;// offload the KQV ops (including the KV cache) to GPU 401bool no_perf ;// measure performance timings 402bool op_offload ;// offload host tensor operations to device 403bool swa_full ;// use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) 404// NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases 405// ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 406bool kv_unified ;// use a unified buffer across the input sequences when computing the attention 407// try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix 408// ref: https://github.com/ggml-org/llama.cpp/pull/14363 409 410// [EXPERIMENTAL] 411// backend sampler chain configuration (make sure the caller keeps the sampler chains alive) 412// note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init) 413struct llama_sampler_seq_config * samplers ; 414size_t n_samplers ; 415 416// a source/target/parent context 417// can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts 418struct llama_context * ctx_other ; 419 }; 420 421struct llama_model_tensor_override { 422const char * pattern ; 423enum ggml_type type ; 424 }; 425 426struct llama_model_imatrix_data { 427const char * name ; 428const float * data ; 429size_t size ; 430 }; 431 432// model quantization parameters 433typedef struct llama_model_quantize_params { 434int32_t nthread ;// number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() 435enum llama_ftype ftype ;// quantize to this llama_ftype 436enum ggml_type output_tensor_type ;// output tensor type 437enum ggml_type token_embedding_type ;// token embeddings tensor type 438bool allow_requantize ;// allow quantizing non-f32/f16 tensors 439bool quantize_output_tensor ;// quantize output.weight 440bool only_copy ;// only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored 441bool pure ;// quantize all tensors to the default type 442bool keep_split ;// quantize to the same number of shards 443bool dry_run ;// calculate and show the final quantization size without performing quantization 444const struct llama_model_imatrix_data * imatrix ;// pointer to importance matrix data 445const struct llama_model_kv_override * kv_overrides ;// pointer to kv overrides 446const struct llama_model_tensor_override * tt_overrides ;// pointer to tensor overrides 447const int32_t * prune_layers ;// pointer to layer indices to prune 448size_t max_buf_size ;// max bytes of tensor rows kept in memory at once, 0 = default (8 GiB) 449 }llama_model_quantize_params ; 450 451typedef struct llama_logit_bias { 452llama_token token ; 453float bias ; 454 }llama_logit_bias ; 455 456typedef struct llama_sampler_chain_params { 457bool no_perf ;// whether to measure performance timings 458 }llama_sampler_chain_params ; 459 460// used in chat template 461typedef struct llama_chat_message { 462const char * role ; 463const char * content ; 464 }llama_chat_message ; 465 466// lora adapter 467struct llama_adapter_lora ; 468 469LLAMA_API const char * llama_version (void ); 470 471// Helpers for getting default parameters 472// TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172) 473LLAMA_API struct llama_model_params llama_model_default_params (void ); 474LLAMA_API struct llama_context_params llama_context_default_params (void ); 475LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params (void ); 476LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params (void ); 477 478// Initialize the llama + ggml backend 479// If numa is true, use NUMA optimizations 480// Call once at the start of the program 481LLAMA_API void llama_backend_init (void ); 482 483// Call once at the end of the program - currently only used for MPI 484LLAMA_API void llama_backend_free (void ); 485 486//optional: 487LLAMA_API void llama_numa_init (enum ggml_numa_strategy numa ); 488 489// Optional: an auto threadpool gets created in ggml if not passed explicitly 490LLAMA_API void llama_attach_threadpool ( 491struct llama_context * ctx , 492ggml_threadpool_t threadpool , 493ggml_threadpool_t threadpool_batch ); 494 495LLAMA_API void llama_detach_threadpool (struct llama_context * ctx ); 496 497typedef void (* llama_model_set_tensor_data_t )(struct ggml_tensor * tensor ,void * userdata ); 498 499// Create a new model from GGUF metadata as well as a function to set the tensor data 500// - tensors are created as GGML_TYPE_F32 by default, 501// override by adding a tensor with the same name but a different name to the context 502LLAMA_API struct llama_model * llama_model_init_from_user ( 503struct gguf_context * metadata , 504llama_model_set_tensor_data_t set_tensor_data ,// function to initialize tensor data with 505void * set_tensor_data_ud ,// userdata for function 506struct llama_model_params params ); 507 508DEPRECATED (LLAMA_API struct llama_model * llama_load_model_from_file ( 509const char * path_model , 510struct llama_model_params params ), 511"use llama_model_load_from_file instead" ); 512 513// Load a model from a file 514// If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf 515// If the split file name does not follow this pattern, use llama_model_load_from_splits 516LLAMA_API struct llama_model * llama_model_load_from_file ( 517const char * path_model , 518struct llama_model_params params ); 519 520// Load a model from an open FILE pointer 521LLAMA_API struct llama_model * llama_model_load_from_file_ptr ( 522FILE * file , 523struct llama_model_params params ); 524 525// Load a model from multiple splits (support custom naming scheme) 526// The paths must be in the correct order 527LLAMA_API struct llama_model * llama_model_load_from_splits ( 528const char ** paths , 529size_t n_paths , 530struct llama_model_params params ); 531 532LLAMA_API void llama_model_save_to_file ( 533const struct llama_model * model , 534const char * path_model ); 535 536DEPRECATED (LLAMA_API void llama_free_model (struct llama_model * model ), 537"use llama_model_free instead" ); 538 539LLAMA_API void llama_model_free (struct llama_model * model ); 540 541LLAMA_API struct llama_context * llama_init_from_model ( 542struct llama_model * model , 543struct llama_context_params params ); 544 545DEPRECATED (LLAMA_API struct llama_context * llama_new_context_with_model ( 546struct llama_model * model , 547struct llama_context_params params ), 548"use llama_init_from_model instead" ); 549 550// Frees all allocated memory 551LLAMA_API void llama_free (struct llama_context * ctx ); 552 553LLAMA_API int64_t llama_time_us (void ); 554 555LLAMA_API size_t llama_max_devices (void ); 556LLAMA_API size_t llama_max_parallel_sequences (void ); 557LLAMA_API size_t llama_max_tensor_buft_overrides (void ); 558 559LLAMA_API bool llama_supports_mmap (void ); 560LLAMA_API bool llama_supports_mlock (void ); 561LLAMA_API bool llama_supports_gpu_offload (void ); 562LLAMA_API bool llama_supports_rpc (void ); 563 564// NOTE: After creating a llama_context, it is recommended to query the actual values using these functions 565// In some cases the requested values via llama_context_params may differ from the actual values used by the context 566// ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732 567LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx ); 568LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx ); 569LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx ); 570LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx ); 571LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx ); 572LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx ); 573 574DEPRECATED (LLAMA_API int32_t llama_n_ctx_train (const struct llama_model * model ),"use llama_model_n_ctx_train instead" ); 575DEPRECATED (LLAMA_API int32_t llama_n_embd (const struct llama_model * model ),"use llama_model_n_embd instead" ); 576DEPRECATED (LLAMA_API int32_t llama_n_layer (const struct llama_model * model ),"use llama_model_n_layer instead" ); 577DEPRECATED (LLAMA_API int32_t llama_n_head (const struct llama_model * model ),"use llama_model_n_head instead" ); 578 579DEPRECATED (LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab ),"use llama_vocab_n_tokens instead" ); 580 581LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx ); 582LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx ); 583LLAMA_API enum llama_pooling_type llama_pooling_type (const struct llama_context * ctx );// TODO: rename to llama_get_pooling_type 584 585LLAMA_API const struct llama_vocab * llama_model_get_vocab (const struct llama_model * model ); 586LLAMA_API enum llama_rope_type llama_model_rope_type (const struct llama_model * model ); 587 588LLAMA_API int32_t llama_model_n_ctx_train (const struct llama_model * model ); 589LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model ); 590LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model ); 591LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model ); 592LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model ); 593LLAMA_API int32_t llama_model_n_layer_nextn (const struct llama_model * model ); 594LLAMA_API int32_t llama_model_n_head (const struct llama_model * model ); 595LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model ); 596LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model ); 597 598// Get the model's RoPE frequency scaling factor 599LLAMA_API float llama_model_rope_freq_scale_train (const struct llama_model * model ); 600 601// Returns the number of classifier outputs (only valid for classifier models) 602// Undefined behavior for non-classifier models 603LLAMA_API uint32_t llama_model_n_cls_out (const struct llama_model * model ); 604 605// Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided 606LLAMA_API const char * llama_model_cls_label (const struct llama_model * model ,uint32_t i ); 607 608LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_vocab * vocab ); 609 610LLAMA_API int32_t llama_vocab_n_tokens (const struct llama_vocab * vocab ); 611 612// Functions to access the model's GGUF metadata scalar values 613// - The functions return the length of the string on success, or -1 on failure 614// - The output string is always null-terminated and cleared on failure 615// - When retrieving a string, an extra byte must be allocated to account for the null terminator 616// - GGUF array values are not supported by these functions 617 618// Get metadata value as a string by key name 619LLAMA_API int32_t llama_model_meta_val_str (const struct llama_model * model ,const char * key ,char * buf ,size_t buf_size ); 620 621// Get the number of metadata key/value pairs 622LLAMA_API int32_t llama_model_meta_count (const struct llama_model * model ); 623 624// Get sampling metadata key name. Returns nullptr if the key is invalid 625LLAMA_API const char * llama_model_meta_key_str (enum llama_model_meta_key key ); 626 627// Get metadata key name by index 628LLAMA_API int32_t llama_model_meta_key_by_index (const struct llama_model * model ,int32_t i ,char * buf ,size_t buf_size ); 629 630// Get metadata value as a string by index 631LLAMA_API int32_t llama_model_meta_val_str_by_index (const struct llama_model * model ,int32_t i ,char * buf ,size_t buf_size ); 632 633// Get a string describing the model type 634LLAMA_API int32_t llama_model_desc (const struct llama_model * model ,char * buf ,size_t buf_size ); 635 636// Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 637LLAMA_API enum llama_ftype llama_model_ftype (const struct llama_model * model ); 638 639// Returns the total size of all the tensors in the model in bytes 640LLAMA_API uint64_t llama_model_size (const struct llama_model * model ); 641 642// Get the default chat template. Returns nullptr if not available 643// If name is NULL, returns the default chat template 644LLAMA_API const char * llama_model_chat_template (const struct llama_model * model ,const char * name ); 645 646// Returns the total number of parameters in the model 647LLAMA_API uint64_t llama_model_n_params (const struct llama_model * model ); 648 649// Returns true if the model contains an encoder that requires llama_encode() call 650LLAMA_API bool llama_model_has_encoder (const struct llama_model * model ); 651 652// Returns true if the model contains a decoder that requires llama_decode() call 653LLAMA_API bool llama_model_has_decoder (const struct llama_model * model ); 654 655// For encoder-decoder models, this function returns id of the token that must be provided 656// to the decoder to start generating output sequence. For other models, it returns -1. 657LLAMA_API llama_token llama_model_decoder_start_token (const struct llama_model * model ); 658 659// Returns true if the model is recurrent (like Mamba, RWKV, etc.) 660LLAMA_API bool llama_model_is_recurrent (const struct llama_model * model ); 661 662// Returns true if the model is hybrid (like Jamba, Granite, etc.) 663LLAMA_API bool llama_model_is_hybrid (const struct llama_model * model ); 664 665// Returns true if the model is diffusion-based (like LLaDA, Dream, etc.) 666LLAMA_API bool llama_model_is_diffusion (const struct llama_model * model ); 667 668// Returns 0 on success 669LLAMA_API uint32_t llama_model_quantize ( 670const char * fname_inp , 671const char * fname_out , 672const llama_model_quantize_params * params ); 673 674// 675// Adapters 676// 677 678// Load a LoRA adapter from file 679// The adapter is valid as long as the associated model is not freed 680LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init ( 681struct llama_model * model , 682const char * path_lora ); 683 684// Functions to access the adapter's GGUF metadata scalar values 685// - The functions return the length of the string on success, or -1 on failure 686// - The output string is always null-terminated and cleared on failure 687// - When retrieving a string, an extra byte must be allocated to account for the null terminator 688// - GGUF array values are not supported by these functions 689 690// Get metadata value as a string by key name 691LLAMA_API int32_t llama_adapter_meta_val_str (const struct llama_adapter_lora * adapter ,const char * key ,char * buf ,size_t buf_size ); 692 693// Get the number of metadata key/value pairs 694LLAMA_API int32_t llama_adapter_meta_count (const struct llama_adapter_lora * adapter ); 695 696// Get metadata key name by index 697LLAMA_API int32_t llama_adapter_meta_key_by_index (const struct llama_adapter_lora * adapter ,int32_t i ,char * buf ,size_t buf_size ); 698 699// Get metadata value as a string by index 700LLAMA_API int32_t llama_adapter_meta_val_str_by_index (const struct llama_adapter_lora * adapter ,int32_t i ,char * buf ,size_t buf_size ); 701 702// Manually free a LoRA adapter 703// NOTE: loaded adapters that are not manually freed will be freed when the associated model is deleted 704LLAMA_API void llama_adapter_lora_free (struct llama_adapter_lora * adapter ); 705 706// Get the invocation tokens if the current lora is an alora 707LLAMA_API uint64_t llama_adapter_get_alora_n_invocation_tokens (const struct llama_adapter_lora * adapter ); 708LLAMA_API const llama_token * llama_adapter_get_alora_invocation_tokens (const struct llama_adapter_lora * adapter ); 709 710// The following functions operate on a llama_context, hence the naming: llama_verb_... 711 712// Set LoRa adapters on the context. Will only modify if the adapters currently in context are different. 713LLAMA_API int32_t llama_set_adapters_lora ( 714struct llama_context * ctx , 715struct llama_adapter_lora ** adapters , 716size_t n_adapters , 717float * scales ); 718 719// Apply a loaded control vector to a llama_context, or if data is NULL, clear 720// the currently loaded vector. 721// n_embd should be the size of a single layer's control, and data should point 722// to an n_embd x n_layers buffer starting from layer 1. 723// il_start and il_end are the layer range the vector should apply to (both inclusive) 724// See llama_control_vector_load in common to load a control vector. 725LLAMA_API int32_t llama_set_adapter_cvec ( 726struct llama_context * ctx , 727const float * data , 728size_t len , 729int32_t n_embd , 730int32_t il_start , 731int32_t il_end ); 732 733// 734// Memory 735// 736 737// Clear the memory contents 738// If data == true, the data buffers will also be cleared together with the metadata 739LLAMA_API void llama_memory_clear ( 740llama_memory_t mem , 741bool data ); 742 743// Removes all tokens that belong to the specified sequence and have positions in [p0, p1) 744// Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails 745// seq_id < 0 : match any sequence [TAG_LLAMA_SEQ_ID_NEG] 746// p0 < 0 : [0, p1] 747// p1 < 0 : [p0, inf) 748LLAMA_API bool llama_memory_seq_rm ( 749llama_memory_t mem , 750llama_seq_id seq_id , 751llama_pos p0 , 752llama_pos p1 ); 753 754// Copy all tokens that belong to the specified sequence to another sequence 755// p0 < 0 : [0, p1] 756// p1 < 0 : [p0, inf) 757LLAMA_API void llama_memory_seq_cp ( 758llama_memory_t mem , 759llama_seq_id seq_id_src , 760llama_seq_id seq_id_dst , 761llama_pos p0 , 762llama_pos p1 ); 763 764// Removes all tokens that do not belong to the specified sequence 765LLAMA_API void llama_memory_seq_keep ( 766llama_memory_t mem , 767llama_seq_id seq_id ); 768 769// Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) 770// p0 < 0 : [0, p1] 771// p1 < 0 : [p0, inf) 772LLAMA_API void llama_memory_seq_add ( 773llama_memory_t mem , 774llama_seq_id seq_id , 775llama_pos p0 , 776llama_pos p1 , 777llama_pos delta ); 778 779// Integer division of the positions by factor of `d > 1` 780// p0 < 0 : [0, p1] 781// p1 < 0 : [p0, inf) 782LLAMA_API void llama_memory_seq_div ( 783llama_memory_t mem , 784llama_seq_id seq_id , 785llama_pos p0 , 786llama_pos p1 , 787int d ); 788 789// Returns the smallest position present in the memory for the specified sequence 790// This is typically non-zero only for SWA caches 791// Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory 792// Return -1 if the sequence is empty 793LLAMA_API llama_pos llama_memory_seq_pos_min ( 794llama_memory_t mem , 795llama_seq_id seq_id ); 796 797// Returns the largest position present in the memory for the specified sequence 798// Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory 799// Return -1 if the sequence is empty 800LLAMA_API llama_pos llama_memory_seq_pos_max ( 801llama_memory_t mem , 802llama_seq_id seq_id ); 803 804// Check if the memory supports shifting 805LLAMA_API bool llama_memory_can_shift (llama_memory_t mem ); 806 807// 808// State / sessions 809// 810 811// Returns the *actual* size in bytes of the state 812// (logits, embedding and memory) 813// Only use when saving the state, not when restoring it, otherwise the size may be too small. 814LLAMA_API size_t llama_state_get_size (struct llama_context * ctx ); 815LLAMA_API DEPRECATED (size_t llama_get_state_size (struct llama_context * ctx ), 816 "use llama_state_get_size instead "); 817 818// Copies the state to the specified destination address. 819// Destination needs to have allocated enough memory. 820// Returns the number of bytes copied 821LLAMA_API size_t llama_state_get_data ( 822struct llama_context * ctx , 823uint8_t * dst , 824size_t size ); 825LLAMA_API DEPRECATED (size_t llama_copy_state_data ( 826struct llama_context * ctx , 827uint8_t * dst ), 828 "use llama_state_get_data instead "); 829 830// Set the state reading from the specified address 831// Returns the number of bytes read 832LLAMA_API size_t llama_state_set_data ( 833struct llama_context * ctx , 834const uint8_t * src , 835size_t size ); 836LLAMA_API DEPRECATED (size_t llama_set_state_data ( 837struct llama_context * ctx , 838const uint8_t * src ), 839 "use llama_state_set_data instead "); 840 841// Save/load session file 842LLAMA_API bool llama_state_load_file ( 843struct llama_context * ctx , 844const char * path_session , 845llama_token * tokens_out , 846size_t n_token_capacity , 847size_t * n_token_count_out ); 848LLAMA_API DEPRECATED (bool llama_load_session_file ( 849struct llama_context * ctx , 850const char * path_session , 851llama_token * tokens_out , 852size_t n_token_capacity , 853size_t * n_token_count_out ), 854 "use llama_state_load_file instead "); 855 856LLAMA_API bool llama_state_save_file ( 857struct llama_context * ctx , 858const char * path_session , 859const llama_token * tokens , 860size_t n_token_count ); 861LLAMA_API DEPRECATED (bool llama_save_session_file ( 862struct llama_context * ctx , 863const char * path_session , 864const llama_token * tokens , 865size_t n_token_count ), 866 "use llama_state_save_file instead "); 867 868// Get the exact size needed to copy the state of a single sequence 869LLAMA_API size_t llama_state_seq_get_size ( 870struct llama_context * ctx , 871llama_seq_id seq_id ); 872 873// Copy the state of a single sequence into the specified buffer 874LLAMA_API size_t llama_state_seq_get_data ( 875struct llama_context * ctx , 876uint8_t * dst , 877size_t size , 878llama_seq_id seq_id ); 879 880// Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence 881// Returns: 882// - Positive: Ok 883// - Zero: Failed to load 884LLAMA_API size_t llama_state_seq_set_data ( 885struct llama_context * ctx , 886const uint8_t * src , 887size_t size , 888llama_seq_id dest_seq_id ); 889 890LLAMA_API size_t llama_state_seq_save_file ( 891struct llama_context * ctx , 892const char * filepath , 893llama_seq_id seq_id , 894const llama_token * tokens , 895size_t n_token_count ); 896 897// If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded 898LLAMA_API size_t llama_state_seq_load_file ( 899struct llama_context * ctx , 900const char * filepath , 901llama_seq_id dest_seq_id , 902llama_token * tokens_out , 903size_t n_token_capacity , 904size_t * n_token_count_out ); 905 906#define LLAMA_STATE_SEQ_FLAGS_NONE 0 907 908// for backwards-compat 909#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 910 911// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) 912#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 913 914// Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load). 915// Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag. 916#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 917 918typedef uint32_t llama_state_seq_flags ; 919 920LLAMA_API size_t llama_state_seq_get_size_ext ( 921struct llama_context * ctx , 922llama_seq_id seq_id , 923llama_state_seq_flags flags ); 924 925LLAMA_API size_t llama_state_seq_get_data_ext ( 926struct llama_context * ctx , 927uint8_t * dst , 928size_t size , 929llama_seq_id seq_id , 930llama_state_seq_flags flags ); 931 932LLAMA_API size_t llama_state_seq_set_data_ext ( 933struct llama_context * ctx , 934const uint8_t * src , 935size_t size , 936llama_seq_id dest_seq_id , 937llama_state_seq_flags flags ); 938 939// 940// Decoding 941// 942 943// Return batch for single sequence of tokens 944// The sequence ID will be fixed to 0 945// The position of the tokens will be tracked automatically by llama_decode 946// 947// NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it 948// 949LLAMA_API struct llama_batch llama_batch_get_one ( 950llama_token * tokens , 951int32_t n_tokens ); 952 953// Allocates a batch of tokens on the heap that can hold a maximum of n_tokens 954// Each token can be assigned up to n_seq_max sequence ids 955// The batch has to be freed with llama_batch_free() 956// If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) 957// Otherwise, llama_batch.token will be allocated to store n_tokens llama_token 958// The rest of the llama_batch members are allocated with size n_tokens 959// All members are left uninitialized 960LLAMA_API struct llama_batch llama_batch_init ( 961int32_t n_tokens , 962int32_t embd , 963int32_t n_seq_max ); 964 965// Frees a batch of tokens allocated with llama_batch_init() 966LLAMA_API void llama_batch_free (struct llama_batch batch ); 967 968// Process a batch of tokens. 969// In contrast to llama_decode() - this call does not use KV cache. 970// For encode-decoder contexts, processes the batch using the encoder. 971// Can store the encoder output internally for later use by the decoder's cross-attention layers. 972// 0 - success 973// < 0 - error. the memory state is restored to the state before this call 974LLAMA_API int32_t llama_encode ( 975struct llama_context * ctx , 976struct llama_batch batch ); 977 978// Process a batch of tokens. 979// Requires the context to have a memory. 980// For encode-decoder contexts, processes the batch using the decoder. 981// Positive return values does not mean a fatal error, but rather a warning. 982// Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context 983// To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() 984// Upon other return values, the memory state is restored to the state before this call 985// 0 - success 986// 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) 987// 2 - aborted (processed ubatches will remain in the context's memory) 988// -1 - invalid input batch 989// < -1 - fatal error (processed ubatches will remain in the context's memory) 990LLAMA_API int32_t llama_decode ( 991struct llama_context * ctx , 992struct llama_batch batch ); 993 994// Set the number of threads used for decoding 995// n_threads is the number of threads used for generation (single token) 996// n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) 997LLAMA_API void llama_set_n_threads (struct llama_context * ctx ,int32_t n_threads ,int32_t n_threads_batch ); 998 999// Get the number of threads used for generation of a single token. 1000LLAMA_API int32_t llama_n_threads (struct llama_context * ctx ); 1001 1002// Get the number of threads used for prompt and batch processing (multiple token). 1003LLAMA_API int32_t llama_n_threads_batch (struct llama_context * ctx ); 1004 1005// Set whether the context outputs embeddings or not 1006// TODO: rename to avoid confusion with llama_get_embeddings() 1007LLAMA_API void llama_set_embeddings (struct llama_context * ctx ,bool embeddings ); 1008 1009// Set whether to use causal attention or not 1010// If set to true, the model will only attend to the past tokens 1011LLAMA_API void llama_set_causal_attn (struct llama_context * ctx ,bool causal_attn ); 1012 1013// Set whether the model is in warmup mode or not 1014// If true, all model tensors are activated during llama_decode() to load and cache their weights. 1015// 1016// note: using this can cause extra graph reallocations because it changes the graph topology with MoE models, 1017// so it is generally not recommended to use in practice. will be removed in the future 1018DEPRECATED (LLAMA_API void llama_set_warmup (struct llama_context * ctx ,bool warmup ), 1019"user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]" ); 1020 1021// Set abort callback 1022LLAMA_API void llama_set_abort_callback (struct llama_context * ctx ,ggml_abort_callback abort_callback ,void * abort_callback_data ); 1023 1024// Wait until all computations are finished 1025// This is automatically done when using one of the functions below to obtain the computation results 1026// and is not necessary to call it explicitly in most cases 1027LLAMA_API void llama_synchronize (struct llama_context * ctx ); 1028 1029// Token logits obtained from the last call to llama_decode() 1030// The logits for which llama_batch.logits[i] != 0 are stored contiguously 1031// in the order they have appeared in the batch. 1032// Rows: number of tokens for which llama_batch.logits[i] != 0 1033// Cols: n_vocab 1034// TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) 1035LLAMA_API float * llama_get_logits (struct llama_context * ctx ); 1036 1037// Logits for the ith token. For positive indices, Equivalent to: 1038// llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab 1039// Negative indices can be used to access logits in reverse order, -1 is the last logit. 1040// returns NULL for invalid ids. 1041LLAMA_API float * llama_get_logits_ith (struct llama_context * ctx ,int32_t i ); 1042 1043// Get all output token embeddings. 1044// when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, 1045// the embeddings for which llama_batch.logits[i] != 0 are stored contiguously 1046// in the order they have appeared in the batch. 1047// shape: [n_outputs*n_embd] 1048// Otherwise, returns NULL. 1049// TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) 1050LLAMA_API float * llama_get_embeddings (struct llama_context * ctx ); 1051 1052// Get the embeddings for the ith token. For positive indices, Equivalent to: 1053// llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd 1054// Negative indices can be used to access embeddings in reverse order, -1 is the last embedding. 1055// shape: [n_embd] (1-dimensional) 1056// returns NULL for invalid ids. 1057LLAMA_API float * llama_get_embeddings_ith (struct llama_context * ctx ,int32_t i ); 1058 1059// Get the embeddings for a sequence id 1060// Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE 1061// when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence 1062// otherwise: float[n_embd] (1-dimensional) 1063LLAMA_API float * llama_get_embeddings_seq (struct llama_context * ctx ,llama_seq_id seq_id ); 1064 1065// 1066// backend sampling API [EXPERIMENTAL] 1067// note: use only if the llama_context was created with at least one llama_sampler_seq_config 1068// 1069 1070// Get the backend sampled token for the ith token. 1071// With multiple outputs, sampler state advances when the token is accepted, 1072// not when it is read through this function. 1073// When accepting multiple outputs, accept a contiguous prefix in output order. 1074// Returns LLAMA_TOKEN_NULL if no token was sampled. 1075LLAMA_API llama_token llama_get_sampled_token_ith (struct llama_context * ctx ,int32_t i ); 1076 1077// Get the backend sampled probabilities for the ith token 1078// The index matches llama_get_sampled_token_ith(). 1079// Returns NULL if no probabilities were generated. 1080LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx ,int32_t i ); 1081LLAMA_API uint32_t llama_get_sampled_probs_count_ith (struct llama_context * ctx ,int32_t i ); 1082 1083// Get the backend sampled logits for the ith token 1084// Returns NULL if no logits were sampled. 1085LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx ,int32_t i ); 1086LLAMA_API uint32_t llama_get_sampled_logits_count_ith (struct llama_context * ctx ,int32_t i ); 1087 1088// Get the backend sampled candidates (token ids) for the ith token 1089// These are needed to map probability/logit indices to vocab token ids. 1090// Returns NULL if no candidates were sampled. 1091LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx ,int32_t i ); 1092LLAMA_API uint32_t llama_get_sampled_candidates_count_ith (struct llama_context * ctx ,int32_t i ); 1093 1094// 1095// Vocab 1096// 1097 1098LLAMA_API const char * llama_vocab_get_text (const struct llama_vocab * vocab ,llama_token token ); 1099 1100LLAMA_API float llama_vocab_get_score (const struct llama_vocab * vocab ,llama_token token ); 1101 1102LLAMA_API enum llama_token_attr llama_vocab_get_attr (const struct llama_vocab * vocab ,llama_token token ); 1103 1104// Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) 1105LLAMA_API bool llama_vocab_is_eog (const struct llama_vocab * vocab ,llama_token token ); 1106 1107// Identify if Token Id is a control token or a render-able token 1108LLAMA_API bool llama_vocab_is_control (const struct llama_vocab * vocab ,llama_token token ); 1109 1110// Special tokens 1111LLAMA_API llama_token llama_vocab_bos (const struct llama_vocab * vocab );// beginning-of-sentence 1112LLAMA_API llama_token llama_vocab_eos (const struct llama_vocab * vocab );// end-of-sentence 1113LLAMA_API llama_token llama_vocab_eot (const struct llama_vocab * vocab );// end-of-turn 1114LLAMA_API llama_token llama_vocab_sep (const struct llama_vocab * vocab );// sentence separator 1115LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab );// next-line 1116LLAMA_API llama_token llama_vocab_pad (const struct llama_vocab * vocab );// padding 1117LLAMA_API llama_token llama_vocab_mask (const struct llama_vocab * vocab );// mask 1118 1119LLAMA_API bool llama_vocab_get_add_bos (const struct llama_vocab * vocab ); 1120LLAMA_API bool llama_vocab_get_add_eos (const struct llama_vocab * vocab ); 1121LLAMA_API bool llama_vocab_get_add_sep (const struct llama_vocab * vocab ); 1122 1123// model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) 1124LLAMA_API const llama_token * llama_vocab_get_suppress_tokens (const struct llama_vocab * vocab ,int32_t * n_suppress_tokens ); 1125 1126LLAMA_API llama_token llama_vocab_fim_pre (const struct llama_vocab * vocab ); 1127LLAMA_API llama_token llama_vocab_fim_suf (const struct llama_vocab * vocab ); 1128LLAMA_API llama_token llama_vocab_fim_mid (const struct llama_vocab * vocab ); 1129LLAMA_API llama_token llama_vocab_fim_pad (const struct llama_vocab * vocab ); 1130LLAMA_API llama_token llama_vocab_fim_rep (const struct llama_vocab * vocab ); 1131LLAMA_API llama_token llama_vocab_fim_sep (const struct llama_vocab * vocab ); 1132 1133DEPRECATED (LLAMA_API const char * llama_token_get_text (const struct llama_vocab * vocab ,llama_token token ),"use llama_vocab_get_text instead" ); 1134DEPRECATED (LLAMA_API float llama_token_get_score (const struct llama_vocab * vocab ,llama_token token ),"use llama_vocab_get_score instead" ); 1135DEPRECATED (LLAMA_API enum llama_token_attr llama_token_get_attr (const struct llama_vocab * vocab ,llama_token token ),"use llama_vocab_get_attr instead" ); 1136DEPRECATED (LLAMA_API bool llama_token_is_eog (const struct llama_vocab * vocab ,llama_token token ),"use llama_vocab_is_eog instead" ); 1137DEPRECATED (LLAMA_API bool llama_token_is_control (const struct llama_vocab * vocab ,llama_token token ),"use llama_vocab_is_control instead" ); 1138DEPRECATED (LLAMA_API llama_token llama_token_bos (const struct llama_vocab * vocab ),"use llama_vocab_bos instead" ); 1139DEPRECATED (LLAMA_API llama_token llama_token_eos (const struct llama_vocab * vocab ),"use llama_vocab_eos instead" ); 1140DEPRECATED (LLAMA_API llama_token llama_token_eot (const struct llama_vocab * vocab ),"use llama_vocab_eot instead" ); 1141DEPRECATED (LLAMA_API llama_token llama_token_cls (const struct llama_vocab * vocab ),"use llama_vocab_cls instead" ); 1142DEPRECATED (LLAMA_API llama_token llama_token_sep (const struct llama_vocab * vocab ),"use llama_vocab_sep instead" ); 1143DEPRECATED (LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab ),"use llama_vocab_nl instead" ); 1144DEPRECATED (LLAMA_API llama_token llama_token_pad (const struct llama_vocab * vocab ),"use llama_vocab_pad instead" ); 1145DEPRECATED (LLAMA_API bool llama_add_bos_token (const struct llama_vocab * vocab ),"use llama_vocab_get_add_bos instead" ); 1146DEPRECATED (LLAMA_API bool llama_add_eos_token (const struct llama_vocab * vocab ),"use llama_vocab_get_add_eos instead" ); 1147DEPRECATED (LLAMA_API llama_token llama_token_fim_pre (const struct llama_vocab * vocab ),"use llama_vocab_fim_pre instead" ); 1148DEPRECATED (LLAMA_API llama_token llama_token_fim_suf (const struct llama_vocab * vocab ),"use llama_vocab_fim_suf instead" ); 1149DEPRECATED (LLAMA_API llama_token llama_token_fim_mid (const struct llama_vocab * vocab ),"use llama_vocab_fim_mid instead" ); 1150DEPRECATED (LLAMA_API llama_token llama_token_fim_pad (const struct llama_vocab * vocab ),"use llama_vocab_fim_pad instead" ); 1151DEPRECATED (LLAMA_API llama_token llama_token_fim_rep (const struct llama_vocab * vocab ),"use llama_vocab_fim_rep instead" ); 1152DEPRECATED (LLAMA_API llama_token llama_token_fim_sep (const struct llama_vocab * vocab ),"use llama_vocab_fim_sep instead" ); 1153 1154// CLS is equivalent to BOS 1155DEPRECATED (LLAMA_API llama_token llama_vocab_cls (const struct llama_vocab * vocab ),// classification 1156"use llama_vocab_bos instead" ); 1157 1158// 1159// Tokenization 1160// 1161// The API is thread-safe. 1162// 1163 1164/// @details Convert the provided text into tokens. 1165/// @param tokens The tokens pointer must be large enough to hold the resulting tokens. 1166/// @return Returns the number of tokens on success, no more than n_tokens_max 1167/// @return Returns a negative number on failure - the number of tokens that would have been returned 1168/// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit) 1169/// @param add_special Allow to add BOS and EOS tokens if model is configured to do so. 1170/// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated 1171/// as plaintext. Does not insert a leading space. 1172LLAMA_API int32_t llama_tokenize ( 1173const struct llama_vocab * vocab , 1174const char * text , 1175int32_t text_len , 1176llama_token * tokens , 1177int32_t n_tokens_max , 1178bool add_special , 1179bool parse_special ); 1180 1181// Token Id -> Piece. 1182// Uses the vocabulary in the provided context. 1183// Does not write null terminator to the buffer. 1184// User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') 1185// @param special If true, special tokens are rendered in the output. 1186LLAMA_API int32_t llama_token_to_piece ( 1187const struct llama_vocab * vocab , 1188llama_token token , 1189char * buf , 1190int32_t length , 1191int32_t lstrip , 1192bool special ); 1193 1194/// @details Convert the provided tokens into text (inverse of llama_tokenize()). 1195/// @param text The char pointer must be large enough to hold the resulting text. 1196/// @return Returns the number of chars/bytes on success, no more than text_len_max. 1197/// @return Returns a negative number on failure - the number of chars/bytes that would have been returned. 1198/// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. 1199/// @param unparse_special If true, special tokens are rendered in the output. 1200LLAMA_API int32_t llama_detokenize ( 1201const struct llama_vocab * vocab , 1202const llama_token * tokens , 1203int32_t n_tokens , 1204char * text , 1205int32_t text_len_max , 1206bool remove_special , 1207bool unparse_special ); 1208 1209// 1210// Chat templates 1211// 1212 1213/// Apply chat template. Inspired by hf apply_chat_template() on python. 1214/// 1215/// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template 1216/// @param tmpl A Jinja template to use for this chat. 1217/// @param chat Pointer to a list of multiple llama_chat_message 1218/// @param n_msg Number of llama_chat_message in this chat 1219/// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. 1220/// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages) 1221/// @param length The size of the allocated buffer 1222/// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. 1223LLAMA_API int32_t llama_chat_apply_template ( 1224const char * tmpl , 1225const struct llama_chat_message * chat , 1226size_t n_msg , 1227bool add_ass , 1228char * buf , 1229int32_t length ); 1230 1231// Get list of built-in chat templates 1232LLAMA_API int32_t llama_chat_builtin_templates (const char ** output ,size_t len ); 1233 1234// 1235// Sampling API 1236// 1237// Sample usage: 1238// 1239// // prepare the sampling chain at the start 1240// auto sparams = llama_sampler_chain_default_params(); 1241// 1242// llama_sampler * smpl = llama_sampler_chain_init(sparams); 1243// 1244// llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50)); 1245// llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); 1246// llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8)); 1247// 1248// // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat" 1249// // this sampler will be responsible to select the actual token 1250// llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); 1251// 1252// ... 1253// 1254// // decoding loop: 1255// while (...) { 1256// ... 1257// 1258// llama_decode(ctx, batch); 1259// 1260// // sample from the logits of the last token in the batch 1261// const llama_token id = llama_sampler_sample(smpl, ctx, -1); 1262// 1263// ... 1264// } 1265// 1266// llama_sampler_free(smpl); 1267// 1268 1269typedef void * llama_sampler_context_t ; 1270 1271struct llama_sampler_data { 1272struct ggml_tensor * logits ; 1273struct ggml_tensor * probs ; 1274struct ggml_tensor * sampled ; 1275struct ggml_tensor * candidates ; 1276 }; 1277 1278// user code can implement the interface below in order to create custom llama_sampler 1279struct llama_sampler_i { 1280const char * (* name ) (const struct llama_sampler * smpl );// can be NULL 1281void (* accept )(struct llama_sampler * smpl ,llama_token token );// can be NULL 1282void (* apply ) (struct llama_sampler * smpl ,llama_token_data_array * cur_p );// required 1283void (* reset ) (struct llama_sampler * smpl );// can be NULL 1284struct llama_sampler * (* clone ) (const struct llama_sampler * smpl );// can be NULL if ctx is NULL 1285void (* free ) (struct llama_sampler * smpl );// can be NULL if ctx is NULL 1286 1287// [EXPERIMENTAL] 1288// backend sampling interface: 1289 1290// return true if the backend supports all ops needed by the sampler and can handle up to n_outputs_max_per_seq outputs per sequence 1291// note: call once per sampler 1292bool (* backend_init )( 1293struct llama_sampler * smpl , 1294ggml_backend_buffer_type_t buft , 1295uint32_t n_outputs_max_per_seq ); 1296 1297// call after .backend_apply() 1298void (* backend_accept )( 1299struct llama_sampler * smpl , 1300struct ggml_context * ctx , 1301struct ggml_cgraph * gf , 1302struct ggml_tensor * selected_token ); 1303 1304// call after .backend_init() 1305void (* backend_apply )( 1306struct llama_sampler * smpl , 1307struct ggml_context * ctx , 1308struct ggml_cgraph * gf , 1309struct llama_sampler_data * data ); 1310 1311// called before graph execution to set inputs for the current ubatch 1312void (* backend_set_input )(struct llama_sampler * smpl ); 1313 1314// called before rebuilding a sampling graph to clear any internal sampler state 1315void (* backend_reset )(struct llama_sampler * smpl ); 1316 1317// copy mutable state from src into dst while keeping dst's references to the current sampling graph 1318// src and dst must have the same type and configuration 1319void (* copy_state )(const struct llama_sampler * src ,struct llama_sampler * dst ); 1320 }; 1321 1322struct llama_sampler { 1323struct llama_sampler_i * iface ; 1324 1325llama_sampler_context_t ctx ; 1326 }; 1327 1328// [EXPERIMENTAL] 1329// attach a sampler to the context 1330// note: prefer initializing the context with llama_context_params.samplers when possible 1331LLAMA_API bool llama_set_sampler (struct llama_context * ctx ,llama_seq_id seq_id ,struct llama_sampler * smpl ); 1332 1333// mirror of llama_sampler_i: 1334LLAMA_API struct llama_sampler * llama_sampler_init (struct llama_sampler_i * iface ,llama_sampler_context_t ctx ); 1335LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl ); 1336LLAMA_API void llama_sampler_accept (struct llama_sampler * smpl ,llama_token token ); 1337LLAMA_API void llama_sampler_apply (struct llama_sampler * smpl ,llama_token_data_array * cur_p ); 1338LLAMA_API void llama_sampler_reset (struct llama_sampler * smpl ); 1339LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl ); 1340LLAMA_API void llama_sampler_copy (const struct llama_sampler * src ,struct llama_sampler * dst ); 1341// important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) 1342LLAMA_API void llama_sampler_free (struct llama_sampler * smpl ); 1343 1344// llama_sampler_chain 1345// a type of llama_sampler that can chain multiple samplers one after another 1346 1347LLAMA_API struct llama_sampler * llama_sampler_chain_init (struct llama_sampler_chain_params params ); 1348 1349// important: takes ownership of the sampler object and will free it when llama_sampler_free is called 1350LLAMA_API void llama_sampler_chain_add (struct llama_sampler * chain ,struct llama_sampler * smpl ); 1351 1352// return NULL if: 1353// - the sampler is NULL 1354// - the sampler is not a llama_sampler_chain 1355// - the index is out of bounds, unless i == -1 1356// - if i == -1, returns the chain itself (can be used to check if the sampler is a chain) 1357LLAMA_API struct llama_sampler * llama_sampler_chain_get (struct llama_sampler * chain ,int32_t i ); 1358 1359// the total number of samplers in the chain 1360LLAMA_API int32_t llama_sampler_chain_n (const struct llama_sampler * chain ); 1361 1362// after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed 1363LLAMA_API struct llama_sampler * llama_sampler_chain_remove (struct llama_sampler * chain ,int32_t i ); 1364 1365// available samplers: 1366 1367LLAMA_API struct llama_sampler * llama_sampler_init_greedy (void ); 1368 1369/// seed == LLAMA_DEFAULT_SEED to use a random seed. 1370LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed ); 1371 1372/// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 1373/// Setting k <= 0 makes this a noop 1374LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k ); 1375 1376/// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 1377LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p ,size_t min_keep ); 1378 1379/// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 1380LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p ,size_t min_keep ); 1381 1382/// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. 1383LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p ,size_t min_keep ); 1384 1385/// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf 1386LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t ); 1387 1388/// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. 1389LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t ,float delta ,float exponent ); 1390 1391/// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 1392LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p ,float t ,size_t min_keep ,uint32_t seed ); 1393 1394/// @details Top n sigma sampling as described in academic paper "Top-nĪ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 1395LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma (float n ); 1396 1397/// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. 1398/// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. 1399/// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. 1400/// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. 1401/// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. 1402/// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. 1403LLAMA_API struct llama_sampler * llama_sampler_init_mirostat ( 1404int32_t n_vocab , 1405uint32_t seed , 1406float tau , 1407float eta , 1408int32_t m ); 1409 1410/// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. 1411/// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. 1412/// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. 1413/// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. 1414/// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. 1415LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2 ( 1416uint32_t seed , 1417float tau , 1418float eta ); 1419 1420/// @details Initializes a GBNF grammar, see grammars/README.md for details. 1421/// @param vocab The vocabulary that this grammar will be used with. 1422/// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. 1423/// @param grammar_root The name of the start symbol for the grammar. 1424LLAMA_API struct llama_sampler * llama_sampler_init_grammar ( 1425const struct llama_vocab * vocab , 1426const char * grammar_str , 1427const char * grammar_root ); 1428 1429DEPRECATED (LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy ( 1430const struct llama_vocab * vocab , 1431const char * grammar_str , 1432const char * grammar_root , 1433const char ** trigger_words , 1434size_t num_trigger_words , 1435const llama_token * trigger_tokens , 1436size_t num_trigger_tokens ), 1437 "use llama_sampler_init_grammar_lazy_patterns instead "); 1438 1439 1440/// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 1441/// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. 1442/// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. 1443LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns ( 1444const struct llama_vocab * vocab , 1445const char * grammar_str , 1446const char * grammar_root , 1447const char ** trigger_patterns , 1448size_t num_trigger_patterns , 1449const llama_token * trigger_tokens , 1450size_t num_trigger_tokens ); 1451 1452 1453/// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. 1454LLAMA_API struct llama_sampler * llama_sampler_init_penalties ( 1455int32_t n_vocab , 1456int32_t penalty_last_n ,// last n tokens to penalize (0 = disable penalty) 1457float penalty_repeat ,// must be > 0.0, 1.0 = disabled 1458float penalty_freq ,// must be finite, 0.0 = disabled 1459float penalty_present );// must be finite, 0.0 = disabled 1460 1461/// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 1462LLAMA_API struct llama_sampler * llama_sampler_init_dry ( 1463const struct llama_vocab * vocab , 1464float dry_multiplier , 1465float dry_base , 1466int32_t dry_allowed_length , 1467int32_t dry_penalty_last_n ,// last n tokens to penalize (0 = disable penalty) 1468const char ** seq_breakers , 1469size_t num_breakers ); 1470 1471/// adaptive-p: select tokens near a configurable target probability over time. 1472/// 1473/// the adaptive-p sampler transforms the token probability distribution to favor tokens 1474/// that fall near a user-configurable probability target. 1475/// 1476/// internally, the sampler maintains an exponential moving average of the *ORIGINAL* 1477/// probabilities of selected tokens at each sampling step. it uses this EMA to compute an 1478/// adapted target probability at each sampling step, thus maintaining the desired target 1479/// probability over time. 1480/// 1481/// adaptive-p selects a token ID rather than just mutating candidates, so it must be last 1482/// in the sampler chain (like mirostat, dist, greedy). 1483/// 1484/// only mild truncation before this sampler is recommended. we suggest applying min-p 1485/// before adaptive-p as the only other active sampler in the chain. 1486/// 1487/// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) 1488/// @param decay EMA decay for adaptation; history â 1/(1-decay) tokens (valid range 0.0 - 0.99) 1489/// @param seed RNG seed 1490/// 1491/// ref: https://github.com/ggml-org/llama.cpp/pull/17927 1492/// 1493LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p ( 1494float target , 1495float decay , 1496uint32_t seed ); 1497 1498LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias ( 1499int32_t n_vocab , 1500int32_t n_logit_bias , 1501const llama_logit_bias * logit_bias ); 1502 1503// this sampler is meant to be used for fill-in-the-middle infilling 1504// it's supposed to be used after top_k + top_p sampling 1505// 1506// 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG 1507// 2. combine probs of tokens that have the same prefix 1508// 1509// example: 1510// 1511// - before: 1512// "hel": 0.5 1513// "hell": 0.2 1514// "hello": 0.1 1515// "dummy": 0.1 1516// 1517// - after: 1518// "hel": 0.8 1519// "dummy": 0.1 1520// 1521// 3. discard non-EOG tokens with low prob 1522// 4. if no tokens are left -> pick EOT 1523// 1524LLAMA_API struct llama_sampler * llama_sampler_init_infill (const struct llama_vocab * vocab ); 1525 1526// Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise 1527LLAMA_API uint32_t llama_sampler_get_seed (const struct llama_sampler * smpl ); 1528 1529/// @details Sample and accept a token from the idx-th output of the last evaluation 1530// For multiple outputs from one sampler, call this function in output order without gaps. 1531// 1532// Shorthand for: 1533// const auto * logits = llama_get_logits_ith(ctx, idx); 1534// llama_token_data_array cur_p = { ... init from logits ... }; 1535// llama_sampler_apply(smpl, &cur_p); 1536// auto token = cur_p.data[cur_p.selected].id; 1537// llama_sampler_accept(smpl, token); 1538// return token; 1539// Returns the sampled token 1540LLAMA_API llama_token llama_sampler_sample (struct llama_sampler * smpl ,struct llama_context * ctx ,int32_t idx ); 1541 1542// TODO: extend in the future 1543//LLAMA_API void llama_decode_with_sampler(struct llama_context * ctx, struct llama_sampler * smpl, struct llama_batch batch, ...); 1544 1545// 1546// Model split 1547// 1548 1549/// @details Build a split GGUF final path for this chunk. 1550/// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" 1551// Returns the split_path length. 1552LLAMA_API int32_t llama_split_path (char * split_path ,size_t maxlen ,const char * path_prefix ,int32_t split_no ,int32_t split_count ); 1553 1554/// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. 1555/// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" 1556// Returns the split_prefix length. 1557LLAMA_API int32_t llama_split_prefix (char * split_prefix ,size_t maxlen ,const char * split_path ,int32_t split_no ,int32_t split_count ); 1558 1559// Print system information 1560LLAMA_API const char * llama_print_system_info (void ); 1561 1562// Set callback for all future logging events. 1563// If this is not called, or NULL is supplied, everything is output on stderr. 1564// The logger state is global so these functions are NOT thread safe. 1565LLAMA_API void llama_log_get (ggml_log_callback * log_callback ,void ** user_data ); 1566LLAMA_API void llama_log_set (ggml_log_callback log_callback ,void * user_data ); 1567 1568// 1569// Performance utils 1570// 1571// NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements. 1572// 1573 1574struct llama_perf_context_data { 1575// ms == milliseconds 1576double t_start_ms ;// absolute start time 1577double t_load_ms ;// time needed for loading the model 1578double t_p_eval_ms ;// time needed for processing the prompt 1579double t_eval_ms ;// time needed for generating tokens 1580 1581int32_t n_p_eval ;// number of prompt tokens 1582int32_t n_eval ;// number of generated tokens 1583int32_t n_reused ;// number of times a ggml compute graph had been reused 1584 }; 1585 1586struct llama_perf_sampler_data { 1587double t_sample_ms ;// time needed for sampling in ms 1588 1589int32_t n_sample ;// number of sampled tokens 1590 }; 1591 1592LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx ); 1593LLAMA_API void llama_perf_context_print (const struct llama_context * ctx ); 1594LLAMA_API void llama_perf_context_reset (struct llama_context * ctx ); 1595 1596// NOTE: the following work only with samplers constructed via llama_sampler_chain_init 1597LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain ); 1598LLAMA_API void llama_perf_sampler_print (const struct llama_sampler * chain ); 1599LLAMA_API void llama_perf_sampler_reset (struct llama_sampler * chain ); 1600 1601// 1602// training 1603// 1604 1605// function that returns whether or not a given tensor contains trainable parameters 1606typedef bool (* llama_opt_param_filter )(const struct ggml_tensor * tensor ,void * userdata ); 1607 1608// always returns true 1609LLAMA_API bool llama_opt_param_filter_all (const struct ggml_tensor * tensor ,void * userdata ); 1610 1611struct llama_opt_params { 1612uint32_t n_ctx_train ;// assumed context size post training, use context size specified in llama_context if 0 1613 1614llama_opt_param_filter param_filter ;// callback for determining which tensors contain trainable parameters 1615void * param_filter_ud ;// userdata for determining which tensors contain trainable parameters 1616 1617ggml_opt_get_optimizer_params get_opt_pars ;// callback for calculating optimizer parameters 1618void * get_opt_pars_ud ;// userdata for calculating optimizer parameters 1619 1620enum ggml_opt_optimizer_type optimizer_type ; 1621 }; 1622 1623LLAMA_API void llama_opt_init (struct llama_context * lctx ,struct llama_model * model ,struct llama_opt_params lopt_params ); 1624 1625LLAMA_API void llama_opt_epoch ( 1626struct llama_context * lctx , 1627ggml_opt_dataset_t dataset , 1628ggml_opt_result_t result_train , 1629ggml_opt_result_t result_eval , 1630int64_t idata_split , 1631ggml_opt_epoch_callback callback_train , 1632ggml_opt_epoch_callback callback_eval ); 1633 1634#ifdef __cplusplus 1635} 1636#endif 1637 1638#endif // LLAMA_H