hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovggml : update ggml_prec specification (#26675)5a6caa05f2026-09-08

1#pragma once
2
3//
4// GGML Tensor Library
5//
6// This documentation is still a work in progress.
7// If you wish some specific topics to be covered, feel free to drop a comment:
8//
9//   https://github.com/ggml-org/whisper.cpp/issues/40
10//
11// ## Overview
12//
13// This library implements:
14//
15//  - a set of tensor operations
16//  - automatic differentiation
17//  - basic optimization algorithms
18//
19// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
20// but is not limited to, the following:
21//
22//  - linear regression
23//  - support vector machines
24//  - neural networks
25//
26// The library allows the user to define a certain function using the available tensor operations. This function
27// definition is represented internally via a computation graph. Each tensor operation in the function definition
28// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
29// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
30// using one of the available optimization algorithms.
31//
32// For example, here we define the function: f(x) = a*x^2 + b
33//
34//   {
35//       struct ggml_init_params params = {
36//           .mem_size   = 16*1024*1024,
37//           .mem_buffer = NULL,
38//       };
39//
40//       // memory allocation happens here
41//       struct ggml_context * ctx = ggml_init(params);
42//
43//       struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
44//
45//       ggml_set_param(ctx, x); // x is an input variable
46//
47//       struct ggml_tensor * a  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
48//       struct ggml_tensor * b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
49//       struct ggml_tensor * x2 = ggml_mul(ctx, x, x);
50//       struct ggml_tensor * f  = ggml_add(ctx, ggml_mul(ctx, a, x2), b);
51//
52//       ...
53//   }
54//
55// Notice that the function definition above does not involve any actual computation. The computation is performed only
56// when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
57//
58//   {
59//       ...
60//
61//       struct ggml_cgraph * gf = ggml_new_graph(ctx);
62//       ggml_build_forward_expand(gf, f);
63//
64//       // set the input variable and parameter values
65//       ggml_set_f32(x, 2.0f);
66//       ggml_set_f32(a, 3.0f);
67//       ggml_set_f32(b, 4.0f);
68//
69//       ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
70//
71//       printf("f = %f\n", ggml_get_f32_1d(f, 0));
72//
73//       ...
74//   }
75//
76// The actual computation is performed in the ggml_graph_compute() function.
77//
78// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
79// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
80// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
81// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was
82// actually needed.
83//
84// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
85// differentiation and optimization algorithms.
86//
87// The described approach allows to define the function graph once and then compute its forward or backward graphs
88// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way
89// the user can avoid the memory allocation overhead at runtime.
90//
91// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
92// citizens, but in theory the library can be extended to support FP8 and integer data types.
93//
94// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
95// and binary operations. Most of the available operations fall into one of these two categories. With time, it became
96// clear that the library needs to support more complex operations. The way to support these operations is not clear
97// yet, but a few examples are demonstrated in the following operations:
98//
99//   - ggml_permute()
100//   - ggml_conv_1d_1s()
101//   - ggml_conv_1d_2s()
102//
103// For each tensor operator, the library implements a forward and backward computation function. The forward function
104// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
105// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
106// calculus class, or watch the following video:
107//
108//   What is Automatic Differentiation?
109//   https://www.youtube.com/watch?v=wG_nF1awSSY
110//
111//
112// ## Tensor data (struct ggml_tensor)
113//
114// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of
115// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
116// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
117//
118//   {
119//       struct ggml_tensor * c = ggml_add(ctx, a, b);
120//
121//       assert(c->src[0] == a);
122//       assert(c->src[1] == b);
123//   }
124//
125// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the
126// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
127// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
128// permutation. All tensor operations have to take the stride into account and not assume that the tensor is
129// contiguous in memory.
130//
131// The data of the tensor is accessed via the "data" pointer. For example:
132//
133//   {
134//       const int nx = 2;
135//       const int ny = 3;
136//
137//       struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);
138//
139//       for (int y = 0; y < ny; y++) {
140//           for (int x = 0; x < nx; x++) {
141//               *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
142//           }
143//       }
144//
145//       ...
146//   }
147//
148// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.
149//
150// ## The matrix multiplication operator (ggml_mul_mat)
151//
152// TODO
153//
154//
155// ## Multi-threading
156//
157// TODO
158//
159//
160// ## Overview of ggml.c
161//
162// TODO
163//
164//
165// ## SIMD optimizations
166//
167// TODO
168//
169//
170// ## Debugging ggml
171//
172// TODO
173//
174//
175
176#ifdef GGML_SHARED
177#    if defined(_WIN32) && !defined(__MINGW32__)
178#        ifdef GGML_BUILD
179#            define GGML_API __declspec(dllexport) extern
180#        else
181#            define GGML_API __declspec(dllimport) extern
182#        endif
183#    else
184#        define GGML_API __attribute__ ((visibility ("default"))) extern
185#    endif
186#else
187#    define GGML_API extern
188#endif
189
190// TODO: support for clang
191#ifdef __GNUC__
192#    define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
193#elif defined(_MSC_VER)
194#    define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
195#else
196#    define GGML_DEPRECATED(func, hint) func
197#endif
198
199#ifndef __GNUC__
200#    define GGML_ATTRIBUTE_FORMAT(...)
201#elif defined(__MINGW32__) && !defined(__clang__)
202#    define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
203#else
204#    define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
205#endif
206
207#if defined(_WIN32) && !defined(_WIN32_WINNT)
208#    define _WIN32_WINNT 0x0A00
209#endif
210
211#include <stdbool.h>
212#include <stddef.h>
213#include <stdint.h>
214#include <stdio.h>
215
216#define GGML_FILE_MAGIC   0x67676d6c // "ggml"
217#define GGML_FILE_VERSION 2
218
219#define GGML_QNT_VERSION        2    // bump this on quantization format changes
220#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
221
222#define GGML_MAX_DIMS           4
223#define GGML_MAX_PARAMS         2048
224#define GGML_MAX_SRC            10
225#define GGML_MAX_N_THREADS      512
226#define GGML_MAX_OP_PARAMS      64
227
228#ifndef GGML_MAX_NAME
229#   define GGML_MAX_NAME        64
230#endif
231
232#define GGML_DEFAULT_N_THREADS  4
233#define GGML_DEFAULT_GRAPH_SIZE 2048
234
235#if UINTPTR_MAX == 0xFFFFFFFF
236    #define GGML_MEM_ALIGN 4
237#elif defined(__EMSCRIPTEN__)
238// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm.
239// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.)
240// ref: https://github.com/ggml-org/llama.cpp/pull/18628
241    #define GGML_MEM_ALIGN 8
242#else
243    #define GGML_MEM_ALIGN 16
244#endif
245
246#define GGML_EXIT_SUCCESS 0
247#define GGML_EXIT_ABORTED 1
248
249// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726
250#define GGML_ROPE_TYPE_NORMAL 0
251#define GGML_ROPE_TYPE_NEOX   2
252#define GGML_ROPE_TYPE_MROPE  8
253#define GGML_ROPE_TYPE_VISION 24
254#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000
255
256#define GGML_MROPE_SECTIONS   4
257
258#define GGML_UNUSED(x) (void)(x)
259#ifdef __CUDACC__
260template<typename... Args>
261__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {}
262#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__)
263#else
264#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0)
265#endif // __CUDACC__
266
267#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
268
269#ifndef NDEBUG
270#   define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0)
271#elif defined(__GNUC__)
272#   define GGML_UNREACHABLE() __builtin_unreachable()
273#elif defined(_MSC_VER)
274#   define GGML_UNREACHABLE() __assume(0)
275#else
276#   define GGML_UNREACHABLE() ((void) 0)
277#endif
278
279#ifdef __cplusplus
280#   define GGML_NORETURN [[noreturn]]
281#elif defined(_MSC_VER)
282#   define GGML_NORETURN __declspec(noreturn)
283#else
284#   define GGML_NORETURN _Noreturn
285#endif
286
287#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__)
288#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x)
289
290// used to copy the number of elements and stride in bytes of tensors into local variables.
291// main purpose is to reduce code duplication and improve readability.
292//
293// example:
294//
295//    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
296//    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb);
297//
298#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
299    const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \
300    GGML_UNUSED(prefix##0);
301#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
302    GGML_TENSOR_LOCALS_1    (type, prefix, pointer, array) \
303    const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \
304    GGML_UNUSED(prefix##1);
305#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
306    GGML_TENSOR_LOCALS_2    (type, prefix, pointer, array) \
307    const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \
308    GGML_UNUSED(prefix##2);
309#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
310    GGML_TENSOR_LOCALS_3  (type, prefix, pointer, array) \
311    const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \
312    GGML_UNUSED(prefix##3);
313
314#define GGML_TENSOR_UNARY_OP_LOCALS \
315    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
316    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb) \
317    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne) \
318    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)
319
320#define GGML_TENSOR_BINARY_OP_LOCALS \
321    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
322    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb) \
323    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
324    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb) \
325    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne) \
326    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)
327
328#define GGML_TENSOR_TERNARY_OP_LOCALS \
329    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
330    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb) \
331    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
332    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb) \
333    GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \
334    GGML_TENSOR_LOCALS(size_t,  nb2, src2, nb) \
335    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne) \
336    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)
337
338#define GGML_TENSOR_BINARY_OP_LOCALS01 \
339    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
340    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb) \
341    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
342    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb)
343
344#ifdef  __cplusplus
345extern "C" {
346#endif
347
348    // Function type used in fatal error callbacks
349    typedef void (*ggml_abort_callback_t)(const char * error_message);
350
351    // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)
352    // Returns the old callback for chaining
353    GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback);
354
355    GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)
356    GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);
357
358    enum ggml_status {
359        GGML_STATUS_ALLOC_FAILED = -2,
360        GGML_STATUS_FAILED = -1,
361        GGML_STATUS_SUCCESS = 0,
362        GGML_STATUS_ABORTED = 1,
363    };
364
365    // get ggml_status name string
366    GGML_API const char * ggml_status_to_string(enum ggml_status status);
367
368    // ieee 754-2008 half-precision float16
369    // todo: make this not an integral type
370    typedef uint16_t ggml_fp16_t;
371    GGML_API float       ggml_fp16_to_fp32(ggml_fp16_t);
372    GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);
373    GGML_API void        ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);
374    GGML_API void        ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);
375
376    // google brain half-precision bfloat16
377    typedef struct { uint16_t bits; } ggml_bf16_t;
378    GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
379    GGML_API float       ggml_bf16_to_fp32(ggml_bf16_t);  // consider just doing << 16
380    GGML_API void        ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
381    GGML_API void        ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);
382    GGML_API void        ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
383
384    struct ggml_object;
385    struct ggml_context;
386    struct ggml_cgraph;
387
388    // NOTE: always add types at the end of the enum to keep backward compatibility
389    enum ggml_type {
390        GGML_TYPE_F32     = 0,
391        GGML_TYPE_F16     = 1,
392        GGML_TYPE_Q4_0    = 2,
393        GGML_TYPE_Q4_1    = 3,
394        // GGML_TYPE_Q4_2 = 4, support has been removed
395        // GGML_TYPE_Q4_3 = 5, support has been removed
396        GGML_TYPE_Q5_0    = 6,
397        GGML_TYPE_Q5_1    = 7,
398        GGML_TYPE_Q8_0    = 8,
399        GGML_TYPE_Q8_1    = 9,
400        GGML_TYPE_Q2_K    = 10,
401        GGML_TYPE_Q3_K    = 11,
402        GGML_TYPE_Q4_K    = 12,
403        GGML_TYPE_Q5_K    = 13,
404        GGML_TYPE_Q6_K    = 14,
405        GGML_TYPE_Q8_K    = 15,
406        GGML_TYPE_IQ2_XXS = 16,
407        GGML_TYPE_IQ2_XS  = 17,
408        GGML_TYPE_IQ3_XXS = 18,
409        GGML_TYPE_IQ1_S   = 19,
410        GGML_TYPE_IQ4_NL  = 20,
411        GGML_TYPE_IQ3_S   = 21,
412        GGML_TYPE_IQ2_S   = 22,
413        GGML_TYPE_IQ4_XS  = 23,
414        GGML_TYPE_I8      = 24,
415        GGML_TYPE_I16     = 25,
416        GGML_TYPE_I32     = 26,
417        GGML_TYPE_I64     = 27,
418        GGML_TYPE_F64     = 28,
419        GGML_TYPE_IQ1_M   = 29,
420        GGML_TYPE_BF16    = 30,
421        // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
422        // GGML_TYPE_Q4_0_4_8 = 32,
423        // GGML_TYPE_Q4_0_8_8 = 33,
424        GGML_TYPE_TQ1_0   = 34,
425        GGML_TYPE_TQ2_0   = 35,
426        // GGML_TYPE_IQ4_NL_4_4 = 36,
427        // GGML_TYPE_IQ4_NL_4_8 = 37,
428        // GGML_TYPE_IQ4_NL_8_8 = 38,
429        GGML_TYPE_MXFP4   = 39, // MXFP4 (1 block)
430        GGML_TYPE_NVFP4   = 40, // NVFP4 (4 blocks, E4M3 scale)
431        GGML_TYPE_Q1_0    = 41,
432        GGML_TYPE_Q2_0    = 42,
433        GGML_TYPE_COUNT   = 43,
434    };
435
436    // [TAG_GGML_PREC]
437    // this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op
438    // the declared types can be:
439    //  - result accumulation type
440    //  - source tensor data representation type
441    //  - etc.
442    // the precision parameters are stored as ggml_tensor.op_params to the respective ops
443    enum ggml_prec {
444        GGML_PREC_UNDEFINED = 0,
445        GGML_PREC_DEFAULT   = 0,  // note: deprecated, use GGML_PREC_UNDEFINED
446        GGML_PREC_F32       = 10,
447        GGML_PREC_BF16      = 15,
448        GGML_PREC_F16       = 20,
449        GGML_PREC_Q8        = 30,
450        GGML_PREC_Q4        = 40,
451    };
452
453    // op hint
454    enum ggml_op_hint {
455        GGML_HINT_NONE             = 0,
456        GGML_HINT_SRC0_IS_HADAMARD = 1,
457    };
458
459    // model file types
460    enum ggml_ftype {
461        GGML_FTYPE_UNKNOWN        = -1,
462        GGML_FTYPE_ALL_F32        = 0,
463        GGML_FTYPE_MOSTLY_F16     = 1,  // except 1d tensors
464        GGML_FTYPE_MOSTLY_Q4_0    = 2,  // except 1d tensors
465        GGML_FTYPE_MOSTLY_Q4_1    = 3,  // except 1d tensors
466        GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
467        GGML_FTYPE_MOSTLY_Q8_0    = 7,  // except 1d tensors
468        GGML_FTYPE_MOSTLY_Q5_0    = 8,  // except 1d tensors
469        GGML_FTYPE_MOSTLY_Q5_1    = 9,  // except 1d tensors
470        GGML_FTYPE_MOSTLY_Q2_K    = 10, // except 1d tensors
471        GGML_FTYPE_MOSTLY_Q3_K    = 11, // except 1d tensors
472        GGML_FTYPE_MOSTLY_Q4_K    = 12, // except 1d tensors
473        GGML_FTYPE_MOSTLY_Q5_K    = 13, // except 1d tensors
474        GGML_FTYPE_MOSTLY_Q6_K    = 14, // except 1d tensors
475        GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
476        GGML_FTYPE_MOSTLY_IQ2_XS  = 16, // except 1d tensors
477        GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
478        GGML_FTYPE_MOSTLY_IQ1_S   = 18, // except 1d tensors
479        GGML_FTYPE_MOSTLY_IQ4_NL  = 19, // except 1d tensors
480        GGML_FTYPE_MOSTLY_IQ3_S   = 20, // except 1d tensors
481        GGML_FTYPE_MOSTLY_IQ2_S   = 21, // except 1d tensors
482        GGML_FTYPE_MOSTLY_IQ4_XS  = 22, // except 1d tensors
483        GGML_FTYPE_MOSTLY_IQ1_M   = 23, // except 1d tensors
484        GGML_FTYPE_MOSTLY_BF16    = 24, // except 1d tensors
485        GGML_FTYPE_MOSTLY_MXFP4   = 25, // except 1d tensors
486        GGML_FTYPE_MOSTLY_NVFP4   = 26, // except 1d tensors
487        GGML_FTYPE_MOSTLY_Q1_0    = 27, // except 1d tensors
488        GGML_FTYPE_MOSTLY_Q2_0    = 28, // except 1d tensors
489    };
490
491    // available tensor operations:
492    enum ggml_op {
493        GGML_OP_NONE = 0,
494
495        GGML_OP_DUP,
496        GGML_OP_ADD,
497        GGML_OP_ADD_ID,
498        GGML_OP_ADD1,
499        GGML_OP_ACC,
500        GGML_OP_SUB,
501        GGML_OP_MUL,
502        GGML_OP_DIV,
503        GGML_OP_SQR,
504        GGML_OP_SQRT,
505        GGML_OP_LOG,
506        GGML_OP_SIN,
507        GGML_OP_COS,
508        GGML_OP_SUM,
509        GGML_OP_SUM_ROWS,
510        GGML_OP_CUMSUM,
511        GGML_OP_MEAN,
512        GGML_OP_ARGMAX,
513        GGML_OP_COUNT_EQUAL,
514        GGML_OP_REPEAT,
515        GGML_OP_REPEAT_BACK,
516        GGML_OP_CONCAT,
517        GGML_OP_SILU_BACK,
518        GGML_OP_NORM, // normalize
519        GGML_OP_RMS_NORM,
520        GGML_OP_RMS_NORM_BACK,
521        GGML_OP_GROUP_NORM,
522        GGML_OP_L2_NORM,
523
524        GGML_OP_MUL_MAT,
525        GGML_OP_MUL_MAT_ID,
526        GGML_OP_OUT_PROD,
527
528        GGML_OP_SCALE,
529        GGML_OP_SET,
530        GGML_OP_CPY,
531        GGML_OP_CONT,
532        GGML_OP_RESHAPE,
533        GGML_OP_VIEW,
534        GGML_OP_PERMUTE,
535        GGML_OP_TRANSPOSE,
536        GGML_OP_GET_ROWS,
537        GGML_OP_GET_ROWS_BACK,
538        GGML_OP_SET_ROWS,
539        GGML_OP_DIAG,
540        GGML_OP_DIAG_MASK_INF,
541        GGML_OP_DIAG_MASK_ZERO,
542        GGML_OP_SOFT_MAX,
543        GGML_OP_SOFT_MAX_BACK,
544        GGML_OP_ROPE,
545        GGML_OP_ROPE_BACK,
546        GGML_OP_CLAMP,
547        GGML_OP_CONV_TRANSPOSE_1D,
548        GGML_OP_IM2COL,
549        GGML_OP_IM2COL_BACK,
550        GGML_OP_IM2COL_3D,
551        GGML_OP_COL2IM_1D,
552        GGML_OP_CONV_2D,
553        GGML_OP_CONV_3D,
554        GGML_OP_CONV_2D_DW,
555        GGML_OP_CONV_TRANSPOSE_2D,
556        GGML_OP_POOL_1D,
557        GGML_OP_POOL_2D,
558        GGML_OP_POOL_2D_BACK,
559        GGML_OP_UPSCALE,
560        GGML_OP_PAD,
561        GGML_OP_PAD_REFLECT_1D,
562        GGML_OP_ROLL,
563        GGML_OP_ARANGE,
564        GGML_OP_TIMESTEP_EMBEDDING,
565        GGML_OP_ARGSORT,
566        GGML_OP_TOP_K,
567        GGML_OP_LEAKY_RELU,
568        GGML_OP_TRI,
569        GGML_OP_FILL,
570
571        GGML_OP_FLASH_ATTN_EXT,
572        GGML_OP_FLASH_ATTN_BACK,
573        GGML_OP_SSM_CONV,
574        GGML_OP_SSM_SCAN,
575        GGML_OP_WIN_PART,
576        GGML_OP_WIN_UNPART,
577        GGML_OP_GET_REL_POS,
578        GGML_OP_ADD_REL_POS,
579        GGML_OP_RWKV_WKV6,
580        GGML_OP_GATED_LINEAR_ATTN,
581        GGML_OP_RWKV_WKV7,
582        GGML_OP_SOLVE_TRI,
583        GGML_OP_GATED_DELTA_NET,
584        GGML_OP_LIGHTNING_INDEXER,
585        GGML_OP_DSV4_HC_COMB,
586        GGML_OP_DSV4_HC_PRE,
587        GGML_OP_DSV4_HC_POST,
588
589        GGML_OP_UNARY,
590
591        GGML_OP_MAP_CUSTOM1,
592        GGML_OP_MAP_CUSTOM2,
593        GGML_OP_MAP_CUSTOM3,
594
595        GGML_OP_CUSTOM,
596
597        GGML_OP_CROSS_ENTROPY_LOSS,
598        GGML_OP_CROSS_ENTROPY_LOSS_BACK,
599        GGML_OP_OPT_STEP_ADAMW,
600        GGML_OP_OPT_STEP_SGD,
601
602        GGML_OP_GLU,
603
604        GGML_OP_COUNT,
605    };
606
607    enum ggml_unary_op {
608        GGML_UNARY_OP_ABS,
609        GGML_UNARY_OP_SGN,
610        GGML_UNARY_OP_NEG,
611        GGML_UNARY_OP_STEP,
612        GGML_UNARY_OP_TANH,
613        GGML_UNARY_OP_ELU,
614        GGML_UNARY_OP_RELU,
615        GGML_UNARY_OP_SIGMOID,
616        GGML_UNARY_OP_GELU,
617        GGML_UNARY_OP_GELU_QUICK,
618        GGML_UNARY_OP_SILU,
619        GGML_UNARY_OP_HARDSWISH,
620        GGML_UNARY_OP_HARDSIGMOID,
621        GGML_UNARY_OP_EXP,
622        GGML_UNARY_OP_EXPM1,
623        GGML_UNARY_OP_SOFTPLUS,
624        GGML_UNARY_OP_GELU_ERF,
625        GGML_UNARY_OP_XIELU,
626        GGML_UNARY_OP_FLOOR,
627        GGML_UNARY_OP_CEIL,
628        GGML_UNARY_OP_ROUND,
629        GGML_UNARY_OP_TRUNC,
630
631        GGML_UNARY_OP_COUNT,
632    };
633
634    enum ggml_glu_op {
635        GGML_GLU_OP_REGLU,
636        GGML_GLU_OP_GEGLU,
637        GGML_GLU_OP_SWIGLU,
638        GGML_GLU_OP_SWIGLU_OAI,
639        GGML_GLU_OP_GEGLU_ERF,
640        GGML_GLU_OP_GEGLU_QUICK,
641        GGML_GLU_OP_SWIGLU_CLAMP,
642
643        GGML_GLU_OP_COUNT,
644    };
645
646    enum ggml_object_type {
647        GGML_OBJECT_TYPE_TENSOR,
648        GGML_OBJECT_TYPE_GRAPH,
649        GGML_OBJECT_TYPE_WORK_BUFFER
650    };
651
652    enum ggml_log_level {
653        GGML_LOG_LEVEL_NONE  = 0,
654        GGML_LOG_LEVEL_DEBUG = 1,
655        GGML_LOG_LEVEL_INFO  = 2,
656        GGML_LOG_LEVEL_WARN  = 3,
657        GGML_LOG_LEVEL_ERROR = 4,
658        GGML_LOG_LEVEL_CONT  = 5, // continue previous log
659    };
660
661    // this tensor...
662    enum ggml_tensor_flag {
663        GGML_TENSOR_FLAG_INPUT   =  1, // ...is an input for the GGML compute graph
664        GGML_TENSOR_FLAG_OUTPUT  =  2, // ...is an output for the GGML compute graph
665        GGML_TENSOR_FLAG_PARAM   =  4, // ...contains trainable parameters
666        GGML_TENSOR_FLAG_LOSS    =  8, // ...defines loss for numerical optimization (multiple loss tensors add up)
667        GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed
668    };
669
670    enum ggml_tri_type {
671        GGML_TRI_TYPE_UPPER_DIAG = 0,
672        GGML_TRI_TYPE_UPPER      = 1,
673        GGML_TRI_TYPE_LOWER_DIAG = 2,
674        GGML_TRI_TYPE_LOWER      = 3
675    };
676
677    struct ggml_init_params {
678        // memory pool
679        size_t mem_size;   // bytes
680        void * mem_buffer; // if NULL, memory will be allocated internally
681        bool   no_alloc;   // don't allocate memory for the tensor data
682    };
683
684    // n-dimensional tensor
685    struct ggml_tensor {
686        enum ggml_type type;
687
688        struct ggml_backend_buffer * buffer;
689
690        int64_t ne[GGML_MAX_DIMS]; // number of elements
691        size_t  nb[GGML_MAX_DIMS]; // stride in bytes:
692                                   // nb[0] = ggml_type_size(type)
693                                   // nb[1] = nb[0]   * (ne[0] / ggml_blck_size(type)) + padding
694                                   // nb[i] = nb[i-1] * ne[i-1]
695
696        // compute data
697        enum ggml_op op;
698
699        // op params - allocated as int32_t for alignment
700        int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
701
702        int32_t flags;
703
704        struct ggml_tensor * src[GGML_MAX_SRC];
705
706        // source tensor and offset for views
707        struct ggml_tensor * view_src;
708        size_t               view_offs;
709
710        void * data;
711
712        char name[GGML_MAX_NAME];
713
714        void * extra; // extra things e.g. for ggml-cuda.cu
715
716        char padding[8];
717    };
718
719    static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
720
721    // Abort callback
722    // If not NULL, called before ggml computation
723    // If it returns true, the computation is aborted
724    typedef bool (*ggml_abort_callback)(void * data);
725
726
727    //
728    // GUID
729    //
730
731    // GUID types
732    typedef uint8_t ggml_guid[16];
733    typedef ggml_guid * ggml_guid_t;
734
735    GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);
736
737    // misc
738
739    GGML_API const char * ggml_version(void);
740    GGML_API const char * ggml_commit(void);
741
742    GGML_API void    ggml_time_init(void); // call this once at the beginning of the program
743    GGML_API int64_t ggml_time_ms(void);
744    GGML_API int64_t ggml_time_us(void);
745    GGML_API int64_t ggml_cycles(void);
746    GGML_API int64_t ggml_cycles_per_ms(void);
747
748    // accepts a UTF-8 path, even on Windows
749    GGML_API FILE *  ggml_fopen(const char * fname, const char * mode);
750
751    GGML_API void    ggml_print_object (const struct ggml_object * obj);
752    GGML_API void    ggml_print_objects(const struct ggml_context * ctx);
753
754    GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);
755    GGML_API int64_t ggml_nrows     (const struct ggml_tensor * tensor);
756    GGML_API size_t  ggml_nbytes    (const struct ggml_tensor * tensor);
757    GGML_API size_t  ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN
758
759    GGML_API int64_t ggml_blck_size(enum ggml_type type);
760    GGML_API size_t  ggml_type_size(enum ggml_type type);             // size in bytes for all elements in a block
761    GGML_API size_t  ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row
762
763    GGML_DEPRECATED(
764    GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float
765    "use ggml_row_size() instead");
766
767    GGML_API const char * ggml_type_name(enum ggml_type type);
768    GGML_API const char * ggml_op_name  (enum ggml_op   op);
769    GGML_API const char * ggml_op_symbol(enum ggml_op   op);
770
771    GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);
772    GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op);
773    GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name
774
775    GGML_API size_t  ggml_element_size(const struct ggml_tensor * tensor);
776
777    GGML_API bool    ggml_is_quantized(enum ggml_type type);
778
779    // TODO: temporary until model loading of ggml examples is refactored
780    GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);
781
782    GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);
783    GGML_API bool ggml_is_permuted  (const struct ggml_tensor * tensor);
784    GGML_API bool ggml_is_empty     (const struct ggml_tensor * tensor);
785    GGML_API bool ggml_is_view      (const struct ggml_tensor * tensor);
786    GGML_API bool ggml_is_scalar    (const struct ggml_tensor * tensor);
787    GGML_API bool ggml_is_vector    (const struct ggml_tensor * tensor);
788    GGML_API bool ggml_is_matrix    (const struct ggml_tensor * tensor);
789    GGML_API bool ggml_is_3d        (const struct ggml_tensor * tensor);
790    GGML_API int  ggml_n_dims       (const struct ggml_tensor * tensor); // returns 1 for scalars
791
792    // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation)
793    GGML_API bool ggml_is_contiguous  (const struct ggml_tensor * tensor);
794    GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()
795    GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1
796    GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2
797
798    GGML_API bool ggml_is_contiguous_to_1(const struct ggml_tensor * tensor); // contiguous for dims < 1
799    GGML_API bool ggml_is_contiguous_to_2(const struct ggml_tensor * tensor); // contiguous for dims < 2
800    GGML_API bool ggml_is_contiguous_to_3(const struct ggml_tensor * tensor); // contiguous for dims < 3
801
802    // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok)
803    GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor);
804
805    // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN
806    GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor);
807
808    // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements
809    GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor);
810
811    GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);
812    GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
813
814    GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
815
816    // use this to compute the memory overhead of a tensor
817    GGML_API size_t ggml_tensor_overhead(void);
818
819    GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);
820
821    // main
822
823    GGML_API struct ggml_context * ggml_init (struct ggml_init_params params);
824    GGML_API void                  ggml_reset(struct ggml_context * ctx);
825    GGML_API void                  ggml_free (struct ggml_context * ctx);
826
827    GGML_API size_t  ggml_used_mem(const struct ggml_context * ctx);
828
829    GGML_API bool    ggml_get_no_alloc(struct ggml_context * ctx);
830    GGML_API void    ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);
831
832    GGML_API void *  ggml_get_mem_buffer     (const struct ggml_context * ctx);
833    GGML_API size_t  ggml_get_mem_size       (const struct ggml_context * ctx);
834    GGML_API size_t  ggml_get_max_tensor_size(const struct ggml_context * ctx);
835
836    GGML_API struct ggml_tensor * ggml_new_tensor(
837            struct ggml_context * ctx,
838            enum   ggml_type type,
839            int    n_dims,
840            const int64_t *ne);
841
842    GGML_API struct ggml_tensor * ggml_new_tensor_1d(
843            struct ggml_context * ctx,
844            enum   ggml_type type,
845            int64_t ne0);
846
847    GGML_API struct ggml_tensor * ggml_new_tensor_2d(
848            struct ggml_context * ctx,
849            enum   ggml_type type,
850            int64_t ne0,
851            int64_t ne1);
852
853    GGML_API struct ggml_tensor * ggml_new_tensor_3d(
854            struct ggml_context * ctx,
855            enum   ggml_type type,
856            int64_t ne0,
857            int64_t ne1,
858            int64_t ne2);
859
860    GGML_API struct ggml_tensor * ggml_new_tensor_4d(
861            struct ggml_context * ctx,
862            enum   ggml_type type,
863            int64_t ne0,
864            int64_t ne1,
865            int64_t ne2,
866            int64_t ne3);
867
868    GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes);
869
870    GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);
871    GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);
872
873    // Context tensor enumeration and lookup
874    GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);
875    GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);
876    GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);
877
878    // Converts a flat index into coordinates
879    GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
880
881    GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);
882    GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor);
883
884    GGML_API void *  ggml_get_data    (const struct ggml_tensor * tensor);
885    GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
886
887    GGML_API const char *         ggml_get_name   (const struct ggml_tensor * tensor);
888    GGML_API struct ggml_tensor * ggml_set_name   (      struct ggml_tensor * tensor, const char * name);
889    GGML_ATTRIBUTE_FORMAT(2, 3)
890    GGML_API struct ggml_tensor * ggml_format_name(      struct ggml_tensor * tensor, const char * fmt, ...);
891
892    // Tensor flags
893    GGML_API void ggml_set_input(struct ggml_tensor * tensor);
894    GGML_API void ggml_set_output(struct ggml_tensor * tensor);
895    GGML_API void ggml_set_param(struct ggml_tensor * tensor);
896    GGML_API void ggml_set_loss(struct ggml_tensor * tensor);
897
898    //
899    // operations on tensors with backpropagation
900    //
901
902    GGML_API struct ggml_tensor * ggml_dup(
903            struct ggml_context * ctx,
904            struct ggml_tensor  * a);
905
906    // in-place, returns view(a)
907    GGML_API struct ggml_tensor * ggml_dup_inplace(
908            struct ggml_context * ctx,
909            struct ggml_tensor  * a);
910
911    GGML_API struct ggml_tensor * ggml_add(
912            struct ggml_context * ctx,
913            struct ggml_tensor  * a,
914            struct ggml_tensor  * b);
915
916    GGML_API struct ggml_tensor * ggml_add_inplace(
917            struct ggml_context * ctx,
918            struct ggml_tensor  * a,
919            struct ggml_tensor  * b);
920
921    GGML_API struct ggml_tensor * ggml_add_cast(
922            struct ggml_context * ctx,
923            struct ggml_tensor  * a,
924            struct ggml_tensor  * b,
925            enum   ggml_type      type);
926
927    // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]]
928    GGML_API struct ggml_tensor * ggml_add_id(
929            struct ggml_context * ctx,
930            struct ggml_tensor  * a,
931            struct ggml_tensor  * b,
932            struct ggml_tensor  * ids);
933
934    GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1(
935            struct ggml_context * ctx,
936            struct ggml_tensor  * a,
937            struct ggml_tensor  * b),
938        "use ggml_add instead");
939
940    GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace(
941            struct ggml_context * ctx,
942            struct ggml_tensor  * a,
943            struct ggml_tensor  * b),
944        "use ggml_add_inplace instead");
945
946    // dst = a
947    // view(dst, nb1, nb2, nb3, offset) += b
948    // return dst
949    GGML_API struct ggml_tensor * ggml_acc(
950            struct ggml_context * ctx,
951            struct ggml_tensor  * a,
952            struct ggml_tensor  * b,
953            size_t                nb1,
954            size_t                nb2,
955            size_t                nb3,
956            size_t                offset);
957
958    GGML_API struct ggml_tensor * ggml_acc_inplace(
959            struct ggml_context * ctx,
960            struct ggml_tensor  * a,
961            struct ggml_tensor  * b,
962            size_t                nb1,
963            size_t                nb2,
964            size_t                nb3,
965            size_t                offset);
966
967    GGML_API struct ggml_tensor * ggml_sub(
968            struct ggml_context * ctx,
969            struct ggml_tensor  * a,
970            struct ggml_tensor  * b);
971
972    GGML_API struct ggml_tensor * ggml_sub_inplace(
973            struct ggml_context * ctx,
974            struct ggml_tensor  * a,
975            struct ggml_tensor  * b);
976
977    GGML_API struct ggml_tensor * ggml_mul(
978            struct ggml_context * ctx,
979            struct ggml_tensor  * a,
980            struct ggml_tensor  * b);
981
982    GGML_API struct ggml_tensor * ggml_mul_inplace(
983            struct ggml_context * ctx,
984            struct ggml_tensor  * a,
985            struct ggml_tensor  * b);
986
987    GGML_API struct ggml_tensor * ggml_div(
988            struct ggml_context * ctx,
989            struct ggml_tensor  * a,
990            struct ggml_tensor  * b);
991
992    GGML_API struct ggml_tensor * ggml_div_inplace(
993            struct ggml_context * ctx,
994            struct ggml_tensor  * a,
995            struct ggml_tensor  * b);
996
997    GGML_API struct ggml_tensor * ggml_sqr(
998            struct ggml_context * ctx,
999            struct ggml_tensor  * a);
1000
1001    GGML_API struct ggml_tensor * ggml_sqr_inplace(
1002            struct ggml_context * ctx,
1003            struct ggml_tensor  * a);
1004
1005    GGML_API struct ggml_tensor * ggml_sqrt(
1006            struct ggml_context * ctx,
1007            struct ggml_tensor  * a);
1008
1009    GGML_API struct ggml_tensor * ggml_sqrt_inplace(
1010            struct ggml_context * ctx,
1011            struct ggml_tensor  * a);
1012
1013    GGML_API struct ggml_tensor * ggml_log(
1014            struct ggml_context * ctx,
1015            struct ggml_tensor  * a);
1016
1017    GGML_API struct ggml_tensor * ggml_log_inplace(
1018            struct ggml_context * ctx,
1019            struct ggml_tensor  * a);
1020
1021    GGML_API struct ggml_tensor * ggml_expm1(
1022            struct ggml_context * ctx,
1023            struct ggml_tensor  * a);
1024
1025    GGML_API struct ggml_tensor * ggml_expm1_inplace(
1026            struct ggml_context * ctx,
1027            struct ggml_tensor  * a);
1028
1029    GGML_API struct ggml_tensor * ggml_softplus(
1030            struct ggml_context * ctx,
1031            struct ggml_tensor  * a);
1032
1033    GGML_API struct ggml_tensor * ggml_softplus_inplace(
1034            struct ggml_context * ctx,
1035            struct ggml_tensor  * a);
1036
1037    GGML_API struct ggml_tensor * ggml_sin(
1038            struct ggml_context * ctx,
1039            struct ggml_tensor  * a);
1040
1041    GGML_API struct ggml_tensor * ggml_sin_inplace(
1042            struct ggml_context * ctx,
1043            struct ggml_tensor  * a);
1044
1045    GGML_API struct ggml_tensor * ggml_cos(
1046            struct ggml_context * ctx,
1047            struct ggml_tensor  * a);
1048
1049    GGML_API struct ggml_tensor * ggml_cos_inplace(
1050            struct ggml_context * ctx,
1051            struct ggml_tensor  * a);
1052
1053    // return scalar
1054    GGML_API struct ggml_tensor * ggml_sum(
1055            struct ggml_context * ctx,
1056            struct ggml_tensor  * a);
1057
1058    // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
1059    GGML_API struct ggml_tensor * ggml_sum_rows(
1060            struct ggml_context * ctx,
1061            struct ggml_tensor  * a);
1062
1063    GGML_API struct ggml_tensor * ggml_cumsum(
1064        struct ggml_context * ctx,
1065        struct ggml_tensor  * a);
1066
1067    // mean along rows
1068    GGML_API struct ggml_tensor * ggml_mean(
1069            struct ggml_context * ctx,
1070            struct ggml_tensor  * a);
1071
1072    // argmax along rows
1073    GGML_API struct ggml_tensor * ggml_argmax(
1074            struct ggml_context * ctx,
1075            struct ggml_tensor  * a);
1076
1077    // count number of equal elements in a and b
1078    GGML_API struct ggml_tensor * ggml_count_equal(
1079            struct ggml_context * ctx,
1080            struct ggml_tensor  * a,
1081            struct ggml_tensor  * b);
1082
1083    // if a is the same shape as b, and a is not parameter, return a
1084    // otherwise, return a new tensor: repeat(a) to fit in b
1085    GGML_API struct ggml_tensor * ggml_repeat(
1086            struct ggml_context * ctx,
1087            struct ggml_tensor  * a,
1088            struct ggml_tensor  * b);
1089
1090    // repeat a to the specified shape
1091    GGML_API struct ggml_tensor * ggml_repeat_4d(
1092            struct ggml_context * ctx,
1093            struct ggml_tensor  * a,
1094                       int64_t    ne0,
1095                       int64_t    ne1,
1096                       int64_t    ne2,
1097                       int64_t    ne3);
1098
1099    // sums repetitions in a into shape of b
1100    GGML_API struct ggml_tensor * ggml_repeat_back(
1101            struct ggml_context * ctx,
1102            struct ggml_tensor  * a,
1103            struct ggml_tensor  * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride
1104
1105    // concat a and b along dim
1106    // used in stable-diffusion
1107    GGML_API struct ggml_tensor * ggml_concat(
1108            struct ggml_context * ctx,
1109            struct ggml_tensor  * a,
1110            struct ggml_tensor  * b,
1111            int                   dim);
1112
1113    GGML_API struct ggml_tensor * ggml_abs(
1114            struct ggml_context * ctx,
1115            struct ggml_tensor  * a);
1116
1117    GGML_API struct ggml_tensor * ggml_abs_inplace(
1118            struct ggml_context * ctx,
1119            struct ggml_tensor  * a);
1120
1121    GGML_API struct ggml_tensor * ggml_sgn(
1122            struct ggml_context * ctx,
1123            struct ggml_tensor  * a);
1124
1125    GGML_API struct ggml_tensor * ggml_sgn_inplace(
1126            struct ggml_context * ctx,
1127            struct ggml_tensor  * a);
1128
1129    GGML_API struct ggml_tensor * ggml_neg(
1130            struct ggml_context * ctx,
1131            struct ggml_tensor  * a);
1132
1133    GGML_API struct ggml_tensor * ggml_neg_inplace(
1134            struct ggml_context * ctx,
1135            struct ggml_tensor  * a);
1136
1137    GGML_API struct ggml_tensor * ggml_step(
1138            struct ggml_context * ctx,
1139            struct ggml_tensor  * a);
1140
1141    GGML_API struct ggml_tensor * ggml_step_inplace(
1142            struct ggml_context * ctx,
1143            struct ggml_tensor  * a);
1144
1145    GGML_API struct ggml_tensor * ggml_tanh(
1146            struct ggml_context * ctx,
1147            struct ggml_tensor  * a);
1148
1149    GGML_API struct ggml_tensor * ggml_tanh_inplace(
1150            struct ggml_context * ctx,
1151            struct ggml_tensor  * a);
1152
1153    GGML_API struct ggml_tensor * ggml_elu(
1154            struct ggml_context * ctx,
1155            struct ggml_tensor  * a);
1156
1157    GGML_API struct ggml_tensor * ggml_elu_inplace(
1158            struct ggml_context * ctx,
1159            struct ggml_tensor  * a);
1160
1161    GGML_API struct ggml_tensor * ggml_relu(
1162            struct ggml_context * ctx,
1163            struct ggml_tensor  * a);
1164
1165    GGML_API struct ggml_tensor * ggml_leaky_relu(
1166            struct ggml_context * ctx,
1167            struct ggml_tensor  * a, float negative_slope, bool inplace);
1168
1169    GGML_API struct ggml_tensor * ggml_relu_inplace(
1170            struct ggml_context * ctx,
1171            struct ggml_tensor  * a);
1172
1173    GGML_API struct ggml_tensor * ggml_sigmoid(
1174            struct ggml_context * ctx,
1175            struct ggml_tensor  * a);
1176
1177    GGML_API struct ggml_tensor * ggml_sigmoid_inplace(
1178            struct ggml_context * ctx,
1179            struct ggml_tensor  * a);
1180
1181    GGML_API struct ggml_tensor * ggml_gelu(
1182            struct ggml_context * ctx,
1183            struct ggml_tensor  * a);
1184
1185    GGML_API struct ggml_tensor * ggml_gelu_inplace(
1186            struct ggml_context * ctx,
1187            struct ggml_tensor  * a);
1188
1189    // GELU using erf (error function) when possible
1190    // some backends may fallback to approximation based on Abramowitz and Stegun formula
1191    GGML_API struct ggml_tensor * ggml_gelu_erf(
1192            struct ggml_context * ctx,
1193            struct ggml_tensor  * a);
1194
1195    GGML_API struct ggml_tensor * ggml_gelu_erf_inplace(
1196            struct ggml_context * ctx,
1197            struct ggml_tensor  * a);
1198
1199    GGML_API struct ggml_tensor * ggml_gelu_quick(
1200            struct ggml_context * ctx,
1201            struct ggml_tensor  * a);
1202
1203    GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(
1204            struct ggml_context * ctx,
1205            struct ggml_tensor  * a);
1206
1207    GGML_API struct ggml_tensor * ggml_silu(
1208            struct ggml_context * ctx,
1209            struct ggml_tensor  * a);
1210
1211    GGML_API struct ggml_tensor * ggml_silu_inplace(
1212            struct ggml_context * ctx,
1213            struct ggml_tensor  * a);
1214
1215    // a - dy
1216    // b - x
1217    GGML_API struct ggml_tensor * ggml_silu_back(
1218            struct ggml_context * ctx,
1219            struct ggml_tensor  * a,
1220            struct ggml_tensor  * b);
1221
1222    // hardswish(x) = x * relu6(x + 3) / 6
1223    GGML_API struct ggml_tensor * ggml_hardswish(
1224            struct ggml_context * ctx,
1225            struct ggml_tensor  * a);
1226
1227    // hardsigmoid(x) = relu6(x + 3) / 6
1228    GGML_API struct ggml_tensor * ggml_hardsigmoid(
1229            struct ggml_context * ctx,
1230            struct ggml_tensor  * a);
1231
1232    GGML_API struct ggml_tensor * ggml_exp(
1233            struct ggml_context * ctx,
1234            struct ggml_tensor  * a);
1235
1236    GGML_API struct ggml_tensor * ggml_exp_inplace(
1237            struct ggml_context * ctx,
1238            struct ggml_tensor  * a);
1239
1240    GGML_API struct ggml_tensor * ggml_floor(
1241            struct ggml_context * ctx,
1242            struct ggml_tensor  * a);
1243
1244    GGML_API struct ggml_tensor * ggml_floor_inplace(
1245            struct ggml_context * ctx,
1246            struct ggml_tensor  * a);
1247
1248    GGML_API struct ggml_tensor * ggml_ceil(
1249            struct ggml_context * ctx,
1250            struct ggml_tensor  * a);
1251
1252    GGML_API struct ggml_tensor * ggml_ceil_inplace(
1253            struct ggml_context * ctx,
1254            struct ggml_tensor  * a);
1255
1256    GGML_API struct ggml_tensor * ggml_round(
1257            struct ggml_context * ctx,
1258            struct ggml_tensor  * a);
1259
1260    GGML_API struct ggml_tensor * ggml_round_inplace(
1261            struct ggml_context * ctx,
1262            struct ggml_tensor  * a);
1263
1264     /**
1265     * Truncates the fractional part of each element in the tensor (towards zero).
1266     * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0
1267     * Similar to std::trunc in C/C++.
1268     */
1269
1270    GGML_API struct ggml_tensor * ggml_trunc(
1271            struct ggml_context * ctx,
1272            struct ggml_tensor  * a);
1273
1274    GGML_API struct ggml_tensor * ggml_trunc_inplace(
1275            struct ggml_context * ctx,
1276            struct ggml_tensor  * a);
1277
1278
1279
1280    // xIELU activation function
1281    // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0)
1282    // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions
1283    // that constrain the positive and negative source alpha values respectively
1284    GGML_API struct ggml_tensor * ggml_xielu(
1285            struct ggml_context * ctx,
1286            struct ggml_tensor  * a,
1287            float alpha_n,
1288            float alpha_p,
1289            float beta,
1290            float eps);
1291
1292    // gated linear unit ops
1293    // A: n columns, r rows,
1294    // result is n / 2 columns, r rows,
1295    // expects gate in second half of row, unless swapped is true
1296    GGML_API struct ggml_tensor * ggml_glu(
1297            struct ggml_context * ctx,
1298             struct ggml_tensor * a,
1299             enum ggml_glu_op     op,
1300             bool                 swapped);
1301
1302    GGML_API struct ggml_tensor * ggml_reglu(
1303            struct ggml_context * ctx,
1304            struct ggml_tensor  * a);
1305
1306    GGML_API struct ggml_tensor * ggml_reglu_swapped(
1307            struct ggml_context * ctx,
1308            struct ggml_tensor  * a);
1309
1310    GGML_API struct ggml_tensor * ggml_geglu(
1311            struct ggml_context * ctx,
1312            struct ggml_tensor  * a);
1313
1314    GGML_API struct ggml_tensor * ggml_geglu_swapped(
1315            struct ggml_context * ctx,
1316            struct ggml_tensor  * a);
1317
1318    GGML_API struct ggml_tensor * ggml_swiglu(
1319            struct ggml_context * ctx,
1320            struct ggml_tensor  * a);
1321
1322    GGML_API struct ggml_tensor * ggml_swiglu_swapped(
1323            struct ggml_context * ctx,
1324            struct ggml_tensor  * a);
1325
1326    GGML_API struct ggml_tensor * ggml_geglu_erf(
1327            struct ggml_context * ctx,
1328            struct ggml_tensor  * a);
1329
1330    GGML_API struct ggml_tensor * ggml_geglu_erf_swapped(
1331            struct ggml_context * ctx,
1332            struct ggml_tensor  * a);
1333
1334    GGML_API struct ggml_tensor * ggml_geglu_quick(
1335            struct ggml_context * ctx,
1336            struct ggml_tensor  * a);
1337
1338    GGML_API struct ggml_tensor * ggml_geglu_quick_swapped(
1339            struct ggml_context * ctx,
1340            struct ggml_tensor  * a);
1341
1342    // A: n columns, r rows,
1343    // B: n columns, r rows,
1344    GGML_API struct ggml_tensor * ggml_glu_split(
1345            struct ggml_context * ctx,
1346             struct ggml_tensor * a,
1347             struct ggml_tensor * b,
1348             enum ggml_glu_op     op);
1349
1350    GGML_API struct ggml_tensor * ggml_reglu_split(
1351            struct ggml_context * ctx,
1352            struct ggml_tensor  * a,
1353            struct ggml_tensor  * b);
1354
1355    GGML_API struct ggml_tensor * ggml_geglu_split(
1356            struct ggml_context * ctx,
1357            struct ggml_tensor  * a,
1358            struct ggml_tensor  * b);
1359
1360    GGML_API struct ggml_tensor * ggml_swiglu_split(
1361            struct ggml_context * ctx,
1362            struct ggml_tensor  * a,
1363            struct ggml_tensor  * b);
1364
1365    GGML_API struct ggml_tensor * ggml_geglu_erf_split(
1366            struct ggml_context * ctx,
1367            struct ggml_tensor  * a,
1368            struct ggml_tensor  * b);
1369
1370    GGML_API struct ggml_tensor * ggml_geglu_quick_split(
1371            struct ggml_context * ctx,
1372            struct ggml_tensor  * a,
1373            struct ggml_tensor  * b);
1374
1375    GGML_API struct ggml_tensor * ggml_swiglu_oai(
1376            struct ggml_context * ctx,
1377            struct ggml_tensor  * a,
1378            struct ggml_tensor  * b,
1379            float                 alpha,
1380            float                 limit);
1381
1382    GGML_API struct ggml_tensor * ggml_swiglu_clamp(
1383            struct ggml_context * ctx,
1384            struct ggml_tensor  * a,
1385            struct ggml_tensor  * b,
1386            float                 limit);
1387
1388    // normalize along rows
1389    GGML_API struct ggml_tensor * ggml_norm(
1390            struct ggml_context * ctx,
1391            struct ggml_tensor  * a,
1392            float                 eps);
1393
1394    GGML_API struct ggml_tensor * ggml_norm_inplace(
1395            struct ggml_context * ctx,
1396            struct ggml_tensor  * a,
1397            float                 eps);
1398
1399    GGML_API struct ggml_tensor * ggml_rms_norm(
1400            struct ggml_context * ctx,
1401            struct ggml_tensor  * a,
1402            float                 eps);
1403
1404    GGML_API struct ggml_tensor * ggml_rms_norm_inplace(
1405            struct ggml_context * ctx,
1406            struct ggml_tensor  * a,
1407            float                 eps);
1408
1409    // group normalize along ne0*ne1*n_groups
1410    // used in stable-diffusion
1411    GGML_API struct ggml_tensor * ggml_group_norm(
1412            struct ggml_context * ctx,
1413            struct ggml_tensor  * a,
1414            int                   n_groups,
1415            float                 eps);
1416
1417    GGML_API struct ggml_tensor * ggml_group_norm_inplace(
1418            struct ggml_context * ctx,
1419            struct ggml_tensor  * a,
1420            int                   n_groups,
1421            float                 eps);
1422
1423    // l2 normalize along rows
1424    // used in rwkv v7
1425    GGML_API struct ggml_tensor * ggml_l2_norm(
1426            struct ggml_context * ctx,
1427            struct ggml_tensor  * a,
1428            float                 eps);
1429
1430    GGML_API struct ggml_tensor * ggml_l2_norm_inplace(
1431            struct ggml_context * ctx,
1432            struct ggml_tensor  * a,
1433            float                 eps);
1434
1435    // a - x
1436    // b - dy
1437    GGML_API struct ggml_tensor * ggml_rms_norm_back(
1438            struct ggml_context * ctx,
1439            struct ggml_tensor  * a,
1440            struct ggml_tensor  * b,
1441            float                 eps);
1442
1443    // [TAG_GGML_PREC]
1444    // set the minimum required accumulator type for the implementation to use during the compute
1445    // for example:
1446    //  - GGML_PREC_F32  - requires accumulation of the results in F32
1447    //  - GGML_PREC_BF16 - can accumulate the results in BF16, F32
1448    //  - GGML_PREC_F16  - can accumulate the results in F16, F32
1449    //  - GGML_PREC_Q8   - not allowed
1450    //  - GGML_PREC_Q4   - not allowed
1451    //
1452    // return false on faliure
1453    GGML_API bool ggml_prec_set_acc(
1454            struct ggml_tensor * a,
1455            enum ggml_prec       prec);
1456
1457    // [TAG_GGML_PREC]
1458    // set the smallest rank that the implementation can use to internally convert the src[idx] data to
1459    // ranks in decreasing order:
1460    //  - GGML_PREC_F32  - GGML_TYPE_F32
1461    //  - GGML_PREC_BF16 - GGML_TYPE_BF16
1462    //  - GGML_PREC_F16  - GGML_TYPE_F16,
1463    //  - GGML_PREC_Q8   - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc.
1464    //  - GGML_PREC_Q4   - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc.
1465    //
1466    // for example:
1467    //   - ggml_prec_set_src(a, GGML_PREC_Q8, 1):
1468    //     - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0
1469    //     - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4
1470    //   - ggml_prec_set_src(a, GGML_PREC_Q4, 1):
1471    //     - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc.
1472    //
1473    // return false on faliure
1474    GGML_API bool ggml_prec_set_src(
1475            struct ggml_tensor * a,
1476            enum ggml_prec       prec,
1477            int                  idx);
1478
1479    // A: k columns, n rows => [ne03, ne02, n, k]
1480    // B: k columns, m rows  (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
1481    // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
1482    GGML_API struct ggml_tensor * ggml_mul_mat(
1483            struct ggml_context * ctx,
1484            struct ggml_tensor  * a,
1485            struct ggml_tensor  * b);
1486
1487    // change the precision of a matrix multiplication
1488    // set to GGML_PREC_F32 for higher precision (useful for phi-2)
1489    GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec(
1490            struct ggml_tensor * a,
1491            enum ggml_prec       prec),
1492        "use ggml_prec_set_acc() instead");
1493
1494    // change the hint of a matrix multiplication
1495    GGML_API void ggml_mul_mat_set_hint(
1496            struct ggml_tensor * a,
1497            enum ggml_op_hint    hint);
1498
1499    // indirect matrix multiplication
1500    GGML_API struct ggml_tensor * ggml_mul_mat_id(
1501            struct ggml_context * ctx,
1502            struct ggml_tensor  * as,
1503            struct ggml_tensor  * b,
1504            struct ggml_tensor  * ids);
1505
1506    // A: m columns, n rows,
1507    // B: p columns, n rows,
1508    // result is m columns, p rows
1509    GGML_API struct ggml_tensor * ggml_out_prod(
1510            struct ggml_context * ctx,
1511            struct ggml_tensor  * a,
1512            struct ggml_tensor  * b);
1513
1514    //
1515    // operations on tensors without backpropagation
1516    //
1517
1518    GGML_API struct ggml_tensor * ggml_scale(
1519            struct ggml_context * ctx,
1520            struct ggml_tensor  * a,
1521            float                 s);
1522
1523    // in-place, returns view(a)
1524    GGML_API struct ggml_tensor * ggml_scale_inplace(
1525            struct ggml_context * ctx,
1526            struct ggml_tensor  * a,
1527            float                 s);
1528
1529    // x = s * a + b
1530    GGML_API struct ggml_tensor * ggml_scale_bias(
1531        struct ggml_context * ctx,
1532        struct ggml_tensor  * a,
1533        float                 s,
1534        float                 b);
1535
1536    GGML_API struct ggml_tensor * ggml_scale_bias_inplace(
1537        struct ggml_context * ctx,
1538        struct ggml_tensor  * a,
1539        float                 s,
1540        float                 b);
1541
1542    // b -> view(a,offset,nb1,nb2,3), return modified a
1543    GGML_API struct ggml_tensor * ggml_set(
1544            struct ggml_context * ctx,
1545            struct ggml_tensor  * a,
1546            struct ggml_tensor  * b,
1547            size_t                nb1,
1548            size_t                nb2,
1549            size_t                nb3,
1550            size_t                offset); // in bytes
1551
1552    // b -> view(a,offset,nb1,nb2,3), return view(a)
1553    GGML_API struct ggml_tensor * ggml_set_inplace(
1554            struct ggml_context * ctx,
1555            struct ggml_tensor  * a,
1556            struct ggml_tensor  * b,
1557            size_t                nb1,
1558            size_t                nb2,
1559            size_t                nb3,
1560            size_t                offset); // in bytes
1561
1562    GGML_API struct ggml_tensor * ggml_set_1d(
1563            struct ggml_context * ctx,
1564            struct ggml_tensor  * a,
1565            struct ggml_tensor  * b,
1566            size_t                offset); // in bytes
1567
1568    GGML_API struct ggml_tensor * ggml_set_1d_inplace(
1569            struct ggml_context * ctx,
1570            struct ggml_tensor  * a,
1571            struct ggml_tensor  * b,
1572            size_t                offset); // in bytes
1573
1574    // b -> view(a,offset,nb1,nb2,3), return modified a
1575    GGML_API struct ggml_tensor * ggml_set_2d(
1576            struct ggml_context * ctx,
1577            struct ggml_tensor  * a,
1578            struct ggml_tensor  * b,
1579            size_t                nb1,
1580            size_t                offset); // in bytes
1581
1582    // b -> view(a,offset,nb1,nb2,3), return view(a)
1583    GGML_API struct ggml_tensor * ggml_set_2d_inplace(
1584            struct ggml_context * ctx,
1585            struct ggml_tensor  * a,
1586            struct ggml_tensor  * b,
1587            size_t                nb1,
1588            size_t                offset); // in bytes
1589
1590    // a -> b, return view(b)
1591    GGML_API struct ggml_tensor * ggml_cpy(
1592            struct ggml_context * ctx,
1593            struct ggml_tensor  * a,
1594            struct ggml_tensor  * b);
1595
1596    // note: casting from f32 to i32 will discard the fractional part
1597    GGML_API struct ggml_tensor * ggml_cast(
1598            struct ggml_context * ctx,
1599            struct ggml_tensor  * a,
1600            enum   ggml_type      type);
1601
1602    // make contiguous
1603    GGML_API struct ggml_tensor * ggml_cont(
1604            struct ggml_context * ctx,
1605            struct ggml_tensor  * a);
1606
1607    // make contiguous, with new shape
1608    GGML_API struct ggml_tensor * ggml_cont_1d(
1609            struct ggml_context * ctx,
1610            struct ggml_tensor  * a,
1611            int64_t               ne0);
1612
1613    GGML_API struct ggml_tensor * ggml_cont_2d(
1614            struct ggml_context * ctx,
1615            struct ggml_tensor  * a,
1616            int64_t               ne0,
1617            int64_t               ne1);
1618
1619    GGML_API struct ggml_tensor * ggml_cont_3d(
1620            struct ggml_context * ctx,
1621            struct ggml_tensor  * a,
1622            int64_t               ne0,
1623            int64_t               ne1,
1624            int64_t               ne2);
1625
1626    GGML_API struct ggml_tensor * ggml_cont_4d(
1627            struct ggml_context * ctx,
1628            struct ggml_tensor  * a,
1629            int64_t               ne0,
1630            int64_t               ne1,
1631            int64_t               ne2,
1632            int64_t               ne3);
1633
1634    // return view(a), b specifies the new shape
1635    // TODO: when we start computing gradient, make a copy instead of view
1636    GGML_API struct ggml_tensor * ggml_reshape(
1637            struct ggml_context * ctx,
1638            struct ggml_tensor  * a,
1639            struct ggml_tensor  * b);
1640
1641    // return view(a)
1642    // TODO: when we start computing gradient, make a copy instead of view
1643    GGML_API struct ggml_tensor * ggml_reshape_1d(
1644            struct ggml_context * ctx,
1645            struct ggml_tensor  * a,
1646            int64_t               ne0);
1647
1648    GGML_API struct ggml_tensor * ggml_reshape_2d(
1649            struct ggml_context * ctx,
1650            struct ggml_tensor  * a,
1651            int64_t               ne0,
1652            int64_t               ne1);
1653
1654    // return view(a)
1655    // TODO: when we start computing gradient, make a copy instead of view
1656    GGML_API struct ggml_tensor * ggml_reshape_3d(
1657            struct ggml_context * ctx,
1658            struct ggml_tensor  * a,
1659            int64_t               ne0,
1660            int64_t               ne1,
1661            int64_t               ne2);
1662
1663    GGML_API struct ggml_tensor * ggml_reshape_4d(
1664            struct ggml_context * ctx,
1665            struct ggml_tensor  * a,
1666            int64_t               ne0,
1667            int64_t               ne1,
1668            int64_t               ne2,
1669            int64_t               ne3);
1670
1671    // offset in bytes
1672    GGML_API struct ggml_tensor * ggml_view_1d(
1673            struct ggml_context * ctx,
1674            struct ggml_tensor  * a,
1675            int64_t               ne0,
1676            size_t                offset);
1677
1678    GGML_API struct ggml_tensor * ggml_view_2d(
1679            struct ggml_context * ctx,
1680            struct ggml_tensor  * a,
1681            int64_t               ne0,
1682            int64_t               ne1,
1683            size_t                nb1, // row stride in bytes
1684            size_t                offset);
1685
1686    GGML_API struct ggml_tensor * ggml_view_3d(
1687            struct ggml_context * ctx,
1688            struct ggml_tensor  * a,
1689            int64_t               ne0,
1690            int64_t               ne1,
1691            int64_t               ne2,
1692            size_t                nb1, // row   stride in bytes
1693            size_t                nb2, // slice stride in bytes
1694            size_t                offset);
1695
1696    GGML_API struct ggml_tensor * ggml_view_4d(
1697            struct ggml_context * ctx,
1698            struct ggml_tensor  * a,
1699            int64_t               ne0,
1700            int64_t               ne1,
1701            int64_t               ne2,
1702            int64_t               ne3,
1703            size_t                nb1, // row   stride in bytes
1704            size_t                nb2, // slice stride in bytes
1705            size_t                nb3,
1706            size_t                offset);
1707
1708    GGML_API struct ggml_tensor * ggml_permute(
1709            struct ggml_context * ctx,
1710            struct ggml_tensor  * a,
1711            int                   axis0,
1712            int                   axis1,
1713            int                   axis2,
1714            int                   axis3);
1715
1716    // alias for ggml_permute(ctx, a, 1, 0, 2, 3)
1717    GGML_API struct ggml_tensor * ggml_transpose(
1718            struct ggml_context * ctx,
1719            struct ggml_tensor  * a);
1720
1721    // supports 4D a:
1722    // a     [n_embd, ne1, ne2, ne3]
1723    // b I32 [n_rows, ne2, ne3, 1]
1724    //
1725    // return [n_embd, n_rows, ne2, ne3]
1726    GGML_API struct ggml_tensor * ggml_get_rows(
1727            struct ggml_context * ctx,
1728            struct ggml_tensor  * a,  // data
1729            struct ggml_tensor  * b); // row indices
1730
1731    GGML_API struct ggml_tensor * ggml_get_rows_back(
1732            struct ggml_context * ctx,
1733            struct ggml_tensor  * a,  // gradients of ggml_get_rows result
1734            struct ggml_tensor  * b,  // row indices
1735            struct ggml_tensor  * c); // data for ggml_get_rows, only used for its shape
1736
1737    // a TD  [n_embd, ne1,    ne2,    ne3]
1738    // b TS  [n_embd, n_rows, ne02,   ne03] | ne02 == ne2, ne03 == ne3
1739    // c I64 [n_rows, ne11,   ne12,   1]    | c[i] in [0, ne1)
1740    //
1741    // undefined behavior if destination rows overlap
1742    //
1743    // broadcast:
1744    //   ne2 % ne11 == 0
1745    //   ne3 % ne12 == 0
1746    //
1747    // return view(a)
1748    GGML_API struct ggml_tensor * ggml_set_rows(
1749            struct ggml_context * ctx,
1750            struct ggml_tensor  * a,  // destination
1751            struct ggml_tensor  * b,  // source
1752            struct ggml_tensor  * c); // row indices
1753
1754    GGML_API struct ggml_tensor * ggml_diag(
1755        struct ggml_context     * ctx,
1756        struct ggml_tensor      * a);
1757
1758    // set elements above the diagonal to -INF
1759    GGML_API struct ggml_tensor * ggml_diag_mask_inf(
1760            struct ggml_context * ctx,
1761            struct ggml_tensor  * a,
1762            int                   n_past);
1763
1764    // in-place, returns view(a)
1765    GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(
1766            struct ggml_context * ctx,
1767            struct ggml_tensor  * a,
1768            int                   n_past);
1769
1770    // set elements above the diagonal to 0
1771    GGML_API struct ggml_tensor * ggml_diag_mask_zero(
1772            struct ggml_context * ctx,
1773            struct ggml_tensor  * a,
1774            int                   n_past);
1775
1776    // in-place, returns view(a)
1777    GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(
1778            struct ggml_context * ctx,
1779            struct ggml_tensor  * a,
1780            int                   n_past);
1781
1782    GGML_API struct ggml_tensor * ggml_clamp(
1783            struct ggml_context * ctx,
1784            struct ggml_tensor  * a,
1785            float                 min,
1786            float                 max);
1787
1788    // in-place, returns view(a)
1789    GGML_API struct ggml_tensor * ggml_clamp_inplace(
1790            struct ggml_context * ctx,
1791            struct ggml_tensor  * a,
1792            float                 min,
1793            float                 max);
1794
1795    GGML_API struct ggml_tensor * ggml_soft_max(
1796            struct ggml_context * ctx,
1797            struct ggml_tensor  * a);
1798
1799    // in-place, returns view(a)
1800    GGML_API struct ggml_tensor * ggml_soft_max_inplace(
1801            struct ggml_context * ctx,
1802            struct ggml_tensor  * a);
1803
1804    // a    [ne0, ne01, ne02, ne03]
1805    // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional
1806    //
1807    // broadcast:
1808    //   ne02 % ne12 == 0
1809    //   ne03 % ne13 == 0
1810    //
1811    // fused soft_max(a*scale + mask*(ALiBi slope))
1812    // max_bias = 0.0f for no ALiBi
1813    GGML_API struct ggml_tensor * ggml_soft_max_ext(
1814            struct ggml_context * ctx,
1815            struct ggml_tensor  * a,
1816            struct ggml_tensor  * mask,
1817            float                 scale,
1818            float                 max_bias);
1819
1820    GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace(
1821            struct ggml_context * ctx,
1822            struct ggml_tensor  * a,
1823            struct ggml_tensor  * mask,
1824            float                 scale,
1825            float                 max_bias);
1826
1827    GGML_API void ggml_soft_max_add_sinks(
1828            struct ggml_tensor * a,
1829            struct ggml_tensor * sinks);
1830
1831    GGML_API struct ggml_tensor * ggml_soft_max_ext_back(
1832            struct ggml_context * ctx,
1833            struct ggml_tensor  * a,
1834            struct ggml_tensor  * b,
1835            float                 scale,
1836            float                 max_bias);
1837
1838    // in-place, returns view(a)
1839    GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace(
1840            struct ggml_context * ctx,
1841            struct ggml_tensor  * a,
1842            struct ggml_tensor  * b,
1843            float                 scale,
1844            float                 max_bias);
1845
1846    // rotary position embedding
1847    // if (mode & 1) - skip n_past elements (NOT SUPPORTED)
1848    // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style
1849    //
1850    // b is an int32 vector with size a->ne[2], it contains the positions
1851    GGML_API struct ggml_tensor * ggml_rope(
1852            struct ggml_context * ctx,
1853            struct ggml_tensor  * a,
1854            struct ggml_tensor  * b,
1855            int                   n_dims,
1856            int                   mode);
1857
1858    // in-place, returns view(a)
1859    GGML_API struct ggml_tensor * ggml_rope_inplace(
1860            struct ggml_context * ctx,
1861            struct ggml_tensor  * a,
1862            struct ggml_tensor  * b,
1863            int                   n_dims,
1864            int                   mode);
1865
1866    // RoPE operations with extended options
1867    // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token]
1868    // b is an int32 vector with size n_token
1869    // c is freq factors (e.g. phi3-128k), (optional)
1870    // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi
1871    //
1872    // pseudo-code for computing theta:
1873    //   for i in [0, n_dims/2):
1874    //     theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims);
1875    //     theta[i] = theta[i] / c[i];  # if c is provided, divide theta by c
1876    //     theta[i] = rope_yarn(theta[i], ...);  # note: theta = theta * freq_scale is applied here
1877    //
1878    // other params are used by YaRN RoPE scaling, these default values will disable YaRN:
1879    //   freq_scale  = 1.0f
1880    //   ext_factor  = 0.0f
1881    //   attn_factor = 1.0f
1882    //   beta_fast   = 0.0f
1883    //   beta_slow   = 0.0f
1884    //
1885    // example:
1886    //   (marking: c = cos, s = sin, 0 = unrotated)
1887    //   given a single head with size = 8 --> [00000000]
1888    //   GGML_ROPE_TYPE_NORMAL  n_dims = 4 --> [cscs0000]
1889    //   GGML_ROPE_TYPE_NORMAL  n_dims = 8 --> [cscscscs]
1890    //   GGML_ROPE_TYPE_NEOX    n_dims = 4 --> [ccss0000]
1891    //   GGML_ROPE_TYPE_NEOX    n_dims = 8 --> [ccccssss]
1892    GGML_API struct ggml_tensor * ggml_rope_ext(
1893            struct ggml_context * ctx,
1894            struct ggml_tensor  * a,
1895            struct ggml_tensor  * b,
1896            struct ggml_tensor  * c,
1897            int                   n_dims,
1898            int                   mode,
1899            int                   n_ctx_orig,
1900            float                 freq_base,
1901            float                 freq_scale,
1902            float                 ext_factor,
1903            float                 attn_factor,
1904            float                 beta_fast,
1905            float                 beta_slow);
1906
1907    // multi-dimensional RoPE, for Qwen-VL and similar vision models
1908    // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX
1909    // sections specify how many dimensions to rotate in each section:
1910    //   section length is equivalent to number of cos/sin pairs, NOT the number of dims
1911    //   (i.e. sum of 4 sections are expected to be n_dims/2)
1912    //   last sections can be 0, means ignored
1913    // all other options are identical to ggml_rope_ext
1914    //
1915    // important note:
1916    //   - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION
1917    //     if you need normal ordering, there are 2 methods:
1918    //     (1) split the tensor manually using ggml_view
1919    //     (2) permute the weight upon conversion
1920    //   - for VISION, n_dims must be head_size/2
1921    //
1922    // example M-RoPE:
1923    //  given sections = [t=4, y=2, x=2, 0]
1924    //  given a single head with size = 18 --> [000000000000000000]
1925    //  GGML_ROPE_TYPE_MROPE   n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering)
1926    //  GGML_ROPE_TYPE_IMROPE  n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering)
1927    //  note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section
1928    //        in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section
1929    //
1930    // example vision RoPE:
1931    //  given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored)
1932    //  given a single head with size = 8 --> [00000000]
1933    //  GGML_ROPE_TYPE_VISION  n_dims = 4 --> [yyyyxxxx]
1934    //  other values of n_dims are untested and is undefined behavior
1935    //  note: unlike MROPE, the theta for each dim is computed differently for each section
1936    //        in other words, idx used for theta: [0123] for y section, then [0123] for x section
1937    GGML_API struct ggml_tensor * ggml_rope_multi(
1938            struct ggml_context * ctx,
1939            struct ggml_tensor  * a,
1940            struct ggml_tensor  * b,
1941            struct ggml_tensor  * c,
1942            int                   n_dims,
1943            int                   sections[GGML_MROPE_SECTIONS],
1944            int                   mode,
1945            int                   n_ctx_orig,
1946            float                 freq_base,
1947            float                 freq_scale,
1948            float                 ext_factor,
1949            float                 attn_factor,
1950            float                 beta_fast,
1951            float                 beta_slow);
1952
1953    // in-place, returns view(a)
1954    GGML_API struct ggml_tensor * ggml_rope_ext_inplace(
1955            struct ggml_context * ctx,
1956            struct ggml_tensor  * a,
1957            struct ggml_tensor  * b,
1958            struct ggml_tensor  * c,
1959            int                   n_dims,
1960            int                   mode,
1961            int                   n_ctx_orig,
1962            float                 freq_base,
1963            float                 freq_scale,
1964            float                 ext_factor,
1965            float                 attn_factor,
1966            float                 beta_fast,
1967            float                 beta_slow);
1968
1969    GGML_API struct ggml_tensor * ggml_rope_multi_inplace(
1970            struct ggml_context * ctx,
1971            struct ggml_tensor  * a,
1972            struct ggml_tensor  * b,
1973            struct ggml_tensor  * c,
1974            int                   n_dims,
1975            int                   sections[GGML_MROPE_SECTIONS],
1976            int                   mode,
1977            int                   n_ctx_orig,
1978            float                 freq_base,
1979            float                 freq_scale,
1980            float                 ext_factor,
1981            float                 attn_factor,
1982            float                 beta_fast,
1983            float                 beta_slow);
1984
1985    GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(
1986            struct ggml_context * ctx,
1987            struct ggml_tensor  * a,
1988            struct ggml_tensor  * b,
1989            int                   n_dims,
1990            int                   mode,
1991            int                   n_ctx_orig,
1992            float                 freq_base,
1993            float                 freq_scale,
1994            float                 ext_factor,
1995            float                 attn_factor,
1996            float                 beta_fast,
1997            float                 beta_slow),
1998        "use ggml_rope_ext instead");
1999
2000    GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(
2001            struct ggml_context * ctx,
2002            struct ggml_tensor  * a,
2003            struct ggml_tensor  * b,
2004            int                   n_dims,
2005            int                   mode,
2006            int                   n_ctx_orig,
2007            float                 freq_base,
2008            float                 freq_scale,
2009            float                 ext_factor,
2010            float                 attn_factor,
2011            float                 beta_fast,
2012            float                 beta_slow),
2013        "use ggml_rope_ext_inplace instead");
2014
2015    // compute correction dims for YaRN RoPE scaling
2016    GGML_API void ggml_rope_yarn_corr_dims(
2017        int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
2018
2019    // rotary position embedding backward, i.e compute dx from dy
2020    // a - dy
2021    GGML_API struct ggml_tensor * ggml_rope_ext_back(
2022            struct ggml_context * ctx,
2023            struct ggml_tensor  * a, // gradients of ggml_rope result
2024            struct ggml_tensor  * b, // positions
2025            struct ggml_tensor  * c, // freq factors
2026            int                   n_dims,
2027            int                   mode,
2028            int                   n_ctx_orig,
2029            float                 freq_base,
2030            float                 freq_scale,
2031            float                 ext_factor,
2032            float                 attn_factor,
2033            float                 beta_fast,
2034            float                 beta_slow);
2035
2036    GGML_API struct ggml_tensor * ggml_rope_multi_back(
2037            struct ggml_context * ctx,
2038            struct ggml_tensor  * a,
2039            struct ggml_tensor  * b,
2040            struct ggml_tensor  * c,
2041            int                   n_dims,
2042            int                   sections[4],
2043            int                   mode,
2044            int                   n_ctx_orig,
2045            float                 freq_base,
2046            float                 freq_scale,
2047            float                 ext_factor,
2048            float                 attn_factor,
2049            float                 beta_fast,
2050            float                 beta_slow);
2051
2052    // set the offset dims for RoPE
2053    // a must be GGML_OP_ROPE or GGML_OP_ROPE_BACK
2054    // vision RoPE is not supported
2055    // example: (marking: x = rotated, 0 = unrotated)
2056    //     n_embd = 10, n_dims = 4, offset = 2 --> [00xxxx0000]
2057    GGML_API struct ggml_tensor * ggml_rope_set_offset(
2058            struct ggml_tensor  * a,
2059            int                   n_offs);
2060
2061    // im2col
2062    // converts data into a format that effectively results in a convolution when combined with matrix multiplication
2063    GGML_API struct ggml_tensor * ggml_im2col(
2064            struct ggml_context * ctx,
2065            struct ggml_tensor  * a,  // convolution kernel
2066            struct ggml_tensor  * b,  // data
2067            int                   s0, // stride dimension 0
2068            int                   s1, // stride dimension 1
2069            int                   p0, // padding dimension 0
2070            int                   p1, // padding dimension 1
2071            int                   d0, // dilation dimension 0
2072            int                   d1, // dilation dimension 1
2073            bool                  is_2D,
2074            enum ggml_type        dst_type);
2075
2076    GGML_API struct ggml_tensor * ggml_im2col_back(
2077        struct ggml_context * ctx,
2078        struct ggml_tensor  * a,  // convolution kernel
2079        struct ggml_tensor  * b,  // gradient of im2col output
2080        int64_t             * ne, // shape of im2col input
2081        int                   s0, // stride dimension 0
2082        int                   s1, // stride dimension 1
2083        int                   p0, // padding dimension 0
2084        int                   p1, // padding dimension 1
2085        int                   d0, // dilation dimension 0
2086        int                   d1, // dilation dimension 1
2087        bool                  is_2D);
2088
2089    // col2im_1d: scatter-add GEMM columns back to 1D signal
2090    // a: [K*OC, T_in]  (columns from matmul, K = a->ne[0]/OC)
2091    // result: [T_out, OC]  where T_out = (T_in - 1)*s0 + K - 2*p0
2092    GGML_API struct ggml_tensor * ggml_col2im_1d(
2093        struct ggml_context * ctx,
2094        struct ggml_tensor  * a,   // columns [K*OC, T_in]
2095        int                   s0,  // stride
2096        int                   oc,  // output channels
2097        int                   p0); // padding to crop from both sides
2098
2099    GGML_API struct ggml_tensor * ggml_conv_1d(
2100            struct ggml_context * ctx,
2101            struct ggml_tensor  * a,   // convolution kernel
2102            struct ggml_tensor  * b,   // data
2103            int                   s0,  // stride
2104            int                   p0,  // padding
2105            int                   d0); // dilation
2106
2107    // conv_1d with padding = half
2108    // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
2109    GGML_API struct ggml_tensor* ggml_conv_1d_ph(
2110            struct ggml_context * ctx,
2111            struct ggml_tensor  * a,  // convolution kernel
2112            struct ggml_tensor  * b,  // data
2113            int                   s,  // stride
2114            int                   d); // dilation
2115
2116    // depthwise
2117    // TODO: this is very likely wrong for some cases! - needs more testing
2118    GGML_API struct ggml_tensor * ggml_conv_1d_dw(
2119            struct ggml_context * ctx,
2120            struct ggml_tensor  * a,   // convolution kernel
2121            struct ggml_tensor  * b,   // data
2122            int                   s0,  // stride
2123            int                   p0,  // padding
2124            int                   d0); // dilation
2125
2126    GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph(
2127            struct ggml_context * ctx,
2128            struct ggml_tensor  * a,   // convolution kernel
2129            struct ggml_tensor  * b,   // data
2130            int                   s0,  // stride
2131            int                   d0); // dilation
2132
2133    GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
2134            struct ggml_context * ctx,
2135            struct ggml_tensor  * a,   // convolution kernel
2136            struct ggml_tensor  * b,   // data
2137            int                   s0,  // stride
2138            int                   p0,  // padding
2139            int                   d0); // dilation
2140
2141    GGML_API struct ggml_tensor * ggml_conv_2d(
2142            struct ggml_context * ctx,
2143            struct ggml_tensor  * a,   // convolution kernel
2144            struct ggml_tensor  * b,   // data
2145            int                   s0,  // stride dimension 0
2146            int                   s1,  // stride dimension 1
2147            int                   p0,  // padding dimension 0
2148            int                   p1,  // padding dimension 1
2149            int                   d0,  // dilation dimension 0
2150            int                   d1); // dilation dimension 1
2151
2152    GGML_API struct ggml_tensor * ggml_im2col_3d(
2153            struct ggml_context * ctx,
2154            struct ggml_tensor  * a,
2155            struct ggml_tensor  * b,
2156            int64_t               IC,
2157            int                   s0, // stride width
2158            int                   s1, // stride height
2159            int                   s2, // stride depth
2160            int                   p0, // padding width
2161            int                   p1, // padding height
2162            int                   p2, // padding depth
2163            int                   d0, // dilation width
2164            int                   d1, // dilation height
2165            int                   d2, // dilation depth
2166            enum ggml_type        dst_type);
2167
2168    // a: [OC*IC, KD, KH, KW]
2169    // b: [N*IC, ID, IH, IW]
2170    // result: [N*OC, OD, OH, OW]
2171    GGML_API struct ggml_tensor * ggml_conv_3d(
2172                struct ggml_context * ctx,
2173                struct ggml_tensor  * a,
2174                struct ggml_tensor  * b,
2175                int64_t               IC,
2176                int                   s0, // stride width
2177                int                   s1, // stride height
2178                int                   s2, // stride depth
2179                int                   p0, // padding width
2180                int                   p1, // padding height
2181                int                   p2, // padding depth
2182                int                   d0, // dilation width
2183                int                   d1, // dilation height
2184                int                   d2  // dilation depth
2185        );
2186
2187    // kernel size is a->ne[0] x a->ne[1]
2188    // stride is equal to kernel size
2189    // padding is zero
2190    // example:
2191    // a:     16   16    3  768
2192    // b:   1024 1024    3    1
2193    // res:   64   64  768    1
2194    // used in sam
2195    GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(
2196            struct ggml_context * ctx,
2197            struct ggml_tensor  * a,
2198            struct ggml_tensor  * b);
2199
2200    // kernel size is a->ne[0] x a->ne[1]
2201    // stride is 1
2202    // padding is half
2203    // example:
2204    // a:      3    3    256  256
2205    // b:     64   64    256    1
2206    // res:   64   64    256    1
2207    // used in sam
2208    GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(
2209            struct ggml_context * ctx,
2210            struct ggml_tensor  * a,
2211            struct ggml_tensor  * b);
2212
2213    // depthwise (via im2col and mul_mat)
2214    GGML_API struct ggml_tensor * ggml_conv_2d_dw(
2215            struct ggml_context * ctx,
2216            struct ggml_tensor  * a,  // convolution kernel
2217            struct ggml_tensor  * b,  // data
2218            int                  s0,  // stride dimension 0
2219            int                  s1,  // stride dimension 1
2220            int                  p0,  // padding dimension 0
2221            int                  p1,  // padding dimension 1
2222            int                  d0,  // dilation dimension 0
2223            int                  d1); // dilation dimension 1
2224
2225    // Depthwise 2D convolution
2226    // may be faster than ggml_conv_2d_dw, but not available in all backends
2227    // a:   KW    KH    1    C    convolution kernel
2228    // b:   W     H     C    N    input data
2229    // res: W_out H_out C    N
2230    GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct(
2231            struct ggml_context * ctx,
2232            struct ggml_tensor  * a,
2233            struct ggml_tensor  * b,
2234            int                   stride0,
2235            int                   stride1,
2236            int                   pad0,
2237            int                   pad1,
2238            int                   dilation0,
2239            int                   dilation1);
2240
2241    GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(
2242            struct ggml_context * ctx,
2243            struct ggml_tensor  * a,
2244            struct ggml_tensor  * b,
2245            int                   stride);
2246
2247    GGML_API struct ggml_tensor * ggml_conv_2d_direct(
2248            struct ggml_context * ctx,
2249            struct ggml_tensor  * a,   // convolution kernel [KW, KH, IC, OC]
2250            struct ggml_tensor  * b,   // input data [W, H, C, N]
2251            int                   s0,  // stride dimension 0
2252            int                   s1,  // stride dimension 1
2253            int                   p0,  // padding dimension 0
2254            int                   p1,  // padding dimension 1
2255            int                   d0,  // dilation dimension 0
2256            int                   d1); // dilation dimension 1
2257
2258    GGML_API struct ggml_tensor * ggml_conv_3d_direct(
2259            struct ggml_context * ctx,
2260            struct ggml_tensor  * a,   // kernel [KW, KH, KD, IC * OC]
2261            struct ggml_tensor  * b,   // input  [W, H, D, C * N]
2262            int                   s0,  // stride
2263            int                   s1,
2264            int                   s2,
2265            int                   p0,  // padding
2266            int                   p1,
2267            int                   p2,
2268            int                   d0,  // dilation
2269            int                   d1,
2270            int                   d2,
2271            int                   n_channels,
2272            int                   n_batch,
2273            int                   n_channels_out);
2274
2275    enum ggml_op_pool {
2276        GGML_OP_POOL_MAX,
2277        GGML_OP_POOL_AVG,
2278        GGML_OP_POOL_COUNT,
2279    };
2280
2281    GGML_API struct ggml_tensor * ggml_pool_1d(
2282            struct ggml_context * ctx,
2283            struct ggml_tensor  * a,
2284            enum ggml_op_pool     op,
2285            int                   k0, // kernel size
2286            int                   s0, // stride
2287            int                   p0); // padding
2288
2289    // the result will have 2*p0 padding for the first dimension
2290    // and 2*p1 padding for the second dimension
2291    GGML_API struct ggml_tensor * ggml_pool_2d(
2292            struct ggml_context * ctx,
2293            struct ggml_tensor  * a,
2294            enum ggml_op_pool     op,
2295            int                   k0,
2296            int                   k1,
2297            int                   s0,
2298            int                   s1,
2299            float                 p0,
2300            float                 p1);
2301
2302    GGML_API struct ggml_tensor * ggml_pool_2d_back(
2303            struct ggml_context * ctx,
2304            struct ggml_tensor  * a,
2305            struct ggml_tensor  * af, // "a"/input used in forward pass
2306            enum ggml_op_pool     op,
2307            int                   k0,
2308            int                   k1,
2309            int                   s0,
2310            int                   s1,
2311            float                 p0,
2312            float                 p1);
2313
2314    enum ggml_scale_mode {
2315        GGML_SCALE_MODE_NEAREST  = 0,
2316        GGML_SCALE_MODE_BILINEAR = 1,
2317        GGML_SCALE_MODE_BICUBIC  = 2,
2318
2319        GGML_SCALE_MODE_COUNT
2320    };
2321
2322    enum ggml_scale_flag {
2323        GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8),
2324        GGML_SCALE_FLAG_ANTIALIAS     = (1 << 9),
2325    };
2326
2327    // interpolate
2328    // multiplies ne0 and ne1 by scale factor
2329    GGML_API struct ggml_tensor * ggml_upscale(
2330            struct ggml_context * ctx,
2331            struct ggml_tensor  * a,
2332            int                   scale_factor,
2333            enum ggml_scale_mode  mode);
2334
2335    // interpolate
2336    // interpolate scale to specified dimensions
2337    GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext(
2338            struct ggml_context * ctx,
2339            struct ggml_tensor  * a,
2340            int                   ne0,
2341            int                   ne1,
2342            int                   ne2,
2343            int                   ne3,
2344            enum ggml_scale_mode  mode),
2345        "use ggml_interpolate instead");
2346
2347    // Up- or downsamples the input to the specified size.
2348    // 2D scale modes (eg. bilinear) are applied to the first two dimensions.
2349    GGML_API struct ggml_tensor * ggml_interpolate(
2350            struct ggml_context * ctx,
2351            struct ggml_tensor  * a,
2352            int64_t               ne0,
2353            int64_t               ne1,
2354            int64_t               ne2,
2355            int64_t               ne3,
2356            uint32_t              mode); // ggml_scale_mode [ | ggml_scale_flag...]
2357
2358    // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
2359    GGML_API struct ggml_tensor * ggml_pad(
2360            struct ggml_context * ctx,
2361            struct ggml_tensor  * a,
2362            int                  p0,
2363            int                  p1,
2364            int                  p2,
2365            int                  p3);
2366
2367    // pad each dimension with values on the other side of the torus (looping around)
2368    GGML_API struct ggml_tensor * ggml_pad_circular(
2369            struct ggml_context * ctx,
2370            struct ggml_tensor  * a,
2371            int                   p0,
2372            int                   p1,
2373            int                   p2,
2374            int                   p3);
2375
2376    GGML_API struct ggml_tensor * ggml_pad_ext(
2377            struct ggml_context * ctx,
2378            struct ggml_tensor  * a,
2379            int                  lp0,
2380            int                  rp0,
2381            int                  lp1,
2382            int                  rp1,
2383            int                  lp2,
2384            int                  rp2,
2385            int                  lp3,
2386            int                  rp3
2387            );
2388
2389    // pad each dimension with values on the other side of the torus (looping around)
2390    GGML_API struct ggml_tensor * ggml_pad_ext_circular(
2391            struct ggml_context * ctx,
2392            struct ggml_tensor  * a,
2393            int                   lp0,
2394            int                   rp0,
2395            int                   lp1,
2396            int                   rp1,
2397            int                   lp2,
2398            int                   rp2,
2399            int                   lp3,
2400            int                   rp3);
2401
2402    // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c]
2403    GGML_API struct ggml_tensor * ggml_pad_reflect_1d(
2404            struct ggml_context * ctx,
2405            struct ggml_tensor  * a,
2406            int                   p0,
2407            int                   p1);
2408
2409    // Move tensor elements by an offset given for each dimension. Elements that
2410    // are shifted beyond the last position are wrapped around to the beginning.
2411    GGML_API struct ggml_tensor * ggml_roll(
2412            struct ggml_context * ctx,
2413            struct ggml_tensor  * a,
2414            int                   shift0,
2415            int                   shift1,
2416            int                   shift2,
2417            int                   shift3);
2418
2419    // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing
2420    // zeroes everywhere outside the masked area
2421    GGML_API struct ggml_tensor * ggml_tri(
2422            struct ggml_context * ctx,
2423            struct ggml_tensor  * a,
2424            enum ggml_tri_type    type);
2425
2426    // Fill tensor a with constant c
2427    GGML_API struct ggml_tensor * ggml_fill(
2428            struct ggml_context * ctx,
2429            struct ggml_tensor  * a,
2430            float                 c);
2431
2432    GGML_API struct ggml_tensor * ggml_fill_inplace(
2433            struct ggml_context * ctx,
2434            struct ggml_tensor  * a,
2435            float                 c);
2436
2437    // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
2438    // timesteps: [N,]
2439    // return: [N, dim]
2440    GGML_API struct ggml_tensor * ggml_timestep_embedding(
2441            struct ggml_context * ctx,
2442            struct ggml_tensor  * timesteps,
2443            int                   dim,
2444            int                   max_period);
2445
2446    // sort rows
2447    enum ggml_sort_order {
2448        GGML_SORT_ORDER_ASC,
2449        GGML_SORT_ORDER_DESC,
2450    };
2451
2452    GGML_API struct ggml_tensor * ggml_argsort(
2453            struct ggml_context * ctx,
2454            struct ggml_tensor  * a,
2455            enum ggml_sort_order  order);
2456
2457    // similar to ggml_top_k but implemented as `argsort` + `view`
2458    GGML_API struct ggml_tensor * ggml_argsort_top_k(
2459            struct ggml_context * ctx,
2460            struct ggml_tensor  * a,
2461            int                   k);
2462
2463    // top k elements per row
2464    // note: the resulting top k indices are in no particular order
2465    GGML_API struct ggml_tensor * ggml_top_k(
2466            struct ggml_context * ctx,
2467            struct ggml_tensor  * a,
2468            int                   k);
2469
2470    GGML_API struct ggml_tensor * ggml_arange(
2471            struct ggml_context * ctx,
2472            float                 start,
2473            float                 stop,
2474            float                 step);
2475
2476    // q:    [n_embd_k, n_batch, n_head,    ne3 ]
2477    // k:    [n_embd_k, n_kv,    n_head_kv, ne3 ]
2478    // v:    [n_embd_v, n_kv,    n_head_kv, ne3 ] !! not transposed !!
2479    // mask: [n_kv,     n_batch, ne32,      ne33]
2480    // res:  [n_embd_v, n_head,  n_batch,   ne3 ] !! permuted !!
2481    //
2482    // broadcast:
2483    //   n_head % n_head_kv == 0
2484    //   n_head % ne32      == 0
2485    //   ne3    % ne33      == 0
2486    //
2487    GGML_API struct ggml_tensor * ggml_flash_attn_ext(
2488            struct ggml_context * ctx,
2489            struct ggml_tensor  * q,
2490            struct ggml_tensor  * k,
2491            struct ggml_tensor  * v,
2492            struct ggml_tensor  * mask,
2493            float                 scale,
2494            float                 max_bias,
2495            float                 logit_softcap);
2496
2497    GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(
2498            struct ggml_tensor * a,
2499            enum ggml_prec       prec),
2500        "use ggml_prec_set_acc() instead");
2501
2502    GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(
2503            const struct ggml_tensor * a);
2504
2505    // Use finite mask entries as a sparse K/V set. Set 0 to disable.
2506    // n_kv_max must bound the number of finite entries in every mask row.
2507    GGML_API void ggml_flash_attn_ext_set_n_kv_max(
2508            struct ggml_tensor * a,
2509            int32_t              n_kv_max);
2510
2511    GGML_API void ggml_flash_attn_ext_add_sinks(
2512            struct ggml_tensor * a,
2513            struct ggml_tensor * sinks);
2514
2515    // TODO: needs to be adapted to ggml_flash_attn_ext
2516    GGML_API struct ggml_tensor * ggml_flash_attn_back(
2517           struct ggml_context * ctx,
2518           struct ggml_tensor  * q,
2519           struct ggml_tensor  * k,
2520           struct ggml_tensor  * v,
2521           struct ggml_tensor  * d,
2522           bool                  masked);
2523
2524    GGML_API struct ggml_tensor * ggml_ssm_conv(
2525            struct ggml_context * ctx,
2526            struct ggml_tensor  * sx,
2527            struct ggml_tensor  * c);
2528
2529    GGML_API struct ggml_tensor * ggml_ssm_scan(
2530            struct ggml_context * ctx,
2531            struct ggml_tensor  * s,
2532            struct ggml_tensor  * x,
2533            struct ggml_tensor  * dt,
2534            struct ggml_tensor  * A,
2535            struct ggml_tensor  * B,
2536            struct ggml_tensor  * C,
2537            struct ggml_tensor  * ids,
2538            int64_t               K);
2539
2540    // partition into non-overlapping windows with padding if needed
2541    // example:
2542    // a:   768   64   64    1
2543    // w:    14
2544    // res: 768   14   14    25
2545    // used in sam
2546    GGML_API struct ggml_tensor * ggml_win_part(
2547            struct ggml_context * ctx,
2548            struct ggml_tensor  * a,
2549            int                   w);
2550
2551    // reverse of ggml_win_part
2552    // used in sam
2553    GGML_API struct ggml_tensor * ggml_win_unpart(
2554            struct ggml_context * ctx,
2555            struct ggml_tensor  * a,
2556            int                   w0,
2557            int                   h0,
2558            int                   w);
2559
2560    GGML_API struct ggml_tensor * ggml_unary(
2561            struct ggml_context * ctx,
2562             struct ggml_tensor * a,
2563             enum ggml_unary_op op);
2564
2565    GGML_API struct ggml_tensor * ggml_unary_inplace(
2566        struct ggml_context * ctx,
2567        struct ggml_tensor  * a,
2568        enum ggml_unary_op op);
2569
2570    // used in sam
2571    GGML_API struct ggml_tensor * ggml_get_rel_pos(
2572            struct ggml_context * ctx,
2573            struct ggml_tensor  * a,
2574            int                   qh,
2575            int                   kh);
2576
2577    // used in sam
2578    GGML_API struct ggml_tensor * ggml_add_rel_pos(
2579            struct ggml_context * ctx,
2580            struct ggml_tensor  * a,
2581            struct ggml_tensor  * pw,
2582            struct ggml_tensor  * ph);
2583
2584    GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(
2585            struct ggml_context * ctx,
2586            struct ggml_tensor  * a,
2587            struct ggml_tensor  * pw,
2588            struct ggml_tensor  * ph);
2589
2590    GGML_API struct ggml_tensor * ggml_rwkv_wkv6(
2591            struct ggml_context * ctx,
2592            struct ggml_tensor  * k,
2593            struct ggml_tensor  * v,
2594            struct ggml_tensor  * r,
2595            struct ggml_tensor  * tf,
2596            struct ggml_tensor  * td,
2597            struct ggml_tensor  * state);
2598
2599    GGML_API struct ggml_tensor * ggml_gated_linear_attn(
2600            struct ggml_context * ctx,
2601            struct ggml_tensor  * k,
2602            struct ggml_tensor  * v,
2603            struct ggml_tensor  * q,
2604            struct ggml_tensor  * g,
2605            struct ggml_tensor  * state,
2606            float scale);
2607
2608    GGML_API struct ggml_tensor * ggml_rwkv_wkv7(
2609            struct ggml_context * ctx,
2610            struct ggml_tensor  * r,
2611            struct ggml_tensor  * w,
2612            struct ggml_tensor  * k,
2613            struct ggml_tensor  * v,
2614            struct ggml_tensor  * a,
2615            struct ggml_tensor  * b,
2616            struct ggml_tensor  * state);
2617
2618    /* Solves a specific equation of the form Ax=B, where A is a triangular matrix
2619    *  without zeroes on the diagonal (i.e. invertible).
2620    *  B can have any number of columns, but must have the same number of rows as A
2621    *  If A is [n, n] and B is [n, m], then the result will be [n, m] as well
2622    *  Has O(n^3) complexity (unlike most matrix ops out there), so use on cases
2623    *  where n > 100 sparingly, pre-chunk if necessary.
2624    *
2625    *  If left = false, solves xA=B instead
2626    *  If lower = false, assumes upper triangular instead
2627    *  If uni = true, assumes diagonal of A to be all ones (will override actual values)
2628    *
2629    *  TODO: currently only lower, right, non-unitriangular variant is implemented
2630    */
2631    GGML_API struct ggml_tensor * ggml_solve_tri(
2632        struct ggml_context * ctx,
2633        struct ggml_tensor  * a,
2634        struct ggml_tensor  * b,
2635        bool                  left,
2636        bool                  lower,
2637        bool                  uni);
2638
2639    // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST]
2640    // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306
2641    //
2642    // tensor shapes (S_k == S_v, H_v % H_k == 0):
2643    //   q, k  : [S_k, H_k, n_tokens, n_seqs]
2644    //   v     : [S_v, H_v, n_tokens, n_seqs]
2645    //   g     : [1, H_v, n_tokens, n_seqs] (scalar gate) or [S_v, H_v, n_tokens, n_seqs] (KDA)
2646    //   beta  : [1, H_v, n_tokens, n_seqs]
2647    //   state : [S_v, S_v, H_v, n_seqs] -- initial recurrent state s0
2648    //
2649    // the output packs the attention scores [S_v, H_v, n_tokens, n_seqs] followed by K state
2650    // snapshots, most-recent first (slot 0 = final state, slot s = state s tokens back). K == 1
2651    // keeps only the final state; when n_tokens < K only slots 0..n_tokens-1 are written.
2652    GGML_API struct ggml_tensor * ggml_gated_delta_net(
2653            struct ggml_context * ctx,
2654            struct ggml_tensor  * q,
2655            struct ggml_tensor  * k,
2656            struct ggml_tensor  * v,
2657            struct ggml_tensor  * g,
2658            struct ggml_tensor  * beta,
2659            struct ggml_tensor  * state,
2660            int64_t               K);
2661
2662    // DSA lightning indexer
2663    //
2664    // q:       [n_embd_idx, n_head_idx, n_batch, ne3 ]
2665    // k:       [n_embd_idx, 1,          n_kv,    ne3 ]
2666    // weights: [n_head_idx, n_batch,    1,       ne3 ] !! prescaled !!
2667    // mask:    [n_kv,       n_batch,    1,       ne33] !! f16 !!
2668    // res:     [n_kv,       n_batch,    1,       ne3 ]
2669    //
2670    // broadcast:
2671    //   ne3 % ne33 == 0
2672    //
2673    GGML_API struct ggml_tensor * ggml_lightning_indexer(
2674        struct ggml_context * ctx,
2675        struct ggml_tensor  * q,
2676        struct ggml_tensor  * k,
2677        struct ggml_tensor  * weights,
2678        struct ggml_tensor  * mask);
2679
2680    // DeepSeek V4 hyper-connections (ref. https://arxiv.org/pdf/2512.24880)
2681    // In short these operations are replacements for the original residual connection (x = transformer(x) + x)
2682    // using a richer representation through streams.
2683    //
2684    // hc_comb: mixes [(2 + hc)*hc, n_tokens], scale [3], base [(2 + hc)*hc]
2685    //          -> [dst_hc, src_hc, n_tokens]
2686    // logits[dst, src, t] = mixes[2*hc + dst + hc*src, t]*scale[2]
2687    //                         + base[2*hc + dst + hc*src]
2688    // Softmax over dst, add eps, normalize over src, then repeat normalization
2689    // over dst followed by src for iterations 1 through n_iter - 1.
2690    GGML_API struct ggml_tensor * ggml_dsv4_hc_comb(
2691            struct ggml_context * ctx,
2692            struct ggml_tensor  * mixes,
2693            struct ggml_tensor  * scale,
2694            struct ggml_tensor  * base,
2695            float                 eps,
2696            int32_t               n_iter);
2697
2698    // hc_pre: x [n_embd, hc, n_tokens], weights [hc, n_tokens] -> [n_embd, n_tokens]
2699    //   result[i, t] = sum_h x[i, h, t]*weights[h, t]
2700    //
2701    GGML_API struct ggml_tensor * ggml_dsv4_hc_pre(
2702            struct ggml_context * ctx,
2703            struct ggml_tensor  * x,
2704            struct ggml_tensor  * weights);
2705
2706    // hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens],
2707    //          post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens]
2708    //          -> [n_embd, hc, n_tokens]
2709    //   result[i, dst, t] = x[i, t]*post[dst, t]
2710    //                       + sum_src residual[i, src, t]*comb[dst, src, t]
2711    //
2712    GGML_API struct ggml_tensor * ggml_dsv4_hc_post(
2713            struct ggml_context * ctx,
2714            struct ggml_tensor  * x,
2715            struct ggml_tensor  * residual,
2716            struct ggml_tensor  * post,
2717            struct ggml_tensor  * comb);
2718
2719    // custom operators
2720
2721    typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
2722    typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);
2723    typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);
2724
2725#define GGML_N_TASKS_MAX (-1)
2726    // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks
2727
2728    GGML_API struct ggml_tensor * ggml_map_custom1(
2729            struct ggml_context   * ctx,
2730            struct ggml_tensor    * a,
2731            ggml_custom1_op_t       fun,
2732            int                     n_tasks,
2733            void                  * userdata);
2734
2735    GGML_API struct ggml_tensor * ggml_map_custom1_inplace(
2736            struct ggml_context   * ctx,
2737            struct ggml_tensor    * a,
2738            ggml_custom1_op_t       fun,
2739            int                     n_tasks,
2740            void                  * userdata);
2741
2742    GGML_API struct ggml_tensor * ggml_map_custom2(
2743            struct ggml_context   * ctx,
2744            struct ggml_tensor    * a,
2745            struct ggml_tensor    * b,
2746            ggml_custom2_op_t       fun,
2747            int                     n_tasks,
2748            void                  * userdata);
2749
2750    GGML_API struct ggml_tensor * ggml_map_custom2_inplace(
2751            struct ggml_context   * ctx,
2752            struct ggml_tensor    * a,
2753            struct ggml_tensor    * b,
2754            ggml_custom2_op_t       fun,
2755            int                     n_tasks,
2756            void                  * userdata);
2757
2758    GGML_API struct ggml_tensor * ggml_map_custom3(
2759            struct ggml_context   * ctx,
2760            struct ggml_tensor    * a,
2761            struct ggml_tensor    * b,
2762            struct ggml_tensor    * c,
2763            ggml_custom3_op_t       fun,
2764            int                     n_tasks,
2765            void                  * userdata);
2766
2767    GGML_API struct ggml_tensor * ggml_map_custom3_inplace(
2768            struct ggml_context   * ctx,
2769            struct ggml_tensor    * a,
2770            struct ggml_tensor    * b,
2771            struct ggml_tensor    * c,
2772            ggml_custom3_op_t       fun,
2773            int                     n_tasks,
2774            void                  * userdata);
2775
2776    typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata);
2777
2778    GGML_API struct ggml_tensor * ggml_custom_4d(
2779            struct ggml_context * ctx,
2780            enum ggml_type        type,
2781            int64_t               ne0,
2782            int64_t               ne1,
2783            int64_t               ne2,
2784            int64_t               ne3,
2785            struct ggml_tensor ** args,
2786            int                   n_args,
2787            ggml_custom_op_t      fun,
2788            int                   n_tasks,
2789            void                * userdata);
2790
2791    GGML_API struct ggml_tensor * ggml_custom_inplace(
2792            struct ggml_context * ctx,
2793            struct ggml_tensor  * a,
2794            struct ggml_tensor ** args,
2795            int                   n_args,
2796            ggml_custom_op_t      fun,
2797            int                   n_tasks,
2798            void                * userdata);
2799
2800    // loss function
2801
2802    GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
2803            struct ggml_context * ctx,
2804            struct ggml_tensor  * a,  // logits
2805            struct ggml_tensor  * b); // labels
2806
2807    GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(
2808            struct ggml_context * ctx,
2809            struct ggml_tensor  * a,  // logits
2810            struct ggml_tensor  * b,  // labels
2811            struct ggml_tensor  * c); // gradients of cross_entropy_loss result
2812
2813    // AdamW optimizer step
2814    // Paper: https://arxiv.org/pdf/1711.05101v3.pdf
2815    // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html
2816    GGML_API struct ggml_tensor * ggml_opt_step_adamw(
2817            struct ggml_context * ctx,
2818            struct ggml_tensor  * a,
2819            struct ggml_tensor  * grad,
2820            struct ggml_tensor  * m,
2821            struct ggml_tensor  * v,
2822            struct ggml_tensor  * adamw_params); // parameters such as the learning rate
2823
2824    // stochastic gradient descent step (with weight decay)
2825    GGML_API struct ggml_tensor * ggml_opt_step_sgd(
2826        struct ggml_context * ctx,
2827        struct ggml_tensor *  a,
2828        struct ggml_tensor *  grad,
2829        struct ggml_tensor *  sgd_params); // alpha, weight decay
2830
2831    // build forward multiple tensors and select one of them for computing
2832    // this is useful for creating graphs that have constant topology but compute different things based on the input
2833    // ref: https://github.com/ggml-org/llama.cpp/pull/18550
2834    //
2835    // nodes:
2836    //   | - build forward into the graph but do not compute
2837    //   c - build forward into the graph and compute
2838    //
2839    //    |  |  ...  c  ...  |
2840    //    |  |  ...  c  ...  |
2841    //    |  |  ...  c  ...  |
2842    //   [0  1  ... idx ...  n-1]        <-- ggml_build_forward_select(..., n, idx)
2843    //               c
2844    //               c
2845    //
2846    // example:
2847    //   struct ggml_tensor * curs[3];
2848    //
2849    //   curs[0]  = compute0(...);
2850    //   curs[1]  = compute1(...);
2851    //   curs[2]  = compute2(...);
2852    //
2853    //   int idx = select_branch(some_input);
2854    //
2855    //   struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx);
2856    //
2857    GGML_API struct ggml_tensor * ggml_build_forward_select(
2858            struct ggml_cgraph  * cgraph,
2859            struct ggml_tensor ** tensors,
2860            int                   n_tensors,
2861            int                   idx);
2862
2863    GGML_API void ggml_build_forward_expand(
2864            struct ggml_cgraph * cgraph,
2865            struct ggml_tensor * tensor);
2866
2867    // add the tensor and its parents to the graph without marking them for compute
2868    // the flag is set later, when the tensor is reached from a node that computes
2869    GGML_API void ggml_build_forward_order(
2870            struct ggml_cgraph * cgraph,
2871            struct ggml_tensor * tensor);
2872
2873    GGML_API void ggml_build_backward_expand(
2874        struct ggml_context *  ctx,        // context for gradient computation
2875        struct ggml_cgraph  *  cgraph,
2876        struct ggml_tensor  ** grad_accs);
2877
2878    // graph allocation in a context
2879    GGML_API struct ggml_cgraph * ggml_new_graph       (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false
2880    GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads);
2881    GGML_API struct ggml_cgraph * ggml_graph_dup       (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads);
2882    GGML_API void                 ggml_graph_cpy       (struct ggml_cgraph * src, struct ggml_cgraph * dst);
2883    GGML_API void                 ggml_graph_reset     (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1
2884    GGML_API void                 ggml_graph_clear     (struct ggml_cgraph * cgraph);
2885
2886    GGML_API int                   ggml_graph_size   (struct ggml_cgraph * cgraph);
2887    GGML_API struct ggml_tensor *  ggml_graph_node   (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i]
2888    GGML_API struct ggml_tensor ** ggml_graph_nodes  (struct ggml_cgraph * cgraph);
2889    GGML_API int                   ggml_graph_n_nodes(struct ggml_cgraph * cgraph);
2890
2891    GGML_API void   ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
2892
2893    GGML_API size_t ggml_graph_overhead(void);
2894    GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);
2895
2896    GGML_API struct ggml_tensor * ggml_graph_get_tensor  (const struct ggml_cgraph * cgraph, const char * name);
2897    GGML_API struct ggml_tensor * ggml_graph_get_grad    (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);
2898    GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);
2899
2900    // print info and performance information for the graph
2901    GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);
2902
2903    // dump the graph into a file using the dot format
2904    GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename);
2905
2906    // TODO these functions were sandwiched in the old optimization interface, is there a better place for them?
2907    typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);
2908
2909    // Set callback for all future logging events.
2910    // If this is not called, or NULL is supplied, everything is output on stderr.
2911    GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data);
2912    GGML_API void ggml_log_set(ggml_log_callback   log_callback, void *  user_data);
2913
2914    GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
2915
2916    //
2917    // quantization
2918    //
2919
2920    // - ggml_quantize_init can be called multiple times with the same type
2921    //   it will only initialize the quantization tables for the first call or after ggml_quantize_free
2922    //   automatically called by ggml_quantize_chunk for convenience
2923    //
2924    // - ggml_quantize_free will free any memory allocated by ggml_quantize_init
2925    //   call this at the end of the program to avoid memory leaks
2926    //
2927    // note: these are thread-safe
2928    //
2929    GGML_API void ggml_quantize_init(enum ggml_type type);
2930    GGML_API void ggml_quantize_free(void);
2931
2932    // some quantization type cannot be used without an importance matrix
2933    GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);
2934
2935    // calls ggml_quantize_init internally (i.e. can allocate memory)
2936    GGML_API size_t ggml_quantize_chunk(
2937            enum ggml_type   type,
2938               const float * src,
2939                      void * dst,
2940                   int64_t   start,
2941                   int64_t   nrows,
2942                   int64_t   n_per_row,
2943               const float * imatrix);
2944
2945#ifdef __cplusplus
2946    // restrict not standard in C++
2947#    if defined(__GNUC__)
2948#        define GGML_RESTRICT __restrict__
2949#    elif defined(__clang__)
2950#        define GGML_RESTRICT __restrict
2951#    elif defined(_MSC_VER)
2952#        define GGML_RESTRICT __restrict
2953#    else
2954#        define GGML_RESTRICT
2955#    endif
2956#else
2957#    if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L)
2958#        define GGML_RESTRICT __restrict
2959#    else
2960#        define GGML_RESTRICT restrict
2961#    endif
2962#endif
2963    typedef void (*ggml_to_float_t)  (const void  * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
2964    typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void  * GGML_RESTRICT y, int64_t k);
2965
2966    struct ggml_type_traits {
2967        const char             * type_name;
2968        int64_t                  blck_size;
2969        int64_t                  blck_size_interleave; // interleave elements in blocks
2970        size_t                   type_size;
2971        bool                     is_quantized;
2972        ggml_to_float_t          to_float;
2973        ggml_from_float_t        from_float_ref;
2974    };
2975
2976    GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);
2977
2978    // ggml threadpool
2979    // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend
2980    // the goal should be to create an API that other backends can use move everything to the ggml base
2981
2982    // scheduling priorities
2983    enum ggml_sched_priority {
2984        GGML_SCHED_PRIO_LOW = -1,
2985        GGML_SCHED_PRIO_NORMAL,
2986        GGML_SCHED_PRIO_MEDIUM,
2987        GGML_SCHED_PRIO_HIGH,
2988        GGML_SCHED_PRIO_REALTIME
2989    };
2990
2991    // threadpool params
2992    // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
2993    struct ggml_threadpool_params {
2994        bool                cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
2995        int                 n_threads;                   // number of threads
2996        enum ggml_sched_priority prio;                   // thread priority
2997        uint32_t            poll;                        // polling level (0 - no polling, 100 - aggressive polling)
2998        bool                strict_cpu;                  // strict cpu placement
2999        bool                paused;                      // start in paused state
3000    };
3001
3002    struct ggml_threadpool;     // forward declaration, see ggml.c
3003
3004    typedef struct ggml_threadpool * ggml_threadpool_t;
3005
3006    GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
3007    GGML_API void                          ggml_threadpool_params_init   (struct ggml_threadpool_params * p, int n_threads);
3008    GGML_API bool                          ggml_threadpool_params_match  (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
3009
3010#ifdef  __cplusplus
3011}
3012#endif