char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

alex-spacemitggml-cpu: Add IME2 Instruction Support for the SpacemiT Backend (#22863)81b0d882a

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.3 KiB95 linesraw
1#pragma once
2
3#include "ggml-cpu-impl.h"
4
5#include <cassert>
6#include <cstddef>
7#include <cstdint>
8#include <functional>
9
10namespace spacemit_kernels {
11
12constexpr auto div_round_up(auto up, auto down) {
13    return (up + down - 1) / down;
14}
15
16// Q8 Blk [f32] [s16] [int8 * blk_len]
17// Q8 Blk N [f32 * N] [s16 * N] [int8 * blk_len * N]
18constexpr size_t q8_blk_size(size_t blk_len, bool with_blk_sum = false) {
19    const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t) + (with_blk_sum ? sizeof(int16_t) : 0);
20    return blk_size;
21}
22
23// Q8 HP row block: K is split into K32 subblocks.
24// Each subblock stores [f32 scale] [int8 * 32], with an optional fp16 sum trailer per subblock.
25constexpr size_t q8_hp_blk_size(size_t blk_len, bool with_blk_sum = false, bool with_blk_scale = false) {
26    const size_t subblk_count = div_round_up(blk_len, size_t(32));
27    const size_t blk_size     = blk_len * sizeof(int8_t) + subblk_count * sizeof(_Float16) +
28                            (with_blk_sum ? subblk_count * sizeof(_Float16) : 0) +
29                            (with_blk_scale ? sizeof(_Float16) : 0);
30    return blk_size;
31}
32
33// Q8K Blk [f32] [s16 * (blk_len / 16)] [int8 * blk_len]
34// Q8K Blk N [f32 * N] [s16 * (blk_len / 16) * N] [int8 * blk_len * N]
35constexpr size_t q8k_blk_size(size_t blk_len) {
36    const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t) + sizeof(int16_t) * blk_len / 16;
37    return blk_size;
38}
39
40using quantize_a_row_def = std::function<void(size_t, const float *, size_t, uint8_t *)>;
41
42namespace rvv {
43void memcpy1d(void * dst, const void * src, int64_t size);
44
45void memcpy2d(void * dst, int64_t dst_stride, const void * src, int64_t src_stride, int64_t tile_rows, int64_t size);
46
47void forward_flash_attn_ext_f16_one_chunk_vlen1024_vf16(const ggml_compute_params * params,
48                                                        ggml_tensor *               dst,
49                                                        int                         ir0,
50                                                        int                         ir1,
51                                                        void *                      tcm_buffer,
52                                                        size_t                      tcm_buffer_size);
53
54void forward_flash_attn_ext_f16_tiled_vlen1024_vf16(const ggml_compute_params * params,
55                                                    ggml_tensor *               dst,
56                                                    int                         ir0,
57                                                    int                         ir1,
58                                                    void *                      tcm_buffer,
59                                                    size_t                      tcm_buffer_size);
60
61void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op);
62
63void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op);
64
65void forward_cont_with_permute(ggml_compute_params * params, ggml_tensor * op);
66
67void forward_cpy_with_permute(ggml_compute_params * params, ggml_tensor * op);
68
69template <typename T> void forward_get_rows(ggml_compute_params * params, ggml_tensor * op);
70
71template <typename T> void forward_concat(ggml_compute_params * params, ggml_tensor * op);
72
73template <ggml_op op_type, typename T> void forward_binary(ggml_compute_params * params, ggml_tensor * op);
74
75template <typename T> void forward_sum_rows(const ggml_compute_params * params, ggml_tensor * op);
76
77template <typename T> void forward_repeat_nrows(ggml_compute_params * params, ggml_tensor * op);
78
79template <typename T> void forward_repeat_dim1(ggml_compute_params * params, ggml_tensor * op);
80
81void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
82
83void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
84
85void quantize_a_row_i8_hp(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
86
87void quantize_a_4row_i8_hp(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
88
89void quantize_a_row_i8k(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
90
91void quantize_a_4row_i8k(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr);
92
93}  // namespace rvv
94
95}  // namespace spacemit_kernels