char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

YiChen Lvmetal: per-op source split + parallel compile (#26561)b615f5b4b

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.4 KiB126 linesraw
1#pragma once
2
3#include "ggml-metal-impl.h"
4
5#include <metal_stdlib>
6
7#ifdef GGML_METAL_HAS_TENSOR
8#include <metal_tensor>
9
10#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>
11#endif
12
13using namespace metal;
14
15#define MAX(x, y) ((x) > (y) ? (x) : (y))
16#define MIN(x, y) ((x) < (y) ? (x) : (y))
17#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }
18
19#define PAD2(x, n) (((x) + (n) - 1) & ~((n) - 1))
20
21#define FOR_UNROLL(x) _Pragma("clang loop unroll(full)") for (x)
22
23#define N_SIMDWIDTH 32 // assuming SIMD group size is 32
24
25// ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
26//
27// cmd:
28//   .../usr/bin/metal -dM -E -c                             ggml/src/ggml-metal/kernels/<src>.metal
29//   .../usr/bin/metal -dM -E -c -target air64-apple-ios14.0 ggml/src/ggml-metal/kernels/<src>.metal
30//
31#if __METAL_VERSION__ < 310 && defined(GGML_METAL_HAS_BF16)
32#undef GGML_METAL_HAS_BF16
33#endif
34
35#if defined(GGML_METAL_HAS_BF16)
36typedef matrix<bfloat, 4, 4> bfloat4x4;
37typedef matrix<bfloat, 2, 4> bfloat2x4;
38#endif
39
40constexpr constant static float kvalues_iq4nl_f[16] = {
41    -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f
42};
43
44constexpr constant static float kvalues_mxfp4_f[16] = {
45    0, .5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f, -0, -.5f, -1.f, -1.5f, -2.f, -3.f, -4.f, -6.f
46};
47
48static inline int best_index_int8(int n, constant float * val, float x) {
49    if (x <= val[0]) return 0;
50    if (x >= val[n-1]) return n-1;
51    int ml = 0, mu = n-1;
52    while (mu-ml > 1) {
53        int mav = (ml+mu)/2;
54        if (x < val[mav]) mu = mav; else ml = mav;
55    }
56    return x - val[mu-1] < val[mu] - x ? mu-1 : mu;
57}
58
59static inline float e8m0_to_fp32(uint8_t x) {
60    uint32_t bits;
61
62    if (x == 0) {
63        bits = 0x00400000;
64    } else {
65        bits = (uint32_t) x << 23;
66    }
67
68    return as_type<float>(bits);
69}
70
71static inline float dot(float x, float y) {
72    return x*y;
73}
74
75static inline float sum(float x) {
76    return x;
77}
78
79static inline float sum(float4 x) {
80    return x[0] + x[1] + x[2] + x[3];
81}
82
83enum ggml_sort_order {
84    GGML_SORT_ORDER_ASC,
85    GGML_SORT_ORDER_DESC,
86};
87
88constant float GELU_COEF_A     = 0.044715f;
89constant float GELU_QUICK_COEF = -1.702f;
90constant float SQRT_2_OVER_PI  = 0.79788456080286535587989211986876f;
91constant float SQRT_2_INV      = 0.70710678118654752440084436210484f;
92
93// based on Abramowitz and Stegun formula 7.1.26 or similar Hastings' approximation
94// ref: https://www.johndcook.com/blog/python_erf/
95constant float p_erf  = 0.3275911f;
96constant float a1_erf = 0.254829592f;
97constant float a2_erf = -0.284496736f;
98constant float a3_erf = 1.421413741f;
99constant float a4_erf = -1.453152027f;
100constant float a5_erf = 1.061405429f;
101
102template<typename T>
103inline T erf_approx(T x) {
104    T sign_x = sign(x);
105    x = fabs(x);
106    T t = 1.0f / (1.0f + p_erf * x);
107    T y = 1.0f - (((((a5_erf * t + a4_erf) * t) + a3_erf) * t + a2_erf) * t + a1_erf) * t * exp(-x * x);
108    return sign_x * y;
109}
110
111template<typename T> T elu_approx(T x);
112
113template<> inline float elu_approx<float>(float x) {
114    return (x > 0.f) ? x : (exp(x) - 1);
115}
116
117template<> inline float4 elu_approx<float4>(float4 x) {
118    float4 res;
119
120    res[0] = (x[0] > 0.0f) ? x[0] : (exp(x[0]) - 1.0f);
121    res[1] = (x[1] > 0.0f) ? x[1] : (exp(x[1]) - 1.0f);
122    res[2] = (x[2] > 0.0f) ? x[2] : (exp(x[2]) - 1.0f);
123    res[3] = (x[3] > 0.0f) ? x[3] : (exp(x[3]) - 1.0f);
124
125    return res;
126}