char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Frosty40sycl: fuse rms_norm+mul+add and add+add residual chains (#27610)6703d7894

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.1 KiB69 linesraw
1#ifndef GGML_SYCL_BINBCAST_HPP
2#define GGML_SYCL_BINBCAST_HPP
3#include "common.hpp"
4
5
6static __dpct_inline__ float op_repeat(const float a, const float b) {
7    return b;
8    GGML_UNUSED(a);
9}
10
11static __dpct_inline__ float op_add(const float a, const float b) {
12    return a + b;
13}
14
15static __dpct_inline__ float op_sub(const float a, const float b) {
16    return a - b;
17}
18
19static __dpct_inline__ float op_mul(const float a, const float b) {
20    return a * b;
21}
22
23static __dpct_inline__ float op_div(const float a, const float b) {
24    return a / b;
25}
26
27void ggml_sycl_add(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
28
29void ggml_sycl_sub(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
30
31void ggml_sycl_mul(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
32
33void ggml_sycl_div(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
34
35void ggml_sycl_repeat(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
36
37void ggml_sycl_op_add_add_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add0, ggml_tensor * add1);
38
39// Type combinations the standalone SYCL add() kernel can run. Fused ADD+ADD
40// uses the same set; anything else falls back to two add() launches.
41inline bool ggml_sycl_add_kernel_supports(enum ggml_type src0, enum ggml_type src1, enum ggml_type dst) {
42    if (src0 == GGML_TYPE_F32 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_F32) {
43        return true;
44    }
45    if (src0 == GGML_TYPE_F16 && src1 == GGML_TYPE_F16 && dst == GGML_TYPE_F16) {
46        return true;
47    }
48    if (src0 == GGML_TYPE_F16 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_F16) {
49        return true;
50    }
51    if (src0 == GGML_TYPE_I32 && src1 == GGML_TYPE_I32 && dst == GGML_TYPE_I32) {
52        return true;
53    }
54    if (src0 == GGML_TYPE_I16 && src1 == GGML_TYPE_I16 && dst == GGML_TYPE_I16) {
55        return true;
56    }
57#ifdef GGML_SYCL_HAS_BF16
58    if (src0 == GGML_TYPE_BF16 && src1 == GGML_TYPE_BF16 && dst == GGML_TYPE_BF16) {
59        return true;
60    }
61    if (src0 == GGML_TYPE_BF16 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_BF16) {
62        return true;
63    }
64#endif
65    return false;
66}
67
68#endif //GGML_SYCL_BINBCAST_HPP
69