char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Neo Zhang[SYCL] supprt Flash Attention for fp32/fp16/Q4/Q5/Q8 (#20190)213c4a0b8

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.5 KiB79 linesraw
1#include "count-equal.hpp"
2
3#include <cstdint>
4
5template <typename T>
6static void count_equal(const T *__restrict__ x, const T *__restrict__ y,
7                        int64_t *__restrict__ dst, const int64_t dk,
8                        const int64_t k) {
9    auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
10    const int64_t i0 = (int64_t)item_ct1.get_group(2) * dk;
11    const int64_t i1 = sycl::min(i0 + dk, k);
12
13    int nequal = 0;
14
15    for (int64_t i = i0 + item_ct1.get_local_id(2); i < i1; i += WARP_SIZE) {
16        const T xi = x[i];
17        const T yi = y[i];
18        nequal += xi == yi;
19    }
20
21    nequal = warp_reduce_sum<WARP_SIZE>(nequal);
22
23    if (item_ct1.get_local_id(2) != 0) {
24        return;
25    }
26
27    dpct::atomic_fetch_add<sycl::access::address_space::generic_space>(
28        (int *)dst, nequal);
29}
30
31void ggml_sycl_count_equal(ggml_backend_sycl_context &ctx, ggml_tensor *dst) {
32    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
33    const ggml_tensor * src0 = dst->src[0];
34    const ggml_tensor * src1 = dst->src[1];
35
36    GGML_ASSERT(src0->type == src1->type);
37    GGML_ASSERT( dst->type == GGML_TYPE_I64);
38
39    GGML_ASSERT(ggml_are_same_shape(src0, src1));
40    GGML_ASSERT(ggml_is_contiguous(src0));
41    GGML_ASSERT(ggml_is_contiguous(src1));
42    GGML_ASSERT(ggml_is_contiguous(dst));
43
44    int64_t * dst_d  = (int64_t *) dst->data;
45
46    dpct::queue_ptr stream = ctx.stream();
47    const int id       = get_current_device_id();
48    const int nsm = ggml_sycl_info().devices[id].nsm;
49
50    const int64_t ne = ggml_nelements(src0);
51    GGML_ASSERT(ne < (1 << 30) && "atomicAdd implementation only supports int");
52    const int64_t dne =
53        GGML_PAD((ne + 4 * nsm - 1) / (4 * nsm), SYCL_COUNT_EQUAL_CHUNK_SIZE);
54
55    SYCL_CHECK(CHECK_TRY_ERROR(stream->memset(dst_d, 0, ggml_nbytes(dst))));
56
57    const dpct::dim3 block_dims(WARP_SIZE, 1, 1);
58    const dpct::dim3 block_nums(
59        std::min((int64_t)4 * nsm, (ne + SYCL_COUNT_EQUAL_CHUNK_SIZE - 1) /
60                                       SYCL_COUNT_EQUAL_CHUNK_SIZE),
61        1, 1);
62
63    switch (src0->type) {
64    case GGML_TYPE_I32: {
65        const int *src0_d = (const int *)src0->data;
66        const int *src1_d = (const int *)src1->data;
67        stream->parallel_for(
68            sycl::nd_range<3>(block_nums * block_dims, block_dims),
69            [=](sycl::nd_item<3> item_ct1) {
70                count_equal(src0_d, src1_d, dst_d, dne, ne);
71                GGML_UNUSED(item_ct1);
72            });
73
74    } break;
75    default:
76        GGML_ASSERT(false);
77        break;
78    }
79}