char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Titaniumtownsycl: fuse mul_mat(gate) + mul_mat(up) + GLU for q4_K dense FFN (#26779)650913862

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.2 KiB76 linesraw
1//
2// MIT license
3// Copyright (C) 2024 Intel Corporation
4// SPDX-License-Identifier: MIT
5//
6
7//
8// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9// See https://llvm.org/LICENSE.txt for license information.
10// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11//
12
13#ifndef GGML_SYCL_MMVQ_HPP
14#define GGML_SYCL_MMVQ_HPP
15
16#include "common.hpp"
17
18
19void ggml_sycl_op_mul_mat_vec_q(
20    ggml_backend_sycl_context & ctx,
21    const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,
22    const char *src0_dd_i, const float *src1_ddf_i, const char *src1_ddq_i,
23    float *dst_dd_i, const int64_t row_low, const int64_t row_high,
24    const int64_t src1_ncols, const int64_t src1_padded_row_size,
25    const dpct::queue_ptr &stream);
26
27// Requires standard (non-reorder) block layout for src0.
28// Returns false if src0_type isn't handled; caller should fall back.
29bool ggml_sycl_mul_mat_vec_q_id(
30    enum ggml_type     src0_type,
31    const void *       vx_base,             // start of stacked expert weights
32    const void *       vy,                  // pre-quantized src1 (Q8_1)
33    const int32_t *    ids_dev,             // device-side int32, length n_experts_used
34    float *            dst_base,
35    int                ncols,
36    int                nrows,
37    int                n_experts_used,
38    size_t             expert_weight_stride, // bytes between experts in vx_base
39    size_t             dst_row_stride,       // bytes between dst rows
40    size_t             src1_row_stride,      // 0 = shared src1, else per-expert stride in bytes
41    dpct::queue_ptr    stream);
42
43// Reorder (SoA) variant of the fused MoE expert GEMV.
44// vx_base: each expert slice (stride expert_weight_stride == src0->nb[2]) is a self-contained reorder/SoA layout.
45// vy: src1 quantized with quantize_and_reorder_q8_1_soa (per-row SoA). Returns false if src0_type isn't handled.
46bool ggml_sycl_mul_mat_vec_q_id_reorder(
47    enum ggml_type     src0_type,
48    const void *       vx_base,
49    const void *       vy,
50    const int32_t *    ids_dev,
51    float *            dst_base,
52    int                ncols,
53    int                nrows,
54    int                n_experts_used,
55    size_t             expert_weight_stride,
56    size_t             dst_row_stride,
57    size_t             src1_row_stride,
58    dpct::queue_ptr    stream);
59
60// Fused dense-FFN GEMV: writes glu(gate . y, up . y) instead of the two mat-vec results.
61// vx / vgate must share shape, stride and reorder layout. Returns false if unhandled.
62bool ggml_sycl_mul_mat_vec_q_glu_reorder(
63    enum ggml_type     src0_type,
64    enum ggml_glu_op   glu_op,
65    const void *       vx,
66    const void *       vgate,
67    const void *       vy,
68    float *            dst,
69    int                ncols,                // K, shared by both weights
70    int                nrows,                // output rows, i.e. weight ne[1]
71    int                ncols_dst,            // activation columns, 1..MMVQ_MAX_BATCH_SIZE
72    int                stride_col_y_bytes,   // bytes between activation columns in vy
73    int                stride_col_dst,       // floats between output columns in dst
74    dpct::queue_ptr    stream);
75
76#endif // GGML_SYCL_MMVQ_HPP