char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Nick Farrellsycl: make --fit respect --fit-target better (#27629)cc83d7b48

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.7 KiB42 linesraw
1//
2// MIT license
3// Copyright (C) 2025 Intel Corporation
4// SPDX-License-Identifier: MIT
5//
6
7//
8// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9// See https://llvm.org/LICENSE.txt for license information.
10// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11//
12
13#ifndef GGML_SYCL_FATTN_HPP
14#define GGML_SYCL_FATTN_HPP
15
16#include "common.hpp"
17
18void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
19
20bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst);
21
22// Scratch that flash attention needs beyond the output tensor
23struct ggml_sycl_fattn_extra {
24    uintptr_t K_buffer_ptr     = 0;   // F16 copy of the K cache
25    uintptr_t V_buffer_ptr     = 0;   // F16 copy of the V cache
26    uintptr_t Q_buffer_ptr     = 0;   // dense F16 copy of Q, oneDNN only
27    uintptr_t scale_buffer_ptr = 0;   // the softmax scale as an F16 scalar, oneDNN only
28    uintptr_t out_buffer_ptr   = 0;   // F16 SDPA output before conversion to F32, oneDNN only
29    uintptr_t end              = 0;   // one past the last reserved byte; sizes the allocation
30};
31
32// ggml_sycl_fattn_get_extra() is the single source of truth for the layout: it both sizes
33// the reservation and hands out the pointers, so the two cannot disagree.
34// Each field is the address of one reserved block, or 0 if that block was not reserved,
35// in which case the caller allocates from the scratch pool instead.
36ggml_sycl_fattn_extra ggml_sycl_fattn_get_extra(const ggml_tensor * dst);
37
38size_t ggml_sycl_flash_attn_ext_get_alloc_size(const ggml_tensor * dst);
39
40void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
41
42#endif // GGML_SYCL_FATTN_HPP