char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Martin Changggml-et: Initial ET backend (#24179)082b326fc

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.5 KiB86 linesraw
1#pragma once
2
3#include "ggml-backend-impl.h"
4#include "ggml-et-uberkernel-common.h"
5
6#include <device-layer/IDeviceLayer.h>
7#include <runtime/IProfiler.h>
8#include <runtime/IRuntime.h>
9
10#include <cstdint>
11#include <fstream>
12#include <string>
13#include <unordered_map>
14#include <vector>
15
16std::shared_ptr<rt::IRuntime> ggml_et_runtime();
17
18struct ggml_backend_et_buffer_type_context {
19    int         devidx;
20    std::string name;
21};
22
23struct ggml_backend_et_buffer_context {
24    int          devidx;
25    void *       data;  // Device memory pointer
26    size_t       size;
27    rt::DeviceId rtid;
28};
29
30struct ggml_backend_et_context {
31    int devidx;
32};
33
34struct ggml_backend_et_device_context;
35
36// One slot in the uberkernel ring. The host vectors back the H2D copy and
37// must outlive the upload; the device buffers feed the kernel that consumes
38// them. pending_event lets us know when both have drained so the slot can
39// be recycled.
40struct ggml_backend_et_uberkernel_slot {
41    std::vector<ggml_et_uberkernel_inst> insts;
42    std::vector<std::byte>               params_blob;
43
44    std::byte * device_insts           = nullptr;
45    std::byte * device_params          = nullptr;
46    size_t      device_insts_capacity  = 0;
47    size_t      device_params_capacity = 0;
48
49    rt::EventId pending_event{};
50    bool        has_pending = false;
51};
52
53struct ggml_backend_et_uberkernel_context {
54    bool     failed     = false;
55    uint64_t shire_mask = 0;
56
57    // Ring of slots. We accumulate into slots[current_slot]; on segment
58    // commit we fire the H2D + launch and rotate to the next slot,
59    // waiting on its previous launch only if it hasn't drained yet.
60    static constexpr size_t         SLOT_COUNT = 4;
61    ggml_backend_et_uberkernel_slot slots[SLOT_COUNT];
62    size_t                          current_slot = 0;
63};
64
65struct ggml_backend_et_device_context {
66    int                        devidx;
67    rt::DeviceId               rtid;
68    std::string                name;
69    std::string                desc;
70    size_t                     total_mem;
71    ggml_backend_buffer_type_t buftype;
72
73    // Kernel management - default stream for ordered execution on this device
74    rt::StreamId                                  default_stream;
75    std::unordered_map<std::string, rt::KernelId> loaded_kernels;
76
77    // trace buffer - for printing support
78    std::byte * trace_buffer;
79
80    bool                               uberkernel_enabled = false;
81    ggml_backend_et_uberkernel_context uberkernel;
82};
83
84struct ggml_backend_et_reg_ctx {
85    std::vector<ggml_backend_dev_t> devices;
86};