hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-device model split (aka row-split) (#28589)eafe15a5e2026-09-11

1#ifndef HTP_OPNODE_H
2#define HTP_OPNODE_H
3
4#define GGML_COMMON_IMPL_CPP
5#include "ggml-backend-impl.h"
6#include "ggml-common.h"
7
8#include <algorithm>
9#include <string>
10#include <vector>
11#include <memory>
12#include <stdio.h>
13#include "htp-ops.h"
14#include "htp/matmul-ops.h"
15#include "htp/flash-attn-ops.h"
16#include "htp/unary-ops.h"
17#include "htp/allreduce-ops.h"
18
19struct htp_opnode {
20    ggml_tensor * node   { nullptr };
21    htp_op_code   opcode { HTP_OP_INVALID };
22    int32_t       kernel_params[HTP_OP_MAX_KERN_PARAMS] {0};
23
24    std::vector<ggml_tensor *>                fused;
25    std::vector<std::shared_ptr<ggml_tensor>> dummy;
26
27    std::vector<const ggml_tensor *> inputs;
28    std::vector<const ggml_tensor *> outputs;
29    std::string                      name;
30
31    int n_active_src(const ggml_tensor * t) const {
32        if (!t) return 0;
33        for (int i = GGML_MAX_SRC - 1; i >= 0; i--) {
34            if (t->src[i]) {
35                return i + 1;
36            }
37        }
38        return 0;
39    }
40
41    void init(ggml_tensor * node) {
42        this->node = node;
43        if (this->node) {
44            this->name = ggml_op_desc(this->node);
45
46            // Build inputs (preserving optional nullptrs)
47            int n_inputs = n_active_src(this->node);
48            this->inputs.resize(n_inputs, nullptr);
49            for (int i = 0; i < n_inputs; i++) {
50                this->inputs[i] = this->node->src[i];
51            }
52
53            // Build outputs
54            this->outputs.push_back(this->dst());
55        }
56    }
57
58    htp_opnode(htp_op_code opcode = HTP_OP_INVALID, ggml_tensor * node = nullptr) : opcode(opcode) {
59        init(node);
60    }
61
62    ggml_op             op()   const { return node->op; }
63    const ggml_tensor * src0() const { return node->src[0]; }
64    const ggml_tensor * src1() const { return node->src[1]; }
65    const ggml_tensor * dst()  const { return outputs.empty() ? node : outputs.back(); }
66
67    ggml_tensor * add_dummy(const ggml_tensor & t) {
68        dummy.push_back(std::make_shared<ggml_tensor>(t));
69        return dummy.back().get();
70    }
71
72    void add_fused(ggml_tensor * t, bool extra_dst = false) {
73        fused.push_back(t);
74
75        name += "+";
76        name += ggml_op_desc(t);
77
78        if (extra_dst) {
79            outputs.push_back(t);
80        } else {
81            outputs.clear();
82            outputs.push_back(t);
83        }
84
85        // Remove the newly fused intermediate output tensor t from inputs (if it was there)
86        inputs.erase(std::remove(inputs.begin(), inputs.end(), t), inputs.end());
87
88        // Append new inputs from t, preserving middle nullptrs
89        int n_inputs = n_active_src(t);
90        for (int i = 0; i < n_inputs; i++) {
91            const auto * src = t->src[i];
92            if (!src) {
93                inputs.push_back(nullptr);
94            } else if (src != node &&
95                       std::find(fused.begin(), fused.end(), src) == fused.end() &&
96                       std::find(inputs.begin(), inputs.end(), src) == inputs.end()) {
97                inputs.push_back(src);
98            }
99        }
100    }
101
102    const std::vector<const ggml_tensor *> & get_inputs() const {
103        return inputs;
104    }
105
106    const std::vector<const ggml_tensor *> & get_outputs() const {
107        return outputs;
108    }
109
110    std::string op_name() const {
111        return name;
112    }
113
114    bool is_empty() const {
115        return ggml_op_is_empty(node->op);
116    }
117
118    bool stackable() const {
119        switch (this->op()) {
120            case GGML_OP_MUL_MAT:
121            case GGML_OP_MUL_MAT_ID:
122                return ggml_is_quantized(this->src0()->type);
123            default:
124                return false;
125        }
126    }
127
128    bool same_input(const htp_opnode& n) const {
129        return n.src1() == this->src1();
130    }
131};
132
133struct htp_opformat {
134    char strides[64 * GGML_MAX_SRC];
135    char dims[64 * GGML_MAX_SRC];
136    char types[16 * GGML_MAX_SRC];
137    char buffs[64 * GGML_MAX_SRC];
138    char names[64 * GGML_MAX_SRC];
139    char kparams[128];
140
141    int format_tensor_dims(char * str, size_t max_size, const struct ggml_tensor * t) {
142        if (!t) {
143            return snprintf(str, max_size, "NONE");
144        }
145        if (t->ne[2] == 1 && t->ne[3] == 1) {
146            return snprintf(str, max_size, "%d:%d", (int) t->ne[0], (int) t->ne[1]);
147        } else {
148            return snprintf(str, max_size, "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]);
149        }
150    }
151
152    void format_op_dims(char * str, size_t max_size, const htp_opnode & node) {
153        char * p = str;
154        char * p_end = str + max_size;
155        auto inputs = node.get_inputs();
156
157        if (!inputs.empty()) {
158            p += std::min((size_t)format_tensor_dims(p, p_end - p, inputs[0]), (size_t)(p_end - p));
159
160            for (size_t i = 1; i < inputs.size(); i++) {
161                if (p < p_end) {
162                    p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
163                }
164                if (p < p_end) {
165                    p += std::min((size_t)format_tensor_dims(p, p_end - p, inputs[i]), (size_t)(p_end - p));
166                }
167            }
168
169            if (p < p_end) {
170                p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
171            }
172        }
173
174        char self[64];
175        format_tensor_dims(self, sizeof(self), node.dst());
176        if (p < p_end) {
177            p += std::min((size_t)snprintf(p, p_end - p, "%s", self), (size_t)(p_end - p));
178        }
179    }
180
181    int format_tensor_strides(char * str, size_t max_size, const struct ggml_tensor * t) {
182        if (!t) {
183            return snprintf(str, max_size, "NONE");
184        }
185        const char * c = ggml_is_contiguous(t) ? "" : "!";
186
187        if (t->ne[2] == 1 && t->ne[3] == 1) {
188            return snprintf(str, max_size, "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c);
189        } else {
190            return snprintf(str, max_size, "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c);
191        }
192    }
193
194    void format_op_strides(char * str, size_t max_size, const htp_opnode & node) {
195        char * p = str;
196        char * p_end = str + max_size;
197        auto inputs = node.get_inputs();
198
199        if (!inputs.empty()) {
200            p += std::min((size_t)format_tensor_strides(p, p_end - p, inputs[0]), (size_t)(p_end - p));
201
202            for (size_t i = 1; i < inputs.size(); i++) {
203                if (p < p_end) {
204                    p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
205                }
206                if (p < p_end) {
207                    p += std::min((size_t)format_tensor_strides(p, p_end - p, inputs[i]), (size_t)(p_end - p));
208                }
209            }
210
211            if (p < p_end) {
212                p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
213            }
214        }
215
216        char self[64];
217        format_tensor_strides(self, sizeof(self), node.dst());
218        if (p < p_end) {
219            p += std::min((size_t)snprintf(p, p_end - p, "%s", self), (size_t)(p_end - p));
220        }
221    }
222
223    void format_op_types(char * str, size_t max_size, const htp_opnode & node) {
224        char * p = str;
225        char * p_end = str + max_size;
226        auto inputs = node.get_inputs();
227
228        if (!inputs.empty()) {
229            if (p < p_end) {
230                p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[0] ? ggml_type_name(inputs[0]->type) : "NONE"), (size_t)(p_end - p));
231            }
232
233            for (size_t i = 1; i < inputs.size(); i++) {
234                if (p < p_end) {
235                    p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
236                }
237                if (p < p_end) {
238                    p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[i] ? ggml_type_name(inputs[i]->type) : "NONE"), (size_t)(p_end - p));
239                }
240            }
241
242            if (p < p_end) {
243                p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
244            }
245        }
246
247        if (p < p_end) {
248            p += std::min((size_t)snprintf(p, p_end - p, "%s", ggml_type_name(node.dst()->type)), (size_t)(p_end - p));
249        }
250    }
251
252    const char * tensor_buff_name(const struct ggml_tensor * t) {
253        if (t && t->buffer) {
254            return ggml_backend_buffer_name(t->buffer);
255        }
256        return "NONE";
257    }
258
259    void format_op_buffs(char * str, size_t max_size, const htp_opnode & node) {
260        char * p = str;
261        char * p_end = str + max_size;
262        auto inputs = node.get_inputs();
263
264        if (!inputs.empty()) {
265            if (p < p_end) {
266                p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(inputs[0])), (size_t)(p_end - p));
267            }
268
269            for (size_t i = 1; i < inputs.size(); i++) {
270                if (p < p_end) {
271                    p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
272                }
273                if (p < p_end) {
274                    p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(inputs[i])), (size_t)(p_end - p));
275                }
276            }
277
278            if (p < p_end) {
279                p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
280            }
281        }
282
283        if (p < p_end) {
284            p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(node.dst())), (size_t)(p_end - p));
285        }
286    }
287
288    void format_op_names(char * str, size_t max_size, const htp_opnode & node) {
289        char * p = str;
290        char * p_end = str + max_size;
291        auto inputs = node.get_inputs();
292
293        if (!inputs.empty()) {
294            if (p < p_end) {
295                p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[0] ? inputs[0]->name : "NONE"), (size_t)(p_end - p));
296            }
297
298            for (size_t i = 1; i < inputs.size(); i++) {
299                if (p < p_end) {
300                    p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));
301                }
302                if (p < p_end) {
303                    p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[i] ? inputs[i]->name : "NONE"), (size_t)(p_end - p));
304                }
305            }
306
307            if (p < p_end) {
308                p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));
309            }
310        }
311
312        if (p < p_end) {
313            p += std::min((size_t)snprintf(p, p_end - p, "%s", node.dst()->name), (size_t)(p_end - p));
314        }
315    }
316    void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) {
317        if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID ||
318            node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX ||
319            node.opcode == HTP_OP_MUL_MAT_ADD) {
320            const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params;
321            const char * path = "unknown";
322            int32_t type = kparams->kernel_type;
323            if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_F16_BATCHED) {
324                path = "hmx-tiled";
325            } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM ||
326                       type == HTP_MM_KERNEL_HVX_QUANT_ROW    || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
327                path = "hvx-tiled";
328            } else if (type == HTP_MM_KERNEL_HVX_F16_F16_DDR  || type == HTP_MM_KERNEL_HVX_F16_F32_DDR ||
329                       type == HTP_MM_KERNEL_HVX_F32_F32_DDR  || type == HTP_MM_KERNEL_HVX_F32_F16_DDR ||
330                       type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
331                path = "hvx-flat";
332            }
333            snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
334        } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {
335            const auto * kparams = (const struct htp_fa_kernel_params *) node.kernel_params;
336            const char * path = "unknown";
337            int32_t type = kparams->kernel_type;
338            if (type == HTP_FA_KERNEL_HMX) {
339                path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq";
340            } else if (type == HTP_FA_KERNEL_HVX) {
341                path = "hvx";
342            }
343            snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
344        } else if (htp_op_is_unary(node.opcode)) {
345            const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params;
346            snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);
347        } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) {
348            snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]);
349        } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) {
350            snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]);
351        } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) {
352            snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]);
353        } else {
354            snprintf(str, max_size, "----");
355        }
356    }
357
358    void format(const htp_opnode & node) {
359        format_op_dims(dims, sizeof(dims), node);
360        format_op_strides(strides, sizeof(strides), node);
361        format_op_types(types, sizeof(types), node);
362        format_op_buffs(buffs, sizeof(buffs), node);
363        format_op_names(names, sizeof(names), node);
364        format_kernel_params(kparams, sizeof(kparams), node);
365    }
366
367    htp_opformat() {
368        strides[0] = '\0';
369        dims[0]    = '\0';
370        types[0]   = '\0';
371        buffs[0]   = '\0';
372        names[0]   = '\0';
373        kparams[0] = '\0';
374    }
375    htp_opformat(const htp_opnode & node) { format(node); }
376};
377
378#endif // HTP_OPNODE_H