char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Bartowskitests : add unit test coverage for llama_tensor_get_type (#20112)7992aa7c8

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.9 KiB50 linesraw
1#pragma once
2
3#include "ggml-cpp.h"
4#include "gguf.h"
5
6#include <cstdint>
7#include <optional>
8#include <string>
9#include <vector>
10
11struct gguf_remote_tensor {
12    std::string  name;
13    ggml_type    type    = GGML_TYPE_F32;
14    int64_t      ne[4]   = {1, 1, 1, 1};  // dimensions, unused dims = 1
15    uint32_t     n_dims  = 0;
16};
17
18struct gguf_remote_model {
19    // Selected KV metadata
20    std::string architecture;               // general.architecture
21    uint32_t    n_embd          = 0;        // <arch>.embedding_length
22    uint32_t    n_ff            = 0;        // <arch>.feed_forward_length
23    uint32_t    n_vocab         = 0;        // inferred from token_embd.weight ne[1]
24    uint32_t    n_layer         = 0;        // <arch>.block_count
25    uint32_t    n_head          = 0;        // <arch>.attention.head_count
26    uint32_t    n_head_kv       = 0;        // <arch>.attention.head_count_kv
27    uint32_t    n_expert        = 0;        // <arch>.expert_count (0 if absent)
28    uint32_t    n_embd_head_k   = 0;        // <arch>.attention.key_length
29    uint32_t    n_embd_head_v   = 0;        // <arch>.attention.value_length
30    uint16_t    n_split         = 0;        // split.count (0 = not split)
31    uint32_t    n_split_tensors = 0;        // split.tensors.count (0 if not split)
32
33    std::vector<gguf_remote_tensor> tensors;
34};
35
36// Fetch model metadata from HuggingFace with local caching.
37// repo: e.g., "ggml-org/Qwen3-32B-GGUF"
38// quant: e.g., "Q8_0" -- auto-detects filename (including first shard of split models)
39// Returns nullopt if download fails or network is unavailable.
40std::optional<gguf_remote_model> gguf_fetch_model_meta(
41    const std::string & repo,
42    const std::string & quant = "Q8_0",
43    const std::string & cache_dir = "",  // empty = default
44    bool verbose = true);
45
46gguf_context_ptr gguf_fetch_gguf_ctx(
47    const std::string & repo,
48    const std::string & quant = "Q8_0",
49    const std::string & cache_dir = "",
50    bool verbose = true);