char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aman Guptallama: refactor fused ops (#24646)90e0f5cfc

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
3.0 KiB105 linesraw
1#pragma once
2
3#include "ggml.h" // for ggml_log_level
4
5#include <string>
6#include <type_traits>
7#include <vector>
8
9#ifdef __GNUC__
10#    if defined(__MINGW32__) && !defined(__clang__)
11#        define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
12#    else
13#        define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
14#    endif
15#else
16#    define LLAMA_ATTRIBUTE_FORMAT(...)
17#endif
18
19//
20// logging
21//
22
23LLAMA_ATTRIBUTE_FORMAT(2, 3)
24void llama_log_internal        (ggml_log_level level, const char * format, ...);
25void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data);
26
27#define LLAMA_LOG(...)       llama_log_internal(GGML_LOG_LEVEL_NONE , __VA_ARGS__)
28#define LLAMA_LOG_INFO(...)  llama_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)
29#define LLAMA_LOG_WARN(...)  llama_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)
30#define LLAMA_LOG_ERROR(...) llama_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
31#define LLAMA_LOG_DEBUG(...) llama_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
32#define LLAMA_LOG_CONT(...)  llama_log_internal(GGML_LOG_LEVEL_CONT , __VA_ARGS__)
33
34//
35// helpers
36//
37
38template <typename T>
39struct no_init {
40    T value;
41    no_init() = default;
42};
43
44template <typename dst_t, typename src_t>
45static inline dst_t llama_cast(src_t v) {
46    if constexpr (std::is_same_v<src_t, dst_t>) {
47        return v;
48    } else if constexpr (std::is_same_v<src_t, ggml_fp16_t> && std::is_same_v<dst_t, float>) {
49        return ggml_fp16_to_fp32(v);
50    } else if constexpr (std::is_same_v<src_t, float> && std::is_same_v<dst_t, ggml_fp16_t>) {
51        return ggml_fp32_to_fp16(v);
52    } else {
53        static_assert(std::is_same_v<dst_t, void>, "unsupported type combination");
54    }
55}
56
57static inline ggml_tensor * llama_mul_mat_hadamard(
58        ggml_context * ctx,
59        ggml_tensor * cur,
60        ggml_tensor * rot) {
61    const auto n = rot->ne[0];
62
63    ggml_tensor * res;
64
65    if (!ggml_is_contiguous(cur)) {
66        res = ggml_cont_2d(ctx, cur, n, ggml_nelements(cur)/n);
67    } else {
68        res = ggml_reshape_2d(ctx, cur, n, ggml_nelements(cur)/n);
69    }
70    res = ggml_mul_mat(ctx, rot, res);
71    ggml_mul_mat_set_hint(res, GGML_HINT_SRC0_IS_HADAMARD);
72    res = ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]);
73
74    return res;
75}
76
77struct time_meas {
78    time_meas(int64_t & t_acc, bool disable = false);
79    ~time_meas();
80
81    const int64_t t_start_us;
82
83    int64_t & t_acc;
84};
85
86template <typename T>
87struct buffer_view {
88    T * data;
89    size_t size = 0;
90
91    bool has_data() const {
92        return data && size > 0;
93    }
94};
95
96void replace_all(std::string & s, const std::string & search, const std::string & replace);
97
98// TODO: rename to llama_format ?
99LLAMA_ATTRIBUTE_FORMAT(1, 2)
100std::string format(const char * fmt, ...);
101
102std::string llama_format_tensor_shape(const std::vector<int64_t> & ne);
103std::string llama_format_tensor_shape(const struct ggml_tensor * t);
104
105std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i);