char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Nick Farrellsycl: attribute device allocations by site (GGML_SYCL_MEMTRACE) (#27631)cd8cdf397

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.4 KiB194 linesraw
1#include "memtrace.hpp"
2
3#include "common.hpp"
4#include "ggml-impl.h"
5
6#include <cstdio>
7#include <mutex>
8#include <unordered_map>
9
10constexpr size_t MIB = 1024 * 1024;
11
12static const char * mem_type_name(ggml_sycl_mem_type type) {
13    switch (type) {
14        case GGML_SYCL_MEM_BUFFER:   return "buffer";
15        case GGML_SYCL_MEM_POOL_LEG: return "pool_leg";
16        case GGML_SYCL_MEM_POOL_VMM: return "pool_vmm";
17        case GGML_SYCL_MEM_ASYNC:    return "async";
18        case GGML_SYCL_MEM_FATTN_KV: return "fattn_kv";
19        case GGML_SYCL_MEM_DIRECT:   return "direct";
20        default:                     GGML_ABORT("[%s] The type value %d is not supported\n", __func__, (int) type);
21    }
22}
23
24struct mem_tracker {
25    std::mutex mutex;
26    std::unordered_map<const void *, std::pair<ggml_sycl_mem_type, size_t>> live_by_ptr;
27    size_t live[GGML_SYCL_MEM_TYPE_COUNT] = {};
28    size_t peak[GGML_SYCL_MEM_TYPE_COUNT] = {};
29    size_t total_live = 0;
30    size_t total_peak = 0;
31    size_t last_logged_peak = 0;
32};
33
34static mem_tracker & get_tracker() {
35    static mem_tracker t;
36    return t;
37}
38
39static size_t step_bytes() {
40    const int mib = g_ggml_sycl_memtrace_step > 0 ? g_ggml_sycl_memtrace_step : 64;
41    return (size_t) mib * MIB;
42}
43
44static void report_sites_locked() {
45    mem_tracker & t = get_tracker();
46    for (int i = 0; i < GGML_SYCL_MEM_TYPE_COUNT; i++) {
47        if (t.peak[i] == 0) {
48            continue;
49        }
50        GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG "   %-9s allocated %5zu MiB, peak %5zu MiB\n",
51                      mem_type_name((ggml_sycl_mem_type) i), t.live[i] / MIB, t.peak[i] / MIB);
52    }
53}
54
55static void report_locked(const char * tag) {
56    mem_tracker & t = get_tracker();
57
58    const size_t allocated = t.total_live / MIB;
59    const size_t buffers   = t.live[GGML_SYCL_MEM_BUFFER] / MIB;
60
61    GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: allocated %5zu MiB (buffers %5zu + scratch %5zu),"
62                  " peak %5zu MiB\n",
63                  tag, allocated, buffers, allocated - buffers, t.total_peak / MIB);
64    report_sites_locked();
65}
66
67static void log_event_locked(const char * op, ggml_sycl_mem_type type, const void * ptr, size_t bytes) {
68    GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " allocated %5zu MiB  %-5s %-9s %9.3f MiB  ptr=%p\n",
69                  get_tracker().total_live / MIB, op, mem_type_name(type),
70                  (double) bytes / MIB, ptr);
71}
72
73bool ggml_sycl_memtrace_enabled() {
74    return g_ggml_sycl_memtrace > 0;
75}
76
77void ggml_sycl_memtrace_add(ggml_sycl_mem_type type, const void * ptr, size_t bytes) {
78    if (!ggml_sycl_memtrace_enabled()) {
79        return;
80    }
81    GGML_ASSERT(ptr != nullptr);
82    GGML_ASSERT(bytes != 0);
83
84    mem_tracker & t = get_tracker();
85    std::lock_guard<std::mutex> lock(t.mutex);
86
87    auto it = t.live_by_ptr.find(ptr);
88    if (it != t.live_by_ptr.end()) {
89        t.live[it->second.first] -= it->second.second;
90        t.total_live -= it->second.second;
91    }
92
93    t.live_by_ptr[ptr] = { type, bytes };
94    t.live[type] += bytes;
95    t.total_live += bytes;
96
97    if (t.live[type] > t.peak[type]) {
98        t.peak[type] = t.live[type];
99    }
100    if (t.total_live > t.total_peak) {
101        t.total_peak = t.total_live;
102    }
103
104    if (g_ggml_sycl_memtrace >= 2) {
105        log_event_locked("alloc", type, ptr, bytes);
106    }
107
108    static const size_t step = step_bytes();
109    if (t.total_peak >= t.last_logged_peak + step) {
110        t.last_logged_peak = t.total_peak;
111        char tag[96];
112        std::snprintf(tag, sizeof(tag), "peak grew (+%zu MiB from %s)", bytes / MIB,
113                      mem_type_name(type));
114        report_locked(tag);
115    }
116}
117
118void ggml_sycl_memtrace_del(const void * ptr) {
119    if (!ggml_sycl_memtrace_enabled() || ptr == nullptr) {
120        return;
121    }
122    mem_tracker & t = get_tracker();
123    std::lock_guard<std::mutex> lock(t.mutex);
124
125    auto it = t.live_by_ptr.find(ptr);
126    if (it == t.live_by_ptr.end()) {
127        return;
128    }
129    const ggml_sycl_mem_type type = it->second.first;
130    const size_t bytes = it->second.second;
131    t.live[type] -= bytes;
132    t.total_live -= bytes;
133    t.live_by_ptr.erase(it);
134
135    if (g_ggml_sycl_memtrace >= 2) {
136        log_event_locked("free", type, ptr, bytes);
137    }
138}
139
140void ggml_sycl_memtrace_fail(ggml_sycl_mem_type type, size_t bytes) {
141    GGML_LOG_ERROR(GGML_SYCL_MEMTRACE_TAG " alloc FAILED: %9.3f MiB %s\n",
142                   (double) bytes / MIB, mem_type_name(type));
143    if (!ggml_sycl_memtrace_enabled()) {
144        return;
145    }
146    mem_tracker & t = get_tracker();
147    std::lock_guard<std::mutex> lock(t.mutex);
148    report_locked("at allocation failure");
149}
150
151void ggml_sycl_memtrace_report(const char * tag) {
152    if (!ggml_sycl_memtrace_enabled()) {
153        return;
154    }
155    mem_tracker & t = get_tracker();
156    std::lock_guard<std::mutex> lock(t.mutex);
157    report_locked(tag);
158}
159
160static bool device_memory_is_dedicated(int device) {
161    if (device < 0 || device >= ggml_sycl_info().device_count) {
162        return false;
163    }
164    const sycl_device_info & info = ggml_sycl_info().devices[device];
165    return info.l0_device_type_valid && info.l0_discrete_gpu;
166}
167
168void ggml_sycl_memtrace_report_device(const char * tag, int device, size_t dev_free, size_t dev_total) {
169    if (!ggml_sycl_memtrace_enabled()) {
170        return;
171    }
172    mem_tracker & t = get_tracker();
173    std::lock_guard<std::mutex> lock(t.mutex);
174
175    const size_t in_use    = dev_total > dev_free ? dev_total - dev_free : 0;
176    const size_t total     = dev_total / MIB;
177    const size_t freed     = dev_free / MIB;
178    const size_t allocated = t.total_live / MIB;
179    const size_t buffers   = t.live[GGML_SYCL_MEM_BUFFER] / MIB;
180    const size_t peak      = t.total_peak / MIB;
181
182    if (in_use >= t.total_live && device_memory_is_dedicated(device) && total >= freed + allocated) {
183        GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: total %5zu MiB = free %5zu + allocated %5zu"
184                      " (buffers %5zu + scratch %5zu) + other %5zu, peak %5zu MiB\n",
185                      tag, total, freed, allocated, buffers, allocated - buffers,
186                      total - freed - allocated, peak);
187    } else {
188        GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: total %5zu MiB, free %5zu, in use %5zu;"
189                      " allocated %5zu (buffers %5zu + scratch %5zu), peak %5zu MiB\n",
190                      tag, total, freed, in_use / MIB, allocated, buffers,
191                      allocated - buffers, peak);
192    }
193    report_sites_locked();
194}