char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Aman Guptallama-batch: add n_keep_tail in split_equal for recurrent models (#25278)230ea9d21

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
11.2 KiB363 linesraw
1#include "llama-kv-cache-iswa.h"
2
3#include "llama-impl.h"
4#include "llama-batch.h"
5#include "llama-model.h"
6
7#include <algorithm>
8#include <cassert>
9
10//
11// llama_kv_cache_iswa
12//
13
14llama_kv_cache_iswa::llama_kv_cache_iswa(
15        const llama_model & model,
16                ggml_type   type_k,
17                ggml_type   type_v,
18                     bool   v_trans,
19                     bool   offload,
20                     bool   swa_full,
21                     bool   unified,
22                 uint32_t   kv_size,
23                 uint32_t   n_seq_max,
24                 uint32_t   n_ubatch,
25                 uint32_t   n_pad,
26           llama_memory_t   mem_other,
27    const layer_filter_cb & filter,
28    const  layer_reuse_cb & reuse,
29    const  layer_share_cb & share) :
30    llama_kv_cache_iswa(model, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified,
31            kv_size, n_seq_max, n_ubatch, n_pad, mem_other, filter, reuse, share) {
32}
33
34llama_kv_cache_iswa::llama_kv_cache_iswa(
35        const llama_model & model,
36        const llama_hparams & hparams,
37                ggml_type   type_k,
38                ggml_type   type_v,
39                     bool   v_trans,
40                     bool   offload,
41                     bool   swa_full,
42                     bool   unified,
43                 uint32_t   kv_size,
44                 uint32_t   n_seq_max,
45                 uint32_t   n_ubatch,
46                 uint32_t   n_pad,
47           llama_memory_t   mem_other,
48    const layer_filter_cb & filter,
49    const  layer_reuse_cb & reuse,
50    const  layer_share_cb & share) : unified(unified) {
51
52    // chain filters
53    const layer_filter_cb filter_base = [&](int32_t il) {
54        if (filter && !filter(il)) {
55            return false;
56        }
57
58        return !model.hparams.is_swa(il);
59    };
60
61    const layer_filter_cb filter_swa  = [&](int32_t il) {
62        if (filter && !filter(il)) {
63            return false;
64        }
65
66        return  model.hparams.is_swa(il);
67    };
68
69    const uint32_t size_base = kv_size;
70
71    // note: the SWA cache is always padded to 256 for performance
72    //       https://github.com/ggml-org/llama.cpp/issues/17037
73    uint32_t size_swa = GGML_PAD(std::min(size_base, hparams.n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256);
74
75    // when using full-size SWA cache, we set the SWA cache size to be equal to the base cache size
76    if (swa_full) {
77        LLAMA_LOG_WARN("%s: using full-size SWA cache (ref: %s)\n",
78                __func__, "https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
79
80        size_swa = size_base;
81    }
82
83    LLAMA_LOG_INFO("%s: creating non-SWA KV cache, size = %u cells\n", __func__, size_base);
84
85    llama_memory_t mem_other_base = nullptr;
86    if (mem_other) {
87        mem_other_base = static_cast<llama_kv_cache_iswa *>(mem_other)->get_base();
88    }
89
90    llama_memory_t mem_other_swa = nullptr;
91    if (mem_other) {
92        mem_other_swa = static_cast<llama_kv_cache_iswa *>(mem_other)->get_swa();
93    }
94
95    kv_base = std::make_unique<llama_kv_cache>(
96            model, hparams, type_k, type_v,
97            v_trans, offload, unified, size_base, n_seq_max, n_pad,
98            0, LLAMA_SWA_TYPE_NONE, mem_other_base, filter_base, reuse, share);
99
100    LLAMA_LOG_INFO("%s: creating     SWA KV cache, size = %u cells\n", __func__, size_swa);
101
102    kv_swa = std::make_unique<llama_kv_cache>(
103            model, hparams, type_k, type_v,
104            v_trans, offload, unified, size_swa, n_seq_max, n_pad,
105            hparams.n_swa, hparams.swa_type, mem_other_swa, filter_swa, reuse, share);
106}
107
108void llama_kv_cache_iswa::clear(bool data) {
109    kv_base->clear(data);
110    kv_swa ->clear(data);
111}
112
113bool llama_kv_cache_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
114    bool res = true;
115
116    res = res & kv_base->seq_rm(seq_id, p0, p1);
117    res = res & kv_swa ->seq_rm(seq_id, p0, p1);
118
119    return res;
120}
121
122void llama_kv_cache_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
123    kv_base->seq_cp(seq_id_src, seq_id_dst, p0, p1);
124    kv_swa ->seq_cp(seq_id_src, seq_id_dst, p0, p1);
125}
126
127void llama_kv_cache_iswa::seq_keep(llama_seq_id seq_id) {
128    kv_base->seq_keep(seq_id);
129    kv_swa ->seq_keep(seq_id);
130}
131
132void llama_kv_cache_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {
133    kv_base->seq_add(seq_id, p0, p1, shift);
134    kv_swa ->seq_add(seq_id, p0, p1, shift);
135}
136
137void llama_kv_cache_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {
138    kv_base->seq_div(seq_id, p0, p1, d);
139    kv_swa ->seq_div(seq_id, p0, p1, d);
140}
141
142llama_pos llama_kv_cache_iswa::seq_pos_min(llama_seq_id seq_id) const {
143    // the base cache is a superset of the SWA cache, so we can just check the SWA cache
144    return kv_swa->seq_pos_min(seq_id);
145}
146
147llama_pos llama_kv_cache_iswa::seq_pos_max(llama_seq_id seq_id) const {
148    return kv_swa->seq_pos_max(seq_id);
149}
150
151std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_iswa::memory_breakdown() const {
152    std::map<ggml_backend_buffer_type_t, size_t> mb = kv_base->memory_breakdown();
153    for (const auto & buft_size : kv_swa->memory_breakdown()) {
154        mb[buft_size.first] += buft_size.second;
155    }
156    return mb;
157}
158
159llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {
160    GGML_UNUSED(embd_all);
161
162    // first try simple split
163    do {
164        if (!unified) {
165            // requires equal splits, so we skip the simple split
166            break;
167        }
168
169        balloc.split_reset();
170
171        std::vector<llama_ubatch> ubatches;
172        while (true) {
173            auto ubatch = balloc.split_simple(n_ubatch);
174
175            if (ubatch.n_tokens == 0) {
176                break;
177            }
178
179            ubatches.push_back(std::move(ubatch)); // NOLINT
180        }
181
182        if (balloc.get_n_used() < balloc.get_n_tokens()) {
183            // failed to find a suitable split
184            break;
185        }
186
187        auto sinfos_base = kv_base->prepare(ubatches);
188        if (sinfos_base.empty()) {
189            break;
190        }
191
192        auto sinfos_swa = kv_swa->prepare(ubatches);
193        if (sinfos_swa.empty()) {
194            break;
195        }
196
197        assert(sinfos_base.size() == sinfos_swa.size());
198
199        return std::make_unique<llama_kv_cache_iswa_context>(
200                this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches));
201    } while (false);
202
203    // if it fails, try equal split
204    do {
205        balloc.split_reset();
206
207        std::vector<llama_ubatch> ubatches;
208        while (true) {
209            auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);
210
211            if (ubatch.n_tokens == 0) {
212                break;
213            }
214
215            ubatches.push_back(std::move(ubatch)); // NOLINT
216        }
217
218        if (balloc.get_n_used() < balloc.get_n_tokens()) {
219            // failed to find a suitable split
220            break;
221        }
222
223        auto sinfos_base = kv_base->prepare(ubatches);
224        if (sinfos_base.empty()) {
225            break;
226        }
227
228        auto sinfos_swa = kv_swa->prepare(ubatches);
229        if (sinfos_swa.empty()) {
230            break;
231        }
232
233        assert(sinfos_base.size() == sinfos_swa.size());
234
235        return std::make_unique<llama_kv_cache_iswa_context>(
236                this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches));
237    } while (false);
238
239    // TODO: if we fail again, we should attempt different splitting strategies
240    //       but to do that properly, we first have to refactor the batches to be more flexible
241
242    return std::make_unique<llama_kv_cache_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);
243}
244
245llama_memory_context_ptr llama_kv_cache_iswa::init_full() {
246    return std::make_unique<llama_kv_cache_iswa_context>(this);
247}
248
249llama_memory_context_ptr llama_kv_cache_iswa::init_update(llama_context * lctx, bool optimize) {
250    return std::make_unique<llama_kv_cache_iswa_context>(this, lctx, optimize);
251}
252
253bool llama_kv_cache_iswa::get_can_shift() const {
254    return kv_base->get_can_shift() &&
255           kv_swa->get_can_shift() &&
256           kv_base->get_size() == kv_swa->get_size();
257}
258
259void llama_kv_cache_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {
260    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {
261        kv_base->state_write(io, seq_id, flags);
262    }
263
264    kv_swa->state_write(io, seq_id, flags);
265}
266
267void llama_kv_cache_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {
268    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {
269        kv_base->state_read(io, seq_id, flags);
270    }
271
272    kv_swa->state_read(io, seq_id, flags);
273}
274
275llama_kv_cache * llama_kv_cache_iswa::get_base() const {
276    return kv_base.get();
277}
278
279llama_kv_cache * llama_kv_cache_iswa::get_swa() const {
280    return kv_swa.get();
281}
282
283//
284// llama_kv_cache_iswa_context
285//
286
287llama_kv_cache_iswa_context::llama_kv_cache_iswa_context(llama_memory_status status) : status(status) {}
288
289llama_kv_cache_iswa_context::llama_kv_cache_iswa_context(
290        llama_kv_cache_iswa * kv) :
291    ctx_base(kv->get_base()->init_full()),
292    ctx_swa (kv->get_swa ()->init_full()),
293    status(llama_memory_status_combine(ctx_base->get_status(), ctx_swa->get_status())) {
294}
295
296llama_kv_cache_iswa_context::llama_kv_cache_iswa_context(
297        llama_kv_cache_iswa * kv,
298        llama_context * lctx,
299        bool optimize) :
300    ctx_base(kv->get_base()->init_update(lctx, optimize)),
301    ctx_swa (kv->get_swa ()->init_update(lctx, optimize)),
302    status(llama_memory_status_combine(ctx_base->get_status(), ctx_swa->get_status())) {
303}
304
305llama_kv_cache_iswa_context::llama_kv_cache_iswa_context(
306        llama_kv_cache_iswa * kv,
307        slot_info_vec_t sinfos_base,
308        slot_info_vec_t sinfos_swa,
309        std::vector<llama_ubatch> ubatches) :
310    ubatches(std::move(ubatches)),
311    // note: here we copy the ubatches. not sure if this is ideal
312    ctx_base(new llama_kv_cache_context(kv->get_base(), std::move(sinfos_base), this->ubatches)),
313    ctx_swa (new llama_kv_cache_context(kv->get_swa (), std::move(sinfos_swa),  this->ubatches)),
314    status(llama_memory_status_combine(ctx_base->get_status(), ctx_swa->get_status())) {
315}
316
317llama_kv_cache_iswa_context:: ~llama_kv_cache_iswa_context() = default;
318
319bool llama_kv_cache_iswa_context::next() {
320    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
321
322    ctx_base->next();
323    ctx_swa ->next();
324
325    if (++i_next >= ubatches.size()) {
326        return false;
327    }
328
329    return true;
330}
331
332bool llama_kv_cache_iswa_context::apply() {
333    assert(!llama_memory_status_is_fail(status));
334
335    bool res = true;
336
337    res = res & ctx_base->apply();
338    res = res & ctx_swa ->apply();
339
340    return res;
341}
342
343llama_memory_status llama_kv_cache_iswa_context::get_status() const {
344    return status;
345}
346
347const llama_ubatch & llama_kv_cache_iswa_context::get_ubatch() const {
348    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
349
350    return ubatches[i_next];
351}
352
353const llama_kv_cache_context * llama_kv_cache_iswa_context::get_base() const {
354    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
355
356    return static_cast<const llama_kv_cache_context *>(ctx_base.get());
357}
358
359const llama_kv_cache_context * llama_kv_cache_iswa_context::get_swa()  const {
360    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
361
362    return static_cast<const llama_kv_cache_context *>(ctx_swa.get());
363}