char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovllama : add option to save memory in device buffers (#22679)d6e7b033a

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
812 B35 linesraw
1#pragma once
2
3#include <cstddef>
4#include <cstdint>
5#include <string>
6
7struct ggml_tensor;
8
9class llama_io_write_i {
10public:
11    llama_io_write_i() = default;
12    virtual ~llama_io_write_i() = default;
13
14    virtual void write(const void * src, size_t size) = 0;
15    virtual void write_tensor(ggml_tensor * tensor, size_t offset, size_t size) = 0;
16
17    // bytes written so far
18    virtual size_t n_bytes() = 0;
19
20    void write_string(const std::string & str);
21};
22
23class llama_io_read_i {
24public:
25    llama_io_read_i() = default;
26    virtual ~llama_io_read_i() = default;
27
28    virtual void read(void * dst, size_t size) = 0;
29    virtual void read_tensor(ggml_tensor * tensor, size_t offset, size_t size) = 0;
30
31    // bytes read so far
32    virtual size_t n_bytes() = 0;
33
34    void read_string(std::string & str);
35};