hey! this is a static git repo viewer. commit diffs, file history, older commits and other revisions won't load without javascript.

you are still able to view source files on branch/tag tips, but the experience is degraded.

char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Pepper Grayllama : add missing headers (#28566)ca86fb2222026-09-08

1#include "llama-mmap.h"
2
3#include "llama-impl.h"
4
5#include "ggml.h"
6
7#include <cstring>
8#include <climits>
9#include <cstdlib>
10#include <stdexcept>
11#include <cerrno>
12#include <algorithm>
13
14#ifdef __has_include
15    #if __has_include(<unistd.h>)
16        #include <unistd.h>
17        #include <fcntl.h>
18        #include <sys/stat.h>
19        #if defined(_POSIX_MAPPED_FILES)
20            #include <sys/mman.h>
21        #endif
22        #if defined(_POSIX_MEMLOCK_RANGE)
23            #include <sys/resource.h>
24        #endif
25    #endif
26#endif
27
28#if defined(_WIN32)
29    #define WIN32_LEAN_AND_MEAN
30    #ifndef NOMINMAX
31        #define NOMINMAX
32    #endif
33    #include <windows.h>
34    #ifndef PATH_MAX
35        #define PATH_MAX MAX_PATH
36    #endif
37    #include <io.h>
38#endif
39
40#if defined(__APPLE__)
41#include <TargetConditionals.h>
42#endif
43
44#ifdef _WIN32
45#    define llama_mmap_ftell _ftelli64
46#    define llama_mmap_fseek _fseeki64
47#else
48#    define llama_mmap_ftell ftello
49#    define llama_mmap_fseek fseeko
50#endif
51
52// TODO: consider moving to llama-impl.h if needed in more places
53#if defined(_WIN32)
54static std::string llama_format_win_err(DWORD err) {
55    LPSTR buf;
56    size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
57                                 NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
58    if (!size) {
59        return "FormatMessageA failed";
60    }
61    std::string ret(buf, size);
62    LocalFree(buf);
63    return ret;
64}
65#endif
66
67// llama_file
68
69struct llama_file::impl {
70#if defined(_WIN32)
71    HANDLE fp_win32;
72    std::string GetErrorMessageWin32(DWORD error_code) const {
73        std::string ret;
74        LPSTR lpMsgBuf = NULL;
75        DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
76                                    NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);
77        if (!bufLen) {
78            ret = format("Win32 error code: %lx", error_code);
79        } else {
80            ret = lpMsgBuf;
81            LocalFree(lpMsgBuf);
82        }
83
84        return ret;
85    }
86
87    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {
88        fp = ggml_fopen(fname, mode);
89        if (fp == NULL) {
90            throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
91        }
92        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
93        seek(0, SEEK_END);
94        size = tell();
95        seek(0, SEEK_SET);
96    }
97
98    impl(FILE * file) : owns_fp(false) {
99        fp = file;
100        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
101        seek(0, SEEK_END);
102        size = tell();
103        seek(0, SEEK_SET);
104    }
105
106    size_t tell() const {
107        LARGE_INTEGER li;
108        li.QuadPart = 0;
109        BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);
110        if (!ret) {
111            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
112        }
113
114        return li.QuadPart;
115    }
116
117    void seek(size_t offset, int whence) const {
118        static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");
119        static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");
120        static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");
121
122        LARGE_INTEGER li;
123        li.QuadPart = offset;
124        BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);
125        if (!ret) {
126            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
127        }
128    }
129
130    void read_raw(void * ptr, size_t len) {
131        size_t bytes_read = 0;
132        while (bytes_read < len) {
133            size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);
134            DWORD chunk_read = 0;
135            BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);
136            if (!result) {
137                throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
138            }
139            if (chunk_read < chunk_size || chunk_read == 0) {
140                throw std::runtime_error("unexpectedly reached end of file");
141            }
142
143            bytes_read += chunk_read;
144        }
145    }
146
147    uint32_t read_u32() {
148        uint32_t val;
149        read_raw(&val, sizeof(val));
150        return val;
151    }
152
153    void write_raw(const void * ptr, size_t len) const {
154        size_t bytes_written = 0;
155        while (bytes_written < len) {
156            size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);
157            DWORD chunk_written = 0;
158            BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);
159            if (!result) {
160                throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
161            }
162            if (chunk_written < chunk_size || chunk_written == 0) {
163                throw std::runtime_error("unexpectedly failed to write bytes");
164            }
165
166            bytes_written += chunk_written;
167        }
168    }
169
170    void write_u32(uint32_t val) const {
171        write_raw(&val, sizeof(val));
172    }
173
174    bool has_direct_io() const {
175        return true;
176    }
177
178    ~impl() {
179        if (fp && owns_fp) {
180            std::fclose(fp);
181        }
182    }
183#else
184    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
185#ifdef __linux__
186        // Try unbuffered I/O for read only
187        if (use_direct_io && std::strcmp(mode, "rb") == 0) {
188            if (init_fd()) {
189                return;
190            }
191            LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O",
192                           fname, strerror(errno));
193        }
194#endif
195        init_fp(mode);
196    }
197
198#ifdef __linux__
199    bool init_fd() {
200        fd = open(fname.c_str(), O_RDONLY | O_DIRECT);
201
202        if (fd != -1) {
203            struct stat file_stats{};
204            fstat(fd, &file_stats);
205
206            size = file_stats.st_size;
207            alignment = file_stats.st_blksize;
208
209            off_t ret = lseek(fd, 0, SEEK_SET);
210            if (ret == -1) {
211                throw std::runtime_error(format("seek error: %s", strerror(errno)));
212            }
213            return true;
214        }
215        return false;
216    }
217#endif
218
219    void init_fp(const char * mode) {
220        fp = ggml_fopen(fname.c_str(), mode);
221        if (fp == NULL) {
222            throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno)));
223        }
224        seek(0, SEEK_END);
225        size = tell();
226        seek(0, SEEK_SET);
227    }
228
229    impl(FILE * file) : fname("(file*)"), owns_fp(false) {
230        fp = file;
231        seek(0, SEEK_END);
232        size = tell();
233        seek(0, SEEK_SET);
234    }
235
236    size_t tell() const {
237        if (fd == -1) {
238            off_t ret = llama_mmap_ftell(fp);
239            if (ret == -1) {
240                throw std::runtime_error(format("ftell error: %s", strerror(errno)));
241            }
242
243            return (size_t) ret;
244        }
245
246        off_t pos = lseek(fd, 0, SEEK_CUR);
247        if (pos == -1) {
248            throw std::runtime_error(format("lseek error: %s", strerror(errno)));
249        }
250        return (size_t) pos;
251    }
252
253    void seek(size_t offset, int whence) const {
254        off_t ret = 0;
255        if (fd == -1) {
256            ret = llama_mmap_fseek(fp, offset, whence);
257        } else {
258            ret = lseek(fd, offset, whence);
259        }
260        if (ret == -1) {
261            throw std::runtime_error(format("seek error: %s", strerror(errno)));
262        }
263    }
264
265    void read_raw_unsafe(void * ptr, size_t len) {
266        if (len == 0) {
267            return;
268        }
269        errno = 0;
270        if (fd == -1) {
271            const size_t curr_off = tell();
272            const size_t to_read = std::min(len, size - curr_off);
273
274            std::size_t ret = std::fread(ptr, to_read, 1, fp);
275            if (ferror(fp)) {
276                throw std::runtime_error(format("read error: %s", strerror(errno)));
277            }
278            if (to_read > 0 && ret != 1) {
279                throw std::runtime_error("unexpectedly reached end of file");
280            }
281        } else {
282            size_t bytes_read = 0;
283            while (bytes_read < len) {
284                const size_t to_read = len - bytes_read;
285                ssize_t ret = ::read(fd, reinterpret_cast<char *>(ptr) + bytes_read, to_read);
286
287                if (ret == -1) {
288                    if (errno == EINTR) {
289                        continue;  // Interrupted by signal, retry
290                    }
291                    // Fallback to std::fread in case the DMA controller cannot access the buffer
292                    if (errno == EFAULT || errno == EINVAL) {
293                        LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno));
294                        auto curr_off = tell();
295                        close(fd);
296                        fd = -1;
297                        alignment = 1;
298                        init_fp("rb");
299                        seek(curr_off, SEEK_SET);
300                        read_raw_unsafe(ptr, len);
301                        return;
302                    }
303                    throw std::runtime_error(format("read error: %s", strerror(errno)));
304                }
305                if (ret == 0) {
306                    // EOF: allow if this read was only pulling alignment padding past file end
307                    off_t pos = lseek(fd, 0, SEEK_CUR);
308                    if (pos != -1 && (size_t) pos == size) {
309                        std::memset(reinterpret_cast<char *>(ptr) + bytes_read, 0, len - bytes_read);
310                        return;
311                    }
312                    throw std::runtime_error("unexpectedly reached end of file");
313                }
314
315                bytes_read += (size_t) ret;
316            }
317        }
318    }
319
320    void read_aligned_chunk(void * dest, size_t size) {
321        size_t offset = tell();
322        off_t aligned_offset = offset & ~(alignment - 1);
323        off_t offset_from_alignment = offset - aligned_offset;
324        size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);
325
326        void * raw_buffer = nullptr;
327        int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read);
328        if (ret != 0) {
329            throw std::runtime_error(format("posix_memalign failed with error %d", ret));
330        }
331
332        struct aligned_buffer_deleter {
333            void operator()(void * p) const { free(p); }
334        };
335        std::unique_ptr<void, aligned_buffer_deleter> buffer(raw_buffer);
336
337        seek(aligned_offset, SEEK_SET);
338        read_raw_unsafe(buffer.get(), bytes_to_read);
339
340        uintptr_t actual_data = reinterpret_cast<uintptr_t>(buffer.get()) + offset_from_alignment;
341        memcpy(dest, reinterpret_cast<void *>(actual_data), size);
342    }
343
344    void read_raw(void * ptr, size_t len) {
345        if (has_direct_io()) {
346            read_aligned_chunk(ptr, len);
347        } else {
348            read_raw_unsafe(ptr, len);
349        }
350    }
351
352    uint32_t read_u32() {
353        uint32_t ret;
354        read_raw(&ret, sizeof(ret));
355        return ret;
356    }
357
358    void write_raw(const void * ptr, size_t len) const {
359        if (len == 0) {
360            return;
361        }
362        errno = 0;
363        size_t ret = std::fwrite(ptr, len, 1, fp);
364        if (ret != 1) {
365            throw std::runtime_error(format("write error: %s", strerror(errno)));
366        }
367    }
368
369    void write_u32(uint32_t val) const {
370        write_raw(&val, sizeof(val));
371    }
372
373    bool has_direct_io() const {
374        return fd != -1 && alignment > 1;
375    }
376
377    ~impl() {
378        if (fd != -1) {
379            close(fd);
380        } else if (owns_fp) {
381            std::fclose(fp);
382        }
383    }
384    int fd = -1;
385    std::string fname;
386#endif
387
388    size_t read_alignment() const {
389        return alignment;
390    }
391
392    size_t alignment = 1;
393
394    FILE * fp{};
395    size_t size{};
396    bool owns_fp = true;
397};
398
399llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :
400    pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}
401
402llama_file::llama_file(FILE * file) : pimpl(std::make_unique<impl>(file)) {}
403
404llama_file::~llama_file() = default;
405
406size_t llama_file::tell() const { return pimpl->tell(); }
407size_t llama_file::size() const { return pimpl->size; }
408
409size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
410bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
411
412int llama_file::file_id() const {
413#ifdef _WIN32
414    return _fileno(pimpl->fp);
415#else
416    if (pimpl->fd != -1) {
417        return pimpl->fd;
418    }
419#if defined(fileno)
420    return fileno(pimpl->fp);
421#else
422    return ::fileno(pimpl->fp);
423#endif
424#endif
425}
426
427void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }
428void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }
429#ifdef _WIN32
430void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }
431#else
432void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); }
433#endif
434
435uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
436
437void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
438void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
439
440// llama_mmap
441
442#if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)
443// merge `ranges` and return their complement within [0, limit)
444static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t limit) {
445    llama_mmap::ranges res;
446    std::sort(ranges.begin(), ranges.end());
447
448    size_t pos = 0;
449    for (const auto & range : ranges) {
450        const size_t beg = std::min(range.first,  limit);
451        const size_t end = std::min(range.second, limit);
452        if (beg > pos) {
453            res.emplace_back(pos, beg);
454        }
455        pos = std::max(pos, end);
456    }
457    if (pos < limit) {
458        res.emplace_back(pos, limit);
459    }
460
461    return res;
462}
463#endif
464
465struct llama_mmap::impl {
466#ifdef _POSIX_MAPPED_FILES
467    std::vector<std::pair<size_t, size_t>> mapped_fragments;
468
469    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
470        size = file->size();
471        int fd = file->file_id();
472        int flags = MAP_SHARED;
473        if (numa) { prefetch = 0; }
474#ifdef __linux__
475        if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {
476            LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",
477                    strerror(errno));
478        }
479        // MAP_POPULATE would fault in the lazy ranges too
480        if (prefetch && lazy_ranges.empty()) { flags |= MAP_POPULATE; }
481#endif
482        addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);
483        if (addr == MAP_FAILED) {
484            throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
485        }
486
487        // page-aligned madvise over [beg, end), clamped to the file
488        auto advise = [&](size_t beg, size_t end, int advice, const char * name) {
489            const size_t page_size = sysconf(_SC_PAGESIZE);
490            beg = beg & ~(page_size - 1);
491            end = std::min((end + page_size - 1) & ~(page_size - 1), file->size());
492            if (beg >= end) {
493                return;
494            }
495            if (posix_madvise((char *) addr + beg, end - beg, advice)) {
496                LLAMA_LOG_WARN("warning: posix_madvise(.., %s) failed: %s\n", name, strerror(errno));
497            }
498        };
499
500        if (prefetch > 0) {
501            for (const auto & range : ranges_complement(lazy_ranges, std::min(file->size(), prefetch))) {
502                advise(range.first, range.second, POSIX_MADV_WILLNEED, "POSIX_MADV_WILLNEED");
503            }
504        }
505        for (const auto & range : lazy_ranges) {
506            advise(range.first, range.second, POSIX_MADV_RANDOM, "POSIX_MADV_RANDOM");
507        }
508        if (numa) {
509            if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {
510                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",
511                        strerror(errno));
512            }
513        }
514
515        mapped_fragments.emplace_back(0, file->size());
516    }
517
518    static void align_range(size_t * first, size_t * last, size_t page_size) {
519        size_t offset_in_page = *first & (page_size - 1);
520        size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;
521        *first += offset_to_page;
522
523        *last = *last & ~(page_size - 1);
524
525        if (*last <= *first) {
526            *last = *first;
527        }
528    }
529
530    void unmap_fragment(size_t first, size_t last) {
531        int page_size = sysconf(_SC_PAGESIZE);
532        align_range(&first, &last, page_size);
533        size_t len = last - first;
534
535        if (len == 0) {
536            return;
537        }
538
539        GGML_ASSERT(first % page_size == 0);
540        GGML_ASSERT(last % page_size == 0);
541        GGML_ASSERT(last > first);
542
543        void * next_page_start = (uint8_t *) addr + first;
544
545        if (munmap(next_page_start, len)) {
546            LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
547        }
548
549        std::vector<std::pair<size_t, size_t>> new_mapped_fragments;
550        for (const auto & frag : mapped_fragments) {
551            if (frag.first < first && frag.second > last) {
552                new_mapped_fragments.emplace_back(frag.first, first);
553                new_mapped_fragments.emplace_back(last, frag.second);
554            } else if (frag.first < first && frag.second > first) {
555                new_mapped_fragments.emplace_back(frag.first, first);
556            } else if (frag.first < last && frag.second > last) {
557                new_mapped_fragments.emplace_back(last, frag.second);
558            } else if (frag.first >= first && frag.second <= last) {
559            } else {
560                new_mapped_fragments.push_back(frag);
561            }
562        }
563        mapped_fragments = std::move(new_mapped_fragments);
564    }
565
566    ~impl() {
567        for (const auto & frag : mapped_fragments) {
568            if (munmap((char *) addr + frag.first, frag.second - frag.first)) {
569                LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
570            }
571        }
572    }
573#elif defined(_WIN32)
574    HANDLE hMapping = nullptr;
575
576    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
577        GGML_UNUSED(numa);
578
579        size = file->size();
580
581        HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());
582
583        hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
584
585        if (hMapping == NULL) {
586            DWORD error = GetLastError();
587            throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
588        }
589
590        addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
591        DWORD error = GetLastError();
592
593        if (addr == NULL) {
594            CloseHandle(hMapping);
595            throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));
596        }
597
598        if (prefetch > 0) {
599#if _WIN32_WINNT >= 0x602
600            BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
601            HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
602
603            pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
604
605            if (pPrefetchVirtualMemory) {
606                std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
607                for (const auto & range : ranges_complement(lazy_ranges, std::min(size, prefetch))) {
608                    WIN32_MEMORY_RANGE_ENTRY entry;
609                    entry.VirtualAddress = (char *) addr + range.first;
610                    entry.NumberOfBytes  = (SIZE_T) (range.second - range.first);
611                    entries.push_back(entry);
612                }
613                if (!entries.empty() &&
614                        !pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0)) {
615                    LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",
616                            llama_format_win_err(GetLastError()).c_str());
617                }
618            }
619#else
620            LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");
621#endif
622        }
623    }
624
625    void unmap_fragment(size_t first, size_t last) {
626        GGML_UNUSED(first);
627        GGML_UNUSED(last);
628    }
629
630    ~impl() {
631        if (hMapping) {
632            if (addr) {
633                if (!UnmapViewOfFile(addr)) {
634                    LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",
635                            llama_format_win_err(GetLastError()).c_str());
636                }
637            }
638            if (!CloseHandle(hMapping)) {
639                LLAMA_LOG_WARN("warning: CloseHandle failed: %s\n",
640                        llama_format_win_err(GetLastError()).c_str());
641            }
642        }
643    }
644#else
645    impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
646        GGML_UNUSED(file);
647        GGML_UNUSED(prefetch);
648        GGML_UNUSED(numa);
649        GGML_UNUSED(lazy_ranges);
650
651        throw std::runtime_error("mmap not supported");
652    }
653
654    void unmap_fragment(size_t first, size_t last) {
655        GGML_UNUSED(first);
656        GGML_UNUSED(last);
657
658        throw std::runtime_error("mmap not supported");
659    }
660#endif
661
662    void * addr;
663    size_t size;
664};
665
666llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa,
667        const ranges & lazy_ranges) : pimpl(std::make_unique<impl>(file, prefetch, numa, lazy_ranges)) {}
668llama_mmap::~llama_mmap() = default;
669
670size_t llama_mmap::size() const { return pimpl->size; }
671void * llama_mmap::addr() const { return pimpl->addr; }
672
673void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }
674
675#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
676const bool llama_mmap::SUPPORTED  = true;
677#else
678const bool llama_mmap::SUPPORTED  = false;
679#endif
680
681// llama_mlock
682
683struct llama_mlock::impl {
684#ifdef _POSIX_MEMLOCK_RANGE
685    static size_t lock_granularity() {
686        return (size_t) sysconf(_SC_PAGESIZE);
687    }
688
689    bool raw_lock(const void * addr, size_t size) const {
690        if (!mlock(addr, size)) {
691            return true;
692        }
693
694#ifdef __APPLE__
695#define MLOCK_SUGGESTION \
696        "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
697        "decreasing 'vm.global_no_user_wire_amount'.  Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"
698#else
699#define MLOCK_SUGGESTION \
700        "Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"
701#endif
702
703        char* errmsg = std::strerror(errno);
704        bool suggest = (errno == ENOMEM);
705#if defined(TARGET_OS_VISION) || defined(TARGET_OS_TV) || defined(_AIX) || defined(__HAIKU__)
706        // visionOS/tvOS/Haiku don't support RLIMIT_MEMLOCK
707        // Skip resource limit checks on these platforms
708        suggest = false;
709#else
710        struct rlimit lock_limit;
711        if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {
712            suggest = false;
713        }
714        if (suggest && ((uint64_t)lock_limit.rlim_max > (uint64_t)lock_limit.rlim_cur + size)) {
715            suggest = false;
716        }
717#endif
718
719        LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
720                size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
721        return false;
722    }
723
724    static void raw_unlock(void * addr, size_t size) {
725        if (munlock(addr, size)) {
726            LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));
727        }
728    }
729#elif defined(_WIN32)
730    static size_t lock_granularity() {
731        SYSTEM_INFO si;
732        GetSystemInfo(&si);
733        return (size_t) si.dwPageSize;
734    }
735
736    bool raw_lock(void * ptr, size_t len) const {
737        for (int tries = 1; ; tries++) {
738            if (VirtualLock(ptr, len)) {
739                return true;
740            }
741            if (tries == 2) {
742                LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
743                    len, size, llama_format_win_err(GetLastError()).c_str());
744                return false;
745            }
746
747            SIZE_T min_ws_size, max_ws_size;
748            if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
749                LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",
750                        llama_format_win_err(GetLastError()).c_str());
751                return false;
752            }
753            size_t increment = len + 1048576;
754            min_ws_size += increment;
755            max_ws_size += increment;
756            if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
757                LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",
758                        llama_format_win_err(GetLastError()).c_str());
759                return false;
760            }
761        }
762    }
763
764    static void raw_unlock(void * ptr, size_t len) {
765        if (!VirtualUnlock(ptr, len)) {
766            LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",
767                    llama_format_win_err(GetLastError()).c_str());
768        }
769    }
770#else
771    static size_t lock_granularity() {
772        return (size_t) 65536;
773    }
774
775    bool raw_lock(const void * addr, size_t len) const {
776        LLAMA_LOG_WARN("warning: mlock not supported on this system\n");
777        return false;
778    }
779
780    static void raw_unlock(const void * addr, size_t len) {}
781#endif
782
783    impl() : addr(NULL), size(0), failed_already(false) {}
784
785    void init(void * ptr) {
786        GGML_ASSERT(addr == NULL && size == 0);
787        addr = ptr;
788    }
789
790    void grow_to(size_t target_size) {
791        GGML_ASSERT(addr);
792        if (failed_already) {
793            return;
794        }
795        size_t granularity = lock_granularity();
796        target_size = (target_size + granularity - 1) & ~(granularity - 1);
797        if (target_size > size) {
798            if (raw_lock((uint8_t *) addr + size, target_size - size)) {
799                size = target_size;
800            } else {
801                failed_already = true;
802            }
803        }
804    }
805
806    void * addr;
807    size_t size;
808
809    bool failed_already;
810};
811
812llama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}
813llama_mlock::~llama_mlock() = default;
814
815void llama_mlock::init(void * ptr) { pimpl->init(ptr); }
816void llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }
817
818#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
819const bool llama_mlock::SUPPORTED = true;
820#else
821const bool llama_mlock::SUPPORTED = false;
822#endif
823
824size_t llama_path_max() {
825    return PATH_MAX;
826}