char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Neo Zhangsycl : Fix get mem error (#28227)b6b003d2c

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
5.1 KiB151 linesraw
1#include <sycl/ext/oneapi/backend/level_zero.hpp>
2#include <sycl/sycl.hpp>
3
4#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
5#include <level_zero/ze_api.h>
6#include <level_zero/zes_api.h>
7#endif
8
9#include <cstdint>
10#include <iostream>
11#include <vector>
12
13#include "base.hpp"
14#include "mem.hpp"
15
16const char * mem_api_int2str(int mem_api) {
17    if (mem_api == MEMORY_API_TYPE_SYCL) {
18        return "SYCL API";
19    } else if (mem_api == MEMORY_API_TYPE_LEVEL_ZERO) {
20        return "Level Zero API";
21    } else {
22        return "Unknown";
23    }
24}
25
26#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
27/*
28* Depend on to call zesInit(0) before any other Level Zero API calls, otherwise the Level Zero API calls may fail.
29*/
30bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & total_bytes) {
31    GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using Level Zero API.\n", __func__);
32
33    free_bytes  = 0;
34    total_bytes = 0;
35
36    uint32_t module_count = 0;
37
38#if defined(SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO)
39    constexpr sycl::backend kL0Backend = sycl::backend::ext_oneapi_level_zero;
40#else
41    constexpr sycl::backend kL0Backend = sycl::backend::level_zero;
42#endif
43
44    try {
45
46        if (dev.get_platform().get_backend() != kL0Backend) {
47            GGML_SYCL_DEBUG("Device backend is not Level Zero.\n");
48            return false;
49        }
50
51        ze_device_handle_t ze_dev = sycl::get_native<kL0Backend>(dev);
52        if (ze_dev == nullptr) {
53            GGML_SYCL_DEBUG("Level Zero device handle is null.\n");
54            return false;
55        }
56
57        ze_result_t r = zesDeviceEnumMemoryModules(ze_dev, &module_count, nullptr);
58        if (r != ZE_RESULT_SUCCESS || module_count == 0) {
59            GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules.\n");
60            return false;
61        }
62
63        std::vector<zes_mem_handle_t> modules(module_count);
64        r = zesDeviceEnumMemoryModules(ze_dev, &module_count, modules.data());
65        if (r != ZE_RESULT_SUCCESS || module_count == 0) {
66            GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules.\n");
67            return false;
68        }
69
70        for (uint32_t i = 0; i < module_count; ++i) {
71            zes_mem_state_t state = {};
72            state.stype           = ZES_STRUCTURE_TYPE_MEM_STATE;
73            state.pNext           = nullptr;
74
75            r = zesMemoryGetState(modules[i], &state);
76            if (r != ZE_RESULT_SUCCESS) {
77                continue;
78            }
79
80            free_bytes += state.free;
81            total_bytes += state.size;
82        }
83
84        if (total_bytes == 0) {
85            GGML_SYCL_DEBUG("Level Zero memory query returned zero total bytes.\n");
86            return false;
87        }
88        return total_bytes >= free_bytes;
89
90    } catch (const sycl::exception & e) {
91        GGML_SYCL_DEBUG("Level Zero memory query failed: %s\n", e.what());
92        return false;
93    }
94}
95#endif
96
97bool get_memory_size_by_sycl_api(sycl::device dev, size_t & free_bytes, size_t & total_bytes) {
98    GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using SYCL API.\n", __func__);
99    total_bytes = dev.get_info<sycl::info::device::global_mem_size>();
100
101#if (defined(__SYCL_COMPILER_VERSION) && __SYCL_COMPILER_VERSION >= 20221105)
102    if (dev.has(sycl::aspect::ext_intel_free_memory)) {
103        try {
104            GGML_SYCL_DEBUG("Querying free memory using SYCL aspect::ext_intel_free_memory.\n");
105            free_bytes = dev.get_info<sycl::ext::intel::info::device::free_memory>();
106            return true;
107        } catch (const sycl::exception &) {
108            GGML_SYCL_DEBUG(
109                "Failed to query free memory using SYCL aspect::ext_intel_free_memory.\n");
110            return false;
111        }
112    } else {
113        GGML_SYCL_DEBUG(
114            "Device does not support SYCL aspect::ext_intel_free_memory.\n");
115    }
116#else
117    GGML_SYCL_DEBUG("SYCL Compiler version is older than 20221105.\n");
118#endif
119    return false;
120}
121
122bool get_memory_size(sycl::device dev, size_t & free_bytes, size_t & total_bytes, MemoryAPIType api_type) {
123
124    GGML_SYCL_DEBUG("[%s]GPU Name:          %s\n", __func__,
125        dev.get_info<sycl::info::device::name>().c_str());
126    GGML_SYCL_DEBUG("[%s]GPU Vendor:        %s\n", __func__,
127        dev.get_info<sycl::info::device::vendor>().c_str());
128
129    if (api_type == MEMORY_API_TYPE_LEVEL_ZERO) {
130#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
131        GGML_SYCL_DEBUG("[%s] Querying free memory using Level Zero API.\n", __func__);
132        if (query_free_memory_by_ze(dev, free_bytes, total_bytes)) {
133            return true;
134        }
135        //fallback to SYCL API if Level Zero API fails
136        GGML_SYCL_DEBUG("[%s] Falling back to SYCL API for memory query.\n", __func__);
137#endif
138    }
139
140    //MEMORY_API_TYPE_SYCL
141    if(get_memory_size_by_sycl_api(dev, free_bytes, total_bytes)){
142        return true;
143    }
144
145    //Todo, fallback to other methods to get free memory size, such as using OS-specific APIs (e.g., /proc/meminfo on Linux, GlobalMemoryStatusEx on Windows, etc.)
146    GGML_SYCL_DEBUG(
147        "[%s] Can't get free mem size by Level Zero and SYCL API. Using total memory as free memory.\n", __func__);
148    free_bytes = total_bytes;
149
150    return true;
151}