char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for device discovery and create sessions on demand (#27785)9e54e687c

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
18.0 KiB425 linesraw
1#include <filesystem>
2#include <set>
3#include <sstream>
4#include <string>
5
6#ifdef _WIN32
7#   define WIN32_LEAN_AND_MEAN
8#   ifndef NOMINMAX
9#       define NOMINMAX
10#   endif
11#   include <windows.h>
12#   include <winevt.h>
13#else
14#   include <dlfcn.h>
15#   include <unistd.h>
16#endif
17
18#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
19#pragma clang diagnostic ignored "-Wmissing-prototypes"
20#pragma clang diagnostic ignored "-Wsign-compare"
21#pragma clang diagnostic ignored "-Wlanguage-extension-token"
22#pragma clang diagnostic ignored "-Wmicrosoft-enum-value"
23#pragma clang diagnostic ignored "-Wnested-anon-types"
24
25#include "ggml-impl.h"
26#include "htp-drv.h"
27#include "libdl.h"
28
29#include <domain.h>
30
31//
32// Driver API types
33//
34
35typedef void * (*rpcmem_alloc_pfn_t)(int heapid, uint32_t flags, int size);
36typedef void * (*rpcmem_alloc2_pfn_t)(int heapid, uint32_t flags, size_t size);
37typedef void   (*rpcmem_free_pfn_t)(void * po);
38typedef int    (*rpcmem_to_fd_pfn_t)(void * po);
39
40typedef AEEResult (*dspqueue_create_pfn_t)(int                 domain,
41                                           uint32_t            flags,
42                                           uint32_t            req_queue_size,
43                                           uint32_t            resp_queue_size,
44                                           dspqueue_callback_t packet_callback,
45                                           dspqueue_callback_t error_callback,
46                                           void *              callback_context,
47                                           dspqueue_t *        queue);
48typedef AEEResult (*dspqueue_close_pfn_t)(dspqueue_t queue);
49typedef AEEResult (*dspqueue_export_pfn_t)(dspqueue_t queue, uint64_t *queue_id);
50typedef AEEResult (*dspqueue_write_pfn_t)(dspqueue_t queue, uint32_t flags,
51                                          uint32_t num_buffers,
52                                          struct dspqueue_buffer *buffers,
53                                          uint32_t message_length,
54                                          const uint8_t *message,
55                                          uint32_t timeout_us);
56typedef AEEResult (*dspqueue_read_pfn_t)(dspqueue_t queue, uint32_t *flags,
57                                         uint32_t max_buffers, uint32_t *num_buffers,
58                                         struct dspqueue_buffer *buffers,
59                                         uint32_t max_message_length,
60                                         uint32_t *message_length, uint8_t *message,
61                                         uint32_t timeout_us);
62typedef AEEResult (*dspqueue_read_noblock_pfn_t)(dspqueue_t queue, uint32_t *flags,
63                                                 uint32_t max_buffers, uint32_t *num_buffers,
64                                                 struct dspqueue_buffer *buffers,
65                                                 uint32_t max_message_length,
66                                                 uint32_t *message_length, uint8_t *message);
67typedef int (*fastrpc_mmap_pfn_t)(int domain, int fd, void *addr, int offset, size_t length, enum fastrpc_map_flags flags);
68typedef int (*fastrpc_munmap_pfn_t)(int domain, int fd, void *addr, size_t length);
69
70typedef int (*remote_handle64_open_pfn_t)(const char* name, remote_handle64 *ph);
71typedef int (*remote_handle64_invoke_pfn_t)(remote_handle64 h, uint32_t dwScalars, remote_arg *pra);
72typedef int (*remote_handle64_close_pfn_t)(remote_handle h);
73typedef int (*remote_handle_control_pfn_t)(uint32_t req, void* data, uint32_t datalen);
74typedef int (*remote_handle64_control_pfn_t)(remote_handle64 h, uint32_t req, void* data, uint32_t datalen);
75typedef int (*remote_session_control_pfn_t)(uint32_t req, void *data, uint32_t datalen);
76typedef int (*remote_system_request_pfn_t)(system_req_payload * req);
77
78//
79// Driver API pfns
80//
81
82rpcmem_alloc_pfn_t  rpcmem_alloc_pfn  = nullptr;
83rpcmem_alloc2_pfn_t rpcmem_alloc2_pfn = nullptr;
84rpcmem_free_pfn_t   rpcmem_free_pfn   = nullptr;
85rpcmem_to_fd_pfn_t  rpcmem_to_fd_pfn  = nullptr;
86
87fastrpc_mmap_pfn_t   fastrpc_mmap_pfn   = nullptr;
88fastrpc_munmap_pfn_t fastrpc_munmap_pfn = nullptr;
89
90dspqueue_create_pfn_t       dspqueue_create_pfn       = nullptr;
91dspqueue_close_pfn_t        dspqueue_close_pfn        = nullptr;
92dspqueue_export_pfn_t       dspqueue_export_pfn       = nullptr;
93dspqueue_write_pfn_t        dspqueue_write_pfn        = nullptr;
94dspqueue_read_pfn_t         dspqueue_read_pfn         = nullptr;
95dspqueue_read_noblock_pfn_t dspqueue_read_noblock_pfn = nullptr;
96
97remote_handle64_open_pfn_t    remote_handle64_open_pfn    = nullptr;
98remote_handle64_invoke_pfn_t  remote_handle64_invoke_pfn  = nullptr;
99remote_handle64_close_pfn_t   remote_handle64_close_pfn   = nullptr;
100remote_handle_control_pfn_t   remote_handle_control_pfn   = nullptr;
101remote_handle64_control_pfn_t remote_handle64_control_pfn = nullptr;
102remote_session_control_pfn_t  remote_session_control_pfn  = nullptr;
103remote_system_request_pfn_t   remote_system_request_pfn   = nullptr;
104
105//
106// Driver API
107//
108
109void * rpcmem_alloc(int heapid, uint32_t flags, int size) {
110    return rpcmem_alloc_pfn(heapid, flags, size);
111}
112
113void * rpcmem_alloc2(int heapid, uint32_t flags, size_t size) {
114    if (rpcmem_alloc2_pfn) {
115        return rpcmem_alloc2_pfn(heapid, flags, size);
116    } else {
117        GGML_LOG_INFO("ggml-hex: rpcmem_alloc2 not found, falling back to rpcmem_alloc\n");
118        return rpcmem_alloc_pfn(heapid, flags, size);
119    }
120}
121
122void rpcmem_free(void * po) {
123    return rpcmem_free_pfn(po);
124}
125
126int rpcmem_to_fd(void * po) {
127    return rpcmem_to_fd_pfn(po);
128}
129
130HTPDRV_API int fastrpc_mmap(int domain, int fd, void * addr, int offset, size_t length, enum fastrpc_map_flags flags) {
131    return fastrpc_mmap_pfn(domain, fd, addr, offset, length, flags);
132}
133
134HTPDRV_API int fastrpc_munmap(int domain, int fd, void * addr, size_t length) {
135    return fastrpc_munmap_pfn(domain, fd, addr, length);
136}
137
138AEEResult dspqueue_create(int                 domain,
139                          uint32_t            flags,
140                          uint32_t            req_queue_size,
141                          uint32_t            resp_queue_size,
142                          dspqueue_callback_t packet_callback,
143                          dspqueue_callback_t error_callback,
144                          void *              callback_context,
145                          dspqueue_t *        queue) {
146    return dspqueue_create_pfn(domain, flags, req_queue_size, resp_queue_size, packet_callback, error_callback,
147                               callback_context, queue);
148}
149
150AEEResult dspqueue_close(dspqueue_t queue) {
151    return dspqueue_close_pfn(queue);
152}
153
154AEEResult dspqueue_export(dspqueue_t queue, uint64_t * queue_id) {
155    return dspqueue_export_pfn(queue, queue_id);
156}
157
158AEEResult dspqueue_write(dspqueue_t               queue,
159                         uint32_t                 flags,
160                         uint32_t                 num_buffers,
161                         struct dspqueue_buffer * buffers,
162                         uint32_t                 message_length,
163                         const uint8_t *          message,
164                         uint32_t                 timeout_us) {
165    return dspqueue_write_pfn(queue, flags, num_buffers, buffers, message_length, message, timeout_us);
166}
167
168AEEResult dspqueue_read(dspqueue_t               queue,
169                        uint32_t *               flags,
170                        uint32_t                 max_buffers,
171                        uint32_t *               num_buffers,
172                        struct dspqueue_buffer * buffers,
173                        uint32_t                 max_message_length,
174                        uint32_t *               message_length,
175                        uint8_t *                message,
176                        uint32_t                 timeout_us) {
177#ifdef _WIN32
178    if (timeout_us == 0) {
179        return dspqueue_read_noblock_pfn(queue, flags, max_buffers, num_buffers, buffers, max_message_length,
180                                     message_length, message);
181    }
182#endif
183    return dspqueue_read_pfn(queue, flags, max_buffers, num_buffers, buffers, max_message_length, message_length,
184                             message, timeout_us);
185}
186
187HTPDRV_API int remote_handle64_open(const char * name, remote_handle64 * ph) {
188    return remote_handle64_open_pfn(name, ph);
189}
190
191HTPDRV_API int remote_handle64_invoke(remote_handle64 h, uint32_t dwScalars, remote_arg * pra) {
192    return remote_handle64_invoke_pfn(h, dwScalars, pra);
193}
194
195HTPDRV_API int remote_handle64_close(remote_handle64 h) {
196    return remote_handle64_close_pfn(h);
197}
198
199HTPDRV_API int remote_handle_control(uint32_t req, void * data, uint32_t datalen) {
200    return remote_handle_control_pfn(req, data, datalen);
201}
202
203HTPDRV_API int remote_handle64_control(remote_handle64 h, uint32_t req, void * data, uint32_t datalen) {
204    return remote_handle64_control_pfn(h, req, data, datalen);
205}
206
207HTPDRV_API int remote_session_control(uint32_t req, void * data, uint32_t datalen) {
208    return remote_session_control_pfn(req, data, datalen);
209}
210
211HTPDRV_API int remote_system_request(system_req_payload * req) {
212    if (!remote_system_request_pfn) {
213        return AEE_EUNSUPPORTEDAPI;
214    }
215    return remote_system_request_pfn(req);
216}
217
218#ifdef _WIN32
219
220static std::string wstr_to_str(std::wstring_view wstr) {
221    std::string result;
222    if (wstr.empty()) {
223        return result;
224    }
225    auto bytes_needed = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS,
226                                            wstr.data(), (int) wstr.size(),
227                                            nullptr, 0, nullptr, nullptr);
228    if (bytes_needed == 0) {
229        GGML_LOG_ERROR("ggml-hex: WideCharToMultiByte failed. Error %lu\n", GetLastError());
230        throw std::runtime_error("Invalid wstring input");
231    }
232
233    result.resize(bytes_needed, '\0');
234    int bytes_written = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS,
235                                            wstr.data(), (int) wstr.size(),
236                                            result.data(), bytes_needed,
237                                            nullptr, nullptr);
238    if (bytes_written == 0) {
239        GGML_LOG_ERROR("ggml-hex: WideCharToMultiByte failed. Error %lu\n", GetLastError());
240        throw std::runtime_error("Wstring conversion failed");
241    }
242    return result;
243}
244
245static std::string get_driver_path() {
246    std::wstring serviceName = L"qcnspmcdm";
247    std::string result;
248
249    // Get a handle to the SCM database.
250    SC_HANDLE schSCManager = OpenSCManagerW(NULL, NULL, STANDARD_RIGHTS_READ);
251    if (nullptr == schSCManager) {
252        GGML_LOG_ERROR("ggml-hex: Failed to open SCManager. Error: %lu\n", GetLastError());
253        return result;
254    }
255
256    // Get a handle to the service.
257    SC_HANDLE schService = OpenServiceW(schSCManager,           // SCM database
258                                        serviceName.c_str(),    // name of service
259                                        SERVICE_QUERY_CONFIG);  // need query config access
260
261    if (nullptr == schService) {
262        GGML_LOG_ERROR("ggml-hex: Failed to open qcnspmcdm service. Error: %lu\n", GetLastError());
263        CloseServiceHandle(schSCManager);
264        return result;
265    }
266
267    // Store the size of buffer used as an output.
268    DWORD bufferSize;
269    if (!QueryServiceConfigW(schService, NULL, 0, &bufferSize) &&
270        (GetLastError() != ERROR_INSUFFICIENT_BUFFER)) {
271        GGML_LOG_ERROR("ggml-hex: Failed to query service config. Error: %lu\n", GetLastError());
272        CloseServiceHandle(schService);
273        CloseServiceHandle(schSCManager);
274        return result;
275    }
276    // Get the configuration of the service.
277    LPQUERY_SERVICE_CONFIGW serviceConfig =
278        static_cast<LPQUERY_SERVICE_CONFIGW>(LocalAlloc(LMEM_FIXED, bufferSize));
279    if (!QueryServiceConfigW(schService, serviceConfig, bufferSize, &bufferSize)) {
280        fprintf(stderr, "ggml-hex: Failed to query service config. Error: %lu\n", GetLastError());
281        LocalFree(serviceConfig);
282        CloseServiceHandle(schService);
283        CloseServiceHandle(schSCManager);
284        return result;
285    }
286
287    // Read the driver file path get its parent directory
288    std::wstring driverPath = std::wstring(serviceConfig->lpBinaryPathName);
289    driverPath = driverPath.substr(0, driverPath.find_last_of(L"\\"));
290
291    // Clean up resources
292    LocalFree(serviceConfig);
293    CloseServiceHandle(schService);
294    CloseServiceHandle(schSCManager);
295
296    // Driver path would contain invalid path string, like:
297    // \SystemRoot\System32\DriverStore\FileRepository\qcadsprpc8280.inf_arm64_c2b9460c9a072f37
298    // "\SystemRoot" should be replace with a correct one (e.g. C:\Windows)
299    const std::wstring systemRootPlaceholder = L"\\SystemRoot";
300    if (0 != driverPath.compare(0, systemRootPlaceholder.length(), systemRootPlaceholder)) {
301        GGML_LOG_ERROR("ggml-hex: String pattern not found in driver path.\n");
302        return result;
303    }
304
305    // Replace \SystemRoot with an absolute path from system ENV windir
306    const std::wstring systemRootEnv = L"windir";
307
308    // Query the number of wide characters this variable requires
309    DWORD numWords = GetEnvironmentVariableW(systemRootEnv.c_str(), NULL, 0);
310    if (numWords == 0) {
311        GGML_LOG_ERROR("ggml-hex: Failed get systemRoot environment variable\n");
312        return result;
313    }
314
315    // Query the actual system root name from environment variable
316    std::vector<wchar_t> systemRoot(numWords + 1);
317    numWords = GetEnvironmentVariableW(systemRootEnv.c_str(), systemRoot.data(), numWords + 1);
318    if (numWords == 0) {
319        GGML_LOG_ERROR("ggml-hex: Failed to read windir environment variable\n");
320        return result;
321    }
322    driverPath.replace(0, systemRootPlaceholder.length(), std::wstring(systemRoot.data()));
323
324    return wstr_to_str(driverPath);
325}
326
327#endif
328
329using dl_handle_ptr = std::unique_ptr<dl_handle, dl_handle_deleter>;
330
331int htpdrv_init() {
332    static dl_handle_ptr lib_cdsp_rpc_handle = nullptr;
333    static bool initialized = false;
334#ifdef _WIN32
335    std::string drv_path = get_driver_path() + "\\" + "libcdsprpc.dll";
336#else
337    std::string drv_path = "libcdsprpc.so";
338#endif
339    if (initialized) {
340        GGML_LOG_INFO("ggml-hex: Driver already loaded\n");
341        return AEE_SUCCESS;
342    }
343    GGML_LOG_INFO("ggml-hex: Loading driver %s\n", drv_path.c_str());
344
345    fs::path path{ drv_path.c_str() };
346    dl_handle_ptr handle { dl_load_library(path) };
347    if (!handle) {
348        GGML_LOG_ERROR("ggml-hex: failed to load %s: %s\n", path.u8string().c_str(), dl_error());
349        return AEE_EUNABLETOLOAD;
350    }
351
352#define dlsym(drv, type, pfn, symbol, ignore)                               \
353    do {                                                                    \
354        pfn = (type) dl_get_sym(drv, #symbol);                              \
355        if (!ignore && nullptr == pfn) {                                    \
356            GGML_LOG_ERROR("ggml-hex: failed to dlsym %s\n", #symbol);      \
357            return AEE_EUNABLETOLOAD;                                       \
358        }                                                                   \
359    } while (0)
360
361    dlsym(handle.get(), rpcmem_alloc_pfn_t, rpcmem_alloc_pfn, rpcmem_alloc, false);
362    dlsym(handle.get(), rpcmem_alloc2_pfn_t, rpcmem_alloc2_pfn, rpcmem_alloc2, true);
363    dlsym(handle.get(), rpcmem_free_pfn_t, rpcmem_free_pfn, rpcmem_free, false);
364    dlsym(handle.get(), rpcmem_to_fd_pfn_t, rpcmem_to_fd_pfn, rpcmem_to_fd, false);
365    dlsym(handle.get(), fastrpc_mmap_pfn_t, fastrpc_mmap_pfn, fastrpc_mmap, false);
366    dlsym(handle.get(), fastrpc_munmap_pfn_t, fastrpc_munmap_pfn, fastrpc_munmap, false);
367    dlsym(handle.get(), dspqueue_create_pfn_t, dspqueue_create_pfn, dspqueue_create, false);
368    dlsym(handle.get(), dspqueue_close_pfn_t, dspqueue_close_pfn, dspqueue_close, false);
369    dlsym(handle.get(), dspqueue_export_pfn_t, dspqueue_export_pfn, dspqueue_export, false);
370    dlsym(handle.get(), dspqueue_write_pfn_t, dspqueue_write_pfn, dspqueue_write, false);
371    dlsym(handle.get(), dspqueue_read_pfn_t, dspqueue_read_pfn, dspqueue_read, false);
372    dlsym(handle.get(), dspqueue_read_noblock_pfn_t, dspqueue_read_noblock_pfn, dspqueue_read_noblock, false);
373    dlsym(handle.get(), remote_handle64_open_pfn_t, remote_handle64_open_pfn, remote_handle64_open, false);
374    dlsym(handle.get(), remote_handle64_invoke_pfn_t, remote_handle64_invoke_pfn, remote_handle64_invoke, false);
375    dlsym(handle.get(), remote_handle_control_pfn_t, remote_handle_control_pfn, remote_handle_control, false);
376    dlsym(handle.get(), remote_handle64_control_pfn_t, remote_handle64_control_pfn, remote_handle64_control, false);
377    dlsym(handle.get(), remote_session_control_pfn_t, remote_session_control_pfn, remote_session_control, false);
378    dlsym(handle.get(), remote_handle64_close_pfn_t, remote_handle64_close_pfn, remote_handle64_close, false);
379    dlsym(handle.get(), remote_system_request_pfn_t, remote_system_request_pfn, remote_system_request, true);
380
381    lib_cdsp_rpc_handle = std::move(handle);
382    initialized         = true;
383
384    return AEE_SUCCESS;
385}
386
387domain * htpdrv_get_domain(int domain_id) {
388    int i    = 0;
389    int size = sizeof(supported_domains) / sizeof(domain);
390
391    for (i = 0; i < size; i++) {
392        if (supported_domains[i].id == domain_id) {
393            return &supported_domains[i];
394        }
395    }
396
397    return NULL;
398}
399
400int htpdrv_get_arch(int domain, int * arch) {
401    if (!remote_handle_control_pfn) {
402        GGML_LOG_ERROR("ggml-hex: remote_handle_control is not supported on this device\n");
403        return AEE_EUNSUPPORTEDAPI;
404    }
405
406    struct remote_dsp_capability arch_ver;
407    arch_ver.domain       = (uint32_t) domain;
408    arch_ver.attribute_ID = ARCH_VER;
409    arch_ver.capability   = (uint32_t) 0;
410
411    int err = remote_handle_control(DSPRPC_GET_DSP_INFO, &arch_ver, sizeof(arch_ver));
412    if ((err & 0xff) == (AEE_EUNSUPPORTEDAPI & 0xff)) {
413        GGML_LOG_ERROR("ggml-hex: FastRPC capability API is not supported on this device\n");
414        return AEE_EUNSUPPORTEDAPI;
415    }
416
417    if (err != AEE_SUCCESS) {
418        GGML_LOG_ERROR("ggml-hex: FastRPC capability query failed (err %d)\n", err);
419        return err;
420    }
421
422    uint32_t val = arch_ver.capability & 0xff;
423    *arch = (int) ((val >> 4) * 10 + (val & 0x0f));
424    return 0;
425}