char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Christian Kastnerdocs : Minor cleanups (#19252)7a4ca3cbd

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
4.4 KiB123 linesraw
1/*
2 * Copyright (c) 2023-2026 The ggml authors
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to
6 * deal in the Software without restriction, including without limitation the
7 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8 * sell copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20 * IN THE SOFTWARE.
21 */
22
23#pragma once
24
25#include "ggml-backend.h"
26#include "ggml.h"
27
28#ifdef __cplusplus
29extern "C" {
30#endif
31
32/**
33 * @brief Maximum number of CANN devices supported.
34 */
35#define GGML_CANN_MAX_DEVICES 16
36
37GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cann_reg(void);
38
39/**
40 * @brief Initializes the CANN backend for a specified device.
41 *
42 * This function initializes the CANN backend for the given device.
43 * It verifies the device index, allocates a context, and creates a backend
44 * instance.
45 *
46 * @param device The index of the device to initialize.
47 * @return A pointer to the initialized backend instance, or nullptr on failure.
48 */
49GGML_BACKEND_API ggml_backend_t ggml_backend_cann_init(int32_t device);
50
51/**
52 * @brief Checks if a given backend is a CANN backend.
53 *
54 * This function verifies if the provided backend is a CANN backend by comparing
55 * its GUID with the CANN backend's GUID.
56 *
57 * @param backend The backend instance to check.
58 * @return True if the backend is a CANN backend, false otherwise.
59 */
60GGML_BACKEND_API bool ggml_backend_is_cann(ggml_backend_t backend);
61
62/**
63 * @brief Retrieves the CANN buffer type for a specified device.
64 *
65 * This function initializes and returns the buffer type interface associated
66 * with the given device. It ensures thread-safe access using a mutex.
67 *
68 * @param device The device index for which to retrieve the buffer type.
69 * @return A pointer to the buffer type interface for the specified device, or
70 * nullptr if the device index is out of range.
71 */
72GGML_BACKEND_API ggml_backend_buffer_type_t
73ggml_backend_cann_buffer_type(int32_t device);
74
75/**
76 * @brief Retrieves the number of CANN devices available.
77 *
78 * This function returns the number of CANN devices available based on
79 * information obtained from `ggml_cann_info()`.
80 *
81 * @return The number of CANN devices available.
82 */
83GGML_BACKEND_API int32_t ggml_backend_cann_get_device_count(void);
84
85/**
86 * @brief pinned host buffer for use with the CPU backend for faster copies between CPU and NPU.
87 *
88 * @return A pointer to the host buffer type interface.
89 */
90GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cann_host_buffer_type(void);
91
92/**
93 * @brief Retrieves the description of a specific CANN device.
94 *
95 * This function sets the specified device, retrieves the SoC name,
96 * and writes it into the provided description buffer.
97 *
98 * @param device The device index to retrieve the description for.
99 * @param description Pointer to a buffer where the description will be written.
100 * @param description_size Size of the description buffer.
101 */
102GGML_BACKEND_API void ggml_backend_cann_get_device_description(
103    int32_t device, char* description, size_t description_size);
104
105/**
106 * @brief Retrieves the memory information of a specific CANN device.
107 *
108 * This function sets the specified device, retrieves the free and total
109 * memory information of the specified type (ACL_HBM_MEM), and stores them
110 * in the provided pointers.
111 *
112 * @param device The device index to retrieve memory information for.
113 * @param free Pointer to a variable where the free memory size will be stored.
114 * @param total Pointer to a variable where the total memory size will be
115 * stored.
116 */
117GGML_BACKEND_API void ggml_backend_cann_get_device_memory(int32_t device,
118                                                  size_t* free,
119                                                  size_t* total);
120
121#ifdef __cplusplus
122}
123#endif