char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Zijun YuOpenVINO: Update OV to 2026.3.1, whisper.cpp support, Qwen3.5 on NPU, and new ops (#27843)511f9c137

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
7.9 KiB237 linesraw
1ARG OPENVINO_VERSION_MAJOR=2026.3.1
2ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d
3ARG UBUNTU_VERSION=24.04
4
5# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
6ARG IGC_VERSION=v2.40.13
7ARG IGC_VERSION_FULL=2_2.40.13+22418
8ARG COMPUTE_RUNTIME_VERSION=26.31.39395.13
9ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
10ARG IGDGMM_VERSION=22.10.0
11
12# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
13ARG NPU_DRIVER_VERSION=v1.35.0
14ARG NPU_DRIVER_FULL=v1.35.0.20260722-29947505341
15ARG LIBZE1_VERSION=1.28.2-1~24.04~ppa1
16
17# Optional proxy build arguments
18ARG http_proxy=
19ARG https_proxy=
20
21ARG BUILD_DATE=N/A
22ARG APP_VERSION=N/A
23ARG APP_REVISION=N/A
24
25ARG NODE_VERSION=24
26
27FROM docker.io/node:$NODE_VERSION AS web
28
29ARG APP_VERSION
30
31WORKDIR /app/tools/ui
32
33COPY tools/ui/package.json tools/ui/package-lock.json ./
34RUN npm ci
35
36COPY tools/ui/ ./
37RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
38
39## Build Image
40FROM docker.io/ubuntu:${UBUNTU_VERSION} AS build
41
42# Pass proxy args to build stage
43ARG http_proxy
44ARG https_proxy
45
46RUN apt-get update && \
47    apt-get install -y --no-install-recommends \
48        ca-certificates \
49        gnupg \
50        wget \
51        git \
52        cmake \
53        ninja-build \
54        build-essential \
55        libtbb12 \
56        libssl-dev \
57        ocl-icd-opencl-dev \
58        opencl-headers \
59        opencl-clhpp-headers \
60        intel-opencl-icd && \
61    rm -rf /var/lib/apt/lists/*
62
63# OpenVINO toolkit and GPU/NPU drivers are cached via BuildKit cache mounts to avoid re-downloading on rebuilds.
64# Install OpenVINO for Ubuntu 24.04.
65ARG OPENVINO_VERSION_MAJOR
66ARG OPENVINO_VERSION_FULL
67RUN --mount=type=cache,target=/var/cache/openvino,sharing=locked \
68    mkdir -p /opt/intel && \
69    TGZ=/var/cache/openvino/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz && \
70    if [ ! -f "$TGZ" ]; then \
71        wget -O "$TGZ" https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION_MAJOR}/linux/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz; \
72    fi && \
73    tar -xf "$TGZ" -C /opt/intel/ && \
74    mv /opt/intel/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64 /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} && \
75    cd /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} && \
76    echo "Y" | ./install_dependencies/install_openvino_dependencies.sh && \
77    cd - && \
78    ln -s /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} /opt/intel/openvino
79
80ENV OpenVINO_DIR=/opt/intel/openvino
81
82WORKDIR /app
83
84COPY . .
85
86COPY --from=web /app/tools/ui/dist tools/ui/dist
87
88# Build Stage
89RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
90    cmake -B build/ReleaseOV -G Ninja \
91        -DCMAKE_BUILD_TYPE=Release \
92        -DLLAMA_BUILD_TESTS=OFF \
93        -DGGML_NATIVE=OFF \
94        -DGGML_BACKEND_DL=ON \
95        -DGGML_CPU_ALL_VARIANTS=ON \
96        -DGGML_OPENVINO=ON && \
97    cmake --build build/ReleaseOV --parallel "
98
99# Copy all necessary libraries (build outputs + OpenVINO runtime libs)
100RUN mkdir -p /app/lib && \
101    find build/ReleaseOV -name '*.so*' -exec cp -P {} /app/lib \; && \
102    find "${OpenVINO_DIR}/runtime/lib/intel64" -name '*.so*' -exec cp -P {} /app/lib \;
103
104# Create runtime directories and copy binaries
105RUN mkdir -p /app/full \
106    && cp build/ReleaseOV/bin/* /app/full/ \
107    && cp *.py /app/full \
108    && cp -r conversion /app/full \
109    && cp -r gguf-py /app/full \
110    && cp -r requirements /app/full \
111    && cp requirements.txt /app/full \
112    && cp .devops/tools.sh /app/full/tools.sh
113
114## Base Runtime Image
115FROM docker.io/ubuntu:${UBUNTU_VERSION} AS base
116
117# Pass proxy args to runtime stage
118ARG http_proxy
119ARG https_proxy
120ARG BUILD_DATE=N/A
121ARG APP_VERSION=N/A
122ARG APP_REVISION=N/A
123ARG IMAGE_URL=https://github.com/ggml-org/llama.cpp
124ARG IMAGE_SOURCE=https://github.com/ggml-org/llama.cpp
125LABEL org.opencontainers.image.created=$BUILD_DATE \
126      org.opencontainers.image.version=$APP_VERSION \
127      org.opencontainers.image.revision=$APP_REVISION \
128      org.opencontainers.image.title="llama.cpp" \
129      org.opencontainers.image.description="LLM inference in C/C++" \
130      org.opencontainers.image.url=$IMAGE_URL \
131      org.opencontainers.image.source=$IMAGE_SOURCE
132
133RUN apt-get update \
134    && apt-get install -y libgomp1 libtbb12 curl wget ffmpeg ocl-icd-libopencl1 \
135    && apt autoremove -y \
136    && apt clean -y \
137    && rm -rf /tmp/* /var/tmp/* \
138    && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \
139    && find /var/cache -type f -delete
140
141# Install GPU drivers
142ARG IGC_VERSION
143ARG IGC_VERSION_FULL
144ARG COMPUTE_RUNTIME_VERSION
145ARG COMPUTE_RUNTIME_VERSION_FULL
146ARG IGDGMM_VERSION
147RUN --mount=type=cache,target=/var/cache/intel-gpu,sharing=locked \
148    set -eux; \
149    cd /var/cache/intel-gpu; \
150    for url in \
151        https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \
152        https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \
153        https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-ocloc_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \
154        https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-opencl-icd_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \
155        https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libigdgmm12_${IGDGMM_VERSION}_amd64.deb \
156        https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb ; do \
157        f=$(basename "$url"); \
158        [ -f "$f" ] || wget -q -O "$f" "$url"; \
159    done; \
160    apt-get update; \
161    apt-get install -y --no-install-recommends ./*.deb; \
162    rm -rf /var/lib/apt/lists/*
163
164# Install NPU drivers
165ARG NPU_DRIVER_VERSION
166ARG NPU_DRIVER_FULL
167ARG LIBZE1_VERSION
168RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \
169    set -eux; \
170    TGZ=/var/cache/intel-npu/linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz; \
171    if [ ! -f "$TGZ" ]; then \
172        wget -q -O "$TGZ" https://github.com/intel/linux-npu-driver/releases/download/${NPU_DRIVER_VERSION}/linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz; \
173    fi; \
174    DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \
175    if [ ! -f "$DEB" ]; then \
176        wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260606T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
177    fi; \
178    mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \
179    apt-get update; \
180    apt-get install -y --no-install-recommends ./*.deb; \
181    rm -rf /tmp/npu/ /var/lib/apt/lists/*
182
183COPY --from=build /app/lib/ /app/
184
185### Full (all binaries)
186FROM base AS full
187
188ARG http_proxy
189ARG https_proxy
190
191COPY --from=build /app/full /app/
192
193WORKDIR /app
194
195RUN apt-get update && \
196    apt-get install -y --no-install-recommends \
197    git \
198    python3 \
199    python3-venv \
200    python3-pip && \
201    python3 -m venv /openvino-venv && \
202    /openvino-venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel && \
203    /openvino-venv/bin/pip install --no-cache-dir -r requirements.txt && \
204    apt-get autoremove -y && \
205    apt-get clean && \
206    rm -rf /tmp/* /var/tmp/* && \
207    find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete && \
208    find /var/cache -type f -delete
209
210# Activate the venv
211ENV VIRTUAL_ENV=/openvino-venv \
212    PATH=/openvino-venv/bin:$PATH
213
214ENTRYPOINT ["/app/tools.sh"]
215
216
217### Light, CLI only
218FROM base AS light
219
220COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
221
222WORKDIR /app
223
224ENTRYPOINT [ "/app/llama-cli" ]
225
226### Server, Server only
227FROM base AS server
228
229ENV LLAMA_ARG_HOST=0.0.0.0
230
231COPY --from=build /app/full/llama /app/full/llama-server /app/
232
233WORKDIR /app
234
235HEALTHCHECK CMD [ "curl", "-f", "http://localhost:8080/health" ]
236
237ENTRYPOINT [ "/app/llama-server" ]