char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Daniel Beveniusci : add PYTEST_WORKERS=1 to fix server-self-hosted job (#28603)1744c6bde

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
7.4 KiB252 linesraw
1name: Server (self-hosted)
2
3on:
4  workflow_dispatch: # allows manual triggering
5    inputs:
6      sha:
7        description: 'Commit SHA1 to build'
8        required: false
9        type: string
10      slow_tests:
11        description: 'Run slow tests'
12        required: true
13        type: boolean
14  push:
15    branches:
16      - master
17    paths: [
18      '.github/workflows/server-self-hosted.yml',
19      '**/CMakeLists.txt',
20      '**/Makefile',
21      '**/*.h',
22      '**/*.hpp',
23      '**/*.c',
24      '**/*.cpp',
25      '**/*.cu',
26      '**/*.swift',
27      '**/*.m',
28      'tools/server/**.*'
29    ]
30
31env:
32  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
33  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
34  LLAMA_ARG_LOG_COLORS: 1
35  LLAMA_ARG_LOG_PREFIX: 1
36  LLAMA_ARG_LOG_TIMESTAMPS: 1
37  LLAMA_ARG_LOG_VERBOSITY: 10
38
39concurrency:
40  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
41  cancel-in-progress: true
42
43jobs:
44  server-metal:
45    runs-on: [self-hosted, llama-server, macOS, ARM64]
46
47    steps:
48      - name: Clone
49        id: checkout
50        uses: actions/checkout@v6
51        with:
52          fetch-depth: 0
53          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
54
55      - name: Build
56        id: cmake_build
57        run: |
58          cmake -B build -DGGML_SCHED_NO_REALLOC=ON
59          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) --target llama-server
60
61      - name: Python setup
62        id: setup_python
63        run: |
64          cd tools/server/tests
65          python3 -m venv venv
66          source venv/bin/activate
67          pip install -r requirements.txt
68
69      - name: Tests (GPUx1)
70        id: server_integration_tests
71        if: ${{ !github.event.pull_request }}
72        run: |
73          cd tools/server/tests
74          source venv/bin/activate
75          PYTEST_WORKERS=1 ./tests.sh
76
77      - name: Tests (GPUx1, backend-sampling)
78        id: server_integration_tests_backend_sampling
79        if: ${{ !github.event.pull_request }}
80        run: |
81          cd tools/server/tests
82          source venv/bin/activate
83          export LLAMA_ARG_BACKEND_SAMPLING=1
84          PYTEST_WORKERS=1 ./tests.sh
85
86      - name: Tests (GPUx2)
87        id: server_integration_tests_gpu2
88        if: ${{ !github.event.pull_request }}
89        run: |
90          cd tools/server/tests
91          source venv/bin/activate
92          export GGML_METAL_DEVICES=2
93          PYTEST_WORKERS=1 ./tests.sh
94
95      - name: Tests (GPUx2, backend-sampling)
96        id: server_integration_tests_gpu2_backend_sampling
97        if: ${{ !github.event.pull_request }}
98        run: |
99          cd tools/server/tests
100          source venv/bin/activate
101          export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
102          PYTEST_WORKERS=1 ./tests.sh
103
104  server-cuda:
105    runs-on: "hf-jobs-t4-small:cuda13"
106
107    steps:
108      - name: Clone
109        id: checkout
110        uses: actions/checkout@v6
111        with:
112          fetch-depth: 0
113          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
114
115      - name: Install dependencies
116        run: |
117          sudo apt update
118          sudo apt install -y cmake libssl-dev python3 python3-venv python3-pip
119
120      - name: ccache
121        uses: ggml-org/ccache-action@v1.2.24
122        with:
123          restore: false
124          save: false
125
126      - name: ccache-buckets-restore
127        uses: ./.github/actions/ccache-buckets
128        with:
129          key: self-hosted-server-cuda
130          folder: llama.cpp
131          hf_bucket: ggml-org/cache
132
133      - name: Build
134        id: cmake_build
135        run: |
136          cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
137          cmake --build build --config Release -j $(nproc) --target llama-server
138
139      - name: ccache-buckets-save
140        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
141        uses: ./.github/actions/ccache-buckets
142        env:
143          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
144        with:
145          key: self-hosted-server-cuda
146          folder: llama.cpp
147          evict-old-files: 1d
148          hf_bucket: ggml-org/cache
149          save: true
150
151      - name: Python setup
152        id: setup_python
153        run: |
154          cd tools/server/tests
155          python3 -m venv venv
156          source venv/bin/activate
157          pip install -r requirements.txt
158
159      - name: Tests (GPUx1)
160        id: server_integration_tests
161        if: ${{ !github.event.pull_request }}
162        run: |
163          cd tools/server/tests
164          source venv/bin/activate
165          PYTEST_WORKERS=1 ./tests.sh
166
167      - name: Tests (GPUx1, backend-sampling)
168        id: server_integration_tests_backend_sampling
169        if: ${{ !github.event.pull_request }}
170        run: |
171          cd tools/server/tests
172          source venv/bin/activate
173          export LLAMA_ARG_BACKEND_SAMPLING=1
174          PYTEST_WORKERS=1 ./tests.sh
175
176      - name: Tests (GPUx2)
177        id: server_integration_tests_gpu2
178        if: ${{ !github.event.pull_request }}
179        run: |
180          cd tools/server/tests
181          source venv/bin/activate
182          export GGML_CUDA_DEVICES=2
183          PYTEST_WORKERS=1 ./tests.sh
184
185      - name: Tests (GPUx2, backend-sampling)
186        id: server_integration_tests_gpu2_backend_sampling
187        if: ${{ !github.event.pull_request }}
188        run: |
189          cd tools/server/tests
190          source venv/bin/activate
191          export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
192          PYTEST_WORKERS=1 ./tests.sh
193
194  server-kleidiai:
195    runs-on: ah-ubuntu_22_04-c8g_8x
196
197    steps:
198      - name: Clone
199        id: checkout
200        uses: actions/checkout@v6
201        with:
202          fetch-depth: 0
203          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
204
205      - name: Dependencies
206        id: depends
207        run: |
208          set -euxo pipefail
209          sudo apt-get update
210          sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
211          apt-get install -y \
212           build-essential \
213           libssl-dev \
214           python3-venv \
215           gpg \
216           wget \
217           time \
218           git-lfs
219
220          git lfs install
221
222          # install the latest cmake
223          sudo install -d /usr/share/keyrings
224          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
225           | gpg --dearmor \
226           | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
227          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
228           | sudo tee /etc/apt/sources.list.d/kitware.list
229          sudo apt-get update
230          sudo apt-get install -y cmake
231
232      - name: Build
233        id: cmake_build
234        run: |
235          cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON
236          cmake --build build --config Release -j $(nproc) --target llama-server
237
238      - name: Python setup
239        id: setup_python
240        run: |
241          cd tools/server/tests
242          python3 -m venv venv
243          source venv/bin/activate
244          pip install -r requirements.txt
245
246      - name: Tests
247        id: server_integration_tests
248        if: ${{ !github.event.pull_request }}
249        run: |
250          cd tools/server/tests
251          source venv/bin/activate
252          ./tests.sh