char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Sigbjørn Skjæretci : fix pre-tokenizer-hashes check (#23651)062d3115a

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
1.5 KiB45 linesraw
1name: Check Pre-Tokenizer Hashes
2
3on:
4    push:
5        paths:
6            - 'conversion/base.py'
7            - 'convert_hf_to_gguf_update.py'
8    pull_request:
9        paths:
10            - 'conversion/base.py'
11            - 'convert_hf_to_gguf_update.py'
12
13jobs:
14    pre-tokenizer-hashes:
15        runs-on: [self-hosted, fast]
16
17        steps:
18        - name: Checkout repository
19          uses: actions/checkout@v6
20
21        - name: Set up Python
22          uses: actions/setup-python@v6
23          with:
24              python-version: '3.11'
25
26        - name: Install Python dependencies
27          run: |
28              python3 -m venv .venv
29              .venv/bin/pip install -r requirements/requirements-convert_hf_to_gguf_update.txt
30
31        - name: Update pre-tokenizer hashes
32          run: |
33              cp conversion/base.py /tmp
34              .venv/bin/python convert_hf_to_gguf_update.py --check-missing
35
36        - name: Check if committed pre-tokenizer hashes matches generated version
37          run: |
38              if ! diff -q conversion/base.py /tmp/base.py; then
39                  echo "Model pre-tokenizer hashes (in conversion/base.py) do not match generated hashes (from convert_hf_to_gguf_update.py)."
40                  echo "To fix: run ./convert_hf_to_gguf_update.py and commit the updated conversion/base.py along with your changes"
41                  echo "Differences found:"
42                  diff conversion/base.py /tmp/base.py || true
43                  exit 1
44              fi
45              echo "Model pre-tokenizer hashes are up to date."