char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Brianconvert.py : add python logging instead of print() (#6511)a2ac89d6e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
2.4 KiB84 linesraw
1#!/usr/bin/env python3
2
3import logging
4import os
5import hashlib
6
7logger = logging.getLogger("verify-checksum-models")
8
9
10def sha256sum(file):
11    block_size = 16 * 1024 * 1024  # 16 MB block size
12    b = bytearray(block_size)
13    file_hash = hashlib.sha256()
14    mv = memoryview(b)
15    with open(file, 'rb', buffering=0) as f:
16        while True:
17            n = f.readinto(mv)
18            if not n:
19                break
20            file_hash.update(mv[:n])
21
22    return file_hash.hexdigest()
23
24
25# Define the path to the llama directory (parent folder of script directory)
26llama_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
27
28# Define the file with the list of hashes and filenames
29hash_list_file = os.path.join(llama_path, "SHA256SUMS")
30
31# Check if the hash list file exists
32if not os.path.exists(hash_list_file):
33    logger.error(f"Hash list file not found: {hash_list_file}")
34    exit(1)
35
36# Read the hash file content and split it into an array of lines
37with open(hash_list_file, "r") as f:
38    hash_list = f.read().splitlines()
39
40# Create an array to store the results
41results = []
42
43# Loop over each line in the hash list
44for line in hash_list:
45    # Split the line into hash and filename
46    hash_value, filename = line.split("  ")
47
48    # Get the full path of the file by joining the llama path and the filename
49    file_path = os.path.join(llama_path, filename)
50
51    # Informing user of the progress of the integrity check
52    logger.info(f"Verifying the checksum of {file_path}")
53
54    # Check if the file exists
55    if os.path.exists(file_path):
56        # Calculate the SHA256 checksum of the file using hashlib
57        file_hash = sha256sum(file_path)
58
59        # Compare the file hash with the expected hash
60        if file_hash == hash_value:
61            valid_checksum = "V"
62            file_missing = ""
63        else:
64            valid_checksum = ""
65            file_missing = ""
66    else:
67        valid_checksum = ""
68        file_missing = "X"
69
70    # Add the results to the array
71    results.append({
72        "filename": filename,
73        "valid checksum": valid_checksum,
74        "file missing": file_missing
75    })
76
77
78# Print column headers for results table
79print("filename".ljust(40) + "valid checksum".center(20) + "file missing".center(20)) # noqa: NP100
80print("-" * 80) # noqa: NP100
81
82# Output the results as a table
83for r in results:
84    print(f"{r['filename']:40} {r['valid checksum']:^20} {r['file missing']:^20}") # noqa: NP100