char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Christian Kastnerscripts: Add script to verify API/ABI compatibility (#28579)dfe45163e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
7.8 KiB272 linesraw
1#!/bin/sh
2# Check for backwards-incompatible API and ABI changes between two builds
3#
4# Backwards-incompatible API changes, such as removing a value from an enum,
5# are checked by abi-compliance-checker. Such changes can break compilation of
6# existing programs.
7#
8# Backwards-incompatible ABI changes, such as the removal of a public function,
9# are checked by libigail-tools. Such changes could break run-time dynamic
10# linking of existing binaries. (We don't use a-c-c for ABI checks because it
11# needs a debug build, whereas abigail does not.)
12#
13# Commands:
14#   --generate <build-dir>: Creates API/ABI dumps in <build-dir>
15#                           <build-dir> is expected to be a CMake build result
16#   --check <dir1> <dir2>:  Compares dumps in <dir1> and <dir2>
17#                           Comparison exit codes
18#                              0: all good
19#                              1: backwards-incompatible changes found
20#
21# Options:
22#   --include-path <dir>:   a-c-c calls gcc on headers; use this option to add
23#                           directories to gcc's search path
24#
25#
26# This script would typically be used before cutting a release:
27#
28#   1. Generate API/ABI dump for the old version
29#
30#      $ check-apiabi-compat.sh --generate <build-dir-old> libfoo [ libbar ...]
31#
32#   2. <update source>
33#
34#   3. Generate API/ABI dump for the new version
35#
36#      $ check-apiabi-compat.sh --generate <build-dir-new> libfoo [ libbar ...]
37#
38#   4. Compare the two dumps
39#
40#      $ check-apiabi-compat.sh --check <old-build-dir> <new-build-dir>
41#
42#      If the check exits 0, all is fine. Otherwise, backwards-incompatible
43#      changes were found, and the librar(ies) need a SOVER bump.
44set -eu
45
46# Preconditions
47if ! command -v abi-compliance-checker >/dev/null 2>&1; then
48    echo "abi-compliance-checker is not installed." >&2
49    exit 1
50elif ! command -v abidw >/dev/null 2>&1; then
51    echo "abigail-tools are not installed." >&2
52    exit 1
53fi
54
55# Some generic functions
56usage() {
57    echo "Usage: $0 [ --include-path <dir> ] --generate <build-dir> libXXX [ libYYY ... ]" >&2
58    echo "       $0 --check <old-build-dir> <new-build-dir>" >&2
59}
60
61get_cmake_project_name() {
62    sed -nr 's/^project\("(.*)".*$/\1/p' CMakeLists.txt
63}
64
65get_cmake_version() {
66    major="$(sed -nr 's/^set\([A-Z]+_VERSION_MAJOR ([0-9]+)\)$/\1/p' CMakeLists.txt)"
67    minor="$(sed -nr 's/^set\([A-Z]+_VERSION_MINOR ([0-9]+)\)$/\1/p' CMakeLists.txt)"
68    patch="$(sed -nr 's/^set\([A-Z]+_VERSION_PATCH ([0-9]+)\)$/\1/p' CMakeLists.txt)"
69    echo "$major.$minor.$patch"
70}
71
72# Option parsing and validation
73DO_GEN=0
74DO_CHECK=0
75BUILD_DIR=
76BUILD_DIR_NEW=
77INCLUDE_PATHS=
78while [ "$#" -gt 0 ]; do
79    case "$1" in
80    --generate=*)
81        DO_GEN=1
82        BUILD_DIR="${1#*=}"
83        shift
84        ;;
85    --generate)
86        DO_GEN=1
87        if [ -z "${2:-}" ]; then
88            usage
89            exit 1
90        fi
91        BUILD_DIR="$2"
92        shift 2
93        ;;
94    --check)
95        DO_CHECK=1
96        if [ -z "${2:-}" ] || [ -z "${3:-}" ]; then
97            usage
98            exit 1
99        elif ! [ -d "$2" ]; then
100            echo "$2 is not a directory." >&2
101            exit 1
102        elif ! [ -d "$3" ]; then
103            echo "$3 is not a directory." >&2
104            exit 1
105        fi
106        BUILD_DIR="$2"
107        BUILD_DIR_NEW="$3"
108        shift 3
109        ;;
110    --include-path=*)
111        INCLUDE_PATHS="$INCLUDE_PATHS ${1#*=}"
112        shift
113        ;;
114    --include-path)
115        if [ -z "${2:-}" ]; then
116            usage
117            exit 1
118        fi
119        INCLUDE_PATHS="$INCLUDE_PATHS $2"
120        shift 2
121        ;;
122    -h | --help)
123        usage
124        exit 1
125        ;;
126    -?*)
127        usage
128        exit 1
129        ;;
130    *)
131        break
132        ;;
133    esac
134done
135if [ $((DO_GEN + DO_CHECK)) -gt 1 ]; then
136    echo "Can only use one --command." >&2
137    exit
138fi
139PROJECT_NAME="$(get_cmake_project_name)"
140PROJECT_VERSION="$(get_cmake_version)"
141LIB_NAMES=""
142while [ "$#" -gt 0 ]; do
143    if [ "${1#lib}" = "$1" ]; then
144        echo "Library to check must start with libXXX." >&2
145        exit 1
146    fi
147    LIB_NAMES="$LIB_NAMES $1"
148    shift
149done
150
151dump_current_api() {
152    echo "Dumping API..."
153
154    DESCRIPTOR="$BUILD_DIR/apiabi/acc-descriptor.xml"
155    mkdir -p "$BUILD_DIR/apiabi"
156    cat >"$DESCRIPTOR" <<EOF
157<version>$PROJECT_VERSION</version>
158<headers>include</headers>
159<add_include_paths>$INCLUDE_PATHS</add_include_paths>
160EOF
161
162    # This addresses a bug between a-c-c and universal-ctags, manifested when
163    # a name is use both for a tag and a function name
164    mkdir -p "$BUILD_DIR/apiabi/.ctags.d"
165    echo "--fields=-t" >"$BUILD_DIR/apiabi/.ctags.d/acc.ctags"
166
167    # Change HOME so that .ctags.d gets picked up by universal-ctags, if used
168    HOME="$BUILD_DIR/apiabi" abi-compliance-checker \
169        -headers-only \
170        -lib "$PROJECT_NAME" \
171        -dump "$DESCRIPTOR" \
172        -log-path "$BUILD_DIR/apiabi/acc.log" \
173        -dump-path "$BUILD_DIR/apiabi/api.dump"
174    # acc generates this file with an ancient timestamp, which confuses gzip
175    touch "$BUILD_DIR/apiabi/api.dump"
176}
177
178dump_current_abi() {
179    echo "Dumping ABIs ..."
180    mkdir -p "$BUILD_DIR/apiabi"
181    # The suppressions are needed to avoid including all the internal C++
182    # symbols, and system types
183    cat >"$BUILD_DIR/apiabi/abidw.suppress" <<EOF
184[suppress_function]
185
186label = suppress internal C++ mangled functions
187symbol_name_regexp = ^_Z
188drop = yes
189
190[suppress_variable]
191label = suppress internal C++ mangled variables
192symbol_name_regexp = ^_Z
193drop = yes
194
195[suppress_type]
196label = Suppress types outside of our own source
197source_location_not_regexp = ^include/
198drop = yes
199EOF
200
201    # In abidw 2.5, handling of undefined stuff was changed a bit
202    abidw_version="$(abidw --version | sed -r 's/^abidw: ([0-9]+\.[0-9]+).*$/\1/')"
203    abidw_major="${abidw_version%.*}"
204    abidw_minor="${abidw_version#*.}"
205    if [ "$abidw_major" -gt 2 ] || [ "$abidw_minor" -gt 4 ]; then
206        abidw_undefined_syms_options="--no-load-undefined-interfaces"
207    else
208        abidw_undefined_syms_options="--drop-undefined-syms"
209    fi
210
211    for lib_name in $LIB_NAMES; do
212        # Depending on where add_library resides, the libraries can end up in
213        # build/src or build/bin
214        lib_path="$BUILD_DIR/src/$lib_name.so"
215        if ! [ -f "$lib_path" ]; then
216            lib_path="$BUILD_DIR/bin/$lib_name.so"
217            if ! [ -f "$lib_path" ]; then
218                echo "Cannot find library $lib_name.so" >&2
219                exit 1
220            fi
221        fi
222        abidw \
223            --headers-dir include \
224            "$abidw_undefined_syms_options" \
225            --suppressions "$BUILD_DIR/apiabi/abidw.suppress" \
226            --out-file "${BUILD_DIR}/apiabi/$lib_name.abi.xml" \
227            "$lib_path"
228    done
229}
230
231# Run the actual commands
232if [ "$DO_GEN" -eq 1 ]; then
233    dump_current_api
234    dump_current_abi
235    exit 0
236elif [ "$DO_CHECK" -eq 1 ]; then
237    # From here on, we don't want to exit on first error
238    set +e
239
240    abi-compliance-checker \
241        -strict \
242        -source \
243        -library "$PROJECT_NAME" \
244        -old "$BUILD_DIR/apiabi/api.dump" \
245        -new "$BUILD_DIR_NEW/apiabi/api.dump" \
246        -src-report-path "$BUILD_DIR_NEW/apiabi/api_compat_report.html"
247    API_RESULT=$?
248
249    ABI_RESULT=0
250    for xml_file in "$BUILD_DIR/apiabi/"lib*.abi.xml; do
251        xml_file_new="$BUILD_DIR_NEW/apiabi/$(basename "$xml_file")"
252
253        if ! [ -f "$xml_file_new" ]; then
254            echo "Cannot compare, missing file: $xml_file_new" >&2
255            exit 1
256        fi
257
258        abidiff "$xml_file" "$xml_file_new"
259        res=$?
260        [ "$((res & 8))" -ne 0 ] && ABI_RESULT=1
261    done
262
263    if [ "$API_RESULT" -gt 0 ]; then
264        echo "ERROR: API changed with possible backwards-compatibility problems." >&2
265    fi
266    if [ "$ABI_RESULT" -gt 0 ]; then
267        echo "ERROR: ABI changed with possible backwards-compatibility problems." >&2
268    fi
269    if [ "$((API_RESULT + ABI_RESULT))" -gt 0 ]; then
270        exit 1
271    fi
272fi