char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Max Krasnyanskyhexagon: support for multi-device model split (aka row-split) (#28589)eafe15a5e

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
11.0 KiB296 linesraw
1#!/usr/bin/env python3
2"""
3align-macros.py - Inspect and align trailing backslashes in multiline C/C++ macros.
4
5Usage:
6    align-macros.py [paths...]                 # Check and report misaligned macros
7    align-macros.py --diff [paths...]          # Show unified diff of fixes
8    align-macros.py --fix [paths...]           # Fix misaligned macros in-place
9    align-macros.py --fix --mode majority ...  # Align to the dominant column
10    align-macros.py --fix --pad 2 ...          # Align to (max_content_len + pad)
11
12Safety rules:
13    - Macros that are ALREADY aligned are NEVER touched (unless --all is given).
14    - Whitespace after trailing backslashes is flagged and cleaned.
15"""
16
17import argparse
18import difflib
19import logging
20import os
21import re
22import sys
23from collections import Counter
24from typing import List, Optional, Tuple, NamedTuple
25
26logger = logging.getLogger("ggml-hexagon-align-macros")
27
28
29class MacroLine(NamedTuple):
30    line_num: int       # 1-indexed
31    raw: str            # Original line including newline
32    content: str        # Line content before trailing backslash (stripped of trailing whitespace)
33    bs_col: Optional[int]  # 1-indexed column of backslash, or None if last line has no backslash
34    trailing_ws: bool   # True if whitespace existed after the backslash
35
36
37class MacroDef(NamedTuple):
38    name: str
39    filepath: str
40    start_line: int
41    end_line: int
42    lines: List[MacroLine]
43
44
45def parse_macros(filepath: str) -> List[MacroDef]:
46    """Extract all multiline macros from a C/C++ source file."""
47    try:
48        with open(filepath, "r", encoding="utf-8", errors="replace") as f:
49            lines = f.readlines()
50    except Exception as e:
51        logger.error(f"Error reading {filepath}: {e}")
52        return []
53
54    macros: List[MacroDef] = []
55    i = 0
56    n = len(lines)
57
58    while i < n:
59        line = lines[i]
60        m = re.match(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z0-9_]*)", line)
61        if m:
62            macro_name = m.group(1)
63            macro_start = i + 1
64            macro_lines: List[MacroLine] = []
65            cur = i
66
67            while cur < n:
68                l_raw = lines[cur]
69                l_rstrip = l_raw.rstrip("\r\n")
70
71                # Check if line has a trailing backslash
72                # Note: handle possible accidental spaces after backslash
73                match_bs = re.search(r"\\([ \t]*)$", l_rstrip)
74                if match_bs:
75                    has_trailing_ws = len(match_bs.group(1)) > 0
76                    bs_index = match_bs.start()
77                    content = l_rstrip[:bs_index].rstrip()
78                    # 1-indexed column of the backslash
79                    bs_col = bs_index + 1
80                    macro_lines.append(MacroLine(
81                        line_num=cur + 1,
82                        raw=l_raw,
83                        content=content,
84                        bs_col=bs_col,
85                        trailing_ws=has_trailing_ws
86                    ))
87                    cur += 1
88                else:
89                    # Line does not end with backslash
90                    if cur == i:
91                        # Single-line macro, not multiline
92                        break
93                    else:
94                        # Final line of a multiline macro
95                        macro_lines.append(MacroLine(
96                            line_num=cur + 1,
97                            raw=l_raw,
98                            content=l_rstrip.rstrip(),
99                            bs_col=None,
100                            trailing_ws=False
101                        ))
102                        break
103
104            # Only record if it is a multiline macro (has at least one continuation line)
105            continuation_lines = [ml for ml in macro_lines if ml.bs_col is not None]
106            if continuation_lines:
107                macro_end = macro_lines[-1].line_num
108                macros.append(MacroDef(
109                    name=macro_name,
110                    filepath=filepath,
111                    start_line=macro_start,
112                    end_line=macro_end,
113                    lines=macro_lines
114                ))
115            i = cur
116        i += 1
117
118    return macros
119
120
121def is_macro_aligned(macro: MacroDef) -> bool:
122    """A macro is aligned if all continuation lines have backslashes at the same column."""
123    bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None]
124    if not bs_cols:
125        return True
126    has_trailing_ws = any(ml.trailing_ws for ml in macro.lines)
127    return len(set(bs_cols)) == 1 and not has_trailing_ws
128
129
130def compute_target_column(macro: MacroDef, mode: str, pad: int, target_col: Optional[int]) -> int:
131    """Determine the column where backslashes should be aligned."""
132    max_content_len = max(len(ml.content) for ml in macro.lines)
133    min_needed = max_content_len + pad
134
135    if target_col is not None:
136        return max(target_col, min_needed)
137
138    bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None]
139    if not bs_cols:
140        return min_needed
141
142    if mode == "min":
143        return min_needed
144    elif mode == "max":
145        return max(max(bs_cols), min_needed)
146    elif mode == "majority":
147        counts = Counter(bs_cols)
148        # Sort by frequency descending, then by column descending
149        majority_col = sorted(counts.items(), key=lambda x: (-x[1], -x[0]))[0][0]
150        return max(majority_col, min_needed)
151    else:
152        return min_needed
153
154
155def realign_macro_lines(macro: MacroDef, target_col: int) -> List[str]:
156    """Format macro lines with backslashes aligned at target_col."""
157    new_lines: List[str] = []
158    for ml in macro.lines:
159        nl = "\r\n" if ml.raw.endswith("\r\n") else "\n"
160        if ml.bs_col is None:
161            # Last line without backslash
162            new_lines.append(ml.raw)
163        else:
164            if not ml.content:
165                spaces = " " * (target_col - 1)
166                new_lines.append(f"{spaces}\\{nl}")
167            else:
168                spaces_needed = max(1, target_col - len(ml.content) - 1)
169                new_lines.append(f"{ml.content}{' ' * spaces_needed}\\{nl}")
170    return new_lines
171
172
173def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Optional[str]]:
174    macros = parse_macros(filepath)
175    if not macros:
176        return 0, 0, None
177
178    with open(filepath, "r", encoding="utf-8", errors="replace") as f:
179        file_lines = f.readlines()
180
181    misaligned_count = 0
182    modified = False
183    new_file_lines = list(file_lines)
184
185    for macro in macros:
186        aligned = is_macro_aligned(macro)
187        if not aligned or args.all:
188            if not aligned:
189                misaligned_count += 1
190
191            bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None]
192            max_content = max(len(ml.content) for ml in macro.lines)
193            col_counts = Counter(bs_cols)
194
195            if not args.quiet:
196                logger.info(f"{filepath}:{macro.start_line}-{macro.end_line} [{macro.name}]")
197                logger.info(f"  Max content width: {max_content}, Min needed column (+{args.pad}): {max_content + args.pad}")
198                logger.info(f"  Current backslash columns: {dict(sorted(col_counts.items()))}")
199                trailing_ws_lines = [ml.line_num for ml in macro.lines if ml.trailing_ws]
200                if trailing_ws_lines:
201                    logger.warning(f"  Warning: Trailing whitespace after backslash on line(s): {trailing_ws_lines}")
202
203            target_col = compute_target_column(macro, args.mode, args.pad, args.target_col)
204            if not args.quiet:
205                logger.info(f"  -> Target alignment column: {target_col}")
206
207            realigned = realign_macro_lines(macro, target_col)
208
209            start_idx = macro.start_line - 1
210            end_idx = start_idx + len(macro.lines)
211            if new_file_lines[start_idx:end_idx] != realigned:
212                new_file_lines[start_idx:end_idx] = realigned
213                modified = True
214
215    diff_text = None
216    if modified:
217        diff = difflib.unified_diff(
218            file_lines,
219            new_file_lines,
220            fromfile=f"a/{filepath}",
221            tofile=f"b/{filepath}",
222            lineterm=""
223        )
224        diff_text = "\n".join(diff)
225
226        if args.fix:
227            with open(filepath, "w", encoding="utf-8") as f:
228                f.writelines(new_file_lines)
229            if not args.quiet:
230                logger.info(f"  [FIXED] Updated {filepath}")
231
232    return len(macros), misaligned_count, diff_text
233
234
235def find_source_files(paths: List[str]) -> List[str]:
236    extensions = {".c", ".cpp", ".cc", ".cxx", ".h", ".hpp", ".inl"}
237    result: List[str] = []
238    for p in paths:
239        if os.path.isfile(p):
240            result.append(p)
241        elif os.path.isdir(p):
242            for root, _, files in os.walk(p):
243                for file in sorted(files):
244                    _, ext = os.path.splitext(file)
245                    if ext.lower() in extensions:
246                        result.append(os.path.join(root, file))
247    return sorted(result)
248
249
250def main():
251    logging.basicConfig(level=logging.INFO, format="%(message)s")
252    parser = argparse.ArgumentParser(
253        description="Inspect and align backslashes in multiline C/C++ macros."
254    )
255    parser.add_argument("paths", nargs="*", default=["."], help="Files or directories to scan (default: current dir)")
256    parser.add_argument("--fix", action="store_true", help="Fix misaligned macros in-place")
257    parser.add_argument("--diff", action="store_true", help="Display unified diff of suggested fixes")
258    parser.add_argument("--check", action="store_true", help="Exit with code 1 if misaligned macros exist")
259    parser.add_argument("--mode", choices=["min", "max", "majority"], default="min",
260                        help="Alignment mode: 'min' (max_len + pad), 'max' (max existing col), 'majority' (dominant col)")
261    parser.add_argument("--pad", type=int, default=2, help="Spaces between longest line and backslash (default: 2)")
262    parser.add_argument("--target-col", type=int, default=None, help="Force alignment to an exact column")
263    parser.add_argument("--all", action="store_true", help="Realign all macros even if already aligned (default: only misaligned)")
264    parser.add_argument("-q", "--quiet", action="store_true", help="Only output errors and diffs/summary")
265
266    args = parser.parse_args()
267
268    files = find_source_files(args.paths)
269    if not files:
270        logger.error("No C/C++ source files found.")
271        sys.exit(0)
272
273    total_macros = 0
274    total_misaligned = 0
275    diffs: List[str] = []
276
277    for filepath in files:
278        num_macros, num_misaligned, diff_text = process_file(filepath, args)
279        total_macros += num_macros
280        total_misaligned += num_misaligned
281        if diff_text:
282            diffs.append(diff_text)
283
284    if args.diff and diffs:
285        logger.info("\n--- Proposed Changes ---\n")
286        for d in diffs:
287            logger.info(d)
288
289    logger.info(f"\nSummary: scanned {len(files)} files, {total_macros} multiline macros, {total_misaligned} misaligned.")
290
291    if args.check and total_misaligned > 0:
292        sys.exit(1)
293
294
295if __name__ == "__main__":
296    main()