char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Sigbjørn Skjæretci : bump ty to 0.0.78 (#28548)67672dc5b

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
28.3 KiB748 linesraw
1#!/usr/bin/env python3
2
3import sys
4import os
5import re
6import argparse
7import statistics
8import logging
9import bisect
10from typing import Any, Dict, List, Optional, Iterable
11
12from collections import defaultdict
13
14# Mapping of cli-friendly names to (internal_data_key, Display Header, numeric_sort_key)
15COL_MAP = {
16    "tot-usec":   ("tot_usec",   "Tot usec",   "_sort_tot_usec"),
17    "op":         ("op",         "Op",         "op"),
18    "dims":       ("dims",       "Dims",       "dims"),
19    "dtypes":     ("dtypes",     "DTypes",     "dtypes"),
20    "count":      ("count",      "Count",      "_sort_count"),
21    "max-usec":   ("max_usec",   "Max usec",   "_sort_max_usec"),
22    "avg-usec":   ("avg_usec",   "Avg usec",   "_sort_avg_usec"),
23    "max-cycles": ("max_cycles", "Max Cycles", "_sort_max_cycles"),
24    "avg-cycles": ("avg_cycles", "Avg Cycles", "_sort_avg_cycles"),
25    "max-pmu":    ("max_pmu",    "Max PMU",    "_sort_max_pmu"),
26    "avg-pmu":    ("avg_pmu",    "Avg PMU",    "_sort_avg_pmu"),
27}
28
29op_pattern = re.compile(
30    r"profile-op\s+(?P<op_name>[A-Z_0-9+]+):\s+.*?\s+:\s+(?P<dims>[\d:x\s\->!]+)\s+:\s+(?P<types>[a-z\d_\s\->x]+)\s+:\s+.*?\s+:\s+(?:op-)?usec\s+(?P<usec>\d+)\s+(?:op-)?cycles\s+(?P<cycles>\d+)(?:\s+start\s+(?P<start>\d+))?(?:\s+mhz\s+(?P<mhz>[\d.]+))?(?:\s+pmu\s+\[(?P<pmu>[\d,\s]+)\])?(?:\s+evt\s+\[(?P<evt>[\d,\s]+)\])?"
31)
32
33trace_pattern = re.compile(
34    r"trace-evt\s+(?P<event>[A-Z_0-9\-]+):\s+thread\s+(?P<thread>\d+)\s+info\s+(?P<info>\d+)\s+(?P<state>start|stop)\s+(?P<cycles>\d+)"
35)
36
37device_pattern = re.compile(r"\b(HTP\d+(?::\d+)?)\s+(?:profile-op|trace-evt)\b")
38
39
40def extract_device(line):
41    m = device_pattern.search(line)
42    if m:
43        return m.group(1)
44    return "HTP0"
45
46
47def device_matches(record_device, target_device):
48    targets = [t.strip() for t in target_device.split(',')]
49    for target in targets:
50        if record_device == target:
51            return True
52        if record_device.startswith(target + ":"):
53            return True
54    return False
55
56
57logger = logging.getLogger("ggml-hexagon-profile")
58
59
60def normalize_event_name(evt_type):
61    if evt_type == "HVX_COMP":
62        return "V-COMP"
63    if evt_type == "HMX_COMP":
64        return "M-COMP"
65
66    # Strip HVX_ or HMX_ prefixes
67    name = evt_type
68    if name.startswith("HVX_") or name.startswith("HMX_"):
69        name = name[4:]
70    return name.replace("_", "-")
71
72
73class CycleUnwrapper:
74    def __init__(self, initial_val=None):
75        if initial_val is not None:
76            self.last_raw = initial_val & 0xFFFFFFFF
77            self.high_part = initial_val & 0xFFFFFFFF00000000
78        else:
79            self.last_raw = None
80            self.high_part = 0
81
82    def unwrap(self, raw):
83        if self.last_raw is None:
84            self.last_raw = raw
85            return raw
86        diff = raw - self.last_raw
87        if diff < -0x80000000:
88            self.high_part += 0x100000000
89        elif diff > 0x80000000:
90            self.high_part -= 0x100000000
91        self.last_raw = raw
92        return raw + self.high_part
93
94
95def parse_log(file_path, pmu_index=None, limit=None, device_filter=None, op_filter_re=None):
96    try:
97        if file_path != "-":
98            f = open(file_path, 'r', encoding='utf-8', errors='ignore')
99        else:
100            f = os.fdopen(0, 'r', encoding='utf-8', errors='ignore')
101    except FileNotFoundError:
102        logger.error(f"file '{file_path}' not found.")
103        sys.exit(1)
104
105    all_ops: List[Dict[str, Any]] = []
106    all_traces: List[Dict[str, Any]] = []
107    current_op: Optional[Dict[str, Any]] = None
108    ops_count_per_device = {}
109    if device_filter is not None:
110        for target in device_filter.split(','):
111            ops_count_per_device[target.strip()] = 0
112    limit_reached = False
113
114    timestamp_pattern = re.compile(r"(?P<min>\d+)\.(?P<sec>\d+)\.(?P<ms>\d+)\.(?P<us>\d+)\s+[A-Z]\s+")
115    unwrappers = {}
116    last_batch_start = {}
117    trace_unwrappers = {}
118
119    for line in f:
120        if "profile-op" not in line and "trace-evt" not in line:
121            continue
122
123        ts_match = timestamp_pattern.search(line)
124        abs_usec = 0
125        if ts_match:
126            abs_usec = (
127                (int(ts_match.group('min')) * 60 + int(ts_match.group('sec'))) * 1000000
128                + int(ts_match.group('ms')) * 1000
129                + int(ts_match.group('us'))
130            )
131
132        device = extract_device(line)
133
134        idx = line.find("profile-op")
135        if idx != -1 and "|" in line[idx:]:
136            parts = [p.strip() for p in line[idx:].split("|")]
137            prefix = parts[0]
138            prefix_match = re.search(r"profile-op\s+(?P<op_name>[A-Z_0-9+]+)", prefix)
139            if not prefix_match:
140                continue
141
142            names = parts[1]
143            if len(parts) == 7:
144                dims, types, timings = parts[2], parts[3], parts[6]
145            elif len(parts) == 6:
146                dims, types, timings = parts[2], parts[3], parts[5]
147            else:
148                continue
149
150            timing_match = re.search(
151                r"(?:op-)?usec\s+(?P<usec>\d+)\s+(?:op-)?cycles\s+(?P<cycles>\d+)(?:\s+start\s+(?P<start>\d+))?(?:\s+mhz\s+(?P<mhz>[\d.]+))?(?:\s+pmu\s+\[(?P<pmu>[\d,\s]+)\])?(?:\s+evt\s+\[(?P<evt>[\d,\s]+)\])?",
152                timings
153            )
154            if not timing_match:
155                continue
156
157            op_match = timing_match
158            op_name = prefix_match.group("op_name")
159        else:
160            op_match = op_pattern.search(line)
161            if op_match:
162                op_name = op_match.group('op_name')
163                names = ""
164                dims = op_match.group('dims').strip()
165                types = op_match.group('types').strip()
166            else:
167                op_match = None
168
169        if op_match:
170            pmu_raw = op_match.group('pmu') if 'pmu' in op_match.groupdict() else None
171            pmu_val = None
172            if pmu_raw and pmu_index is not None:
173                try:
174                    pmu_list = [int(x.strip()) for x in pmu_raw.split(',')]
175                    if len(pmu_list) > pmu_index:
176                        pmu_val = pmu_list[pmu_index]
177                except (ValueError, IndexError):
178                    pmu_val = None
179
180            evt_val = None
181            if types.startswith("evt-cnt "):
182                try:
183                    evt_val = [int(x.strip()) for x in types[8:].split(',')]
184                except ValueError:
185                    evt_val = None
186
187            cycles_start_raw = op_match.group('start')
188            unwrapped_cycles_start = None
189            if op_name == "OPBATCH":
190                if cycles_start_raw:
191                    unwrapped_cycles_start = int(cycles_start_raw)
192                    unwrappers[device] = CycleUnwrapper(unwrapped_cycles_start)
193                    last_batch_start[device] = unwrapped_cycles_start
194                    for k in list(trace_unwrappers.keys()):
195                        if k[0] == device:
196                            del trace_unwrappers[k]
197            else:
198                if cycles_start_raw:
199                    device_unwrapper = unwrappers.get(device)
200                    if device_unwrapper is not None:
201                        unwrapped_cycles_start = device_unwrapper.unwrap(int(cycles_start_raw))
202
203            op_text = re.sub(r"^profile-op\s+", "", line[idx:]).strip() if idx != -1 else line.strip()
204
205            current_op = {
206                'name':         op_name,
207                'names':        names,
208                'dims':         dims,
209                'types':        types,
210                'op_text':      op_text,
211                'usec':         int(op_match.group('usec')),
212                'cycles':       int(op_match.group('cycles')),
213                'cycles_start': int(cycles_start_raw) if cycles_start_raw else None,
214                'unwrapped_cycles_start': unwrapped_cycles_start,
215                'pmu_val':      pmu_val,
216                'evt_val':      evt_val,
217                'abs_usec':     abs_usec,
218                'trace_events': [],
219                'device':       device
220            }
221            all_ops.append(current_op)
222
223            # Check if matching early exit criteria
224            matched = False
225            matched_target = None
226            if device_filter is not None:
227                targets = [t.strip() for t in device_filter.split(',')]
228                for target in targets:
229                    if device == target or device.startswith(target + ":"):
230                        matched = True
231                        matched_target = target
232                        break
233            else:
234                matched = True
235                matched_target = device
236
237            if op_filter_re is not None and not op_filter_re.search(op_text):
238                matched = False
239
240            if matched:
241                if matched_target not in ops_count_per_device:
242                    ops_count_per_device[matched_target] = 0
243                ops_count_per_device[matched_target] += 1
244
245            if limit is not None and len(ops_count_per_device) > 0 and all(count >= limit for count in ops_count_per_device.values()):
246                limit_reached = True
247
248            if limit_reached and op_name == "OPBATCH":
249                break
250            continue
251
252        trace_match = trace_pattern.search(line)
253        if trace_match:
254            thread = int(trace_match.group('thread'))
255            raw_cyc = int(trace_match.group('cycles'))
256            unwrapped_cyc = None
257            th_key = (device, thread)
258            if th_key not in trace_unwrappers:
259                batch_start = last_batch_start.get(device)
260                trace_unwrappers[th_key] = CycleUnwrapper(batch_start)
261            unwrapped_cyc = trace_unwrappers[th_key].unwrap(raw_cyc)
262            all_traces.append({
263                'thread': thread,
264                'event':  trace_match.group('event'),
265                'info':   int(trace_match.group('info')),
266                'cycles': raw_cyc,
267                'unwrapped_cycles': unwrapped_cyc,
268                'state':  trace_match.group('state'),
269                'device': device
270            })
271
272    f.close()
273
274    # Assign start/end cycles to all ops
275    for op in all_ops:
276        op['start_cycles'] = op['unwrapped_cycles_start']
277        op['end_cycles'] = op['start_cycles'] + op['cycles'] if op['start_cycles'] is not None else None
278
279    # Group ops by device
280    valid_ops_by_dev = defaultdict(list)
281    for op in all_ops:
282        if op['start_cycles'] is not None and op['end_cycles'] is not None:
283            valid_ops_by_dev[op['device']].append(op)
284
285    # Group trace events by device
286    traces_by_dev = defaultdict(list)
287    for e in all_traces:
288        if e['unwrapped_cycles'] is not None:
289            traces_by_dev[e['device']].append(e)
290
291    for device, dev_ops in valid_ops_by_dev.items():
292        opbatch_ops = [op for op in dev_ops if op['name'] == "OPBATCH"]
293        other_ops = [op for op in dev_ops if op['name'] != "OPBATCH"]
294
295        opbatch_ops.sort(key=lambda op: op['start_cycles'])
296        other_ops.sort(key=lambda op: op['start_cycles'])
297
298        opbatch_starts = [op['start_cycles'] for op in opbatch_ops]
299        other_starts = [op['start_cycles'] for op in other_ops]
300
301        dev_traces = traces_by_dev.get(device, [])
302        for e in dev_traces:
303            cyc = e['unwrapped_cycles']
304
305            # Map to OPBATCH
306            idx = bisect.bisect_right(opbatch_starts, cyc) - 1
307            if idx >= 0:
308                op = opbatch_ops[idx]
309                if op['start_cycles'] <= cyc <= op['end_cycles']:
310                    op['trace_events'].append(e)
311
312            # Map to other ops
313            idx = bisect.bisect_right(other_starts, cyc) - 1
314            if idx >= 0:
315                op = other_ops[idx]
316                if op['start_cycles'] <= cyc <= op['end_cycles']:
317                    op['trace_events'].append(e)
318
319    return all_ops
320
321
322def print_bubbles_timeline(op):
323    op_name = op['name']
324    dims = op['dims']
325    types = op['types']
326    usec = op['usec']
327    cycles = op['cycles']
328    events = op['trace_events']
329    logger.info("=" * 100)
330    logger.info(f"{op_name} ({dims} : {types}) - {usec} usec {cycles} cycles")
331    logger.info("=" * 100)
332
333    if not events:
334        logger.info("  No trace events recorded.")
335        return
336
337    # Identify start and end cycles for this operator
338    op_start = op['start_cycles']
339    op_end = op['end_cycles']
340    if op_start is None or op_end is None:
341        logger.info("  Cannot analyze bubbles: missing start/end cycle counts.")
342        return
343
344    batch_duration = op_end - op_start
345    if batch_duration <= 0:
346        logger.info("  Cannot analyze bubbles: batch duration is 0.")
347        return
348
349    # Group events by (thread, track_type)
350    tracks = defaultdict(list)
351    for e in events:
352        t = e['thread']
353        is_dma = (normalize_event_name(e['event']) == 'DMA')
354        track_type = 'dma' if is_dma else 'compute'
355        tracks[(t, track_type)].append(e)
356
357    active_threads = sorted(list(set(t for (t, track_type) in tracks.keys())))
358    if not active_threads:
359        logger.info("  No active threads in trace.")
360        return
361
362    bubble_threshold = 10000  # 10k cycles
363
364    thread_stats = {}
365    for t in active_threads:
366        thread_stats[t] = {
367            'compute_idle_cycles': batch_duration,
368            'compute_idle_pct': 100.0,
369            'compute_bubbles': [],
370
371            'dma_idle_cycles': batch_duration,
372            'dma_idle_pct': 100.0,
373            'dma_bubbles': []
374        }
375
376    total_compute_idle_pct = 0.0
377    total_dma_idle_pct = 0.0
378
379    for t in active_threads:
380        for track_type in ['compute', 'dma']:
381            key = (t, track_type)
382            track_events = tracks.get(key, [])
383
384            if not track_events:
385                gaps = [(op_start, op_end)]
386                idle_cycles = batch_duration
387            else:
388                track_events = sorted(track_events, key=lambda e: e.get('unwrapped_cycles') or e['cycles'])
389
390                active_intervals = []
391                active_count = 0
392                curr_start = None
393
394                for e in track_events:
395                    cyc = e.get('unwrapped_cycles') or e['cycles']
396                    cyc = max(op_start, min(op_end, cyc))
397                    state = e['state']
398
399                    if state == 'start':
400                        if active_count == 0:
401                            curr_start = cyc
402                        active_count += 1
403                    elif state == 'stop':
404                        if active_count > 0:
405                            active_count -= 1
406                            if active_count == 0:
407                                active_intervals.append((curr_start, cyc))
408                        else:
409                            active_intervals.append((op_start, cyc))
410
411                if active_count > 0 and curr_start is not None:
412                    active_intervals.append((curr_start, op_end))
413
414                # Merge intervals
415                active_intervals.sort(key=lambda x: x[0])
416                merged_intervals = []
417                for start, end in active_intervals:
418                    if not merged_intervals:
419                        merged_intervals.append([start, end])
420                    else:
421                        last_start, last_end = merged_intervals[-1]
422                        if start <= last_end:
423                            merged_intervals[-1][1] = max(last_end, end)
424                        else:
425                            merged_intervals.append([start, end])
426
427                # Calculate gaps
428                gaps = []
429                curr_time = op_start
430                for start, end in merged_intervals:
431                    if start > curr_time:
432                        gaps.append((curr_time, start))
433                    curr_time = max(curr_time, end)
434                if curr_time < op_end:
435                    gaps.append((curr_time, op_end))
436
437                idle_cycles = sum(end - start for start, end in gaps)
438
439            idle_pct = (idle_cycles / batch_duration) * 100.0
440
441            bubbles = []
442            for start, end in gaps:
443                dur = end - start
444                if dur >= bubble_threshold:
445                    bubbles.append((start, end, dur))
446
447            if track_type == 'compute':
448                thread_stats[t]['compute_idle_cycles'] = idle_cycles
449                thread_stats[t]['compute_idle_pct'] = idle_pct
450                thread_stats[t]['compute_bubbles'] = bubbles
451                total_compute_idle_pct += idle_pct
452            else:
453                thread_stats[t]['dma_idle_cycles'] = idle_cycles
454                thread_stats[t]['dma_idle_pct'] = idle_pct
455                thread_stats[t]['dma_bubbles'] = bubbles
456                total_dma_idle_pct += idle_pct
457
458    avg_compute_idle = total_compute_idle_pct / len(active_threads)
459    avg_dma_idle = total_dma_idle_pct / len(active_threads)
460
461    logger.info("  Combined Idle Statistics:")
462    logger.info(f"    Active Threads   : {', '.join(str(t) for t in active_threads)}")
463    logger.info(f"    Avg Thread Compute IDLE : {avg_compute_idle:.1f}%")
464    logger.info(f"    Avg Thread DMA IDLE     : {avg_dma_idle:.1f}%")
465    logger.info("-" * 100)
466
467    logger.info("  Per-Thread Idle Analysis:")
468    for t in active_threads:
469        stats = thread_stats[t]
470        thread_name = f"Thread {t:<2} (HVX)" if t != 10 else "Thread 10 (HMX)"
471        logger.info(f"    {thread_name} -> Compute Idle: {stats['compute_idle_pct']:.1f}% | DMA Idle: {stats['dma_idle_pct']:.1f}%")
472
473    all_bubbles = []
474    for t in active_threads:
475        stats = thread_stats[t]
476        assert isinstance(stats['dma_bubbles'], Iterable)
477        assert isinstance(stats['compute_bubbles'], Iterable)
478        for start, end, dur in stats['compute_bubbles']:
479            pct = (dur / batch_duration) * 100.0
480            all_bubbles.append((dur, f"Thread {t} Compute: bubble of {dur} cycles ({pct:.1f}%) at {start - op_start} to {end - op_start}"))
481        for start, end, dur in stats['dma_bubbles']:
482            pct = (dur / batch_duration) * 100.0
483            all_bubbles.append((dur, f"Thread {t} DMA    : bubble of {dur} cycles ({pct:.1f}%) at {start - op_start} to {end - op_start}"))
484
485    if all_bubbles:
486        logger.info("-" * 100)
487        logger.info(f"  Significant Bubbles (>= {bubble_threshold} cycles):")
488        all_bubbles.sort(key=lambda x: x[0], reverse=True)
489        for dur, desc in all_bubbles[:15]:
490            logger.info(f"    {desc}")
491    else:
492        logger.info("-" * 100)
493        logger.info(f"  No significant bubbles detected (all idle gaps < {bubble_threshold} cycles).")
494
495
496def print_ascii_summary(op_name, dims, types, usec, cycles, events):
497    logger.info("=" * 100)
498    logger.info(f"{op_name} ({dims} : {types}) - {usec} usec {cycles} cycles")
499    logger.info("=" * 100)
500
501    events = sorted(events, key=lambda e: e['cycles'])
502    if not events:
503        logger.info("  No trace events recorded.")
504        return
505
506    active_starts = {}
507    thread_totals = defaultdict(lambda: defaultdict(int))
508
509    for e in events:
510        t = e['thread']
511        evt = e['event']
512        info = e['info']
513        cyc = e['cycles']
514        state = e['state']
515
516        key = (t, evt, info)
517        if state == 'start':
518            active_starts[key] = cyc
519        elif state == 'stop':
520            if key in active_starts:
521                start_cyc = active_starts[key]
522                del active_starts[key]
523
524                if cyc >= start_cyc:
525                    dur = cyc - start_cyc
526                else:
527                    dur = (cyc + 0x100000000) - start_cyc
528
529                norm_evt = normalize_event_name(evt)
530                thread_totals[t][norm_evt] += dur
531
532    for t in sorted(thread_totals.keys()):
533        thread_name = f"Thread {t} (HVX)" if t != 10 else "Thread 10 (HMX)"
534        sorted_evts = sorted(thread_totals[t].items(), key=lambda item: item[0])
535
536        evt_strs = []
537        for evt, dur in sorted_evts:
538            pct = (dur / cycles * 100) if cycles > 0 else 0
539            evt_strs.append(f"{evt} {dur} ({pct:.1f}%)")
540
541        logger.info(f"  {thread_name:<16}: " + " | ".join(evt_strs))
542
543
544def generate_report(ops, top_n, width_overrides, sort_col, pmu_name=None):
545    if not ops:
546        logger.info("No valid records found.")
547        return
548
549    grouped = defaultdict(list)
550    for op in ops:
551        key = (op['name'], op['dims'], op['types'])
552        grouped[key].append(op)
553
554    group_stats = []
555    for (name, dims, types), group_ops in grouped.items():
556        usecs = [o['usec'] for o in group_ops]
557        cycles = [o['cycles'] for o in group_ops]
558        pmu_vals = [o['pmu_val'] for o in group_ops if o['pmu_val'] is not None]
559
560        avg_usec_val = statistics.mean(usecs)
561        count_val = len(group_ops)
562        tot_usec_val = avg_usec_val * count_val
563
564        group_stats.append({
565            'op':               name,
566            'dims':             dims,
567            'dtypes':           types,
568            'count':            str(count_val),
569            'max_usec':         str(max(usecs)),
570            'avg_usec':         f"{avg_usec_val:.2f}",
571            'tot_usec':         f"{tot_usec_val:.2f}",
572            'max_cycles':       str(max(cycles)),
573            'avg_cycles':       f"{statistics.mean(cycles):.2f}",
574            'max_pmu':          str(max(pmu_vals)) if pmu_vals else "0",
575            'avg_pmu':          f"{statistics.mean(pmu_vals):.2f}" if pmu_vals else "0.00",
576            # Numeric values for accurate sorting
577            '_sort_count':      count_val,
578            '_sort_max_usec':   max(usecs),
579            '_sort_avg_usec':   avg_usec_val,
580            '_sort_tot_usec':   tot_usec_val,
581            '_sort_max_cycles': max(cycles),
582            '_sort_avg_cycles': statistics.mean(cycles),
583            '_sort_max_pmu':    max(pmu_vals) if pmu_vals else 0,
584            '_sort_avg_pmu':    statistics.mean(pmu_vals) if pmu_vals else 0
585        })
586
587    # Sorting logic
588    actual_sort_key = COL_MAP[sort_col][2]
589    is_numeric    = actual_sort_key.startswith("_") or actual_sort_key == "count"
590    sorted_groups = sorted(group_stats, key=lambda x: x[actual_sort_key], reverse=is_numeric)[:top_n]
591
592    # Define initial column order
593    active_cols = ["op", "dims", "dtypes"]
594    if pmu_name:
595        active_cols += ["max-pmu", "avg-pmu"]
596    active_cols += ["tot-usec", "avg-usec", "avg-cycles", "max-usec", "max-cycles", "count"]
597
598    final_headers, final_keys, final_widths = [], [], []
599
600    for col_name in active_cols:
601        data_key, header_text, _ = COL_MAP[col_name]
602        if "pmu" in col_name and pmu_name:
603            header_text = header_text.replace("PMU", pmu_name)
604
605        natural_width = max([len(str(row[data_key])) for row in sorted_groups] + [len(header_text)])
606        target_width  = width_overrides.get(col_name, natural_width)
607
608        if target_width == 0:
609            continue
610
611        final_headers.append(header_text)
612        final_keys.append(data_key)
613        final_widths.append(target_width)
614
615    # Print Report
616    logger.info(f"\n# Profile Report (Top {top_n} Ops sorted by {sort_col})\n")
617    header_line = "| " + " | ".join(f"{h:<{final_widths[i]}}" for i, h in enumerate(final_headers)) + " |"
618    sep_line    = "| " + " | ".join("-" * final_widths[i] for i in range(len(final_headers))) + " |"
619    logger.info(header_line)
620    logger.info(sep_line)
621
622    for group in sorted_groups:
623        row_vals = []
624        for i, key in enumerate(final_keys):
625            val = str(group[key])
626            if len(val) > final_widths[i]:
627                val = val[:final_widths[i] - 3] + "..."
628            row_vals.append(f"{val:<{final_widths[i]}}")
629        logger.info("| " + " | ".join(row_vals) + " |")
630
631
632def main():
633    parser = argparse.ArgumentParser(description="Post-process Op profile info.")
634    parser.add_argument("logfile")
635    parser.add_argument("-n", "--top", type=int, default=100)
636    parser.add_argument("--sort", type=str, default="tot-usec", choices=list(COL_MAP.keys()))
637    parser.add_argument("--pmu-index", type=int)
638    parser.add_argument("--pmu-name", type=str)
639    parser.add_argument("--width", action='append', default=['dims:40'], help="Override column width, e.g. --width dims:50")
640    parser.add_argument("--timeline", type=str, nargs='?', const='summary', choices=["summary", "bubbles"],
641                        help="Output ASCII art event summary or thread idle bubble analysis (default: summary)")
642    parser.add_argument("--filter", type=str, help="Regex filter matching against the original profile-op line")
643    parser.add_argument("--device", type=str, help="Device to filter by (e.g. HTP0, HTP0:0) or 'split' to generate separate reports per device")
644
645    group = parser.add_mutually_exclusive_group()
646    group.add_argument("--head", type=int, help="Limit to first N ops")
647    group.add_argument("--tail", type=int, help="Limit to last N ops")
648
649    args = parser.parse_args()
650
651    logging.basicConfig(level=logging.INFO, format='%(message)s')
652
653    if "pmu" in args.sort and args.pmu_index is None:
654        logger.error(f"Cannot sort by '{args.sort}' without --pmu-index.")
655        sys.exit(1)
656
657    overrides = {}
658    if args.width:
659        for w in args.width:
660            try:
661                name, val = w.split(':')
662                overrides[name.lower()] = int(val)
663            except ValueError:
664                logger.warning(f"Invalid width format '{w}'")
665
666    final_pmu_name = (args.pmu_name or f"#{args.pmu_index}") if args.pmu_index is not None else None
667
668    op_filter_re = None
669    if args.filter:
670        try:
671            op_filter_re = re.compile(args.filter)
672        except re.error as e:
673            logger.error(f"Invalid regex filter: {e}")
674            sys.exit(1)
675
676    limit = args.head if args.head is not None else None
677    device_filter = args.device if (args.device and args.device != "split") else None
678    ops = parse_log(args.logfile, pmu_index=args.pmu_index, limit=limit, device_filter=device_filter, op_filter_re=op_filter_re)
679
680    if args.device and args.device != "split":
681        ops = [op for op in ops if device_matches(op['device'], args.device)]
682
683    if args.device == "split":
684        unique_devices = sorted(list(set(op['device'] for op in ops)))
685        for dev in unique_devices:
686            dev_ops = [op for op in ops if device_matches(op['device'], dev)]
687
688            if args.filter:
689                try:
690                    filter_re = re.compile(args.filter)
691                except re.error as e:
692                    logger.error(f"Invalid regex filter: {e}")
693                    sys.exit(1)
694                dev_ops = [op for op in dev_ops if filter_re.search(op['op_text'])]
695
696            if args.head is not None:
697                dev_ops = dev_ops[:args.head]
698            elif args.tail is not None:
699                dev_ops = dev_ops[-args.tail:]
700
701            logger.info("\n=========================================")
702            logger.info(f" Device: {dev}")
703            logger.info("=========================================")
704
705            if args.timeline:
706                for op in dev_ops:
707                    if args.timeline == "summary":
708                        print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'])
709                    elif args.timeline == "bubbles":
710                        print_bubbles_timeline(op)
711            else:
712                generate_report(dev_ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)
713    else:
714        if args.filter:
715            try:
716                filter_re = re.compile(args.filter)
717            except re.error as e:
718                logger.error(f"Invalid regex filter: {e}")
719                sys.exit(1)
720            ops = [op for op in ops if filter_re.search(op['op_text'])]
721
722        if args.head is not None or args.tail is not None:
723            ops_by_dev = defaultdict(list)
724            for op in ops:
725                ops_by_dev[op['device']].append(op)
726
727            filtered_ops = []
728            for dev in sorted(ops_by_dev.keys()):
729                dev_ops = ops_by_dev[dev]
730                if args.head is not None:
731                    dev_ops = dev_ops[:args.head]
732                elif args.tail is not None:
733                    dev_ops = dev_ops[-args.tail:]
734                filtered_ops.extend(dev_ops)
735            ops = filtered_ops
736
737        if args.timeline:
738            for op in ops:
739                if args.timeline == "summary":
740                    print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'])
741                elif args.timeline == "bubbles":
742                    print_bubbles_timeline(op)
743        else:
744            generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)
745
746
747if __name__ == "__main__":
748    main()