Source

`scripts/bench_gate.sh`

1#!/usr/bin/env bash

2# bench_gate.sh — cheatah::linalg::Fixed must never become slower than GLM.

3#

4# tests/benchmarks/fixed_glm_bench.cpp measures the COMPLETE overlap of the two APIs as

5# BM_<op>_fixed / BM_<op>_glm pairs. This gate runs them and fails if any pair regresses. Without it,

6# "Fixed is as fast as GLM" is a claim that was true once, on one commit, on one machine.

7#

8# Three things keep it from being a flaky gate:

9#

10# 1. A TOLERANCE. Most of these operations take one or two cycles, where measurement noise is a

11# larger effect than any code change; run-to-run sign flips of +/-5% are normal. A pair fails

12# only above THRESHOLD (default 1.15x).

13# 2. An ABSOLUTE FLOOR. A ratio is meaningless on a sub-nanosecond operation. `dot` on a 3-vector

14# of doubles compiles to instruction-identical code in both libraries — the same movsd/mulsd/

15# addsd sequence, verified by reading the assembly — and still measures ~0.09 ns apart, because

16# at that scale the harness's own DoNotOptimize scaffolding dominates. So a pair must ALSO be

17# slower by more than MIN_GAP_NS (default 0.25 ns, about one cycle) to fail. Anything under

18# that is reported, never fatal. The floor is one cycle, not half, because in the QA gate the

19# benchmarks run LAST — after the ASan/TSan/Valgrind stages have pinned the CPU for ~15 min —

20# so boost clocks are depressed and the smallest ops (a 16-double mat4 add is ~1.4 ns) drift a

21# few tenths of a nanosecond hotter than a cold run; that thermal tail must not fail the build.

22# 3. CONFIRMATION. Anything that trips both is re-measured, alone, with more repetitions. A single

23# noisy sample never fails the build; a real regression survives the second look.

24#

25# scripts/bench_gate.sh # gate (build if needed, then check)

26# THRESHOLD=1.10 scripts/bench_gate.sh

27# scripts/bench_gate.sh report # print every pair, fail nothing

28set -uo pipefail

29cd "$(git rev-parse --show-toplevel)"

31MODE="${1:-gate}"

32THRESHOLD="${THRESHOLD:-1.15}"

33MIN_GAP_NS="${MIN_GAP_NS:-0.25}"

34BIN=build/release/bin/cheatah_benchmarks

36bold() { printf '\n\033[1m[bench-gate] %s\033[0m\n' "$*"; }

37skip() { printf '\033[33m[bench-gate] SKIP: %s\033[0m\n' "$*"; exit 0; }

38fail() { printf '\033[31m[bench-gate] FAILED: %s\033[0m\n' "$*"; exit 1; }

40command -v cmake >/dev/null 2>&1 || skip "no cmake"

41if [ ! -x "$BIN" ]; then

42 bold "building the benchmarks (release)…"

43 cmake --preset release -DCHEATAH_BUILD_BENCHMARKS=ON >/tmp/cheatah_bench_cfg.log 2>&1 \

44 || skip "benchmark configure failed (GLM missing?) — see /tmp/cheatah_bench_cfg.log"

45 cmake --build --preset release --target cheatah_benchmarks -j"$(nproc)" >/tmp/cheatah_bench_build.log 2>&1 \

46 || skip "benchmark build failed — see /tmp/cheatah_bench_build.log"

47fi

48[ -x "$BIN" ] || skip "no benchmark binary"

50run_pairs() { # run_pairs <filter> <reps> <min_time> -> csv on stdout

51 # --benchmark_enable_random_interleaving shuffles the flat list of repetition slots across

52 # every filtered case (benchmark.cc RunBenchmarks), so a pair's _fixed and _glm repetitions

53 # are scattered through the run instead of being measured as two consecutive blocks. Without

54 # it, any clock or thermal drift over the block lands entirely on one side and shows up as

55 # a ratio change; with it, the same drift is zero-mean noise that the median absorbs. The

56 # iteration count is calibrated on the first repetition and reused for the rest

57 # (benchmark_runner.cc), so scattering the reps does not change what each one measures.

58 "$BIN" --benchmark_filter="$1" --benchmark_repetitions="$2" --benchmark_min_time="$3" \

59 --benchmark_enable_random_interleaving=true \

60 --benchmark_report_aggregates_only=true --benchmark_format=csv 2>/dev/null

61}

63# ---- pass 1: the SCREEN, pair-sharded across cores --------------------------------------------

64# Pass 1 only nominates suspects; pass 2 below re-measures them serially, alone, with more

65# repetitions, and is the ONLY place a failure is declared. That authority split is what makes

66# sharding the screen safe: concurrency noise can at worst nominate extra suspects, each of which

67# pass 2 then measures under the same quiet conditions as always. A pair's _fixed and _glm case

68# are kept in the SAME shard (the comparison is intra-pair, so shared-core noise hits both sides

69# alike); glm-only orphans become one-name units, measured exactly as before. BENCH_GATE_JOBS=1

70# reproduces the old single-process pass 1 verbatim. Default 8 (not nproc): a lightly-loaded box

71# keeps the suspect rate near zero, and pass 1 is already ~6x faster at 8.

72BENCH_GATE_JOBS="${BENCH_GATE_JOBS:-8}"

73esc() { printf '%s' "$1" | sed 's/[][\.|$(){}?+*^\\]/\\&/g'; }

75bold "measuring the fixed/glm pairs (pass 1 across ${BENCH_GATE_JOBS} shards)…"

76# Names in this subset never contain whitespace (PAIR-macro generated, BM_<op>_<type>_<side>);

77# if a future one did, the read below would mis-split and the completeness check fails loudly.

78PAIR_NAMES=()

79while IFS= read -r _n; do PAIR_NAMES+=("$_n"); done < <("$BIN" --benchmark_list_tests | grep -E '_(fixed|glm)$')

80[ "${#PAIR_NAMES[@]}" -gt 0 ] || fail "no fixed/glm benchmark cases listed"

82# Group names into pair units: key = name minus its _fixed/_glm suffix. A stable sort by key

83# makes a pair's two sides ADJACENT (fixed before glm, registration order preserved within a

84# key), so one walk builds each unit's alternation. Plain indexed arrays only — macOS stock

85# bash is 3.2 (no associative arrays) and the gate runs there too.

86UNIT_FILTERS=(); _prev_key=""

87while read -r _k _n; do

88 _e=$(esc "$_n")

89 if [ "$_k" = "$_prev_key" ]; then

90 _last=$(( ${#UNIT_FILTERS[@]} - 1 ))

91 UNIT_FILTERS[_last]="${UNIT_FILTERS[_last]}|$_e"

92 else

93 UNIT_FILTERS+=("$_e"); _prev_key="$_k"

94 fi

95done < <(for _n in "${PAIR_NAMES[@]}"; do

96 _k="${_n%_fixed}"; _k="${_k%_glm}"; printf '%s %s\n' "$_k" "$_n"

97 done | sort -s -k1,1)

98shards=$(( BENCH_GATE_JOBS < ${#UNIT_FILTERS[@]} ? BENCH_GATE_JOBS : ${#UNIT_FILTERS[@]} ))

99SHARD_FILTER=()

100for ((i = 0; i < shards; i++)); do SHARD_FILTER[i]=""; done

101for ((i = 0; i < ${#UNIT_FILTERS[@]}; i++)); do

102 s=$((i % shards))

103 SHARD_FILTER[s]="${SHARD_FILTER[s]}${SHARD_FILTER[s]:+|}${UNIT_FILTERS[i]}"

104done

105

106pass1_pids=()

107for ((i = 0; i < shards; i++)); do

108 : > "/tmp/cheatah_bench_pass1_shard$i.csv" # truncate: a lost shard must yield 0 rows

109 # 7 reps, not 5: interleaving raises the per-repetition variance of the sub-nanosecond

110 # cases (each rep now starts cold rather than warm behind its own predecessor), and pass 1

111 # is only a screen. Two extra reps keep the suspect rate near zero so pass 2 stays cheap.

112 run_pairs "^(${SHARD_FILTER[i]})"'$' 7 0.2s > "/tmp/cheatah_bench_pass1_shard$i.csv" &

113 pass1_pids+=($!)

114done

115for p in "${pass1_pids[@]}"; do wait "$p" || fail "benchmark run"; done

116: > /tmp/cheatah_bench_pass1.csv

117for ((i = 0; i < shards; i++)); do cat "/tmp/cheatah_bench_pass1_shard$i.csv" >> /tmp/cheatah_bench_pass1.csv; done

118# Completeness: one _median aggregate row per case; the union must equal the listed set.

119measured=$(grep -cE '^"?BM_[^,]*_median"?,' /tmp/cheatah_bench_pass1.csv || true)

120[ "${measured:-0}" -eq "${#PAIR_NAMES[@]}" ] || \

121 fail "pass 1 incomplete: measured ${measured:-0}/${#PAIR_NAMES[@]} fixed/glm cases (a shard was lost?)"

122

123SUSPECTS="$(python3 - "$THRESHOLD" "$MIN_GAP_NS" <<'PY'

124import csv, sys

125threshold, min_gap = float(sys.argv[1]), float(sys.argv[2])

126t = {}

127for row in csv.reader(open('/tmp/cheatah_bench_pass1.csv')):

128 if not row or not row[0].startswith('BM_') or not row[0].endswith('_median'):

129 continue

130 name, ns = row[0][:-len('_median')], float(row[2])

131 if name.endswith('_fixed'):

132 t.setdefault(name[3:-6], {})['f'] = ns

133 elif name.endswith('_glm'):

134 t.setdefault(name[3:-4], {})['g'] = ns

135pairs = {k: v for k, v in t.items() if 'f' in v and 'g' in v}

136if not pairs:

137 sys.stderr.write("no fixed/glm pairs found — did the benchmark names change?\n")

138 sys.exit(2)

139print(' '.join(k for k, v in pairs.items()

140 if v['g'] > 0 and v['f'] / v['g'] > threshold and (v['f'] - v['g']) > min_gap))

141PY

142)" || fail "could not parse the benchmark output"

143

144if [ -z "$SUSPECTS" ]; then

145

    printf '\n\033[32m[bench-gate] OK — Fixed is within %sx (or %s ns) of GLM on every operation.\033[0m\n' "$THRESHOLD" "$MIN_GAP_NS"

146 exit 0

147fi

148

149bold "re-measuring under suspicion (noise, or a real regression?): $SUSPECTS"

150FILTER="BM_($(echo "$SUSPECTS" | tr ' ' '|'))_(fixed|glm)$"

151run_pairs "$FILTER" 15 0.5s > /tmp/cheatah_bench_pass2.csv || fail "confirmation run"

152

153python3 - "$THRESHOLD" "$MIN_GAP_NS" "$MODE" <<'PY' || fail "Fixed regressed against GLM (see above)"

154import csv, sys

155threshold, min_gap, mode = float(sys.argv[1]), float(sys.argv[2]), sys.argv[3]

156t = {}

157for row in csv.reader(open('/tmp/cheatah_bench_pass2.csv')):

158 if not row or not row[0].startswith('BM_') or not row[0].endswith('_median'):

159 continue

160 name, ns = row[0][:-len('_median')], float(row[2])

161 if name.endswith('_fixed'):

162 t.setdefault(name[3:-6], {})['f'] = ns

163 elif name.endswith('_glm'):

164 t.setdefault(name[3:-4], {})['g'] = ns

165

166bad = []

167for k, v in sorted(t.items()):

168 if 'f' not in v or 'g' not in v or v['g'] <= 0:

169 continue

170 ratio, gap = v['f'] / v['g'], v['f'] - v['g']

171 regressed = ratio > threshold and gap > min_gap

172 verdict = 'REGRESSED' if regressed else ('below-floor' if ratio > threshold else 'noise')

173 print(f" {verdict:12s} {k:22s} fixed {v['f']:8.3f} ns glm {v['g']:8.3f} ns {ratio:.2f}x (+{gap:.3f} ns)")

174 if regressed:

175 bad.append((k, ratio, gap))

176

177if bad and mode != 'report':

178 print()

179 for k, ratio, gap in bad:

180 print(f" {k} is {ratio:.2f}x GLM (+{gap:.3f} ns) and stayed that way under 15 repetitions.")

181 sys.exit(1)

182PY

183

184printf '\n\033[32m[bench-gate] OK — every flagged pair was noise; Fixed holds against GLM.\033[0m\n'