Source
scripts/perf_suite.py
1
#!/usr/bin/env python32
"""The cheatah Performance-row suite — regenerates per-function speed numbers.4
This is the PERIODIC suite (NOT part of the QA gate — benchmarks are slow, noisy, and5
machine-specific). Run it now and then on a fixed reference machine — after a codegen/6
stdlib change, or a new CPython release — and commit the regenerated7
`docs/perf_data.json`. The docs generator reads that file and renders a "Performance"8
row on every function it covers, so the numbers live in ONE generated, provenance-9
tagged file instead of being hand-written into 218 headers.11
Each case times a real cheatah `.purr` (compiled, opaque library call) against the12
equivalent CPython, ELISION-PROOF: the input varies each iteration, results are13
accumulated, and the total is printed — so the compiler can't delete, fold, or hoist14
the body (an empty-loop guard flags any case that slips through). Functions with no15
honest Python twin get a cheatah-only row; numpy-backed numeric ops point at the16
dedicated NumPy comparison on the performance page.18
STRIATED, WITH STATISTICS. This suite used to time each side as its own consecutive19
block of TRIALS runs and report the MINIMUM of each. That is two separate problems.20
A minimum has no dispersion, so a case that swung 40% between runs was indistinguishable21
from one that was rock steady; and taking the two minima independently pairs cheatah's22
luckiest run against CPython's, which is not a measurement of anything. Worse, running23
all of one language's trials and then all of the other's means any thermal or clock24
drift over the window lands entirely on one side of the ratio.26
So a case is now measured in ROUNDS passes, and each pass runs the cheatah side and the27
CPython side ADJACENTLY. Each pass yields one PAIRED ratio, and the headline speedup is28
the median of those ratios — the estimator that stays unbiased when the machine drifts29
under both sides at once. Each side additionally reports its own median and IQR, and the30
range of the per-round ratios is kept so a reader can see how stable the number is.32
python3 scripts/perf_suite.py # run everything, rewrite perf_data.json33
python3 scripts/perf_suite.py math io # only the named modules (faster iteration)34
"""35
import json36
import os37
import platform38
import subprocess39
import sys40
import tempfile41
import time43
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))44
PURRC = os.path.join(ROOT, "build", "release", "bin", "purrc")45
CHEATAH = os.path.join(ROOT, "build", "release", "bin", "cheatah")46
OUT = os.path.join(ROOT, "docs", "perf_data.json")47
# 7 rounds, not 3 trials: enough for a median with a real middle and a meaningful IQR,48
# while keeping a full sweep of the suite to a sitting.49
ROUNDS = 752
def C(body, py=None, imp="", setup="", py_setup=None, acc="0.0", iters=10_000_000,53
vs="cpython"):54
"""A compared/cheatah-only case. `body`/`py` accumulate into `acc` per iteration.55
py=None → cheatah-only row (no honest Python twin). `vs` is the comparison target56
label ("cpython" or "numpy") — numeric modules with a NumPy equivalent compare57
against NumPy (the fast baseline), not pure Python."""58
return dict(kind="compared" if py else "cheatah_only", body=body, py=py, imp=imp,59
setup=setup, py_setup=py_setup, acc=acc, iters=iters, vs=vs)62
def NUMPY():63
"""A numeric op whose honest comparison is vs NumPy (see the performance page)."""64
return dict(kind="numpy")67
def NOTE(text):68
"""A function we deliberately don't micro-benchmark (I/O-bound, blocking, etc.)."""69
return dict(kind="note", note=text)72
# ---------------------------------------------------------------------------73
# Registry — keyed by the docs name `<module>.<function>`.74
# ---------------------------------------------------------------------------75
REG = {}77
# ---- math: clean Python twins (math module / builtins) --------------------78
_MATH = {79
"sqrt": C("acc = acc + math.sqrt(1.0 + i)", "acc = acc + math.sqrt(1.0 + i)"),80
"cbrt": C("acc = acc + math.cbrt(1.0 + i)", "acc = acc + (1.0 + i) ** (1.0/3.0)"),81
"fabs": C("acc = acc + math.fabs(-1.0 * i)", "acc = acc + math.fabs(-1.0 * i)"),82
"floor": C("acc = acc + math.floor(0.5 + 0.001 * i)", "acc = acc + math.floor(0.5 + 0.001 * i)"),83
"ceil": C("acc = acc + math.ceil(0.5 + 0.001 * i)", "acc = acc + math.ceil(0.5 + 0.001 * i)"),84
"trunc": C("acc = acc + math.trunc(0.5 + 0.001 * i)", "acc = acc + math.trunc(0.5 + 0.001 * i)"),85
"round": C("acc = acc + math.round(0.5 + 0.001 * i)", "acc = acc + round(0.5 + 0.001 * i)"),86
"exp": C("acc = acc + math.exp(0.0000001 * i)", "acc = acc + math.exp(0.0000001 * i)"),87
"log": C("acc = acc + math.log(1.0 + i)", "acc = acc + math.log(1.0 + i)"),88
"log10": C("acc = acc + math.log10(1.0 + i)", "acc = acc + math.log10(1.0 + i)"),89
"log2": C("acc = acc + math.log2(1.0 + i)", "acc = acc + math.log2(1.0 + i)"),90
"sin": C("acc = acc + math.sin(0.001 * i)", "acc = acc + math.sin(0.001 * i)"),91
"cos": C("acc = acc + math.cos(0.001 * i)", "acc = acc + math.cos(0.001 * i)"),92
"tan": C("acc = acc + math.tan(0.001 * i)", "acc = acc + math.tan(0.001 * i)"),93
"asin": C("acc = acc + math.asin(math.sin(0.001 * i))", "acc = acc + math.asin(math.sin(0.001 * i))"),94
"acos": C("acc = acc + math.acos(math.sin(0.001 * i))", "acc = acc + math.acos(math.sin(0.001 * i))"),95
"atan": C("acc = acc + math.atan(0.001 * i)", "acc = acc + math.atan(0.001 * i)"),96
"atan2": C("acc = acc + math.atan2(1.0 + i, 2.0)", "acc = acc + math.atan2(1.0 + i, 2.0)"),97
"hypot": C("acc = acc + math.hypot(1.0 * i, 2.0)", "acc = acc + math.hypot(1.0 * i, 2.0)"),98
"fmod": C("acc = acc + math.fmod(1.0 + i, 7.0)", "acc = acc + math.fmod(1.0 + i, 7.0)"),99
"copysign": C("acc = acc + math.copysign(2.0, -1.0 * i)", "acc = acc + math.copysign(2.0, -1.0 * i)"),100
"degrees": C("acc = acc + math.degrees(0.01 * i)", "acc = acc + math.degrees(0.01 * i)"),101
"radians": C("acc = acc + math.radians(0.01 * i)", "acc = acc + math.radians(0.01 * i)"),102
"pow": C("acc = acc + math.pow(1.0 + 0.000001 * i, 2.0)", "acc = acc + math.pow(1.0 + 0.000001 * i, 2.0)"),103
"abs": C("acc = acc + math.abs(-1.0 * i)", "acc = acc + abs(-1.0 * i)"),104
"min": C("acc = acc + math.min(1.0 * i, 2.0)", "acc = acc + min(1.0 * i, 2.0)"),105
"max": C("acc = acc + math.max(1.0 * i, 2.0)", "acc = acc + max(1.0 * i, 2.0)"),106
"gcd": C("acc = acc + math.gcd(i + 1, 48)", "acc = acc + math.gcd(i + 1, 48)", iters=2_000_000),107
"factorial": C("acc = acc + math.factorial(10)", "acc = acc + math.factorial(10)", iters=2_000_000),108
"isnan": C("if math.isnan(1.0 + i) { acc = acc + 1.0 }", "acc = acc + (1.0 if math.isnan(1.0 + i) else 0.0)"),109
"isinf": C("if math.isinf(1.0 + i) { acc = acc + 1.0 }", "acc = acc + (1.0 if math.isinf(1.0 + i) else 0.0)"),110
"isfinite": C("if math.isfinite(1.0 + i) { acc = acc + 1.0 }", "acc = acc + (1.0 if math.isfinite(1.0 + i) else 0.0)"),111
}112
for k, v in _MATH.items():113
v["imp"] = "math"114
REG[f"math.{k}"] = v116
# ---- string: cheatah `string.f(s)` vs Python `s.f()` (str methods) --------117
# These are opaque library calls (string.cpp), so a fixed input is fine — the call118
# runs every iteration. Bool results are consumed via a branch; others via len/+.119
_SS, _PSS = 'let s = "Hello, World! 123 hello"', 's = "Hello, World! 123 hello"'120
_STR = {121
"upper": C("acc = acc + len(string.upper(s))", "acc = acc + len(s.upper())"),122
"lower": C("acc = acc + len(string.lower(s))", "acc = acc + len(s.lower())"),123
"capitalize": C("acc = acc + len(string.capitalize(s))", "acc = acc + len(s.capitalize())"),124
"title": C("acc = acc + len(string.title(s))", "acc = acc + len(s.title())"),125
"swapcase": C("acc = acc + len(string.swapcase(s))", "acc = acc + len(s.swapcase())"),126
"strip": C("acc = acc + len(string.strip(s))", "acc = acc + len(s.strip())"),127
"lstrip": C("acc = acc + len(string.lstrip(s))", "acc = acc + len(s.lstrip())"),128
"rstrip": C("acc = acc + len(string.rstrip(s))", "acc = acc + len(s.rstrip())"),129
"replace": C('acc = acc + len(string.replace(s, "l", "L"))', 'acc = acc + len(s.replace("l", "L"))'),130
"center": C("acc = acc + len(string.center(s, 40))", "acc = acc + len(s.center(40))"),131
"ljust": C("acc = acc + len(string.ljust(s, 40))", "acc = acc + len(s.ljust(40))"),132
"rjust": C("acc = acc + len(string.rjust(s, 40))", "acc = acc + len(s.rjust(40))"),133
"zfill": C("acc = acc + len(string.zfill(s, 40))", "acc = acc + len(s.zfill(40))"),134
"count": C('acc = acc + string.count(s, "l")', 'acc = acc + s.count("l")'),135
"find": C('acc = acc + string.find(s, "World")', 'acc = acc + s.find("World")'),136
"rfind": C('acc = acc + string.rfind(s, "l")', 'acc = acc + s.rfind("l")'),137
"split": C("acc = acc + len(string.split(s))", "acc = acc + len(s.split())"),138
"splitlines": C("acc = acc + len(string.splitlines(s))", "acc = acc + len(s.splitlines())"),139
"startswith": C('if string.startswith(s, "Hello") { acc = acc + 1.0 }', "acc = acc + (1.0 if s.startswith('Hello') else 0.0)"),140
"endswith": C('if string.endswith(s, "hello") { acc = acc + 1.0 }', "acc = acc + (1.0 if s.endswith('hello') else 0.0)"),141
"contains": C('if string.contains(s, "World") { acc = acc + 1.0 }', "acc = acc + (1.0 if 'World' in s else 0.0)"),142
"isalnum": C("if string.isalnum(s) { acc = acc + 1.0 }", "acc = acc + (1.0 if s.isalnum() else 0.0)"),143
"isalpha": C("if string.isalpha(s) { acc = acc + 1.0 }", "acc = acc + (1.0 if s.isalpha() else 0.0)"),144
"isdigit": C("if string.isdigit(s) { acc = acc + 1.0 }", "acc = acc + (1.0 if s.isdigit() else 0.0)"),145
"islower": C("if string.islower(s) { acc = acc + 1.0 }", "acc = acc + (1.0 if s.islower() else 0.0)"),146
"isupper": C("if string.isupper(s) { acc = acc + 1.0 }", "acc = acc + (1.0 if s.isupper() else 0.0)"),147
"isspace": C("if string.isspace(s) { acc = acc + 1.0 }", "acc = acc + (1.0 if s.isspace() else 0.0)"),148
}149
for k, v in _STR.items():150
v.update(imp="string", setup=_SS, py_setup=_PSS, iters=2_000_000)151
REG[f"string.{k}"] = v152
# join + capwords need their own operands.153
REG["string.join"] = C('acc = acc + len(string.join(", ", parts))', 'acc = acc + len(", ".join(parts))',154
imp="string", setup='let parts = ["alpha", "beta", "gamma", "delta"]',155
py_setup='parts = ["alpha", "beta", "gamma", "delta"]', iters=2_000_000)156
REG["string.capwords"] = C("acc = acc + len(string.capwords(s))", "acc = acc + len(string.capwords(s))",157
imp="string", setup=_SS, py_setup="import string\n" + _PSS, iters=2_000_000)159
# ---- statistics: vs NumPy (the fast numeric baseline; Python's `statistics`160
# module uses slow exact arithmetic, so it's not the honest comparison) -------161
_XS_LIST = "[4.0, 8.0, 15.0, 16.0, 23.0, 42.0, 1.0, 2.0, 3.0, 5.0, 7.0, 11.0, 13.0, 17.0, 19.0, 29.0]"162
_XS = f"let xs = {_XS_LIST}"163
# cheatah call -> NumPy equivalent. statistics.hpp is header-only, so a constant list164
# folds; we feed `acc` back into xs[0] (a loop-carried dependency, bounded via fmod)165
# so the O(n) reduction genuinely runs every iteration on BOTH sides.166
_STAT = {167
"mean": ("statistics.mean(xs)", "np.mean(xs)"),168
"median": ("statistics.median(xs)", "np.median(xs)"),169
"pstdev": ("statistics.pstdev(xs)", "np.std(xs)"),170
"pvariance": ("statistics.pvariance(xs)", "np.var(xs)"),171
"stdev": ("statistics.stdev(xs)", "np.std(xs, ddof=1)"),172
"variance": ("statistics.variance(xs)", "np.var(xs, ddof=1)"),173
"sum": ("statistics.sum(xs)", "np.sum(xs)"),174
"count": ("statistics.count(xs)", "xs.size"),175
}176
for k, (call, npcall) in _STAT.items():177
REG[f"statistics.{k}"] = C(178
f"xs[0] = math.fmod(acc, 100.0) + 1.0\n acc = acc + {call}",179
f"xs[0] = math.fmod(acc, 100.0) + 1.0; acc = acc + {npcall}",180
imp="statistics math", setup=_XS,181
py_setup=f"import numpy as np\nimport math\nxs = np.array({_XS_LIST})",182
iters=1_000_000, vs="numpy")184
# ---- hashlib / html: opaque library calls with clean twins ----------------185
REG["hashlib.sha256"] = C("acc = acc + len(hashlib.sha256(io.str(i)))",186
"acc = acc + len(hashlib.sha256(str(i).encode()).hexdigest())",187
imp="hashlib", iters=500_000)188
REG["parsers.html.escape"] = C("acc = acc + len(parsers.html.escape(s))", "acc = acc + len(html.escape(s))",189
imp="parsers.html", setup='let s = "<a href=\\"x\\">A & B < C > D</a>"',190
py_setup='import html\ns = "<a href=\\"x\\">A & B < C > D</a>"', iters=2_000_000)191
REG["parsers.html.unescape"] = C("acc = acc + len(parsers.html.unescape(s))", "acc = acc + len(html.unescape(s))",192
imp="parsers.html", setup='let s = "A & B < C > D "q""',193
py_setup='import html\ns = "A & B < C > D "q""', iters=2_000_000)195
# ---- os.path: path-string ops (opaque, clean twins) ----------------------196
_P, _PP = 'let p = "/usr/local/bin/python3.12"', 'import os.path\np = "/usr/local/bin/python3.12"'197
for k in ["basename", "dirname", "normpath", "abspath"]:198
REG[f"os.path.{k}"] = C(f"acc = acc + len(os.path.{k}(p))", f"acc = acc + len(os.path.{k}(p))",199
imp="os", setup=_P, py_setup=_PP, iters=2_000_000)200
REG["os.path.join"] = C('acc = acc + len(os.path.join("/usr/local", "bin"))',201
'acc = acc + len(os.path.join("/usr/local", "bin"))',202
imp="os", py_setup="import os.path", iters=2_000_000)203
for k in ["exists", "isfile", "isdir"]: # stat() syscall each call204
REG[f"os.path.{k}"] = C(f'if os.path.{k}("/usr/bin") {{ acc = acc + 1.0 }}',205
f'acc = acc + (1.0 if os.path.{k}("/usr/bin") else 0.0)',206
imp="os", py_setup="import os.path", iters=500_000)207
REG["os.path.getsize"] = C('acc = acc + os.path.getsize("/etc/passwd")',208
'acc = acc + os.path.getsize("/etc/passwd")',209
imp="os", py_setup="import os.path", iters=500_000)210
REG["os.path.splitext"] = NOTE("returns a (root, ext) pair — not reduced to one scalar here")212
# ---- os: read-only syscalls (clean twins); mutating ones are noted above --213
REG["os.getpid"] = C("acc = acc + os.getpid()", "acc = acc + os.getpid()", imp="os", py_setup="import os", iters=2_000_000)214
REG["os.getcwd"] = C("acc = acc + len(os.getcwd())", "acc = acc + len(os.getcwd())", imp="os", py_setup="import os", iters=1_000_000)215
REG["os.cpu_count"] = C("acc = acc + os.cpu_count()", "acc = acc + os.cpu_count()", imp="os", py_setup="import os", iters=2_000_000)216
REG["os.getenv"] = C('acc = acc + len(os.getenv("PATH"))', 'acc = acc + len(os.getenv("PATH"))', imp="os", py_setup="import os", iters=1_000_000)218
# ---- datetime: calendar ops over an epoch. format() builds a medium-length219
# timestamp string (the natural string workload here); the component extractors run220
# on the same representative epoch. vs CPython's datetime (fromtimestamp + strftime /221
# the component attributes). ----------------------------------------------------222
_DT = 'let e = 1700000000.0\nlet fmt = "%a %b %d %H:%M:%S %Y"' # -> "Tue Nov 14 22:13:20 2023" (24 chars)223
_DTP = 'import datetime as _dt\ne = 1700000000.0\nfmt = "%a %b %d %H:%M:%S %Y"'224
REG["datetime.format"] = C('acc = acc + len(datetime.format(e, fmt))',225
'acc = acc + len(_dt.datetime.fromtimestamp(e).strftime(fmt))',226
imp="datetime", setup=_DT, py_setup=_DTP, iters=1_000_000)227
for k in ["year", "month", "day", "hour", "minute", "second", "weekday"]:228
_pyattr = "weekday()" if k == "weekday" else k229
REG[f"datetime.{k}"] = C(f"acc = acc + datetime.{k}(e)",230
f"acc = acc + _dt.datetime.fromtimestamp(e).{_pyattr}",231
imp="datetime", setup=_DT, py_setup=_DTP, iters=2_000_000)232
REG["datetime.now"] = C("acc = acc + len(datetime.now())", "acc = acc + len(str(_dt.datetime.now()))",233
imp="datetime", py_setup="import datetime as _dt", iters=300_000)234
REG["datetime.utcnow"] = C("acc = acc + len(datetime.utcnow())", "acc = acc + len(str(_dt.datetime.utcnow()))",235
imp="datetime", py_setup="import datetime as _dt", iters=300_000)236
REG["datetime.today"] = C("acc = acc + len(datetime.today())", "acc = acc + len(str(_dt.date.today()))",237
imp="datetime", py_setup="import datetime as _dt", iters=300_000)238
REG["datetime.timestamp"] = C("acc = acc + datetime.timestamp()", "acc = acc + _dt.datetime.now().timestamp()",239
imp="datetime", py_setup="import datetime as _dt", iters=500_000)241
# ---- random: vs NumPy's random (the fast/vectorized equivalent). NumPy's242
# strength is bulk array generation; per-scalar it carries dispatch overhead. -----243
_NPR = "import numpy as np\nrng = np.random.default_rng(1)"244
REG["random.random"] = C("acc = acc + random.random()", "acc = acc + rng.random()",245
imp="random", setup="random.seed(1)", py_setup=_NPR, vs="numpy")246
REG["random.randint"] = C("acc = acc + random.randint(1, 100)", "acc = acc + int(rng.integers(1, 101))",247
imp="random", setup="random.seed(1)", py_setup=_NPR, vs="numpy")248
REG["random.uniform"] = C("acc = acc + random.uniform(0.0, 1.0)", "acc = acc + rng.uniform(0.0, 1.0)",249
imp="random", setup="random.seed(1)", py_setup=_NPR, vs="numpy")250
REG["random.gauss"] = C("acc = acc + random.gauss(0.0, 1.0)", "acc = acc + rng.normal(0.0, 1.0)",251
imp="random", setup="random.seed(1)", py_setup=_NPR, vs="numpy")252
REG["random.choice"] = C("acc = acc + random.choice(xs)", "acc = acc + rng.choice(xs)",253
imp="random", setup="random.seed(1)\nlet xs = [10.0, 20.0, 30.0, 40.0, 50.0]",254
py_setup=_NPR + "\nxs = np.array([10.0, 20.0, 30.0, 40.0, 50.0])", vs="numpy")255
REG["random.seed"] = NOTE("one-time initialization — not a hot path")257
# ---- time: clock reads (syscall/vDSO); same on both sides -----------------258
for k in ["monotonic", "perf_counter", "process_time", "time"]:259
REG[f"time.{k}"] = C(f"acc = acc + time.{k}()", f"acc = acc + time.{k}()",260
imp="time", py_setup="import time", iters=2_000_000)261
for k in ["monotonic_ns", "perf_counter_ns", "time_ns"]:262
REG[f"time.{k}"] = C(f"acc = acc + time.{k}()", f"acc = acc + time.{k}()",263
imp="time", py_setup="import time", iters=2_000_000)264
REG["time.sleep"] = NOTE("blocks for a requested duration — not micro-benchmarked")266
# ---- io: str/repr have clean twins; format/input/open/read_file noted ------267
REG["io.str"] = C("acc = acc + len(io.str(i))", "acc = acc + len(str(i))", iters=2_000_000)268
REG["io.repr"] = C("acc = acc + len(io.repr(1.0 * i))", "acc = acc + len(repr(1.0 * i))", iters=2_000_000)269
REG["io.format"] = NOTE("string templating — closest CPython twin (f-strings) isn't a function call")271
# ---- builtins: int-input ones measure cleanly; string-input ones inline ----272
REG["builtins.hex"] = C("acc = acc + len(hex(i))", "acc = acc + len(hex(i))", iters=5_000_000)273
REG["builtins.bin"] = C("acc = acc + len(bin(i))", "acc = acc + len(bin(i))", iters=5_000_000)274
REG["builtins.oct"] = C("acc = acc + len(oct(i))", "acc = acc + len(oct(i))", iters=5_000_000)275
REG["builtins.chr"] = C("acc = acc + ord(chr(65 + (i - (i / 26) * 26)))",276
"acc = acc + ord(chr(65 + i % 26))", iters=5_000_000)277
for k in ["len", "ord", "ascii", "hash", "bool", "int", "float", "contains", "startswith",278
"endswith", "index", "slice", "range", "append", "to_bool", "to_int", "to_float"]:279
REG[f"builtins.{k}"] = NOTE("header-inlined to ~sub-nanosecond; the win over CPython "280
"is its eliminated ~60 ns per-call interpreter overhead")282
# ---- parsers.html: object/structure returns, not one scalar (datetime is283
# benchmarked above against CPython's datetime) -----------------------------284
for k in ["get_attr", "has_attr", "parse"]:285
REG[f"parsers.html.{k}"] = NOTE("returns a parse structure — not reduced to one scalar here")287
# ---- numeric ops: the honest comparison is vs NumPy (perf page) -----------288
for fn in ["cholesky", "cond", "conj_transpose", "det", "dot", "eig", "eigh", "eigvals",289
"eigvalsh", "inner", "inv", "kron", "lstsq", "matmul", "matrix_power",290
"matrix_rank", "norm", "outer", "pinv", "qr", "slogdet", "solve", "svd",291
"trace", "vdot"]:292
REG[f"linalg.{fn}"] = NUMPY()293
REG["linalg.simd_features"] = NOTE("queries CPU SIMD support — not a hot path")294
REG["linalg.simd_lane_doubles"] = NOTE("queries CPU SIMD width — not a hot path")295
for fn in ["abs", "add", "arange", "array", "binary_op", "broadcast_shapes",296
"broadcast_to", "cbrt", "complex", "conj", "cos", "divide", "exp", "full",297
"get", "imag", "is_contiguous", "log", "mean", "mul", "ones", "real",298
"reshape", "scalar", "shape_of", "size_of", "sin", "sqrt", "sub", "sum",299
"tan", "to_string", "zeros"]:300
REG[f"ndarray.{fn}"] = NUMPY()302
# ---- I/O / system / network: no honest in-loop Python twin ----------------303
for fn in ["input", "open", "read_file", "print"]:304
REG[f"io.{fn}"] = NOTE("I/O-bound — dominated by the OS, not micro-benchmarked")305
for fn in ["accept", "bind", "close", "connect", "last_error", "listen", "local_port",306
"recv", "send", "sendall", "set_reuseaddr", "socket", "tcp_connect", "tcp_listen"]:307
REG[f"socket.{fn}"] = NOTE("network/syscall-bound — not micro-benchmarked")308
for fn in ["chdir", "listdir", "makedirs", "mkdir", "remove",309
"rename", "rmdir", "setenv", "system"]:310
REG[f"os.{fn}"] = NOTE("filesystem/process syscall — not micro-benchmarked")313
# ---------------------------------------------------------------------------314
# Timing315
# ---------------------------------------------------------------------------316
def _run(argv):317
return subprocess.run(argv, capture_output=True, text=True)320
def _imports(imp):321
return "".join(f"import {m}\n" for m in imp.split()) if imp else ""324
def _machine_string():325
"""A machine string someone could actually reproduce on. platform.processor() returns326
bare "x86_64" on Linux, which identifies nothing — read the real model name out of327
/proc/cpuinfo and note whether frequency scaling was left on, since that is the single328
biggest source of run-to-run drift on a laptop part."""329
model = platform.processor() or platform.machine()330
try:331
for line in open("/proc/cpuinfo"):332
if line.startswith("model name"):333
model = line.split(":", 1)[1].strip()334
break335
except OSError:336
pass337
try:338
gov = open("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor").read().strip()339
model += f" (governor={gov})"340
except OSError:341
pass342
return f"{model}, {os.cpu_count()} CPUs, {platform.system()} {platform.release()}"345
def _median(xs):346
ys = sorted(xs)347
n = len(ys)348
return ys[n // 2] if n % 2 else 0.5 * (ys[n // 2 - 1] + ys[n // 2])351
def _iqr(xs):352
"""Inter-quartile range. Preferred over stddev because a timing sample is not normal:353
one descheduled run moves a standard deviation far more than it moves the middle 50%."""354
if len(xs) < 4:355
return 0.0356
ys = sorted(xs)358
def q(p):359
pos = p * (len(ys) - 1)360
lo = int(pos)361
hi = min(lo + 1, len(ys) - 1)362
return ys[lo] + (pos - lo) * (ys[hi] - ys[lo])364
return q(0.75) - q(0.25)367
def cheatah_src(case, body):368
return ("import io\nimport time\n" + _imports(case["imp"]) + case["setup"] + "\n"369
f"let acc = {case['acc']}\n"370
"let t0 = time.monotonic()\n"371
f"for i in range(0, {case['iters']}) {{\n {body}\n}}\n"372
"let t1 = time.monotonic()\n"373
"io.print(acc)\nio.print(t1 - t0)\n")376
def python_src(case):377
setup = case["py_setup"] if case["py_setup"] is not None else \378
"".join(f"import {m}\n" for m in case["imp"].split() if m != "io")379
return ("import time\n" + setup + "\n"380
+ f"acc = {case['acc']}\n"381
"t0 = time.monotonic()\n"382
f"for i in range({case['iters']}):\n {case['py']}\n"383
"t1 = time.monotonic()\nprint(acc)\nprint(t1 - t0)\n")386
def build_cheatah(case, body, d, stem):387
"""Compile ONCE, outside the timing loop. Compilation is not what we are measuring, and388
re-running purrc between rounds would put a multi-second gap between the two sides."""389
purr, so = os.path.join(d, stem + ".purr"), os.path.join(d, stem + ".so")390
open(purr, "w").write(cheatah_src(case, body))391
return so if _run([PURRC, purr, "-o", so]).returncode == 0 else None394
def _elapsed(argv):395
out = _run(argv).stdout.strip().splitlines()396
try:397
return float(out[-1])398
except (IndexError, ValueError):399
return None402
def measure(case):403
"""One case, ROUNDS striated rounds. Returns None if the cheatah side cannot be built404
or run; the CPython side may legitimately be absent (cheatah-only rows)."""405
compared = case["kind"] == "compared"406
with tempfile.TemporaryDirectory() as d:407
so = build_cheatah(case, case["body"], d, "b")408
if so is None:409
return None410
# The elision guard: an empty loop built from the same scaffolding. Timed in the411
# same rounds so it sees the same machine conditions as the body it is judging.412
so_empty = build_cheatah(case, "acc = acc", d, "e")413
py = python_src(case) if compared else None415
ch, cp, empty, ratios = [], [], [], []416
for _ in range(ROUNDS):417
# cheatah and CPython back to back — the whole point of the round.418
t = _elapsed([CHEATAH, so])419
if t is None:420
return None421
ch.append(t)422
if py is not None:423
p = _elapsed([sys.executable, "-c", py])424
if p is not None:425
cp.append(p)426
ratios.append(p / t) # PAIRED: same round, same conditions427
if so_empty is not None:428
e = _elapsed([CHEATAH, so_empty])429
if e is not None:430
empty.append(e)432
out = {"cheatah": _median(ch), "cheatah_iqr": _iqr(ch),433
"empty": _median(empty) if empty else None}434
if cp:435
out.update(compare=_median(cp), compare_iqr=_iqr(cp),436
speedup=_median(ratios), speedup_lo=min(ratios), speedup_hi=max(ratios))437
return out440
def main():441
if not (os.path.exists(PURRC) and os.path.exists(CHEATAH)):442
sys.exit("perf_suite: build the `release` preset first (need purrc + cheatah).")443
only = set(sys.argv[1:])444
commit = _run(["git", "-C", ROOT, "rev-parse", "--short", "HEAD"]).stdout.strip()445
results = {}446
existing = {}447
if os.path.exists(OUT):448
existing = {e["name"]: e for e in json.load(open(OUT)).get("functions", [])}449
for key, case in sorted(REG.items()):450
mod = key.split(".")[0]451
if only and mod not in only:452
results[key] = existing.get(key, case if case["kind"] in ("numpy", "note") else {})453
continue454
kind = case["kind"]455
if kind in ("numpy", "note"):456
results[key] = case457
print(f"{key:<26} {kind}")458
continue459
m = measure(case)460
if m is None:461
print(f"{key:<26} ⚠ cheatah failed")462
continue463
per_iter = 1e9 / case["iters"]464
ch_ns = m["cheatah"] * per_iter465
elided = m["empty"] is not None and m["cheatah"] < m["empty"] * 1.3466
row = {"kind": kind, "cheatah_ns": round(ch_ns, 2),467
"cheatah_ns_iqr": round(m["cheatah_iqr"] * per_iter, 2),468
"rounds": ROUNDS, "statistic": "median; spread = IQR over rounds"}469
if elided:470
row["warn"] = "elided"471
if kind == "compared":472
row["vs"] = case.get("vs", "cpython") # comparison target: cpython | numpy473
if "compare" in m:474
row["compare_ns"] = round(m["compare"] * per_iter, 2)475
row["compare_ns_iqr"] = round(m["compare_iqr"] * per_iter, 2)476
# The headline is the MEDIAN OF THE PAIRED RATIOS, not the ratio of the two477
# medians — the two differ whenever the machine drifts, and only the former478
# stays unbiased. lo/hi bound how much the ratio moved across rounds.479
row["speedup"] = round(m["speedup"], 1)480
row["speedup_lo"] = round(m["speedup_lo"], 1)481
row["speedup_hi"] = round(m["speedup_hi"], 1)482
results[key] = row483
extra = " ⚠ELIDED" if elided else ""484
if kind == "compared" and "speedup" in row:485
sp = (f" {row['speedup']}× vs {row['vs']}"486
f" [{row['speedup_lo']}–{row['speedup_hi']}]")487
elif kind == "compared":488
sp = f" —× vs {row.get('vs','')}"489
else:490
sp = " (cheatah-only)"491
print(f"{key:<26} {ch_ns:>8.2f} ns ±{row['cheatah_ns_iqr']:<6.2f}{sp}{extra}")493
try:494
import numpy as _np495
npv = _np.__version__496
except Exception:497
npv = "?"498
meta = {"machine": _machine_string(),499
"cheatah_commit": commit,500
"cpython": f"{sys.version_info.major}.{sys.version_info.minor}."501
f"{sys.version_info.micro}",502
"numpy": npv,503
"generated": time.strftime("%Y-%m-%d"),504
"rounds": ROUNDS,505
"statistic": "median of paired per-round ratios; spread = IQR over rounds",506
"striated": True}507
# A partial run (`perf_suite.py math io`) keeps every other module's PREVIOUS row while508
# stamping a fresh date/commit over the whole file — which would quietly claim the untouched509
# rows were measured today, on this commit, under this methodology. Name the modules that510
# actually ran so the stamp cannot overstate its own coverage.511
if only:512
meta["partial"] = sorted(only)513
meta["generated_note"] = ("PARTIAL RUN — only the listed modules were re-measured; "514
"every other row is carried over from the previous file")515
# A LIST of full records (every field present + has_compare), sorted by name — the516
# shape parsers.json's typed reader consumes in the pure-cheatah docs generator.517
FIELDS = {"kind": "", "note": "", "cheatah_ns": 0.0, "compare_ns": 0.0, "speedup": 0.0,518
"vs": "", "cheatah_us": 0.0, "numpy_us": 0.0, "dims": "", "warn": "",519
# Added with the striated/median rewrite. ADDITIVE ONLY: the four keys above520
# keep their names so docs/gen/generate.py and docs/gen-cheatah/gen.purr —521
# the latter reading through a FIXED typed field list — keep working. Any522
# future removal has to change all three files in one commit.523
"cheatah_ns_iqr": 0.0, "compare_ns_iqr": 0.0,524
"speedup_lo": 0.0, "speedup_hi": 0.0,525
"rounds": 0, "statistic": ""}526
recs = []527
for name in sorted(results):528
e = results[name]529
rec = {"name": name}530
rec.update({f: e.get(f, dflt) for f, dflt in FIELDS.items()})531
rec["has_compare"] = "compare_ns" in e532
recs.append(rec)533
json.dump({"meta": meta, "functions": recs}, open(OUT, "w"), indent=1, sort_keys=True)534
print(f"\nwrote {OUT} ({len([1 for r in results.values() if r.get('kind')=='compared'])} compared, "535
f"machine={meta['machine']}, cheatah@{commit}, CPython {meta['cpython']})")538
if __name__ == "__main__":539
main()