45 lines
1.8 KiB
Python
45 lines
1.8 KiB
Python
|
"""Summarize retained raw samples; nearest-rank p95, no gate claims."""
|
||
|
|
import json
|
||
|
|
import math
|
||
|
|
import sys
|
||
|
|
|
||
|
|
|
||
|
|
def percentile(values, probability):
|
||
|
|
if not values or not 0 < probability <= 1:
|
||
|
|
raise ValueError("nonempty samples and probability in (0, 1] required")
|
||
|
|
if any(not math.isfinite(value) or value < 0 for value in values):
|
||
|
|
raise ValueError("samples must be finite nonnegative milliseconds")
|
||
|
|
ordered = sorted(values)
|
||
|
|
return ordered[math.ceil(len(ordered) * probability) - 1]
|
||
|
|
|
||
|
|
|
||
|
|
def summarize(report):
|
||
|
|
if report.get("kind") != "synthetic_lexical_index_only":
|
||
|
|
raise ValueError("unsupported report")
|
||
|
|
rows = []
|
||
|
|
for case in report["cases"]:
|
||
|
|
if len(case["basic"]) != len(case["debug"]) or not case["basic"]:
|
||
|
|
raise ValueError("paired basic/debug samples required")
|
||
|
|
row = {"case": case["case"], "samples_per_mode": len(case["basic"])}
|
||
|
|
for mode in ("basic", "debug"):
|
||
|
|
values = [sample["wall_ms"] for sample in case[mode]]
|
||
|
|
row[mode] = {
|
||
|
|
"p50_ms": percentile(values, 0.5),
|
||
|
|
"p95_ms": percentile(values, 0.95),
|
||
|
|
"maximum_ms": max(values),
|
||
|
|
"engine_p95_ms": percentile(
|
||
|
|
[sample["metrics"]["total_ms"] for sample in case[mode]], 0.95
|
||
|
|
),
|
||
|
|
}
|
||
|
|
# Pair order follows the same deterministic request IDs in both arrays.
|
||
|
|
row["mean_paired_debug_minus_basic_ms"] = sum(
|
||
|
|
debug["wall_ms"] - basic["wall_ms"]
|
||
|
|
for basic, debug in zip(case["basic"], case["debug"])
|
||
|
|
) / len(case["basic"])
|
||
|
|
rows.append(row)
|
||
|
|
return {"kind": report["kind"], "production_gate_pass": None, "cases": rows}
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
||
|
|
print(json.dumps(summarize(json.load(source)), indent=2))
|