38 lines
1.6 KiB
Python
38 lines
1.6 KiB
Python
|
"""Summarize predeclared image-budget experiments without fitting thresholds."""
|
||
|
|
import argparse
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from score import metrics
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser()
|
||
|
|
parser.add_argument("report", type=Path)
|
||
|
|
parser.add_argument("output", type=Path)
|
||
|
|
args = parser.parse_args()
|
||
|
|
report = json.loads(args.report.read_text(encoding="utf-8"))
|
||
|
|
fixture_hash = hashlib.sha256(Path("fixtures/media/fixture.json").read_bytes()).hexdigest()
|
||
|
|
if report["fixture_sha256"] != fixture_hash:
|
||
|
|
raise ValueError("Frozen fixture changed")
|
||
|
|
if [budget["token_budget"] for budget in report["budgets"]] != [70, 280, 1120]:
|
||
|
|
raise ValueError("Expected predeclared budgets in order")
|
||
|
|
summary = {
|
||
|
|
"fixture_sha256": fixture_hash,
|
||
|
|
"worker": report["worker"],
|
||
|
|
"photos": report["photos"],
|
||
|
|
"queries": report["queries"],
|
||
|
|
"query_embedding_ms": report["query_embedding_ms"],
|
||
|
|
"budgets": [dict(token_budget=budget["token_budget"],
|
||
|
|
encoding_ms=budget["encoding_ms"],
|
||
|
|
metrics=metrics(budget["rows"]))
|
||
|
|
for budget in report["budgets"]],
|
||
|
|
"interpretation": "Same frozen 12 photos and 24 correlated EN/DE queries. Query embeddings shared across budgets. Encoding time is CPU batch cost; ranking-only time excludes inference. Equal small-corpus recall does not prove equal quality on larger or more difficult images.",
|
||
|
|
}
|
||
|
|
args.output.write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||
|
|
print(json.dumps(summary["budgets"]))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|