108 lines
5.9 KiB
Python
108 lines
5.9 KiB
Python
"""Bounded source acquisition and lossless metadata stress variants; no inference."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import struct
|
|
import urllib.request
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def execute(args):
|
|
return subprocess.run(args, check=True, capture_output=True, timeout=60).stdout
|
|
|
|
|
|
def probe(path, ffprobe):
|
|
value = json.loads(execute([str(ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", str(path)]))
|
|
tags = {k.lower(): v for k, v in value.get("format", {}).get("tags", {}).items()}
|
|
for stream in value["streams"]:
|
|
tags.update({k.lower(): v for k, v in stream.get("tags", {}).items()})
|
|
return tags
|
|
|
|
|
|
def pcm(path, ffmpeg, ffprobe):
|
|
metadata = json.loads(execute([str(ffprobe), "-v", "error", "-show_format", "-of", "json", str(path)]))
|
|
duration_ms = min(30000, int(float(metadata["format"]["duration"]) * 1000 + 0.5))
|
|
data = execute([str(ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-ss", "0.000", "-t", f"{duration_ms / 1000:.3f}", "-i", str(path), "-af", f"aresample=16000,atrim=end_sample={duration_ms * 16},asetpts=PTS-STARTPTS", "-ac", "1", "-ar", "16000", "-threads", "1", "-f", "s16le", "pipe:1"])
|
|
if not data or len(data) > 480000 * 2 or len(data) % 2:
|
|
raise ValueError("Invalid bounded PCM")
|
|
return sha(b"".join(struct.pack("<f", value[0] / 32768) for value in struct.iter_unpack("<h", data)))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("fixture", type=Path)
|
|
parser.add_argument("output", type=Path)
|
|
parser.add_argument("ffmpeg", type=Path)
|
|
parser.add_argument("ffprobe", type=Path)
|
|
args = parser.parse_args()
|
|
raw = args.fixture.read_bytes()
|
|
fixture = json.loads(raw)
|
|
lock = json.loads(args.fixture.with_name("labels.lock.json").read_text())
|
|
if sha(raw) != lock["fixture_sha256"]:
|
|
raise ValueError("Frozen fixture changed")
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
plan_path = args.output / "prepared.json"
|
|
if plan_path.exists():
|
|
raise ValueError("Preserve existing prepared plan; use a fresh output directory")
|
|
sources = []
|
|
for asset in fixture["assets"]:
|
|
destination = args.output / (asset["id"] + "-source.ogg")
|
|
if destination.exists():
|
|
data = destination.read_bytes()
|
|
else:
|
|
request = urllib.request.Request(asset["url"] + "?download=1", headers={"User-Agent": "SearchiMediaEvaluation/1.0"})
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
data = response.read(asset["bytes"] + 1)
|
|
if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]:
|
|
raise ValueError("Publisher bytes changed")
|
|
destination.write_bytes(data)
|
|
if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]:
|
|
raise ValueError("Existing source changed")
|
|
sources.append({"asset": asset, "path": destination.resolve(), "sha256": sha(data), "tags": probe(destination, args.ffprobe)})
|
|
prepared = []
|
|
whitelist = ["artist", "title", "album", "genre", "composer"]
|
|
for i, source in enumerate(sources):
|
|
original_pcm = None
|
|
for variant in fixture["variants"]:
|
|
path = args.output / (source["asset"]["id"] + "-" + variant + ".flac")
|
|
tags = source["tags"] if variant == "original" else (sources[(i + 1) % len(sources)]["tags"] if variant == "misleading_metadata" else {})
|
|
command = [str(args.ffmpeg), "-v", "error", "-nostdin", "-n", "-threads", "1", "-i", str(source["path"]), "-t", "30", "-ac", "1", "-ar", "16000", "-map_metadata", "-1", "-c:a", "flac"]
|
|
for key in whitelist:
|
|
if key in tags:
|
|
command.extend(["-metadata", f"{key}={tags[key]}"])
|
|
command.append(str(path))
|
|
execute(command)
|
|
pcm_hash = pcm(path, args.ffmpeg, args.ffprobe)
|
|
if original_pcm is None:
|
|
original_pcm = pcm_hash
|
|
if pcm_hash != original_pcm:
|
|
raise ValueError("Metadata variant changed decoded audio")
|
|
prepared.append({"id": source["asset"]["id"], "variant": variant, "kind": "audio", "path": str(path.resolve()), "sha256": sha(path.read_bytes()), "pcm_sha256": pcm_hash, "actual_tags": probe(path, args.ffprobe), "source_sha256": source["sha256"], "source_page": source["asset"]["source_page"], "license": source["asset"]["license"]})
|
|
# Previously evaluated assets are distractors only, never new positive evidence.
|
|
legacy_root = Path("fixtures/media")
|
|
legacy_raw = (legacy_root / "fixture.json").read_bytes()
|
|
legacy = json.loads(legacy_raw)
|
|
legacy_lock = json.loads((legacy_root / "labels.lock.json").read_text())
|
|
if sha(legacy_raw) != legacy_lock["fixture_sha256"]:
|
|
raise ValueError("Legacy distractor fixture changed")
|
|
for asset in legacy["assets"]:
|
|
if asset["id"] not in fixture["legacy_distractors"]:
|
|
continue
|
|
path = (legacy_root / asset["path"]).resolve()
|
|
if sha(path.read_bytes()) != asset["sha256"]:
|
|
raise ValueError("Legacy distractor changed")
|
|
prepared.append({"id": asset["id"], "variant": "fixed", "kind": "image" if asset["kind"] == "image" else "audio", "path": str(path), "sha256": asset["sha256"], "actual_tags": {} if asset["kind"] == "image" else probe(path, args.ffprobe), "legacy_distractor": True})
|
|
result = {"fixture_sha256": sha(raw), "fixture": fixture, "candidates": prepared, "ffmpeg_sha256": sha(args.ffmpeg.read_bytes()), "ffprobe_sha256": sha(args.ffprobe.read_bytes()), "source_info": [{**s, "path": str(s["path"])} for s in sources]}
|
|
with plan_path.open("x", encoding="utf-8") as output:
|
|
json.dump(result, output, ensure_ascii=False, indent=2)
|
|
print("Prepared", len(prepared), "containers; no inference. Plan SHA256", sha(plan_path.read_bytes()))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|