karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Keep real model benchmark validity checks active under optimization

Browse Source
This commit is contained in:
karl committed 2026-10-13 13:54:33 +02:00
1 parent d568bdb2b1
commit 66bf7d7b47
4 files changed
+79 -27
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/embeddinggemma2/benchmark.py
+27 -20
View File
@@ -38,6 +38,11 @@ def distribution(values):
'p50_ms': statistics.median(values),
'p95_ms': ordered[max(0, math.ceil(len(values) * .95) - 1)]}
def require(condition, message):
if not condition:
raise ValueError(message)
class Session:
def __init__(self, command):
environment = os.environ.copy()
@@ -81,7 +86,7 @@ class Session:
response = self.next()
if not response.get('ok'):
raise RuntimeError(response.get('error', 'Worker request failed'))
assert response['device'] == 'cpu'
require(response['device'] == 'cpu', 'Benchmark contract failed at benchmark.py:84')
return response, elapsed(start)
def mcp(self, identifier, method, params):
@@ -90,7 +95,7 @@ class Session:
while True:
response = self.next()
if response.get('id') == identifier:
assert 'error' not in response, response
require('error' not in response, response)
return response['result'], elapsed(start)
def close(self):
@@ -103,19 +108,19 @@ class Session:
def vectors(response, count):
result = response['vectors']
assert len(result) == count
require(len(result) == count, 'Benchmark contract failed at benchmark.py:106')
for vector in result:
assert len(vector) == 768 and all(math.isfinite(n) for n in vector)
assert abs(sum(n*n for n in vector) - 1) < .0001
require(len(vector) == 768 and all(math.isfinite(n) for n in vector), 'Benchmark contract failed at benchmark.py:108')
require(abs(sum(n*n for n in vector) - 1) < .0001, 'Benchmark contract failed at benchmark.py:109')
return result
def native(args):
manifest = json.loads((ROOT / 'tools/embeddinggemma2/model-manifest.json').read_text())
model = args.models / manifest['files'][0]['path']
start = time.perf_counter()
assert model.stat().st_size == manifest['files'][0]['bytes']
require(model.stat().st_size == manifest['files'][0]['bytes'], 'Benchmark contract failed at benchmark.py:116')
with model.open('rb') as stream:
assert hashlib.file_digest(stream, 'sha256').hexdigest() == manifest['files'][0]['sha256']
require(hashlib.file_digest(stream, 'sha256').hexdigest() == manifest['files'][0]['sha256'], 'Benchmark contract failed at benchmark.py:118')
verification_ms = elapsed(start)
reports = []
for threads in args.threads:
@@ -124,7 +129,7 @@ def native(args):
try:
hello = session.next()
load_ms = elapsed(session.started)
assert hello['device'] == 'cpu' and hello['model'] == manifest['model_id']
require(hello['device'] == 'cpu' and hello['model'] == manifest['model_id'], 'Benchmark contract failed at benchmark.py:127')
first, first_ms = session.worker({'op':'query', 'text':'first unseen foreground query',
'debug_performance':True})
vectors(first, 1)
@@ -143,8 +148,8 @@ def native(args):
response, duration = session.worker({'op':'windows','text':passage,'title':'Agreement'})
encoded = passage.encode()
for window in response['windows']:
assert encoded[window['start_byte']:window['end_byte']].decode() == window['text']
assert response['windows'][-1]['end_byte'] == len(encoded)
require(encoded[window['start_byte']:window['end_byte']].decode() == window['text'], 'Benchmark contract failed at benchmark.py:146')
require(response['windows'][-1]['end_byte'] == len(encoded), 'Benchmark contract failed at benchmark.py:147')
window_samples.append({'bytes':len(encoded),'windows':len(response['windows']),
'round_trip_ms':duration})
documents = [(f'Document {n}. Supplier cancellation requires sixty days written notice. '
@@ -158,7 +163,7 @@ def native(args):
response, batch_ms = session.worker({'op':'documents','texts':documents,'title':'Agreement'})
batched = vectors(response, 8)
cosines = [sum(a*b for a,b in zip(left,right)) for left,right in zip(singles,batched)]
assert min(cosines) > .99999
require(min(cosines) > .99999, 'Benchmark contract failed at benchmark.py:161')
reports.append({'threads':threads,'worker_info':hello,'process_load_ms':load_ms,
'first_query_ms':first_ms,'first_query_worker':first['performance'],
'warm_queries':distribution([sample['round_trip_ms'] for sample in samples]),
@@ -174,8 +179,8 @@ def native(args):
'First query precedes warm samples. Documents measure IPC batch, not assumed parallelism.'}
def mcp(args):
assert args.db and args.cli
assert args.db.is_file()
require(args.db and args.cli, 'Benchmark contract failed at benchmark.py:177')
require(args.db.is_file(), 'Benchmark contract failed at benchmark.py:178')
session = Session([str(args.cli.resolve()),'mcp','--db',str(args.db.resolve()),
'--model-dir',str(args.models.resolve()),'--embedding-worker',str(args.worker.resolve()),
'--cpu-threads','1'])
@@ -185,8 +190,8 @@ def mcp(args):
startup_ms = elapsed(session.started)
session.send({'jsonrpc':'2.0','method':'notifications/initialized'})
stats,_ = session.mcp(2,'tools/call',{'name':'stats','arguments':{}})
assert stats['structuredContent']['mode'] == 'hybrid'
assert stats['structuredContent']['worker']['device'] == 'cpu'
require(stats['structuredContent']['mode'] == 'hybrid', 'Benchmark contract failed at benchmark.py:188')
require(stats['structuredContent']['worker']['device'] == 'cpu', 'Benchmark contract failed at benchmark.py:189')
samples=[]
identifier=3
cases = [('first_unseen',QUERIES[0],None)]
@@ -199,11 +204,11 @@ def mcp(args):
arguments['extension']=extension
result,duration=session.mcp(identifier,'tools/call',{'name':'search','arguments':arguments})
identifier+=1
assert not result.get('isError'), result
require(not result.get('isError'), result)
response=result['structuredContent']
assert response['mode']=='hybrid'
require(response['mode']=='hybrid', 'Benchmark contract failed at benchmark.py:204')
if extension:
assert all(item['filename'].lower().endswith('.'+extension) for item in response['results'])
require(all(item['filename'].lower().endswith('.'+extension) for item in response['results']), 'Benchmark contract failed at benchmark.py:206')
samples.append({'case':case,'query':query,'extension':extension,'request_ms':duration,
'engine_ms':response['metrics']['total_ms'],
'matches':[{'filename':item['filename'],'source':item['source'],
@@ -230,7 +235,8 @@ def main():
parser.add_argument('--rust-profile',choices=['debug','release','not-applicable'],default='not-applicable')
parser.add_argument('--label',default='')
args=parser.parse_args()
assert 1<=args.repeats<=20 and all(1<=n<=8 for n in args.threads)
require(not args.output.exists(), 'Output must be new; preserve previous evidence')
require(1<=args.repeats<=20 and all(1<=n<=8 for n in args.threads), 'Benchmark contract failed at benchmark.py:233')
def digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream,'sha256').hexdigest()
@@ -241,7 +247,8 @@ def main():
'cli_sha256':digest(args.cli) if args.cli else None,
'results':native(args) if args.mode=='native' else mcp(args)}
args.output.parent.mkdir(parents=True,exist_ok=True)
args.output.write_text(json.dumps(report,indent=2,ensure_ascii=False)+'\n',encoding='utf-8')
with args.output.open('x', encoding='utf-8') as output:
output.write(json.dumps(report,indent=2,ensure_ascii=False)+'\n')
print(f'Saved {args.output}',flush=True)
if __name__=='__main__':