safetynet/scripts/windows-semantic-dlp-smoke.mjs

// Real pinned local Gemma inference, reviewed teaching and a private HTTPS receiver.
import assert from 'node:assert/strict';
import {createServer} from 'node:https';
import {createRequire} from 'node:module';
import {mkdtempSync,mkdirSync,readFileSync,writeFileSync,rmSync,existsSync} from 'node:fs';
import {tmpdir} from 'node:os';
import {join,resolve,sep} from 'node:path';
import {spawn} from 'node:child_process';
import {randomBytes} from 'node:crypto';
import {createApp} from '../apps/server/app.mjs';
import {openStore,digest} from '../apps/server/store.mjs';
import {inspect,verifyPolicy} from '../packages/core/policy.mjs';
import {startTelemetry} from '../apps/server/telemetry.mjs';
const collector='http://127.0.0.1:4318';assert.equal((await fetch(collector+'/health')).ok,true);process.env.OTEL_EXPORTER_OTLP_ENDPOINT=collector;const telemetry=startTelemetry(),temporary=mkdtempSync(join(tmpdir(),'safetynet-semantic-mvp-'));mkdirSync('artifacts',{recursive:true});assert.ok(resolve(temporary).startsWith(resolve(tmpdir())+sep)&&temporary.split(/[\\/]/).at(-1).startsWith('safetynet-semantic-mvp-'));
const fixtures={reference:'Confidential corporate restructuring will close the Lisbon office next autumn and eliminate two hundred positions before the public announcement. Leadership is negotiating a six-million-euro acquisition of a rival. These undisclosed staffing reductions and acquisition plans are restricted internal company information.',paraphrase:'Private workforce planning calls for shutting the Portugal branch after summer and cutting roughly two hundred jobs. Executives are discussing a six-million-euro purchase of a competitor; the timetable and personnel reductions remain undisclosed.',publicText:'A simple tomato pasta recipe combines ripe tomatoes, olive oil, garlic and fresh basil. Boil the pasta in salted water and serve it warm.'};
const runtime=resolve('.runtime/windows/semantic'),python=resolve('.runtime/windows/python/python.exe');assert.ok(existsSync(join(runtime,'models/model-manifest.json')));const publicFile=join(temporary,'public.txt'),secretFile=join(temporary,'paraphrase.txt');writeFileSync(publicFile,fixtures.publicText);writeFileSync(secretFile,fixtures.paraphrase);
const facts={posts:0,publicPosts:0,paraphrasePosts:0,filePosts:0,exactPublicFiles:0};let inspector,management,sink,store,child,stage='startup';
try{const inspectorToken=randomBytes(32).toString('base64url'),adminToken=randomBytes(32).toString('base64url'),code=randomBytes(32).toString('base64url');inspector=spawn(python,['apps/inspector/server.py'],{windowsHide:true,stdio:['ignore','pipe','pipe'],env:{...process.env,EMBEDDING_MODEL_DIR:join(runtime,'models'),EMBEDDING_WORKER_PATH:join(runtime,'searchi-gemma-worker.exe'),INSPECTOR_TOKEN:inspectorToken,INSPECTOR_PORT:'0',INSPECTOR_PARENT_PID:String(process.pid)}});inspector.stderr.on('data',()=>{});const inspectorPort=await new Promise((accept,reject)=>{let output='';const timer=setTimeout(()=>{inspector.kill();reject(Error('Private semantic inspector startup timed out'));},40000);inspector.once('error',()=>reject(Error('Private semantic inspector startup failed')));inspector.once('exit',()=>{clearTimeout(timer);reject(Error('Private semantic inspector exited'));});inspector.stdout.on('data',chunk=>{output+=chunk;if(output.includes('\n')){clearTimeout(timer);try{const startup=JSON.parse(output.split('\n')[0]);assert.equal(startup.containment.parserIsolation.active,true);assert.equal(startup.containment.parserIsolation.networkCapabilities,0);accept(startup.port);}catch{reject(Error('Private semantic inspector attestation invalid'));}}});});
 sink=createServer({cert:readFileSync('tests/fixtures/icap/cert.pem'),key:readFileSync('tests/fixtures/icap/key.pem')},(req,res)=>{if(req.url==='/facts'){res.setHeader('content-type','application/json');res.end(JSON.stringify(facts));return;}if(req.method==='POST'){let chunks=[],size=0;req.on('data',chunk=>{size+=chunk.length;if(size>1048576){req.destroy();return;}chunks.push(chunk);});req.on('end',()=>{facts.posts++;const bytes=Buffer.concat(chunks);if(bytes.includes(Buffer.from(fixtures.paraphrase)))facts.paraphrasePosts++;if(bytes.includes(Buffer.from(fixtures.publicText)))facts.publicPosts++;if(req.url==='/receiver/file'){facts.filePosts++;const boundary=/boundary=(?:"([^"]+)"|([^;]+))/.exec(req.headers['content-type']||'');if(boundary){const start=bytes.indexOf(Buffer.from('\r\n\r\n')),end=bytes.indexOf(Buffer.from('\r\n--'+(boundary[1]||boundary[2])),start+4);if(start>=0&&end>=0&&bytes.subarray(start+4,end).equals(Buffer.from(fixtures.publicText)))facts.exactPublicFiles++;}}bytes.fill(0);for(const chunk of chunks)chunk.fill(0);chunks=[];res.end('accepted');});return;}res.setHeader('content-type','text/html');res.end('<!doctype html><title>Public local AI workspace</title><style>body{font:18px Arial;padding:25px}[contenteditable]{border:1px solid #999;width:85%;min-height:60px;margin:25px 0}</style><h1>Public AI workspace</h1><div id="editor" role="textbox" contenteditable="true"><p><br></p></div><input id="file" type="file"><script>window.submitPrompt=()=>fetch("/receiver/prompt",{method:"POST",headers:{"content-type":"text/plain"},body:document.getElementById("editor").innerText.trimEnd()}).then(r=>r.ok).catch(()=>false)</script>');}).listen(0,'127.0.0.1');await new Promise(r=>sink.once('listening',r));const origin=`https://127.0.0.1:${sink.address().port}`;
 store=openStore(join(temporary,'db'));const p=store.get('policy');p.browser.homeUrl=origin+'/workspace';p.browser.allowedHosts=['127.0.0.1'];p.agents.enabled=false;p.dlp.rules=[];p.security.yara.enabled=false;p.diagnostics={...p.diagnostics,enabled:true,mode:'debug',endpoint:collector,sampleRate:1,debugSampleRate:1,flushIntervalMs:250};store.put('policy',p);store.db.prepare('INSERT INTO enrollments VALUES(?,?,?)').run(digest(code),Date.now()+1800000,'semantic-mvp-user');const config={secure:false,publicUrl:'http://127.0.0.1',adminToken,requirePQManagement:true,requireIndependentPolicyApproval:true,inspectorUrl:`http://127.0.0.1:${inspectorPort}`,inspectorToken};management=createApp(store,config).listen(0,'127.0.0.1');await new Promise(r=>management.once('listening',r));config.publicUrl=`http://127.0.0.1:${management.address().port}`;const reviewer=randomBytes(32).toString('base64url');store.db.prepare('INSERT INTO sessions VALUES(?,?,?)').run(digest(reviewer),Date.now()+1800000,JSON.stringify({sub:'independent-semantic-reviewer',role:'admin'}));const headers={authorization:'Bearer '+adminToken,'content-type':'application/json'};
 async function admin(route,body,method='POST',status=200){const result=await fetch(config.publicUrl+route,{method,headers,body:JSON.stringify(body),signal:AbortSignal.timeout(120000)});assert.equal(result.status,status,'Semantic admin operation refused');return result.json();}
 async function approve(review){const result=await fetch(config.publicUrl+`/api/admin/policy-reviews/${review.reviewId}/approve`,{method:'POST',headers:{cookie:'sn_session='+reviewer,origin:config.publicUrl,'content-type':'application/json'},body:JSON.stringify({payloadHash:review.payloadHash,expectedVersion:review.expectedVersion,approve:true})});assert.equal(result.status,200);}
 stage='reviewed document teaching';const teaching=await admin('/api/admin/teaching',{label:'Synthetic restricted workforce reference',filename:'synthetic-reference.txt',data:Buffer.from(fixtures.reference).toString('base64')});assert.equal(teaching.reference.registrationReady,true);assert.ok(teaching.reference.fragmentHashes.length>0);const publication=await admin(`/api/admin/teaching/${teaching.id}/publish`,{expectedVersion:p.version,approve:true},'POST',202);await approve(publication);const retained=JSON.parse(store.db.prepare('SELECT reference FROM teaching WHERE id=?').get(teaching.id).reference);assert.deepEqual(Object.keys(retained).sort(),['bytes','contentType','coverage','fragmentHashes','registrationReady','sha256']);assert.ok(!JSON.stringify(retained).includes(fixtures.reference));assert.ok(!JSON.stringify(retained).includes('Lisbon office'));assert.ok(retained.fragmentHashes.every(hash=>/^[a-f0-9]{64}$/.test(hash)));
 stage='real pinned local document embedding';const reference=await admin('/api/admin/semantic/document-reference',{filename:'synthetic-reference.txt',data:Buffer.from(fixtures.reference).toString('base64')});const model={model:'google/embeddinggemma-2',revision:'bfcd298762cc34d0357ece5ebdd31791a3a374d8',dimensions:768,profile:'document-v1',artifactFormat:'gguf',quantization:'Q8_0',runtimeRevision:'b11461',local:true};for(const [key,value]of Object.entries(model))assert.equal(reference[key],value,'Pinned local model contract changed');assert.equal(reference.coverage.complete,true);const semanticPolicy=store.get('policy');semanticPolicy.version++;semanticPolicy.dlp.semantic={enabled:true,model:reference.model,revision:reference.revision,dimensions:reference.dimensions,profile:reference.profile,artifactFormat:reference.artifactFormat,quantization:reference.quantization,runtimeRevision:reference.runtimeRevision,categories:[{id:'semantic-workforce-fixture',label:'Synthetic restricted workforce plan',threshold:0.65,prototype:reference.vector,prototypes:reference.vectors}]};assert.equal(inspect(Buffer.from(fixtures.paraphrase),semanticPolicy).decision,'allow','Paraphrase matches deterministic teaching; semantic proof invalid');const secretResult=await admin('/api/admin/policy/test',{text:fixtures.paraphrase,policy:semanticPolicy}),publicResult=await admin('/api/admin/policy/test',{text:fixtures.publicText,policy:semanticPolicy});assert.equal(secretResult.ruleId,'semantic-workforce-fixture');assert.equal(publicResult.decision,'allow');await approve(await admin('/api/admin/policy',semanticPolicy,'PUT',202));const signed=verifyPolicy(store.envelope(),store.publicKey());assert.equal(signed.dlp.semantic.revision,model.revision);assert.ok(!JSON.stringify(signed).includes(fixtures.reference));
 const teachingProof={independentlyReviewedTeaching:true,independentlyReviewedSemanticPolicy:true,fullDocumentNotRetained:true,retainedKeys:Object.keys(retained).sort(),retainedFragmentCount:retained.fragmentHashes.length,referenceCoverageComplete:true,deterministicParaphraseAllowed:true,semanticParaphraseBlocked:true,publicSemanticAllowed:true,policyVersion:signed.version,model,paraphraseSimilarity:secretResult.similarity?.[0]?.score,publicSimilarity:publicResult.similarity?.[0]?.score};writeFileSync('artifacts/windows-semantic-teaching-proof.json',JSON.stringify(teachingProof,null,2));
 stage='actual native semantic paste and upload';const env={...process.env,SAFETYNET_SMOKE_SERVER:config.publicUrl,SAFETYNET_SMOKE_CODE:code,SAFETYNET_SMOKE_ADMIN:adminToken,SAFETYNET_SMOKE_USERDATA:join(temporary,'browser'),SAFETYNET_SMOKE_ORIGIN:origin,SAFETYNET_SMOKE_PUBLIC_FILE:publicFile,SAFETYNET_SMOKE_SECRET_FILE:secretFile,SAFETYNET_SEMANTIC_FIXTURES:JSON.stringify(fixtures),SAFETYNET_SEMANTIC_POLICY:String(signed.version)};delete env.EMBEDDING_MODEL_DIR;delete env.EMBEDDING_WORKER_PATH;child=spawn(createRequire(import.meta.url)('electron'),['scripts/windows-semantic-dlp-smoke.cjs'],{windowsHide:true,stdio:['ignore','pipe','pipe'],env});child.stdout.on('data',chunk=>process.stdout.write(chunk));child.stderr.on('data',chunk=>process.stderr.write(chunk));const timer=setTimeout(()=>child.kill(),240000),exit=await new Promise(r=>{child.once('exit',v=>r(v??1));child.once('error',()=>r(1));});clearTimeout(timer);assert.equal(exit,0,'Native semantic MVP failed');assert.equal(facts.paraphrasePosts,0);assert.equal(facts.filePosts,1);assert.equal(facts.exactPublicFiles,1);console.log('Reviewed local Gemma semantic MVP proof passed');
}catch(error){writeFileSync('artifacts/windows-semantic-harness-error.json',JSON.stringify({stage,name:error.name},null,2));console.error('Semantic MVP harness failed at '+stage+': '+error.name);process.exitCode=1;}finally{child?.kill();inspector?.kill();for(const server of [management,sink])if(server){server.closeAllConnections();await new Promise(r=>server.close(r));}store?.db.close();await telemetry?.shutdown();rmSync(temporary,{recursive:true,force:true});}