Preserve broker audit causation in central ingestion and SIEM exports

apps/server/app.mjs

@@ -5,6 +5,7 @@ import requestTransferFacts from '../../packages/core/request-transfer-facts.cjs
 import agentAudit from '../../packages/core/agent-audit.cjs';
 import interactionAudit from '../../packages/core/interaction-facts.cjs';
 import transportAudit from '../../packages/core/transport-facts.cjs';
+import brokerAudit from '../../packages/core/broker-audit.cjs';
 import {installEvidence} from './evidence.mjs';
 import {installIncidents} from './incidents.mjs';
 import { installBookmarkRoutes,globalBookmarksValid } from './bookmarks.mjs';
@@ -108,7 +109,7 @@ export function createApp(store,config) {
     store.db.prepare('UPDATE devices SET last_seen=?,policy_version=? WHERE id=?').run(new Date().toISOString(),b.policyVersion,req.device.id);res.json({ok:true});
   });
   app.post('/api/device/events',device,(req,res)=>{
-    const b=z.object({eventId:z.string().uuid(),kind:z.enum([...transportAudit.TRANSPORT_KINDS,'browser.interaction','browser.interaction.dropped','dlp.block','navigation.block','download.block','policy.applied','browser.started','agent.action','agent.denied','library.ai.proposed','library.ai.applied','library.ai.reconcile','devtools.open-requested','devtools.opened','devtools.closed','devtools.denied','download.scan','download.released','upload.authorized','upload.block','dns.block','content.block','vault.submission-completed','vault.submission.failure','vault.submission-prepared','vault.submission-authorized','vault.submission-denied','vault.created','vault.unlocked','vault.locked','vault.saved','vault.deleted','vault.filled','vault.fill-requested','phishing.block','phishing.error','phishing.scanned','clipboard.block','clipboard.copy','clipboard.cut','clipboard.paste']),ruleId:z.string().max(100).optional(),host:z.string().max(253).optional(),policyVersion:z.number().int().positive(),traceId:z.string().regex(/^[a-f0-9]{32}$/).optional(),spanId:z.string().regex(/^[a-f0-9]{16}$/).optional(),file:transferFacts.FileFactsSchema.optional(),request:requestTransferFacts.RequestFactsSchema.optional(),agent:agentAudit.AgentAuditSchema.optional(),interaction:interactionAudit.InteractionFactsSchema.optional(),transport:transportAudit.TransportFactsSchema.optional()}).strict().parse(req.body);
+    const b=z.object({eventId:z.string().uuid(),kind:z.enum([...transportAudit.TRANSPORT_KINDS,'browser.interaction','browser.interaction.dropped','dlp.block','navigation.block','download.block','policy.applied','browser.started','agent.action','agent.denied','library.ai.proposed','library.ai.applied','library.ai.reconcile','devtools.open-requested','devtools.opened','devtools.closed','devtools.denied','download.scan','download.released','upload.authorized','upload.block','dns.block','content.block','vault.submission-completed','vault.submission.failure','vault.submission-prepared','vault.submission-authorized','vault.submission-denied','vault.created','vault.unlocked','vault.locked','vault.saved','vault.deleted','vault.filled','vault.fill-requested','phishing.block','phishing.error','phishing.scanned','clipboard.block','clipboard.copy','clipboard.cut','clipboard.paste']),ruleId:z.string().max(100).optional(),host:z.string().max(253).optional(),policyVersion:z.number().int().positive(),traceId:z.string().regex(/^[a-f0-9]{32}$/).optional(),spanId:z.string().regex(/^[a-f0-9]{16}$/).optional(),file:transferFacts.FileFactsSchema.optional(),request:requestTransferFacts.RequestFactsSchema.optional(),agent:agentAudit.AgentAuditSchema.optional(),interaction:interactionAudit.InteractionFactsSchema.optional(),transport:transportAudit.TransportFactsSchema.optional(),broker:brokerAudit.BrokerFactsSchema.optional()}).strict().superRefine(brokerAudit.validateBrokerSemantics).parse(req.body);
     if(b.kind.startsWith('transport.')){
       const unavailable=b.kind==='transport.monitor-unavailable';
       if(b.file||b.request||b.agent||b.interaction|| (unavailable?(b.host!==undefined||b.transport!==undefined||!transportAudit.TRANSPORT_MONITOR_REASONS.includes(b.ruleId)):(b.ruleId!==undefined||!transportAudit.canonicalTransportHostname(b.host)||!transportAudit.transportFactsForEvent(b.kind,b.transport))))return res.status(400).json({error:'Transport audit events require matching lifecycle metadata'});

docs/broker-siem.md

@@ -0,0 +1,22 @@
+# Broker lifecycle evidence in SIEM exports
+
+Broker events use a request UUID separate from OpenTelemetry trace identifiers.
+A terminal forwarding fault references the durable authorization event UUID;
+authorization alone does not prove delivery. Strict shared schema validation
+rejects unsupported phases, reasons, self-links and raw content fields.
+
+ECS exports the projected facts under `safetynet.broker`, including request,
+connection and original event IDs. Forwarding/inspection faults have an error
+type and failure outcome. An authorization has an unknown delivery outcome.
+
+CEF uses the standard custom string `cs2`, labeled `BrokerLifecycle`, for bounded
+JSON facts, `cs3` for the original event UUID and `requestContext` for the request
+UUID. This preserves the existing rule, file type, trace, span and audit-integrity
+fields. Configure the SIEM to parse the custom JSON string when joining causal
+events. The standard `outcome` field reflects unknown authorization or failure;
+Microsoft documents its mapping to `EventOutcome` in
+[CommonSecurityLog](https://learn.microsoft.com/en-us/azure/sentinel/cef-name-mapping).
+
+These format and exporter tests do not establish a customer's remote SIEM
+deployment, retention, key custody or incident-response configuration. Production
+browser-to-broker integration remains a separate acceptance gate.

packages/core/broker-audit.cjs

@@ -0,0 +1,21 @@
+'use strict';
+const {z}=require('zod');
+const {RequestFactsSchema}=require('./request-transfer-facts.cjs');
+const BROKER_REASONS=Object.freeze(['broker-approved','broker-policy-block','broker-inspection-fault','broker-authority-expired','broker-audit-failed','broker-connect-denied','broker-auth-denied','broker-framing-block','broker-size-block','broker-request-aborted','broker-request-timeout','broker-upstream-failed','broker-response-invalid']);
+const BrokerFactsSchema=z.object({requestId:z.string().uuid(),connectionId:z.string().uuid().optional(),phase:z.enum(['connect.block','request.authorized','request.block','request.fault','forward.fault']),reason:z.enum(BROKER_REASONS),authorizedEventId:z.string().uuid().optional()}).strict();
+const phaseReasons=Object.freeze({
+  'connect.block':['broker-connect-denied','broker-auth-denied','broker-framing-block','broker-authority-expired','broker-audit-failed','broker-request-timeout','broker-size-block','broker-request-aborted'],
+  'request.authorized':['broker-approved'],
+  'request.block':['broker-policy-block','broker-authority-expired','broker-framing-block','broker-size-block','broker-request-aborted','broker-request-timeout'],
+  'request.fault':['broker-inspection-fault','broker-audit-failed'],
+  'forward.fault':['broker-upstream-failed','broker-response-invalid','broker-request-aborted','broker-request-timeout','broker-authority-expired','broker-audit-failed']
+});
+function validateBrokerSemantics(event,context){
+  if(!event.broker)return;
+  const value=event.broker;
+  if(!event.request||event.file||event.kind!==(value.phase==='request.authorized'?'upload.authorized':'upload.block')||!phaseReasons[value.phase].includes(value.reason)||value.phase==='forward.fault'&&!value.authorizedEventId||value.phase!=='forward.fault'&&value.authorizedEventId||value.authorizedEventId===event.eventId){
+    context.addIssue({code:'custom',path:['broker'],message:'Broker lifecycle semantics rejected'});
+  }
+}
+const BrokerAuditEventSchema=z.object({eventId:z.string().uuid(),kind:z.enum(['upload.authorized','upload.block']),ruleId:z.string().min(1).max(100),policyVersion:z.number().int().min(1).max(Number.MAX_SAFE_INTEGER),host:z.string().max(253).optional(),request:RequestFactsSchema,broker:BrokerFactsSchema.optional(),traceId:z.string().regex(/^[a-f0-9]{32}$/).optional(),spanId:z.string().regex(/^[a-f0-9]{16}$/).optional()}).strict().superRefine(validateBrokerSemantics);
+module.exports={BrokerAuditEventSchema,BrokerFactsSchema,BROKER_REASONS,validateBrokerSemantics};

packages/core/siem.mjs

@@ -3,17 +3,24 @@ import transportAudit from './transport-facts.cjs';
 import agentAudit from './agent-audit.cjs';
 import transferFacts from './transfer-facts.cjs';
 import requestTransferFacts from './request-transfer-facts.cjs';
+import brokerAudit from './broker-audit.cjs';
+const brokerFacts=event=>{
+ const m=event.metadata;if(!m.broker)return null;
+ const parsed=brokerAudit.BrokerAuditEventSchema.safeParse({eventId:m.eventId,kind:event.kind,ruleId:m.ruleId,policyVersion:m.policyVersion,request:m.request,broker:m.broker,...(m.host?{host:m.host}:{}),...(m.file?{file:m.file}:{})});
+ return parsed.success?{eventId:parsed.data.eventId,...parsed.data.broker}:null;
+};
 const requestFacts=(metadata,kind)=>{if(!kind?.startsWith('upload.'))return null;const old=metadata.file;const parsed=requestTransferFacts.RequestFactsSchema.safeParse(metadata.request||(old?{bytes:old.bytes,detectedType:old.detectedType}:undefined));return parsed.success?parsed.data:null;};
 const facts=metadata=>{const parsed=transferFacts.FileFactsSchema.safeParse(metadata.file);return parsed.success?parsed.data:null;};
 export function toECS(event,installationId='safetynet') {
   const m=event.metadata,transport=transportAudit.canonicalTransportHostname(m.host)?transportAudit.transportFactsForEvent(event.kind,event.metadata.transport):null,transportEvent=transportAudit.TRANSPORT_KINDS.includes(event.kind),interaction=interactionAudit.interactionFactsForEvent(event.kind,event.metadata.interaction),agent=agentAudit.agentFactsForEvent(event.kind,event.metadata.agent),block=event.kind.includes('block')||event.kind.endsWith('.denied')||event.kind==='vault.submission-denied'||m.decision==='block',failure=event.kind.endsWith('.failure')||agent?.outcome==='failed',alert=event.kind==='sigma.alert';
+  const broker=brokerFacts(event),brokerFailure=broker&&['request.fault','forward.fault'].includes(broker.phase);
   const config=event.kind.startsWith('policy.')||event.kind.startsWith('enrollment.')||event.kind==='device.revoked';
   const auth=event.kind.startsWith('sso.'),clipboard=event.kind.startsWith('clipboard.');
-  return {
+  const output={
     '@timestamp':event.time,ecs:{version:'8.17.0'},
     event:{id:`${installationId}:${event.id}`,kind:alert?'alert':'event',module:'safetynet',dataset:'safetynet.security',
       action:event.kind,category:[alert?'intrusion_detection':auth?'authentication':config?'configuration':clipboard?'process':'web'],
-      type:[transportEvent||alert||interaction?.outcome==='observed'?'info':failure?'error':auth?'start':config?'change':block?'denied':clipboard?'access':'allowed'],outcome:transportEvent||interaction?.outcome==='observed'?'unknown':failure?'failure':'success',...(alert?{severity:severity(m.level)}:{})},
+      type:[transportEvent||alert||interaction?.outcome==='observed'?'info':failure||brokerFailure?'error':auth?'start':config?'change':block?'denied':clipboard?'access':'allowed'],outcome:transportEvent||interaction?.outcome==='observed'||broker?.phase==='request.authorized'?'unknown':failure||brokerFailure?'failure':'success',...(alert?{severity:severity(m.level)}:{})},
     observer:{vendor:'SafetyNet',product:'Managed Enterprise Browser',type:'dlp',version:'0.1.0'},
     ...(event.device_id?{host:{id:event.device_id}}:{}),
     ...(m.actor?{user:{id:m.actor}}:{}),...(m.host&&(!transportEvent||transport)?{destination:{domain:m.host}}:{}),
@@ -25,12 +32,18 @@ export function toECS(event,installationId='safetynet') {
       content_type:m.contentType??null,model:m.model??null,coverage:m.coverage??null,classification:m.classification==='local-only'?'local-only':null,
       ...(alert?{source_event_id:`${installationId}:${m.sourceEventId}`,severity:m.level}:{})}
   };
+  if(broker)output.safetynet.broker={event_id:broker.eventId,request_id:broker.requestId,phase:broker.phase,reason:broker.reason,...(broker.connectionId?{connection_id:broker.connectionId}:{}),...(broker.authorizedEventId?{authorized_event_id:broker.authorizedEventId}:{})};
+  return output;
 }
 const header=v=>String(v).replace(/\\/g,'\\\\').replace(/\|/g,'\\|').replace(/[\r\n]/g,' ');
 const ext=v=>String(v).replace(/\\/g,'\\\\').replace(/=/g,'\\=').replace(/\r/g,'\\r').replace(/\n/g,'\\n');
 const severity=level=>({low:3,medium:5,high:7,critical:10}[level]??3);
 export function toCEF(event,installationId='safetynet') {
   const m=event.metadata,transport=transportAudit.canonicalTransportHostname(m.host)?transportAudit.transportFactsForEvent(event.kind,event.metadata.transport):null,transportEvent=transportAudit.TRANSPORT_KINDS.includes(event.kind),interaction=interactionAudit.interactionFactsForEvent(event.kind,event.metadata.interaction),agent=agentAudit.agentFactsForEvent(event.kind,event.metadata.agent),blocked=event.kind.includes('block')||event.kind.endsWith('.denied')||event.kind==='vault.submission-denied'||m.decision==='block',alert=event.kind==='sigma.alert';
+  const broker=brokerFacts(event);
+  // CEF has six custom strings, already used for rule/trace/span/type metadata.
+  // A bounded, schema-projected JSON value preserves every causal UUID in one
+  // standard custom string without replacing audit integrity or file facts.
   // suid identifies the verified agent capability principal, not an employee account.
   // suser, when present, remains the independently supplied human actor.
   const fields={...(transport?{cs2Label:'TransportProtocol',cs2:transport.protocol,cs3Label:'TransportPhase',cs3:transport.phase,cs6Label:'TransportRealm',cs6:transport.realm,requestContext:transport.sessionId,cs1Label:'TransportId',cs1:transport.transportId,flexNumber2Label:'ConnectionEstablished',flexNumber2:transport.connectionEstablished?1:0}:{}),...(event.kind==='transport.monitor-unavailable'&&transportAudit.TRANSPORT_MONITOR_REASONS.includes(m.ruleId)?{reason:m.ruleId}:{}),...(interaction?{cs2Label:'InteractionAction',cs2:interaction.action,cs3Label:'InteractionOutcome',cs3:interaction.outcome,requestContext:interaction.sessionId}:{}),...(agent?{cs2Label:'AgentAction',cs2:agent.action,cs3Label:'AgentScope',cs3:agent.scope,cs6Label:'AgentOutcome',cs6:agent.outcome,...(agent.sessionId?{requestContext:agent.sessionId}:{}),...(agent.principalId?{suid:agent.principalId,flexNumber1Label:'AgentCredentialGeneration',flexNumber1:agent.credentialGeneration}:{})}:{}),...(requestFacts(m,event.kind)?{cn3Label:'RequestBytes',cn3:requestFacts(m,event.kind).bytes,cs6Label:'DetectedRequestType',cs6:requestFacts(m,event.kind).detectedType}:{}),...(event.audit?{flexString1Label:'AuditRecordHash',flexString1:event.audit.hash,flexString2Label:'AuditSequence',flexString2:event.audit.sequence}:{}),externalId:`${installationId}:${event.id}`,rt:Date.parse(event.time),act:event.kind,
@@ -40,6 +53,7 @@ export function toCEF(event,installationId='safetynet') {
     ...(event.kind.startsWith('download.')&&facts(m)?{cn2Label:'FileBytes',cn2:facts(m).bytes,cs6Label:'DetectedFileType',cs6:facts(m).detectedType,fileHash:facts(m).sha256}:{}),
     ...(/^[a-f0-9]{32}$/.test(m.traceId||'')?{cs4Label:'TraceId',cs4:m.traceId}:{}),
     ...(/^[a-f0-9]{16}$/.test(m.spanId||'')?{cs5Label:'SpanId',cs5:m.spanId}:{}),
+    ...(broker?{cs2Label:'BrokerLifecycle',cs2:JSON.stringify(broker),cs3Label:'BrokerEventId',cs3:broker.eventId,requestContext:broker.requestId,reason:broker.reason,outcome:broker.phase==='request.authorized'?'unknown':['request.fault','forward.fault'].includes(broker.phase)?'failure':'success'}:{}),
     ...(alert?{cs2Label:'SourceEventId',cs2:`${installationId}:${m.sourceEventId}`,cs3Label:'AlertLevel',cs3:m.level}:{})};
   return `CEF:0|SafetyNet|Enterprise Browser|0.1.0|${header(event.kind)}|${header(event.kind)}|${alert?severity(m.level):!transportEvent&&blocked?7:3}|`+Object.entries(fields).map(([k,v])=>`${k}=${ext(v)}`).join(' ');
 }

tests/broker-siem.test.mjs

@@ -0,0 +1,56 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {randomUUID} from 'node:crypto';
+import {toECS,toCEF} from '../packages/core/siem.mjs';
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import {openStore} from '../apps/server/store.mjs';
+import {startSIEM} from '../apps/server/siem.mjs';
+function event(phase='forward.fault'){
+ const authorizedEventId=randomUUID();
+ return {id:2,time:new Date().toISOString(),device_id:'synthetic',kind:phase==='request.authorized'?'upload.authorized':'upload.block',metadata:{eventId:randomUUID(),ruleId:phase==='request.authorized'?'broker-approved':'broker-upstream-failed',policyVersion:4,request:{bytes:128,detectedType:'pdf'},broker:{requestId:randomUUID(),connectionId:randomUUID(),phase,reason:phase==='request.authorized'?'broker-approved':'broker-upstream-failed',...(phase==='forward.fault'?{authorizedEventId}:{})}}};
+}
+test('ECS retains actual causal UUIDs and marks terminal forwarding failure as error',()=>{
+ const input=event(),output=toECS(input);
+ assert.equal(output.event.outcome,'failure');assert.deepEqual(output.event.type,['error']);
+ assert.equal(output.safetynet.broker.event_id,input.metadata.eventId);
+ assert.equal(output.safetynet.broker.authorized_event_id,input.metadata.broker.authorizedEventId);
+ assert.equal(output.safetynet.broker.connection_id,input.metadata.broker.connectionId);
+ assert.equal(output.trace,undefined);assert.equal(output.span,undefined);
+});
+test('a durable authorization is not reported as completed delivery',()=>{
+ const output=toECS(event('request.authorized'));assert.equal(output.event.outcome,'unknown');assert.deepEqual(output.event.type,['allowed']);
+});
+test('CEF standard custom string preserves all causation without overwriting integrity or type',()=>{
+ const input=event();input.audit={hash:'synthetic-hash',sequence:3};
+ const output=toCEF(input);assert.ok(output.includes('cs2Label=BrokerLifecycle'));
+ for(const value of [input.metadata.eventId,input.metadata.broker.requestId,input.metadata.broker.connectionId,input.metadata.broker.authorizedEventId])assert.ok(output.includes(value));
+ assert.ok(output.includes('flexString1=synthetic-hash'));assert.ok(output.includes('cs6=pdf'));
+ assert.ok(output.includes('outcome=failure'));assert.ok(toCEF(event('request.authorized')).includes('outcome=unknown'));
+});
+test('malformed or content-bearing broker facts are never exported as lifecycle metadata',()=>{
+ for(const modify of [input=>{input.metadata.broker.body='private-body';},input=>{input.metadata.broker.authorizedEventId=input.metadata.eventId;},input=>{input.metadata.broker.reason='private-reason';}]){
+  const input=event();modify(input);assert.equal(toECS(input).safetynet.broker,undefined);
+  const cef=toCEF(input);assert.equal(cef.includes('BrokerLifecycle'),false);assert.equal(cef.includes('private-'),false);
+ }
+});
+test('actual durable exporter preserves the authorization-to-fault UUID join',async()=>{
+ const root=path.resolve(os.tmpdir()),directory=fs.mkdtempSync(path.join(root,'safetynet-broker-siem-'));
+ if(path.dirname(path.resolve(directory))!==root||!path.basename(directory).startsWith('safetynet-broker-siem-'))throw Error('Synthetic cleanup path rejected');
+ const store=openStore(directory);let exporter;
+ try{
+  const authorized=event('request.authorized'),fault=event();
+  fault.metadata.broker.authorizedEventId=authorized.metadata.eventId;
+  fault.metadata.broker.requestId=authorized.metadata.broker.requestId;
+  store.event('synthetic','upload.authorized',authorized.metadata);store.event('synthetic','upload.block',fault.metadata);
+  exporter=startSIEM(store,directory,{});await exporter.flush();assert.equal(exporter.status().cursor,2);
+  const rows=fs.readFileSync(path.join(directory,'security.ndjson'),'utf8').trim().split('\n').map(JSON.parse);
+  assert.equal(rows.length,2);assert.equal(rows[0].event.outcome,'unknown');assert.equal(rows[1].event.outcome,'failure');
+  assert.equal(rows[1].safetynet.broker.authorized_event_id,rows[0].safetynet.broker.event_id);
+  assert.equal(rows[1].safetynet.broker.request_id,rows[0].safetynet.broker.request_id);
+ }finally{
+  exporter?.stop();store.db.close();
+  fs.rmSync(directory,{recursive:true,force:true});
+ }
+});

tests/request-audit.test.mjs

@@ -26,3 +26,18 @@ test('SIEM suppresses historical upload hashes while preserving transfer amounts
  const current={...legacy,metadata:{request:{bytes:18,detectedType:'text'}}};assert.deepEqual(toECS(current).safetynet.request,{bytes:18,detected_type:'text'});assert.match(toCEF(current),/cn3=18/);assert.equal(JSON.stringify(toECS(current)).includes(hash),false);
  legacy.kind='download.scan';assert.equal(toECS(legacy).file.hash.sha256,hash);
 });
+
+test('central API accepts typed broker causation and rejects invalid lifecycle semantics',async t=>{
+ const dir=mkdtempSync(join(tmpdir(),'safetynet-request-audit-')),store=openStore(dir),credential='synthetic-device-broker-token';
+ store.db.prepare('INSERT INTO devices(id,name,token_hash) VALUES(?,?,?)').run('broker-device','Synthetic',digest(credential));
+ const server=createApp(store,{publicUrl:'http://127.0.0.1',adminToken:'synthetic-administrator-token-123456789',secure:false}).listen(0,'127.0.0.1');await new Promise(r=>server.once('listening',r));
+ t.after(async()=>{server.closeAllConnections();await new Promise(r=>server.close(r));store.db.close();rmSync(dir,{recursive:true,force:true});});
+ const send=body=>fetch(`http://127.0.0.1:${server.address().port}/api/device/events`,{method:'POST',headers:{authorization:`Bearer ${credential}`,'content-type':'application/json'},body:JSON.stringify(body)});
+ const requestId=randomUUID(),authorizedEventId=randomUUID();
+ const allow={eventId:authorizedEventId,policyVersion:1,kind:'upload.authorized',ruleId:'broker-approved',request:{bytes:7,detectedType:'text'},broker:{requestId,phase:'request.authorized',reason:'broker-approved'}};
+ assert.equal((await send(allow)).status,200);
+ const fault={...allow,eventId:randomUUID(),kind:'upload.block',ruleId:'broker-upstream-failed',broker:{requestId,phase:'forward.fault',reason:'broker-upstream-failed',authorizedEventId}};
+ assert.equal((await send(fault)).status,200);
+ for(const broker of [{...fault.broker,authorizedEventId:undefined},{...fault.broker,authorizedEventId:fault.eventId},{...allow.broker,phase:'connect.block'},{...allow.broker,reason:'arbitrary secret details'},{...allow.broker,body:'private'}])assert.equal((await send({...fault,eventId:broker.authorizedEventId===fault.eventId?fault.eventId:randomUUID(),broker})).status,400);
+ const rows=store.db.prepare('SELECT metadata FROM events ORDER BY id').all();assert.equal(rows.length,2);assert.deepEqual(JSON.parse(rows[1].metadata).broker,fault.broker);assert.equal(store.auditIntegrity.verify().valid,true);
+});