Implement managed browser security, individual agents and observability

apps/inspector/engine.py

@@ -8,9 +8,11 @@ import re
 import shutil
 import subprocess
 import tempfile
+import time
 import unicodedata
 import zipfile
 from pathlib import Path
+from stage_timings import timed, stage
 
 os.environ['HF_HUB_OFFLINE']='1'
 os.environ['TRANSFORMERS_OFFLINE']='1'
@@ -22,6 +24,7 @@ _MODEL = None
 _OCR = None
 
 
+@timed("ocr.load")
 def local_ocr():
     global _OCR
     if _OCR is None:
@@ -46,6 +49,7 @@ def local_ocr():
     return _OCR
 
 
+@timed("yara")
 def scan_download(data, policy):
     """Malware gate only: never implies document/DLP coverage."""
     result={"decision":"block","ruleId":"yara-unavailable","engine":"yara",
@@ -72,8 +76,26 @@ def scan_download(data, policy):
         import yara
         rules=yara.compile(sources=sources,includes=False,error_on_warning=True)
         matches=rules.match(data=data,timeout=5)
+        if config.get("scanOfficeMacros",True) is not False:
+            from office_macros import macro_payloads
+            try:
+                with stage("macros"):
+                    summary,payloads=macro_payloads(data)
+                result["officeMacros"]=summary
+                deadline=time.monotonic()+10
+                for payload in payloads:
+                    if time.monotonic()>=deadline: raise ValueError('macro-scan-timeout')
+                    matches.extend(rules.match(data=payload,timeout=5))
+            except Exception:
+                result["ruleId"]="office-macro-inspection-failed"
+                result["officeMacros"]={"office":data.startswith(b'\xd0\xcf\x11\xe0') or data.startswith(b'PK'),
+                                       "present":False,"modules":0,"complete":False}
+                return result
+        else:
+            result["officeMacros"]={"office":False,"present":False,"modules":0,"complete":False}
         result["coverage"]["complete"]=True
-        result["matchedRules"]=[{"id":ids[item.namespace],"rule":item.rule} for item in matches]
+        result["matchedRules"]=[{"id":identifier,"rule":rule} for identifier,rule in
+                                dict.fromkeys((ids[item.namespace],item.rule) for item in matches)]
         if matches: result["ruleId"]=ids[matches[0].namespace]
         else: result["decision"]="allow";result["ruleId"]=None
     except ImportError: pass
@@ -81,6 +103,7 @@ def scan_download(data, policy):
     return result
 
 
+@timed("inspect")
 def reference_document(data):
     """Return privacy-preserving registration features, without extracted plaintext."""
     result={"sha256":hashlib.sha256(data).hexdigest(),"bytes":len(data),
@@ -143,6 +166,7 @@ def content_kind(data):
     except UnicodeError: return "unknown"
 
 
+@timed("ocr")
 def ocr_image(data):
     from PIL import Image, ImageSequence
     image=Image.open(io.BytesIO(data))
@@ -169,6 +193,7 @@ def ocr_image(data):
     return "\n".join(parts), True
 
 
+@timed("parse")
 def extract(data, kind):
     if kind=="text": return data.decode("utf-8"), True, []
     if kind=="image":
@@ -224,6 +249,7 @@ def extract(data, kind):
     raise ValueError("Unsupported content type")
 
 
+@timed("semantic.load")
 def get_model():
     global _MODEL
     directory=os.environ.get("EMBEDDING_MODEL_DIR", "")
@@ -235,6 +261,7 @@ def get_model():
     return _MODEL
 
 
+@timed("semantic")
 def embed(data, kind):
     model=get_model()
     if kind=="text": vectors=model.text_vectors(data.decode("utf-8"))
@@ -253,6 +280,7 @@ def aggregate_vectors(vectors):
     return [value/norm for value in vector]
 
 
+@timed("semantic")
 def semantic_vector_features(text,media):
     model=get_model();vectors=model.text_vectors(text) if text.strip() else []
     text_chunks=len(vectors)
@@ -269,6 +297,7 @@ def semantic_vectors(text,media):
     return semantic_vector_features(text,media)[0]
 
 
+@timed("semantic")
 def embedding_features(data):
     kind=content_kind(data)
     if len(data)>MAX_BYTES: raise ValueError("Embedding input budget exceeded")
@@ -297,6 +326,7 @@ def deterministic(text, data, policy):
     return None
 
 
+@timed("inspect")
 def inspect_content(data, policy, encoder=embed):
     kind=content_kind(data)
     result={"decision":"block","ruleId":"uninspectable","contentType":kind,"coverage":{"complete":False},"model":None}