114 lines
12 KiB
Python
114 lines
12 KiB
Python
|
"""Real OPC ZIP fixtures for strict static OOXML coverage and hostile parts."""
|
||
|
|
import base64
|
||
|
|
import io
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
import unittest
|
||
|
|
import zipfile
|
||
|
|
from pathlib import Path
|
||
|
|
sys.path.insert(0,str(Path(__file__).resolve().parent.parent/'apps/inspector'))
|
||
|
|
from stage_timings import collect,stage,metadata,validated
|
||
|
|
from office_content import extract_office,OfficeCoverageError,CT,REL,R,W,S,P,A,CP,MAIN,TYPE_PREFIX
|
||
|
|
PNG=base64.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Wl6b5sAAAAASUVORK5CYII=')
|
||
|
|
def xml(tag,namespace,inside=''):return f'<{tag} xmlns="{namespace}" xmlns:r="{R}">{inside}</{tag}>'.encode()
|
||
|
|
def relations(items):return xml('Relationships',REL,''.join(f'<Relationship Id="{id_}" Type="{type_}" Target="{target}"'+(f' TargetMode="{mode}"' if mode else '')+'/>' for id_,type_,target,mode in items))
|
||
|
|
def package(family='docx',parts=None,main=None,relationships=None,extra_info=None):
|
||
|
|
main_name={'docx':'word/document.xml','xlsx':'xl/workbook.xml','pptx':'ppt/presentation.xml'}[family]
|
||
|
|
main_type=next(type_ for type_,values in MAIN.items() if values[0]==family)
|
||
|
|
default={'docx':xml('document',W,'<body><p><r><t>Public company newsletter</t></r></p></body>'),'xlsx':xml('workbook',S,'<sheets><sheet name="Public budget" sheetId="1" r:id="rId1"/></sheets>'),'pptx':xml('presentation',P,'<sldIdLst><sldId id="256" r:id="rId1"/></sldIdLst>')}
|
||
|
|
values={main_name:(main_type,main or default[family])};values.update(parts or {})
|
||
|
|
if family=='xlsx' and 'xl/worksheets/sheet1.xml' not in values:values['xl/worksheets/sheet1.xml']=(TYPE_PREFIX+'spreadsheetml.worksheet+xml',xml('worksheet',S,'<sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>Public projected revenue</t></is></c></row></sheetData>'))
|
||
|
|
if family=='pptx' and 'ppt/slides/slide1.xml' not in values:values['ppt/slides/slide1.xml']=(TYPE_PREFIX+'presentationml.slide+xml',xml('sld',P,f'<cSld><spTree><nvGrpSpPr/><grpSpPr/><sp><nvSpPr/><spPr/><txBody xmlns:a="{A}"><a:bodyPr/><a:lstStyle/><a:p><a:r><a:t>Public quarterly update</a:t></a:r></a:p></txBody></sp></spTree></cSld>'))
|
||
|
|
rels={'_rels/.rels':[('root',R+'/officeDocument',main_name,None)]}
|
||
|
|
if family=='xlsx':rels['xl/_rels/workbook.xml.rels']=[('rId1',R+'/worksheet','worksheets/sheet1.xml',None)]
|
||
|
|
if family=='pptx':rels['ppt/_rels/presentation.xml.rels']=[('rId1',R+'/slide','slides/slide1.xml',None)]
|
||
|
|
rels.update(relationships or {})
|
||
|
|
types=f'<Types xmlns="{CT}"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'+''.join(f'<Override PartName="/{name}" ContentType="{type_}"/>' for name,(type_,_payload) in values.items())+'</Types>'
|
||
|
|
output=io.BytesIO()
|
||
|
|
with zipfile.ZipFile(output,'w',zipfile.ZIP_DEFLATED) as archive:
|
||
|
|
archive.writestr('[Content_Types].xml',types)
|
||
|
|
for name,(_type,payload) in values.items():archive.writestr(name,payload)
|
||
|
|
for name,items in rels.items():archive.writestr(name,relations(items))
|
||
|
|
if extra_info:archive.writestr(extra_info,b'PUBLIC')
|
||
|
|
return output.getvalue()
|
||
|
|
def extract(data,callback=lambda _data:('',True),**limits):return extract_office(data,callback,**limits)
|
||
|
|
class OfficeTests(unittest.TestCase):
|
||
|
|
def test_public_valid_docx_xlsx_and_pptx_static_parts_complete(self):
|
||
|
|
for family in ['docx','xlsx','pptx']:
|
||
|
|
result=extract(package(family));self.assertTrue(result['complete'],result['reasons']);self.assertEqual(result['family'],family);self.assertIn('Public',result['text'])
|
||
|
|
def test_core_metadata_attributes_namespace_and_split_runs_included(self):
|
||
|
|
main=xml('document',W,'<body><p secret="ATTR-SECRET"><r><t>TOP</t></r><r><t>SECRET</t></r></p></body>')
|
||
|
|
metadata=xml('coreProperties',CP,'<title xmlns="http://purl.org/dc/elements/1.1/">METADATA-SECRET</title>')
|
||
|
|
result=extract(package(parts={'docProps/core.xml':('application/vnd.openxmlformats-package.core-properties+xml',metadata)},main=main))
|
||
|
|
self.assertTrue(result['complete']);self.assertIn('ATTR-SECRET',result['text']);self.assertIn('METADATA-SECRET',result['text']);self.assertIn('TOPSECRET',result['text']);self.assertIn(W,result['text'])
|
||
|
|
def test_shared_strings_and_hidden_unreferenced_parts_are_inspected(self):
|
||
|
|
parts={'xl/sharedStrings.xml':(TYPE_PREFIX+'spreadsheetml.sharedStrings+xml',xml('sst',S,'<si><t>SHARED-SECRET</t></si>')),'xl/worksheets/hidden.xml':(TYPE_PREFIX+'spreadsheetml.worksheet+xml',xml('worksheet',S,'<sheetData><row><c t="inlineStr"><is><t>HIDDEN-SECRET</t></is></c></row></sheetData>'))}
|
||
|
|
result=extract(package('xlsx',parts));self.assertTrue(result['complete']);self.assertIn('SHARED-SECRET',result['text']);self.assertIn('HIDDEN-SECRET',result['text'])
|
||
|
|
def test_image_text_receives_ocr_and_cannot_escape_dlp_input(self):
|
||
|
|
parts={'word/media/image1.png':('image/png',PNG)};calls=[]
|
||
|
|
def ocr(payload):calls.append(payload);return ('RASTER-SECRET',True)
|
||
|
|
result=extract(package(parts=parts),ocr);self.assertTrue(result['complete']);self.assertEqual(calls,[PNG]);self.assertIn('RASTER-SECRET',result['text']);self.assertEqual(len(result['media']),1)
|
||
|
|
self.assertFalse(extract(package(parts=parts),lambda _data:('',False))['complete'])
|
||
|
|
def test_external_relationship_metadata_scanned_but_coverage_incomplete(self):
|
||
|
|
result=extract(package(relationships={'word/_rels/document.xml.rels':[('external',R+'/hyperlink','https://host/RELATIONSHIP-SECRET','External')]}))
|
||
|
|
self.assertFalse(result['complete']);self.assertIn('RELATIONSHIP-SECRET',result['text'])
|
||
|
|
def test_unsupported_font_ole_macro_equation_chart_and_vector_fail_closed(self):
|
||
|
|
for name,type_,payload in [('word/fonts/private.odttf','application/vnd.openxmlformats-officedocument.obfuscatedFont',b'font'),('word/embeddings/a.bin','application/vnd.openxmlformats-officedocument.oleObject',b'OLE'),('word/vbaProject.bin','application/vnd.ms-office.vbaProject',b'VBA'),('word/chart.xml',TYPE_PREFIX+'drawingml.chart+xml',xml('chartSpace',A,'<chart/>'))]:
|
||
|
|
self.assertFalse(extract(package(parts={name:(type_,payload)}))['complete'])
|
||
|
|
for content in ['<oMath xmlns="http://schemas.openxmlformats.org/officeDocument/2006/math"/>',f'<custGeom xmlns="{A}"/>','<fldSimple instr="DDE secret"/>']:
|
||
|
|
self.assertFalse(extract(package(main=xml('document',W,'<body>'+content+'</body>')))['complete'])
|
||
|
|
def test_unknown_xml_content_type_is_incomplete_even_if_text_readable(self):
|
||
|
|
result=extract(package(parts={'word/private.xml':('application/x-private+xml',xml('document',W,'<t>SECRET</t>'))}));self.assertFalse(result['complete']);self.assertIn('SECRET',result['text'])
|
||
|
|
def test_duplicate_case_paths_traversal_and_compression_bombs_rejected(self):
|
||
|
|
for extra in ['word/../outside','WORD/document.xml']:
|
||
|
|
with self.assertRaises(OfficeCoverageError):extract(package(extra_info=extra))
|
||
|
|
with self.assertRaises(OfficeCoverageError):extract(package(main=xml('document',W,'A'*100000)))
|
||
|
|
def test_relationship_target_missing_escape_id_duplicate_and_role_mismatch_rejected(self):
|
||
|
|
for items in [[('x',R+'/styles','missing.xml',None)],[('x',R+'/styles','../../../escape.xml',None)],[('x',R+'/styles','document.xml',None)],[('x',R+'/officeDocument','document.xml',None),('x',R+'/officeDocument','document.xml',None)]]:
|
||
|
|
with self.assertRaises(OfficeCoverageError):extract(package(relationships={'word/_rels/document.xml.rels':items}))
|
||
|
|
def test_dtd_and_bad_xml_are_rejected(self):
|
||
|
|
for payload in [b'<!DOCTYPE x [<!ENTITY e "SECRET">]><x>&e;</x>',b'<x>']:
|
||
|
|
with self.assertRaises(Exception):extract(package(main=payload))
|
||
|
|
def test_comments_and_processing_instructions_are_not_silently_discarded(self):
|
||
|
|
result=extract(package(main=xml('document',W,'<!--COMMENT-SECRET--><body><p><r><t>Public</t></r></p></body>')))
|
||
|
|
self.assertTrue(result['complete']);self.assertIn('COMMENT-SECRET',result['text'])
|
||
|
|
result=extract(package(main=xml('document',W,'<?xml-stylesheet href="PI-SECRET"?><body/>')))
|
||
|
|
self.assertFalse(result['complete']);self.assertIn('PI-SECRET',result['text'])
|
||
|
|
def test_simple_formula_cache_covered_dynamic_dde_and_missing_cache_incomplete(self):
|
||
|
|
def sheet(formula,cache):return xml('worksheet',S,'<sheetData><row><c r="A1"><f>'+formula+'</f><v>'+cache+'</v></c></row></sheetData>')
|
||
|
|
for formula,cache,complete in [('SUM(1,2)','3',True),('IF(1,"CACHE-SECRET","Public")','CACHE-SECRET',True),('CHAR(83)&CHAR(69)','SE',False),('WEBSERVICE("https://host")','Public',False),("cmd|'/c calc'!A0",'0',False),('SUM(1,2)','',False)]:
|
||
|
|
result=extract(package('xlsx',{'xl/worksheets/sheet1.xml':(TYPE_PREFIX+'spreadsheetml.worksheet+xml',sheet(formula,cache))}))
|
||
|
|
self.assertEqual(result['complete'],complete,formula)
|
||
|
|
if cache:self.assertIn(cache,result['text'])
|
||
|
|
def test_trailing_data_and_local_extra_fields_cannot_claim_coverage(self):
|
||
|
|
with self.assertRaises(OfficeCoverageError):extract(package()+b'TRAILING-SECRET')
|
||
|
|
info=zipfile.ZipInfo('word/extra.bin');info.extra=b'\x01\x00\x03\x00ABC'
|
||
|
|
with self.assertRaises(OfficeCoverageError):extract(package(extra_info=info))
|
||
|
|
def test_aggregate_stages_have_real_parent_timing_and_no_document_content(self):
|
||
|
|
parts={'word/header'+str(index)+'.xml':(TYPE_PREFIX+'wordprocessingml.header+xml',xml('hdr',W,'<p><r><t>SYNTHETIC-PRIVATE-CONTENT</t></r></p>')) for index in range(90)}
|
||
|
|
with collect({'diagnostics':{'enabled':True,'mode':'debug'}}) as state:
|
||
|
|
with stage('parse'):result=extract(package(parts=parts))
|
||
|
|
self.assertTrue(result['complete']);proof=metadata(state);self.assertIs(validated(proof),proof)
|
||
|
|
self.assertEqual([record['stage'] for record in proof['records']],['parse','office.archive','office.xml','office.relationships','office.ocr'])
|
||
|
|
parent=proof['records'][0]
|
||
|
|
for record in proof['records'][1:]:
|
||
|
|
self.assertEqual(record['parentId'],parent['id']);self.assertGreaterEqual(record['startOffsetMs'],parent['startOffsetMs']);self.assertLessEqual(record['startOffsetMs']+record['durationMs'],parent['startOffsetMs']+parent['durationMs']);self.assertEqual(record['status'],'ok')
|
||
|
|
self.assertNotIn('SYNTHETIC-PRIVATE-CONTENT',json.dumps(proof));self.assertNotIn('header90.xml',json.dumps(proof))
|
||
|
|
def test_error_stages_preserve_correlation_without_exception_text(self):
|
||
|
|
def bad_ocr(_payload):raise ValueError('SYNTHETIC-PRIVATE-ERROR')
|
||
|
|
cases=[(package()+b'TAIL',None,'office.archive'),(package(main=b'<BROKEN>'),None,'office.xml'),(package(relationships={'word/_rels/document.xml.rels':[('x',R+'/styles','missing.xml',None)]}),None,'office.relationships'),(package(parts={'word/media/image.png':('image/png',PNG)}),bad_ocr,'office.ocr')]
|
||
|
|
for data,callback,expected in cases:
|
||
|
|
with collect({'diagnostics':{'enabled':True,'mode':'debug'}}) as state:
|
||
|
|
with self.assertRaises(Exception):
|
||
|
|
with stage('parse'):extract(data,callback or (lambda _image:('',True)))
|
||
|
|
proof=metadata(state);self.assertIs(validated(proof),proof);self.assertEqual(proof['records'][0]['status'],'error')
|
||
|
|
record=next(record for record in proof['records'] if record['stage']==expected);self.assertEqual(record['status'],'error');self.assertEqual(record['parentId'],1);self.assertNotIn('SYNTHETIC-PRIVATE-ERROR',json.dumps(proof))
|
||
|
|
def test_disabled_or_operational_diagnostics_do_not_change_extraction(self):
|
||
|
|
for policy in [{},{'diagnostics':{'enabled':False,'mode':'debug'}},{'diagnostics':{'enabled':True,'mode':'operational'}}]:
|
||
|
|
with collect(policy) as state:result=extract(package())
|
||
|
|
self.assertTrue(result['complete']);self.assertIsNone(metadata(state))
|
||
|
|
def test_bounds_cannot_return_complete(self):
|
||
|
|
data=package()
|
||
|
|
for limits in [{'max_bytes':len(data)-1},{'max_parts':1},{'max_text':10}]:
|
||
|
|
with self.assertRaises(OfficeCoverageError):extract(data,**limits)
|
||
|
|
if __name__=='__main__':unittest.main()
|