216 lines
19 KiB
Python
216 lines
19 KiB
Python
|
"""Bounded static OOXML content coverage, not an Office renderer or macro engine.
|
||
|
|
|
||
|
|
OPC and part families: Microsoft Open XML SDK structure documentation:
|
||
|
|
https://learn.microsoft.com/en-us/office/open-xml/general/how-to-create-a-package
|
||
|
|
https://learn.microsoft.com/en-us/office/open-xml/spreadsheet/structure-of-a-spreadsheetml-document
|
||
|
|
https://learn.microsoft.com/en-us/office/open-xml/presentation/structure-of-a-presentationml-document
|
||
|
|
All XML text, tails, attributes, namespace declarations and ZIP metadata are inputs.
|
||
|
|
Only known static XML parts and successfully OCR-inspected raster assets are covered.
|
||
|
|
Unknown parts, equations, vector/custom layouts and active features remain incomplete.
|
||
|
|
"""
|
||
|
|
import io
|
||
|
|
import posixpath
|
||
|
|
import re
|
||
|
|
import struct
|
||
|
|
import unicodedata
|
||
|
|
import zipfile
|
||
|
|
from urllib.parse import urlsplit
|
||
|
|
from defusedxml.ElementTree import fromstring,iterparse
|
||
|
|
from stage_timings import stage
|
||
|
|
CT='http://schemas.openxmlformats.org/package/2006/content-types'
|
||
|
|
REL='http://schemas.openxmlformats.org/package/2006/relationships'
|
||
|
|
R='http://schemas.openxmlformats.org/officeDocument/2006/relationships'
|
||
|
|
W='http://schemas.openxmlformats.org/wordprocessingml/2006/main'
|
||
|
|
S='http://schemas.openxmlformats.org/spreadsheetml/2006/main'
|
||
|
|
P='http://schemas.openxmlformats.org/presentationml/2006/main'
|
||
|
|
A='http://schemas.openxmlformats.org/drawingml/2006/main'
|
||
|
|
CP='http://schemas.openxmlformats.org/package/2006/metadata/core-properties'
|
||
|
|
EP='http://schemas.openxmlformats.org/officeDocument/2006/extended-properties'
|
||
|
|
CUSTOM='http://schemas.openxmlformats.org/officeDocument/2006/custom-properties'
|
||
|
|
TYPE_PREFIX='application/vnd.openxmlformats-officedocument.'
|
||
|
|
MAIN={TYPE_PREFIX+'wordprocessingml.document.main+xml':('docx',W,'document'),TYPE_PREFIX+'spreadsheetml.sheet.main+xml':('xlsx',S,'workbook'),TYPE_PREFIX+'presentationml.presentation.main+xml':('pptx',P,'presentation')}
|
||
|
|
XML_TYPES={TYPE_PREFIX+'wordprocessingml.'+name+'+xml':(W,root) for name,root in [('styles','styles'),('settings','settings'),('fontTable','fonts'),('numbering','numbering'),('header','hdr'),('footer','ftr'),('comments','comments'),('footnotes','footnotes'),('endnotes','endnotes'),('webSettings','webSettings')]}
|
||
|
|
XML_TYPES.update({TYPE_PREFIX+'spreadsheetml.'+name+'+xml':(S,root) for name,root in [('worksheet','worksheet'),('sharedStrings','sst'),('styles','styleSheet'),('table','table')]})
|
||
|
|
XML_TYPES.update({TYPE_PREFIX+'presentationml.'+name+'+xml':(P,root) for name,root in [('slide','sld'),('notesSlide','notes'),('notesMaster','notesMaster'),('slideMaster','sldMaster'),('slideLayout','sldLayout'),('presProps','presentationPr'),('viewProps','viewPr'),('tableStyles','tblStyleLst')]})
|
||
|
|
XML_TYPES['application/vnd.ms-word.stylesWithEffects+xml']=(W,'styles')
|
||
|
|
XML_TYPES[TYPE_PREFIX+'customXmlProperties+xml']=('http://schemas.openxmlformats.org/officeDocument/2006/customXml','datastoreItem')
|
||
|
|
XML_TYPES[TYPE_PREFIX+'presentationml.tableStyles+xml']=(A,'tblStyleLst')
|
||
|
|
XML_TYPES.update({'application/vnd.openxmlformats-package.core-properties+xml':(CP,'coreProperties'),TYPE_PREFIX+'extended-properties+xml':(EP,'Properties'),TYPE_PREFIX+'custom-properties+xml':(CUSTOM,'Properties'),TYPE_PREFIX+'theme+xml':(A,'theme')})
|
||
|
|
SAFE_REL={R+'/'+name for name in ['officeDocument','extended-properties','custom-properties','styles','settings','fontTable','numbering','header','footer','comments','footnotes','endnotes','worksheet','sharedStrings','table','theme','slide','notesSlide','notesMaster','slideMaster','slideLayout','presProps','viewProps','tableStyles','image']}
|
||
|
|
SAFE_REL.update({R+'/customXml',R+'/customXmlProps',R+'/webSettings','http://schemas.microsoft.com/office/2007/relationships/stylesWithEffects','http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties','http://schemas.openxmlformats.org/package/2006/relationships/metadata/thumbnail'})
|
||
|
|
SAFE_NS={CT,REL,R,W,S,P,A,CP,EP,CUSTOM,'http://schemas.openxmlformats.org/drawingml/2006/picture','http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing','http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes','http://purl.org/dc/elements/1.1/','http://purl.org/dc/terms/','http://purl.org/dc/dcmitype/','http://www.w3.org/2001/XMLSchema-instance','http://www.w3.org/XML/1998/namespace','http://schemas.openxmlformats.org/markup-compatibility/2006','http://schemas.openxmlformats.org/officeDocument/2006/bibliography','http://schemas.openxmlformats.org/officeDocument/2006/customXml','http://schemas.openxmlformats.org/officeDocument/2006/math'}
|
||
|
|
SAFE_EXTENSION_TAGS={'{urn:schemas-microsoft-com:office:office}'+name for name in ['shapedefaults','shapelayout','idmap']}|{'{http://schemas.microsoft.com/office/word/2010/wordml}'+name for name in ['docId','defaultImageDpi']}
|
||
|
|
UNSUPPORTED_TAGS={'AlternateContent','oMath','oMathPara','custGeom','pict','object','control','oleObj','graphicFrame','chart','grpSp','cxnSp','fldSimple','fldChar','instrText','altChunk','externalLink','externalReference','externalReferences','blob','hlinkClick','hlinkHover','animation','transition','timing','embedRegular','embedBold','embedItalic','embedBoldItalic'}
|
||
|
|
class OfficeCoverageError(ValueError):pass
|
||
|
|
|
||
|
|
def _part(name):
|
||
|
|
if not isinstance(name,str) or not name or len(name)>240 or '\\' in name or '%' in name or name.startswith('/') or any(part in {'','.', '..'} for part in name.split('/')) or any(ord(c)<32 for c in name) or unicodedata.normalize('NFC',name)!=name:raise OfficeCoverageError('office-part-name-invalid')
|
||
|
|
return name
|
||
|
|
|
||
|
|
def _physical_zip(data,archive,infos):
|
||
|
|
if not data.startswith(b'PK\x03\x04'):raise OfficeCoverageError('office-archive-prefix-invalid')
|
||
|
|
cursor=0
|
||
|
|
for info in sorted(infos,key=lambda item:item.header_offset):
|
||
|
|
if info.header_offset!=cursor or cursor+30>len(data):raise OfficeCoverageError('office-archive-gap')
|
||
|
|
signature,version,flags,method,time,date,crc,compressed,size,name_length,extra_length=struct.unpack_from('<IHHHHHIIIHH',data,cursor)
|
||
|
|
if signature!=0x04034b50 or flags!=info.flag_bits or method!=info.compress_type:raise OfficeCoverageError('office-local-header-invalid')
|
||
|
|
if extra_length:raise OfficeCoverageError('office-local-extra-uncovered')
|
||
|
|
cursor+=30+name_length+extra_length+info.compress_size
|
||
|
|
if flags&8:
|
||
|
|
if data[cursor:cursor+4]==b'PK\x07\x08':cursor+=4
|
||
|
|
if cursor+12>len(data) or struct.unpack_from('<III',data,cursor)!=(info.CRC,info.compress_size,info.file_size):raise OfficeCoverageError('office-descriptor-invalid')
|
||
|
|
cursor+=12
|
||
|
|
elif (crc,compressed,size)!=(info.CRC,info.compress_size,info.file_size):raise OfficeCoverageError('office-local-size-invalid')
|
||
|
|
if cursor!=archive.start_dir:raise OfficeCoverageError('office-archive-gap')
|
||
|
|
for _info in infos:
|
||
|
|
if data[cursor:cursor+4]!=b'PK\x01\x02' or cursor+46>len(data):raise OfficeCoverageError('office-central-header-invalid')
|
||
|
|
name_length,extra_length,comment_length=struct.unpack_from('<HHH',data,cursor+28);cursor+=46+name_length+extra_length+comment_length
|
||
|
|
if data[cursor:cursor+4]!=b'PK\x05\x06' or cursor+22>len(data):raise OfficeCoverageError('office-end-record-invalid')
|
||
|
|
disk,start_disk,count_disk,count,total_size,start,comment=struct.unpack_from('<HHHHIIH',data,cursor+4)
|
||
|
|
if disk or start_disk or count_disk!=len(infos) or count!=len(infos) or start!=archive.start_dir or total_size!=cursor-archive.start_dir or cursor+22+comment!=len(data):raise OfficeCoverageError('office-end-record-invalid')
|
||
|
|
|
||
|
|
def extract_office(data,ocr_image,*,max_bytes=20*1024*1024,max_parts=2000,max_media=100,max_text=2*1024*1024):
|
||
|
|
if not isinstance(data,bytes) or len(data)>max_bytes or not callable(ocr_image):raise OfficeCoverageError('office-input-invalid')
|
||
|
|
texts=[];seen_text=set();media=[];reasons=set();xml={};images={};types={};members={};payloads={};total=0;nodes=0
|
||
|
|
def append(value):
|
||
|
|
nonlocal total
|
||
|
|
if value:
|
||
|
|
value=str(value)
|
||
|
|
if value in seen_text:return
|
||
|
|
seen_text.add(value);total+=len(value)
|
||
|
|
if total>max_text:raise OfficeCoverageError('office-text-limit')
|
||
|
|
texts.append(value)
|
||
|
|
with stage('office.archive'):
|
||
|
|
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||
|
|
infos=archive.infolist()
|
||
|
|
_physical_zip(data,archive,infos)
|
||
|
|
if len(infos)>max_parts or sum(item.file_size for item in infos)>max_bytes:raise OfficeCoverageError('office-expansion-limit')
|
||
|
|
append(archive.comment.decode('utf-8','strict'))
|
||
|
|
for info in infos:
|
||
|
|
name=_part(info.filename)
|
||
|
|
if name.casefold() in members:raise OfficeCoverageError('office-duplicate-part')
|
||
|
|
members[name.casefold()]=(name,info)
|
||
|
|
if info.is_dir() or info.flag_bits&1 or info.compress_type not in {zipfile.ZIP_STORED,zipfile.ZIP_DEFLATED} or info.file_size>max_bytes or info.file_size>max(1,info.compress_size)*200 or (info.external_attr>>16)&0o170000==0o120000:raise OfficeCoverageError('office-archive-unsafe')
|
||
|
|
# Unknown local/central extra fields may hide material outside parts.
|
||
|
|
if info.extra:reasons.add('zip-extra-field-uncovered')
|
||
|
|
append(name);append(info.comment.decode('utf-8','strict'))
|
||
|
|
payload=archive.read(info)
|
||
|
|
if len(payload)!=info.file_size:raise OfficeCoverageError('office-part-size-invalid')
|
||
|
|
payloads[name]=payload
|
||
|
|
with stage('office.xml'):
|
||
|
|
for name,payload in payloads.items():
|
||
|
|
if name.endswith(('.xml','.rels')) or name=='[Content_Types].xml':
|
||
|
|
root=fromstring(payload,forbid_dtd=True,forbid_entities=True,forbid_external=True);xml[name]=root
|
||
|
|
for _event,namespace in iterparse(io.BytesIO(payload),events=['start-ns'],forbid_dtd=True,forbid_entities=True,forbid_external=True):append(namespace[0]);append(namespace[1])
|
||
|
|
for event,item in iterparse(io.BytesIO(payload),events=['comment','pi'],forbid_dtd=True,forbid_entities=True,forbid_external=True):
|
||
|
|
append(item.text)
|
||
|
|
if event=='pi':reasons.add('xml-processing-instruction-unsupported')
|
||
|
|
parents={child:parent for parent in root.iter() for child in parent}
|
||
|
|
append(''.join(value for value in root.itertext() if value.strip()))
|
||
|
|
for element in root.iter():
|
||
|
|
nodes+=1
|
||
|
|
if nodes>100000:raise OfficeCoverageError('office-xml-node-limit')
|
||
|
|
append(element.tag);append(element.text);append(element.tail)
|
||
|
|
for key,value in element.attrib.items():
|
||
|
|
append(key);append(value)
|
||
|
|
if key.startswith('{') and key[1:].split('}',1)[0] not in SAFE_NS and key not in {'{http://schemas.microsoft.com/office/word/2010/wordml}paraId','{http://schemas.microsoft.com/office/word/2010/wordml}textId'} and not (element.tag in SAFE_EXTENSION_TAGS and ((key=='{urn:schemas-microsoft-com:vml}ext' and value=='edit') or (key=='{http://schemas.microsoft.com/office/word/2010/wordml}val' and value.isdecimal()))):reasons.add('xml-namespace-unsupported')
|
||
|
|
namespace=element.tag[1:].split('}',1)[0] if element.tag.startswith('{') else ''
|
||
|
|
if namespace not in SAFE_NS and element.tag not in SAFE_EXTENSION_TAGS:reasons.add('xml-namespace-unsupported')
|
||
|
|
if element.tag.rsplit('}',1)[-1] in UNSUPPORTED_TAGS:reasons.add('xml-feature-unsupported')
|
||
|
|
if element.tag=='{'+A+'}prstGeom' and element.get('prst') not in {'rect','roundRect'}:reasons.add('vector-preset-unsupported')
|
||
|
|
if element.tag=='{'+S+'}f':
|
||
|
|
formula=element.text or '';cached=parents.get(element)
|
||
|
|
cached=cached.find('{'+S+'}v') if cached is not None else None
|
||
|
|
without_strings=re.sub(r'"(?:[^"]|"")*"','',formula)
|
||
|
|
functions={name.upper() for name in re.findall(r'([A-Za-z_][A-Za-z0-9_.]*)\s*\(',without_strings)}
|
||
|
|
safe_functions={'SUM','SUMIF','SUMIFS','COUNT','COUNTIF','COUNTIFS','COUNTA','AVERAGE','AVERAGEIF','AVERAGEIFS','MIN','MAX','ROUND','ROUNDDOWN','ROUNDUP','ABS','IF','AND','OR','NOT'}
|
||
|
|
if not formula or set(element.attrib)-{'ca'} or any(c in without_strings for c in '[]|') or not functions<=safe_functions or cached is None or not (cached.text or '').strip():reasons.add('formula-coverage-incomplete')
|
||
|
|
|
||
|
|
elif payload.startswith((b'\x89PNG\r\n\x1a\n',b'\xff\xd8\xff',b'GIF87a',b'GIF89a')):images[name]=payload
|
||
|
|
else:reasons.add('binary-part-unsupported')
|
||
|
|
with stage('office.relationships'):
|
||
|
|
content=xml.get('[Content_Types].xml')
|
||
|
|
if content is None or content.tag!='{'+CT+'}Types':raise OfficeCoverageError('office-content-types-invalid')
|
||
|
|
defaults={};overrides={}
|
||
|
|
for item in content:
|
||
|
|
if item.tag=='{'+CT+'}Default' and set(item.attrib)=={'Extension','ContentType'}:
|
||
|
|
ext=item.attrib['Extension'].lower()
|
||
|
|
if not re.fullmatch('[a-z0-9]+',ext) or ext in defaults:raise OfficeCoverageError('office-content-types-invalid')
|
||
|
|
defaults[ext]=item.attrib['ContentType']
|
||
|
|
elif item.tag=='{'+CT+'}Override' and set(item.attrib)=={'PartName','ContentType'}:
|
||
|
|
part=item.attrib['PartName']
|
||
|
|
if not part.startswith('/'):raise OfficeCoverageError('office-content-types-invalid')
|
||
|
|
part=_part(part[1:])
|
||
|
|
if part.casefold() in {p.casefold() for p in overrides}:raise OfficeCoverageError('office-content-types-invalid')
|
||
|
|
overrides[part]=item.attrib['ContentType']
|
||
|
|
else:raise OfficeCoverageError('office-content-types-invalid')
|
||
|
|
if any(name.casefold() not in members for name in overrides):raise OfficeCoverageError('office-content-types-dangling')
|
||
|
|
main=[]
|
||
|
|
for name,_info in members.values():
|
||
|
|
if name=='[Content_Types].xml':continue
|
||
|
|
type_=overrides.get(name,defaults.get(name.rsplit('.',1)[-1].lower()))
|
||
|
|
if not type_:raise OfficeCoverageError('office-content-type-missing')
|
||
|
|
types[name]=type_
|
||
|
|
if type_ in MAIN:
|
||
|
|
family,namespace,rootname=MAIN[type_];main.append((name,family))
|
||
|
|
if name not in xml or xml[name].tag!='{'+namespace+'}'+rootname:raise OfficeCoverageError('office-main-part-invalid')
|
||
|
|
elif name.endswith('.rels'):
|
||
|
|
if type_!='application/vnd.openxmlformats-package.relationships+xml':raise OfficeCoverageError('office-relationship-type-invalid')
|
||
|
|
elif type_=='application/xml' and name in xml and xml[name].tag=='{http://schemas.openxmlformats.org/officeDocument/2006/bibliography}Sources':pass
|
||
|
|
elif type_ in XML_TYPES:
|
||
|
|
namespace,rootname=XML_TYPES[type_]
|
||
|
|
if name not in xml or xml[name].tag!='{'+namespace+'}'+rootname:raise OfficeCoverageError('office-xml-part-invalid')
|
||
|
|
elif name in images:
|
||
|
|
expected='image/png' if images[name].startswith(b'\x89PNG') else 'image/jpeg' if images[name].startswith(b'\xff\xd8') else 'image/gif'
|
||
|
|
if type_!=expected:raise OfficeCoverageError('office-image-type-invalid')
|
||
|
|
else:reasons.add('content-type-unsupported')
|
||
|
|
if len(main)!=1:raise OfficeCoverageError('office-main-count-invalid')
|
||
|
|
relationships={};top_main=[]
|
||
|
|
for name,root in xml.items():
|
||
|
|
if not name.endswith('.rels'):continue
|
||
|
|
if root.tag!='{'+REL+'}Relationships':raise OfficeCoverageError('office-relationships-invalid')
|
||
|
|
if name=='_rels/.rels':owner=''
|
||
|
|
elif '/_rels/' in name:
|
||
|
|
prefix,file=name.rsplit('/_rels/',1);owner=prefix+'/'+file[:-5]
|
||
|
|
if owner.casefold() not in members:raise OfficeCoverageError('office-relationships-owner-invalid')
|
||
|
|
else:raise OfficeCoverageError('office-relationships-location-invalid')
|
||
|
|
ids=set()
|
||
|
|
for item in root:
|
||
|
|
if item.tag!='{'+REL+'}Relationship' or not {'Id','Type','Target'}<=set(item.attrib) or set(item.attrib)-{'Id','Type','Target','TargetMode'}:raise OfficeCoverageError('office-relationship-invalid')
|
||
|
|
id_=item.attrib['Id'];target=item.attrib['Target'];type_=item.attrib['Type']
|
||
|
|
if not id_ or id_ in ids:raise OfficeCoverageError('office-relationship-id-invalid')
|
||
|
|
ids.add(id_)
|
||
|
|
if item.attrib.get('TargetMode','Internal')!='Internal':reasons.add('external-relationship-unsupported');continue
|
||
|
|
if urlsplit(target).scheme or '%' in target or '\\' in target or '#' in target or '?' in target:raise OfficeCoverageError('office-relationship-target-invalid')
|
||
|
|
resolved=posixpath.normpath(posixpath.join(posixpath.dirname(owner),target)) if not target.startswith('/') else target[1:]
|
||
|
|
_part(resolved)
|
||
|
|
if resolved.casefold() not in members:raise OfficeCoverageError('office-relationship-target-missing')
|
||
|
|
if type_ not in SAFE_REL:reasons.add('relationship-type-unsupported')
|
||
|
|
else:
|
||
|
|
target_type=types.get(resolved)
|
||
|
|
role=type_.rsplit('/',1)[-1]
|
||
|
|
if role=='officeDocument':valid=target_type in MAIN
|
||
|
|
elif role in {'image','thumbnail'}:valid=resolved in images
|
||
|
|
elif role=='customXml':valid=target_type=='application/xml' and resolved in xml and xml[resolved].tag=='{http://schemas.openxmlformats.org/officeDocument/2006/bibliography}Sources'
|
||
|
|
elif role=='customXmlProps':valid=target_type==TYPE_PREFIX+'customXmlProperties+xml'
|
||
|
|
elif role=='stylesWithEffects':valid=target_type=='application/vnd.ms-word.stylesWithEffects+xml'
|
||
|
|
elif role=='core-properties':valid=target_type=='application/vnd.openxmlformats-package.core-properties+xml'
|
||
|
|
elif role in {'extended-properties','custom-properties'}:valid=target_type==TYPE_PREFIX+role+'+xml'
|
||
|
|
else:valid=target_type in XML_TYPES and (target_type==TYPE_PREFIX+'theme+xml' if role=='theme' else target_type.endswith('.'+role+'+xml'))
|
||
|
|
if not valid:raise OfficeCoverageError('office-relationship-type-target-mismatch')
|
||
|
|
if type_==R+'/officeDocument' and not owner:top_main.append(resolved)
|
||
|
|
relationships[owner]=ids
|
||
|
|
if top_main!=[main[0][0]]:raise OfficeCoverageError('office-root-relationship-invalid')
|
||
|
|
for name,root in xml.items():
|
||
|
|
if name.endswith('.rels') or name=='[Content_Types].xml':continue
|
||
|
|
for element in root.iter():
|
||
|
|
for key,value in element.attrib.items():
|
||
|
|
if key.startswith('{'+R+'}') and value not in relationships.get(name,set()):raise OfficeCoverageError('office-dangling-relationship-reference')
|
||
|
|
with stage('office.ocr'):
|
||
|
|
for name,payload in images.items():
|
||
|
|
if len(media)>=max_media:raise OfficeCoverageError('office-media-limit')
|
||
|
|
scanned,complete=ocr_image(payload)
|
||
|
|
if not isinstance(scanned,str) or complete is not True:reasons.add('image-ocr-incomplete')
|
||
|
|
else:append(scanned)
|
||
|
|
media.append(('image',payload))
|
||
|
|
return {'text':'\n'.join(texts),'complete':not reasons,'media':media,'family':main[0][1],'reasons':sorted(reasons),'parts':len(members)}
|