Inspect static PDF and OOXML content with bounded local coverage
This commit is contained in:
Karl committed
1 parent
69d820e315
commit
9e7cb57093
14 files changed
+758
-51
No files matched your search
@@ -0,0 +1,186 @@
|
||||
"""Bounded static PDF inspection: decoded content plus every-page rendering/OCR.
|
||||
|
||||
Active, layered, annotated, embedded-file and incremental documents are outside
|
||||
the supported subset and fail closed. This runs inside the existing isolated
|
||||
parser process; PDFium has no JavaScript/XFA support and forms are never enabled.
|
||||
"""
|
||||
import io
|
||||
import math
|
||||
import time
|
||||
from stage_timings import stage
|
||||
|
||||
MAX_PAGES = 32
|
||||
MAX_OBJECTS = 10000
|
||||
MAX_PIXELS = 50_000_000
|
||||
MAX_PAGE_PIXELS = 20_000_000
|
||||
MAX_BYTES = 20 * 1024 * 1024
|
||||
MAX_TEXT = 1_000_000
|
||||
FORBIDDEN = frozenset(("/Annots", "/AcroForm", "/XFA", "/OpenAction", "/AA", "/A", "/JS", "/JavaScript", "/Names", "/EmbeddedFiles", "/EF", "/AF", "/OCProperties", "/Collection", "/RichMedia", "/Movie", "/Sound", "/3D", "/Rendition", "/Launch", "/URI", "/GoToR", "/GoToE", "/SubmitForm", "/ImportData", "/Encrypt", "/PieceInfo"))
|
||||
STANDARD_FONTS = frozenset(("/Helvetica", "/Helvetica-Bold", "/Helvetica-Oblique", "/Helvetica-BoldOblique", "/Times-Roman", "/Times-Bold", "/Times-Italic", "/Times-BoldItalic", "/Courier", "/Courier-Bold", "/Courier-Oblique", "/Courier-BoldOblique", "/Symbol", "/ZapfDingbats"))
|
||||
|
||||
|
||||
def extract_pdf(data, ocr_image):
|
||||
from pypdf import apply_configuration
|
||||
# Enforce decompressor limits BEFORE any stream get_data()/image allocation.
|
||||
with apply_configuration(maximum_declared_stream_length=MAX_BYTES,
|
||||
array_based_stream_maximum_output_length=MAX_BYTES,
|
||||
zlib_maximum_output_length=MAX_BYTES,
|
||||
lzw_maximum_output_length=MAX_BYTES,
|
||||
run_length_maximum_output_length=MAX_BYTES,
|
||||
brotli_maximum_output_length=MAX_BYTES,
|
||||
jbig2_maximum_output_length=MAX_BYTES,
|
||||
image_maximum_buffer_size=MAX_BYTES,
|
||||
stream_filters_maximum_length=4,
|
||||
stream_decoding_work_maximum_length=MAX_BYTES,
|
||||
zlib_maximum_recovery_input_length=1048576):
|
||||
return _extract_pdf(data, ocr_image)
|
||||
|
||||
|
||||
def _extract_pdf(data, ocr_image):
|
||||
from pypdf import PdfReader
|
||||
from pypdf.generic import IndirectObject, DictionaryObject, ArrayObject, StreamObject, TextStringObject, ByteStringObject, NameObject, ContentStream
|
||||
import pypdfium2 as pdfium
|
||||
if len(data) > MAX_BYTES or data.count(b"%%EOF") != 1 or data.rsplit(b"%%EOF", 1)[-1].strip():
|
||||
raise ValueError("Unsupported PDF container")
|
||||
if pdfium.PDFIUM_INFO.flags or pdfium.PYPDFIUM_INFO.api_tag != (5, 14, 0):
|
||||
raise ValueError("Unverified PDF renderer")
|
||||
deadline = time.monotonic() + 20
|
||||
def budget():
|
||||
if time.monotonic() >= deadline:
|
||||
raise ValueError("PDF processing deadline")
|
||||
text, media, seen, indirect = [], [], {}, set()
|
||||
extracted_bytes, characters = 0, 0
|
||||
allowed_streams = set()
|
||||
def append(value):
|
||||
nonlocal characters
|
||||
characters += len(value)
|
||||
text.append(value)
|
||||
if characters > MAX_TEXT:
|
||||
raise ValueError("PDF text budget exceeded")
|
||||
def walk(value, depth=0):
|
||||
nonlocal extracted_bytes
|
||||
budget()
|
||||
if depth > 40 or len(seen) + len(indirect) > MAX_OBJECTS:
|
||||
raise ValueError("PDF object budget exceeded")
|
||||
if isinstance(value, IndirectObject):
|
||||
identity = (value.idnum, value.generation)
|
||||
if identity in indirect: return
|
||||
indirect.add(identity)
|
||||
walk(value.get_object(), depth + 1)
|
||||
return
|
||||
if isinstance(value, (DictionaryObject, ArrayObject)):
|
||||
if id(value) in seen: return
|
||||
# Retain objects as well as IDs: parsed operand containers could
|
||||
# otherwise be collected and their IDs reused across streams.
|
||||
seen[id(value)] = value
|
||||
if isinstance(value, DictionaryObject):
|
||||
if any(str(key) in FORBIDDEN for key in value) or str(value.get("/Type", "")) in ("/Filespec", "/Action", "/ObjStm", "/EmbeddedFile", "/XRef"):
|
||||
raise ValueError("Unsupported active PDF content")
|
||||
if value.get("/Type") == "/Font" and (value.get("/Subtype") != "/Type1" or value.get("/BaseFont") not in STANDARD_FONTS or "/FontDescriptor" in value or "/ToUnicode" in value or value.get("/Encoding") not in (None, "/WinAnsiEncoding", "/MacRomanEncoding", "/StandardEncoding")):
|
||||
raise ValueError("Unsupported PDF font encoding")
|
||||
if isinstance(value, StreamObject):
|
||||
if id(value) not in allowed_streams:
|
||||
raise ValueError("Unsupported hidden PDF stream")
|
||||
decoded = value.get_data()
|
||||
extracted_bytes += len(decoded)
|
||||
if extracted_bytes > MAX_BYTES:
|
||||
raise ValueError("PDF expansion budget exceeded")
|
||||
try: append(decoded.decode("utf-8", errors="strict"))
|
||||
except UnicodeError: pass
|
||||
if value.get("/Subtype") != "/Image":
|
||||
operations = ContentStream(value, reader).operations
|
||||
if len(operations) > MAX_OBJECTS: raise ValueError("PDF operation budget exceeded")
|
||||
for operands, _operator in operations:
|
||||
for operand in operands: walk(operand, depth + 1)
|
||||
for key, item in value.items():
|
||||
append(str(key))
|
||||
walk(item, depth + 1)
|
||||
elif isinstance(value, ArrayObject):
|
||||
for item in value: walk(item, depth + 1)
|
||||
elif isinstance(value, (TextStringObject, NameObject)): append(str(value))
|
||||
elif isinstance(value, ByteStringObject):
|
||||
try: append(bytes(value).decode("utf-8", errors="strict"))
|
||||
except UnicodeError: raise ValueError("Unsupported PDF string encoding")
|
||||
with stage("pdf.structure"):
|
||||
reader = PdfReader(io.BytesIO(data), strict=True)
|
||||
if reader.is_encrypted or reader.trailer.get("/Prev") or not 0 < len(reader.pages) <= MAX_PAGES:
|
||||
raise ValueError("Unsupported PDF document")
|
||||
# Only page content and page-reachable image resources may carry streams.
|
||||
# Fonts/alternate encodings, hidden orphan streams and XMP streams remain
|
||||
# outside this deliberately explicit static PDF subset.
|
||||
def allow_resource(value, depth=0, visited=None):
|
||||
if visited is None: visited = set()
|
||||
budget()
|
||||
if depth > 40 or len(visited) > MAX_OBJECTS: raise ValueError("PDF resource budget exceeded")
|
||||
if isinstance(value, IndirectObject): value = value.get_object()
|
||||
if isinstance(value, (DictionaryObject, ArrayObject)):
|
||||
if id(value) in visited: return
|
||||
visited.add(id(value))
|
||||
if isinstance(value, StreamObject):
|
||||
if value.get("/Subtype") not in ("/Image", "/Form"):
|
||||
raise ValueError("Unsupported PDF resource stream")
|
||||
allowed_streams.add(id(value))
|
||||
if isinstance(value, DictionaryObject):
|
||||
for item in value.values(): allow_resource(item, depth + 1, visited)
|
||||
elif isinstance(value, ArrayObject):
|
||||
for item in value: allow_resource(item, depth + 1, visited)
|
||||
for page in reader.pages:
|
||||
contents = page.get("/Contents")
|
||||
if isinstance(contents, IndirectObject): contents = contents.get_object()
|
||||
if isinstance(contents, ArrayObject):
|
||||
for item in contents:
|
||||
item = item.get_object()
|
||||
if not isinstance(item, StreamObject): raise ValueError("Unsupported PDF page content")
|
||||
allowed_streams.add(id(item))
|
||||
elif isinstance(contents, StreamObject): allowed_streams.add(id(contents))
|
||||
elif contents is not None: raise ValueError("Unsupported PDF page content")
|
||||
if page.get("/Resources") is not None: allow_resource(page["/Resources"])
|
||||
walk(reader.trailer)
|
||||
# Inspect every indexed object, including unreferenced metadata/actions.
|
||||
for generation, entries in reader.xref.items():
|
||||
for number in entries:
|
||||
if number: walk(IndirectObject(number, generation, reader))
|
||||
for number in reader.xref_objStm:
|
||||
walk(IndirectObject(number, 0, reader))
|
||||
media_bytes, pixels = 0, 0
|
||||
with pdfium.PdfDocument(data) as document:
|
||||
if len(document) != len(reader.pages): raise ValueError("PDF parser disagreement")
|
||||
for index, source in enumerate(reader.pages):
|
||||
budget()
|
||||
append(source.extract_text() or "")
|
||||
# Hidden/unpainted image resources also require independent OCR.
|
||||
with stage("pdf.images"):
|
||||
for image in source.images:
|
||||
if len(image.data) > MAX_BYTES: raise ValueError("PDF image budget exceeded")
|
||||
media_bytes += len(image.data)
|
||||
if media_bytes > MAX_BYTES or len(media) >= 64: raise ValueError("PDF media budget exceeded")
|
||||
value, complete = ocr_image(image.data)
|
||||
if not complete: raise ValueError("Incomplete PDF image OCR")
|
||||
append(value)
|
||||
media.append(("image", image.data))
|
||||
page = document[index]
|
||||
try:
|
||||
width, height = page.get_size()
|
||||
if not all(math.isfinite(value) and value > 0 for value in (width, height)):
|
||||
raise ValueError("Invalid PDF page dimensions")
|
||||
count = math.ceil(width * 2) * math.ceil(height * 2)
|
||||
pixels += count
|
||||
if count > MAX_PAGE_PIXELS or pixels > MAX_PIXELS:
|
||||
raise ValueError("PDF rendering budget exceeded")
|
||||
with stage("pdf.render"):
|
||||
bitmap = page.render(scale=2, draw_annots=False)
|
||||
try:
|
||||
rendered = io.BytesIO()
|
||||
bitmap.to_pil().save(rendered, format="PNG")
|
||||
payload = rendered.getvalue()
|
||||
finally: bitmap.close()
|
||||
media_bytes += len(payload)
|
||||
if media_bytes > MAX_BYTES or len(media) >= 64: raise ValueError("PDF rendered media budget exceeded")
|
||||
with stage("pdf.ocr"):
|
||||
value, complete = ocr_image(payload)
|
||||
if not complete: raise ValueError("Incomplete PDF rendered OCR")
|
||||
append(value)
|
||||
media.append(("image", payload))
|
||||
finally: page.close()
|
||||
budget()
|
||||
return "\n".join(text), True, media
|
||||
Reference in new issue
Block a user