187 lines
10 KiB
Python
187 lines
10 KiB
Python
"""Bounded static PDF inspection: decoded content plus every-page rendering/OCR.
|
|
|
|
Active, layered, annotated, embedded-file and incremental documents are outside
|
|
the supported subset and fail closed. This runs inside the existing isolated
|
|
parser process; PDFium has no JavaScript/XFA support and forms are never enabled.
|
|
"""
|
|
import io
|
|
import math
|
|
import time
|
|
from stage_timings import stage
|
|
|
|
MAX_PAGES = 32
|
|
MAX_OBJECTS = 10000
|
|
MAX_PIXELS = 50_000_000
|
|
MAX_PAGE_PIXELS = 20_000_000
|
|
MAX_BYTES = 20 * 1024 * 1024
|
|
MAX_TEXT = 1_000_000
|
|
FORBIDDEN = frozenset(("/Annots", "/AcroForm", "/XFA", "/OpenAction", "/AA", "/A", "/JS", "/JavaScript", "/Names", "/EmbeddedFiles", "/EF", "/AF", "/OCProperties", "/Collection", "/RichMedia", "/Movie", "/Sound", "/3D", "/Rendition", "/Launch", "/URI", "/GoToR", "/GoToE", "/SubmitForm", "/ImportData", "/Encrypt", "/PieceInfo"))
|
|
STANDARD_FONTS = frozenset(("/Helvetica", "/Helvetica-Bold", "/Helvetica-Oblique", "/Helvetica-BoldOblique", "/Times-Roman", "/Times-Bold", "/Times-Italic", "/Times-BoldItalic", "/Courier", "/Courier-Bold", "/Courier-Oblique", "/Courier-BoldOblique", "/Symbol", "/ZapfDingbats"))
|
|
|
|
|
|
def extract_pdf(data, ocr_image):
|
|
from pypdf import apply_configuration
|
|
# Enforce decompressor limits BEFORE any stream get_data()/image allocation.
|
|
with apply_configuration(maximum_declared_stream_length=MAX_BYTES,
|
|
array_based_stream_maximum_output_length=MAX_BYTES,
|
|
zlib_maximum_output_length=MAX_BYTES,
|
|
lzw_maximum_output_length=MAX_BYTES,
|
|
run_length_maximum_output_length=MAX_BYTES,
|
|
brotli_maximum_output_length=MAX_BYTES,
|
|
jbig2_maximum_output_length=MAX_BYTES,
|
|
image_maximum_buffer_size=MAX_BYTES,
|
|
stream_filters_maximum_length=4,
|
|
stream_decoding_work_maximum_length=MAX_BYTES,
|
|
zlib_maximum_recovery_input_length=1048576):
|
|
return _extract_pdf(data, ocr_image)
|
|
|
|
|
|
def _extract_pdf(data, ocr_image):
|
|
from pypdf import PdfReader
|
|
from pypdf.generic import IndirectObject, DictionaryObject, ArrayObject, StreamObject, TextStringObject, ByteStringObject, NameObject, ContentStream
|
|
import pypdfium2 as pdfium
|
|
if len(data) > MAX_BYTES or data.count(b"%%EOF") != 1 or data.rsplit(b"%%EOF", 1)[-1].strip():
|
|
raise ValueError("Unsupported PDF container")
|
|
if pdfium.PDFIUM_INFO.flags or pdfium.PYPDFIUM_INFO.api_tag != (5, 14, 0):
|
|
raise ValueError("Unverified PDF renderer")
|
|
deadline = time.monotonic() + 20
|
|
def budget():
|
|
if time.monotonic() >= deadline:
|
|
raise ValueError("PDF processing deadline")
|
|
text, media, seen, indirect = [], [], {}, set()
|
|
extracted_bytes, characters = 0, 0
|
|
allowed_streams = set()
|
|
def append(value):
|
|
nonlocal characters
|
|
characters += len(value)
|
|
text.append(value)
|
|
if characters > MAX_TEXT:
|
|
raise ValueError("PDF text budget exceeded")
|
|
def walk(value, depth=0):
|
|
nonlocal extracted_bytes
|
|
budget()
|
|
if depth > 40 or len(seen) + len(indirect) > MAX_OBJECTS:
|
|
raise ValueError("PDF object budget exceeded")
|
|
if isinstance(value, IndirectObject):
|
|
identity = (value.idnum, value.generation)
|
|
if identity in indirect: return
|
|
indirect.add(identity)
|
|
walk(value.get_object(), depth + 1)
|
|
return
|
|
if isinstance(value, (DictionaryObject, ArrayObject)):
|
|
if id(value) in seen: return
|
|
# Retain objects as well as IDs: parsed operand containers could
|
|
# otherwise be collected and their IDs reused across streams.
|
|
seen[id(value)] = value
|
|
if isinstance(value, DictionaryObject):
|
|
if any(str(key) in FORBIDDEN for key in value) or str(value.get("/Type", "")) in ("/Filespec", "/Action", "/ObjStm", "/EmbeddedFile", "/XRef"):
|
|
raise ValueError("Unsupported active PDF content")
|
|
if value.get("/Type") == "/Font" and (value.get("/Subtype") != "/Type1" or value.get("/BaseFont") not in STANDARD_FONTS or "/FontDescriptor" in value or "/ToUnicode" in value or value.get("/Encoding") not in (None, "/WinAnsiEncoding", "/MacRomanEncoding", "/StandardEncoding")):
|
|
raise ValueError("Unsupported PDF font encoding")
|
|
if isinstance(value, StreamObject):
|
|
if id(value) not in allowed_streams:
|
|
raise ValueError("Unsupported hidden PDF stream")
|
|
decoded = value.get_data()
|
|
extracted_bytes += len(decoded)
|
|
if extracted_bytes > MAX_BYTES:
|
|
raise ValueError("PDF expansion budget exceeded")
|
|
try: append(decoded.decode("utf-8", errors="strict"))
|
|
except UnicodeError: pass
|
|
if value.get("/Subtype") != "/Image":
|
|
operations = ContentStream(value, reader).operations
|
|
if len(operations) > MAX_OBJECTS: raise ValueError("PDF operation budget exceeded")
|
|
for operands, _operator in operations:
|
|
for operand in operands: walk(operand, depth + 1)
|
|
for key, item in value.items():
|
|
append(str(key))
|
|
walk(item, depth + 1)
|
|
elif isinstance(value, ArrayObject):
|
|
for item in value: walk(item, depth + 1)
|
|
elif isinstance(value, (TextStringObject, NameObject)): append(str(value))
|
|
elif isinstance(value, ByteStringObject):
|
|
try: append(bytes(value).decode("utf-8", errors="strict"))
|
|
except UnicodeError: raise ValueError("Unsupported PDF string encoding")
|
|
with stage("pdf.structure"):
|
|
reader = PdfReader(io.BytesIO(data), strict=True)
|
|
if reader.is_encrypted or reader.trailer.get("/Prev") or not 0 < len(reader.pages) <= MAX_PAGES:
|
|
raise ValueError("Unsupported PDF document")
|
|
# Only page content and page-reachable image resources may carry streams.
|
|
# Fonts/alternate encodings, hidden orphan streams and XMP streams remain
|
|
# outside this deliberately explicit static PDF subset.
|
|
def allow_resource(value, depth=0, visited=None):
|
|
if visited is None: visited = set()
|
|
budget()
|
|
if depth > 40 or len(visited) > MAX_OBJECTS: raise ValueError("PDF resource budget exceeded")
|
|
if isinstance(value, IndirectObject): value = value.get_object()
|
|
if isinstance(value, (DictionaryObject, ArrayObject)):
|
|
if id(value) in visited: return
|
|
visited.add(id(value))
|
|
if isinstance(value, StreamObject):
|
|
if value.get("/Subtype") not in ("/Image", "/Form"):
|
|
raise ValueError("Unsupported PDF resource stream")
|
|
allowed_streams.add(id(value))
|
|
if isinstance(value, DictionaryObject):
|
|
for item in value.values(): allow_resource(item, depth + 1, visited)
|
|
elif isinstance(value, ArrayObject):
|
|
for item in value: allow_resource(item, depth + 1, visited)
|
|
for page in reader.pages:
|
|
contents = page.get("/Contents")
|
|
if isinstance(contents, IndirectObject): contents = contents.get_object()
|
|
if isinstance(contents, ArrayObject):
|
|
for item in contents:
|
|
item = item.get_object()
|
|
if not isinstance(item, StreamObject): raise ValueError("Unsupported PDF page content")
|
|
allowed_streams.add(id(item))
|
|
elif isinstance(contents, StreamObject): allowed_streams.add(id(contents))
|
|
elif contents is not None: raise ValueError("Unsupported PDF page content")
|
|
if page.get("/Resources") is not None: allow_resource(page["/Resources"])
|
|
walk(reader.trailer)
|
|
# Inspect every indexed object, including unreferenced metadata/actions.
|
|
for generation, entries in reader.xref.items():
|
|
for number in entries:
|
|
if number: walk(IndirectObject(number, generation, reader))
|
|
for number in reader.xref_objStm:
|
|
walk(IndirectObject(number, 0, reader))
|
|
media_bytes, pixels = 0, 0
|
|
with pdfium.PdfDocument(data) as document:
|
|
if len(document) != len(reader.pages): raise ValueError("PDF parser disagreement")
|
|
for index, source in enumerate(reader.pages):
|
|
budget()
|
|
append(source.extract_text() or "")
|
|
# Hidden/unpainted image resources also require independent OCR.
|
|
with stage("pdf.images"):
|
|
for image in source.images:
|
|
if len(image.data) > MAX_BYTES: raise ValueError("PDF image budget exceeded")
|
|
media_bytes += len(image.data)
|
|
if media_bytes > MAX_BYTES or len(media) >= 64: raise ValueError("PDF media budget exceeded")
|
|
value, complete = ocr_image(image.data)
|
|
if not complete: raise ValueError("Incomplete PDF image OCR")
|
|
append(value)
|
|
media.append(("image", image.data))
|
|
page = document[index]
|
|
try:
|
|
width, height = page.get_size()
|
|
if not all(math.isfinite(value) and value > 0 for value in (width, height)):
|
|
raise ValueError("Invalid PDF page dimensions")
|
|
count = math.ceil(width * 2) * math.ceil(height * 2)
|
|
pixels += count
|
|
if count > MAX_PAGE_PIXELS or pixels > MAX_PIXELS:
|
|
raise ValueError("PDF rendering budget exceeded")
|
|
with stage("pdf.render"):
|
|
bitmap = page.render(scale=2, draw_annots=False)
|
|
try:
|
|
rendered = io.BytesIO()
|
|
bitmap.to_pil().save(rendered, format="PNG")
|
|
payload = rendered.getvalue()
|
|
finally: bitmap.close()
|
|
media_bytes += len(payload)
|
|
if media_bytes > MAX_BYTES or len(media) >= 64: raise ValueError("PDF rendered media budget exceeded")
|
|
with stage("pdf.ocr"):
|
|
value, complete = ocr_image(payload)
|
|
if not complete: raise ValueError("Incomplete PDF rendered OCR")
|
|
append(value)
|
|
media.append(("image", payload))
|
|
finally: page.close()
|
|
budget()
|
|
return "\n".join(text), True, media
|