docintel-platform 1.0.2__py3-none-any.whl

docintel/services/pdf/presets.py

@@ -0,0 +1,26 @@
+"""Default Presidio entity presets (extend via API or custom recognizers)."""
+
+# Core Presidio entities suitable for HR, legal, and compliance workflows.
+DEFAULT_PII_ENTITIES: tuple[str, ...] = (
+    "EMAIL_ADDRESS",
+    "PHONE_NUMBER",
+    "US_SSN",
+    "CREDIT_CARD",
+    "US_BANK_NUMBER",
+    "US_DRIVER_LICENSE",
+    "US_ITIN",
+    "US_PASSPORT",
+    "PERSON",
+    "LOCATION",
+    "DATE_TIME",
+    "IP_ADDRESS",
+    "IBAN_CODE",
+    "MEDICAL_LICENSE",
+    "URL",
+)
+
+# Minimum extracted characters before a page is treated as scanned (OCR fallback).
+MIN_NATIVE_TEXT_CHARS = 20
+
+# EasyOCR render scale (higher improves accuracy, increases memory).
+OCR_RENDER_SCALE = 2.0

docintel/services/pdf/search.py

@@ -0,0 +1,29 @@
+"""Text search helpers for PDF pages."""
+
+from __future__ import annotations
+
+import re
+from typing import Iterable
+
+import fitz
+
+
+def search_for_text(lines: Iterable[str], pattern: str) -> Iterable[str]:
+    """Yield regex matches from each line (case insensitive)."""
+    for line in lines:
+        for result in re.findall(pattern, line, re.IGNORECASE):
+            yield result
+
+
+def extract_info(input_file: str) -> dict:
+    """Return basic metadata for a PDF file."""
+    pdf_doc = fitz.open(input_file)
+    info = {
+        "file": input_file,
+        "page_count": pdf_doc.page_count,
+        "encrypted": pdf_doc.is_encrypted,
+    }
+    if not pdf_doc.is_encrypted:
+        info.update({key: value for key, value in pdf_doc.metadata.items() if value})
+    pdf_doc.close()
+    return info

docintel/services/pdf/sensitive.py

@@ -0,0 +1,212 @@
+"""Scanned and native PDF sensitive-data detection with OCR + Presidio."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Protocol, Sequence
+
+import fitz
+
+from docintel.services.pdf.annotator import _open_pdf, _save_pdf, highlight_matches, redact_matches
+from docintel.services.pdf.models import Action, PIIDetectionResult
+from docintel.services.pdf.ocr import (
+    build_indexed_text,
+    embed_invisible_text_layer,
+    extract_page_ocr,
+    merge_rects,
+    page_has_native_text,
+    rects_for_char_range,
+)
+from docintel.services.pdf.pii import PIIHit, detect_pii_in_text
+from docintel.services.pdf.search import search_for_text
+
+
+class ProgressCallback(Protocol):
+    def __call__(
+        self,
+        *,
+        stage: str,
+        pages_done: int,
+        pages_total: int,
+        message: str,
+    ) -> None: ...
+
+
+def _ensure_ocr_stack() -> None:
+    try:
+        import easyocr  # noqa: F401
+        import presidio_analyzer  # noqa: F401
+    except ImportError as exc:
+        raise RuntimeError(
+            "OCR and Presidio dependencies are not installed. "
+            "Run: pip install -e '.[ocr]' && python -m spacy download en_core_web_sm"
+        ) from exc
+
+
+def _apply_rect_action(page: fitz.Page, rect: fitz.Rect, action: Action) -> bool:
+    if action == Action.REDACT:
+        page.add_redact_annot(rect, text=" ", fill=(0, 0, 0))
+        return True
+    if action == Action.FRAME:
+        annot = page.add_rect_annot(rect)
+        annot.set_colors(stroke=fitz.utils.getColor("red"))
+        annot.update()
+        return True
+    if action == Action.UNDERLINE:
+        annot = page.add_underline_annot([rect])
+        annot.update()
+        return True
+    if action == Action.SQUIGGLY:
+        annot = page.add_squiggly_annot([rect])
+        annot.update()
+        return True
+    if action == Action.STRIKEOUT:
+        annot = page.add_strikeout_annot([rect])
+        annot.update()
+        return True
+
+    annot = page.add_highlight_annot([rect])
+    annot.update()
+    return True
+
+
+def _annotate_rects(page: fitz.Page, rects: list[fitz.Rect], action: Action) -> int:
+    applied = 0
+    for rect in rects:
+        if _apply_rect_action(page, rect, action):
+            applied += 1
+    if action == Action.REDACT and applied:
+        page.apply_redactions()
+    return applied
+
+
+def _native_rects_for_hit(page: fitz.Page, hit: PIIHit) -> list[fitz.Rect]:
+    return page.search_for(hit.text)
+
+
+def _ocr_rects_for_hit(hit: PIIHit, indexed) -> list[fitz.Rect]:
+    rects = rects_for_char_range(hit.start, hit.end, indexed)
+    merged = merge_rects(rects)
+    return [merged] if merged else []
+
+
+def _regex_hits(page_text: str, pattern: str) -> list[PIIHit]:
+    hits: list[PIIHit] = []
+    for match in re.finditer(pattern, page_text, flags=re.IGNORECASE):
+        hits.append(
+            PIIHit(
+                entity_type="REGEX",
+                text=match.group(0),
+                start=match.start(),
+                end=match.end(),
+                score=1.0,
+            )
+        )
+    return hits
+
+
+def detect_sensitive_pdf(
+    input_file: str | Path,
+    output_file: str | Path,
+    *,
+    entities: Sequence[str] | None = None,
+    action: Action | str = Action.HIGHLIGHT,
+    force_ocr: bool = False,
+    add_text_layer: bool = True,
+    pattern: str | None = None,
+    min_score: float = 0.35,
+    password: str | None = None,
+    progress_callback: ProgressCallback | None = None,
+) -> PIIDetectionResult:
+    """
+    Detect sensitive information with Presidio (and optional regex), annotate PDF.
+
+    Uses native PDF text when available. Falls back to EasyOCR for scanned pages.
+    """
+    _ensure_ocr_stack()
+    selected_action = action if isinstance(action, Action) else Action.from_value(action)
+    if selected_action == Action.REMOVE:
+        raise ValueError("Action 'Remove' is not supported for sensitive detection.")
+
+    pdf_doc = _open_pdf(input_file, password)
+    total_annotations = 0
+    pages_processed = 0
+    ocr_pages: list[int] = []
+    findings: list[dict] = []
+
+    total_pages = pdf_doc.page_count
+    for page_index in range(total_pages):
+        if progress_callback is not None:
+            progress_callback(
+                stage="detecting",
+                pages_done=page_index,
+                pages_total=total_pages,
+                message=f"Processing page {page_index + 1} of {total_pages}",
+            )
+        page = pdf_doc[page_index]
+        pages_processed += 1
+        use_ocr = force_ocr or not page_has_native_text(page)
+
+        if use_ocr:
+            ocr_pages.append(page_index)
+            ocr_spans = extract_page_ocr(page)
+            if add_text_layer and ocr_spans:
+                embed_invisible_text_layer(page, ocr_spans)
+            page_text, indexed = build_indexed_text(ocr_spans)
+        else:
+            ocr_spans = []
+            indexed = []
+            page_text = page.get_text("text")
+
+        hits = detect_pii_in_text(page_text, entities=entities, min_score=min_score)
+        if pattern:
+            hits.extend(_regex_hits(page_text, pattern))
+
+        for hit in hits:
+            if use_ocr:
+                rects = _ocr_rects_for_hit(hit, indexed)
+            else:
+                rects = _native_rects_for_hit(page, hit)
+
+            if not rects:
+                continue
+
+            total_annotations += _annotate_rects(page, rects, selected_action)
+            findings.append(
+                {
+                    "page": page_index,
+                    "entity_type": hit.entity_type,
+                    "text": hit.text,
+                    "score": round(hit.score, 4),
+                    "ocr_used": use_ocr,
+                }
+            )
+
+        # Legacy regex path for native PDFs when pattern matches line fragments
+        if pattern and not use_ocr:
+            matched_values = list(search_for_text(page_text.split("\n"), pattern))
+            if matched_values:
+                if selected_action == Action.REDACT:
+                    total_annotations += redact_matches(page, matched_values)
+                else:
+                    total_annotations += highlight_matches(page, matched_values, selected_action)
+
+    if progress_callback is not None:
+        progress_callback(
+            stage="detecting",
+            pages_done=total_pages,
+            pages_total=total_pages,
+            message="Detection complete",
+        )
+
+    _save_pdf(pdf_doc, Path(output_file))
+    return PIIDetectionResult(
+        input_path=str(input_file),
+        output_path=str(output_file),
+        action=selected_action,
+        matches=total_annotations,
+        pages_processed=pages_processed,
+        ocr_pages=ocr_pages,
+        findings=findings,
+    )

docintel/services/pdf/structure.py

@@ -0,0 +1,118 @@
+"""LLM-backed PDF structuring: unstructured scan to curated digital PDF."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable, Protocol
+
+import fitz
+
+from docintel.services.pdf.annotator import _open_pdf
+from docintel.services.pdf.models import StructureMode, StructureResult
+from docintel.services.pdf.ocr import build_indexed_text, extract_page_ocr, page_has_native_text
+from docintel.services.pdf.structure_llm import structure_document
+from docintel.services.pdf.structure_render import render_curated_pdf, render_searchable_pdf
+from docintel.services.pdf.structure_schema import StructuredDocument
+
+
+class ProgressCallback(Protocol):
+    def __call__(
+        self,
+        *,
+        stage: str,
+        pages_done: int,
+        pages_total: int,
+        message: str,
+    ) -> None: ...
+
+
+def _ensure_ocr_stack() -> None:
+    try:
+        import easyocr  # noqa: F401
+    except ImportError as exc:
+        raise RuntimeError(
+            "OCR dependencies are not installed. Run: pip install -e '.[ocr]'"
+        ) from exc
+
+
+def _extract_page_text(page: fitz.Page, force_ocr: bool) -> tuple[str, bool]:
+    use_ocr = force_ocr or not page_has_native_text(page)
+    if use_ocr:
+        _ensure_ocr_stack()
+        spans = extract_page_ocr(page)
+        text, _ = build_indexed_text(spans)
+        return text, True
+    return page.get_text("text"), False
+
+
+def structure_pdf(
+    input_file: str | Path,
+    output_file: str | Path,
+    *,
+    mode: StructureMode | str = StructureMode.CURATE,
+    force_ocr: bool = False,
+    redact_before_llm: bool = False,
+    structure_fn: Callable[[list[tuple[int, str]]], StructuredDocument] | None = None,
+    progress_callback: ProgressCallback | None = None,
+    password: str | None = None,
+) -> StructureResult:
+    """
+    Convert an unstructured or scanned PDF into a curated structured PDF.
+
+    Uses EasyOCR when native text is missing, then an LLM to clean and structure
+    content before rendering a new PDF (curate) or embedding a searchable layer
+    (searchable).
+    """
+    selected_mode = mode if isinstance(mode, StructureMode) else StructureMode.from_value(mode)
+
+    pdf_doc = _open_pdf(input_file, password)
+    page_texts: list[tuple[int, str]] = []
+    ocr_pages: list[int] = []
+
+    total_pages = pdf_doc.page_count
+    for page_index in range(total_pages):
+        if progress_callback is not None:
+            progress_callback(
+                stage="extracting",
+                pages_done=page_index,
+                pages_total=total_pages,
+                message=f"Extracting page {page_index + 1} of {total_pages}",
+            )
+        text, used_ocr = _extract_page_text(pdf_doc[page_index], force_ocr=force_ocr)
+        if used_ocr:
+            ocr_pages.append(page_index)
+        if redact_before_llm and text.strip():
+            from docintel.services.pdf.pii import mask_pii_in_text
+
+            text, _ = mask_pii_in_text(text)
+        page_texts.append((page_index, text))
+
+    if progress_callback is not None:
+        progress_callback(
+            stage="extracting",
+            pages_done=total_pages,
+            pages_total=total_pages,
+            message="Extraction complete",
+        )
+
+    if structure_fn is not None:
+        document = structure_fn(page_texts)
+    else:
+        document = structure_document(page_texts, progress_callback=progress_callback)
+
+    output_path = Path(output_file)
+    pages_processed = pdf_doc.page_count
+    if selected_mode == StructureMode.SEARCHABLE:
+        render_searchable_pdf(pdf_doc, document.pages, output_path)
+    else:
+        pdf_doc.close()
+        render_curated_pdf(document, output_path)
+
+    return StructureResult(
+        input_path=str(input_file),
+        output_path=str(output_path),
+        mode=selected_mode,
+        pages_processed=pages_processed,
+        ocr_pages=ocr_pages,
+        document_title=document.title,
+    )

docintel/services/pdf/structure_llm.py

@@ -0,0 +1,136 @@
+"""LLM structuring for OCR and native PDF text."""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+from docintel.services.pdf.structure_schema import StructuredDocument, StructuredPage
+
+STRUCTURE_SYSTEM_PROMPT = """You convert noisy OCR or unstructured PDF text into clean structured JSON.
+Rules:
+- Fix OCR typos only when the intended word is obvious from context.
+- Do not invent facts, numbers, names, or clauses that are not in the source.
+- Preserve reading order.
+- Use headings only when the source clearly has a section title.
+- Return valid JSON only, matching the schema exactly."""
+
+STRUCTURE_USER_TEMPLATE = """Page number: {page_number}
+Source text:
+---
+{source_text}
+---
+
+Return JSON with this schema:
+{{
+  "page_title": "document or section title if visible on this page, else empty string",
+  "plain_text": "full cleaned page text in reading order with paragraph breaks",
+  "sections": [
+    {{
+      "heading": "section heading or empty string",
+      "level": 1,
+      "paragraphs": ["paragraph text"],
+      "list_items": ["bullet items"],
+      "tables": [{{"headers": ["col"], "rows": [["value"]]}}]
+    }}
+  ]
+}}"""
+
+
+def _ensure_llm_stack() -> None:
+    try:
+        import openai  # noqa: F401
+    except ImportError as exc:
+        raise RuntimeError(
+            "LLM dependencies are not installed. Run: pip install -e '.[llm]'"
+        ) from exc
+
+
+def _llm_settings() -> tuple[str, str, str | None]:
+    api_key = os.getenv("DOCINTEL_LLM_API_KEY", "").strip()
+    if not api_key:
+        raise RuntimeError(
+            "DOCINTEL_LLM_API_KEY is not set. Configure an OpenAI-compatible API key."
+        )
+    model = os.getenv("DOCINTEL_LLM_MODEL", "gpt-4o-mini").strip()
+    base_url = os.getenv("DOCINTEL_LLM_BASE_URL", "").strip() or None
+    return api_key, model, base_url
+
+
+def _parse_json_response(raw: str) -> dict[str, Any]:
+    cleaned = raw.strip()
+    if cleaned.startswith("```"):
+        lines = cleaned.splitlines()
+        if lines and lines[0].startswith("```"):
+            lines = lines[1:]
+        if lines and lines[-1].strip() == "```":
+            lines = lines[:-1]
+        cleaned = "\n".join(lines).strip()
+    payload = json.loads(cleaned)
+    if not isinstance(payload, dict):
+        raise ValueError("LLM response must be a JSON object.")
+    return payload
+
+
+def structure_page_text(page_index: int, source_text: str) -> StructuredPage:
+    """Send one page of source text to the LLM and return structured output."""
+    _ensure_llm_stack()
+    api_key, model, base_url = _llm_settings()
+
+    from openai import OpenAI
+
+    client_kwargs: dict[str, Any] = {"api_key": api_key}
+    if base_url:
+        client_kwargs["base_url"] = base_url
+    client = OpenAI(**client_kwargs)
+
+    user_prompt = STRUCTURE_USER_TEMPLATE.format(
+        page_number=page_index + 1,
+        source_text=source_text[:12000],
+    )
+    response = client.chat.completions.create(
+        model=model,
+        temperature=0.1,
+        response_format={"type": "json_object"},
+        messages=[
+            {"role": "system", "content": STRUCTURE_SYSTEM_PROMPT},
+            {"role": "user", "content": user_prompt},
+        ],
+    )
+    content = response.choices[0].message.content or "{}"
+    payload = _parse_json_response(content)
+    return StructuredPage.from_llm_payload(page_index, payload)
+
+
+def structure_document(
+    page_texts: list[tuple[int, str]],
+    *,
+    progress_callback=None,
+) -> StructuredDocument:
+    """Structure each page with the LLM and merge into one document model."""
+    structured_pages: list[StructuredPage] = []
+    total = len(page_texts)
+    for offset, (page_index, text) in enumerate(page_texts):
+        cleaned = text.strip()
+        if progress_callback is not None:
+            progress_callback(
+                stage="structuring",
+                pages_done=offset,
+                pages_total=total,
+                message=f"Structuring page {page_index + 1} of {total}",
+            )
+        if not cleaned:
+            structured_pages.append(
+                StructuredPage(page_index=page_index, title="", sections=[], plain_text="")
+            )
+            continue
+        structured_pages.append(structure_page_text(page_index, cleaned))
+    if progress_callback is not None:
+        progress_callback(
+            stage="structuring",
+            pages_done=total,
+            pages_total=total,
+            message="Structuring complete",
+        )
+    return StructuredDocument.from_pages(structured_pages)

docintel/services/pdf/structure_render.py

@@ -0,0 +1,136 @@
+"""Render structured documents into curated or searchable PDFs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import fitz
+
+from docintel.services.pdf.annotator import _save_pdf
+from docintel.services.pdf.structure_schema import SectionBlock, StructuredDocument, StructuredPage
+
+
+@dataclass
+class _RenderContext:
+    pdf: fitz.Document
+    page: fitz.Page
+    margin: float
+    y: float
+
+
+def _new_page(ctx: _RenderContext) -> None:
+    ctx.page = ctx.pdf.new_page()
+    ctx.y = ctx.margin
+
+
+def _ensure_space(ctx: _RenderContext, height: float) -> None:
+    if ctx.y + height > ctx.page.rect.height - ctx.margin:
+        _new_page(ctx)
+
+
+def _write_line(ctx: _RenderContext, text: str, fontsize: float, indent: float = 0) -> None:
+    if not text.strip():
+        return
+    line_height = fontsize * 1.45
+    _ensure_space(ctx, line_height)
+    ctx.page.insert_text(
+        (ctx.margin + indent, ctx.y),
+        text,
+        fontsize=fontsize,
+        fontname="helv",
+    )
+    ctx.y += line_height
+
+
+def _write_wrapped_paragraph(ctx: _RenderContext, text: str, fontsize: float = 11, indent: float = 0) -> None:
+    if not text.strip():
+        return
+    usable_width = ctx.page.rect.width - (2 * ctx.margin) - indent
+    approx_chars = max(40, int(usable_width / (fontsize * 0.55)))
+    words = text.split()
+    line: list[str] = []
+    line_len = 0
+    for word in words:
+        extra = len(word) + (1 if line else 0)
+        if line and line_len + extra > approx_chars:
+            _write_line(ctx, " ".join(line), fontsize, indent=indent)
+            line = [word]
+            line_len = len(word)
+        else:
+            line.append(word)
+            line_len += extra
+    if line:
+        _write_line(ctx, " ".join(line), fontsize, indent=indent)
+    ctx.y += 4
+
+
+def _write_section(ctx: _RenderContext, section: SectionBlock) -> None:
+    if section.heading:
+        heading_size = max(12, 18 - section.level)
+        _write_line(ctx, section.heading, heading_size)
+        ctx.y += 4
+    for paragraph in section.paragraphs:
+        _write_wrapped_paragraph(ctx, paragraph)
+    for item in section.list_items:
+        _write_wrapped_paragraph(ctx, f"- {item}", indent=12)
+    for table in section.tables:
+        if table.headers:
+            _write_line(ctx, " | ".join(table.headers), 10)
+        for row in table.rows:
+            _write_line(ctx, " | ".join(row), 10)
+        ctx.y += 6
+
+
+def render_curated_pdf(document: StructuredDocument, output_path: Path) -> None:
+    """Build a new typeset PDF from structured content."""
+    pdf = fitz.open()
+    page = pdf.new_page()
+    ctx = _RenderContext(pdf=pdf, page=page, margin=72, y=72)
+
+    _write_line(ctx, document.title, 18)
+    ctx.y += 10
+
+    for structured_page in document.pages:
+        for section in structured_page.sections:
+            _write_section(ctx, section)
+        if structured_page.sections:
+            continue
+        if structured_page.plain_text:
+            for line in structured_page.plain_text.splitlines():
+                _write_wrapped_paragraph(ctx, line)
+
+    _save_pdf(pdf, output_path)
+
+
+def render_searchable_pdf(
+    source_doc: fitz.Document,
+    pages: list[StructuredPage],
+    output_path: Path,
+) -> None:
+    """Keep original page layout and embed an invisible curated text layer."""
+    for structured_page in pages:
+        if structured_page.page_index >= source_doc.page_count:
+            continue
+        page = source_doc[structured_page.page_index]
+        text = structured_page.plain_text.strip()
+        if not text:
+            continue
+        y = 72
+        line_height = 13
+        for line in text.splitlines():
+            cleaned = line.strip()
+            if not cleaned:
+                continue
+            if y > page.rect.height - 72:
+                break
+            page.insert_text(
+                (72, y),
+                cleaned,
+                fontsize=10,
+                fontname="helv",
+                render_mode=3,
+            )
+            y += line_height
+
+    _save_pdf(source_doc, output_path)