docintel-platform 1.0.2__py3-none-any.whl

docintel/routes/text.py

@@ -0,0 +1,41 @@
+"""Text summarization API routes."""
+
+from flask import Blueprint, jsonify, request
+
+from docintel.auth.limiter import limiter
+from docintel.services.summary import summarize_text
+from docintel.services.summary.textrank import DEFAULT_SENTENCE_COUNT, MAX_SENTENCE_COUNT
+
+text_bp = Blueprint("text", __name__, url_prefix="/v1/text")
+
+
+@text_bp.post("/summarize")
+@limiter.limit("100 per hour")
+def summarize():
+    """Extractively summarize plain text using TextRank sentence ranking."""
+    payload = request.get_json(silent=True)
+    if not isinstance(payload, dict):
+        return jsonify({"error": "Request body must be JSON."}), 400
+
+    text = payload.get("text", "")
+    sentences = payload.get("sentences", DEFAULT_SENTENCE_COUNT)
+
+    if not isinstance(text, str):
+        return jsonify({"error": "Field 'text' must be a string."}), 400
+
+    try:
+        sentences = int(sentences)
+    except (TypeError, ValueError):
+        return jsonify({"error": "Field 'sentences' must be an integer."}), 400
+
+    if sentences < 1 or sentences > MAX_SENTENCE_COUNT:
+        return jsonify(
+            {"error": f"Field 'sentences' must be between 1 and {MAX_SENTENCE_COUNT}."}
+        ), 400
+
+    try:
+        result = summarize_text(text=text, sentence_count=sentences)
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 400
+
+    return jsonify({"status": "ok", **result.to_dict()}), 200

docintel/services/__init__.py

@@ -0,0 +1 @@
+"""Document processing services."""

docintel/services/matching/__init__.py

@@ -0,0 +1,6 @@
+"""Resume-to-job matching service."""
+
+from docintel.services.matching.models import MatchResult
+from docintel.services.matching.scorer import match_resume_to_job
+
+__all__ = ["MatchResult", "match_resume_to_job"]

docintel/services/matching/models.py

@@ -0,0 +1,19 @@
+"""Types for resume matching."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class MatchResult:
+    score: float
+    matched_keywords: list[str]
+    missing_keywords: list[str]
+
+    def to_dict(self) -> dict:
+        return {
+            "score": self.score,
+            "matched_keywords": self.matched_keywords,
+            "missing_keywords": self.missing_keywords,
+        }

docintel/services/matching/scorer.py

@@ -0,0 +1,64 @@
+"""TF-IDF resume matching engine."""
+
+from __future__ import annotations
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+from docintel.services.matching.models import MatchResult
+
+DEFAULT_TOP_KEYWORDS = 25
+
+
+def _clean_text(text: str) -> str:
+    return " ".join(text.strip().split())
+
+
+def match_resume_to_job(
+    resume: str,
+    job_description: str,
+    *,
+    top_keywords: int = DEFAULT_TOP_KEYWORDS,
+) -> MatchResult:
+    """Score resume fit against a job description using TF-IDF cosine similarity."""
+    resume_text = _clean_text(resume)
+    job_text = _clean_text(job_description)
+
+    if not resume_text:
+        raise ValueError("Resume text is required.")
+    if not job_text:
+        raise ValueError("Job description text is required.")
+
+    vectorizer = TfidfVectorizer(
+        stop_words="english",
+        token_pattern=r"(?u)\b[a-zA-Z][a-zA-Z0-9+#.]+\b",
+    )
+    matrix = vectorizer.fit_transform([resume_text, job_text])
+    similarity = cosine_similarity(matrix[0:1], matrix[1:2])[0][0]
+    score = round(float(similarity) * 100, 2)
+
+    features = vectorizer.get_feature_names_out()
+    resume_weights = matrix[0].toarray()[0]
+    job_weights = matrix[1].toarray()[0]
+
+    matched: list[tuple[str, float]] = []
+    missing: list[tuple[str, float]] = []
+
+    for index, term in enumerate(features):
+        job_weight = job_weights[index]
+        if job_weight <= 0:
+            continue
+        if resume_weights[index] > 0:
+            matched.append((term, job_weight))
+        else:
+            missing.append((term, job_weight))
+
+    matched.sort(key=lambda item: item[1], reverse=True)
+    missing.sort(key=lambda item: item[1], reverse=True)
+
+    limit = max(1, top_keywords)
+    return MatchResult(
+        score=score,
+        matched_keywords=[term for term, _ in matched[:limit]],
+        missing_keywords=[term for term, _ in missing[:limit]],
+    )

docintel/services/pdf/__init__.py

@@ -0,0 +1,26 @@
+"""PDF search and annotation service."""
+
+from docintel.services.pdf.annotator import PDFAnnotator, annotate_pdf
+from docintel.services.pdf.models import Action, PIIDetectionResult, ProcessResult, StructureMode, StructureResult
+from docintel.services.pdf.pii import detect_pii_in_text, list_supported_entities
+from docintel.services.pdf.presets import DEFAULT_PII_ENTITIES
+from docintel.services.pdf.search import extract_info, search_for_text
+from docintel.services.pdf.sensitive import detect_sensitive_pdf
+from docintel.services.pdf.structure import structure_pdf
+
+__all__ = [
+    "Action",
+    "DEFAULT_PII_ENTITIES",
+    "PDFAnnotator",
+    "PIIDetectionResult",
+    "ProcessResult",
+    "StructureMode",
+    "StructureResult",
+    "annotate_pdf",
+    "detect_pii_in_text",
+    "detect_sensitive_pdf",
+    "extract_info",
+    "list_supported_entities",
+    "search_for_text",
+    "structure_pdf",
+]

docintel/services/pdf/annotator.py

@@ -0,0 +1,188 @@
+"""Core PDF annotation engine."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable, Sequence
+
+import fitz
+
+from docintel.services.pdf.models import Action, ProcessResult
+from docintel.services.pdf.search import search_for_text
+
+
+def _normalize_pages(pages: Sequence[int | str] | None) -> list[int] | None:
+    if pages is None:
+        return None
+    return [int(page) for page in pages]
+
+
+def _open_pdf(path: str | Path, password: str | None = None) -> fitz.Document:
+    pdf_path = Path(path)
+    if not pdf_path.is_file():
+        raise FileNotFoundError(f"PDF not found: {pdf_path}")
+
+    pdf_doc = fitz.open(pdf_path)
+    if pdf_doc.is_encrypted:
+        if not password or not pdf_doc.authenticate(password):
+            pdf_doc.close()
+            raise PermissionError(f"PDF is encrypted: {pdf_path}")
+    return pdf_doc
+
+
+def _save_pdf(pdf_doc: fitz.Document, output_path: Path) -> None:
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    pdf_doc.save(output_path)
+    pdf_doc.close()
+
+
+def redact_matches(page: fitz.Page, matched_values: Iterable[str]) -> int:
+    count = 0
+    for value in matched_values:
+        areas = page.search_for(value)
+        if not areas:
+            continue
+        count += 1
+        for area in areas:
+            page.add_redact_annot(area, text=" ", fill=(0, 0, 0))
+    if count:
+        page.apply_redactions()
+    return count
+
+
+def frame_matches(page: fitz.Page, matched_values: Iterable[str]) -> int:
+    count = 0
+    for value in matched_values:
+        areas = page.search_for(value)
+        if not areas:
+            continue
+        count += 1
+        for area in areas:
+            annot = page.add_rect_annot(area)
+            annot.set_colors(stroke=fitz.utils.getColor("red"))
+            annot.update()
+    return count
+
+
+def highlight_matches(page: fitz.Page, matched_values: Iterable[str], action: Action) -> int:
+    count = 0
+    for value in matched_values:
+        areas = page.search_for(value)
+        if not areas:
+            continue
+        count += 1
+        if action == Action.SQUIGGLY:
+            annot = page.add_squiggly_annot(areas)
+        elif action == Action.UNDERLINE:
+            annot = page.add_underline_annot(areas)
+        elif action == Action.STRIKEOUT:
+            annot = page.add_strikeout_annot(areas)
+        else:
+            annot = page.add_highlight_annot(areas)
+        annot.update()
+    return count
+
+
+def remove_annotations(
+    input_file: str | Path,
+    output_file: str | Path,
+    pages: Sequence[int] | None = None,
+    password: str | None = None,
+) -> ProcessResult:
+    pdf_doc = _open_pdf(input_file, password)
+    removed = 0
+    pages_processed = 0
+    page_list = _normalize_pages(pages)
+
+    for page_index in range(pdf_doc.page_count):
+        if page_list is not None and page_index not in page_list:
+            continue
+        pages_processed += 1
+        page = pdf_doc[page_index]
+        annot = page.first_annot
+        while annot:
+            removed += 1
+            next_annot = annot.next
+            page.delete_annot(annot)
+            annot = next_annot
+
+    _save_pdf(pdf_doc, Path(output_file))
+    return ProcessResult(
+        input_path=str(input_file),
+        output_path=str(output_file),
+        action=Action.REMOVE,
+        matches=removed,
+        pages_processed=pages_processed,
+    )
+
+
+class PDFAnnotator:
+    """High-level API for annotating PDF files."""
+
+    def __init__(
+        self,
+        pattern: str,
+        action: Action | str = Action.HIGHLIGHT,
+        pages: Sequence[int | str] | None = None,
+        password: str | None = None,
+    ):
+        self.pattern = pattern
+        self.action = action if isinstance(action, Action) else Action.from_value(action)
+        self.pages = _normalize_pages(pages)
+        self.password = password
+
+    def annotate(self, input_file: str | Path, output_file: str | Path) -> ProcessResult:
+        return annotate_pdf(
+            input_file=input_file,
+            output_file=output_file,
+            pattern=self.pattern,
+            action=self.action,
+            pages=self.pages,
+            password=self.password,
+        )
+
+
+def annotate_pdf(
+    input_file: str | Path,
+    output_file: str | Path,
+    pattern: str,
+    action: Action | str = Action.HIGHLIGHT,
+    pages: Sequence[int | str] | None = None,
+    password: str | None = None,
+) -> ProcessResult:
+    """Search a PDF and apply the requested annotation action."""
+    selected_action = action if isinstance(action, Action) else Action.from_value(action)
+    page_list = _normalize_pages(pages)
+
+    if selected_action == Action.REMOVE:
+        return remove_annotations(input_file, output_file, page_list, password)
+
+    pdf_doc = _open_pdf(input_file, password)
+    total_matches = 0
+    pages_processed = 0
+
+    for page_index in range(pdf_doc.page_count):
+        if page_list is not None and page_index not in page_list:
+            continue
+        pages_processed += 1
+        page = pdf_doc[page_index]
+        page_lines = page.get_text("text").split("\n")
+        matched_values = list(search_for_text(page_lines, pattern))
+        if not matched_values:
+            continue
+
+        if selected_action == Action.REDACT:
+            total_matches += redact_matches(page, matched_values)
+        elif selected_action == Action.FRAME:
+            total_matches += frame_matches(page, matched_values)
+        else:
+            total_matches += highlight_matches(page, matched_values, selected_action)
+
+    _save_pdf(pdf_doc, Path(output_file))
+    return ProcessResult(
+        input_path=str(input_file),
+        output_path=str(output_file),
+        action=selected_action,
+        matches=total_matches,
+        pages_processed=pages_processed,
+    )

docintel/services/pdf/models.py

@@ -0,0 +1,104 @@
+"""Shared types for PDF annotation."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class StructureMode(str, Enum):
+    CURATE = "curate"
+    SEARCHABLE = "searchable"
+
+    @classmethod
+    def from_value(cls, value: str) -> "StructureMode":
+        normalized = value.strip().lower()
+        for mode in cls:
+            if mode.value == normalized:
+                return mode
+        valid = ", ".join(mode.value for mode in cls)
+        raise ValueError(f"Unsupported mode '{value}'. Choose from: {valid}")
+
+
+class Action(str, Enum):
+    HIGHLIGHT = "Highlight"
+    SQUIGGLY = "Squiggly"
+    UNDERLINE = "Underline"
+    STRIKEOUT = "Strikeout"
+    REDACT = "Redact"
+    FRAME = "Frame"
+    REMOVE = "Remove"
+
+    @classmethod
+    def choices(cls) -> tuple[str, ...]:
+        return tuple(action.value for action in cls)
+
+    @classmethod
+    def from_value(cls, value: str) -> "Action":
+        normalized = value.strip().lower()
+        for action in cls:
+            if action.value.lower() == normalized:
+                return action
+        valid = ", ".join(cls.choices())
+        raise ValueError(f"Unsupported action '{value}'. Choose from: {valid}")
+
+
+@dataclass(frozen=True)
+class ProcessResult:
+    input_path: str
+    output_path: str
+    action: Action
+    matches: int
+    pages_processed: int
+
+    def to_dict(self) -> dict:
+        return {
+            "input_path": self.input_path,
+            "output_path": self.output_path,
+            "action": self.action.value,
+            "matches": self.matches,
+            "pages_processed": self.pages_processed,
+        }
+
+    def __str__(self) -> str:
+        return (
+            f"{self.matches} matches annotated in {self.pages_processed} pages "
+            f"-> {self.output_path}"
+        )
+
+
+@dataclass(frozen=True)
+class StructureResult:
+    input_path: str
+    output_path: str
+    mode: StructureMode
+    pages_processed: int
+    ocr_pages: list[int]
+    document_title: str
+
+    def to_dict(self) -> dict:
+        return {
+            "input_path": self.input_path,
+            "output_path": self.output_path,
+            "mode": self.mode.value,
+            "pages_processed": self.pages_processed,
+            "ocr_pages": self.ocr_pages,
+            "document_title": self.document_title,
+        }
+
+
+@dataclass(frozen=True)
+class PIIDetectionResult(ProcessResult):
+    ocr_pages: list[int]
+    findings: list[dict]
+
+    def to_dict(self) -> dict:
+        payload = super().to_dict()
+        payload.update(
+            {
+                "ocr_pages": self.ocr_pages,
+                "findings": self.findings,
+                "finding_count": len(self.findings),
+            }
+        )
+        return payload

docintel/services/pdf/ocr.py

@@ -0,0 +1,130 @@
+"""EasyOCR extraction for scanned PDF pages."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import TYPE_CHECKING
+
+import fitz
+
+from docintel.services.pdf.presets import MIN_NATIVE_TEXT_CHARS, OCR_RENDER_SCALE
+
+if TYPE_CHECKING:
+    import numpy as np
+
+
+@dataclass(frozen=True)
+class OCRSpan:
+    """A text region detected by OCR with PDF coordinates."""
+
+    text: str
+    rect: fitz.Rect
+    confidence: float
+    char_start: int = 0
+    char_end: int = 0
+
+
+@dataclass(frozen=True)
+class IndexedSpan:
+    """Character offsets mapped to a PDF rectangle."""
+
+    char_start: int
+    char_end: int
+    rect: fitz.Rect
+
+
+def page_has_native_text(page: fitz.Page, min_chars: int = MIN_NATIVE_TEXT_CHARS) -> bool:
+    """Return True when the PDF text layer has enough content to skip OCR."""
+    return len(page.get_text("text").strip()) >= min_chars
+
+
+@lru_cache(maxsize=1)
+def _easyocr_reader():
+    import easyocr
+
+    return easyocr.Reader(["en"], gpu=False, verbose=False)
+
+
+def _pixmap_to_array(pix: fitz.Pixmap) -> np.ndarray:
+    import numpy as np
+
+    return np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.height, pix.width, pix.n)
+
+
+def extract_page_ocr(page: fitz.Page, scale: float = OCR_RENDER_SCALE) -> list[OCRSpan]:
+    """Run EasyOCR on a PDF page and return text boxes in PDF coordinates."""
+    matrix = fitz.Matrix(scale, scale)
+    pixmap = page.get_pixmap(matrix=matrix, alpha=False)
+    image = _pixmap_to_array(pixmap)
+
+    reader = _easyocr_reader()
+    detections = reader.readtext(image)
+
+    spans: list[OCRSpan] = []
+    for bbox, text, confidence in detections:
+        cleaned = str(text).strip()
+        if not cleaned:
+            continue
+        xs = [point[0] for point in bbox]
+        ys = [point[1] for point in bbox]
+        rect = fitz.Rect(
+            min(xs) / scale,
+            min(ys) / scale,
+            max(xs) / scale,
+            max(ys) / scale,
+        )
+        spans.append(OCRSpan(text=cleaned, rect=rect, confidence=float(confidence)))
+
+    return spans
+
+
+def build_indexed_text(spans: list[OCRSpan]) -> tuple[str, list[IndexedSpan]]:
+    """Join OCR spans into page text and track character offsets per box."""
+    chunks: list[str] = []
+    indexed: list[IndexedSpan] = []
+    position = 0
+
+    for index, span in enumerate(spans):
+        if index > 0:
+            chunks.append(" ")
+            position += 1
+        start = position
+        chunks.append(span.text)
+        position += len(span.text)
+        indexed.append(IndexedSpan(char_start=start, char_end=position, rect=span.rect))
+
+    return "".join(chunks), indexed
+
+
+def rects_for_char_range(start: int, end: int, indexed: list[IndexedSpan]) -> list[fitz.Rect]:
+    """Map a character span to one or more PDF rectangles."""
+    rects: list[fitz.Rect] = []
+    for item in indexed:
+        if item.char_end <= start or item.char_start >= end:
+            continue
+        rects.append(item.rect)
+    return rects
+
+
+def merge_rects(rects: list[fitz.Rect]) -> fitz.Rect | None:
+    """Merge rectangles into a single bounding box."""
+    if not rects:
+        return None
+    return fitz.Rect(
+        min(rect.x0 for rect in rects),
+        min(rect.y0 for rect in rects),
+        max(rect.x1 for rect in rects),
+        max(rect.y1 for rect in rects),
+    )
+
+
+def embed_invisible_text_layer(page: fitz.Page, spans: list[OCRSpan]) -> None:
+    """Add a searchable text layer from OCR spans (invisible rendering)."""
+    for span in spans:
+        page.insert_text(
+            (span.rect.x0, span.rect.y1),
+            span.text,
+            fontsize=max(6, span.rect.height * 0.8),
+            render_mode=3,
+        )

docintel/services/pdf/pii.py

@@ -0,0 +1,105 @@
+"""Microsoft Presidio PII detection."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Sequence
+
+from docintel.services.pdf.presets import DEFAULT_PII_ENTITIES
+
+
+@dataclass(frozen=True)
+class PIIHit:
+    """A sensitive entity detected in text."""
+
+    entity_type: str
+    text: str
+    start: int
+    end: int
+    score: float
+
+    def to_dict(self) -> dict:
+        return {
+            "entity_type": self.entity_type,
+            "text": self.text,
+            "start": self.start,
+            "end": self.end,
+            "score": round(self.score, 4),
+        }
+
+
+@lru_cache(maxsize=1)
+def _analyzer_engine():
+    from presidio_analyzer import AnalyzerEngine
+
+    return AnalyzerEngine()
+
+
+def detect_pii_in_text(
+    text: str,
+    *,
+    entities: Sequence[str] | None = None,
+    language: str = "en",
+    min_score: float = 0.35,
+) -> list[PIIHit]:
+    """Run Presidio analyzer on plain text."""
+    if not text.strip():
+        return []
+
+    selected_entities = list(entities) if entities else list(DEFAULT_PII_ENTITIES)
+    analyzer = _analyzer_engine()
+    results = analyzer.analyze(
+        text=text,
+        language=language,
+        entities=selected_entities,
+    )
+
+    hits: list[PIIHit] = []
+    for result in results:
+        if result.score < min_score:
+            continue
+        hits.append(
+            PIIHit(
+                entity_type=result.entity_type,
+                text=text[result.start : result.end],
+                start=result.start,
+                end=result.end,
+                score=float(result.score),
+            )
+        )
+    return hits
+
+
+def list_supported_entities(language: str = "en") -> list[str]:
+    """Return Presidio-supported entity types for a language."""
+    return sorted(_analyzer_engine().get_supported_entities(language=language))
+
+
+def mask_pii_in_text(
+    text: str,
+    *,
+    entities: Sequence[str] | None = None,
+    language: str = "en",
+    min_score: float = 0.35,
+    mask_template: str = "[REDACTED_{entity}]",
+) -> tuple[str, int]:
+    """
+    Replace detected PII spans with redaction tokens before external LLM calls.
+
+    Returns masked text and the number of entities redacted.
+    """
+    hits = detect_pii_in_text(
+        text,
+        entities=entities,
+        language=language,
+        min_score=min_score,
+    )
+    if not hits:
+        return text, 0
+
+    masked = text
+    for hit in sorted(hits, key=lambda item: item.start, reverse=True):
+        token = mask_template.format(entity=hit.entity_type)
+        masked = masked[: hit.start] + token + masked[hit.end :]
+    return masked, len(hits)