docfgp 0.1.0__py3-none-any.whl

docfgp/__init__.py

@@ -0,0 +1,6 @@
+"""Create deterministic document fingerprints."""
+
+from .fingerprint import Fingerprinter
+from .schemas import DocumentFingerprint, FingerprintConfig, FingerprintError
+
+__all__ = ["Fingerprinter", "DocumentFingerprint", "FingerprintConfig", "FingerprintError"]

docfgp/cli.py

@@ -0,0 +1,49 @@
+"""Expose document fingerprinting as a CLI."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from .fingerprint import Fingerprinter
+from .schemas import FingerprintConfig, FingerprintError
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Print a document fingerprint as JSON.
+
+    Args:
+        argv: Optional command-line arguments.
+
+    Returns:
+        Zero on success and nonzero on processing errors.
+    """
+    parser = argparse.ArgumentParser(description="Fingerprint a document")
+    parser.add_argument("path", help="PDF, DOCX, PPTX, or image path")
+    parser.add_argument("--office2pdf", help="Path to the office2pdf binary")
+    parser.add_argument("--tesseract", help="Path to the Tesseract binary")
+    parser.add_argument("--ocr-language", default="eng", help="Installed Tesseract language codes")
+    parser.add_argument(
+        "--max-document-seconds", type=float, default=180.0, help="Processing deadline in seconds"
+    )
+    parser.add_argument(
+        "--no-shingle-counts", action="store_true", help="Omit precise overlap counts"
+    )
+    args = parser.parse_args(argv)
+    try:
+        config = FingerprintConfig(
+            office2pdf_path=args.office2pdf,
+            tesseract_path=args.tesseract,
+            ocr_language=args.ocr_language,
+            max_document_seconds=args.max_document_seconds,
+            include_shingle_counts=not args.no_shingle_counts,
+        )
+        print(Fingerprinter(config).fingerprint(args.path).to_json())
+    except (FingerprintError, ValueError) as exc:
+        print(f"docfgp: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docfgp/extract.py

@@ -0,0 +1,274 @@
+"""Validate input and extract PDF or image pages with bounded decoding."""
+
+from __future__ import annotations
+
+import contextlib
+import math
+import shutil
+import subprocess
+import tempfile
+import unicodedata
+import zipfile
+from collections.abc import Generator, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+import pypdfium2 as pdfium
+from PIL import Image, UnidentifiedImageError
+
+from .normalize import strict_text, word_tokens
+from .schemas import FingerprintConfig, FingerprintError, OCRReason, _remaining_seconds
+
+_IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp"}
+_IMAGE_FORMATS = {"PNG", "JPEG", "TIFF", "BMP", "WEBP"}
+_OFFICE_SUFFIXES = {".docx", ".pptx"}
+RASTER_OCR_COVERAGE = 0.5
+
+
+@dataclass(frozen=True)
+class ExtractedPage:
+    """Carry one page's native text, image renders, and OCR trigger."""
+
+    number: int
+    native_text: str
+    visual: Image.Image
+    ocr_image: Image.Image | None
+    ocr_reason: OCRReason | None
+
+
+def identify(path: Path, config: FingerprintConfig) -> tuple[str, int]:
+    """Validate the format signature, size, and Office ZIP structure.
+
+    Args:
+        path: Existing input path.
+        config: Input and expansion limits.
+
+    Returns:
+        Format name and original byte size.
+
+    Raises:
+        FingerprintError: The file is unsupported, unsafe, or invalid.
+    """
+    try:
+        size = path.stat().st_size
+        with path.open("rb") as stream:
+            header = stream.read(16)
+    except OSError as exc:
+        raise FingerprintError(f"Cannot read document: {exc}") from exc
+    if size == 0 or size > config.max_file_bytes:
+        raise FingerprintError("Document is empty or exceeds max_file_bytes")
+    suffix = path.suffix.lower()
+    if suffix == ".pdf" and header.startswith(b"%PDF-"):
+        return "pdf", size
+    if suffix in _OFFICE_SUFFIXES and header.startswith(b"PK\x03\x04"):
+        try:
+            with zipfile.ZipFile(path) as archive:
+                infos = archive.infolist()
+                if len(infos) > 10_000 or sum(item.file_size for item in infos) > 200_000_000:
+                    raise FingerprintError("Office package exceeds expansion limits")
+                names = set(archive.namelist())
+                required = "word/document.xml" if suffix == ".docx" else "ppt/presentation.xml"
+                if "[Content_Types].xml" not in names or required not in names:
+                    raise FingerprintError("Office package is missing required parts")
+                if any(item.flag_bits & 1 for item in infos):
+                    raise FingerprintError("Encrypted Office packages are unsupported")
+                if archive.testzip() is not None:
+                    raise FingerprintError("Office package failed CRC validation")
+        except (OSError, zipfile.BadZipFile, RuntimeError, ValueError, EOFError) as exc:
+            raise FingerprintError(f"Invalid Office package: {exc}") from exc
+        return suffix[1:], size
+    if suffix in _IMAGE_SUFFIXES:
+        try:
+            with Image.open(path) as image:
+                image_format = image.format
+                if image_format not in _IMAGE_FORMATS:
+                    raise FingerprintError("Unsupported image format")
+                image.verify()
+        except (OSError, UnidentifiedImageError, ValueError) as exc:
+            raise FingerprintError(f"Invalid image: {exc}") from exc
+        return image_format.lower(), size
+    raise FingerprintError("Unsupported format or file signature")
+
+
+@contextlib.contextmanager
+def pdf_path(
+    path: Path, source_format: str, config: FingerprintConfig, deadline: float
+) -> Iterator[Path]:
+    """Yield a PDF path, converting Office files in a temporary directory.
+
+    Args:
+        path: Original path.
+        source_format: Identified input format.
+        config: Converter path and timeout.
+        deadline: Cooperative document deadline.
+
+    Yields:
+        Original or temporary PDF path.
+
+    Raises:
+        FingerprintError: Conversion fails or produces invalid output.
+    """
+    if source_format == "pdf":
+        _remaining_seconds(deadline)
+        yield path
+        return
+    configured = config.office2pdf_path
+    executable = (
+        str(Path(configured).expanduser().resolve())
+        if configured and Path(configured).expanduser().is_file()
+        else shutil.which(configured or "office2pdf")
+    )
+    if not executable:
+        raise FingerprintError("office2pdf is required for DOCX and PPTX")
+    with tempfile.TemporaryDirectory(prefix="docfgp-office-") as directory:
+        output = Path(directory) / "converted.pdf"
+        error_path = Path(directory) / "converter.stderr"
+        try:
+            with error_path.open("wb") as errors:
+                result = subprocess.run(
+                    [executable, str(path.resolve()), "-o", str(output)],
+                    cwd=directory,
+                    stdout=subprocess.DEVNULL,
+                    stderr=errors,
+                    timeout=min(config.process_timeout, _remaining_seconds(deadline)),
+                    check=False,
+                )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            _remaining_seconds(deadline)
+            raise FingerprintError(f"office2pdf could not complete: {exc}") from exc
+        _remaining_seconds(deadline)
+        if result.returncode or not output.is_file():
+            with error_path.open("rb") as errors:
+                message = errors.read(500).decode("utf-8", "replace").strip()
+            raise FingerprintError(f"office2pdf failed: {message}")
+        with output.open("rb") as stream:
+            signature = stream.read(5)
+        if output.stat().st_size > config.max_file_bytes or signature != b"%PDF-":
+            raise FingerprintError("office2pdf produced an invalid or oversized PDF")
+        yield output
+
+
+def _render(page: pdfium.PdfPage, dpi: int, config: FingerprintConfig) -> Image.Image:
+    width, height = page.get_size()
+    if not (math.isfinite(width) and math.isfinite(height) and width > 0 and height > 0):
+        raise FingerprintError("Invalid PDF page geometry")
+    scale = min(dpi / 72, math.sqrt(config.max_page_pixels / (width * height)))
+    if scale <= 0:
+        raise FingerprintError("PDF page exceeds rendering limits")
+    image = page.render(scale=scale).to_pil()
+    if image.width * image.height > config.max_page_pixels:
+        image.close()
+        raise FingerprintError("PDF page exceeds max_page_pixels")
+    return image
+
+
+def _ocr_reason(page: pdfium.PdfPage, text: str) -> OCRReason | None:
+    if any(
+        character == "\ufffd"
+        or (unicodedata.category(character) == "Cc" and character not in "\r\n\t\f")
+        for character in text
+    ):
+        return "invalid_native_text"
+    if not word_tokens(strict_text(text)):
+        return "missing_native_text"
+    width, height = page.get_size()
+    page_area = width * height
+    if not math.isfinite(page_area) or page_area <= 0:
+        raise FingerprintError("Invalid PDF page geometry")
+    image_area = 0.0
+    for image in page.get_objects(filter=[pdfium.raw.FPDF_PAGEOBJ_IMAGE]):
+        left, bottom, right, top = image.get_bounds()
+        if not all(math.isfinite(value) for value in (left, bottom, right, top)):
+            continue
+        visible_width = max(0.0, min(right, width) - max(left, 0.0))
+        visible_height = max(0.0, min(top, height) - max(bottom, 0.0))
+        image_area += visible_width * visible_height
+        if image_area >= page_area * RASTER_OCR_COVERAGE:
+            return "rasterized_content"
+    return None
+
+
+def pdf_pages(
+    path: Path, config: FingerprintConfig, deadline: float
+) -> Generator[ExtractedPage, None, None]:
+    """Yield native text plus visual and optional OCR render for each PDF page.
+
+    Args:
+        path: Input PDF.
+        config: Page, pixel, and resolution limits.
+        deadline: Cooperative document deadline.
+
+    Yields:
+        Page data with native text and bounded image renders.
+
+    Raises:
+        FingerprintError: PDF opening, page decoding, or limits fail.
+    """
+    try:
+        document = pdfium.PdfDocument(str(path))
+    except Exception as exc:
+        raise FingerprintError(f"Cannot open PDF: {exc}") from exc
+    try:
+        count = len(document)
+        if count == 0 or count > config.max_pages:
+            raise FingerprintError("PDF has no pages or exceeds max_pages")
+        for index in range(count):
+            _remaining_seconds(deadline)
+            try:
+                page = document[index]
+            except Exception as exc:
+                raise FingerprintError(f"Cannot open PDF page {index + 1}: {exc}") from exc
+            try:
+                textpage = page.get_textpage()
+                try:
+                    text = textpage.get_text_range()
+                finally:
+                    textpage.close()
+                reason = _ocr_reason(page, text)
+                visual = _render(page, config.render_dpi, config)
+                try:
+                    ocr_image = _render(page, config.ocr_dpi, config) if reason else None
+                except Exception:
+                    visual.close()
+                    raise
+                yield ExtractedPage(index + 1, text, visual, ocr_image, reason)
+            except FingerprintError:
+                raise
+            except Exception as exc:
+                raise FingerprintError(f"Cannot process PDF page {index + 1}: {exc}") from exc
+            finally:
+                page.close()
+    finally:
+        document.close()
+
+
+def image_pages(
+    path: Path, config: FingerprintConfig, deadline: float
+) -> Generator[ExtractedPage, None, None]:
+    """Yield decoded image frames within configured limits.
+
+    Args:
+        path: Standalone image path.
+        config: Frame and pixel limits.
+        deadline: Cooperative document deadline.
+
+    Yields:
+        One-based image frames with OCR provenance.
+
+    Raises:
+        FingerprintError: Decoding fails or limits are exceeded.
+    """
+    try:
+        with Image.open(path) as image:
+            frames = getattr(image, "n_frames", 1)
+            if frames > config.max_pages:
+                raise FingerprintError("Image exceeds max_pages")
+            for index in range(frames):
+                _remaining_seconds(deadline)
+                image.seek(index)
+                if image.width * image.height > config.max_page_pixels:
+                    raise FingerprintError("Image frame exceeds max_page_pixels")
+                frame = image.copy()
+                yield ExtractedPage(index + 1, "", frame, frame, "image_input")
+    except (OSError, UnidentifiedImageError) as exc:
+        raise FingerprintError(f"Cannot decode image: {exc}") from exc

docfgp/fingerprint.py

@@ -0,0 +1,288 @@
+"""Coordinate one stateless document fingerprinting invocation."""
+
+from __future__ import annotations
+
+import hashlib
+import time
+from collections import Counter
+from collections.abc import Iterable, Iterator
+from contextlib import closing
+from pathlib import Path
+
+from . import extract, ocr
+from .minhash import BANDS, COMPONENTS, ROWS, band_keys, sketch, stable_digest
+from .normalize import strict_text, tolerant_text, word_tokens
+from .schemas import (
+    DocumentFingerprint,
+    FingerprintConfig,
+    FingerprintError,
+    PageHash,
+    PageQuality,
+    PageSource,
+    Quality,
+    QualityStatus,
+    TextEvidence,
+    _remaining_seconds,
+)
+from .visual import page_phash, visual_keys
+from .winnowing import anchors
+
+SCHEMA_VERSION = "1.1"
+ALGORITHM_VERSION = "1.3"
+WORD_SHINGLE_SIZE = 5
+CHAR_SHINGLE_SIZE = 13
+ANCHOR_WINDOW = 32
+OCR_MIN_CONFIDENCE = 30.0
+OCR_REPLACEMENT_CONFIDENCE = 60.0
+
+
+def _raw_hash(path: Path, deadline: float) -> str:
+    digest = hashlib.sha256()
+    try:
+        with path.open("rb") as stream:
+            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+                _remaining_seconds(deadline)
+                digest.update(chunk)
+    except OSError as exc:
+        raise FingerprintError(f"Cannot hash document: {exc}") from exc
+    return digest.hexdigest()
+
+
+def _shingles(tokens: tuple[str, ...]) -> Iterator[str]:
+    for index in range(len(tokens) - WORD_SHINGLE_SIZE + 1):
+        yield stable_digest("\x1f".join(tokens[index : index + WORD_SHINGLE_SIZE]).encode("utf-8"))
+
+
+def _assemble_fingerprint(
+    text: str,
+    *,
+    config: FingerprintConfig,
+    deadline: float,
+    source_format: str,
+    byte_size: int,
+    raw_sha256: str,
+    page_quality: tuple[PageQuality, ...],
+    page_hashes: tuple[PageHash, ...] = (),
+    warnings: tuple[str, ...] = (),
+) -> DocumentFingerprint:
+    """Build shared text features and a versioned result from extracted content."""
+    tokens = word_tokens(text)
+    canonical = " ".join(tokens)
+    word_hashes: list[str] = []
+    counts: Counter[str] = Counter()
+    for value in _shingles(tokens):
+        if len(word_hashes) % 4096 == 0:
+            _remaining_seconds(deadline)
+        word_hashes.append(value)
+        counts[value] += 1
+        if len(counts) > config.max_features:
+            raise FingerprintError("Word shingles exceed max_features")
+    word_sketch = sketch(counts)
+    _remaining_seconds(deadline)
+    characters = tolerant_text(canonical)
+    char_features = set()
+    for index in range(len(characters) - CHAR_SHINGLE_SIZE + 1):
+        if index % 4096 == 0:
+            _remaining_seconds(deadline)
+        char_features.add(stable_digest(characters[index : index + CHAR_SHINGLE_SIZE].encode()))
+        if len(char_features) > config.max_features:
+            raise FingerprintError("Character shingles exceed max_features")
+    char_sketch = sketch(char_features)
+    _remaining_seconds(deadline)
+    passage_anchors = anchors(word_hashes, ANCHOR_WINDOW)
+    visual_lookup = tuple(sorted({key for page in page_hashes for key in page.lookup_keys}))
+    if word_sketch:
+        text_evidence: TextEvidence = "word_shingles"
+    elif char_sketch:
+        text_evidence = "character_shingles"
+    elif canonical:
+        text_evidence = "exact_only"
+    else:
+        text_evidence = "none"
+    if tokens and page_hashes:
+        status: QualityStatus = "text_and_visual"
+    elif tokens:
+        status = "text_only"
+    elif page_hashes:
+        status = "visual_only"
+    else:
+        status = "insufficient_evidence"
+    quality_warnings = list(warnings)
+    if status == "visual_only":
+        quality_warnings.append("No reliable text was extracted; textual matching is unavailable")
+    if status == "insufficient_evidence":
+        quality_warnings.append("Neither text nor nonblank visual evidence was extracted")
+    if text_evidence == "exact_only":
+        quality_warnings.append(
+            "Text supports exact matching only; no shingle sketch was available"
+        )
+    elif text_evidence == "character_shingles":
+        quality_warnings.append(
+            "Word-shingle matching is unavailable; character retrieval remains available"
+        )
+    return DocumentFingerprint(
+        schema_version=SCHEMA_VERSION,
+        algorithm_version=ALGORITHM_VERSION,
+        source_format=source_format,
+        byte_size=byte_size,
+        page_count=len(page_quality),
+        exact_hashes={
+            "raw_sha256": raw_sha256,
+            "canonical_sha256": stable_digest(canonical.encode()) if canonical else None,
+        },
+        lookup_keys={
+            "word_lsh": band_keys(word_sketch, "word"),
+            "char_lsh": band_keys(char_sketch, "char"),
+            "passage": tuple(f"passage:v1:{value}" for value in passage_anchors),
+            "visual": visual_lookup,
+        },
+        minhash=word_sketch,
+        char_minhash=char_sketch,
+        shingle_counts=tuple(sorted(counts.items())) if config.include_shingle_counts else (),
+        shingle_total=sum(counts.values()),
+        shingle_unique=len(counts),
+        passage_anchors=passage_anchors,
+        page_hashes=page_hashes,
+        quality=Quality(status, text_evidence, page_quality, tuple(quality_warnings)),
+        parameters={
+            "word_shingle_size": WORD_SHINGLE_SIZE,
+            "char_shingle_size": CHAR_SHINGLE_SIZE,
+            "minhash_components": COMPONENTS,
+            "lsh_bands": BANDS,
+            "lsh_rows": ROWS,
+            "anchor_window": ANCHOR_WINDOW,
+            "visual_hash": "phash64-dct32",
+            "render_dpi": config.render_dpi,
+            "ocr_dpi": config.ocr_dpi,
+            "ocr_language": config.ocr_language,
+            "ocr_min_confidence": OCR_MIN_CONFIDENCE,
+            "ocr_replacement_confidence": OCR_REPLACEMENT_CONFIDENCE,
+            "raster_ocr_coverage": extract.RASTER_OCR_COVERAGE,
+            "shingle_counts_included": config.include_shingle_counts,
+        },
+    )
+
+
+class Fingerprinter:
+    """Generate stateless document fingerprints.
+
+    Args:
+        config: Optional bounded processing settings.
+    """
+
+    def __init__(self, config: FingerprintConfig | None = None) -> None:
+        """Store immutable processing limits.
+
+        Args:
+            config: Optional processing settings.
+        """
+        self.config = config or FingerprintConfig()
+
+    def fingerprint(self, path: str | Path) -> DocumentFingerprint:
+        """Create a document's textual and visual fingerprint.
+
+        Args:
+            path: PDF, DOCX, PPTX, or supported image path.
+
+        Returns:
+            Versioned fingerprint.
+
+        Raises:
+            FingerprintError: Input is invalid, limits are exceeded, or processing fails.
+        """
+        deadline = time.monotonic() + self.config.max_document_seconds
+        source = Path(path).expanduser()
+        source_format, byte_size = extract.identify(source, self.config)
+        _remaining_seconds(deadline)
+        raw_sha256 = _raw_hash(source, deadline)
+        page_text: list[str] = []
+        page_hashes: list[PageHash] = []
+        page_quality: list[PageQuality] = []
+        warnings: list[str] = []
+        total_text_chars = 0
+
+        def consume(
+            pages: Iterable[extract.ExtractedPage],
+        ) -> None:
+            nonlocal total_text_chars
+            for page in pages:
+                page_number = page.number
+                visual = page.visual
+                ocr_image = page.ocr_image
+                try:
+                    _remaining_seconds(deadline)
+                    native = strict_text(page.native_text)
+                    extracted = "" if page.ocr_reason == "invalid_native_text" else native
+                    source: PageSource = "native" if extracted else "none"
+                    attempted = ocr_image is not None
+                    warning = None
+                    confidence = None
+                    if ocr_image is not None:
+                        try:
+                            result = ocr.read_text(ocr_image, self.config, deadline)
+                            confidence = result.confidence
+                            recognized = strict_text(result.text)
+                            if not recognized or confidence is None:
+                                warning = "OCR found no reliable text"
+                            elif page.ocr_reason == "rasterized_content":
+                                novel = set(word_tokens(recognized)) - set(word_tokens(native))
+                                if confidence >= OCR_REPLACEMENT_CONFIDENCE and len(novel) >= 3:
+                                    extracted = recognized
+                                    source = "ocr"
+                                elif novel:
+                                    warning = "OCR did not provide confident additional text"
+                            elif confidence >= OCR_MIN_CONFIDENCE:
+                                extracted = recognized
+                                source = "ocr"
+                            else:
+                                warning = "OCR confidence was too low to use its text"
+                        except FingerprintError as exc:
+                            warning = str(exc)
+                    if warning:
+                        warnings.append(f"page {page_number}: {warning}")
+                    page_text.append(extracted)
+                    total_text_chars += len(extracted)
+                    if total_text_chars > self.config.max_text_chars:
+                        raise FingerprintError("Extracted text exceeds max_text_chars")
+                    value = page_phash(visual)
+                    if value is not None:
+                        page_hashes.append(PageHash(page_number, value, visual_keys(value)))
+                    page_quality.append(
+                        PageQuality(
+                            page=page_number,
+                            source=source,
+                            text_chars=len(extracted),
+                            native_text_chars=len(native),
+                            ocr_attempted=attempted,
+                            ocr_reason=page.ocr_reason,
+                            ocr_confidence=confidence,
+                            warning=warning,
+                        )
+                    )
+                    _remaining_seconds(deadline)
+                except (OSError, ValueError) as exc:
+                    raise FingerprintError(f"Cannot process page {page_number}: {exc}") from exc
+                finally:
+                    visual.close()
+                    if ocr_image is not None and ocr_image is not visual:
+                        ocr_image.close()
+
+        if source_format not in {"pdf", "docx", "pptx"}:
+            with closing(extract.image_pages(source, self.config, deadline)) as image_frames:
+                consume(image_frames)
+        else:
+            with extract.pdf_path(source, source_format, self.config, deadline) as pdf:
+                with closing(extract.pdf_pages(pdf, self.config, deadline)) as pdf_frames:
+                    consume(pdf_frames)
+
+        return _assemble_fingerprint(
+            " ".join(page_text),
+            config=self.config,
+            deadline=deadline,
+            source_format=source_format,
+            byte_size=byte_size,
+            raw_sha256=raw_sha256,
+            page_quality=tuple(page_quality),
+            page_hashes=tuple(page_hashes),
+            warnings=tuple(warnings),
+        )

docfgp/minhash.py

@@ -0,0 +1,81 @@
+"""Stable 512-permutation MinHash and indexed LSH keys."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Iterable
+
+COMPONENTS = 512
+BANDS = 32
+ROWS = 16
+_PRIME = (1 << 61) - 1
+
+
+def _coefficients(index: int) -> tuple[int, int]:
+    # Derive every permutation from bytes, independent of Python's random module.
+    digest = hashlib.sha256(b"docfgp-minhash-v2" + index.to_bytes(2, "big")).digest()
+    return int.from_bytes(digest[:8], "big") % (_PRIME - 1) + 1, int.from_bytes(
+        digest[8:16], "big"
+    ) % _PRIME
+
+
+_COEFFICIENTS = tuple(_coefficients(index) for index in range(COMPONENTS))
+
+
+def stable_digest(data: bytes) -> str:
+    """Return a stable SHA-256 digest.
+
+    Args:
+        data: Input bytes.
+
+    Returns:
+        Lowercase hexadecimal digest.
+    """
+    return hashlib.sha256(data).hexdigest()
+
+
+def sketch(features: Iterable[str]) -> tuple[str, ...]:
+    """Compute a deterministic, 512-component MinHash over unique features.
+
+    Args:
+        features: Textual shingles or their stable hexadecimal digests.
+
+    Returns:
+        Fixed-width hexadecimal values, or an empty tuple without features.
+    """
+    minima = [_PRIME] * COMPONENTS
+    seen: set[str] = set()
+    for feature in features:
+        if feature in seen:
+            continue
+        seen.add(feature)
+        x = int.from_bytes(hashlib.sha256(feature.encode("utf-8")).digest()[:8], "big") % _PRIME
+        for index, (a, b) in enumerate(_COEFFICIENTS):
+            value = (a * x + b) % _PRIME
+            if value < minima[index]:
+                minima[index] = value
+    return tuple(f"{value:016x}" for value in minima) if seen else ()
+
+
+def band_keys(values: tuple[str, ...], namespace: str) -> tuple[str, ...]:
+    """Build 32 stable keys from consecutive groups of 16 components.
+
+    Args:
+        values: A complete MinHash sketch.
+        namespace: Word or character sketch identifier.
+
+    Returns:
+        Indexed keys, or no keys for an empty sketch.
+
+    Raises:
+        ValueError: The sketch has the wrong length.
+    """
+    if not values:
+        return ()
+    if len(values) != COMPONENTS:
+        raise ValueError("A banded sketch must contain 512 components")
+    keys = []
+    for band in range(BANDS):
+        data = "".join(values[band * ROWS : (band + 1) * ROWS]).encode()
+        keys.append(f"{namespace}:v2:{band:02d}:{stable_digest(data)[:24]}")
+    return tuple(keys)

docfgp/normalize.py

@@ -0,0 +1,49 @@
+"""Canonicalize extracted text while retaining significant numbers."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+_HYPHENATION = re.compile(r"(?<=\w)[-\u00ad][ \t]*\r?\n[ \t]*(?=\w)")
+_WHITESPACE = re.compile(r"\s+")
+_WORDS = re.compile(r"[^\W_]+(?:['’][^\W_]+)?", re.UNICODE)
+
+
+def strict_text(text: str) -> str:
+    """Normalize Unicode, soft line breaks, and whitespace.
+
+    Args:
+        text: Raw extracted page text.
+
+    Returns:
+        Canonical casefolded text.
+    """
+    text = unicodedata.normalize("NFKC", text)
+    text = _HYPHENATION.sub("", text)
+    text = text.replace("\u00ad", "")
+    return _WHITESPACE.sub(" ", text).strip().casefold()
+
+
+def word_tokens(text: str) -> tuple[str, ...]:
+    """Extract Unicode word tokens from normalized text.
+
+    Args:
+        text: Canonical text.
+
+    Returns:
+        Tokens in reading order.
+    """
+    return tuple(_WORDS.findall(text))
+
+
+def tolerant_text(text: str) -> str:
+    """Remove punctuation and spacing for OCR-tolerant character retrieval.
+
+    Args:
+        text: Canonical text.
+
+    Returns:
+        Alphanumeric Unicode characters, including digits.
+    """
+    return "".join(ch for ch in text if ch.isalnum())

docfgp/ocr.py

@@ -0,0 +1,111 @@
+"""Run bounded Tesseract OCR and retain word confidence evidence."""
+
+from __future__ import annotations
+
+import csv
+import math
+import os
+import shutil
+import subprocess
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from PIL import Image
+
+from .schemas import FingerprintConfig, FingerprintError, _remaining_seconds
+from .visual import normalized_grayscale
+
+
+@dataclass(frozen=True)
+class OCRResult:
+    """Hold recognized text and mean confidence over recognized words."""
+
+    text: str
+    confidence: float | None
+
+
+def read_text(image: Image.Image, config: FingerprintConfig, deadline: float) -> OCRResult:
+    """Extract text from a rendered page in an isolated temporary directory.
+
+    Args:
+        image: Page image within configured pixel limits.
+        config: OCR executable, language, and timeout settings.
+        deadline: Cooperative document deadline.
+
+    Returns:
+        Text and Tesseract word confidence from the same OCR invocation.
+
+    Raises:
+        FingerprintError: Tesseract is missing, times out, or fails.
+    """
+    configured = config.tesseract_path
+    executable = (
+        str(Path(configured).expanduser().resolve())
+        if configured and Path(configured).expanduser().is_file()
+        else shutil.which(configured or "tesseract")
+    )
+    if not executable:
+        raise FingerprintError("Tesseract 5 is not available")
+    with tempfile.TemporaryDirectory(prefix="docfgp-ocr-") as directory:
+        input_path = Path(directory) / "page.png"
+        output_base = Path(directory) / "result"
+        output_path = output_base.with_suffix(".txt")
+        confidence_path = output_base.with_suffix(".tsv")
+        error_path = Path(directory) / "ocr.stderr"
+        with normalized_grayscale(image) as prepared:
+            prepared.save(input_path)
+        command = [
+            executable,
+            str(input_path),
+            str(output_base),
+            "-l",
+            config.ocr_language,
+            "--psm",
+            "3",
+            "txt",
+            "tsv",
+        ]
+        try:
+            with error_path.open("wb") as errors:
+                result = subprocess.run(
+                    command,
+                    cwd=directory,
+                    stdout=subprocess.DEVNULL,
+                    stderr=errors,
+                    timeout=min(config.process_timeout, _remaining_seconds(deadline)),
+                    check=False,
+                    env={**os.environ, "OMP_THREAD_LIMIT": "1"},
+                )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            _remaining_seconds(deadline)
+            raise FingerprintError(f"Tesseract could not complete: {exc}") from exc
+        _remaining_seconds(deadline)
+        if result.returncode:
+            with error_path.open("rb") as errors:
+                message = errors.read(500).decode("utf-8", "replace").strip()
+            raise FingerprintError(f"Tesseract failed: {message}")
+        if not output_path.is_file() or output_path.stat().st_size > config.max_text_chars * 4:
+            raise FingerprintError("Tesseract produced missing or oversized text")
+        if (
+            not confidence_path.is_file()
+            or confidence_path.stat().st_size > config.max_text_chars * 16
+        ):
+            raise FingerprintError("Tesseract produced missing or oversized confidence data")
+        total_confidence = 0.0
+        word_count = 0
+        try:
+            with confidence_path.open(encoding="utf-8", errors="replace", newline="") as stream:
+                for row in csv.DictReader(stream, delimiter="\t"):
+                    if row["level"] != "5" or not row["text"].strip():
+                        continue
+                    confidence = float(row["conf"])
+                    if math.isfinite(confidence) and confidence >= 0:
+                        total_confidence += confidence
+                        word_count += 1
+        except (OSError, KeyError, ValueError, TypeError, AttributeError) as exc:
+            raise FingerprintError(f"Cannot read Tesseract confidence data: {exc}") from exc
+        return OCRResult(
+            output_path.read_text(encoding="utf-8", errors="replace"),
+            round(total_confidence / word_count, 2) if word_count else None,
+        )

docfgp/schemas.py

@@ -0,0 +1,222 @@
+"""Versioned public schemas and processing limits for document fingerprints."""
+
+from __future__ import annotations
+
+import json
+import math
+import time
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import MappingProxyType
+from typing import Any, Literal
+
+QualityStatus = Literal["text_and_visual", "text_only", "visual_only", "insufficient_evidence"]
+TextEvidence = Literal["word_shingles", "character_shingles", "exact_only", "none"]
+PageSource = Literal["native", "ocr", "none"]
+OCRReason = Literal[
+    "missing_native_text", "invalid_native_text", "rasterized_content", "image_input"
+]
+
+
+class FingerprintError(Exception):
+    """Report invalid input, unavailable dependencies, or incomplete processing."""
+
+
+def _remaining_seconds(deadline: float) -> float:
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        raise FingerprintError("Document exceeded max_document_seconds")
+    return remaining
+
+
+@dataclass(frozen=True)
+class FingerprintConfig:
+    """Configure bounded document processing.
+
+    Attributes:
+        max_file_bytes: Maximum original input size.
+        max_pages: Maximum PDF pages or image frames.
+        max_page_pixels: Maximum pixels decoded for one page or image.
+        max_text_chars: Maximum total extracted characters.
+        max_features: Maximum distinct word or character shingles.
+        render_dpi: Visual rendering resolution, bounded by max_page_pixels.
+        ocr_dpi: OCR rendering resolution, bounded by max_page_pixels.
+        process_timeout: Seconds allowed for each external process.
+        max_document_seconds: Cooperative deadline checked between processing stages.
+        office2pdf_path: Optional explicit converter executable.
+        tesseract_path: Optional explicit OCR executable.
+        ocr_language: Installed Tesseract language codes.
+        include_shingle_counts: Include sorted exact occurrence counts.
+    """
+
+    max_file_bytes: int = 50_000_000
+    max_pages: int = 200
+    max_page_pixels: int = 8_000_000
+    max_text_chars: int = 2_000_000
+    max_features: int = 200_000
+    render_dpi: int = 96
+    ocr_dpi: int = 180
+    process_timeout: float = 60.0
+    max_document_seconds: float = 180.0
+    office2pdf_path: str | Path | None = None
+    tesseract_path: str | Path | None = None
+    ocr_language: str = "eng"
+    include_shingle_counts: bool = True
+
+    def __post_init__(self) -> None:
+        """Reject limits that would make bounded processing impossible.
+
+        Raises:
+            ValueError: A limit or resolution is invalid.
+        """
+        values = (
+            self.max_file_bytes,
+            self.max_pages,
+            self.max_page_pixels,
+            self.max_text_chars,
+            self.max_features,
+            self.render_dpi,
+            self.ocr_dpi,
+        )
+        durations = (self.process_timeout, self.max_document_seconds)
+        if any(
+            not isinstance(value, int) or isinstance(value, bool) or value <= 0 for value in values
+        ):
+            raise ValueError("All integer processing limits must be positive")
+        if any(
+            isinstance(value, bool)
+            or not isinstance(value, (int, float))
+            or not math.isfinite(value)
+            or value <= 0
+            for value in durations
+        ):
+            raise ValueError("All processing limits must be positive")
+        if self.render_dpi > 300 or self.ocr_dpi > 300:
+            raise ValueError("Rendering resolution cannot exceed 300 DPI")
+        if not isinstance(self.ocr_language, str) or not self.ocr_language.strip():
+            raise ValueError("ocr_language must not be empty")
+        for name in ("office2pdf_path", "tesseract_path"):
+            value = getattr(self, name)
+            if value is not None:
+                if not isinstance(value, (str, Path)):
+                    raise ValueError(f"{name} must be a path")
+                object.__setattr__(self, name, str(value))
+
+
+@dataclass(frozen=True)
+class PageQuality:
+    """Record extraction provenance for one page or image frame."""
+
+    page: int
+    source: PageSource
+    text_chars: int
+    native_text_chars: int
+    ocr_attempted: bool
+    ocr_reason: OCRReason | None = None
+    ocr_confidence: float | None = None
+    warning: str | None = None
+
+
+@dataclass(frozen=True)
+class Quality:
+    """State whether the fingerprint carries text, visual, or insufficient evidence."""
+
+    status: QualityStatus
+    text_evidence: TextEvidence
+    pages: tuple[PageQuality, ...]
+    warnings: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class PageHash:
+    """Store a page perceptual hash and its indexed visual keys."""
+
+    page: int
+    phash: str
+    lookup_keys: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class DocumentFingerprint:
+    """Serializable, versioned document fingerprint."""
+
+    schema_version: str
+    algorithm_version: str
+    source_format: str
+    byte_size: int
+    page_count: int
+    exact_hashes: Mapping[str, str | None]
+    lookup_keys: Mapping[str, tuple[str, ...]]
+    minhash: tuple[str, ...]
+    char_minhash: tuple[str, ...]
+    shingle_counts: tuple[tuple[str, int], ...]
+    shingle_total: int
+    shingle_unique: int
+    passage_anchors: tuple[str, ...]
+    page_hashes: tuple[PageHash, ...]
+    quality: Quality
+    parameters: Mapping[str, int | float | str | bool] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        """Freeze mapping fields so the fingerprint cannot change after creation."""
+        object.__setattr__(self, "exact_hashes", MappingProxyType(dict(self.exact_hashes)))
+        object.__setattr__(
+            self,
+            "lookup_keys",
+            MappingProxyType({name: tuple(keys) for name, keys in self.lookup_keys.items()}),
+        )
+        object.__setattr__(self, "parameters", MappingProxyType(dict(self.parameters)))
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-compatible representation.
+
+        Returns:
+            Recursively serialized fingerprint with stable field order.
+        """
+        return {
+            "schema_version": self.schema_version,
+            "algorithm_version": self.algorithm_version,
+            "source_format": self.source_format,
+            "byte_size": self.byte_size,
+            "page_count": self.page_count,
+            "exact_hashes": dict(self.exact_hashes),
+            "lookup_keys": {name: list(keys) for name, keys in self.lookup_keys.items()},
+            "minhash": list(self.minhash),
+            "char_minhash": list(self.char_minhash),
+            "shingle_counts": [[value, count] for value, count in self.shingle_counts],
+            "shingle_total": self.shingle_total,
+            "shingle_unique": self.shingle_unique,
+            "passage_anchors": list(self.passage_anchors),
+            "page_hashes": [
+                {"page": item.page, "phash": item.phash, "lookup_keys": list(item.lookup_keys)}
+                for item in self.page_hashes
+            ],
+            "quality": {
+                "status": self.quality.status,
+                "text_evidence": self.quality.text_evidence,
+                "pages": [
+                    {
+                        "page": page.page,
+                        "source": page.source,
+                        "text_chars": page.text_chars,
+                        "native_text_chars": page.native_text_chars,
+                        "ocr_attempted": page.ocr_attempted,
+                        "ocr_reason": page.ocr_reason,
+                        "ocr_confidence": page.ocr_confidence,
+                        "warning": page.warning,
+                    }
+                    for page in self.quality.pages
+                ],
+                "warnings": list(self.quality.warnings),
+            },
+            "parameters": dict(self.parameters),
+        }
+
+    def to_json(self) -> str:
+        """Serialize the fingerprint as compact JSON.
+
+        Returns:
+            JSON string.
+        """
+        return json.dumps(self.to_dict(), ensure_ascii=False, separators=(",", ":"))

docfgp/visual.py

@@ -0,0 +1,72 @@
+"""Generate CPU perceptual hashes and indexed keys for page images."""
+
+from __future__ import annotations
+
+import math
+import statistics
+from typing import cast
+
+from PIL import Image, ImageOps
+
+_COSINES = tuple(
+    tuple(math.cos(math.pi * (2 * x + 1) * frequency / 64) for x in range(32))
+    for frequency in range(8)
+)
+
+
+def normalized_grayscale(image: Image.Image) -> Image.Image:
+    """Apply orientation and transparency before visual analysis.
+
+    Args:
+        image: One decoded page or image frame.
+
+    Returns:
+        Grayscale image with transparent pixels composited on white.
+    """
+    with ImageOps.exif_transpose(image) as oriented:
+        if oriented.mode in {"RGBA", "LA"} or "transparency" in oriented.info:
+            with oriented.convert("RGBA") as rgba, Image.new("RGBA", rgba.size, "white") as white:
+                with Image.alpha_composite(white, rgba) as composited:
+                    return composited.convert("L")
+        return oriented.convert("L")
+
+
+def page_phash(image: Image.Image) -> str | None:
+    """Hash a nonblank page after consistent visual normalization.
+
+    Args:
+        image: One decoded page or image frame.
+
+    Returns:
+        A 64-bit hash as 16 hexadecimal digits, or None for a blank page.
+    """
+    with normalized_grayscale(image) as grayscale:
+        low, high = cast(tuple[int, int], grayscale.getextrema())
+        if high - low < 5:
+            return None
+        with grayscale.resize((32, 32), Image.Resampling.LANCZOS) as reduced:
+            pixels = list(reduced.tobytes())
+    rows = [pixels[offset : offset + 32] for offset in range(0, 1024, 32)]
+    horizontal = [
+        [sum(row[x] * _COSINES[u][x] for x in range(32)) for u in range(8)] for row in rows
+    ]
+    coefficients = [
+        sum(horizontal[y][u] * _COSINES[v][y] for y in range(32))
+        for v in range(8)
+        for u in range(8)
+    ]
+    median = statistics.median(coefficients[1:])
+    bits = sum(1 << index for index, value in enumerate(coefficients) if value > median)
+    return f"{bits:016x}"
+
+
+def visual_keys(value: str) -> tuple[str, ...]:
+    """Split a 64-bit page hash into four indexed chunks.
+
+    Args:
+        value: Page hash returned by `page_phash`.
+
+    Returns:
+        Four versioned lookup keys.
+    """
+    return tuple(f"visual:v1:{index}:{value[index * 4 : (index + 1) * 4]}" for index in range(4))

docfgp/winnowing.py

@@ -0,0 +1,32 @@
+"""Select deterministic rightmost-minimum passage anchors."""
+
+from __future__ import annotations
+
+from collections import deque
+from collections.abc import Sequence
+
+
+def anchors(hashes: Sequence[str], window: int = 32) -> tuple[str, ...]:
+    """Winnow ordered shingle hashes with a rightmost minimum tie rule.
+
+    Args:
+        hashes: SHA-256 word-shingle hashes in document order.
+        window: Number of consecutive hashes per selection window.
+
+    Returns:
+        Distinct anchor hashes in first occurrence order.
+    """
+    if not hashes:
+        return ()
+    window = max(1, window)
+    queue: deque[int] = deque()
+    selected: dict[str, None] = {}
+    for index, value in enumerate(hashes):
+        while queue and hashes[queue[-1]] >= value:
+            queue.pop()
+        queue.append(index)
+        while queue and queue[0] <= index - window:
+            queue.popleft()
+        if index >= min(window, len(hashes)) - 1:
+            selected.setdefault(hashes[queue[0]], None)
+    return tuple(selected)

docfgp-0.1.0.dist-info/METADATA

@@ -0,0 +1,84 @@
+Metadata-Version: 2.5
+Name: docfgp
+Version: 0.1.0
+Summary: Generate deterministic fingerprints for documents
+License-Expression: MIT
+License-File: LICENSE
+Keywords: document,fingerprint,minhash,pdf
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Requires-Dist: pillow<14,>=10
+Requires-Dist: pypdfium2<6,>=4.30
+Provides-Extra: benchmark
+Requires-Dist: datasets<5,>=3; extra == 'benchmark'
+Requires-Dist: psutil>=6; extra == 'benchmark'
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.15; extra == 'dev'
+Requires-Dist: psutil>=6; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: reportlab>=4; extra == 'dev'
+Requires-Dist: ruff>=0.11; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# DocFGP
+
+[![PyPI version](https://img.shields.io/pypi/v/docfgp)](https://pypi.org/project/docfgp/)
+[![PyPI downloads](https://img.shields.io/pepy/dt/docfgp)](https://pepy.tech/projects/docfgp)
+[![CI](https://github.com/iamzices/docfgp-document-fingerprint/actions/workflows/ci.yml/badge.svg)](https://github.com/iamzices/docfgp-document-fingerprint/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/github/license/iamzices/docfgp-document-fingerprint)](LICENSE)
+
+DocFGP creates text and visual fingerprints for finding duplicates, near-duplicates, and shared passages across document formats.
+
+## Features
+
+- PDF, DOCX, PPTX, PNG, JPEG, TIFF, BMP, and WebP support
+- Exact, near-duplicate, and partial-match
+- OCR and visual fingerprints for scanned documents
+
+## Requirements
+
+- Python 3.11+
+- [office2pdf](https://github.com/developer0hye/office2pdf)
+- [Tesseract 5](https://github.com/tesseract-ocr/tesseract)
+
+## Installation
+
+```bash
+pip install docfgp
+```
+
+## Python API
+
+```python
+from docfgp import Fingerprinter
+
+fingerprint = Fingerprinter().fingerprint("report.pdf")
+record = fingerprint.to_dict()
+```
+
+| Field | Description |
+| --- | --- |
+| `schema_version` | Fingerprint data format version |
+| `algorithm_version` | Feature-generation version |
+| `source_format` | Detected input format |
+| `byte_size` | Original file size in bytes |
+| `page_count` | Number of pages or image frames |
+| `exact_hashes` | Raw and canonical-text SHA-256 hashes |
+| `lookup_keys` | Word, character, passage, and visual candidate keys |
+| `minhash` | Word-shingle MinHash sketch |
+| `char_minhash` | Character-shingle MinHash sketch |
+| `shingle_counts` | Hashed five-word shingle occurrence counts |
+| `shingle_total` | Total five-word shingle occurrences |
+| `shingle_unique` | Distinct five-word shingle hashes |
+| `passage_anchors` | Selected passage fingerprints |
+| `page_hashes` | Per-page visual hashes and lookup keys |
+| `quality` | Evidence status, page provenance, and warnings |
+| `parameters` | Parameters used to construct the fingerprint |
+
+## CLI
+
+```bash
+docfgp report.pdf > report.fingerprint.json
+```

docfgp-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+docfgp/__init__.py,sha256=TO51MIL6EqOStlTB8HgDvykcWYmVAYe-oLer5RMMCWE,261
+docfgp/cli.py,sha256=lydu7_tZ0igU0Wl74AZK6k1CVTdmVP2K11kjgHwiDm4,1691
+docfgp/extract.py,sha256=DUjIEqgvWqUaYyXakltbOhE0dGu-U6L6JHB2IP8gGlA,10777
+docfgp/fingerprint.py,sha256=BAdjlZEsX-YLbKZsScggvY9Xa9NjjDDk5NBW2ZAWerM,11507
+docfgp/minhash.py,sha256=WbrsVqtHoThlzam6Ppi0SgyqVMH99oGt4Q1XxX6S7p4,2432
+docfgp/normalize.py,sha256=F_hV5UChAuaymiv2LplutfjCd3wiXXMk3NiaHuncW9A,1219
+docfgp/ocr.py,sha256=l_ZwQUfgORTWG6Pk_7KGtky5JfBTeFmb4vixkyfoRrw,4294
+docfgp/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docfgp/schemas.py,sha256=MM_bKuHHx5KYFN_h-j9fmcT7cVNSjCxgDXwWpZPNVNw,8108
+docfgp/visual.py,sha256=5iw744jE2T2fed5zCYgMSs-dlIt8hhbJtvk5p39RksA,2386
+docfgp/winnowing.py,sha256=AsldFZJ-r4SXNVEQoD8xZjwksVyIH8F9hFvNTk__4sw,1026
+docfgp-0.1.0.dist-info/METADATA,sha256=orXHQeA3xWSmrrse31Wj2uatGJNBQnnmEceKzI-1Z6Q,2952
+docfgp-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+docfgp-0.1.0.dist-info/entry_points.txt,sha256=LmR7UIjgOuj_WM7dabj1SjS9h3OwEPCHC4HdlAKAum8,43
+docfgp-0.1.0.dist-info/licenses/LICENSE,sha256=0XIJv9rwK4ANSjTRJwHQixCR2c68CnPRC6i-horntwI,1062
+docfgp-0.1.0.dist-info/RECORD,,

docfgp-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

docfgp-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+docfgp = docfgp.cli:main

docfgp-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zices
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.