docfgp 0.1.0__tar.gz

docfgp-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+.venv/
+.tools/
+.benchmark/
+benchmark.json
+__pycache__/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+dist/
+*.egg-info/

docfgp-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zices
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docfgp-0.1.0/PKG-INFO

@@ -0,0 +1,84 @@
+Metadata-Version: 2.5
+Name: docfgp
+Version: 0.1.0
+Summary: Generate deterministic fingerprints for documents
+License-Expression: MIT
+License-File: LICENSE
+Keywords: document,fingerprint,minhash,pdf
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Requires-Dist: pillow<14,>=10
+Requires-Dist: pypdfium2<6,>=4.30
+Provides-Extra: benchmark
+Requires-Dist: datasets<5,>=3; extra == 'benchmark'
+Requires-Dist: psutil>=6; extra == 'benchmark'
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.15; extra == 'dev'
+Requires-Dist: psutil>=6; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: reportlab>=4; extra == 'dev'
+Requires-Dist: ruff>=0.11; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# DocFGP
+
+[![PyPI version](https://img.shields.io/pypi/v/docfgp)](https://pypi.org/project/docfgp/)
+[![PyPI downloads](https://img.shields.io/pepy/dt/docfgp)](https://pepy.tech/projects/docfgp)
+[![CI](https://github.com/iamzices/docfgp-document-fingerprint/actions/workflows/ci.yml/badge.svg)](https://github.com/iamzices/docfgp-document-fingerprint/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/github/license/iamzices/docfgp-document-fingerprint)](LICENSE)
+
+DocFGP creates text and visual fingerprints for finding duplicates, near-duplicates, and shared passages across document formats.
+
+## Features
+
+- PDF, DOCX, PPTX, PNG, JPEG, TIFF, BMP, and WebP support
+- Exact, near-duplicate, and partial-match
+- OCR and visual fingerprints for scanned documents
+
+## Requirements
+
+- Python 3.11+
+- [office2pdf](https://github.com/developer0hye/office2pdf)
+- [Tesseract 5](https://github.com/tesseract-ocr/tesseract)
+
+## Installation
+
+```bash
+pip install docfgp
+```
+
+## Python API
+
+```python
+from docfgp import Fingerprinter
+
+fingerprint = Fingerprinter().fingerprint("report.pdf")
+record = fingerprint.to_dict()
+```
+
+| Field | Description |
+| --- | --- |
+| `schema_version` | Fingerprint data format version |
+| `algorithm_version` | Feature-generation version |
+| `source_format` | Detected input format |
+| `byte_size` | Original file size in bytes |
+| `page_count` | Number of pages or image frames |
+| `exact_hashes` | Raw and canonical-text SHA-256 hashes |
+| `lookup_keys` | Word, character, passage, and visual candidate keys |
+| `minhash` | Word-shingle MinHash sketch |
+| `char_minhash` | Character-shingle MinHash sketch |
+| `shingle_counts` | Hashed five-word shingle occurrence counts |
+| `shingle_total` | Total five-word shingle occurrences |
+| `shingle_unique` | Distinct five-word shingle hashes |
+| `passage_anchors` | Selected passage fingerprints |
+| `page_hashes` | Per-page visual hashes and lookup keys |
+| `quality` | Evidence status, page provenance, and warnings |
+| `parameters` | Parameters used to construct the fingerprint |
+
+## CLI
+
+```bash
+docfgp report.pdf > report.fingerprint.json
+```

docfgp-0.1.0/README.md

@@ -0,0 +1,60 @@
+# DocFGP
+
+[![PyPI version](https://img.shields.io/pypi/v/docfgp)](https://pypi.org/project/docfgp/)
+[![PyPI downloads](https://img.shields.io/pepy/dt/docfgp)](https://pepy.tech/projects/docfgp)
+[![CI](https://github.com/iamzices/docfgp-document-fingerprint/actions/workflows/ci.yml/badge.svg)](https://github.com/iamzices/docfgp-document-fingerprint/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/github/license/iamzices/docfgp-document-fingerprint)](LICENSE)
+
+DocFGP creates text and visual fingerprints for finding duplicates, near-duplicates, and shared passages across document formats.
+
+## Features
+
+- PDF, DOCX, PPTX, PNG, JPEG, TIFF, BMP, and WebP support
+- Exact, near-duplicate, and partial-match
+- OCR and visual fingerprints for scanned documents
+
+## Requirements
+
+- Python 3.11+
+- [office2pdf](https://github.com/developer0hye/office2pdf)
+- [Tesseract 5](https://github.com/tesseract-ocr/tesseract)
+
+## Installation
+
+```bash
+pip install docfgp
+```
+
+## Python API
+
+```python
+from docfgp import Fingerprinter
+
+fingerprint = Fingerprinter().fingerprint("report.pdf")
+record = fingerprint.to_dict()
+```
+
+| Field | Description |
+| --- | --- |
+| `schema_version` | Fingerprint data format version |
+| `algorithm_version` | Feature-generation version |
+| `source_format` | Detected input format |
+| `byte_size` | Original file size in bytes |
+| `page_count` | Number of pages or image frames |
+| `exact_hashes` | Raw and canonical-text SHA-256 hashes |
+| `lookup_keys` | Word, character, passage, and visual candidate keys |
+| `minhash` | Word-shingle MinHash sketch |
+| `char_minhash` | Character-shingle MinHash sketch |
+| `shingle_counts` | Hashed five-word shingle occurrence counts |
+| `shingle_total` | Total five-word shingle occurrences |
+| `shingle_unique` | Distinct five-word shingle hashes |
+| `passage_anchors` | Selected passage fingerprints |
+| `page_hashes` | Per-page visual hashes and lookup keys |
+| `quality` | Evidence status, page provenance, and warnings |
+| `parameters` | Parameters used to construct the fingerprint |
+
+## CLI
+
+```bash
+docfgp report.pdf > report.fingerprint.json
+```

docfgp-0.1.0/docs/applications.md

@@ -0,0 +1,30 @@
+# Applications
+
+DocFGP generates fingerprints for document collections. The application stores them, indexes their keys, scores candidate pairs, and makes ingestion decisions.
+
+## Using a fingerprint in a collection
+
+1. Generate a fingerprint with `Fingerprinter().fingerprint(path)`.
+2. Inspect `quality.status`, page provenance, and warnings. An `insufficient_evidence` result supports raw-byte identity only.
+3. Store the serialized result with its schema and algorithm versions. Index the exact hashes and the relevant entries of `lookup_keys`.
+4. Retrieve candidate fingerprints through those indexes. Compare only compatible versions and calculate the evidence needed for the application's decision.
+
+## Questions the evidence can answer
+
+| Question | Relevant evidence | Interpretation |
+| --- | --- | --- |
+| Are the files byte-identical? | Raw SHA-256 | Equality establishes byte identity. |
+| Did extraction produce the same text across formats? | Nonempty canonical SHA-256 and extraction quality | Equality establishes identity of normalized extracted tokens; layout can differ. |
+| Are the documents broadly similar? | Word and character MinHash, reached through LSH keys | Sketch agreement estimates set resemblance. |
+| Is one document substantially contained in another? | Passage anchors, then hashed-shingle occurrence counts | Anchors retrieve candidates; directional counts quantify shared textual features. |
+| Do scanned pages look alike? | Per-page perceptual hashes and visual keys | Hamming distance describes visual similarity. |
+
+For example, if a document with 100 hashed five-word occurrences shares 95 occurrences with a document containing 1,000, coverage is 95% **from the smaller document's direction** and 9.5% from the larger document's direction. Coverage counts shared hashed five-word occurrences. Retain the counts in both fingerprints for this calculation; MinHash estimates set resemblance.
+
+## Decisions and limits
+
+Raw-hash equality establishes byte identity. Canonical-text equality establishes equality of extracted normalized tokens. Shared templates, repeated footers, and OCR errors can affect approximate and visual scores. The application should define its review or skip policy using the relevant modality, page coverage, and extraction quality.
+
+For a native PDF and its scanned counterpart, OCR text and page hashes provide separate retrieval routes. Their effectiveness depends on recognition and rendering quality. The available evidence and measured results are described in [Methodology](methodology.md) and [Evaluation](evaluation.md).
+
+Fingerprinting runs locally on the CPU. DOCX/PPTX conversion uses office2pdf; scanned-text recognition uses Tesseract. The application owns storage and retrieval.

docfgp-0.1.0/docs/evaluation.md

@@ -0,0 +1,55 @@
+# Evaluation
+
+This report covers two completed checks: cross-format retrieval on the office2pdf business golden corpus and validation of the published RVL-CDIP duplicate metadata. The retrieval figures below come from the Office corpus.
+
+## Office source-to-PDF retrieval
+
+The [business golden corpus](https://github.com/developer0hye/office2pdf/blob/main/tests/golden_mocks/business/README.md) pairs 10 DOCX and 10 PPTX files with PDF exports from Microsoft Office. The evaluator fingerprinted the 20 Office sources once, indexed their fingerprint evidence, then fingerprinted and queried the 20 PDF exports once each. The 20 manifest pairs were positives; the other 380 source/PDF pairs were treated as negatives.
+
+The run took place on **9 November 2026** on Windows with Python 3.14.7, office2pdf 0.10.0, Tesseract 5.5.3, and an AMD Ryzen 5 7500F.
+
+| Measure | Result |
+| --- | ---: |
+| Historical candidates / queries | 20 / 20 |
+| Positive / negative pairs | 20 / 380 |
+| Precision | 1.000 |
+| Recall | 1.000 |
+| Retrieval recall | 1.000 |
+| Recall@10 | 1.000 |
+| False-positive rate | 0.000 |
+| CPU / wall time | 15.08 / 14.09 seconds |
+| Sampled peak RAM | 150.24 MB |
+
+The corpus is small and synthetic. Office observations informed the final native-text provenance rule, so these numbers describe performance on a development corpus. Accuracy on other collections requires independent evaluation.
+
+## RVL-CDIP metadata validation
+
+The evaluator joined the `documents` and `pairs` configurations of [RVL-CDIP Duplicates](https://huggingface.co/datasets/stefan-hf/rvlcdip-duplicates) at revision `457749089e2b729d8b82f64166a379a0c6c4e21e`. The check validated query IDs, original image-path agreement, and consistency between each query's duplicate flag and its pair judgments.
+
+| Split | Queries with judgments | Positive pairs | Other judged pairs | Excluded query without judgments |
+| --- | ---: | ---: | ---: | ---: |
+| Validation | 40,000 | 69,172 | 330,828 | 0 |
+| Test | 39,999 | 68,290 | 331,700 | 1 |
+
+This check measured label and path consistency. The dataset's pairs cover ten same-class candidates per query selected by another system, and its duplicate labels are heuristic. The evaluator scores precision and false-positive rate on judged pairs and reports predictions on unjudged pairs separately. The accuracy figures above come from the Office retrieval run.
+
+## Retrieval and scoring protocol
+
+The corpus evaluators share a sequential protocol. They fingerprint each historical candidate once and retain fingerprint-derived evidence in a temporary disk-backed index. Each query is fingerprinted once. Exact hashes, MinHash bands, and visual keys retrieve candidates from that index. Candidate indexing and ranking use fingerprints; corpus labels enter after ranking to score the results. The temporary index is deleted after the run.
+
+For a retrieved pair, the text score is the larger of word and character MinHash agreement. Equal raw hashes or equal nonempty canonical hashes score 1. The visual score greedily assigns distinct page-hash pairs and divides their summed similarity by the larger page count. When every page on both sides has usable native text and OCR was not attempted, the text score takes precedence over visual layout. Otherwise the larger score is used. A score of at least **0.8** is counted as a predicted match. This threshold is an evaluator policy. These corpora measure document equivalence, so the evaluators omit shingle occurrence counts; applications can retain counts for directional overlap calculations.
+
+Precision is $TP/(TP+FP)$, recall is $TP/(TP+FN)$, and false-positive rate is $FP/(FP+TN)$. Retrieval recall counts labeled positives reached by any lookup key before score filtering. Recall@K counts labeled positives in the first K ranked candidates, divided by all labeled positives. The report includes raw pair counts to make each denominator visible.
+
+CPU time combines Python process time with sampled child-process time. Peak RAM is the maximum sampled resident memory of the process tree at 10 ms intervals; short-lived peaks can be missed. A document-processing failure stops a run instead of yielding a complete-looking report.
+
+## Reproducing the Office run
+
+Install the library with its development dependencies, obtain the business golden corpus and the office2pdf and Tesseract executables, then run from the repository root:
+
+```bash
+pip install ".[dev]"
+python -m scripts.evaluate_office /path/to/business --office2pdf /path/to/office2pdf --tesseract /path/to/tesseract
+```
+
+The command prints the metrics and counts as JSON. Record the corpus revision, command, hardware, operating system, Python version, and converter/OCR versions with any subsequent result. Corpus evaluations are separate from CI.

docfgp-0.1.0/docs/methodology.md

@@ -0,0 +1,65 @@
+# Methodology
+
+DocFGP computes a `DocumentFingerprint` for PDF, DOCX, PPTX, PNG, JPEG, TIFF, BMP, and WebP files.
+
+## From file to evidence
+
+| Stage | Implementation | Result |
+| --- | --- | --- |
+| Input validation | Format signature, file size, and Office package checks | Identified format or an explicit error |
+| Office conversion | [office2pdf](https://github.com/developer0hye/office2pdf) converts DOCX/PPTX to a temporary PDF | A common page representation |
+| Page extraction | [pypdfium2](https://github.com/pypdfium2-team/pypdfium2) extracts PDF text and renders pages; Pillow decodes images | Text, page images, and extraction provenance |
+| OCR when indicated | [Tesseract](https://github.com/tesseract-ocr/tesseract) runs on a bounded page image | Recognized text and mean word confidence |
+| Feature construction | SHA-256, shingles, MinHash, winnowing, and perceptual hashing | A versioned, serializable fingerprint |
+
+Pages are processed in order. Native PDF text is used when it has word tokens and valid decoded characters. OCR is attempted when native text is missing or invalid, or when an embedded image covers at least half of a page. When an image-heavy page also has native text, OCR replaces that text if its mean recognized-word confidence is at least 60 and it contributes at least three new tokens. For missing or invalid native text, accepted OCR needs confidence of at least 30. Page quality records OCR attempts, results, and warnings. These thresholds are deterministic text-selection rules.
+
+## Text representation
+
+Selected page text is normalized with Unicode NFKC, case folding, whitespace compaction, and joining of words divided by a line-break hyphen. Unicode word tokenization retains numbers. The **canonical text** is the ordered token sequence joined with spaces. A second representation removes punctuation and spacing from canonical text before character shingling. It tolerates some OCR spacing differences while retaining letters and digits.
+
+Reading order and OCR quality influence the token sequence. Canonical equality means that the *extracted and normalized tokens* match.
+
+## Fingerprint components
+
+| Component | Construction | Intended use |
+| --- | --- | --- |
+| Raw hash | SHA-256 of original bytes | Byte identity |
+| Canonical hash | SHA-256 of nonempty canonical text | Exact extracted-text identity across formats |
+| Word features | Consecutive five-token shingles, each hashed with SHA-256 | Text resemblance and occurrence counts |
+| Character features | Consecutive 13-character shingles of the tolerant text | Retrieval across some OCR differences |
+| MinHash | 512 deterministic components for each shingle set | Approximate set similarity |
+| LSH keys | 32 bands of 16 MinHash components per sketch | Indexed candidate retrieval |
+| Passage anchors | Rightmost minimum word-shingle hash in each 32-shingle window | Retrieval of shared passages |
+| Page hashes | 64-bit perceptual hash per nonblank rendered page; four 16-bit lookup keys | Visual candidate retrieval |
+
+Exact hashes and text-feature identifiers use SHA-256 for stable values across Python processes. MinHash permutations are derived from fixed seed material. Empty feature sets yield empty sketches and lookup-key groups. The current result uses schema version `1.1` and algorithm version `1.3`; it also records feature parameters, per-page provenance, warnings, and a quality status.
+
+For two documents with word-shingle sets $S_A$ and $S_B$, the fraction of equal MinHash components estimates set Jaccard similarity:
+
+$$
+J(S_A,S_B)=\frac{|S_A\cap S_B|}{|S_A\cup S_B|}.
+$$
+
+With 32 bands of 16 components, the usual independent-MinHash model gives a probability of at least one shared band of $1-(1-s^{16})^{32}$ at Jaccard similarity $s$. Empirical retrieval recall also depends on extraction quality and the implemented hash family. The sketch estimates *set* similarity and discards repetition. The underlying resemblance and containment approach is described by [Broder](https://www.cs.princeton.edu/courses/archive/spr05/cos598E/bib/broder97resemblance.pdf).
+
+Occurrence counts retain repetition. If $c_A(h)$ is the number of occurrences of shingle hash $h$ in A, the application can calculate
+
+$$
+I(A,B)=\sum_h\min(c_A(h),c_B(h)),\qquad
+C(A\mid B)=\frac{I(A,B)}{\sum_h c_A(h)}.
+$$
+
+Coverage is directional because $C(A\mid B)$ and $C(B\mid A)$ use different denominators. A value of 0.95 means that 95% of A's **hashed five-word shingle occurrences** also occur in B, up to hash collision. Set `include_shingle_counts=True` to retain the counts for this calculation.
+
+Winnowing provides a separate route to partial matches. A contiguous passage of at least 36 identical normalized words contains a complete window of 32 five-word shingles and therefore shares an anchor under this selection rule. Shorter or edited passages depend on the anchors they retain. Repeated boilerplate can also share anchors. This method measures shared surface text. See the [winnowing paper](https://www.cs.princeton.edu/courses/archive/spr05/cos598E/bib/p76-schleimer.pdf).
+
+Page perceptual hashes use a 32×32 grayscale discrete cosine transform. Their Hamming distance describes visual similarity independently of text coverage. Candidate retrieval through visual keys requires at least one matching 16-bit chunk; the application can then compare complete page hashes.
+
+## Limits and reproducibility
+
+Default limits are 50 MB per file, 200 pages, 8 million decoded pixels per page, 2 million extracted characters, and 200,000 distinct word or character shingles. PDF rendering uses 96 DPI for page hashes and up to 180 DPI for OCR, subject to the pixel limit; standalone images retain their source dimensions within that limit. External processes have a 60-second timeout; the document has a 180-second cooperative deadline. Configuration can change these limits.
+
+`quality.status` identifies the available text and visual evidence. `insufficient_evidence` retains the raw hash and has empty approximate lookup-key groups. Invalid documents and exceeded processing limits raise `FingerprintError`; OCR failure can leave a warning and a visual-only result. Temporary conversion and OCR files are removed after the call. The cooperative deadline is checked between in-process decoding stages; services accepting untrusted uploads should also apply operating-system resource limits.
+
+Fixed feature parameters and deterministic serialization support repeatable results. Changes in office2pdf, PDFium, Pillow, Tesseract, fonts, or OCR language can change extracted evidence, so experiments should record those versions and settings.

docfgp-0.1.0/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "docfgp"
+version = "0.1.0"
+description = "Generate deterministic fingerprints for documents"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = ["document", "fingerprint", "minhash", "pdf"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = ["pypdfium2>=4.30,<6", "Pillow>=10,<14"]
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "ruff>=0.11", "mypy>=1.15", "build>=1.2", "psutil>=6", "reportlab>=4"]
+benchmark = ["psutil>=6", "datasets>=3,<5"]
+
+[project.scripts]
+docfgp = "docfgp.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/docfgp"]
+
+[tool.hatch.build.targets.sdist]
+exclude = ["/.github", "/.gitattributes", "/uv.lock"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+extend-exclude = [".tools"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "D"]
+ignore = ["D203", "D213", "D413"]
+
+[tool.mypy]
+python_version = "3.11"
+files = ["src/docfgp", "scripts"]
+ignore_missing_imports = true

docfgp-0.1.0/scripts/__init__.py

@@ -0,0 +1 @@
+"""Standalone benchmark evaluators distributed with the source repository."""