autosxtract 0.1.0__tar.gz

autosxtract-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,107 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  quality:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint
+        run: |
+          ruff check .
+          ruff format --check .
+
+      - name: Types
+        run: mypy autosxtract
+        continue-on-error: true
+
+      # No OCR extra: the suite has to pass on the bare core, because that is
+      # how many people will install it. The tests that need an engine mark
+      # themselves as skipped.
+      - name: Tests (core)
+        run: pytest --cov=autosxtract --cov-report=term-missing
+
+      # Runs on EVERY push, not only on a release: it is cheap, and it is the
+      # only automatic defence against an accidental `git add -A` in a directory
+      # that sits next to the real documents the library extracts.
+      - name: Privacy scan
+        run: python scripts/privacy_check.py .
+
+  # Step 2 off Apple hardware, with the real engine. Kept separate because it
+  # downloads ONNX Runtime and the PP-OCRv6 weights — expensive to run across
+  # the whole matrix.
+  with-ocr:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install with the engine
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,paddle]"
+      - name: Integration tests
+        run: pytest -m "not apple" -q
+
+  # Apple's cascade can only be exercised on a macOS runner.
+  apple:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install with Vision
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,apple]"
+      - name: Diagnose
+        run: python -m autosxtract.cli diagnose
+      - name: Tests
+        run: pytest -q
+
+  packaging:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Build the wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      # Importing in a clean environment catches the commonest packaging
+      # error: a subpackage forgotten in the wheel, which only shows outside the
+      # repository.
+      - name: Install in a clean environment and import
+        run: |
+          python -m venv /tmp/clean
+          /tmp/clean/bin/pip install dist/*.whl
+          /tmp/clean/bin/python -c "import autosxtract; print(autosxtract.__version__)"
+          /tmp/clean/bin/autosxtract diagnose

autosxtract-0.1.0/.gitignore

@@ -0,0 +1,41 @@
+# ── dados e artefatos privados ───────────────────────────────────────────
+# Nenhum documento real entra no repositório. Os fixtures da suíte são
+# gerados na hora pelo PyMuPDF, com texto inventado.
+data/
+private/
+acervo/
+pdfs/
+saidas/
+*.pdf
+*.csv
+*.jsonl
+
+# ── pesos de modelo (baixados, nunca versionados) ────────────────────────
+*.onnx
+modelos/
+
+.env
+.env.*
+credentials*
+secrets*
+
+# ── exceções: fixtures sintéticos ────────────────────────────────────────
+!tests/fixtures/
+!tests/fixtures/**
+
+# ── python ───────────────────────────────────────────────────────────────
+__pycache__/
+*.pyc
+*.pyo
+.venv/
+venv/
+*.egg-info/
+build/
+dist/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+.DS_Store

autosxtract-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+# Install with: pre-commit install
+#
+# Order matters: the privacy scan runs FIRST, because it is the only one that
+# prevents a leak — the others only prevent a red CI. A commit blocked on
+# formatting costs thirty seconds; a real document published has no undo.
+repos:
+  - repo: local
+    hooks:
+      - id: privacy-check
+        name: privacy scan (staged)
+        entry: python scripts/privacy_check.py . --staged
+        language: system
+        pass_filenames: false
+        always_run: true
+
+      - id: ruff
+        name: ruff check
+        entry: ruff check --fix
+        language: system
+        types: [python]
+
+      # Without this hook, `ruff check` passes and CI fails on `ruff format
+      # --check`. They are different checks: the first looks at lint rules, the
+      # second at formatting.
+      - id: ruff-format
+        name: ruff format
+        entry: ruff format
+        language: system
+        types: [python]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      # Blocks any file above 1 MB. It is the net that catches the archive PDF
+      # that slipped past .gitignore — right extension, wrong directory.
+      - id: check-added-large-files
+        args: ["--maxkb=1024"]
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      # Stops a private file entering through a symlink pointing outside the
+      # repository.
+      - id: check-symlinks
+      - id: destroyed-symlinks
+      - id: detect-private-key

autosxtract-0.1.0/CLAUDE.md

@@ -0,0 +1,268 @@
+# AutosXtract — project rules
+
+This file is for whoever touches the code (person or agent). It records the
+decisions that are **not** obvious from the code and that have already cost
+something once.
+
+## 1. The default cascade does no networking, and networking is never accidental
+
+`Cascade()` assembles local steps only. No engine and no gate opens a socket.
+There are exactly two exceptions, and both are explicit:
+
+- `engines/models.py` downloads the PP-OCRv6 weights **once**, and extraction
+  works without it (falling back to rapidocr's embedded model).
+- `steps/remote.py` brings `DoclingStep` and `VLMStep`, which **require `url` in
+  the constructor**. They are not in the default cascade, there is no discovery
+  through an environment variable, no built-in default and no fallback to a
+  known endpoint.
+
+`steps/docling_local.py` is the useful counterexample: the same engine as
+`DoclingStep`, with no networking at all. It stays out of the default cascade
+for a **different** reason — it weighs ~2 GB of models and ~4 s per document —
+and that is why it lives outside `remote.py`. Confusing "expensive" with
+"remote" would make this paragraph's invariant meaningless.
+
+This is not a preference: the previous version of this pipeline reached the OCR
+engine on a Mac over a reverse SSH tunnel, and that worker going down
+**silently degraded the text** — 488 documents re-extracted down the worse path,
+19.5 min instead of 4.9, 28,239 characters lost, and nobody noticed until
+someone checked. A remote step nobody declared must not exist.
+
+The corollary that holds the rest together: **`Config` has not a single host,
+port, URL or credential field.** All networking lives in the constructor of a
+step somebody wrote by hand. `tests/test_config.py::test_no_field_points_at_a_network`
+and `tests/test_remote.py` are the guard rails.
+
+## 2. One acceptance criterion
+
+`quality/gate.py::evaluate` decides, for every step, whether the text suffices.
+Whoever decides the current step solved it and whoever decides the next one is
+worth paying for ask the same question, with the same code.
+
+Two competing notions of "adequate extraction" in one pipeline was the defect
+this function exists to avoid repeating: the step approved itself by one
+criterion and the cascade refused it by another.
+
+## 3. A missing engine is never an exception
+
+`available()` returns `(False, reason)`, the step goes inert and the cascade
+moves on. **The absence of a tool is not evidence about the document** —
+treating "I have no OCR" as "the page is empty" switches the pipeline off in
+silence.
+
+Corollary: degrading without breaking is right; degrading without warning is
+not. Every reason goes to the provenance and to `autosxtract diagnose`.
+
+## 4. Two different gates, and the difference is what happens to the refused
+
+**Acceptance gate** (`quality/gate.py`): "is this text good enough to stop the
+cascade?". Something refused here **stays in the contest** — it may be the best
+reading there is.
+
+**Replacement gate** (`quality/rejection.py`): runs only after an `expensive`
+step and asks "is it better than what I already had, and did it lose nothing?".
+Something refused here is **discarded**. Letting it compete would cancel the
+gate, because volume is usually on the wrong side: the corrupted text is
+precisely the longest one.
+
+Confusing the two is the easy mistake. The first compares against a threshold;
+the second compares against a concrete text and has already concluded the new
+one is worse.
+
+## 5. Refused text still competes
+
+`StepResult` separates the **verdict** (does the cascade stop?) from the
+**candidate** (does the text enter the contest?). A refused step may have
+produced the best reading there is. Discarding it left 682 documents with zero
+characters while the PDF had a text layer.
+
+## 6. PyMuPDF is serialised, and that is not negotiable
+
+It **crashes the process** with several threads: a segfault in
+`page_get_textpage`, captured with `faulthandler` and reproduced with 489 PDFs
+across 12 threads. `try/except` does not protect you — a segmentation fault is
+not a Python exception.
+
+Every access goes through `pdf/lock.py`. The measured cost is ~4% (37.2 s with 4
+threads against 38.6 s with 24). The useful parallelism is **per document**.
+
+The same goes for `get_image_rects` / `get_image_info`: `pdf/coverage.py` uses a
+single `get_text("dict")` traversal precisely because the per-image version
+segfaulted under concurrency.
+
+## 7. Engine confidence does not arbitrate quality
+
+Measured on 60 documents audited by four reviewers: engine confidence does not
+separate a good reading from an unsafe one — there was an unsafe document at
+confidence 100. It enters only as a floor against degenerate output, never as a
+criterion.
+
+## 8. Measured numbers go in the comment
+
+Every threshold in `config.py` carries the measurement that fixed it. Changing a
+number without measuring is the mistake this project tries to make difficult.
+
+And the methodological lesson, which is worth more than any specific number: **an
+isolated measurement has already lied here.** Turning off Vision's language
+correction improved anchors across 60 documents (+4) and worsened them across
+the whole cascade (−227), because the worse text failed the gate and fell to
+worse engines. What decides is the cascade's behaviour, not one engine's output.
+Use `scripts/compare_engines.py`, which measures both.
+
+## 9. No real document in the repository
+
+The fixtures are generated on the fly by PyMuPDF, with invented text
+(`tests/conftest.py`). `.gitignore` blocks `*.pdf` at the root. A failing test
+must point at the code, not at one specific archive file.
+
+That applies to **identifiers inside comments and docstrings** too. Documenting a
+measurement with the case number it was made on looks harmless and is not:
+`scripts/privacy_check.py` caught a real case number that had made its way into
+the examples here, with a valid check digit. The examples use numbers with an
+**invalid** check digit on purpose — that way the scanner stays quiet and nobody
+has to decide case by case whether a number exists.
+
+The scanner runs in `pre-commit` (the first hook, before the style ones) and on
+every CI push. It validates tax IDs, company IDs and case numbers by their check
+digit: precision matters more than raw recall, because a noisy scanner gets
+switched off.
+
+Watch out for one trap: `pre-commit` stashes unstaged changes before running. It
+inspects **what will be committed**, not what is in your working tree — fixing a
+file without `git add` leaves the hook looking at the old version.
+
+## 10. Layers
+
+```
+pdf/         knows only the file    — does not know what an engine is
+quality/     knows only text        — does no I/O
+engines/     reads pixels           — does not know what a cascade is
+steps/       composes the two       — does not know which engine is behind it
+cascade.py   orchestrates
+```
+
+If an import crosses the arrow backwards, the design has broken. It was that
+boundary that let the OCR step be **a single one**, generic, for Vision and
+PP-OCRv6 alike.
+
+## 11. The platform decides twice, and both are necessary
+
+    install    a PEP 508 marker in `pyproject.toml`, evaluated by pip
+    runtime    `platform.py` plus the registry in `engines/base.py`
+
+The first makes `pip install autosxtract` bring Vision on macOS and PP-OCRv6
+elsewhere, with no extra. The second is what stops the library from blowing up
+when the marker did not match — an image built for another platform,
+`--no-deps`, a lockfile for another `sys_platform`, an incomplete pyobjc.
+
+A temptation to resist: deleting the second layer because "the first already
+guarantees it". It does not guarantee it — it guarantees the common case.
+`tests/test_packaging.py` pins the first; `tests/test_engines.py` pins the second.
+
+## 12. English in the code, Portuguese in the patterns
+
+Names, docstrings and messages are in English, so the library is usable outside
+Brazil. The **regexes and word lists stay in Portuguese**, because they describe
+the corpus: the conformity stamp, the enclitic pronouns, the abbreviations, the
+identity-card markers, the legal vocabulary of the lexicon.
+
+That is the adaptation seam. Anyone porting the library to another language
+swaps `quality/stamp.py`, `quality/lexicon.py`, `quality/prose.py` and
+`quality/screening.py` — the code around them does not change.
+
+## 13. What runs BEFORE the expensive step
+
+`quality/vetoes.py` holds the five vetoes, and the order is by rising cost:
+pixel statistics at 40 DPI (ms), then a real local OCR (~1 s), then comparing
+text already read (free).
+
+Two warnings that have already cost time:
+
+- **The first two are only valid together with "extracted no text".** On their
+  own they would discard an old photocopy on dark paper, which is continuous
+  tone and carries thousands of legitimate characters (measured: 0.99 / 0.99 /
+  0.83 mid-tone with 1,001, 2,612 and 632 characters).
+- **The witness has to be of another architecture.** A second engine of the same
+  family is not independent evidence, and the agreement veto stops meaning what
+  it says. That is why `veto_engine` points at Tesseract and not at a second
+  PP-OCR.
+
+`local_reading=None` means "I don't know" and skips the last three vetoes. It
+never becomes "there is no text".
+
+## 14. Parallelism is decided by the machine, not by the code
+
+The three fields accept `None` = "decide here", and that is the default. An
+explicit number is obeyed; what it is not, is a promise.
+
+    threads   72 cores     2 cores
+       1       1.36 pg/s   1.44 pg/s
+       2       1.74        1.68     <- plateau
+       4       1.99        1.58
+       8       2.18        1.54     <- worse than 2 threads
+      16       2.27        1.71
+
+Three things that have bitten and are pinned in code:
+
+- **`os.cpu_count()` lies inside a container.** It reports the host, not the
+  quota. `resources.cores()` crosses affinity, cgroup v1/v2 and `cpu_count` and
+  keeps the smallest — none of the three alone covers `taskset` AND `--cpus`.
+- **The product multiplies silently.** `documents × pages` reaches 32 pages in
+  flight from values that look modest. The aggregate cap cuts the PAGES, never
+  the documents: cutting documents raises total time predictably, cutting pages
+  costs almost nothing.
+- **The engine has the last word.** Whoever configures the cascade does not know
+  whether a hardware queue sits behind it. `OCREngine.scales_with_threads =
+  False` makes the engine use 1 thread, and `OCRStep` records the effective
+  value in the provenance when it differs from the requested one — clamping
+  silently would be the same antipattern as section 1.
+
+Resolution is a **method**, not a field computed in the constructor: the machine
+that resolves may not be the one that serialised the configuration.
+
+## 15. The engine contract has two levels, and the detailed one is optional
+
+`transcribe_page` returns `(text, confidence)`; `read_page` returns it line by
+line, with polygon and score. The second is **optional** — `None` is the honest
+answer from an engine without geometry, and the cascade falls back to the first.
+
+Except that without it there are no containment layers, and they are the
+pipeline's cheapest measured gain (entity recall 0.902 → 0.921, and latency
+FALLS). When writing a new engine, implement the detailed contract if the
+backend exposes geometry.
+
+`Transcription.pages` is only filled when **every** page answered in detail. A
+partial list would make the layers operate on a different document from the one
+transcribed, and the page index would stop lining up.
+
+## 16. Never reuse the main OCR instance to recognise a crop
+
+Calling the main `rapidocr` with `use_det=False` turns detection off
+**permanently on that object**. Measured: the next whole-page read returned 1
+line where it had returned 56, and the document came out with 1 character
+instead of 3,900.
+
+The defect is of the worst kind — silent, order-dependent, and only visible from
+the second page of the batch, because the first still uses a clean object. It
+passed a whole test suite without showing; it only appeared on measuring
+document by document.
+
+`PaddleEngine._recognizer()` keeps a **separate** instance for Layer 2.
+`tests/test_paddle.py` pins that.
+
+The general corollary: a third-party library with per-instance state is a shared
+resource. If one path changes configuration, that path needs its own object.
+
+## 17. A visual detector does not decide on its own
+
+The signature detector (YOLO) was measured on a real archive: 19% of pages with
+a detection, most of them **false positives** on seals, stamps, logos, QR codes
+and coats of arms. A model that is good on a public benchmark can be useless in
+your domain, and the way to find out is to run it on your archive, not to read
+the mAP.
+
+The answer was neither to throw the detector away nor to trust it: it was to
+**cross it with the text**. A box only counts if some overlapping line is
+illegible, and it is discarded if any overlapping line is stamp text. That is the
+pattern to repeat whenever a visual signal enters the pipeline — alone it errs,
+crossed with what is already known it helps.

autosxtract-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 AutosXtract
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

autosxtract-0.1.0/Makefile

@@ -0,0 +1,17 @@
+PY := .venv/bin/python
+
+.PHONY: test lint fmt typecheck diagnose privacy hooks all
+
+test:      ; $(PY) -m pytest
+# BOTH commands, as in CI: `ruff check` does not verify formatting, and running
+# only it leaves CI red with the local tree green.
+lint:      ; $(PY) -m ruff check . && $(PY) -m ruff format --check .
+fmt:       ; $(PY) -m ruff format .
+typecheck: ; $(PY) -m mypy autosxtract
+# Prints what THIS machine can run. Worth more than any log when an extraction
+# result surprises you.
+diagnose:  ; $(PY) -m autosxtract.cli diagnose
+# The only check that prevents a leak rather than merely a red CI.
+privacy:   ; $(PY) scripts/privacy_check.py .
+hooks:     ; $(PY) -m pre_commit install
+all: lint typecheck test privacy