docvortex 0.4.24__tar.gz → 0.5.0__tar.gz

docvortex-0.5.0/Cargo.lock

@@ -0,0 +1,138 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "docvortex-core"
+version = "0.1.0"
+
+[[package]]
+name = "docvortex-python"
+version = "0.1.0"
+dependencies = [
+ "docvortex-core",
+ "pyo3",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.28.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.28.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.28.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.28.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.28.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"

docvortex-0.5.0/Cargo.toml

@@ -0,0 +1,12 @@
+[workspace]
+members = ["rust/docvortex-core", "rust/docvortex-python"]
+resolver = "2"
+
+[profile.release]
+lto = "thin"
+codegen-units = 1
+
+[workspace.package]
+version = "0.1.0"
+edition = "2021"
+license = "MIT"

{docvortex-0.4.24 → docvortex-0.5.0}/MANIFEST.in

@@ -7,3 +7,7 @@ recursive-include tests *.py *.json *.pdf *.xor *.rtf
 recursive-include demo *.py *.md *.pdf *.doc *.docx *.ppt *.pptx *.xls *.xlsx *.rtf *.odt *.odp *.ods *.epub *.html *.mhtml *.ofd
 recursive-include src/docvortex/resources *
 global-exclude __pycache__ *.py[cod] .DS_Store
+include Cargo.toml Cargo.lock rust-toolchain.toml
+recursive-include rust *.toml *.rs
+include tools/verify_native_wheel.py tools/verify_native_abi.py
+global-exclude *.so *.pyd

{docvortex-0.4.24/src/docvortex.egg-info → docvortex-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docvortex
-Version: 0.4.24
+Version: 0.5.0
 Summary: A fast, multi-format document parsing and conversion engine
 License-Expression: MIT
 Project-URL: Repository, https://github.com/myhloli/DocVortex
@@ -201,6 +201,9 @@ reads metadata without parsing the document body.
 
 ## Development
 
+See [Repository architecture](docs/ARCHITECTURE.md) for directory responsibilities,
+the Python/Rust boundary, and pure Python or native development commands.
+
 From a local checkout:
 
 ```bash

{docvortex-0.4.24 → docvortex-0.5.0}/README.md

@@ -147,6 +147,9 @@ reads metadata without parsing the document body.
 
 ## Development
 
+See [Repository architecture](docs/ARCHITECTURE.md) for directory responsibilities,
+the Python/Rust boundary, and pure Python or native development commands.
+
 From a local checkout:
 
 ```bash

docvortex-0.5.0/docs/ARCHITECTURE.md

@@ -0,0 +1,138 @@
+# Repository architecture
+
+DocVortex is a Python document engine with optional Rust acceleration. The Python
+package owns the public SDK and document pipeline; the Rust crates implement private
+batch computations and their Python bindings. They are built and released together.
+
+## Repository layout
+
+| Location | Responsibility |
+| --- | --- |
+| `src/docvortex/` | Importable Python package and runtime resources |
+| `rust/docvortex-core/` | Python-independent numeric kernels and Rust contract tests |
+| `rust/docvortex-python/` | PyO3 argument validation, conversion, bindings and PDFium bridge |
+| Root `pyproject.toml`, `setup.py`, `MANIFEST.in` | Python metadata, optional extension build and source distribution |
+| Root `Cargo.toml`, `Cargo.lock`, `rust-toolchain.toml` | Shared Rust workspace, dependencies and compiler |
+| `tests/` | Python unit/integration tests, Python/Rust parity tests and benchmarks |
+| `tools/` | Development, regression review and distribution verification tools |
+| `docs/`, `schemas/`, `demo/` | Architecture/protocol documentation, JSON schemas and runnable examples |
+
+The Python `src` layout separates importable code from repository tools. It does not
+require other languages to live under `src`. The Cargo workspace at the repository
+root manages both crates, so Python and Rust development commands use the same working
+directory. Rust sources are included in the sdist; the compiled extension is installed
+as `docvortex._native`, alongside the Python package, rather than as a second SDK.
+
+## Document pipeline and ownership
+
+The public entry points live in `docvortex.api` and are re-exported by `docvortex`:
+
+```text
+source -> document preparation -> analyzers -> ModelJson
+       -> postprocess + asset materialization -> MiddleJson / DocumentResult
+       -> render -> RenderArtifact -> export / saved files
+```
+
+`parse` combines analysis and postprocessing; `convert` additionally renders and
+exports the result. The major package boundaries are:
+
+- `document`: source preparation, document access and PDF primitives/handle lifetime.
+- `analyzers`: format-specific interpretation and ModelJson construction.
+- `postprocess`: deterministic transformation into the common document representation.
+- `render`: output-format layout and rendering.
+- `export`: materialized files and result bundles.
+- `schema`, `result`, `assets`: document protocols, stage results and asset ownership.
+
+`analyzers/native` means parsing information already present in the source document.
+It is Python code and works without Rust. By contrast, `docvortex._native` is a private
+compiled extension. These two uses of “native” describe different responsibilities.
+
+## Optional compute backend
+
+```text
+Python document/PDF algorithms
+  -> _compute_backend.get_native()
+     -> Python reference implementation, or
+     -> docvortex._native (PyO3 validation and conversion)
+        -> docvortex-core (numeric batches)
+        -> Python-owned objects and parsing decisions
+```
+
+Python retains Unicode/text interpretation, public objects, algorithmic decisions and
+reference implementations. The core crate does not access Python objects or PDFium.
+Bindings validate transport records and preserve Python error and object-reuse
+semantics. Numeric work releases the GIL only at the existing explicit boundaries.
+
+The PDFium bridge is a separate boundary: it borrows function pointers and a text-page
+handle from the active pypdfium2 instance, under the existing Python guard and GIL.
+It does not load another PDFium library, take ownership of handles or close them.
+Subsequent numeric geometry conversion can release the GIL after reading finishes.
+
+`DOCVORTEX_COMPUTE_BACKEND=python|rust|auto` selects the backend once per process.
+`python` never imports the extension; `rust` requires a compatible extension; `auto`
+falls back on a missing or incompatible extension. Computation errors propagate.
+The current private protocol is **6**. Rebuild an editable extension after changing
+native sources and restart processes that cached the backend. Module-only refactors
+preserve the protocol and all Python-visible registrations.
+
+Within the binding crate, `lib.rs` declares modules and registers the existing flat
+Python extension interface. Rust modules do not become Python submodules:
+
+| Module | Responsibility |
+| --- | --- |
+| `geometry` | Character geometry, coordinate conversion, visual runs, anchor pairs and coordinate-object reuse |
+| `statistics` | Ordered clustering, typography and lane-gap statistics |
+| `tables` | Table kernels, column accumulation, row geometry and note-metric state |
+| `spatial` | Baseline indices, title gaps, neighboring lines and annotation geometry |
+| `scripts` | Script-role classification and inline pairing |
+| `dedup` | Duplicate-paint, hidden-text and mapping-run candidates |
+| `conversion` | Shared box extraction, tuple conversion and transport aliases |
+| `pdfium` | Borrowed PDFium calls and bounded character batches |
+
+Keep binding helpers internal to the crate. New functions and methods must include
+Chinese comments explaining their responsibility. Place numeric kernels in the core
+crate, Python transport handling in the bindings, and document decisions in Python.
+
+## Development and verification
+
+Run commands from the repository root in a virtual environment. For pure Python
+development (no native compilation):
+
+```sh
+DOCVORTEX_BUILD_NATIVE=0 python -m pip install -e ".[test,dev]"
+DOCVORTEX_COMPUTE_BACKEND=python python -m pytest -q
+```
+
+For release-mode native development:
+
+```sh
+DOCVORTEX_BUILD_NATIVE=1 python -m pip install -e ".[test,dev]"
+DOCVORTEX_COMPUTE_BACKEND=rust python -m pytest -q tests/test_compute_backend.py tests/test_native_parity.py tests/test_native_statistics.py tests/test_native_round3.py tests/test_native_round4.py tests/test_native_round5.py tests/test_pdfium_bridge.py tests/test_architecture.py
+cargo fmt --all --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test -p docvortex-core --locked
+```
+
+These environment assignments use POSIX shell syntax. In PowerShell, set the matching
+`$env:DOCVORTEX_BUILD_NATIVE` or `$env:DOCVORTEX_COMPUTE_BACKEND` before the command.
+For source-only Python runs, `PYTHONPATH=src` is available; installed-wheel validation
+must omit it and use a fresh environment without an editable DocVortex installation.
+Disabling native compilation does not remove a binary left by an earlier editable
+build; use the explicit Python runtime backend or a clean wheel installation.
+
+Use the existing parity suites for transport, numeric and object-identity contracts,
+and the core crate's tests for Python-independent kernels. Distribution checks must
+cover a complete sdist, a pure Python wheel and a native wheel built from that sdist.
+The existing CI owns the cross-platform matrix. See [Rust acceleration](rust-acceleration.md)
+for installed-wheel smoke checks, real-document comparisons and benchmark commands.
+
+## Layout rationale
+
+Keep `src/docvortex` and `rust` while Python remains the public product. Renaming `rust`
+to `crates` would not change these boundaries. Independent `python` and `rust` product
+roots become useful if Rust gains its own supported SDK and release lifecycle; that
+is not the current distribution model.
+
+References: [Python src layout](https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/),
+[Cargo workspaces](https://doc.rust-lang.org/cargo/reference/workspaces.html),
+[setuptools-rust mixed packages](https://setuptools-rust.readthedocs.io/en/latest/).

docvortex-0.5.0/docs/rust-acceleration.md

@@ -0,0 +1,234 @@
+# Optional Rust PDF kernels
+
+See [Repository architecture](ARCHITECTURE.md) for the Python/Rust ownership boundary
+and binding-module layout. The current private extension protocol is **6**, shared by
+the Python loader and Rust core. Protocol 4 and 5 references below describe historical
+interfaces or rollout stages, not the current loader requirement. Rebuild editable
+native installs and restart existing processes after updating native sources.
+
+DocVortex keeps its Python API, document types, parsing decisions and PDFium runtime.
+The private `docvortex._native` extension accelerates batches of already materialized
+numbers and indices. It does not bundle or call a second PDFium library and does not
+create a thread pool. The existing PDFium lock and handle lifetimes are unchanged.
+
+## Runtime selection
+
+Set `DOCVORTEX_COMPUTE_BACKEND` before the first PDF computation in a process:
+
+| Value | Behavior |
+| --- | --- |
+| `auto` (default) | Load a compatible native extension when available; otherwise use Python. |
+| `python` | Use the Python reference kernels without importing the extension. |
+| `rust` | Require the extension and its matching private protocol; missing or incompatible binaries raise an error. |
+
+Selection is cached for the process. Restart the process after changing the variable
+or rebuilding an editable extension. Computation errors are never silently retried
+in Python. The few explicit algorithmic reference paths preserve extreme coordinates,
+unusual Python inputs, and script-reference ties whose choice depends on Python set
+iteration. These paths do not weaken any parsing decision or output comparison.
+
+Python owns Unicode classification, normalization, font equality, version-dependent
+rounding, HTML generation and public object construction. Stable table-column means
+use an incremental accumulator matching the running CPython version's float-only
+`sum`: sequential for 3.10–3.11 and compensated for 3.12–3.14. Integer/mixed/custom
+inputs, other runtimes and intermediate overflow retain Python computation. Rust owns
+numeric batches and returns source ranges or indices. Canonical geometry reuses
+unchanged Python float objects and zero-rotation tuples to bound retained memory. Original `Char` references and
+copy-on-merge source-index behavior are retained. Small bounded caches contain only
+immutable Unicode features; they do not retain documents or use object addresses.
+
+## Kernels
+
+- Ordered statistics: exact median clustering for baseline and body-height groups;
+  stable first/last-cluster decisions return only source indices.
+- Typography and lanes: bounded font metadata reuse and batched numeric statistics;
+  original lane-member sorting and Python text/font-family classification remain.
+- Table text: stable visual rows and lazy per-recovery occupancy reuse without changing
+  candidates, interval boundary ownership or diagnostics.
+- Shared script geometry: baseline clusters, component membership and script roles.
+- Character geometry: clipping, rotation, visual run boundaries and canonical samples.
+- Character extraction: the private bridge borrows function pointers from the active
+  pypdfium2 instance and reads characters in one synchronous call under the original
+  lock and GIL. It owns no PDFium handles or callbacks and loads no second library.
+  Unsupported ctypes ABI/inputs use the reference reader with a recorded reason;
+  errors after entering the bridge propagate. Coordinate conversion can then release
+  the GIL. Raw numerical records stay within the call; Python records and coordinate
+  results are materialized in batches of at most 1,024. Font sharing, object/source IDs
+  and writing-direction assignment retain whole-page scope. The original protocol-4
+  list reader remains compatible. Wheel smoke checks require an actually completed
+  bridge call using bounded batches.
+- Deduplication: paint buckets, translated-run evidence, source components and hidden
+  OCR geometry. Unicode matching and final copies remain in Python.
+- Tables: region selection, batched rule coverage and merging, grid connectivity,
+  rectangular component checks and glyph-to-cell assignment. Candidate ordering,
+  confidence thresholds, diagnostics, primitive limits and HTML remain unchanged.
+- Dense pages: bounded interval queries replace the quadratic candidate-list fallback;
+  candidate consumption preserves original line indices. Rule-table candidates retain
+  compact interval drafts and share note statistics before stable score materialization.
+  Short-tail attachment sweeps preceding rows without repeatedly scanning whole lanes.
+- Visual assets: the existing rendering workers crop, orient and JPEG-encode images,
+  returning indexed encoded blocks instead of full-page pixels. Scheduling limits,
+  timeout/recovery, crop/encoding parameters and public asset bytes are preserved.
+  After a crop task closes its images, the worker collects the youngest Python GC
+  generation: closed pypdfium2 document/page reference cycles can otherwise retain
+  whole PDF inputs across tasks. This does not change PDFium handle ownership or
+  require allocator-specific memory trimming.
+
+## Development and distribution
+
+The Cargo workspace contains a Python-independent core crate and a PyO3 binding crate.
+`rust-toolchain.toml` pins the compiler; `Cargo.lock` pins dependencies. Build release
+kernels even for editable performance testing:
+
+```sh
+DOCVORTEX_BUILD_NATIVE=1 python -m pip install -e .
+DOCVORTEX_COMPUTE_BACKEND=rust python -m pytest tests/test_native_parity.py
+cargo test -p docvortex-core --locked
+cargo clippy --workspace --all-targets -- -D warnings
+```
+
+`DOCVORTEX_BUILD_NATIVE=0` builds a pure Python wheel. `auto` builds native code when
+Cargo is available and otherwise builds pure Python. If a compiler is available but
+compilation fails, installation fails visibly instead of publishing a broken binary.
+Official binary builds always set `DOCVORTEX_BUILD_NATIVE=1`.
+
+GIL-enabled CPython 3.10–3.14 uses `cp310-abi3` wheels for Linux x86_64/aarch64,
+Windows x86_64 and macOS arm64/x86_64. Linux binary builds target manylinux_2_28;
+older glibc systems can use the pure Python wheel. Other interpreters can build the
+pure package; this does not extend compatibility guarantees of its dependencies.
+Free-threaded native wheels are not part of this change.
+
+The native wheel workflow tests installed artifacts against Python output and exercises
+real parsing, both on Python 3.10 and 3.14. The regular CI matrix exercises Python and
+Rust backends on Python 3.10–3.14. The release workflow retains the main-CI/tag gate,
+then builds/tests every artifact before publishing all platform wheels, the pure wheel
+and the source distribution. A local build is not evidence that remote matrix jobs passed.
+
+On Intel macOS, the existing Magika/ONNX Runtime dependency stack currently prevents
+full application installation on Python 3.14. That wheel is tested with full parsing
+on Python 3.10 and 3.13, and its binary kernels are loaded and executed separately on
+Python 3.14. This ABI check is not a claim that the complete dependency stack installs
+there. The Rust change does not alter those existing dependency requirements.
+
+## Benchmarks and regression gates
+
+Use `PYTHONPATH=src` for a source checkout; omit it when validating an installed wheel.
+Every output directory must be new. Run CPU benchmarks without concurrent tests/builds.
+The independent process-tree memory auditor additionally requires `psutil`.
+
+```sh
+python tests/benchmarks/flash_pdf.py --backend python --runs 5 --profile --output output/rust/reference
+python tests/benchmarks/flash_pdf.py --backend rust --runs 5 --profile --output output/rust/native --baseline output/rust/reference
+python tests/benchmarks/native_pdf_table.py --backend python --runs 5 --output output/rust/table-python
+python tests/benchmarks/native_pdf_table.py --backend rust --runs 5 --output output/rust/table-rust --baseline output/rust/table-python
+python tests/benchmarks/rust_pdf.py --backend python --suite public --output output/rust/public-python
+python tests/benchmarks/rust_pdf.py --backend rust --suite public --output output/rust/public-rust --baseline output/rust/public-python
+python tests/benchmarks/rust_pdf.py --backend python --suite shared --flash-baseline output/rust/reference --output output/rust/shared-python
+python tests/benchmarks/rust_pdf.py --backend rust --suite shared --flash-baseline output/rust/reference --output output/rust/shared-rust --baseline output/rust/shared-python
+```
+
+The public/shared benchmark defaults to `caibao1`, `demo1` and `demo2`. Use repeatable
+`--path` arguments for other cases. Shared benchmarking consumes frozen baseline table
+regions, so a different candidate set cannot masquerade as acceleration. Timings include
+conversion into Rust and Python result construction. First invocation is separate from
+five warmed runs. Output comparison and memory sampling are outside timing. Public
+`parse` includes geometry, image materialization and postprocessing, but not rendering
+an export. Shared timings separate extraction, reuse, evidence and table recovery;
+they do not include model inference or represent MinerU end-to-end latency.
+
+Acceptance requires unchanged full outputs, related historical/manual tests, host API
+and primitive-reuse tests, and installed-wheel verification. Target a 30% reduction in
+each demo's public parse and shared-entry timing. Investigate sustained regressions
+above 5% in time or RSS. Never refresh a gold file or remove fields to achieve parity;
+record any remaining performance gap explicitly.
+
+
+Optional host replay lives outside the independent engine test tree:
+
+```sh
+python tools/mineru_native_replay.py --mineru-source ../Magic-PDF --backend python --flash-baseline output/rust/reference --output output/rust/host-python
+python tools/mineru_native_replay.py --mineru-source ../Magic-PDF --backend rust --flash-baseline output/rust/reference --output output/rust/host-rust --baseline output/rust/host-python
+python tests/benchmarks/compare_native_corpus.py --runs 5 --output output/rust/corpus
+```
+
+Host replay compares actual medium/high native stages with frozen model inputs and
+records fallback inputs and per-page extraction counts; it does not measure live inference.
+
+MinerU revisions whose scheduling tests mock the old full-page raster entry can use
+`python -m pytest -p tools.mineru_render_test_adapter -o addopts= HOST_TEST_PATH`.
+This explicit test-only plugin forwards that mock to the encoded-crop worker boundary;
+it retains page-selection, configuration, title-gating and image-closure assertions.
+The frozen third-round source reproduces the unadapted mock failures too. Real worker
+behavior is checked separately with engine tests and complete PDF/asset comparisons;
+the plugin does not change MinerU application code or production dispatch.
+
+
+Fourth-round kernels require private protocol 5. Rebuild editable native installs after
+updating Python sources; an older extension is rejected by forced Rust selection.
+Statistics preserve even-median arithmetic and stable tie order. Non-finite inputs use
+explicit reference paths. Occupancy caches are local to one table recovery and do not
+retain documents. Font metadata reuse is limited to one typography call and primitive
+dictionary values.
+
+Dense table-column clustering preserves first-match decisions and strict-prefix reuse,
+but no longer re-sums each cluster's complete history. Each corridor's anchors are
+prepared once; repeated row appearances still count as separate query positions.
+Table-note body heights use persistent counts indexed by vertical prefixes and stable
+height ranks. A core-row range may skip extra exclusion only when all occurrences of
+every source ID lie inside the exclusion band; other ranges use exact member filtering.
+Marker interpretation remains Python-owned and cached within one candidate build.
+Marker queries visit only untested rows in the current core range and stop on the
+first match. Two compact row bitmaps record tested and matching positions, with at
+most 128 recently used markers per corridor. Eviction recomputes evidence; it never
+removes candidates. No dense source-by-marker matrix of false results is populated.
+Existing first-source cache decisions are respected, including duplicate source IDs.
+Non-contiguous row selections retain the reference path.
+
+Row geometry indices preserve original ordering, including non-monotonic bottoms.
+They return coordinate source indices so Python reuses the original float objects;
+ties retain the earlier value, including signed zero. Candidate scoring, all interval
+candidates, annotation stopping rules and merge ordering are unchanged. Contexts keep
+strong row references only within the active call and are never global caches.
+
+Installed-wheel checks include `tests/test_native_round4.py`: every prefix's float bits,
+coverage, first-match clustering, quantile ranks, repeated sources, coordinate identity
+and indexed annotation chains are checked against the independent reference. The tests
+assert native state is actually used, not merely that an extension can be imported.
+
+The experimental native mapping pre-grouping kernel remains disabled in production:
+whole-entry measurements showed its packing cost exceeded its calculation benefit.
+It is retained for differential coverage. No additional environment switch is exposed.
+
+For four-way revision acceptance, freeze the old checkout with its matching binary and
+write `benchmark-revision.json` containing its `commit`. Then run:
+
+```sh
+python tests/benchmarks/compare_pdf_revisions.py --reference-source OLD_CHECKOUT --extra-path LOCAL_EXTERNAL.pdf --output output/rust/revisions
+python tools/generate_dense_pdf.py output/dense-table.pdf
+python tools/verify_native_wheel.py output/dense-table.pdf
+```
+
+The revision benchmark compares complete ModelJson, MiddleJson, diagnostics and asset
+hashes, records real source fingerprints, checkpoints every timing run, samples process
+tree RSS separately, and reverses order for regressions above 5%. Local external PDFs
+are identified by SHA256 and are not uploaded to CI. The wheel matrix instead generates
+a deterministic 120-row synthetic table and exercises both pypdfium2 5.10.1 and 5.13.0.
+
+Validation tools set the upstream `ORT_DISABLE_TELEMETRY=1` before importing ONNX Runtime
+and record the setting. API suppression alone cannot retract its initialization event
+([upstream details](https://github.com/microsoft/onnxruntime/blob/main/docs/Privacy.md)).
+This prevents a telemetry SDK shutdown race observed on the
+local macOS stack from masking completed PDF results; production runtime settings are
+unchanged. Failed runs remain excluded, and formal comparisons use the same telemetry
+setting for both revisions and backends. Regression retests reverse only affected pairs.
+
+RSS is collected in a separate process after one entrypoint warmup. Public parsing is
+sampled before model serialization; shared sampling retains the evidence snapshots but
+does not encode or hash JSON. Output equality is checked after sampling. The complete
+descendant tree includes rendering workers and multiprocessing helpers. This avoids
+mistaking allocator retention from full-output JSON validation for parser memory use.
+Existing timing runs can be audited without discarding their original memory records:
+
+```sh
+python tests/benchmarks/pdf_memory.py --timing-report output/rust/revisions/report.json --output output/rust/memory-audit
+```

{docvortex-0.4.24 → docvortex-0.5.0}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=77.0.3", "wheel"]
+requires = ["setuptools>=77.0.3", "wheel", "setuptools-rust>=1.11,<2"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -9,7 +9,7 @@ description = "A fast, multi-format document parsing and conversion engine"
 readme = "README.md"
 requires-python = ">=3.10,<3.15"
 license = "MIT"
-license-files = ["LICENSE.md"]
+license-files = ["LICENSE.md", "licenses/rust/*/LICENSE*", "licenses/rust/README.md"]
 classifiers = [
     "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.10",

docvortex-0.5.0/rust/docvortex-core/Cargo.toml

@@ -0,0 +1,5 @@
+[package]
+name = "docvortex-core"
+version.workspace = true
+edition.workspace = true
+license.workspace = true

docvortex-0.5.0/rust/docvortex-core/src/annotation_geometry.rs

@@ -0,0 +1,82 @@
+//! 表注片段按原顺序筛选并聚合，只返回原坐标来源，不创建新的浮点对象。
+use crate::geometry::Box4;
+use std::collections::{HashMap, HashSet};
+
+pub type Fragment = (usize, Box4, Box4);
+pub type Output = (Vec<(usize, [usize; 4], usize)>, Option<[usize; 4]>);
+
+pub struct AnnotationGeometry {
+    fragments: Vec<Fragment>,
+}
+
+impl AnnotationGeometry {
+    /// 保存走廊内只读片段，限制有限数值幅度以保证中间运算安全。
+    pub fn new(fragments: Vec<Fragment>) -> Option<Self> {
+        if fragments.iter().any(|f| {
+            f.1.iter()
+                .chain(f.2.iter())
+                .any(|v| !v.is_finite() || v.abs() > 1e100)
+        }) {
+            return None;
+        }
+        Some(Self { fragments })
+    }
+
+    /// 严格比较保留先出现的坐标来源，包括相等值与正负零。
+    fn combine(&self, a: [usize; 4], b: [usize; 4]) -> [usize; 4] {
+        let mut result = a;
+        for k in 0..4 {
+            let av = self.fragments[a[k]].1[k];
+            let bv = self.fragments[b[k]].1[k];
+            if (k < 2 && bv < av) || (k >= 2 && bv > av) {
+                result[k] = b[k];
+            }
+        }
+        result
+    }
+
+    /// 按当前候选的原片段序处理排除与重复来源，错误索引明确拒绝。
+    pub fn aggregate(
+        &self,
+        selected: Vec<usize>,
+        excluded: Vec<usize>,
+        local: Option<Box4>,
+    ) -> Option<Output> {
+        if selected.iter().any(|i| *i >= self.fragments.len())
+            || local.is_some_and(|b| b.iter().any(|v| !v.is_finite() || v.abs() > 1e100))
+        {
+            return None;
+        }
+        let excluded: HashSet<_> = excluded.into_iter().collect();
+        let mut positions = HashMap::new();
+        let mut lines: Vec<(usize, [usize; 4], usize)> = Vec::new();
+        for i in selected {
+            let (source, _, b) = self.fragments[i];
+            if excluded.contains(&source) {
+                continue;
+            }
+            if let Some(region) = local {
+                let smaller = (b[2] - b[0]).min(region[2] - region[0]);
+                let overlap = 0.0_f64.max(b[2].min(region[2]) - b[0].max(region[0]));
+                let ratio = if smaller > 0.0 {
+                    overlap / smaller
+                } else {
+                    0.0
+                };
+                if b[3] > region[1] && ratio >= 0.05 {
+                    continue;
+                }
+            }
+            if let Some(&position) = positions.get(&source) {
+                let line: &mut (usize, [usize; 4], usize) = &mut lines[position];
+                line.1 = self.combine(line.1, [i; 4]);
+                line.2 += 1;
+            } else {
+                positions.insert(source, lines.len());
+                lines.push((source, [i; 4], 1));
+            }
+        }
+        let union = lines.iter().map(|v| v.1).reduce(|a, b| self.combine(a, b));
+        Some((lines, union))
+    }
+}