noirdoc 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. noirdoc-0.1.0/.github/workflows/ci.yml +46 -0
  2. noirdoc-0.1.0/.github/workflows/release.yml +130 -0
  3. noirdoc-0.1.0/.gitignore +69 -0
  4. noirdoc-0.1.0/.pre-commit-config.yaml +33 -0
  5. noirdoc-0.1.0/.python-version +1 -0
  6. noirdoc-0.1.0/CHANGELOG.md +40 -0
  7. noirdoc-0.1.0/CONTRIBUTING.md +63 -0
  8. noirdoc-0.1.0/LICENSE +21 -0
  9. noirdoc-0.1.0/PKG-INFO +221 -0
  10. noirdoc-0.1.0/README.md +171 -0
  11. noirdoc-0.1.0/SECURITY.md +33 -0
  12. noirdoc-0.1.0/docs/RELEASING.md +93 -0
  13. noirdoc-0.1.0/docs/assets/banner.svg +41 -0
  14. noirdoc-0.1.0/docs/assets/noirdoc-logo-light-xxl.png +0 -0
  15. noirdoc-0.1.0/pyproject.toml +124 -0
  16. noirdoc-0.1.0/src/noirdoc/__init__.py +12 -0
  17. noirdoc-0.1.0/src/noirdoc/_version.py +24 -0
  18. noirdoc-0.1.0/src/noirdoc/cli.py +255 -0
  19. noirdoc-0.1.0/src/noirdoc/detection/__init__.py +0 -0
  20. noirdoc-0.1.0/src/noirdoc/detection/base.py +44 -0
  21. noirdoc-0.1.0/src/noirdoc/detection/ensemble.py +170 -0
  22. noirdoc-0.1.0/src/noirdoc/detection/flair_detector.py +57 -0
  23. noirdoc-0.1.0/src/noirdoc/detection/flair_recognizer.py +69 -0
  24. noirdoc-0.1.0/src/noirdoc/detection/gliner_detector.py +89 -0
  25. noirdoc-0.1.0/src/noirdoc/detection/model_manager.py +26 -0
  26. noirdoc-0.1.0/src/noirdoc/detection/presidio_detector.py +234 -0
  27. noirdoc-0.1.0/src/noirdoc/file_analysis/__init__.py +4 -0
  28. noirdoc-0.1.0/src/noirdoc/file_analysis/body_walker.py +459 -0
  29. noirdoc-0.1.0/src/noirdoc/file_analysis/extractor.py +81 -0
  30. noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/__init__.py +0 -0
  31. noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/docx_ext.py +27 -0
  32. noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/ocr.py +33 -0
  33. noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/pdf.py +79 -0
  34. noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/plain.py +8 -0
  35. noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/xlsx.py +23 -0
  36. noirdoc-0.1.0/src/noirdoc/file_analysis/mime.py +70 -0
  37. noirdoc-0.1.0/src/noirdoc/file_analysis/models.py +45 -0
  38. noirdoc-0.1.0/src/noirdoc/file_analysis/pipeline.py +263 -0
  39. noirdoc-0.1.0/src/noirdoc/file_analysis/policy.py +28 -0
  40. noirdoc-0.1.0/src/noirdoc/file_analysis/reconstruction.py +192 -0
  41. noirdoc-0.1.0/src/noirdoc/file_analysis/xlsx_inference.py +229 -0
  42. noirdoc-0.1.0/src/noirdoc/file_reidentification/__init__.py +0 -0
  43. noirdoc-0.1.0/src/noirdoc/file_reidentification/service.py +157 -0
  44. noirdoc-0.1.0/src/noirdoc/mappings/__init__.py +0 -0
  45. noirdoc-0.1.0/src/noirdoc/mappings/backends/__init__.py +20 -0
  46. noirdoc-0.1.0/src/noirdoc/mappings/backends/base.py +25 -0
  47. noirdoc-0.1.0/src/noirdoc/mappings/backends/file.py +88 -0
  48. noirdoc-0.1.0/src/noirdoc/mappings/backends/memory.py +44 -0
  49. noirdoc-0.1.0/src/noirdoc/mappings/backends/redis_backend.py +30 -0
  50. noirdoc-0.1.0/src/noirdoc/mappings/hydration.py +48 -0
  51. noirdoc-0.1.0/src/noirdoc/mappings/store.py +130 -0
  52. noirdoc-0.1.0/src/noirdoc/namespace.py +99 -0
  53. noirdoc-0.1.0/src/noirdoc/pseudonymization/__init__.py +0 -0
  54. noirdoc-0.1.0/src/noirdoc/pseudonymization/engine.py +24 -0
  55. noirdoc-0.1.0/src/noirdoc/pseudonymization/mapper.py +83 -0
  56. noirdoc-0.1.0/src/noirdoc/pseudonymization/system_prompt.py +78 -0
  57. noirdoc-0.1.0/src/noirdoc/reidentification/__init__.py +0 -0
  58. noirdoc-0.1.0/src/noirdoc/reidentification/engine.py +105 -0
  59. noirdoc-0.1.0/src/noirdoc/sdk.py +339 -0
  60. noirdoc-0.1.0/tests/__init__.py +0 -0
  61. noirdoc-0.1.0/tests/file_analysis/__init__.py +0 -0
  62. noirdoc-0.1.0/tests/file_analysis/test_body_walker.py +353 -0
  63. noirdoc-0.1.0/tests/file_analysis/test_extractor.py +73 -0
  64. noirdoc-0.1.0/tests/file_analysis/test_mime.py +64 -0
  65. noirdoc-0.1.0/tests/file_analysis/test_pdf_extractor.py +174 -0
  66. noirdoc-0.1.0/tests/file_analysis/test_pipeline.py +334 -0
  67. noirdoc-0.1.0/tests/file_analysis/test_policy.py +25 -0
  68. noirdoc-0.1.0/tests/file_reidentification/__init__.py +0 -0
  69. noirdoc-0.1.0/tests/file_reidentification/test_service.py +175 -0
  70. noirdoc-0.1.0/tests/mappings/__init__.py +0 -0
  71. noirdoc-0.1.0/tests/mappings/test_backends.py +80 -0
  72. noirdoc-0.1.0/tests/mappings/test_hydration.py +166 -0
  73. noirdoc-0.1.0/tests/mappings/test_store.py +153 -0
  74. noirdoc-0.1.0/tests/pseudonymization/__init__.py +0 -0
  75. noirdoc-0.1.0/tests/pseudonymization/test_system_prompt.py +110 -0
  76. noirdoc-0.1.0/tests/test_ensemble.py +302 -0
  77. noirdoc-0.1.0/tests/test_flair_recognizer.py +53 -0
  78. noirdoc-0.1.0/tests/test_gliner_detector.py +87 -0
  79. noirdoc-0.1.0/tests/test_integration_pipeline.py +90 -0
  80. noirdoc-0.1.0/tests/test_mapper.py +120 -0
  81. noirdoc-0.1.0/tests/test_model_manager.py +37 -0
  82. noirdoc-0.1.0/tests/test_namespace.py +98 -0
  83. noirdoc-0.1.0/tests/test_presidio_detector.py +159 -0
  84. noirdoc-0.1.0/tests/test_pseudonymization.py +83 -0
  85. noirdoc-0.1.0/tests/test_reidentification.py +235 -0
@@ -0,0 +1,46 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ concurrency:
12
+ group: ${{ github.workflow }}-${{ github.ref }}
13
+ cancel-in-progress: true
14
+
15
+ jobs:
16
+ lint:
17
+ name: Lint (pre-commit)
18
+ runs-on: ubuntu-latest
19
+ timeout-minutes: 10
20
+ steps:
21
+ - uses: actions/checkout@v4
22
+ - uses: actions/setup-python@v5
23
+ with:
24
+ python-version: "3.12"
25
+ - uses: pre-commit/action@v3.0.1
26
+
27
+ test:
28
+ name: Tests (Python ${{ matrix.python-version }})
29
+ runs-on: ubuntu-latest
30
+ timeout-minutes: 15
31
+ strategy:
32
+ fail-fast: false
33
+ matrix:
34
+ python-version: ["3.12", "3.13"]
35
+ steps:
36
+ - uses: actions/checkout@v4
37
+ - uses: actions/setup-python@v5
38
+ with:
39
+ python-version: ${{ matrix.python-version }}
40
+ cache: "pip"
41
+ - name: Install baseline + dev deps
42
+ run: pip install -e ".[dev]"
43
+ - name: Download spaCy model
44
+ run: python -m spacy download de_core_news_lg
45
+ - name: Run tests
46
+ run: pytest -m "not slow" --tb=short
@@ -0,0 +1,130 @@
1
+ name: Release
2
+
3
+ # Tag-triggered publish. Pre-release tags (a/b/rc/dev) go to TestPyPI;
4
+ # final tags (vX.Y.Z, optionally .postN) go to PyPI. Auth via PyPI Trusted
5
+ # Publishing (OIDC) — no secrets stored in GitHub. See docs/RELEASING.md.
6
+
7
+ on:
8
+ push:
9
+ tags: ["v*"]
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ concurrency:
15
+ group: release-${{ github.ref }}
16
+ cancel-in-progress: false
17
+
18
+ jobs:
19
+ determine:
20
+ name: Determine channel
21
+ runs-on: ubuntu-latest
22
+ timeout-minutes: 2
23
+ outputs:
24
+ channel: ${{ steps.pick.outputs.channel }}
25
+ repository-url: ${{ steps.pick.outputs.repository-url }}
26
+ prerelease: ${{ steps.pick.outputs.prerelease }}
27
+ steps:
28
+ - id: pick
29
+ env:
30
+ TAG: ${{ github.ref_name }}
31
+ run: |
32
+ if [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?$ ]]; then
33
+ echo "channel=pypi" >> "$GITHUB_OUTPUT"
34
+ echo "repository-url=https://upload.pypi.org/legacy/" >> "$GITHUB_OUTPUT"
35
+ echo "prerelease=false" >> "$GITHUB_OUTPUT"
36
+ else
37
+ echo "channel=testpypi" >> "$GITHUB_OUTPUT"
38
+ echo "repository-url=https://test.pypi.org/legacy/" >> "$GITHUB_OUTPUT"
39
+ echo "prerelease=true" >> "$GITHUB_OUTPUT"
40
+ fi
41
+
42
+ build:
43
+ name: Build distribution
44
+ runs-on: ubuntu-latest
45
+ needs: determine
46
+ timeout-minutes: 10
47
+ steps:
48
+ - uses: actions/checkout@v4
49
+ with:
50
+ fetch-depth: 0 # hatch-vcs needs the full history to resolve the tag
51
+ - uses: actions/setup-python@v5
52
+ with:
53
+ python-version: "3.12"
54
+ - name: Install build tooling
55
+ run: pip install --upgrade build
56
+ - name: Build sdist and wheel
57
+ run: python -m build
58
+ - name: Confirm version matches tag
59
+ env:
60
+ TAG: ${{ github.ref_name }}
61
+ run: |
62
+ expected="${TAG#v}"
63
+ built=$(ls dist/*.whl | head -n1 | sed -E 's|.*/noirdoc-([^-]+)-.*|\1|')
64
+ echo "Tag expects: $expected"
65
+ echo "Wheel reports: $built"
66
+ test "$expected" = "$built"
67
+ - uses: actions/upload-artifact@v4
68
+ with:
69
+ name: dist
70
+ path: dist/
71
+ if-no-files-found: error
72
+
73
+ publish:
74
+ name: Publish to ${{ needs.determine.outputs.channel }}
75
+ runs-on: ubuntu-latest
76
+ needs: [determine, build]
77
+ timeout-minutes: 10
78
+ environment:
79
+ name: ${{ needs.determine.outputs.channel }}
80
+ permissions:
81
+ id-token: write # OIDC for Trusted Publishing
82
+ attestations: write # PEP 740 build provenance
83
+ contents: read
84
+ steps:
85
+ - uses: actions/download-artifact@v4
86
+ with:
87
+ name: dist
88
+ path: dist/
89
+ - name: Publish
90
+ uses: pypa/gh-action-pypi-publish@release/v1
91
+ with:
92
+ repository-url: ${{ needs.determine.outputs.repository-url }}
93
+ attestations: true
94
+
95
+ github-release:
96
+ name: Create GitHub Release
97
+ runs-on: ubuntu-latest
98
+ needs: [determine, publish]
99
+ timeout-minutes: 5
100
+ permissions:
101
+ contents: write
102
+ steps:
103
+ - uses: actions/checkout@v4
104
+ - uses: actions/download-artifact@v4
105
+ with:
106
+ name: dist
107
+ path: dist/
108
+ - name: Extract changelog section for this tag
109
+ env:
110
+ TAG: ${{ github.ref_name }}
111
+ run: |
112
+ version="${TAG#v}"
113
+ # Pull the section between `## [<version>]` and the next `## [`.
114
+ awk -v ver="$version" '
115
+ $0 ~ "^## \\[" ver "\\]" {found=1; next}
116
+ found && /^## \[/ {exit}
117
+ found {print}
118
+ ' CHANGELOG.md > release-notes.md
119
+ if [ ! -s release-notes.md ]; then
120
+ echo "Release $TAG — see CHANGELOG.md for details." > release-notes.md
121
+ fi
122
+ - name: Create release
123
+ uses: softprops/action-gh-release@v2
124
+ with:
125
+ tag_name: ${{ github.ref_name }}
126
+ name: ${{ github.ref_name }}
127
+ body_path: release-notes.md
128
+ prerelease: ${{ needs.determine.outputs.prerelease }}
129
+ files: dist/*
130
+ fail_on_unmatched_files: true
@@ -0,0 +1,69 @@
1
+ # Byte-compiled / optimized
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+
8
+ # Virtual environments
9
+ .venv/
10
+ venv/
11
+ env/
12
+ .env
13
+ .env.*
14
+ !.env.example
15
+
16
+ # Packaging
17
+ build/
18
+ dist/
19
+ *.egg-info/
20
+ *.egg
21
+ wheels/
22
+ .eggs/
23
+ pip-wheel-metadata/
24
+
25
+ # Lock files (library — downstream should re-resolve)
26
+ poetry.lock
27
+
28
+ # Test / type / coverage artefacts
29
+ .pytest_cache/
30
+ .ruff_cache/
31
+ .mypy_cache/
32
+ .pyre/
33
+ .pytype/
34
+ .tox/
35
+ .nox/
36
+ .hypothesis/
37
+ .coverage
38
+ .coverage.*
39
+ coverage.xml
40
+ htmlcov/
41
+
42
+ # OS
43
+ .DS_Store
44
+ Thumbs.db
45
+ ehthumbs.db
46
+ Desktop.ini
47
+
48
+ # Editors / IDEs
49
+ .idea/
50
+ .vscode/
51
+ *.swp
52
+ *.swo
53
+ *~
54
+ *.bak
55
+
56
+ # Jupyter
57
+ .ipynb_checkpoints/
58
+
59
+ # Local model cache and user data
60
+ .noirdoc/
61
+
62
+ # Build backends
63
+ .hatch/
64
+
65
+ # hatch-vcs generated version file
66
+ src/noirdoc/_version.py
67
+
68
+ # Claude Code working notes (brainstorm plans + specs)
69
+ docs/superpowers/
@@ -0,0 +1,33 @@
1
+ repos:
2
+ - repo: https://github.com/pre-commit/pre-commit-hooks
3
+ rev: v6.0.0
4
+ hooks:
5
+ - id: trailing-whitespace
6
+ - id: end-of-file-fixer
7
+ - id: check-yaml
8
+ - id: debug-statements
9
+ - id: name-tests-test
10
+ args: [--pytest-test-first]
11
+ - repo: https://github.com/asottile/add-trailing-comma
12
+ rev: v4.0.0
13
+ hooks:
14
+ - id: add-trailing-comma
15
+ - repo: https://github.com/asottile/pyupgrade
16
+ rev: v3.21.2
17
+ hooks:
18
+ - id: pyupgrade
19
+ args: [--py312-plus]
20
+ - repo: https://github.com/pre-commit/mirrors-mypy
21
+ rev: v1.19.1
22
+ hooks:
23
+ - id: mypy
24
+ - repo: https://github.com/astral-sh/ruff-pre-commit
25
+ rev: v0.15.2
26
+ hooks:
27
+ - id: ruff-check
28
+ args: [--fix]
29
+ - id: ruff-format
30
+ - repo: https://github.com/gitleaks/gitleaks
31
+ rev: v8.30.1
32
+ hooks:
33
+ - id: gitleaks
@@ -0,0 +1 @@
1
+ 3.12.12
@@ -0,0 +1,40 @@
1
+ # Changelog
2
+
3
+ All notable changes follow [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
4
+ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
5
+
6
+ ## [Unreleased]
7
+
8
+ ## [0.1.0] — 2026-04-24
9
+
10
+ First public alpha on PyPI.
11
+
12
+ ### Added
13
+ - Initial scaffold: pyproject, MIT LICENSE, README, package skeleton.
14
+ - Detection ensemble (Presidio + optional GLiNER + Flair) extracted from the
15
+ Noirdoc Cloud reverse proxy.
16
+ - File extractors for PDF (with optional OCR fallback), DOCX, XLSX, images, and
17
+ plain text formats (TXT/CSV/MD/HTML).
18
+ - Reversible pseudonymization: `PseudonymMapper` with `to_dict`/`from_dict`
19
+ serialization, `PseudonymizationEngine`, `ReidentificationEngine`, and
20
+ `file_reidentification.service.reidentify_file_bytes` (DOCX/XLSX/plaintext
21
+ roundtrip; PDF/PPTX/images return `None`).
22
+ - `MappingStore` with a pluggable `MappingBackend` protocol; concrete
23
+ `MemoryMappingBackend`, `FileMappingBackend`, and `RedisMappingBackend`
24
+ (via `noirdoc[redis]`).
25
+ - Persistent namespaces under `~/.noirdoc/namespaces/<name>/` with a per-
26
+ namespace Fernet key (`0600`).
27
+ - High-level SDK: `noirdoc.redact(...)` and `noirdoc.Redactor(namespace=...)`.
28
+ - `noirdoc` CLI (Click) with `redact`, `reveal`, `lookup`, `ns {list,show,delete}`,
29
+ and `models pull` subcommands.
30
+ - Project URLs in `pyproject.toml` (Repository, Issues, Changelog) for PyPI sidebar.
31
+
32
+ ### Fixed
33
+ - Baseline install (`pip install noirdoc`) crashed with `ModuleNotFoundError:
34
+ gliner` because the default `detector="ensemble"` eagerly imported GLiNER.
35
+ The ensemble now detects an absent GLiNER, falls back to Presidio-only with
36
+ a `UserWarning`, and keeps working. Explicit `--detector gliner` still fails
37
+ loudly when the `[full]` extra isn't installed.
38
+
39
+ [Unreleased]: https://github.com/nextaim-de/noirdoc/compare/v0.1.0...HEAD
40
+ [0.1.0]: https://github.com/nextaim-de/noirdoc/releases/tag/v0.1.0
@@ -0,0 +1,63 @@
1
+ # Contributing to noirdoc
2
+
3
+ Thanks for your interest. Noirdoc is early — API will change — but contributions are welcome, especially around detectors, formats, and German-language coverage.
4
+
5
+ ## Prerequisites
6
+
7
+ - Python 3.12 or 3.13
8
+ - [Poetry](https://python-poetry.org/) 2.0 or later — `pipx install poetry` if you don't have it.
9
+ - ~1 GB free disk for the `full` extra (GLiNER + Flair weights).
10
+
11
+ ## Dev setup
12
+
13
+ ```bash
14
+ git clone https://github.com/nextaim-de/noirdoc.git
15
+ cd noirdoc
16
+ poetry env use 3.12 # or 3.13
17
+ poetry install --all-extras # dev + full + redis
18
+ poetry run python -m spacy download de_core_news_lg
19
+ poetry run pre-commit install
20
+ ```
21
+
22
+ Poetry manages the local environment and resolves dependencies fresh from `pyproject.toml` on each checkout — noirdoc is a library, so the lockfile is not tracked (downstream consumers re-resolve). The build backend is `hatchling`; the published wheel is built from `[project]` metadata. CI uses plain `pip install -e ".[dev]"` for speed.
23
+
24
+ ## Running tests
25
+
26
+ ```bash
27
+ poetry run pytest # fast tier (CI default)
28
+ poetry run pytest -m slow # includes model-loading tests
29
+ poetry run pytest -m "not slow" # same as CI
30
+ ```
31
+
32
+ Prefer `poetry shell` if you'd rather not prefix every command.
33
+
34
+ ## Lint and formatting
35
+
36
+ `pre-commit` runs ruff (check + format), mypy, pyupgrade, gitleaks, and a few hygiene hooks. CI enforces the same config.
37
+
38
+ ```bash
39
+ poetry run pre-commit run --all-files
40
+ ```
41
+
42
+ ## Adding a detector
43
+
44
+ Custom Presidio recognizers live in `src/noirdoc/detection/presidio_detector.py`. The German set — `GermanPhoneRecognizer`, `GermanSVNRRecognizer`, `GermanSteuerIDRecognizer`, `InvertedNameRecognizer` — is the reference pattern. Each one sets `supported_language="de"`; multi-language detectors register twice, once per language (see the `InvertedNameRecognizer` registration in the same file).
45
+
46
+ Tests go in `tests/test_presidio_detector.py`. Add both positive and negative cases, and cover the German edge cases you're designing for — lowercase terms, IBAN formatting, Steuer-ID checksums.
47
+
48
+ ## Pull requests
49
+
50
+ - Keep PRs small and focused. One detector per PR beats a mega-PR.
51
+ - Every PR includes a test.
52
+ - Update `CHANGELOG.md` under `## [Unreleased]` with a one-line entry in the appropriate subsection (`### Added`, `### Changed`, `### Fixed`).
53
+ - CI must be green before review.
54
+
55
+ ## Releasing
56
+
57
+ Only maintainers cut releases. The flow is tag-driven and uses PyPI Trusted Publishing — see [docs/RELEASING.md](docs/RELEASING.md) for the per-release checklist, the first-release rehearsal, and the one-time PyPI setup.
58
+
59
+ ## Reporting bugs
60
+
61
+ Open an issue at <https://github.com/nextaim-de/noirdoc/issues>. Include Python version, OS, the command or code that triggered the bug, and the traceback if any. A minimal reproducer helps a lot.
62
+
63
+ For security-sensitive bugs, see [SECURITY.md](SECURITY.md) — do not open a public issue.
noirdoc-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Antonio Maiolo / Nextaim GmbH
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
noirdoc-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,221 @@
1
+ Metadata-Version: 2.4
2
+ Name: noirdoc
3
+ Version: 0.1.0
4
+ Summary: Local PII redaction and pseudonymization for documents.
5
+ Project-URL: Homepage, https://noirdoc.de
6
+ Project-URL: Repository, https://github.com/nextaim-de/noirdoc
7
+ Project-URL: Documentation, https://github.com/nextaim-de/noirdoc#readme
8
+ Project-URL: Issues, https://github.com/nextaim-de/noirdoc/issues
9
+ Project-URL: Changelog, https://github.com/nextaim-de/noirdoc/blob/main/CHANGELOG.md
10
+ Author: Antonio Maiolo
11
+ Maintainer-email: Nextaim GmbH <hello@noirdoc.de>
12
+ License-Expression: MIT
13
+ License-File: LICENSE
14
+ Keywords: anonymization,gdpr,nlp,pii,presidio,privacy,pseudonymization,redaction
15
+ Classifier: Development Status :: 3 - Alpha
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: License :: OSI Approved :: MIT License
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Security
23
+ Classifier: Topic :: Text Processing :: Linguistic
24
+ Requires-Python: <3.14,>=3.12
25
+ Requires-Dist: click>=8.1.0
26
+ Requires-Dist: cryptography>=46.0.0
27
+ Requires-Dist: openpyxl>=3.1.0
28
+ Requires-Dist: pillow>=10.0.0
29
+ Requires-Dist: presidio-analyzer>=2.2.361
30
+ Requires-Dist: presidio-anonymizer>=2.2.361
31
+ Requires-Dist: pydantic>=2.0
32
+ Requires-Dist: pypdfium2>=4.0.0
33
+ Requires-Dist: pytesseract>=0.3.10
34
+ Requires-Dist: python-docx>=1.1.0
35
+ Requires-Dist: python-magic>=0.4.27
36
+ Requires-Dist: spacy>=3.8
37
+ Requires-Dist: structlog>=25.0.0
38
+ Provides-Extra: dev
39
+ Requires-Dist: fakeredis>=2.21.0; extra == 'dev'
40
+ Requires-Dist: mypy>=1.19.1; extra == 'dev'
41
+ Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
42
+ Requires-Dist: pytest>=8.0; extra == 'dev'
43
+ Requires-Dist: ruff>=0.15.0; extra == 'dev'
44
+ Provides-Extra: full
45
+ Requires-Dist: flair>=0.14; extra == 'full'
46
+ Requires-Dist: gliner>=0.2.25; extra == 'full'
47
+ Provides-Extra: redis
48
+ Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
49
+ Description-Content-Type: text/markdown
50
+
51
+ <p align="center">
52
+ <img src="https://raw.githubusercontent.com/nextaim-de/noirdoc/main/docs/assets/banner.svg" alt="noirdoc — German-first PII redaction, local by default." width="800">
53
+ </p>
54
+
55
+ <p align="center">
56
+ <a href="https://github.com/nextaim-de/noirdoc/actions/workflows/ci.yml"><img src="https://github.com/nextaim-de/noirdoc/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
57
+ <img src="https://img.shields.io/badge/python-3.12%20%7C%203.13-blue" alt="Python 3.12 | 3.13">
58
+ <img src="https://img.shields.io/badge/license-MIT-green" alt="License: MIT">
59
+ <a href="https://github.com/pre-commit/pre-commit"><img src="https://img.shields.io/badge/pre--commit-enabled-brightgreen" alt="pre-commit enabled"></a>
60
+ </p>
61
+
62
+ # noirdoc
63
+
64
+ **German-first PII redaction and pseudonymization for documents. Local by default. Reversible when you need it.**
65
+
66
+ Noirdoc redacts names, addresses, phone numbers, IBANs, Steuer-IDs, SVNRs, and the rest — from PDFs, DOCX, XLSX, and plain text — without sending anything to a third party. Under the hood it's a rules-based Presidio pipeline by default, and an ensemble (Presidio + GLiNER + Flair) when the `[full]` extra is installed. It's built for real-world German documents and mixed DE/EN text — the kind of stuff Mittelstand actually runs through an LLM.
67
+
68
+ > **Status:** alpha (`0.1.x`). API will change before `1.0`. Pin the minor version.
69
+
70
+ ## Prerequisites
71
+
72
+ - Python **3.12** or **3.13**
73
+ - ~1 GB free disk if you install the `[full]` extra (spaCy + Flair + GLiNER weights)
74
+ - Optional: a Redis instance if you want shared mapping storage across workers (`[redis]` extra)
75
+
76
+ ## Install
77
+
78
+ ```bash
79
+ # Baseline — Presidio + all file extractors + reversible mapper.
80
+ pip install noirdoc
81
+
82
+ # Full ensemble (adds GLiNER + Flair, large ML weights). Recommended for real work.
83
+ pip install noirdoc[full]
84
+ noirdoc models pull
85
+
86
+ # Optional distributed mapper backend.
87
+ pip install noirdoc[redis]
88
+ ```
89
+
90
+ For anything beyond toy examples, use `noirdoc[full]` — the ensemble catches what the baseline misses, especially on German lowercase text.
91
+
92
+ ## Quickstart
93
+
94
+ ```bash
95
+ # One-shot redact (ephemeral mapping, discarded on exit).
96
+ noirdoc redact vertrag.pdf -o vertrag-clean.pdf
97
+
98
+ # Persistent namespace — placeholders stay consistent across files and sessions.
99
+ noirdoc redact --namespace mandant-mueller brief.docx -o brief-clean.docx
100
+ noirdoc reveal --namespace mandant-mueller brief-clean.docx -o brief-revealed.docx
101
+ noirdoc lookup --namespace mandant-mueller "<<PERSON_3>>"
102
+ ```
103
+
104
+ ```python
105
+ from noirdoc import Redactor
106
+
107
+ r = Redactor(namespace="mandant-mueller")
108
+ r.redact_file("vertrag.pdf", output="vertrag-clean.pdf")
109
+ r.redact_file("brief.docx", output="brief-clean.docx")
110
+ r.reveal_text(llm_response) # un-redact the model's reply
111
+ ```
112
+
113
+ Input:
114
+
115
+ > Anna Müller, geboren am 12.03.1981 in München, erreichbar unter 0171-2345678, Steuer-ID 12 345 678 901, IBAN DE89 3704 0044 0532 0130 00.
116
+
117
+ Output:
118
+
119
+ > `<<PERSON_1>>`, geboren am `<<DATE_TIME_1>>` in `<<LOCATION_1>>`, erreichbar unter `<<PHONE_NUMBER_1>>`, Steuer-ID `<<DE_STEUER_ID_1>>`, IBAN `<<IBAN_CODE_1>>`.
120
+
121
+ ## Commands
122
+
123
+ | Command | What it does |
124
+ |------------------------------|----------------------------------------------------------------------------------------|
125
+ | `noirdoc redact <files>` | Redact one or more files (accepts directories; `-o FILE` or `--output-dir DIR`). |
126
+ | `noirdoc reveal <file>` | Reverse pseudonyms back to originals (DOCX / XLSX / plain; `--namespace` required). |
127
+ | `noirdoc lookup <token>` | Resolve a pseudonym like `<<PERSON_1>>` to its original value. |
128
+ | `noirdoc ns list` | List persistent namespaces under `~/.noirdoc/namespaces/`. |
129
+ | `noirdoc ns show <name>` | Print the mapping summary for a namespace as JSON. |
130
+ | `noirdoc ns delete <name>` | Delete a namespace (prompts for confirmation). |
131
+ | `noirdoc models pull` | Download spaCy models and (optionally) GLiNER weights up front. |
132
+
133
+ Run `noirdoc <cmd> --help` for the full flag list on any subcommand.
134
+
135
+ ## Before you start
136
+
137
+ A few honest caveats before you ship this into a pipeline:
138
+
139
+ - **Best results need `[full]`.** On first use (or via `noirdoc models pull`) the full extra downloads roughly **560 MB** of weights: spaCy `de_core_news_lg`, Flair `ner-german-large`, and a GLiNER multilingual model. Budget disk and bandwidth.
140
+ - **PDF reveal is not supported yet.** Round-tripping placeholders back into a PDF is a hard problem (position drift, font metrics, image-based redactions). PDFs redact cleanly; reveal is pass-through. DOCX, XLSX, and plain text round-trip fully.
141
+ - **Alpha API.** Classes and CLI flags may change between `0.1.x` and `0.2.x`. Pin accordingly.
142
+ - **Detector quality depends on the upstream models.** Presidio + Flair + GLiNER do the heavy lifting. Noirdoc adds German-specific recognizers on top, but it does not train models.
143
+
144
+ ## German-first
145
+
146
+ Noirdoc defaults to German (`language="de"`) with fallback to `["de", "en"]` for mixed documents. What that actually means:
147
+
148
+ - **Custom recognizers** in `src/noirdoc/detection/presidio_detector.py`:
149
+ - `GermanPhoneRecognizer` — German phone formats (0171-..., +49...)
150
+ - `GermanSVNRRecognizer` — Sozialversicherungsnummer with checksum
151
+ - `GermanSteuerIDRecognizer` — 11-digit Steuer-ID with checksum
152
+ - `InvertedNameRecognizer` — registered for both `de` and `en` to catch "Nachname, Vorname" patterns
153
+ - **Flair `ner-german-large`** (XLM-R, F1 92.3 % on CoNLL-03 DE) handles lowercase German text — the case where spaCy tends to drop names.
154
+ - **GLiNER multilingual** catches entity types the others miss.
155
+ - **German-style lowercase** financial terms, German IBANs, German date formats, and German address patterns are covered in the test suite (`tests/test_presidio_detector.py`).
156
+
157
+ If you're working with German legal, medical, HR, or financial documents, this is what the defaults are tuned for.
158
+
159
+ ## Supported formats
160
+
161
+ | Format | Redact | Reveal (round-trip) |
162
+ |------------------------------|--------|---------------------|
163
+ | PDF | ✓ | ✗ (pass-through) |
164
+ | DOCX | ✓ | ✓ |
165
+ | XLSX | ✓ | ✓ |
166
+ | Plain text / CSV / MD / HTML | ✓ | ✓ |
167
+ | PPTX / images | ✓ | ✗ (pass-through) |
168
+
169
+ PDF reveal is an open contribution target — see [CONTRIBUTING.md](https://github.com/nextaim-de/noirdoc/blob/main/CONTRIBUTING.md).
170
+
171
+ ## Advanced: shared mapping storage
172
+
173
+ The `[redis]` extra ships a `RedisMappingBackend` that plugs into the lower-level `MappingStore` — the same primitive Noirdoc Cloud uses for request-scoped, encrypted, TTL-bounded mapping persistence across workers. It is **not** wired into `Redactor(namespace=...)`, which persists to the local filesystem under `~/.noirdoc/namespaces/`. Use `MappingStore` when you have multiple workers that need to share pseudonym mappings for the same request, or when you want encrypted-at-rest mappings with automatic expiry.
174
+
175
+ ```python
176
+ import asyncio
177
+ from cryptography.fernet import Fernet
178
+ from redis.asyncio import Redis
179
+
180
+ from noirdoc.mappings.backends.redis_backend import RedisMappingBackend
181
+ from noirdoc.mappings.store import MappingStore
182
+
183
+ async def main() -> None:
184
+ redis = Redis.from_url("redis://localhost:6379")
185
+ store = MappingStore(
186
+ backend=RedisMappingBackend(redis),
187
+ encryption_key=Fernet.generate_key(), # keep stable across workers
188
+ )
189
+ # store.save(request_id=..., tenant_id=..., mapper=...)
190
+ # mappings = await store.load(request_id)
191
+
192
+ asyncio.run(main())
193
+ ```
194
+
195
+ The `encryption_key` must be identical across workers that need to read the same mappings. `MappingStore.save()` accepts a `ttl_days` kwarg (default 30).
196
+
197
+ ## Noirdoc Cloud
198
+
199
+ Don't want to run this yourself? **[Noirdoc Cloud](https://noirdoc.de)** is the hosted API wrapper: a privacy-preserving reverse proxy for LLM calls that uses this exact pipeline, plus multi-tenancy, audit, and provider key management. Compliance story: what's on GitHub is what the cloud runs.
200
+
201
+ ## Contributing
202
+
203
+ Bug reports, detectors, and format support are all welcome. See [CONTRIBUTING.md](https://github.com/nextaim-de/noirdoc/blob/main/CONTRIBUTING.md) for dev setup, tests, and the recognizer pattern.
204
+
205
+ ## Security
206
+
207
+ Report vulnerabilities via GitHub's private vulnerability reporting — see [SECURITY.md](https://github.com/nextaim-de/noirdoc/blob/main/SECURITY.md). Please don't open public issues for security bugs.
208
+
209
+ ## Changelog
210
+
211
+ See [CHANGELOG.md](https://github.com/nextaim-de/noirdoc/blob/main/CHANGELOG.md). Follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [SemVer](https://semver.org/).
212
+
213
+ ## License
214
+
215
+ MIT © 2026 Antonio Maiolo / [Nextaim GmbH](https://nextaim.de). See [LICENSE](https://github.com/nextaim-de/noirdoc/blob/main/LICENSE).
216
+
217
+ ---
218
+
219
+ <p align="center">
220
+ Built by <a href="https://nextaim.de">Nextaim GmbH</a> · <a href="https://noirdoc.de">noirdoc.de</a>
221
+ </p>