noirdoc 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- noirdoc-0.1.0/.github/workflows/ci.yml +46 -0
- noirdoc-0.1.0/.github/workflows/release.yml +130 -0
- noirdoc-0.1.0/.gitignore +69 -0
- noirdoc-0.1.0/.pre-commit-config.yaml +33 -0
- noirdoc-0.1.0/.python-version +1 -0
- noirdoc-0.1.0/CHANGELOG.md +40 -0
- noirdoc-0.1.0/CONTRIBUTING.md +63 -0
- noirdoc-0.1.0/LICENSE +21 -0
- noirdoc-0.1.0/PKG-INFO +221 -0
- noirdoc-0.1.0/README.md +171 -0
- noirdoc-0.1.0/SECURITY.md +33 -0
- noirdoc-0.1.0/docs/RELEASING.md +93 -0
- noirdoc-0.1.0/docs/assets/banner.svg +41 -0
- noirdoc-0.1.0/docs/assets/noirdoc-logo-light-xxl.png +0 -0
- noirdoc-0.1.0/pyproject.toml +124 -0
- noirdoc-0.1.0/src/noirdoc/__init__.py +12 -0
- noirdoc-0.1.0/src/noirdoc/_version.py +24 -0
- noirdoc-0.1.0/src/noirdoc/cli.py +255 -0
- noirdoc-0.1.0/src/noirdoc/detection/__init__.py +0 -0
- noirdoc-0.1.0/src/noirdoc/detection/base.py +44 -0
- noirdoc-0.1.0/src/noirdoc/detection/ensemble.py +170 -0
- noirdoc-0.1.0/src/noirdoc/detection/flair_detector.py +57 -0
- noirdoc-0.1.0/src/noirdoc/detection/flair_recognizer.py +69 -0
- noirdoc-0.1.0/src/noirdoc/detection/gliner_detector.py +89 -0
- noirdoc-0.1.0/src/noirdoc/detection/model_manager.py +26 -0
- noirdoc-0.1.0/src/noirdoc/detection/presidio_detector.py +234 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/__init__.py +4 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/body_walker.py +459 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/extractor.py +81 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/__init__.py +0 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/docx_ext.py +27 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/ocr.py +33 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/pdf.py +79 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/plain.py +8 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/extractors/xlsx.py +23 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/mime.py +70 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/models.py +45 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/pipeline.py +263 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/policy.py +28 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/reconstruction.py +192 -0
- noirdoc-0.1.0/src/noirdoc/file_analysis/xlsx_inference.py +229 -0
- noirdoc-0.1.0/src/noirdoc/file_reidentification/__init__.py +0 -0
- noirdoc-0.1.0/src/noirdoc/file_reidentification/service.py +157 -0
- noirdoc-0.1.0/src/noirdoc/mappings/__init__.py +0 -0
- noirdoc-0.1.0/src/noirdoc/mappings/backends/__init__.py +20 -0
- noirdoc-0.1.0/src/noirdoc/mappings/backends/base.py +25 -0
- noirdoc-0.1.0/src/noirdoc/mappings/backends/file.py +88 -0
- noirdoc-0.1.0/src/noirdoc/mappings/backends/memory.py +44 -0
- noirdoc-0.1.0/src/noirdoc/mappings/backends/redis_backend.py +30 -0
- noirdoc-0.1.0/src/noirdoc/mappings/hydration.py +48 -0
- noirdoc-0.1.0/src/noirdoc/mappings/store.py +130 -0
- noirdoc-0.1.0/src/noirdoc/namespace.py +99 -0
- noirdoc-0.1.0/src/noirdoc/pseudonymization/__init__.py +0 -0
- noirdoc-0.1.0/src/noirdoc/pseudonymization/engine.py +24 -0
- noirdoc-0.1.0/src/noirdoc/pseudonymization/mapper.py +83 -0
- noirdoc-0.1.0/src/noirdoc/pseudonymization/system_prompt.py +78 -0
- noirdoc-0.1.0/src/noirdoc/reidentification/__init__.py +0 -0
- noirdoc-0.1.0/src/noirdoc/reidentification/engine.py +105 -0
- noirdoc-0.1.0/src/noirdoc/sdk.py +339 -0
- noirdoc-0.1.0/tests/__init__.py +0 -0
- noirdoc-0.1.0/tests/file_analysis/__init__.py +0 -0
- noirdoc-0.1.0/tests/file_analysis/test_body_walker.py +353 -0
- noirdoc-0.1.0/tests/file_analysis/test_extractor.py +73 -0
- noirdoc-0.1.0/tests/file_analysis/test_mime.py +64 -0
- noirdoc-0.1.0/tests/file_analysis/test_pdf_extractor.py +174 -0
- noirdoc-0.1.0/tests/file_analysis/test_pipeline.py +334 -0
- noirdoc-0.1.0/tests/file_analysis/test_policy.py +25 -0
- noirdoc-0.1.0/tests/file_reidentification/__init__.py +0 -0
- noirdoc-0.1.0/tests/file_reidentification/test_service.py +175 -0
- noirdoc-0.1.0/tests/mappings/__init__.py +0 -0
- noirdoc-0.1.0/tests/mappings/test_backends.py +80 -0
- noirdoc-0.1.0/tests/mappings/test_hydration.py +166 -0
- noirdoc-0.1.0/tests/mappings/test_store.py +153 -0
- noirdoc-0.1.0/tests/pseudonymization/__init__.py +0 -0
- noirdoc-0.1.0/tests/pseudonymization/test_system_prompt.py +110 -0
- noirdoc-0.1.0/tests/test_ensemble.py +302 -0
- noirdoc-0.1.0/tests/test_flair_recognizer.py +53 -0
- noirdoc-0.1.0/tests/test_gliner_detector.py +87 -0
- noirdoc-0.1.0/tests/test_integration_pipeline.py +90 -0
- noirdoc-0.1.0/tests/test_mapper.py +120 -0
- noirdoc-0.1.0/tests/test_model_manager.py +37 -0
- noirdoc-0.1.0/tests/test_namespace.py +98 -0
- noirdoc-0.1.0/tests/test_presidio_detector.py +159 -0
- noirdoc-0.1.0/tests/test_pseudonymization.py +83 -0
- noirdoc-0.1.0/tests/test_reidentification.py +235 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
13
|
+
cancel-in-progress: true
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
lint:
|
|
17
|
+
name: Lint (pre-commit)
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
timeout-minutes: 10
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v4
|
|
22
|
+
- uses: actions/setup-python@v5
|
|
23
|
+
with:
|
|
24
|
+
python-version: "3.12"
|
|
25
|
+
- uses: pre-commit/action@v3.0.1
|
|
26
|
+
|
|
27
|
+
test:
|
|
28
|
+
name: Tests (Python ${{ matrix.python-version }})
|
|
29
|
+
runs-on: ubuntu-latest
|
|
30
|
+
timeout-minutes: 15
|
|
31
|
+
strategy:
|
|
32
|
+
fail-fast: false
|
|
33
|
+
matrix:
|
|
34
|
+
python-version: ["3.12", "3.13"]
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/checkout@v4
|
|
37
|
+
- uses: actions/setup-python@v5
|
|
38
|
+
with:
|
|
39
|
+
python-version: ${{ matrix.python-version }}
|
|
40
|
+
cache: "pip"
|
|
41
|
+
- name: Install baseline + dev deps
|
|
42
|
+
run: pip install -e ".[dev]"
|
|
43
|
+
- name: Download spaCy model
|
|
44
|
+
run: python -m spacy download de_core_news_lg
|
|
45
|
+
- name: Run tests
|
|
46
|
+
run: pytest -m "not slow" --tb=short
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Tag-triggered publish. Pre-release tags (a/b/rc/dev) go to TestPyPI;
|
|
4
|
+
# final tags (vX.Y.Z, optionally .postN) go to PyPI. Auth via PyPI Trusted
|
|
5
|
+
# Publishing (OIDC) — no secrets stored in GitHub. See docs/RELEASING.md.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
push:
|
|
9
|
+
tags: ["v*"]
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
|
|
14
|
+
concurrency:
|
|
15
|
+
group: release-${{ github.ref }}
|
|
16
|
+
cancel-in-progress: false
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
determine:
|
|
20
|
+
name: Determine channel
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
timeout-minutes: 2
|
|
23
|
+
outputs:
|
|
24
|
+
channel: ${{ steps.pick.outputs.channel }}
|
|
25
|
+
repository-url: ${{ steps.pick.outputs.repository-url }}
|
|
26
|
+
prerelease: ${{ steps.pick.outputs.prerelease }}
|
|
27
|
+
steps:
|
|
28
|
+
- id: pick
|
|
29
|
+
env:
|
|
30
|
+
TAG: ${{ github.ref_name }}
|
|
31
|
+
run: |
|
|
32
|
+
if [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?$ ]]; then
|
|
33
|
+
echo "channel=pypi" >> "$GITHUB_OUTPUT"
|
|
34
|
+
echo "repository-url=https://upload.pypi.org/legacy/" >> "$GITHUB_OUTPUT"
|
|
35
|
+
echo "prerelease=false" >> "$GITHUB_OUTPUT"
|
|
36
|
+
else
|
|
37
|
+
echo "channel=testpypi" >> "$GITHUB_OUTPUT"
|
|
38
|
+
echo "repository-url=https://test.pypi.org/legacy/" >> "$GITHUB_OUTPUT"
|
|
39
|
+
echo "prerelease=true" >> "$GITHUB_OUTPUT"
|
|
40
|
+
fi
|
|
41
|
+
|
|
42
|
+
build:
|
|
43
|
+
name: Build distribution
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
needs: determine
|
|
46
|
+
timeout-minutes: 10
|
|
47
|
+
steps:
|
|
48
|
+
- uses: actions/checkout@v4
|
|
49
|
+
with:
|
|
50
|
+
fetch-depth: 0 # hatch-vcs needs the full history to resolve the tag
|
|
51
|
+
- uses: actions/setup-python@v5
|
|
52
|
+
with:
|
|
53
|
+
python-version: "3.12"
|
|
54
|
+
- name: Install build tooling
|
|
55
|
+
run: pip install --upgrade build
|
|
56
|
+
- name: Build sdist and wheel
|
|
57
|
+
run: python -m build
|
|
58
|
+
- name: Confirm version matches tag
|
|
59
|
+
env:
|
|
60
|
+
TAG: ${{ github.ref_name }}
|
|
61
|
+
run: |
|
|
62
|
+
expected="${TAG#v}"
|
|
63
|
+
built=$(ls dist/*.whl | head -n1 | sed -E 's|.*/noirdoc-([^-]+)-.*|\1|')
|
|
64
|
+
echo "Tag expects: $expected"
|
|
65
|
+
echo "Wheel reports: $built"
|
|
66
|
+
test "$expected" = "$built"
|
|
67
|
+
- uses: actions/upload-artifact@v4
|
|
68
|
+
with:
|
|
69
|
+
name: dist
|
|
70
|
+
path: dist/
|
|
71
|
+
if-no-files-found: error
|
|
72
|
+
|
|
73
|
+
publish:
|
|
74
|
+
name: Publish to ${{ needs.determine.outputs.channel }}
|
|
75
|
+
runs-on: ubuntu-latest
|
|
76
|
+
needs: [determine, build]
|
|
77
|
+
timeout-minutes: 10
|
|
78
|
+
environment:
|
|
79
|
+
name: ${{ needs.determine.outputs.channel }}
|
|
80
|
+
permissions:
|
|
81
|
+
id-token: write # OIDC for Trusted Publishing
|
|
82
|
+
attestations: write # PEP 740 build provenance
|
|
83
|
+
contents: read
|
|
84
|
+
steps:
|
|
85
|
+
- uses: actions/download-artifact@v4
|
|
86
|
+
with:
|
|
87
|
+
name: dist
|
|
88
|
+
path: dist/
|
|
89
|
+
- name: Publish
|
|
90
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
91
|
+
with:
|
|
92
|
+
repository-url: ${{ needs.determine.outputs.repository-url }}
|
|
93
|
+
attestations: true
|
|
94
|
+
|
|
95
|
+
github-release:
|
|
96
|
+
name: Create GitHub Release
|
|
97
|
+
runs-on: ubuntu-latest
|
|
98
|
+
needs: [determine, publish]
|
|
99
|
+
timeout-minutes: 5
|
|
100
|
+
permissions:
|
|
101
|
+
contents: write
|
|
102
|
+
steps:
|
|
103
|
+
- uses: actions/checkout@v4
|
|
104
|
+
- uses: actions/download-artifact@v4
|
|
105
|
+
with:
|
|
106
|
+
name: dist
|
|
107
|
+
path: dist/
|
|
108
|
+
- name: Extract changelog section for this tag
|
|
109
|
+
env:
|
|
110
|
+
TAG: ${{ github.ref_name }}
|
|
111
|
+
run: |
|
|
112
|
+
version="${TAG#v}"
|
|
113
|
+
# Pull the section between `## [<version>]` and the next `## [`.
|
|
114
|
+
awk -v ver="$version" '
|
|
115
|
+
$0 ~ "^## \\[" ver "\\]" {found=1; next}
|
|
116
|
+
found && /^## \[/ {exit}
|
|
117
|
+
found {print}
|
|
118
|
+
' CHANGELOG.md > release-notes.md
|
|
119
|
+
if [ ! -s release-notes.md ]; then
|
|
120
|
+
echo "Release $TAG — see CHANGELOG.md for details." > release-notes.md
|
|
121
|
+
fi
|
|
122
|
+
- name: Create release
|
|
123
|
+
uses: softprops/action-gh-release@v2
|
|
124
|
+
with:
|
|
125
|
+
tag_name: ${{ github.ref_name }}
|
|
126
|
+
name: ${{ github.ref_name }}
|
|
127
|
+
body_path: release-notes.md
|
|
128
|
+
prerelease: ${{ needs.determine.outputs.prerelease }}
|
|
129
|
+
files: dist/*
|
|
130
|
+
fail_on_unmatched_files: true
|
noirdoc-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Byte-compiled / optimized
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
|
|
8
|
+
# Virtual environments
|
|
9
|
+
.venv/
|
|
10
|
+
venv/
|
|
11
|
+
env/
|
|
12
|
+
.env
|
|
13
|
+
.env.*
|
|
14
|
+
!.env.example
|
|
15
|
+
|
|
16
|
+
# Packaging
|
|
17
|
+
build/
|
|
18
|
+
dist/
|
|
19
|
+
*.egg-info/
|
|
20
|
+
*.egg
|
|
21
|
+
wheels/
|
|
22
|
+
.eggs/
|
|
23
|
+
pip-wheel-metadata/
|
|
24
|
+
|
|
25
|
+
# Lock files (library — downstream should re-resolve)
|
|
26
|
+
poetry.lock
|
|
27
|
+
|
|
28
|
+
# Test / type / coverage artefacts
|
|
29
|
+
.pytest_cache/
|
|
30
|
+
.ruff_cache/
|
|
31
|
+
.mypy_cache/
|
|
32
|
+
.pyre/
|
|
33
|
+
.pytype/
|
|
34
|
+
.tox/
|
|
35
|
+
.nox/
|
|
36
|
+
.hypothesis/
|
|
37
|
+
.coverage
|
|
38
|
+
.coverage.*
|
|
39
|
+
coverage.xml
|
|
40
|
+
htmlcov/
|
|
41
|
+
|
|
42
|
+
# OS
|
|
43
|
+
.DS_Store
|
|
44
|
+
Thumbs.db
|
|
45
|
+
ehthumbs.db
|
|
46
|
+
Desktop.ini
|
|
47
|
+
|
|
48
|
+
# Editors / IDEs
|
|
49
|
+
.idea/
|
|
50
|
+
.vscode/
|
|
51
|
+
*.swp
|
|
52
|
+
*.swo
|
|
53
|
+
*~
|
|
54
|
+
*.bak
|
|
55
|
+
|
|
56
|
+
# Jupyter
|
|
57
|
+
.ipynb_checkpoints/
|
|
58
|
+
|
|
59
|
+
# Local model cache and user data
|
|
60
|
+
.noirdoc/
|
|
61
|
+
|
|
62
|
+
# Build backends
|
|
63
|
+
.hatch/
|
|
64
|
+
|
|
65
|
+
# hatch-vcs generated version file
|
|
66
|
+
src/noirdoc/_version.py
|
|
67
|
+
|
|
68
|
+
# Claude Code working notes (brainstorm plans + specs)
|
|
69
|
+
docs/superpowers/
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
3
|
+
rev: v6.0.0
|
|
4
|
+
hooks:
|
|
5
|
+
- id: trailing-whitespace
|
|
6
|
+
- id: end-of-file-fixer
|
|
7
|
+
- id: check-yaml
|
|
8
|
+
- id: debug-statements
|
|
9
|
+
- id: name-tests-test
|
|
10
|
+
args: [--pytest-test-first]
|
|
11
|
+
- repo: https://github.com/asottile/add-trailing-comma
|
|
12
|
+
rev: v4.0.0
|
|
13
|
+
hooks:
|
|
14
|
+
- id: add-trailing-comma
|
|
15
|
+
- repo: https://github.com/asottile/pyupgrade
|
|
16
|
+
rev: v3.21.2
|
|
17
|
+
hooks:
|
|
18
|
+
- id: pyupgrade
|
|
19
|
+
args: [--py312-plus]
|
|
20
|
+
- repo: https://github.com/pre-commit/mirrors-mypy
|
|
21
|
+
rev: v1.19.1
|
|
22
|
+
hooks:
|
|
23
|
+
- id: mypy
|
|
24
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
25
|
+
rev: v0.15.2
|
|
26
|
+
hooks:
|
|
27
|
+
- id: ruff-check
|
|
28
|
+
args: [--fix]
|
|
29
|
+
- id: ruff-format
|
|
30
|
+
- repo: https://github.com/gitleaks/gitleaks
|
|
31
|
+
rev: v8.30.1
|
|
32
|
+
hooks:
|
|
33
|
+
- id: gitleaks
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12.12
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes follow [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
|
|
4
|
+
and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
5
|
+
|
|
6
|
+
## [Unreleased]
|
|
7
|
+
|
|
8
|
+
## [0.1.0] — 2026-04-24
|
|
9
|
+
|
|
10
|
+
First public alpha on PyPI.
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- Initial scaffold: pyproject, MIT LICENSE, README, package skeleton.
|
|
14
|
+
- Detection ensemble (Presidio + optional GLiNER + Flair) extracted from the
|
|
15
|
+
Noirdoc Cloud reverse proxy.
|
|
16
|
+
- File extractors for PDF (with optional OCR fallback), DOCX, XLSX, images, and
|
|
17
|
+
plain text formats (TXT/CSV/MD/HTML).
|
|
18
|
+
- Reversible pseudonymization: `PseudonymMapper` with `to_dict`/`from_dict`
|
|
19
|
+
serialization, `PseudonymizationEngine`, `ReidentificationEngine`, and
|
|
20
|
+
`file_reidentification.service.reidentify_file_bytes` (DOCX/XLSX/plaintext
|
|
21
|
+
roundtrip; PDF/PPTX/images return `None`).
|
|
22
|
+
- `MappingStore` with a pluggable `MappingBackend` protocol; concrete
|
|
23
|
+
`MemoryMappingBackend`, `FileMappingBackend`, and `RedisMappingBackend`
|
|
24
|
+
(via `noirdoc[redis]`).
|
|
25
|
+
- Persistent namespaces under `~/.noirdoc/namespaces/<name>/` with a per-
|
|
26
|
+
namespace Fernet key (`0600`).
|
|
27
|
+
- High-level SDK: `noirdoc.redact(...)` and `noirdoc.Redactor(namespace=...)`.
|
|
28
|
+
- `noirdoc` CLI (Click) with `redact`, `reveal`, `lookup`, `ns {list,show,delete}`,
|
|
29
|
+
and `models pull` subcommands.
|
|
30
|
+
- Project URLs in `pyproject.toml` (Repository, Issues, Changelog) for PyPI sidebar.
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
- Baseline install (`pip install noirdoc`) crashed with `ModuleNotFoundError:
|
|
34
|
+
gliner` because the default `detector="ensemble"` eagerly imported GLiNER.
|
|
35
|
+
The ensemble now detects an absent GLiNER, falls back to Presidio-only with
|
|
36
|
+
a `UserWarning`, and keeps working. Explicit `--detector gliner` still fails
|
|
37
|
+
loudly when the `[full]` extra isn't installed.
|
|
38
|
+
|
|
39
|
+
[Unreleased]: https://github.com/nextaim-de/noirdoc/compare/v0.1.0...HEAD
|
|
40
|
+
[0.1.0]: https://github.com/nextaim-de/noirdoc/releases/tag/v0.1.0
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# Contributing to noirdoc
|
|
2
|
+
|
|
3
|
+
Thanks for your interest. Noirdoc is early — API will change — but contributions are welcome, especially around detectors, formats, and German-language coverage.
|
|
4
|
+
|
|
5
|
+
## Prerequisites
|
|
6
|
+
|
|
7
|
+
- Python 3.12 or 3.13
|
|
8
|
+
- [Poetry](https://python-poetry.org/) 2.0 or later — `pipx install poetry` if you don't have it.
|
|
9
|
+
- ~1 GB free disk for the `full` extra (GLiNER + Flair weights).
|
|
10
|
+
|
|
11
|
+
## Dev setup
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
git clone https://github.com/nextaim-de/noirdoc.git
|
|
15
|
+
cd noirdoc
|
|
16
|
+
poetry env use 3.12 # or 3.13
|
|
17
|
+
poetry install --all-extras # dev + full + redis
|
|
18
|
+
poetry run python -m spacy download de_core_news_lg
|
|
19
|
+
poetry run pre-commit install
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Poetry manages the local environment and resolves dependencies fresh from `pyproject.toml` on each checkout — noirdoc is a library, so the lockfile is not tracked (downstream consumers re-resolve). The build backend is `hatchling`; the published wheel is built from `[project]` metadata. CI uses plain `pip install -e ".[dev]"` for speed.
|
|
23
|
+
|
|
24
|
+
## Running tests
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
poetry run pytest # fast tier (CI default)
|
|
28
|
+
poetry run pytest -m slow # includes model-loading tests
|
|
29
|
+
poetry run pytest -m "not slow" # same as CI
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Prefer `poetry shell` if you'd rather not prefix every command.
|
|
33
|
+
|
|
34
|
+
## Lint and formatting
|
|
35
|
+
|
|
36
|
+
`pre-commit` runs ruff (check + format), mypy, pyupgrade, gitleaks, and a few hygiene hooks. CI enforces the same config.
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
poetry run pre-commit run --all-files
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Adding a detector
|
|
43
|
+
|
|
44
|
+
Custom Presidio recognizers live in `src/noirdoc/detection/presidio_detector.py`. The German set — `GermanPhoneRecognizer`, `GermanSVNRRecognizer`, `GermanSteuerIDRecognizer`, `InvertedNameRecognizer` — is the reference pattern. Each one sets `supported_language="de"`; multi-language detectors register twice, once per language (see the `InvertedNameRecognizer` registration in the same file).
|
|
45
|
+
|
|
46
|
+
Tests go in `tests/test_presidio_detector.py`. Add both positive and negative cases, and cover the German edge cases you're designing for — lowercase terms, IBAN formatting, Steuer-ID checksums.
|
|
47
|
+
|
|
48
|
+
## Pull requests
|
|
49
|
+
|
|
50
|
+
- Keep PRs small and focused. One detector per PR beats a mega-PR.
|
|
51
|
+
- Every PR includes a test.
|
|
52
|
+
- Update `CHANGELOG.md` under `## [Unreleased]` with a one-line entry in the appropriate subsection (`### Added`, `### Changed`, `### Fixed`).
|
|
53
|
+
- CI must be green before review.
|
|
54
|
+
|
|
55
|
+
## Releasing
|
|
56
|
+
|
|
57
|
+
Only maintainers cut releases. The flow is tag-driven and uses PyPI Trusted Publishing — see [docs/RELEASING.md](docs/RELEASING.md) for the per-release checklist, the first-release rehearsal, and the one-time PyPI setup.
|
|
58
|
+
|
|
59
|
+
## Reporting bugs
|
|
60
|
+
|
|
61
|
+
Open an issue at <https://github.com/nextaim-de/noirdoc/issues>. Include Python version, OS, the command or code that triggered the bug, and the traceback if any. A minimal reproducer helps a lot.
|
|
62
|
+
|
|
63
|
+
For security-sensitive bugs, see [SECURITY.md](SECURITY.md) — do not open a public issue.
|
noirdoc-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Antonio Maiolo / Nextaim GmbH
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
noirdoc-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: noirdoc
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local PII redaction and pseudonymization for documents.
|
|
5
|
+
Project-URL: Homepage, https://noirdoc.de
|
|
6
|
+
Project-URL: Repository, https://github.com/nextaim-de/noirdoc
|
|
7
|
+
Project-URL: Documentation, https://github.com/nextaim-de/noirdoc#readme
|
|
8
|
+
Project-URL: Issues, https://github.com/nextaim-de/noirdoc/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/nextaim-de/noirdoc/blob/main/CHANGELOG.md
|
|
10
|
+
Author: Antonio Maiolo
|
|
11
|
+
Maintainer-email: Nextaim GmbH <hello@noirdoc.de>
|
|
12
|
+
License-Expression: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: anonymization,gdpr,nlp,pii,presidio,privacy,pseudonymization,redaction
|
|
15
|
+
Classifier: Development Status :: 3 - Alpha
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Security
|
|
23
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
24
|
+
Requires-Python: <3.14,>=3.12
|
|
25
|
+
Requires-Dist: click>=8.1.0
|
|
26
|
+
Requires-Dist: cryptography>=46.0.0
|
|
27
|
+
Requires-Dist: openpyxl>=3.1.0
|
|
28
|
+
Requires-Dist: pillow>=10.0.0
|
|
29
|
+
Requires-Dist: presidio-analyzer>=2.2.361
|
|
30
|
+
Requires-Dist: presidio-anonymizer>=2.2.361
|
|
31
|
+
Requires-Dist: pydantic>=2.0
|
|
32
|
+
Requires-Dist: pypdfium2>=4.0.0
|
|
33
|
+
Requires-Dist: pytesseract>=0.3.10
|
|
34
|
+
Requires-Dist: python-docx>=1.1.0
|
|
35
|
+
Requires-Dist: python-magic>=0.4.27
|
|
36
|
+
Requires-Dist: spacy>=3.8
|
|
37
|
+
Requires-Dist: structlog>=25.0.0
|
|
38
|
+
Provides-Extra: dev
|
|
39
|
+
Requires-Dist: fakeredis>=2.21.0; extra == 'dev'
|
|
40
|
+
Requires-Dist: mypy>=1.19.1; extra == 'dev'
|
|
41
|
+
Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
|
|
42
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
43
|
+
Requires-Dist: ruff>=0.15.0; extra == 'dev'
|
|
44
|
+
Provides-Extra: full
|
|
45
|
+
Requires-Dist: flair>=0.14; extra == 'full'
|
|
46
|
+
Requires-Dist: gliner>=0.2.25; extra == 'full'
|
|
47
|
+
Provides-Extra: redis
|
|
48
|
+
Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
|
|
49
|
+
Description-Content-Type: text/markdown
|
|
50
|
+
|
|
51
|
+
<p align="center">
|
|
52
|
+
<img src="https://raw.githubusercontent.com/nextaim-de/noirdoc/main/docs/assets/banner.svg" alt="noirdoc — German-first PII redaction, local by default." width="800">
|
|
53
|
+
</p>
|
|
54
|
+
|
|
55
|
+
<p align="center">
|
|
56
|
+
<a href="https://github.com/nextaim-de/noirdoc/actions/workflows/ci.yml"><img src="https://github.com/nextaim-de/noirdoc/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
57
|
+
<img src="https://img.shields.io/badge/python-3.12%20%7C%203.13-blue" alt="Python 3.12 | 3.13">
|
|
58
|
+
<img src="https://img.shields.io/badge/license-MIT-green" alt="License: MIT">
|
|
59
|
+
<a href="https://github.com/pre-commit/pre-commit"><img src="https://img.shields.io/badge/pre--commit-enabled-brightgreen" alt="pre-commit enabled"></a>
|
|
60
|
+
</p>
|
|
61
|
+
|
|
62
|
+
# noirdoc
|
|
63
|
+
|
|
64
|
+
**German-first PII redaction and pseudonymization for documents. Local by default. Reversible when you need it.**
|
|
65
|
+
|
|
66
|
+
Noirdoc redacts names, addresses, phone numbers, IBANs, Steuer-IDs, SVNRs, and the rest — from PDFs, DOCX, XLSX, and plain text — without sending anything to a third party. Under the hood it's a rules-based Presidio pipeline by default, and an ensemble (Presidio + GLiNER + Flair) when the `[full]` extra is installed. It's built for real-world German documents and mixed DE/EN text — the kind of stuff Mittelstand actually runs through an LLM.
|
|
67
|
+
|
|
68
|
+
> **Status:** alpha (`0.1.x`). API will change before `1.0`. Pin the minor version.
|
|
69
|
+
|
|
70
|
+
## Prerequisites
|
|
71
|
+
|
|
72
|
+
- Python **3.12** or **3.13**
|
|
73
|
+
- ~1 GB free disk if you install the `[full]` extra (spaCy + Flair + GLiNER weights)
|
|
74
|
+
- Optional: a Redis instance if you want shared mapping storage across workers (`[redis]` extra)
|
|
75
|
+
|
|
76
|
+
## Install
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
# Baseline — Presidio + all file extractors + reversible mapper.
|
|
80
|
+
pip install noirdoc
|
|
81
|
+
|
|
82
|
+
# Full ensemble (adds GLiNER + Flair, large ML weights). Recommended for real work.
|
|
83
|
+
pip install noirdoc[full]
|
|
84
|
+
noirdoc models pull
|
|
85
|
+
|
|
86
|
+
# Optional distributed mapper backend.
|
|
87
|
+
pip install noirdoc[redis]
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
For anything beyond toy examples, use `noirdoc[full]` — the ensemble catches what the baseline misses, especially on German lowercase text.
|
|
91
|
+
|
|
92
|
+
## Quickstart
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
# One-shot redact (ephemeral mapping, discarded on exit).
|
|
96
|
+
noirdoc redact vertrag.pdf -o vertrag-clean.pdf
|
|
97
|
+
|
|
98
|
+
# Persistent namespace — placeholders stay consistent across files and sessions.
|
|
99
|
+
noirdoc redact --namespace mandant-mueller brief.docx -o brief-clean.docx
|
|
100
|
+
noirdoc reveal --namespace mandant-mueller brief-clean.docx -o brief-revealed.docx
|
|
101
|
+
noirdoc lookup --namespace mandant-mueller "<<PERSON_3>>"
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from noirdoc import Redactor
|
|
106
|
+
|
|
107
|
+
r = Redactor(namespace="mandant-mueller")
|
|
108
|
+
r.redact_file("vertrag.pdf", output="vertrag-clean.pdf")
|
|
109
|
+
r.redact_file("brief.docx", output="brief-clean.docx")
|
|
110
|
+
r.reveal_text(llm_response) # un-redact the model's reply
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Input:
|
|
114
|
+
|
|
115
|
+
> Anna Müller, geboren am 12.03.1981 in München, erreichbar unter 0171-2345678, Steuer-ID 12 345 678 901, IBAN DE89 3704 0044 0532 0130 00.
|
|
116
|
+
|
|
117
|
+
Output:
|
|
118
|
+
|
|
119
|
+
> `<<PERSON_1>>`, geboren am `<<DATE_TIME_1>>` in `<<LOCATION_1>>`, erreichbar unter `<<PHONE_NUMBER_1>>`, Steuer-ID `<<DE_STEUER_ID_1>>`, IBAN `<<IBAN_CODE_1>>`.
|
|
120
|
+
|
|
121
|
+
## Commands
|
|
122
|
+
|
|
123
|
+
| Command | What it does |
|
|
124
|
+
|------------------------------|----------------------------------------------------------------------------------------|
|
|
125
|
+
| `noirdoc redact <files>` | Redact one or more files (accepts directories; `-o FILE` or `--output-dir DIR`). |
|
|
126
|
+
| `noirdoc reveal <file>` | Reverse pseudonyms back to originals (DOCX / XLSX / plain; `--namespace` required). |
|
|
127
|
+
| `noirdoc lookup <token>` | Resolve a pseudonym like `<<PERSON_1>>` to its original value. |
|
|
128
|
+
| `noirdoc ns list` | List persistent namespaces under `~/.noirdoc/namespaces/`. |
|
|
129
|
+
| `noirdoc ns show <name>` | Print the mapping summary for a namespace as JSON. |
|
|
130
|
+
| `noirdoc ns delete <name>` | Delete a namespace (prompts for confirmation). |
|
|
131
|
+
| `noirdoc models pull` | Download spaCy models and (optionally) GLiNER weights up front. |
|
|
132
|
+
|
|
133
|
+
Run `noirdoc <cmd> --help` for the full flag list on any subcommand.
|
|
134
|
+
|
|
135
|
+
## Before you start
|
|
136
|
+
|
|
137
|
+
A few honest caveats before you ship this into a pipeline:
|
|
138
|
+
|
|
139
|
+
- **Best results need `[full]`.** On first use (or via `noirdoc models pull`) the full extra downloads roughly **560 MB** of weights: spaCy `de_core_news_lg`, Flair `ner-german-large`, and a GLiNER multilingual model. Budget disk and bandwidth.
|
|
140
|
+
- **PDF reveal is not supported yet.** Round-tripping placeholders back into a PDF is a hard problem (position drift, font metrics, image-based redactions). PDFs redact cleanly; reveal is pass-through. DOCX, XLSX, and plain text round-trip fully.
|
|
141
|
+
- **Alpha API.** Classes and CLI flags may change between `0.1.x` and `0.2.x`. Pin accordingly.
|
|
142
|
+
- **Detector quality depends on the upstream models.** Presidio + Flair + GLiNER do the heavy lifting. Noirdoc adds German-specific recognizers on top, but it does not train models.
|
|
143
|
+
|
|
144
|
+
## German-first
|
|
145
|
+
|
|
146
|
+
Noirdoc defaults to German (`language="de"`) with fallback to `["de", "en"]` for mixed documents. What that actually means:
|
|
147
|
+
|
|
148
|
+
- **Custom recognizers** in `src/noirdoc/detection/presidio_detector.py`:
|
|
149
|
+
- `GermanPhoneRecognizer` — German phone formats (0171-..., +49...)
|
|
150
|
+
- `GermanSVNRRecognizer` — Sozialversicherungsnummer with checksum
|
|
151
|
+
- `GermanSteuerIDRecognizer` — 11-digit Steuer-ID with checksum
|
|
152
|
+
- `InvertedNameRecognizer` — registered for both `de` and `en` to catch "Nachname, Vorname" patterns
|
|
153
|
+
- **Flair `ner-german-large`** (XLM-R, F1 92.3 % on CoNLL-03 DE) handles lowercase German text — the case where spaCy tends to drop names.
|
|
154
|
+
- **GLiNER multilingual** catches entity types the others miss.
|
|
155
|
+
- **German-style lowercase** financial terms, German IBANs, German date formats, and German address patterns are covered in the test suite (`tests/test_presidio_detector.py`).
|
|
156
|
+
|
|
157
|
+
If you're working with German legal, medical, HR, or financial documents, this is what the defaults are tuned for.
|
|
158
|
+
|
|
159
|
+
## Supported formats
|
|
160
|
+
|
|
161
|
+
| Format | Redact | Reveal (round-trip) |
|
|
162
|
+
|------------------------------|--------|---------------------|
|
|
163
|
+
| PDF | ✓ | ✗ (pass-through) |
|
|
164
|
+
| DOCX | ✓ | ✓ |
|
|
165
|
+
| XLSX | ✓ | ✓ |
|
|
166
|
+
| Plain text / CSV / MD / HTML | ✓ | ✓ |
|
|
167
|
+
| PPTX / images | ✓ | ✗ (pass-through) |
|
|
168
|
+
|
|
169
|
+
PDF reveal is an open contribution target — see [CONTRIBUTING.md](https://github.com/nextaim-de/noirdoc/blob/main/CONTRIBUTING.md).
|
|
170
|
+
|
|
171
|
+
## Advanced: shared mapping storage
|
|
172
|
+
|
|
173
|
+
The `[redis]` extra ships a `RedisMappingBackend` that plugs into the lower-level `MappingStore` — the same primitive Noirdoc Cloud uses for request-scoped, encrypted, TTL-bounded mapping persistence across workers. It is **not** wired into `Redactor(namespace=...)`, which persists to the local filesystem under `~/.noirdoc/namespaces/`. Use `MappingStore` when you have multiple workers that need to share pseudonym mappings for the same request, or when you want encrypted-at-rest mappings with automatic expiry.
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
import asyncio
|
|
177
|
+
from cryptography.fernet import Fernet
|
|
178
|
+
from redis.asyncio import Redis
|
|
179
|
+
|
|
180
|
+
from noirdoc.mappings.backends.redis_backend import RedisMappingBackend
|
|
181
|
+
from noirdoc.mappings.store import MappingStore
|
|
182
|
+
|
|
183
|
+
async def main() -> None:
|
|
184
|
+
redis = Redis.from_url("redis://localhost:6379")
|
|
185
|
+
store = MappingStore(
|
|
186
|
+
backend=RedisMappingBackend(redis),
|
|
187
|
+
encryption_key=Fernet.generate_key(), # keep stable across workers
|
|
188
|
+
)
|
|
189
|
+
# store.save(request_id=..., tenant_id=..., mapper=...)
|
|
190
|
+
# mappings = await store.load(request_id)
|
|
191
|
+
|
|
192
|
+
asyncio.run(main())
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
The `encryption_key` must be identical across workers that need to read the same mappings. `MappingStore.save()` accepts a `ttl_days` kwarg (default 30).
|
|
196
|
+
|
|
197
|
+
## Noirdoc Cloud
|
|
198
|
+
|
|
199
|
+
Don't want to run this yourself? **[Noirdoc Cloud](https://noirdoc.de)** is the hosted API wrapper: a privacy-preserving reverse proxy for LLM calls that uses this exact pipeline, plus multi-tenancy, audit, and provider key management. Compliance story: what's on GitHub is what the cloud runs.
|
|
200
|
+
|
|
201
|
+
## Contributing
|
|
202
|
+
|
|
203
|
+
Bug reports, detectors, and format support are all welcome. See [CONTRIBUTING.md](https://github.com/nextaim-de/noirdoc/blob/main/CONTRIBUTING.md) for dev setup, tests, and the recognizer pattern.
|
|
204
|
+
|
|
205
|
+
## Security
|
|
206
|
+
|
|
207
|
+
Report vulnerabilities via GitHub's private vulnerability reporting — see [SECURITY.md](https://github.com/nextaim-de/noirdoc/blob/main/SECURITY.md). Please don't open public issues for security bugs.
|
|
208
|
+
|
|
209
|
+
## Changelog
|
|
210
|
+
|
|
211
|
+
See [CHANGELOG.md](https://github.com/nextaim-de/noirdoc/blob/main/CHANGELOG.md). Follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [SemVer](https://semver.org/).
|
|
212
|
+
|
|
213
|
+
## License
|
|
214
|
+
|
|
215
|
+
MIT © 2026 Antonio Maiolo / [Nextaim GmbH](https://nextaim.de). See [LICENSE](https://github.com/nextaim-de/noirdoc/blob/main/LICENSE).
|
|
216
|
+
|
|
217
|
+
---
|
|
218
|
+
|
|
219
|
+
<p align="center">
|
|
220
|
+
Built by <a href="https://nextaim.de">Nextaim GmbH</a> · <a href="https://noirdoc.de">noirdoc.de</a>
|
|
221
|
+
</p>
|