gdpr-pseudonymizer 2.1.3__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/PKG-INFO +69 -28
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/README.md +67 -26
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/batch.py +37 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/cli/commands/delete_mapping.py +375 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/destroy_table.py +9 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/import_mappings.py +20 -6
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/list_mappings.py +9 -4
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/process.py +2 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/config.py +19 -2
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/main.py +9 -3
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/validators.py +17 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/core/custom_pseudonym.py +906 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/core/document_processor.py +536 -66
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/core/legacy_names.py +690 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/core/mapping_key.py +227 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/data/database.py +872 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/models.py +11 -8
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/repositories/audit_repository.py +5 -3
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/data/repositories/mapping_repository.py +1201 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/exceptions.py +42 -1
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/en.qm +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/en.ts +69 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/fr.ts +67 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/main_window.py +30 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/gui/models/validation_state.py +1622 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/batch.py +205 -21
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/database.py +10 -4
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/home.py +47 -4
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/validation.py +349 -36
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/entity_panel.py +5 -3
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/batch_worker.py +31 -2
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/database_worker.py +8 -3
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/detection_worker.py +11 -7
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/finalization_worker.py +19 -5
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/locale/fr/LC_MESSAGES/cli.mo +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/locale/fr/LC_MESSAGES/cli.po +4 -4
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/nlp/hybrid_detector.py +2790 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/nlp/model_names.py +46 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/regex_matcher.py +40 -2
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/library_manager.py +2 -2
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/__init__.py +5 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/detection_patterns.yaml +16 -9
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/location_noise_filter.yaml +74 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/org_role_filter.yaml +138 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/person_boundaries.yaml +64 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/place_abbreviations.yaml +14 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/salutations.yaml +17 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/file_handler.py +19 -7
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/tabular_reader.py +5 -2
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/tabular_writer.py +5 -3
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/utils/text_form.py +79 -0
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/validation/models.py +830 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/validation/ui.py +27 -6
- gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/validation/workflow.py +1082 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/pyproject.toml +2 -2
- gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/cli/commands/delete_mapping.py +0 -283
- gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/data/database.py +0 -285
- gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/data/repositories/mapping_repository.py +0 -545
- gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/gui/models/validation_state.py +0 -568
- gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/locale/fr/LC_MESSAGES/cli.mo +0 -0
- gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/nlp/hybrid_detector.py +0 -400
- gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/validation/models.py +0 -422
- gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/validation/workflow.py +0 -529
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/LICENSE +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/config_show.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/export.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/init.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/list_entities.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/stats.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/validate_mappings.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/formatters.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/i18n.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/passphrase.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/progress.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/validation_stub.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/core/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/core/naive_processor.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/encryption.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/naive_data.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/repositories/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/repositories/metadata_repository.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/accessibility/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/accessibility/focus_manager.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/accessibility/shortcuts.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/app.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/config.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/dialogs/shortcuts_help.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/error_handler.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/fr.qm +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/translator.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/models/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/app.ico +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_16.png +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_256.png +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_32.png +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_48.png +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_512.png +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/themes/dark.qss +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/themes/high-contrast.qss +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/themes/light.qss +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/processing.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/results.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/settings.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/stub.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/confirm_dialog.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/drop_zone.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/entity_editor.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/passphrase_dialog.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/step_indicator.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/toast.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/model_manager.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/processing_worker.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/signals.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/locale/en/LC_MESSAGES/cli.mo +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/locale/en/LC_MESSAGES/cli.po +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/entity_detector.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/entity_grouping.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/geography_dictionary.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/name_dictionary.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/spacy_detector.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/stanza_detector.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/assignment_engine.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/gender_detector.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/neutral_id_generator.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/french_gender_lookup.json +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/french_geography.json +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/french_names.json +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/pseudonyms/lotr.json +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/pseudonyms/neutral.json +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/pseudonyms/star_wars.json +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/config_manager.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/french_patterns.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/logger.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/validation/__init__.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/validation/actions.py +0 -0
- {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/validation/context_precomputer.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gdpr-pseudonymizer
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: CLI and GUI tool for GDPR-compliant pseudonymization of French text documents using NLP-based entity detection and reversible mapping
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -30,7 +30,7 @@ Provides-Extra: gui
|
|
|
30
30
|
Provides-Extra: pdf
|
|
31
31
|
Requires-Dist: PySide6-Essentials (>=6.7.0,<6.8.0) ; (python_version >= "3.10" and python_version < "3.13") and (extra == "gui" or extra == "all")
|
|
32
32
|
Requires-Dist: PyYAML (>=6.0,<7.0)
|
|
33
|
-
Requires-Dist: SQLAlchemy (>=2.0.0,<
|
|
33
|
+
Requires-Dist: SQLAlchemy (>=2.0.0,<2.1.0)
|
|
34
34
|
Requires-Dist: click (>=8.0.0,<8.2.0)
|
|
35
35
|
Requires-Dist: cryptography (>=50.0.1,<51.0.0)
|
|
36
36
|
Requires-Dist: markdown-it-py (>=3.0.0,<4.0.0)
|
|
@@ -63,7 +63,16 @@ Transform sensitive French documents for safe AI analysis with local processing,
|
|
|
63
63
|
|
|
64
64
|
---
|
|
65
65
|
|
|
66
|
-
## What's New in v2.
|
|
66
|
+
## What's New in v2.2
|
|
67
|
+
|
|
68
|
+
- **Choose the spaCy model** — `--model` / `pseudonymization.model` accept any spaCy package name (e.g. `en_core_web_trf` for English); `spacy` still selects the French `fr_core_news_lg`. The model only drives NLP detection: regex patterns, dictionaries and pseudonym libraries stay French. See the [CLI Reference](docs/CLI-REFERENCE.md).
|
|
69
|
+
- **Retype entities during CLI validation (`[T]`)** — Switch a detected entity between PERSON, LOCATION and ORG for every occurrence, instead of rejecting and re-adding it
|
|
70
|
+
- **Cleaner entity spans** — Editing with `[E]` no longer erases neighbouring text; "Last, First" matching now requires a known first name; timestamps and stray punctuation are trimmed from NER span edges; names with non-French letters ("María", "Yılmaz") and hyphenated names after titles are detected whole
|
|
71
|
+
- **Fresh `pip install` fixed** — v2.1.3 crashed on import when pip resolved SQLAlchemy 2.1; SQLAlchemy is now capped below 2.1
|
|
72
|
+
- **Audit log records the real model** — e.g. `fr_core_news_lg-3.8.0` instead of `spacy-unknown`
|
|
73
|
+
|
|
74
|
+
<details>
|
|
75
|
+
<summary>v2.1 highlights</summary>
|
|
67
76
|
|
|
68
77
|
- **Validate-once-per-entity** — Accept or reject one entity occurrence and it applies to all same-text occurrences in the document (productivity boost for repeated names)
|
|
69
78
|
- **Excel/CSV Format Support** — Process `.xlsx` and `.csv` files with cell-aware pseudonymization for HR/compliance use cases (`pip install gdpr-pseudonymizer[excel]`)
|
|
@@ -71,6 +80,8 @@ Transform sensitive French documents for safe AI analysis with local processing,
|
|
|
71
80
|
- **NER Accuracy Improvements** — Expanded ORG detection patterns, POS-tag disambiguation for geography matching; LOCATION false-negative rate reduced from 27% to 13%
|
|
72
81
|
- **GUI Discoverability** — F1 keyboard shortcuts help dialog with all shortcut groups, database path persistence across sessions, "Hide confirmed" toggle
|
|
73
82
|
|
|
83
|
+
</details>
|
|
84
|
+
|
|
74
85
|
<!-- TODO: Add GUI screenshot -->
|
|
75
86
|
|
|
76
87
|
**Upgrade:** `pip install --upgrade gdpr-pseudonymizer[gui,excel]`
|
|
@@ -85,10 +96,14 @@ Pre-built standalone executables are available for Windows, macOS, and Linux. No
|
|
|
85
96
|
|
|
86
97
|
| Platform | File | Notes |
|
|
87
98
|
|----------|------|-------|
|
|
88
|
-
| **Windows** | `gdpr-pseudonymizer-2.
|
|
89
|
-
| **macOS (Apple Silicon)** | `gdpr-pseudonymizer-2.
|
|
90
|
-
| **
|
|
91
|
-
|
|
99
|
+
| **Windows** | `gdpr-pseudonymizer-2.2.0-windows-setup.exe` | Run the installer. Adds Start Menu shortcut. |
|
|
100
|
+
| **macOS (Apple Silicon)** | `gdpr-pseudonymizer-2.2.0-macos-arm64.dmg` | Open DMG, drag to Applications. |
|
|
101
|
+
| **Linux** | `gdpr-pseudonymizer-2.2.0-linux.AppImage` | `chmod +x` then run. |
|
|
102
|
+
|
|
103
|
+
> **Intel Macs:** no DMG is published. GitHub retired its Intel macOS runners, and an
|
|
104
|
+
> Apple Silicon runner cannot cross-build a working x86_64 bundle. Install with
|
|
105
|
+
> `pip install gdpr-pseudonymizer` instead — the CLI and GUI both work on Intel.
|
|
106
|
+
|
|
92
107
|
|
|
93
108
|
### Platform Notes
|
|
94
109
|
|
|
@@ -126,7 +141,7 @@ GDPR Pseudonymizer is a **privacy-first CLI and GUI tool** that combines AI effi
|
|
|
126
141
|
- ✅ **Zero telemetry** - No analytics, crash reporting, or external communication
|
|
127
142
|
|
|
128
143
|
### 🤝 **AI + Human Verification**
|
|
129
|
-
- ✅ **Hybrid detection** -
|
|
144
|
+
- ✅ **Hybrid detection** - NLP + regex + geography dictionary: F1 61.4% on 30 unseen documents (CI run 38382553368). Improving this is a priority — see [Technical Details](#-technical-details)
|
|
130
145
|
- ✅ **Mandatory validation** - You review and confirm all entities (ensures 100% accuracy)
|
|
131
146
|
- ✅ **Fast validation UI** - Rich CLI interface with keyboard shortcuts, <2 min per document
|
|
132
147
|
- ✅ **Smart workflow** - Entity-by-type grouping (PERSON → ORG → LOCATION) with context display
|
|
@@ -161,7 +176,7 @@ GDPR Pseudonymizer is a **privacy-first CLI and GUI tool** that combines AI effi
|
|
|
161
176
|
|
|
162
177
|
## 🚀 Quick Start
|
|
163
178
|
|
|
164
|
-
**Status:** 🎉 **v2.
|
|
179
|
+
**Status:** 🎉 **v2.2.0** (September 2026) — spaCy Model Choice, Entity Retyping & Span Fixes
|
|
165
180
|
|
|
166
181
|
### Getting Started
|
|
167
182
|
|
|
@@ -183,23 +198,25 @@ pip install gdpr-pseudonymizer[excel]
|
|
|
183
198
|
pip install gdpr-pseudonymizer[formats]
|
|
184
199
|
```
|
|
185
200
|
|
|
186
|
-
### What v2.
|
|
201
|
+
### What v2.2 Delivers
|
|
187
202
|
|
|
188
203
|
- 🖥️ **Desktop GUI** — Visual entity validation with drag-and-drop, batch dashboard, and database management
|
|
189
|
-
- 📦 **Standalone executables** — Windows .exe, macOS .dmg, Linux AppImage — no Python required
|
|
204
|
+
- 📦 **Standalone executables** — Windows .exe, macOS .dmg (Apple Silicon), Linux AppImage — no Python required
|
|
190
205
|
- ♿ **WCAG 2.1 AA accessibility** — Keyboard navigation, screen reader, high contrast mode
|
|
191
206
|
- 🌐 **French UI** — Complete FR/EN interface with live language switching
|
|
192
|
-
- 🤖 **AI-assisted detection** — Hybrid NLP + regex
|
|
207
|
+
- 🤖 **AI-assisted detection** — Hybrid NLP + regex detection: F1 61.4% (recall 60.6%) on 30 unseen documents by a different author (CI run 38382553368)
|
|
193
208
|
- ✅ **Mandatory human verification** — You review and confirm all entities (ensures 100% accuracy)
|
|
194
209
|
- 🔒 **100% local processing** — Your data never leaves your machine
|
|
195
210
|
- 📄 **PDF/DOCX support** — Process PDF and DOCX files directly (optional extras)
|
|
196
211
|
- 📊 **Excel/CSV support** — Process .xlsx and .csv files with cell-aware pseudonymization (optional extra: `[excel]`)
|
|
197
212
|
- 🆔 **Neutral ID theme** — Counter-based identifiers (PERSON-001, LIEU-001) for formal/legal contexts
|
|
198
213
|
- 🎯 **NER accuracy** — LOCATION false-negative rate reduced from 27% to 13% via regex expansion & POS disambiguation
|
|
214
|
+
- 🧠 **Choice of spaCy model** — French by default; any spaCy model for detection (e.g. `--model en_core_web_trf`)
|
|
215
|
+
- 🏷️ **Entity retyping** — Change an entity's type (PERSON/LOCATION/ORG) during CLI validation with `[T]`
|
|
199
216
|
|
|
200
|
-
**What v2.
|
|
217
|
+
**What v2.2 does NOT deliver:**
|
|
201
218
|
- ❌ Fully automatic "set and forget" processing
|
|
202
|
-
- ❌ 85%+ AI accuracy (current:
|
|
219
|
+
- ❌ 85%+ AI accuracy (current: F1 61.4% on unseen documents, 82.6% on our development corpus — we plan to raise it)
|
|
203
220
|
- ❌ Optional validation mode (validation is mandatory)
|
|
204
221
|
|
|
205
222
|
### Roadmap
|
|
@@ -210,10 +227,12 @@ pip install gdpr-pseudonymizer[formats]
|
|
|
210
227
|
|
|
211
228
|
**v2.0 (Q1 2026):** Desktop GUI, standalone executables, WCAG AA accessibility, French UI, batch validation, core hardening
|
|
212
229
|
|
|
213
|
-
**v2.1 (Q1 2026)
|
|
230
|
+
**v2.1 (Q1 2026):** GUI polish, Excel/CSV support, neutral ID theme, NER accuracy improvements, keyboard shortcuts help
|
|
231
|
+
|
|
232
|
+
**v2.2 (Q3 2026) — CURRENT RELEASE:** Choice of spaCy model, entity retyping in CLI validation, entity-span accuracy fixes, fresh-install fix (SQLAlchemy < 2.1)
|
|
214
233
|
|
|
215
234
|
**v3.0 (2027+):** NLP accuracy & automation
|
|
216
|
-
- Fine-tuned French NER model (70-85% F1 target, up from
|
|
235
|
+
- Fine-tuned French NER model (70-85% F1 target, up from 61.4% on unseen documents today)
|
|
217
236
|
- Optional `--no-validate` flag for high-confidence workflows
|
|
218
237
|
- Confidence-based auto-processing (85%+ F1 target)
|
|
219
238
|
- Multi-language support (English, Spanish, German)
|
|
@@ -362,6 +381,20 @@ GDPR_PSEUDO_LANG=fr gdpr-pseudo --help
|
|
|
362
381
|
|
|
363
382
|
## 🔬 Technical Details
|
|
364
383
|
|
|
384
|
+
### Detection accuracy (Epic 10, October 2026)
|
|
385
|
+
|
|
386
|
+
**On unseen documents: F1 61.4%** (95% range 59.3–63.6%), recall 60.6%, precision 62.3%. This is a blind final exam: 30 French documents written by a different author in messy formats (email threads with quoted replies, PDF-style line breaks, tables flattened to text, mixed case), scored once (CI run 38382553368). By type: PERSON 69.2% F1, LOCATION 68.7%, ORG 34.8%; organisations remain the weak point.
|
|
387
|
+
|
|
388
|
+
**For context:** 82.6% F1 on the development corpus (25 documents) and 78.1% on a held-out set (6 documents) (CI run 38212303870). Both are closer to the documents the rules were built on.
|
|
389
|
+
|
|
390
|
+
How the final exam was made, and its limits:
|
|
391
|
+
- An LLM (GPT) wrote the 30 documents from a brief that named the hard cases to include but quoted no corpus text.
|
|
392
|
+
- Two models (GPT and Gemini) annotated them independently, and the maintainer settled their disagreements.
|
|
393
|
+
- The set was frozen before the last detection change, then scored once.
|
|
394
|
+
- The documents are synthetic, so real documents may score differently.
|
|
395
|
+
|
|
396
|
+
Details: [docs/qa/ner-accuracy-report.md](docs/qa/ner-accuracy-report.md).
|
|
397
|
+
|
|
365
398
|
### NLP Library Selection (Story 1.2 - Completed)
|
|
366
399
|
|
|
367
400
|
After comprehensive benchmarking on 25 French interview/business documents (1,737 annotated entities):
|
|
@@ -369,10 +402,18 @@ After comprehensive benchmarking on 25 French interview/business documents (1,73
|
|
|
369
402
|
| Approach | F1 Score | Precision | Recall | Notes |
|
|
370
403
|
|----------|----------|-----------|--------|-------|
|
|
371
404
|
| **spaCy only** `fr_core_news_lg` | 29.5% | 27.0% | 32.7% | Story 1.2 baseline |
|
|
372
|
-
| **Hybrid** (spaCy + regex) | 59.97% | 48.17% | 79.45% | Story 5.3 |
|
|
373
|
-
| **Hybrid
|
|
405
|
+
| **Hybrid** (spaCy + regex) | 59.97% | 48.17% | 79.45% | Story 5.3 (see scoring note) |
|
|
406
|
+
| **Hybrid (v2.2.0)** | **58.53%** | 48.42% | 73.98% | Scored the way the app normalizes entities |
|
|
407
|
+
|
|
408
|
+
By entity type (v2.2.0): PERSON 72.9% F1, LOCATION 45.0%, ORG 12.9%. Organisation detection is the weak point: job titles such as CTO, DPO or RSSI are often mistaken for companies.
|
|
409
|
+
|
|
410
|
+
**How these numbers are scored (corrected October 2026).** The app strips titles and prepositions before pseudonymizing ("Mme Isabelle Moreau" is handled as "Isabelle Moreau", "à Paris" as "Paris"), and the benchmark now scores detections the same way. Until then the committed scorer compared raw text, so every correctly detected "Mme Isabelle Moreau" counted as a miss plus a false alarm. Under that strict scoring the same detector measured ~32% F1, a figure briefly published on this page in October 2026; it understated real-use detection. The Story 5.3 figure (59.97%) appears to have been measured with title-aware matching and is consistent with v2.2.0's 58.53%. Each number in the table above was measured with `pytest tests/accuracy -m accuracy -s` on the version it names.
|
|
411
|
+
|
|
412
|
+
**Read these numbers with two caveats.** They come from one 25-document French interview/business corpus, and part of its ground truth was auto-annotated with the tool's own regex patterns, which skews results in both directions. And they measure *pre-detection* only: mandatory validation is what makes the final output correct.
|
|
413
|
+
|
|
414
|
+
**Epic 10 (October 2026)** repaired that benchmark against written annotation guidelines, added a held-out set and the blind final exam above, and improved organisation detection. A fine-tuned French NER model remains the v3.0 target (70-85% F1).
|
|
374
415
|
|
|
375
|
-
**Accuracy trajectory:**
|
|
416
|
+
**Accuracy trajectory:** Story 7.5 added 12 ORG pattern keywords, POS-tag disambiguation for geography matching, and 7 international locations — reducing LOCATION false-negative rate from 27.42% to 12.90%.
|
|
376
417
|
|
|
377
418
|
**Approved Solution:**
|
|
378
419
|
- ✅ **Hybrid approach** (NLP + regex + geography dictionary + POS disambiguation)
|
|
@@ -389,7 +430,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
|
|
|
389
430
|
- ✅ **Entity-by-type grouping** - Review PERSON → ORG → LOCATION in logical order
|
|
390
431
|
- ✅ **Context display** - See 10 words before/after each entity with highlighting
|
|
391
432
|
- ✅ **Confidence scores** - Color-coded confidence from spaCy NER (green >80%, yellow 60-80%, red <60%)
|
|
392
|
-
- ✅ **Keyboard shortcuts** - Single-key actions: [Space] Confirm, [R] Reject, [E] Modify, [A] Add, [C] Change pseudonym
|
|
433
|
+
- ✅ **Keyboard shortcuts** - Single-key actions: [Space] Confirm, [R] Reject, [E] Modify, [T] Change type, [A] Add, [C] Change pseudonym
|
|
393
434
|
- ✅ **Batch operations** - Accept/reject all entities of a type at once (Shift+A/R) with entity count feedback
|
|
394
435
|
- ✅ **Context cycling indicator** - Dot indicator (`● ○ ○ ○ ○`) shows current context position; `[Press X to cycle]` hint improves discoverability
|
|
395
436
|
- ✅ **Help overlay** - Press [H] for full command reference
|
|
@@ -417,7 +458,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
|
|
|
417
458
|
| **CLI Framework** | Typer | 0.9+ | Command-line interface |
|
|
418
459
|
| **Database** | SQLite | 3.35+ | Local mapping table storage with WAL mode |
|
|
419
460
|
| **Encryption** | cryptography (AESSIV) | 44.0+ | AES-256-SIV encryption for sensitive fields (PBKDF2 key derivation, passphrase-protected) |
|
|
420
|
-
| **ORM** | SQLAlchemy | 2.0
|
|
461
|
+
| **ORM** | SQLAlchemy | 2.0.x (<2.1) | Database abstraction and session management |
|
|
421
462
|
| **Desktop GUI** | PySide6 | 6.7+ | Desktop application (optional: `pip install gdpr-pseudonymizer[gui]`) |
|
|
422
463
|
| **Validation UI** | rich | 13.7+ | Interactive CLI entity review |
|
|
423
464
|
| **Keyboard Input** | readchar | 4.2+ | Single-keypress capture for validation UI |
|
|
@@ -433,7 +474,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
|
|
|
433
474
|
**Long answer:**
|
|
434
475
|
1. **GDPR defensibility** - Human verification provides legal audit trail
|
|
435
476
|
2. **Zero false negatives** - AI misses entities, humans catch them (100% coverage)
|
|
436
|
-
3. **Current NLP limitations** - French models on interview/business docs: 29.5% F1 out-of-box (hybrid approach
|
|
477
|
+
3. **Current NLP limitations** - French models on interview/business docs: 29.5% F1 out-of-box (hybrid approach: F1 61.4% and recall 60.6% on unseen documents, 82.6% F1 on our development corpus; organisations are the weak point; improvement planned)
|
|
437
478
|
4. **Better than alternatives:**
|
|
438
479
|
- ✅ **vs Manual redaction:** 50%+ faster (AI pre-detection)
|
|
439
480
|
- ✅ **vs Cloud services:** 100% local processing (no data leakage)
|
|
@@ -519,13 +560,13 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
|
|
|
519
560
|
|
|
520
561
|
## 🛠️ Development Status
|
|
521
562
|
|
|
522
|
-
**Epics 1-7 Complete** — v2.
|
|
563
|
+
**Epics 1-7 Complete** — current release v2.2.0 (September 2026): spaCy model choice, entity retyping, entity-span fixes. v2.1.0 (March 2026) closed Epic 7.
|
|
523
564
|
|
|
524
565
|
- ✅ **Epic 1:** Foundation & NLP Validation (9 stories) — spaCy integration, validation UI, hybrid detection, entity deduplication
|
|
525
566
|
- ✅ **Epic 2:** Core Pseudonymization Engine (9 stories) — pseudonym libraries, encryption, audit logging, batch processing, GDPR 1:1 mapping
|
|
526
567
|
- ✅ **Epic 3:** CLI Interface & Batch Processing (7 stories) — 8 CLI commands, progress reporting, config files, parallel batch, UX polish
|
|
527
568
|
- ✅ **Epic 4:** Launch Readiness (8 stories) — LLM utility validation, cross-platform testing, documentation, NER accuracy suite, performance validation, beta feedback integration, codebase refactoring, launch preparation
|
|
528
|
-
- ✅ **Epic 5:** Quick Wins & GDPR Compliance (7 stories) — GDPR Article 17 erasure, gender-aware pseudonyms, NER accuracy
|
|
569
|
+
- ✅ **Epic 5:** Quick Wins & GDPR Compliance (7 stories) — GDPR Article 17 erasure, gender-aware pseudonyms, NER accuracy work (annotation cleanup, expanded regex, geography dictionary), French documentation translation, PDF/DOCX support, CLI polish & benchmarks, v1.1 release
|
|
529
570
|
- ✅ **Epic 6:** v2.0 Desktop GUI & Broader Accessibility (9 stories) — PySide6 desktop application, visual entity validation, batch GUI, i18n, WCAG AA, standalone executables
|
|
530
571
|
- ✅ Story 6.1: UX Architecture & GUI Framework Selection
|
|
531
572
|
- ✅ Story 6.2: GUI Application Foundation (main window, theming, home screen, settings, 77 GUI tests)
|
|
@@ -609,7 +650,7 @@ This project is licensed under the [MIT License](LICENSE).
|
|
|
609
650
|
- ⚠️ Test thoroughly before production use
|
|
610
651
|
|
|
611
652
|
**Current limitations:**
|
|
612
|
-
- AI detection:
|
|
653
|
+
- AI detection: F1 61.4% on unseen documents (82.6% on our development corpus), not 85%+; improving it is planned
|
|
613
654
|
- Validation required for ALL documents (not optional)
|
|
614
655
|
- French documents only (English, Spanish, etc. in future versions)
|
|
615
656
|
- Text-based formats: .txt, .md, .pdf, .docx, .xlsx, .csv (PDF/DOCX/Excel require optional extras: `pip install gdpr-pseudonymizer[formats]`)
|
|
@@ -730,7 +771,7 @@ The integration test suite covers:
|
|
|
730
771
|
|
|
731
772
|
| Metric | Value | Status |
|
|
732
773
|
|--------|-------|--------|
|
|
733
|
-
| **Development Progress** | v2.
|
|
774
|
+
| **Development Progress** | v2.2.0 | ✅ Epics 1-7 complete |
|
|
734
775
|
| **Stories Complete** | 60 (Epics 1-7) | ✅ All epics complete |
|
|
735
776
|
| **LLM Utility (NFR10)** | 4.27/5.0 (85.4%) | ✅ PASSED (threshold: 80%) |
|
|
736
777
|
| **Installation Success (NFR3)** | 87.5% (7/8 platforms) | ✅ PASSED (threshold: 85%) |
|
|
@@ -738,7 +779,7 @@ The integration test suite covers:
|
|
|
738
779
|
| **Critical Bugs Found** | 1 (Story 2.8) | ✅ RESOLVED - Epic 3 Unblocked |
|
|
739
780
|
| **Test Corpus Size** | 25 docs, 1,737 entities | ✅ Complete (post-cleanup) |
|
|
740
781
|
| **NLP Accuracy (Baseline)** | 29.5% F1 (spaCy only) | ✅ Measured (Story 1.2) |
|
|
741
|
-
| **Hybrid Accuracy (NLP+Regex)** |
|
|
782
|
+
| **Hybrid Accuracy (NLP+Regex)** | 61.4% F1 on unseen documents (final exam, CI run 38382553368); 82.6% development corpus, 78.1% held-out (CI run 38212303870) | ⚠️ Below target — improvement planned |
|
|
742
783
|
| **Final Accuracy (AI+Human)** | 100% (validated) | 🎯 By Design |
|
|
743
784
|
| **Pseudonym Libraries** | 3 themes (2,426 names + 240 locations + 588 orgs) | ✅ Stories 2.1, 3.0, 4.6 Complete |
|
|
744
785
|
| **Compositional Matching** | Operational (component reuse + title stripping + compound names) | ✅ Stories 2.2, 2.3 Complete |
|
|
@@ -770,5 +811,5 @@ The integration test suite covers:
|
|
|
770
811
|
|
|
771
812
|
---
|
|
772
813
|
|
|
773
|
-
**Last Updated:** 2026-
|
|
814
|
+
**Last Updated:** 2026-10-01 (v2.2.0 — spaCy model choice, entity retyping, entity-span fixes)
|
|
774
815
|
|
|
@@ -14,7 +14,16 @@ Transform sensitive French documents for safe AI analysis with local processing,
|
|
|
14
14
|
|
|
15
15
|
---
|
|
16
16
|
|
|
17
|
-
## What's New in v2.
|
|
17
|
+
## What's New in v2.2
|
|
18
|
+
|
|
19
|
+
- **Choose the spaCy model** — `--model` / `pseudonymization.model` accept any spaCy package name (e.g. `en_core_web_trf` for English); `spacy` still selects the French `fr_core_news_lg`. The model only drives NLP detection: regex patterns, dictionaries and pseudonym libraries stay French. See the [CLI Reference](docs/CLI-REFERENCE.md).
|
|
20
|
+
- **Retype entities during CLI validation (`[T]`)** — Switch a detected entity between PERSON, LOCATION and ORG for every occurrence, instead of rejecting and re-adding it
|
|
21
|
+
- **Cleaner entity spans** — Editing with `[E]` no longer erases neighbouring text; "Last, First" matching now requires a known first name; timestamps and stray punctuation are trimmed from NER span edges; names with non-French letters ("María", "Yılmaz") and hyphenated names after titles are detected whole
|
|
22
|
+
- **Fresh `pip install` fixed** — v2.1.3 crashed on import when pip resolved SQLAlchemy 2.1; SQLAlchemy is now capped below 2.1
|
|
23
|
+
- **Audit log records the real model** — e.g. `fr_core_news_lg-3.8.0` instead of `spacy-unknown`
|
|
24
|
+
|
|
25
|
+
<details>
|
|
26
|
+
<summary>v2.1 highlights</summary>
|
|
18
27
|
|
|
19
28
|
- **Validate-once-per-entity** — Accept or reject one entity occurrence and it applies to all same-text occurrences in the document (productivity boost for repeated names)
|
|
20
29
|
- **Excel/CSV Format Support** — Process `.xlsx` and `.csv` files with cell-aware pseudonymization for HR/compliance use cases (`pip install gdpr-pseudonymizer[excel]`)
|
|
@@ -22,6 +31,8 @@ Transform sensitive French documents for safe AI analysis with local processing,
|
|
|
22
31
|
- **NER Accuracy Improvements** — Expanded ORG detection patterns, POS-tag disambiguation for geography matching; LOCATION false-negative rate reduced from 27% to 13%
|
|
23
32
|
- **GUI Discoverability** — F1 keyboard shortcuts help dialog with all shortcut groups, database path persistence across sessions, "Hide confirmed" toggle
|
|
24
33
|
|
|
34
|
+
</details>
|
|
35
|
+
|
|
25
36
|
<!-- TODO: Add GUI screenshot -->
|
|
26
37
|
|
|
27
38
|
**Upgrade:** `pip install --upgrade gdpr-pseudonymizer[gui,excel]`
|
|
@@ -36,10 +47,14 @@ Pre-built standalone executables are available for Windows, macOS, and Linux. No
|
|
|
36
47
|
|
|
37
48
|
| Platform | File | Notes |
|
|
38
49
|
|----------|------|-------|
|
|
39
|
-
| **Windows** | `gdpr-pseudonymizer-2.
|
|
40
|
-
| **macOS (Apple Silicon)** | `gdpr-pseudonymizer-2.
|
|
41
|
-
| **
|
|
42
|
-
|
|
50
|
+
| **Windows** | `gdpr-pseudonymizer-2.2.0-windows-setup.exe` | Run the installer. Adds Start Menu shortcut. |
|
|
51
|
+
| **macOS (Apple Silicon)** | `gdpr-pseudonymizer-2.2.0-macos-arm64.dmg` | Open DMG, drag to Applications. |
|
|
52
|
+
| **Linux** | `gdpr-pseudonymizer-2.2.0-linux.AppImage` | `chmod +x` then run. |
|
|
53
|
+
|
|
54
|
+
> **Intel Macs:** no DMG is published. GitHub retired its Intel macOS runners, and an
|
|
55
|
+
> Apple Silicon runner cannot cross-build a working x86_64 bundle. Install with
|
|
56
|
+
> `pip install gdpr-pseudonymizer` instead — the CLI and GUI both work on Intel.
|
|
57
|
+
|
|
43
58
|
|
|
44
59
|
### Platform Notes
|
|
45
60
|
|
|
@@ -77,7 +92,7 @@ GDPR Pseudonymizer is a **privacy-first CLI and GUI tool** that combines AI effi
|
|
|
77
92
|
- ✅ **Zero telemetry** - No analytics, crash reporting, or external communication
|
|
78
93
|
|
|
79
94
|
### 🤝 **AI + Human Verification**
|
|
80
|
-
- ✅ **Hybrid detection** -
|
|
95
|
+
- ✅ **Hybrid detection** - NLP + regex + geography dictionary: F1 61.4% on 30 unseen documents (CI run 38382553368). Improving this is a priority — see [Technical Details](#-technical-details)
|
|
81
96
|
- ✅ **Mandatory validation** - You review and confirm all entities (ensures 100% accuracy)
|
|
82
97
|
- ✅ **Fast validation UI** - Rich CLI interface with keyboard shortcuts, <2 min per document
|
|
83
98
|
- ✅ **Smart workflow** - Entity-by-type grouping (PERSON → ORG → LOCATION) with context display
|
|
@@ -112,7 +127,7 @@ GDPR Pseudonymizer is a **privacy-first CLI and GUI tool** that combines AI effi
|
|
|
112
127
|
|
|
113
128
|
## 🚀 Quick Start
|
|
114
129
|
|
|
115
|
-
**Status:** 🎉 **v2.
|
|
130
|
+
**Status:** 🎉 **v2.2.0** (September 2026) — spaCy Model Choice, Entity Retyping & Span Fixes
|
|
116
131
|
|
|
117
132
|
### Getting Started
|
|
118
133
|
|
|
@@ -134,23 +149,25 @@ pip install gdpr-pseudonymizer[excel]
|
|
|
134
149
|
pip install gdpr-pseudonymizer[formats]
|
|
135
150
|
```
|
|
136
151
|
|
|
137
|
-
### What v2.
|
|
152
|
+
### What v2.2 Delivers
|
|
138
153
|
|
|
139
154
|
- 🖥️ **Desktop GUI** — Visual entity validation with drag-and-drop, batch dashboard, and database management
|
|
140
|
-
- 📦 **Standalone executables** — Windows .exe, macOS .dmg, Linux AppImage — no Python required
|
|
155
|
+
- 📦 **Standalone executables** — Windows .exe, macOS .dmg (Apple Silicon), Linux AppImage — no Python required
|
|
141
156
|
- ♿ **WCAG 2.1 AA accessibility** — Keyboard navigation, screen reader, high contrast mode
|
|
142
157
|
- 🌐 **French UI** — Complete FR/EN interface with live language switching
|
|
143
|
-
- 🤖 **AI-assisted detection** — Hybrid NLP + regex
|
|
158
|
+
- 🤖 **AI-assisted detection** — Hybrid NLP + regex detection: F1 61.4% (recall 60.6%) on 30 unseen documents by a different author (CI run 38382553368)
|
|
144
159
|
- ✅ **Mandatory human verification** — You review and confirm all entities (ensures 100% accuracy)
|
|
145
160
|
- 🔒 **100% local processing** — Your data never leaves your machine
|
|
146
161
|
- 📄 **PDF/DOCX support** — Process PDF and DOCX files directly (optional extras)
|
|
147
162
|
- 📊 **Excel/CSV support** — Process .xlsx and .csv files with cell-aware pseudonymization (optional extra: `[excel]`)
|
|
148
163
|
- 🆔 **Neutral ID theme** — Counter-based identifiers (PERSON-001, LIEU-001) for formal/legal contexts
|
|
149
164
|
- 🎯 **NER accuracy** — LOCATION false-negative rate reduced from 27% to 13% via regex expansion & POS disambiguation
|
|
165
|
+
- 🧠 **Choice of spaCy model** — French by default; any spaCy model for detection (e.g. `--model en_core_web_trf`)
|
|
166
|
+
- 🏷️ **Entity retyping** — Change an entity's type (PERSON/LOCATION/ORG) during CLI validation with `[T]`
|
|
150
167
|
|
|
151
|
-
**What v2.
|
|
168
|
+
**What v2.2 does NOT deliver:**
|
|
152
169
|
- ❌ Fully automatic "set and forget" processing
|
|
153
|
-
- ❌ 85%+ AI accuracy (current:
|
|
170
|
+
- ❌ 85%+ AI accuracy (current: F1 61.4% on unseen documents, 82.6% on our development corpus — we plan to raise it)
|
|
154
171
|
- ❌ Optional validation mode (validation is mandatory)
|
|
155
172
|
|
|
156
173
|
### Roadmap
|
|
@@ -161,10 +178,12 @@ pip install gdpr-pseudonymizer[formats]
|
|
|
161
178
|
|
|
162
179
|
**v2.0 (Q1 2026):** Desktop GUI, standalone executables, WCAG AA accessibility, French UI, batch validation, core hardening
|
|
163
180
|
|
|
164
|
-
**v2.1 (Q1 2026)
|
|
181
|
+
**v2.1 (Q1 2026):** GUI polish, Excel/CSV support, neutral ID theme, NER accuracy improvements, keyboard shortcuts help
|
|
182
|
+
|
|
183
|
+
**v2.2 (Q3 2026) — CURRENT RELEASE:** Choice of spaCy model, entity retyping in CLI validation, entity-span accuracy fixes, fresh-install fix (SQLAlchemy < 2.1)
|
|
165
184
|
|
|
166
185
|
**v3.0 (2027+):** NLP accuracy & automation
|
|
167
|
-
- Fine-tuned French NER model (70-85% F1 target, up from
|
|
186
|
+
- Fine-tuned French NER model (70-85% F1 target, up from 61.4% on unseen documents today)
|
|
168
187
|
- Optional `--no-validate` flag for high-confidence workflows
|
|
169
188
|
- Confidence-based auto-processing (85%+ F1 target)
|
|
170
189
|
- Multi-language support (English, Spanish, German)
|
|
@@ -313,6 +332,20 @@ GDPR_PSEUDO_LANG=fr gdpr-pseudo --help
|
|
|
313
332
|
|
|
314
333
|
## 🔬 Technical Details
|
|
315
334
|
|
|
335
|
+
### Detection accuracy (Epic 10, October 2026)
|
|
336
|
+
|
|
337
|
+
**On unseen documents: F1 61.4%** (95% range 59.3–63.6%), recall 60.6%, precision 62.3%. This is a blind final exam: 30 French documents written by a different author in messy formats (email threads with quoted replies, PDF-style line breaks, tables flattened to text, mixed case), scored once (CI run 38382553368). By type: PERSON 69.2% F1, LOCATION 68.7%, ORG 34.8%; organisations remain the weak point.
|
|
338
|
+
|
|
339
|
+
**For context:** 82.6% F1 on the development corpus (25 documents) and 78.1% on a held-out set (6 documents) (CI run 38212303870). Both are closer to the documents the rules were built on.
|
|
340
|
+
|
|
341
|
+
How the final exam was made, and its limits:
|
|
342
|
+
- An LLM (GPT) wrote the 30 documents from a brief that named the hard cases to include but quoted no corpus text.
|
|
343
|
+
- Two models (GPT and Gemini) annotated them independently, and the maintainer settled their disagreements.
|
|
344
|
+
- The set was frozen before the last detection change, then scored once.
|
|
345
|
+
- The documents are synthetic, so real documents may score differently.
|
|
346
|
+
|
|
347
|
+
Details: [docs/qa/ner-accuracy-report.md](docs/qa/ner-accuracy-report.md).
|
|
348
|
+
|
|
316
349
|
### NLP Library Selection (Story 1.2 - Completed)
|
|
317
350
|
|
|
318
351
|
After comprehensive benchmarking on 25 French interview/business documents (1,737 annotated entities):
|
|
@@ -320,10 +353,18 @@ After comprehensive benchmarking on 25 French interview/business documents (1,73
|
|
|
320
353
|
| Approach | F1 Score | Precision | Recall | Notes |
|
|
321
354
|
|----------|----------|-----------|--------|-------|
|
|
322
355
|
| **spaCy only** `fr_core_news_lg` | 29.5% | 27.0% | 32.7% | Story 1.2 baseline |
|
|
323
|
-
| **Hybrid** (spaCy + regex) | 59.97% | 48.17% | 79.45% | Story 5.3 |
|
|
324
|
-
| **Hybrid
|
|
356
|
+
| **Hybrid** (spaCy + regex) | 59.97% | 48.17% | 79.45% | Story 5.3 (see scoring note) |
|
|
357
|
+
| **Hybrid (v2.2.0)** | **58.53%** | 48.42% | 73.98% | Scored the way the app normalizes entities |
|
|
358
|
+
|
|
359
|
+
By entity type (v2.2.0): PERSON 72.9% F1, LOCATION 45.0%, ORG 12.9%. Organisation detection is the weak point: job titles such as CTO, DPO or RSSI are often mistaken for companies.
|
|
360
|
+
|
|
361
|
+
**How these numbers are scored (corrected October 2026).** The app strips titles and prepositions before pseudonymizing ("Mme Isabelle Moreau" is handled as "Isabelle Moreau", "à Paris" as "Paris"), and the benchmark now scores detections the same way. Until then the committed scorer compared raw text, so every correctly detected "Mme Isabelle Moreau" counted as a miss plus a false alarm. Under that strict scoring the same detector measured ~32% F1, a figure briefly published on this page in October 2026; it understated real-use detection. The Story 5.3 figure (59.97%) appears to have been measured with title-aware matching and is consistent with v2.2.0's 58.53%. Each number in the table above was measured with `pytest tests/accuracy -m accuracy -s` on the version it names.
|
|
362
|
+
|
|
363
|
+
**Read these numbers with two caveats.** They come from one 25-document French interview/business corpus, and part of its ground truth was auto-annotated with the tool's own regex patterns, which skews results in both directions. And they measure *pre-detection* only: mandatory validation is what makes the final output correct.
|
|
364
|
+
|
|
365
|
+
**Epic 10 (October 2026)** repaired that benchmark against written annotation guidelines, added a held-out set and the blind final exam above, and improved organisation detection. A fine-tuned French NER model remains the v3.0 target (70-85% F1).
|
|
325
366
|
|
|
326
|
-
**Accuracy trajectory:**
|
|
367
|
+
**Accuracy trajectory:** Story 7.5 added 12 ORG pattern keywords, POS-tag disambiguation for geography matching, and 7 international locations — reducing LOCATION false-negative rate from 27.42% to 12.90%.
|
|
327
368
|
|
|
328
369
|
**Approved Solution:**
|
|
329
370
|
- ✅ **Hybrid approach** (NLP + regex + geography dictionary + POS disambiguation)
|
|
@@ -340,7 +381,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
|
|
|
340
381
|
- ✅ **Entity-by-type grouping** - Review PERSON → ORG → LOCATION in logical order
|
|
341
382
|
- ✅ **Context display** - See 10 words before/after each entity with highlighting
|
|
342
383
|
- ✅ **Confidence scores** - Color-coded confidence from spaCy NER (green >80%, yellow 60-80%, red <60%)
|
|
343
|
-
- ✅ **Keyboard shortcuts** - Single-key actions: [Space] Confirm, [R] Reject, [E] Modify, [A] Add, [C] Change pseudonym
|
|
384
|
+
- ✅ **Keyboard shortcuts** - Single-key actions: [Space] Confirm, [R] Reject, [E] Modify, [T] Change type, [A] Add, [C] Change pseudonym
|
|
344
385
|
- ✅ **Batch operations** - Accept/reject all entities of a type at once (Shift+A/R) with entity count feedback
|
|
345
386
|
- ✅ **Context cycling indicator** - Dot indicator (`● ○ ○ ○ ○`) shows current context position; `[Press X to cycle]` hint improves discoverability
|
|
346
387
|
- ✅ **Help overlay** - Press [H] for full command reference
|
|
@@ -368,7 +409,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
|
|
|
368
409
|
| **CLI Framework** | Typer | 0.9+ | Command-line interface |
|
|
369
410
|
| **Database** | SQLite | 3.35+ | Local mapping table storage with WAL mode |
|
|
370
411
|
| **Encryption** | cryptography (AESSIV) | 44.0+ | AES-256-SIV encryption for sensitive fields (PBKDF2 key derivation, passphrase-protected) |
|
|
371
|
-
| **ORM** | SQLAlchemy | 2.0
|
|
412
|
+
| **ORM** | SQLAlchemy | 2.0.x (<2.1) | Database abstraction and session management |
|
|
372
413
|
| **Desktop GUI** | PySide6 | 6.7+ | Desktop application (optional: `pip install gdpr-pseudonymizer[gui]`) |
|
|
373
414
|
| **Validation UI** | rich | 13.7+ | Interactive CLI entity review |
|
|
374
415
|
| **Keyboard Input** | readchar | 4.2+ | Single-keypress capture for validation UI |
|
|
@@ -384,7 +425,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
|
|
|
384
425
|
**Long answer:**
|
|
385
426
|
1. **GDPR defensibility** - Human verification provides legal audit trail
|
|
386
427
|
2. **Zero false negatives** - AI misses entities, humans catch them (100% coverage)
|
|
387
|
-
3. **Current NLP limitations** - French models on interview/business docs: 29.5% F1 out-of-box (hybrid approach
|
|
428
|
+
3. **Current NLP limitations** - French models on interview/business docs: 29.5% F1 out-of-box (hybrid approach: F1 61.4% and recall 60.6% on unseen documents, 82.6% F1 on our development corpus; organisations are the weak point; improvement planned)
|
|
388
429
|
4. **Better than alternatives:**
|
|
389
430
|
- ✅ **vs Manual redaction:** 50%+ faster (AI pre-detection)
|
|
390
431
|
- ✅ **vs Cloud services:** 100% local processing (no data leakage)
|
|
@@ -470,13 +511,13 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
|
|
|
470
511
|
|
|
471
512
|
## 🛠️ Development Status
|
|
472
513
|
|
|
473
|
-
**Epics 1-7 Complete** — v2.
|
|
514
|
+
**Epics 1-7 Complete** — current release v2.2.0 (September 2026): spaCy model choice, entity retyping, entity-span fixes. v2.1.0 (March 2026) closed Epic 7.
|
|
474
515
|
|
|
475
516
|
- ✅ **Epic 1:** Foundation & NLP Validation (9 stories) — spaCy integration, validation UI, hybrid detection, entity deduplication
|
|
476
517
|
- ✅ **Epic 2:** Core Pseudonymization Engine (9 stories) — pseudonym libraries, encryption, audit logging, batch processing, GDPR 1:1 mapping
|
|
477
518
|
- ✅ **Epic 3:** CLI Interface & Batch Processing (7 stories) — 8 CLI commands, progress reporting, config files, parallel batch, UX polish
|
|
478
519
|
- ✅ **Epic 4:** Launch Readiness (8 stories) — LLM utility validation, cross-platform testing, documentation, NER accuracy suite, performance validation, beta feedback integration, codebase refactoring, launch preparation
|
|
479
|
-
- ✅ **Epic 5:** Quick Wins & GDPR Compliance (7 stories) — GDPR Article 17 erasure, gender-aware pseudonyms, NER accuracy
|
|
520
|
+
- ✅ **Epic 5:** Quick Wins & GDPR Compliance (7 stories) — GDPR Article 17 erasure, gender-aware pseudonyms, NER accuracy work (annotation cleanup, expanded regex, geography dictionary), French documentation translation, PDF/DOCX support, CLI polish & benchmarks, v1.1 release
|
|
480
521
|
- ✅ **Epic 6:** v2.0 Desktop GUI & Broader Accessibility (9 stories) — PySide6 desktop application, visual entity validation, batch GUI, i18n, WCAG AA, standalone executables
|
|
481
522
|
- ✅ Story 6.1: UX Architecture & GUI Framework Selection
|
|
482
523
|
- ✅ Story 6.2: GUI Application Foundation (main window, theming, home screen, settings, 77 GUI tests)
|
|
@@ -560,7 +601,7 @@ This project is licensed under the [MIT License](LICENSE).
|
|
|
560
601
|
- ⚠️ Test thoroughly before production use
|
|
561
602
|
|
|
562
603
|
**Current limitations:**
|
|
563
|
-
- AI detection:
|
|
604
|
+
- AI detection: F1 61.4% on unseen documents (82.6% on our development corpus), not 85%+; improving it is planned
|
|
564
605
|
- Validation required for ALL documents (not optional)
|
|
565
606
|
- French documents only (English, Spanish, etc. in future versions)
|
|
566
607
|
- Text-based formats: .txt, .md, .pdf, .docx, .xlsx, .csv (PDF/DOCX/Excel require optional extras: `pip install gdpr-pseudonymizer[formats]`)
|
|
@@ -681,7 +722,7 @@ The integration test suite covers:
|
|
|
681
722
|
|
|
682
723
|
| Metric | Value | Status |
|
|
683
724
|
|--------|-------|--------|
|
|
684
|
-
| **Development Progress** | v2.
|
|
725
|
+
| **Development Progress** | v2.2.0 | ✅ Epics 1-7 complete |
|
|
685
726
|
| **Stories Complete** | 60 (Epics 1-7) | ✅ All epics complete |
|
|
686
727
|
| **LLM Utility (NFR10)** | 4.27/5.0 (85.4%) | ✅ PASSED (threshold: 80%) |
|
|
687
728
|
| **Installation Success (NFR3)** | 87.5% (7/8 platforms) | ✅ PASSED (threshold: 85%) |
|
|
@@ -689,7 +730,7 @@ The integration test suite covers:
|
|
|
689
730
|
| **Critical Bugs Found** | 1 (Story 2.8) | ✅ RESOLVED - Epic 3 Unblocked |
|
|
690
731
|
| **Test Corpus Size** | 25 docs, 1,737 entities | ✅ Complete (post-cleanup) |
|
|
691
732
|
| **NLP Accuracy (Baseline)** | 29.5% F1 (spaCy only) | ✅ Measured (Story 1.2) |
|
|
692
|
-
| **Hybrid Accuracy (NLP+Regex)** |
|
|
733
|
+
| **Hybrid Accuracy (NLP+Regex)** | 61.4% F1 on unseen documents (final exam, CI run 38382553368); 82.6% development corpus, 78.1% held-out (CI run 38212303870) | ⚠️ Below target — improvement planned |
|
|
693
734
|
| **Final Accuracy (AI+Human)** | 100% (validated) | 🎯 By Design |
|
|
694
735
|
| **Pseudonym Libraries** | 3 themes (2,426 names + 240 locations + 588 orgs) | ✅ Stories 2.1, 3.0, 4.6 Complete |
|
|
695
736
|
| **Compositional Matching** | Operational (component reuse + title stripping + compound names) | ✅ Stories 2.2, 2.3 Complete |
|
|
@@ -721,4 +762,4 @@ The integration test suite covers:
|
|
|
721
762
|
|
|
722
763
|
---
|
|
723
764
|
|
|
724
|
-
**Last Updated:** 2026-
|
|
765
|
+
**Last Updated:** 2026-10-01 (v2.2.0 — spaCy model choice, entity retyping, entity-span fixes)
|
{gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/batch.py
RENAMED
|
@@ -37,9 +37,12 @@ from gdpr_pseudonymizer.cli.progress import ETAColumn, ProgressTracker
|
|
|
37
37
|
from gdpr_pseudonymizer.cli.validators import (
|
|
38
38
|
ensure_database,
|
|
39
39
|
parse_entity_type_filter,
|
|
40
|
+
validate_model_or_exit,
|
|
40
41
|
validate_theme_or_exit,
|
|
41
42
|
)
|
|
42
43
|
from gdpr_pseudonymizer.core.document_processor import DocumentProcessor
|
|
44
|
+
from gdpr_pseudonymizer.data.database import open_database
|
|
45
|
+
from gdpr_pseudonymizer.exceptions import DatabaseConversionError
|
|
43
46
|
from gdpr_pseudonymizer.utils.logger import configure_logging, get_logger
|
|
44
47
|
|
|
45
48
|
# Configure logging
|
|
@@ -438,6 +441,7 @@ def batch_command(
|
|
|
438
441
|
|
|
439
442
|
# Validate theme
|
|
440
443
|
validate_theme_or_exit(effective_theme)
|
|
444
|
+
validate_model_or_exit(effective_model)
|
|
441
445
|
|
|
442
446
|
# Get passphrase
|
|
443
447
|
resolved_passphrase = resolve_passphrase(
|
|
@@ -449,6 +453,10 @@ def batch_command(
|
|
|
449
453
|
# Initialize database if needed
|
|
450
454
|
ensure_database(effective_db_path, resolved_passphrase, console)
|
|
451
455
|
|
|
456
|
+
# Open it once before any document: a pre-10.7 file is converted here,
|
|
457
|
+
# once, before any worker opens it (Story 10.7, AC3)
|
|
458
|
+
_convert_database_before_workers(effective_db_path, resolved_passphrase)
|
|
459
|
+
|
|
452
460
|
# Determine processing mode based on workers parameter
|
|
453
461
|
actual_workers = min(cpu_count(), effective_workers, 8)
|
|
454
462
|
actual_workers = max(1, actual_workers)
|
|
@@ -712,6 +720,35 @@ def batch_command(
|
|
|
712
720
|
sys.exit(2)
|
|
713
721
|
|
|
714
722
|
|
|
723
|
+
def _convert_database_before_workers(db_path: str, passphrase: str) -> None:
|
|
724
|
+
"""Open the database once before any document is processed (Story 10.7).
|
|
725
|
+
|
|
726
|
+
A database made before Story 10.7 is converted by its first open, so it
|
|
727
|
+
is converted here, once, before the workers start. A conversion that
|
|
728
|
+
cannot be made stops the run before any document, with its message. Any
|
|
729
|
+
other open error is left to the per-document processing, as before.
|
|
730
|
+
|
|
731
|
+
Args:
|
|
732
|
+
db_path: Path to the database (created by ensure_database if missing)
|
|
733
|
+
passphrase: Database passphrase
|
|
734
|
+
"""
|
|
735
|
+
if not Path(db_path).exists():
|
|
736
|
+
return
|
|
737
|
+
try:
|
|
738
|
+
with open_database(db_path, passphrase):
|
|
739
|
+
pass
|
|
740
|
+
except DatabaseConversionError as e:
|
|
741
|
+
format_error_message(
|
|
742
|
+
"Database Conversion Failed",
|
|
743
|
+
str(e),
|
|
744
|
+
"Make the database file and its folder writable, free some disk "
|
|
745
|
+
"space or close other programs using it, then run the command again.",
|
|
746
|
+
)
|
|
747
|
+
sys.exit(1)
|
|
748
|
+
except Exception as e:
|
|
749
|
+
logger.info("batch_database_precheck_skipped", error_type=type(e).__name__)
|
|
750
|
+
|
|
751
|
+
|
|
715
752
|
def _display_batch_summary(result: BatchResult) -> None:
|
|
716
753
|
"""Display batch processing summary report.
|
|
717
754
|
|