gdpr-pseudonymizer 2.1.3__tar.gz → 2.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/PKG-INFO +69 -28
  2. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/README.md +67 -26
  3. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/batch.py +37 -0
  4. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/cli/commands/delete_mapping.py +375 -0
  5. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/destroy_table.py +9 -0
  6. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/import_mappings.py +20 -6
  7. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/list_mappings.py +9 -4
  8. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/process.py +2 -0
  9. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/config.py +19 -2
  10. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/main.py +9 -3
  11. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/validators.py +17 -0
  12. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/core/custom_pseudonym.py +906 -0
  13. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/core/document_processor.py +536 -66
  14. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/core/legacy_names.py +690 -0
  15. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/core/mapping_key.py +227 -0
  16. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/data/database.py +872 -0
  17. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/models.py +11 -8
  18. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/repositories/audit_repository.py +5 -3
  19. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/data/repositories/mapping_repository.py +1201 -0
  20. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/exceptions.py +42 -1
  21. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/en.qm +0 -0
  22. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/en.ts +69 -0
  23. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/fr.ts +67 -0
  24. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/main_window.py +30 -0
  25. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/gui/models/validation_state.py +1622 -0
  26. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/batch.py +205 -21
  27. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/database.py +10 -4
  28. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/home.py +47 -4
  29. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/validation.py +349 -36
  30. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/entity_panel.py +5 -3
  31. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/batch_worker.py +31 -2
  32. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/database_worker.py +8 -3
  33. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/detection_worker.py +11 -7
  34. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/finalization_worker.py +19 -5
  35. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/locale/fr/LC_MESSAGES/cli.mo +0 -0
  36. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/locale/fr/LC_MESSAGES/cli.po +4 -4
  37. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/nlp/hybrid_detector.py +2790 -0
  38. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/nlp/model_names.py +46 -0
  39. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/regex_matcher.py +40 -2
  40. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/library_manager.py +2 -2
  41. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/__init__.py +5 -0
  42. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/detection_patterns.yaml +16 -9
  43. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/location_noise_filter.yaml +74 -0
  44. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/org_role_filter.yaml +138 -0
  45. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/person_boundaries.yaml +64 -0
  46. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/place_abbreviations.yaml +14 -0
  47. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/resources/salutations.yaml +17 -0
  48. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/file_handler.py +19 -7
  49. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/tabular_reader.py +5 -2
  50. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/tabular_writer.py +5 -3
  51. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/utils/text_form.py +79 -0
  52. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/validation/models.py +830 -0
  53. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/validation/ui.py +27 -6
  54. gdpr_pseudonymizer-2.3.0/gdpr_pseudonymizer/validation/workflow.py +1082 -0
  55. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/pyproject.toml +2 -2
  56. gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/cli/commands/delete_mapping.py +0 -283
  57. gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/data/database.py +0 -285
  58. gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/data/repositories/mapping_repository.py +0 -545
  59. gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/gui/models/validation_state.py +0 -568
  60. gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/locale/fr/LC_MESSAGES/cli.mo +0 -0
  61. gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/nlp/hybrid_detector.py +0 -400
  62. gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/validation/models.py +0 -422
  63. gdpr_pseudonymizer-2.1.3/gdpr_pseudonymizer/validation/workflow.py +0 -529
  64. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/LICENSE +0 -0
  65. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/__init__.py +0 -0
  66. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/__init__.py +0 -0
  67. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/__init__.py +0 -0
  68. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/config_show.py +0 -0
  69. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/export.py +0 -0
  70. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/init.py +0 -0
  71. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/list_entities.py +0 -0
  72. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/stats.py +0 -0
  73. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/commands/validate_mappings.py +0 -0
  74. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/formatters.py +0 -0
  75. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/i18n.py +0 -0
  76. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/passphrase.py +0 -0
  77. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/progress.py +0 -0
  78. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/cli/validation_stub.py +0 -0
  79. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/core/__init__.py +0 -0
  80. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/core/naive_processor.py +0 -0
  81. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/__init__.py +0 -0
  82. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/encryption.py +0 -0
  83. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/naive_data.py +0 -0
  84. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/repositories/__init__.py +0 -0
  85. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/data/repositories/metadata_repository.py +0 -0
  86. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/__init__.py +0 -0
  87. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/accessibility/__init__.py +0 -0
  88. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/accessibility/focus_manager.py +0 -0
  89. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/accessibility/shortcuts.py +0 -0
  90. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/app.py +0 -0
  91. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/config.py +0 -0
  92. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/dialogs/shortcuts_help.py +0 -0
  93. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/error_handler.py +0 -0
  94. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/__init__.py +0 -0
  95. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/fr.qm +0 -0
  96. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/i18n/translator.py +0 -0
  97. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/models/__init__.py +0 -0
  98. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/app.ico +0 -0
  99. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_16.png +0 -0
  100. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_256.png +0 -0
  101. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_32.png +0 -0
  102. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_48.png +0 -0
  103. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/icons/icon_512.png +0 -0
  104. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/themes/dark.qss +0 -0
  105. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/themes/high-contrast.qss +0 -0
  106. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/resources/themes/light.qss +0 -0
  107. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/__init__.py +0 -0
  108. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/processing.py +0 -0
  109. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/results.py +0 -0
  110. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/settings.py +0 -0
  111. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/screens/stub.py +0 -0
  112. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/__init__.py +0 -0
  113. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/confirm_dialog.py +0 -0
  114. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/drop_zone.py +0 -0
  115. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/entity_editor.py +0 -0
  116. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/passphrase_dialog.py +0 -0
  117. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/step_indicator.py +0 -0
  118. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/widgets/toast.py +0 -0
  119. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/__init__.py +0 -0
  120. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/model_manager.py +0 -0
  121. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/processing_worker.py +0 -0
  122. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/gui/workers/signals.py +0 -0
  123. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/locale/en/LC_MESSAGES/cli.mo +0 -0
  124. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/locale/en/LC_MESSAGES/cli.po +0 -0
  125. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/__init__.py +0 -0
  126. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/entity_detector.py +0 -0
  127. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/entity_grouping.py +0 -0
  128. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/geography_dictionary.py +0 -0
  129. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/name_dictionary.py +0 -0
  130. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/spacy_detector.py +0 -0
  131. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/nlp/stanza_detector.py +0 -0
  132. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/__init__.py +0 -0
  133. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/assignment_engine.py +0 -0
  134. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/gender_detector.py +0 -0
  135. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/pseudonym/neutral_id_generator.py +0 -0
  136. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/french_gender_lookup.json +0 -0
  137. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/french_geography.json +0 -0
  138. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/french_names.json +0 -0
  139. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/pseudonyms/lotr.json +0 -0
  140. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/pseudonyms/neutral.json +0 -0
  141. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/resources/pseudonyms/star_wars.json +0 -0
  142. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/__init__.py +0 -0
  143. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/config_manager.py +0 -0
  144. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/french_patterns.py +0 -0
  145. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/utils/logger.py +0 -0
  146. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/validation/__init__.py +0 -0
  147. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/validation/actions.py +0 -0
  148. {gdpr_pseudonymizer-2.1.3 → gdpr_pseudonymizer-2.3.0}/gdpr_pseudonymizer/validation/context_precomputer.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gdpr-pseudonymizer
3
- Version: 2.1.3
3
+ Version: 2.3.0
4
4
  Summary: CLI and GUI tool for GDPR-compliant pseudonymization of French text documents using NLP-based entity detection and reversible mapping
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -30,7 +30,7 @@ Provides-Extra: gui
30
30
  Provides-Extra: pdf
31
31
  Requires-Dist: PySide6-Essentials (>=6.7.0,<6.8.0) ; (python_version >= "3.10" and python_version < "3.13") and (extra == "gui" or extra == "all")
32
32
  Requires-Dist: PyYAML (>=6.0,<7.0)
33
- Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0)
33
+ Requires-Dist: SQLAlchemy (>=2.0.0,<2.1.0)
34
34
  Requires-Dist: click (>=8.0.0,<8.2.0)
35
35
  Requires-Dist: cryptography (>=50.0.1,<51.0.0)
36
36
  Requires-Dist: markdown-it-py (>=3.0.0,<4.0.0)
@@ -63,7 +63,16 @@ Transform sensitive French documents for safe AI analysis with local processing,
63
63
 
64
64
  ---
65
65
 
66
- ## What's New in v2.1
66
+ ## What's New in v2.2
67
+
68
+ - **Choose the spaCy model** — `--model` / `pseudonymization.model` accept any spaCy package name (e.g. `en_core_web_trf` for English); `spacy` still selects the French `fr_core_news_lg`. The model only drives NLP detection: regex patterns, dictionaries and pseudonym libraries stay French. See the [CLI Reference](docs/CLI-REFERENCE.md).
69
+ - **Retype entities during CLI validation (`[T]`)** — Switch a detected entity between PERSON, LOCATION and ORG for every occurrence, instead of rejecting and re-adding it
70
+ - **Cleaner entity spans** — Editing with `[E]` no longer erases neighbouring text; "Last, First" matching now requires a known first name; timestamps and stray punctuation are trimmed from NER span edges; names with non-French letters ("María", "Yılmaz") and hyphenated names after titles are detected whole
71
+ - **Fresh `pip install` fixed** — v2.1.3 crashed on import when pip resolved SQLAlchemy 2.1; SQLAlchemy is now capped below 2.1
72
+ - **Audit log records the real model** — e.g. `fr_core_news_lg-3.8.0` instead of `spacy-unknown`
73
+
74
+ <details>
75
+ <summary>v2.1 highlights</summary>
67
76
 
68
77
  - **Validate-once-per-entity** — Accept or reject one entity occurrence and it applies to all same-text occurrences in the document (productivity boost for repeated names)
69
78
  - **Excel/CSV Format Support** — Process `.xlsx` and `.csv` files with cell-aware pseudonymization for HR/compliance use cases (`pip install gdpr-pseudonymizer[excel]`)
@@ -71,6 +80,8 @@ Transform sensitive French documents for safe AI analysis with local processing,
71
80
  - **NER Accuracy Improvements** — Expanded ORG detection patterns, POS-tag disambiguation for geography matching; LOCATION false-negative rate reduced from 27% to 13%
72
81
  - **GUI Discoverability** — F1 keyboard shortcuts help dialog with all shortcut groups, database path persistence across sessions, "Hide confirmed" toggle
73
82
 
83
+ </details>
84
+
74
85
  <!-- TODO: Add GUI screenshot -->
75
86
 
76
87
  **Upgrade:** `pip install --upgrade gdpr-pseudonymizer[gui,excel]`
@@ -85,10 +96,14 @@ Pre-built standalone executables are available for Windows, macOS, and Linux. No
85
96
 
86
97
  | Platform | File | Notes |
87
98
  |----------|------|-------|
88
- | **Windows** | `gdpr-pseudonymizer-2.1.0-windows-setup.exe` | Run the installer. Adds Start Menu shortcut. |
89
- | **macOS (Apple Silicon)** | `gdpr-pseudonymizer-2.1.0-macos-arm64.dmg` | Open DMG, drag to Applications. |
90
- | **macOS (Intel)** | `gdpr-pseudonymizer-2.1.0-macos-x86_64.dmg` | Open DMG, drag to Applications. |
91
- | **Linux** | `gdpr-pseudonymizer-2.1.0-linux.AppImage` | `chmod +x` then run. |
99
+ | **Windows** | `gdpr-pseudonymizer-2.2.0-windows-setup.exe` | Run the installer. Adds Start Menu shortcut. |
100
+ | **macOS (Apple Silicon)** | `gdpr-pseudonymizer-2.2.0-macos-arm64.dmg` | Open DMG, drag to Applications. |
101
+ | **Linux** | `gdpr-pseudonymizer-2.2.0-linux.AppImage` | `chmod +x` then run. |
102
+
103
+ > **Intel Macs:** no DMG is published. GitHub retired its Intel macOS runners, and an
104
+ > Apple Silicon runner cannot cross-build a working x86_64 bundle. Install with
105
+ > `pip install gdpr-pseudonymizer` instead — the CLI and GUI both work on Intel.
106
+
92
107
 
93
108
  ### Platform Notes
94
109
 
@@ -126,7 +141,7 @@ GDPR Pseudonymizer is a **privacy-first CLI and GUI tool** that combines AI effi
126
141
  - ✅ **Zero telemetry** - No analytics, crash reporting, or external communication
127
142
 
128
143
  ### 🤝 **AI + Human Verification**
129
- - ✅ **Hybrid detection** - AI pre-detects ~60% of entities (NLP + regex + geography dictionary)
144
+ - ✅ **Hybrid detection** - NLP + regex + geography dictionary: F1 61.4% on 30 unseen documents (CI run 38382553368). Improving this is a priority — see [Technical Details](#-technical-details)
130
145
  - ✅ **Mandatory validation** - You review and confirm all entities (ensures 100% accuracy)
131
146
  - ✅ **Fast validation UI** - Rich CLI interface with keyboard shortcuts, <2 min per document
132
147
  - ✅ **Smart workflow** - Entity-by-type grouping (PERSON → ORG → LOCATION) with context display
@@ -161,7 +176,7 @@ GDPR Pseudonymizer is a **privacy-first CLI and GUI tool** that combines AI effi
161
176
 
162
177
  ## 🚀 Quick Start
163
178
 
164
- **Status:** 🎉 **v2.1.0** (March 2026) — GUI Polish, Excel/CSV Support & NER Accuracy
179
+ **Status:** 🎉 **v2.2.0** (September 2026) — spaCy Model Choice, Entity Retyping & Span Fixes
165
180
 
166
181
  ### Getting Started
167
182
 
@@ -183,23 +198,25 @@ pip install gdpr-pseudonymizer[excel]
183
198
  pip install gdpr-pseudonymizer[formats]
184
199
  ```
185
200
 
186
- ### What v2.1 Delivers
201
+ ### What v2.2 Delivers
187
202
 
188
203
  - 🖥️ **Desktop GUI** — Visual entity validation with drag-and-drop, batch dashboard, and database management
189
- - 📦 **Standalone executables** — Windows .exe, macOS .dmg, Linux AppImage — no Python required
204
+ - 📦 **Standalone executables** — Windows .exe, macOS .dmg (Apple Silicon), Linux AppImage — no Python required
190
205
  - ♿ **WCAG 2.1 AA accessibility** — Keyboard navigation, screen reader, high contrast mode
191
206
  - 🌐 **French UI** — Complete FR/EN interface with live language switching
192
- - 🤖 **AI-assisted detection** — Hybrid NLP + regex detects ~60% of entities automatically
207
+ - 🤖 **AI-assisted detection** — Hybrid NLP + regex detection: F1 61.4% (recall 60.6%) on 30 unseen documents by a different author (CI run 38382553368)
193
208
  - ✅ **Mandatory human verification** — You review and confirm all entities (ensures 100% accuracy)
194
209
  - 🔒 **100% local processing** — Your data never leaves your machine
195
210
  - 📄 **PDF/DOCX support** — Process PDF and DOCX files directly (optional extras)
196
211
  - 📊 **Excel/CSV support** — Process .xlsx and .csv files with cell-aware pseudonymization (optional extra: `[excel]`)
197
212
  - 🆔 **Neutral ID theme** — Counter-based identifiers (PERSON-001, LIEU-001) for formal/legal contexts
198
213
  - 🎯 **NER accuracy** — LOCATION false-negative rate reduced from 27% to 13% via regex expansion & POS disambiguation
214
+ - 🧠 **Choice of spaCy model** — French by default; any spaCy model for detection (e.g. `--model en_core_web_trf`)
215
+ - 🏷️ **Entity retyping** — Change an entity's type (PERSON/LOCATION/ORG) during CLI validation with `[T]`
199
216
 
200
- **What v2.1 does NOT deliver:**
217
+ **What v2.2 does NOT deliver:**
201
218
  - ❌ Fully automatic "set and forget" processing
202
- - ❌ 85%+ AI accuracy (current: ~60% F1 with hybrid approach)
219
+ - ❌ 85%+ AI accuracy (current: F1 61.4% on unseen documents, 82.6% on our development corpus — we plan to raise it)
203
220
  - ❌ Optional validation mode (validation is mandatory)
204
221
 
205
222
  ### Roadmap
@@ -210,10 +227,12 @@ pip install gdpr-pseudonymizer[formats]
210
227
 
211
228
  **v2.0 (Q1 2026):** Desktop GUI, standalone executables, WCAG AA accessibility, French UI, batch validation, core hardening
212
229
 
213
- **v2.1 (Q1 2026) — CURRENT RELEASE:** GUI polish, Excel/CSV support, neutral ID theme, NER accuracy improvements, keyboard shortcuts help
230
+ **v2.1 (Q1 2026):** GUI polish, Excel/CSV support, neutral ID theme, NER accuracy improvements, keyboard shortcuts help
231
+
232
+ **v2.2 (Q3 2026) — CURRENT RELEASE:** Choice of spaCy model, entity retyping in CLI validation, entity-span accuracy fixes, fresh-install fix (SQLAlchemy < 2.1)
214
233
 
215
234
  **v3.0 (2027+):** NLP accuracy & automation
216
- - Fine-tuned French NER model (70-85% F1 target, up from ~60%)
235
+ - Fine-tuned French NER model (70-85% F1 target, up from 61.4% on unseen documents today)
217
236
  - Optional `--no-validate` flag for high-confidence workflows
218
237
  - Confidence-based auto-processing (85%+ F1 target)
219
238
  - Multi-language support (English, Spanish, German)
@@ -362,6 +381,20 @@ GDPR_PSEUDO_LANG=fr gdpr-pseudo --help
362
381
 
363
382
  ## 🔬 Technical Details
364
383
 
384
+ ### Detection accuracy (Epic 10, October 2026)
385
+
386
+ **On unseen documents: F1 61.4%** (95% range 59.3–63.6%), recall 60.6%, precision 62.3%. This is a blind final exam: 30 French documents written by a different author in messy formats (email threads with quoted replies, PDF-style line breaks, tables flattened to text, mixed case), scored once (CI run 38382553368). By type: PERSON 69.2% F1, LOCATION 68.7%, ORG 34.8%; organisations remain the weak point.
387
+
388
+ **For context:** 82.6% F1 on the development corpus (25 documents) and 78.1% on a held-out set (6 documents) (CI run 38212303870). Both are closer to the documents the rules were built on.
389
+
390
+ How the final exam was made, and its limits:
391
+ - An LLM (GPT) wrote the 30 documents from a brief that named the hard cases to include but quoted no corpus text.
392
+ - Two models (GPT and Gemini) annotated them independently, and the maintainer settled their disagreements.
393
+ - The set was frozen before the last detection change, then scored once.
394
+ - The documents are synthetic, so real documents may score differently.
395
+
396
+ Details: [docs/qa/ner-accuracy-report.md](docs/qa/ner-accuracy-report.md).
397
+
365
398
  ### NLP Library Selection (Story 1.2 - Completed)
366
399
 
367
400
  After comprehensive benchmarking on 25 French interview/business documents (1,737 annotated entities):
@@ -369,10 +402,18 @@ After comprehensive benchmarking on 25 French interview/business documents (1,73
369
402
  | Approach | F1 Score | Precision | Recall | Notes |
370
403
  |----------|----------|-----------|--------|-------|
371
404
  | **spaCy only** `fr_core_news_lg` | 29.5% | 27.0% | 32.7% | Story 1.2 baseline |
372
- | **Hybrid** (spaCy + regex) | 59.97% | 48.17% | 79.45% | Story 5.3 |
373
- | **Hybrid + expanded patterns** | 31.79% | 19.49% | 85.15% | Story 7.5 (current) |
405
+ | **Hybrid** (spaCy + regex) | 59.97% | 48.17% | 79.45% | Story 5.3 (see scoring note) |
406
+ | **Hybrid (v2.2.0)** | **58.53%** | 48.42% | 73.98% | Scored the way the app normalizes entities |
407
+
408
+ By entity type (v2.2.0): PERSON 72.9% F1, LOCATION 45.0%, ORG 12.9%. Organisation detection is the weak point: job titles such as CTO, DPO or RSSI are often mistaken for companies.
409
+
410
+ **How these numbers are scored (corrected October 2026).** The app strips titles and prepositions before pseudonymizing ("Mme Isabelle Moreau" is handled as "Isabelle Moreau", "à Paris" as "Paris"), and the benchmark now scores detections the same way. Until then the committed scorer compared raw text, so every correctly detected "Mme Isabelle Moreau" counted as a miss plus a false alarm. Under that strict scoring the same detector measured ~32% F1, a figure briefly published on this page in October 2026; it understated real-use detection. The Story 5.3 figure (59.97%) appears to have been measured with title-aware matching and is consistent with v2.2.0's 58.53%. Each number in the table above was measured with `pytest tests/accuracy -m accuracy -s` on the version it names.
411
+
412
+ **Read these numbers with two caveats.** They come from one 25-document French interview/business corpus, and part of its ground truth was auto-annotated with the tool's own regex patterns, which skews results in both directions. And they measure *pre-detection* only: mandatory validation is what makes the final output correct.
413
+
414
+ **Epic 10 (October 2026)** repaired that benchmark against written annotation guidelines, added a held-out set and the blind final exam above, and improved organisation detection. A fine-tuned French NER model remains the v3.0 target (70-85% F1).
374
415
 
375
- **Accuracy trajectory:** spaCy-only baseline → hybrid approach with annotation cleanup, expanded regex patterns, and French geography dictionary doubled F1 score. Story 7.5 added 12 ORG pattern keywords, POS-tag disambiguation for geography matching, and 7 international locations — reducing LOCATION false-negative rate from 27.42% to 12.90%.
416
+ **Accuracy trajectory:** Story 7.5 added 12 ORG pattern keywords, POS-tag disambiguation for geography matching, and 7 international locations — reducing LOCATION false-negative rate from 27.42% to 12.90%.
376
417
 
377
418
  **Approved Solution:**
378
419
  - ✅ **Hybrid approach** (NLP + regex + geography dictionary + POS disambiguation)
@@ -389,7 +430,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
389
430
  - ✅ **Entity-by-type grouping** - Review PERSON → ORG → LOCATION in logical order
390
431
  - ✅ **Context display** - See 10 words before/after each entity with highlighting
391
432
  - ✅ **Confidence scores** - Color-coded confidence from spaCy NER (green >80%, yellow 60-80%, red <60%)
392
- - ✅ **Keyboard shortcuts** - Single-key actions: [Space] Confirm, [R] Reject, [E] Modify, [A] Add, [C] Change pseudonym
433
+ - ✅ **Keyboard shortcuts** - Single-key actions: [Space] Confirm, [R] Reject, [E] Modify, [T] Change type, [A] Add, [C] Change pseudonym
393
434
  - ✅ **Batch operations** - Accept/reject all entities of a type at once (Shift+A/R) with entity count feedback
394
435
  - ✅ **Context cycling indicator** - Dot indicator (`● ○ ○ ○ ○`) shows current context position; `[Press X to cycle]` hint improves discoverability
395
436
  - ✅ **Help overlay** - Press [H] for full command reference
@@ -417,7 +458,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
417
458
  | **CLI Framework** | Typer | 0.9+ | Command-line interface |
418
459
  | **Database** | SQLite | 3.35+ | Local mapping table storage with WAL mode |
419
460
  | **Encryption** | cryptography (AESSIV) | 44.0+ | AES-256-SIV encryption for sensitive fields (PBKDF2 key derivation, passphrase-protected) |
420
- | **ORM** | SQLAlchemy | 2.0+ | Database abstraction and session management |
461
+ | **ORM** | SQLAlchemy | 2.0.x (<2.1) | Database abstraction and session management |
421
462
  | **Desktop GUI** | PySide6 | 6.7+ | Desktop application (optional: `pip install gdpr-pseudonymizer[gui]`) |
422
463
  | **Validation UI** | rich | 13.7+ | Interactive CLI entity review |
423
464
  | **Keyboard Input** | readchar | 4.2+ | Single-keypress capture for validation UI |
@@ -433,7 +474,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
433
474
  **Long answer:**
434
475
  1. **GDPR defensibility** - Human verification provides legal audit trail
435
476
  2. **Zero false negatives** - AI misses entities, humans catch them (100% coverage)
436
- 3. **Current NLP limitations** - French models on interview/business docs: 29.5% F1 out-of-box (hybrid approach reaches ~60%)
477
+ 3. **Current NLP limitations** - French models on interview/business docs: 29.5% F1 out-of-box (hybrid approach: F1 61.4% and recall 60.6% on unseen documents, 82.6% F1 on our development corpus; organisations are the weak point; improvement planned)
437
478
  4. **Better than alternatives:**
438
479
  - ✅ **vs Manual redaction:** 50%+ faster (AI pre-detection)
439
480
  - ✅ **vs Cloud services:** 100% local processing (no data leakage)
@@ -519,13 +560,13 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
519
560
 
520
561
  ## 🛠️ Development Status
521
562
 
522
- **Epics 1-7 Complete** — v2.1.0 (March 2026). GUI polish, Excel/CSV support, NER accuracy improvements.
563
+ **Epics 1-7 Complete** — current release v2.2.0 (September 2026): spaCy model choice, entity retyping, entity-span fixes. v2.1.0 (March 2026) closed Epic 7.
523
564
 
524
565
  - ✅ **Epic 1:** Foundation & NLP Validation (9 stories) — spaCy integration, validation UI, hybrid detection, entity deduplication
525
566
  - ✅ **Epic 2:** Core Pseudonymization Engine (9 stories) — pseudonym libraries, encryption, audit logging, batch processing, GDPR 1:1 mapping
526
567
  - ✅ **Epic 3:** CLI Interface & Batch Processing (7 stories) — 8 CLI commands, progress reporting, config files, parallel batch, UX polish
527
568
  - ✅ **Epic 4:** Launch Readiness (8 stories) — LLM utility validation, cross-platform testing, documentation, NER accuracy suite, performance validation, beta feedback integration, codebase refactoring, launch preparation
528
- - ✅ **Epic 5:** Quick Wins & GDPR Compliance (7 stories) — GDPR Article 17 erasure, gender-aware pseudonyms, NER accuracy improvements (F1 29.74% → 59.97%), French documentation translation, PDF/DOCX support, CLI polish & benchmarks, v1.1 release
569
+ - ✅ **Epic 5:** Quick Wins & GDPR Compliance (7 stories) — GDPR Article 17 erasure, gender-aware pseudonyms, NER accuracy work (annotation cleanup, expanded regex, geography dictionary), French documentation translation, PDF/DOCX support, CLI polish & benchmarks, v1.1 release
529
570
  - ✅ **Epic 6:** v2.0 Desktop GUI & Broader Accessibility (9 stories) — PySide6 desktop application, visual entity validation, batch GUI, i18n, WCAG AA, standalone executables
530
571
  - ✅ Story 6.1: UX Architecture & GUI Framework Selection
531
572
  - ✅ Story 6.2: GUI Application Foundation (main window, theming, home screen, settings, 77 GUI tests)
@@ -609,7 +650,7 @@ This project is licensed under the [MIT License](LICENSE).
609
650
  - ⚠️ Test thoroughly before production use
610
651
 
611
652
  **Current limitations:**
612
- - AI detection: ~60% F1 baseline (not 85%+)
653
+ - AI detection: F1 61.4% on unseen documents (82.6% on our development corpus), not 85%+; improving it is planned
613
654
  - Validation required for ALL documents (not optional)
614
655
  - French documents only (English, Spanish, etc. in future versions)
615
656
  - Text-based formats: .txt, .md, .pdf, .docx, .xlsx, .csv (PDF/DOCX/Excel require optional extras: `pip install gdpr-pseudonymizer[formats]`)
@@ -730,7 +771,7 @@ The integration test suite covers:
730
771
 
731
772
  | Metric | Value | Status |
732
773
  |--------|-------|--------|
733
- | **Development Progress** | v2.1.0 | ✅ Epics 1-7 complete |
774
+ | **Development Progress** | v2.2.0 | ✅ Epics 1-7 complete |
734
775
  | **Stories Complete** | 60 (Epics 1-7) | ✅ All epics complete |
735
776
  | **LLM Utility (NFR10)** | 4.27/5.0 (85.4%) | ✅ PASSED (threshold: 80%) |
736
777
  | **Installation Success (NFR3)** | 87.5% (7/8 platforms) | ✅ PASSED (threshold: 85%) |
@@ -738,7 +779,7 @@ The integration test suite covers:
738
779
  | **Critical Bugs Found** | 1 (Story 2.8) | ✅ RESOLVED - Epic 3 Unblocked |
739
780
  | **Test Corpus Size** | 25 docs, 1,737 entities | ✅ Complete (post-cleanup) |
740
781
  | **NLP Accuracy (Baseline)** | 29.5% F1 (spaCy only) | ✅ Measured (Story 1.2) |
741
- | **Hybrid Accuracy (NLP+Regex)** | 59.97% F1 (+30.23pp vs baseline) | ✅ Story 5.3 Complete |
782
+ | **Hybrid Accuracy (NLP+Regex)** | 61.4% F1 on unseen documents (final exam, CI run 38382553368); 82.6% development corpus, 78.1% held-out (CI run 38212303870) | ⚠️ Below target — improvement planned |
742
783
  | **Final Accuracy (AI+Human)** | 100% (validated) | 🎯 By Design |
743
784
  | **Pseudonym Libraries** | 3 themes (2,426 names + 240 locations + 588 orgs) | ✅ Stories 2.1, 3.0, 4.6 Complete |
744
785
  | **Compositional Matching** | Operational (component reuse + title stripping + compound names) | ✅ Stories 2.2, 2.3 Complete |
@@ -770,5 +811,5 @@ The integration test suite covers:
770
811
 
771
812
  ---
772
813
 
773
- **Last Updated:** 2026-03-17 (v2.1.0 — GUI polish, Excel/CSV support, neutral ID theme, NER accuracy improvements, 1670+ tests)
814
+ **Last Updated:** 2026-10-01 (v2.2.0 — spaCy model choice, entity retyping, entity-span fixes)
774
815
 
@@ -14,7 +14,16 @@ Transform sensitive French documents for safe AI analysis with local processing,
14
14
 
15
15
  ---
16
16
 
17
- ## What's New in v2.1
17
+ ## What's New in v2.2
18
+
19
+ - **Choose the spaCy model** — `--model` / `pseudonymization.model` accept any spaCy package name (e.g. `en_core_web_trf` for English); `spacy` still selects the French `fr_core_news_lg`. The model only drives NLP detection: regex patterns, dictionaries and pseudonym libraries stay French. See the [CLI Reference](docs/CLI-REFERENCE.md).
20
+ - **Retype entities during CLI validation (`[T]`)** — Switch a detected entity between PERSON, LOCATION and ORG for every occurrence, instead of rejecting and re-adding it
21
+ - **Cleaner entity spans** — Editing with `[E]` no longer erases neighbouring text; "Last, First" matching now requires a known first name; timestamps and stray punctuation are trimmed from NER span edges; names with non-French letters ("María", "Yılmaz") and hyphenated names after titles are detected whole
22
+ - **Fresh `pip install` fixed** — v2.1.3 crashed on import when pip resolved SQLAlchemy 2.1; SQLAlchemy is now capped below 2.1
23
+ - **Audit log records the real model** — e.g. `fr_core_news_lg-3.8.0` instead of `spacy-unknown`
24
+
25
+ <details>
26
+ <summary>v2.1 highlights</summary>
18
27
 
19
28
  - **Validate-once-per-entity** — Accept or reject one entity occurrence and it applies to all same-text occurrences in the document (productivity boost for repeated names)
20
29
  - **Excel/CSV Format Support** — Process `.xlsx` and `.csv` files with cell-aware pseudonymization for HR/compliance use cases (`pip install gdpr-pseudonymizer[excel]`)
@@ -22,6 +31,8 @@ Transform sensitive French documents for safe AI analysis with local processing,
22
31
  - **NER Accuracy Improvements** — Expanded ORG detection patterns, POS-tag disambiguation for geography matching; LOCATION false-negative rate reduced from 27% to 13%
23
32
  - **GUI Discoverability** — F1 keyboard shortcuts help dialog with all shortcut groups, database path persistence across sessions, "Hide confirmed" toggle
24
33
 
34
+ </details>
35
+
25
36
  <!-- TODO: Add GUI screenshot -->
26
37
 
27
38
  **Upgrade:** `pip install --upgrade gdpr-pseudonymizer[gui,excel]`
@@ -36,10 +47,14 @@ Pre-built standalone executables are available for Windows, macOS, and Linux. No
36
47
 
37
48
  | Platform | File | Notes |
38
49
  |----------|------|-------|
39
- | **Windows** | `gdpr-pseudonymizer-2.1.0-windows-setup.exe` | Run the installer. Adds Start Menu shortcut. |
40
- | **macOS (Apple Silicon)** | `gdpr-pseudonymizer-2.1.0-macos-arm64.dmg` | Open DMG, drag to Applications. |
41
- | **macOS (Intel)** | `gdpr-pseudonymizer-2.1.0-macos-x86_64.dmg` | Open DMG, drag to Applications. |
42
- | **Linux** | `gdpr-pseudonymizer-2.1.0-linux.AppImage` | `chmod +x` then run. |
50
+ | **Windows** | `gdpr-pseudonymizer-2.2.0-windows-setup.exe` | Run the installer. Adds Start Menu shortcut. |
51
+ | **macOS (Apple Silicon)** | `gdpr-pseudonymizer-2.2.0-macos-arm64.dmg` | Open DMG, drag to Applications. |
52
+ | **Linux** | `gdpr-pseudonymizer-2.2.0-linux.AppImage` | `chmod +x` then run. |
53
+
54
+ > **Intel Macs:** no DMG is published. GitHub retired its Intel macOS runners, and an
55
+ > Apple Silicon runner cannot cross-build a working x86_64 bundle. Install with
56
+ > `pip install gdpr-pseudonymizer` instead — the CLI and GUI both work on Intel.
57
+
43
58
 
44
59
  ### Platform Notes
45
60
 
@@ -77,7 +92,7 @@ GDPR Pseudonymizer is a **privacy-first CLI and GUI tool** that combines AI effi
77
92
  - ✅ **Zero telemetry** - No analytics, crash reporting, or external communication
78
93
 
79
94
  ### 🤝 **AI + Human Verification**
80
- - ✅ **Hybrid detection** - AI pre-detects ~60% of entities (NLP + regex + geography dictionary)
95
+ - ✅ **Hybrid detection** - NLP + regex + geography dictionary: F1 61.4% on 30 unseen documents (CI run 38382553368). Improving this is a priority — see [Technical Details](#-technical-details)
81
96
  - ✅ **Mandatory validation** - You review and confirm all entities (ensures 100% accuracy)
82
97
  - ✅ **Fast validation UI** - Rich CLI interface with keyboard shortcuts, <2 min per document
83
98
  - ✅ **Smart workflow** - Entity-by-type grouping (PERSON → ORG → LOCATION) with context display
@@ -112,7 +127,7 @@ GDPR Pseudonymizer is a **privacy-first CLI and GUI tool** that combines AI effi
112
127
 
113
128
  ## 🚀 Quick Start
114
129
 
115
- **Status:** 🎉 **v2.1.0** (March 2026) — GUI Polish, Excel/CSV Support & NER Accuracy
130
+ **Status:** 🎉 **v2.2.0** (September 2026) — spaCy Model Choice, Entity Retyping & Span Fixes
116
131
 
117
132
  ### Getting Started
118
133
 
@@ -134,23 +149,25 @@ pip install gdpr-pseudonymizer[excel]
134
149
  pip install gdpr-pseudonymizer[formats]
135
150
  ```
136
151
 
137
- ### What v2.1 Delivers
152
+ ### What v2.2 Delivers
138
153
 
139
154
  - 🖥️ **Desktop GUI** — Visual entity validation with drag-and-drop, batch dashboard, and database management
140
- - 📦 **Standalone executables** — Windows .exe, macOS .dmg, Linux AppImage — no Python required
155
+ - 📦 **Standalone executables** — Windows .exe, macOS .dmg (Apple Silicon), Linux AppImage — no Python required
141
156
  - ♿ **WCAG 2.1 AA accessibility** — Keyboard navigation, screen reader, high contrast mode
142
157
  - 🌐 **French UI** — Complete FR/EN interface with live language switching
143
- - 🤖 **AI-assisted detection** — Hybrid NLP + regex detects ~60% of entities automatically
158
+ - 🤖 **AI-assisted detection** — Hybrid NLP + regex detection: F1 61.4% (recall 60.6%) on 30 unseen documents by a different author (CI run 38382553368)
144
159
  - ✅ **Mandatory human verification** — You review and confirm all entities (ensures 100% accuracy)
145
160
  - 🔒 **100% local processing** — Your data never leaves your machine
146
161
  - 📄 **PDF/DOCX support** — Process PDF and DOCX files directly (optional extras)
147
162
  - 📊 **Excel/CSV support** — Process .xlsx and .csv files with cell-aware pseudonymization (optional extra: `[excel]`)
148
163
  - 🆔 **Neutral ID theme** — Counter-based identifiers (PERSON-001, LIEU-001) for formal/legal contexts
149
164
  - 🎯 **NER accuracy** — LOCATION false-negative rate reduced from 27% to 13% via regex expansion & POS disambiguation
165
+ - 🧠 **Choice of spaCy model** — French by default; any spaCy model for detection (e.g. `--model en_core_web_trf`)
166
+ - 🏷️ **Entity retyping** — Change an entity's type (PERSON/LOCATION/ORG) during CLI validation with `[T]`
150
167
 
151
- **What v2.1 does NOT deliver:**
168
+ **What v2.2 does NOT deliver:**
152
169
  - ❌ Fully automatic "set and forget" processing
153
- - ❌ 85%+ AI accuracy (current: ~60% F1 with hybrid approach)
170
+ - ❌ 85%+ AI accuracy (current: F1 61.4% on unseen documents, 82.6% on our development corpus — we plan to raise it)
154
171
  - ❌ Optional validation mode (validation is mandatory)
155
172
 
156
173
  ### Roadmap
@@ -161,10 +178,12 @@ pip install gdpr-pseudonymizer[formats]
161
178
 
162
179
  **v2.0 (Q1 2026):** Desktop GUI, standalone executables, WCAG AA accessibility, French UI, batch validation, core hardening
163
180
 
164
- **v2.1 (Q1 2026) — CURRENT RELEASE:** GUI polish, Excel/CSV support, neutral ID theme, NER accuracy improvements, keyboard shortcuts help
181
+ **v2.1 (Q1 2026):** GUI polish, Excel/CSV support, neutral ID theme, NER accuracy improvements, keyboard shortcuts help
182
+
183
+ **v2.2 (Q3 2026) — CURRENT RELEASE:** Choice of spaCy model, entity retyping in CLI validation, entity-span accuracy fixes, fresh-install fix (SQLAlchemy < 2.1)
165
184
 
166
185
  **v3.0 (2027+):** NLP accuracy & automation
167
- - Fine-tuned French NER model (70-85% F1 target, up from ~60%)
186
+ - Fine-tuned French NER model (70-85% F1 target, up from 61.4% on unseen documents today)
168
187
  - Optional `--no-validate` flag for high-confidence workflows
169
188
  - Confidence-based auto-processing (85%+ F1 target)
170
189
  - Multi-language support (English, Spanish, German)
@@ -313,6 +332,20 @@ GDPR_PSEUDO_LANG=fr gdpr-pseudo --help
313
332
 
314
333
  ## 🔬 Technical Details
315
334
 
335
+ ### Detection accuracy (Epic 10, October 2026)
336
+
337
+ **On unseen documents: F1 61.4%** (95% range 59.3–63.6%), recall 60.6%, precision 62.3%. This is a blind final exam: 30 French documents written by a different author in messy formats (email threads with quoted replies, PDF-style line breaks, tables flattened to text, mixed case), scored once (CI run 38382553368). By type: PERSON 69.2% F1, LOCATION 68.7%, ORG 34.8%; organisations remain the weak point.
338
+
339
+ **For context:** 82.6% F1 on the development corpus (25 documents) and 78.1% on a held-out set (6 documents) (CI run 38212303870). Both are closer to the documents the rules were built on.
340
+
341
+ How the final exam was made, and its limits:
342
+ - An LLM (GPT) wrote the 30 documents from a brief that named the hard cases to include but quoted no corpus text.
343
+ - Two models (GPT and Gemini) annotated them independently, and the maintainer settled their disagreements.
344
+ - The set was frozen before the last detection change, then scored once.
345
+ - The documents are synthetic, so real documents may score differently.
346
+
347
+ Details: [docs/qa/ner-accuracy-report.md](docs/qa/ner-accuracy-report.md).
348
+
316
349
  ### NLP Library Selection (Story 1.2 - Completed)
317
350
 
318
351
  After comprehensive benchmarking on 25 French interview/business documents (1,737 annotated entities):
@@ -320,10 +353,18 @@ After comprehensive benchmarking on 25 French interview/business documents (1,73
320
353
  | Approach | F1 Score | Precision | Recall | Notes |
321
354
  |----------|----------|-----------|--------|-------|
322
355
  | **spaCy only** `fr_core_news_lg` | 29.5% | 27.0% | 32.7% | Story 1.2 baseline |
323
- | **Hybrid** (spaCy + regex) | 59.97% | 48.17% | 79.45% | Story 5.3 |
324
- | **Hybrid + expanded patterns** | 31.79% | 19.49% | 85.15% | Story 7.5 (current) |
356
+ | **Hybrid** (spaCy + regex) | 59.97% | 48.17% | 79.45% | Story 5.3 (see scoring note) |
357
+ | **Hybrid (v2.2.0)** | **58.53%** | 48.42% | 73.98% | Scored the way the app normalizes entities |
358
+
359
+ By entity type (v2.2.0): PERSON 72.9% F1, LOCATION 45.0%, ORG 12.9%. Organisation detection is the weak point: job titles such as CTO, DPO or RSSI are often mistaken for companies.
360
+
361
+ **How these numbers are scored (corrected October 2026).** The app strips titles and prepositions before pseudonymizing ("Mme Isabelle Moreau" is handled as "Isabelle Moreau", "à Paris" as "Paris"), and the benchmark now scores detections the same way. Until then the committed scorer compared raw text, so every correctly detected "Mme Isabelle Moreau" counted as a miss plus a false alarm. Under that strict scoring the same detector measured ~32% F1, a figure briefly published on this page in October 2026; it understated real-use detection. The Story 5.3 figure (59.97%) appears to have been measured with title-aware matching and is consistent with v2.2.0's 58.53%. Each number in the table above was measured with `pytest tests/accuracy -m accuracy -s` on the version it names.
362
+
363
+ **Read these numbers with two caveats.** They come from one 25-document French interview/business corpus, and part of its ground truth was auto-annotated with the tool's own regex patterns, which skews results in both directions. And they measure *pre-detection* only: mandatory validation is what makes the final output correct.
364
+
365
+ **Epic 10 (October 2026)** repaired that benchmark against written annotation guidelines, added a held-out set and the blind final exam above, and improved organisation detection. A fine-tuned French NER model remains the v3.0 target (70-85% F1).
325
366
 
326
- **Accuracy trajectory:** spaCy-only baseline → hybrid approach with annotation cleanup, expanded regex patterns, and French geography dictionary doubled F1 score. Story 7.5 added 12 ORG pattern keywords, POS-tag disambiguation for geography matching, and 7 international locations — reducing LOCATION false-negative rate from 27.42% to 12.90%.
367
+ **Accuracy trajectory:** Story 7.5 added 12 ORG pattern keywords, POS-tag disambiguation for geography matching, and 7 international locations — reducing LOCATION false-negative rate from 27.42% to 12.90%.
327
368
 
328
369
  **Approved Solution:**
329
370
  - ✅ **Hybrid approach** (NLP + regex + geography dictionary + POS disambiguation)
@@ -340,7 +381,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
340
381
  - ✅ **Entity-by-type grouping** - Review PERSON → ORG → LOCATION in logical order
341
382
  - ✅ **Context display** - See 10 words before/after each entity with highlighting
342
383
  - ✅ **Confidence scores** - Color-coded confidence from spaCy NER (green >80%, yellow 60-80%, red <60%)
343
- - ✅ **Keyboard shortcuts** - Single-key actions: [Space] Confirm, [R] Reject, [E] Modify, [A] Add, [C] Change pseudonym
384
+ - ✅ **Keyboard shortcuts** - Single-key actions: [Space] Confirm, [R] Reject, [E] Modify, [T] Change type, [A] Add, [C] Change pseudonym
344
385
  - ✅ **Batch operations** - Accept/reject all entities of a type at once (Shift+A/R) with entity count feedback
345
386
  - ✅ **Context cycling indicator** - Dot indicator (`● ○ ○ ○ ○`) shows current context position; `[Press X to cycle]` hint improves discoverability
346
387
  - ✅ **Help overlay** - Press [H] for full command reference
@@ -368,7 +409,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
368
409
  | **CLI Framework** | Typer | 0.9+ | Command-line interface |
369
410
  | **Database** | SQLite | 3.35+ | Local mapping table storage with WAL mode |
370
411
  | **Encryption** | cryptography (AESSIV) | 44.0+ | AES-256-SIV encryption for sensitive fields (PBKDF2 key derivation, passphrase-protected) |
371
- | **ORM** | SQLAlchemy | 2.0+ | Database abstraction and session management |
412
+ | **ORM** | SQLAlchemy | 2.0.x (<2.1) | Database abstraction and session management |
372
413
  | **Desktop GUI** | PySide6 | 6.7+ | Desktop application (optional: `pip install gdpr-pseudonymizer[gui]`) |
373
414
  | **Validation UI** | rich | 13.7+ | Interactive CLI entity review |
374
415
  | **Keyboard Input** | readchar | 4.2+ | Single-keypress capture for validation UI |
@@ -384,7 +425,7 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
384
425
  **Long answer:**
385
426
  1. **GDPR defensibility** - Human verification provides legal audit trail
386
427
  2. **Zero false negatives** - AI misses entities, humans catch them (100% coverage)
387
- 3. **Current NLP limitations** - French models on interview/business docs: 29.5% F1 out-of-box (hybrid approach reaches ~60%)
428
+ 3. **Current NLP limitations** - French models on interview/business docs: 29.5% F1 out-of-box (hybrid approach: F1 61.4% and recall 60.6% on unseen documents, 82.6% F1 on our development corpus; organisations are the weak point; improvement planned)
388
429
  4. **Better than alternatives:**
389
430
  - ✅ **vs Manual redaction:** 50%+ faster (AI pre-detection)
390
431
  - ✅ **vs Cloud services:** 100% local processing (no data leakage)
@@ -470,13 +511,13 @@ The validation UI provides an intuitive keyboard-driven interface for reviewing
470
511
 
471
512
  ## 🛠️ Development Status
472
513
 
473
- **Epics 1-7 Complete** — v2.1.0 (March 2026). GUI polish, Excel/CSV support, NER accuracy improvements.
514
+ **Epics 1-7 Complete** — current release v2.2.0 (September 2026): spaCy model choice, entity retyping, entity-span fixes. v2.1.0 (March 2026) closed Epic 7.
474
515
 
475
516
  - ✅ **Epic 1:** Foundation & NLP Validation (9 stories) — spaCy integration, validation UI, hybrid detection, entity deduplication
476
517
  - ✅ **Epic 2:** Core Pseudonymization Engine (9 stories) — pseudonym libraries, encryption, audit logging, batch processing, GDPR 1:1 mapping
477
518
  - ✅ **Epic 3:** CLI Interface & Batch Processing (7 stories) — 8 CLI commands, progress reporting, config files, parallel batch, UX polish
478
519
  - ✅ **Epic 4:** Launch Readiness (8 stories) — LLM utility validation, cross-platform testing, documentation, NER accuracy suite, performance validation, beta feedback integration, codebase refactoring, launch preparation
479
- - ✅ **Epic 5:** Quick Wins & GDPR Compliance (7 stories) — GDPR Article 17 erasure, gender-aware pseudonyms, NER accuracy improvements (F1 29.74% → 59.97%), French documentation translation, PDF/DOCX support, CLI polish & benchmarks, v1.1 release
520
+ - ✅ **Epic 5:** Quick Wins & GDPR Compliance (7 stories) — GDPR Article 17 erasure, gender-aware pseudonyms, NER accuracy work (annotation cleanup, expanded regex, geography dictionary), French documentation translation, PDF/DOCX support, CLI polish & benchmarks, v1.1 release
480
521
  - ✅ **Epic 6:** v2.0 Desktop GUI & Broader Accessibility (9 stories) — PySide6 desktop application, visual entity validation, batch GUI, i18n, WCAG AA, standalone executables
481
522
  - ✅ Story 6.1: UX Architecture & GUI Framework Selection
482
523
  - ✅ Story 6.2: GUI Application Foundation (main window, theming, home screen, settings, 77 GUI tests)
@@ -560,7 +601,7 @@ This project is licensed under the [MIT License](LICENSE).
560
601
  - ⚠️ Test thoroughly before production use
561
602
 
562
603
  **Current limitations:**
563
- - AI detection: ~60% F1 baseline (not 85%+)
604
+ - AI detection: F1 61.4% on unseen documents (82.6% on our development corpus), not 85%+; improving it is planned
564
605
  - Validation required for ALL documents (not optional)
565
606
  - French documents only (English, Spanish, etc. in future versions)
566
607
  - Text-based formats: .txt, .md, .pdf, .docx, .xlsx, .csv (PDF/DOCX/Excel require optional extras: `pip install gdpr-pseudonymizer[formats]`)
@@ -681,7 +722,7 @@ The integration test suite covers:
681
722
 
682
723
  | Metric | Value | Status |
683
724
  |--------|-------|--------|
684
- | **Development Progress** | v2.1.0 | ✅ Epics 1-7 complete |
725
+ | **Development Progress** | v2.2.0 | ✅ Epics 1-7 complete |
685
726
  | **Stories Complete** | 60 (Epics 1-7) | ✅ All epics complete |
686
727
  | **LLM Utility (NFR10)** | 4.27/5.0 (85.4%) | ✅ PASSED (threshold: 80%) |
687
728
  | **Installation Success (NFR3)** | 87.5% (7/8 platforms) | ✅ PASSED (threshold: 85%) |
@@ -689,7 +730,7 @@ The integration test suite covers:
689
730
  | **Critical Bugs Found** | 1 (Story 2.8) | ✅ RESOLVED - Epic 3 Unblocked |
690
731
  | **Test Corpus Size** | 25 docs, 1,737 entities | ✅ Complete (post-cleanup) |
691
732
  | **NLP Accuracy (Baseline)** | 29.5% F1 (spaCy only) | ✅ Measured (Story 1.2) |
692
- | **Hybrid Accuracy (NLP+Regex)** | 59.97% F1 (+30.23pp vs baseline) | ✅ Story 5.3 Complete |
733
+ | **Hybrid Accuracy (NLP+Regex)** | 61.4% F1 on unseen documents (final exam, CI run 38382553368); 82.6% development corpus, 78.1% held-out (CI run 38212303870) | ⚠️ Below target — improvement planned |
693
734
  | **Final Accuracy (AI+Human)** | 100% (validated) | 🎯 By Design |
694
735
  | **Pseudonym Libraries** | 3 themes (2,426 names + 240 locations + 588 orgs) | ✅ Stories 2.1, 3.0, 4.6 Complete |
695
736
  | **Compositional Matching** | Operational (component reuse + title stripping + compound names) | ✅ Stories 2.2, 2.3 Complete |
@@ -721,4 +762,4 @@ The integration test suite covers:
721
762
 
722
763
  ---
723
764
 
724
- **Last Updated:** 2026-03-17 (v2.1.0 — GUI polish, Excel/CSV support, neutral ID theme, NER accuracy improvements, 1670+ tests)
765
+ **Last Updated:** 2026-10-01 (v2.2.0 — spaCy model choice, entity retyping, entity-span fixes)
@@ -37,9 +37,12 @@ from gdpr_pseudonymizer.cli.progress import ETAColumn, ProgressTracker
37
37
  from gdpr_pseudonymizer.cli.validators import (
38
38
  ensure_database,
39
39
  parse_entity_type_filter,
40
+ validate_model_or_exit,
40
41
  validate_theme_or_exit,
41
42
  )
42
43
  from gdpr_pseudonymizer.core.document_processor import DocumentProcessor
44
+ from gdpr_pseudonymizer.data.database import open_database
45
+ from gdpr_pseudonymizer.exceptions import DatabaseConversionError
43
46
  from gdpr_pseudonymizer.utils.logger import configure_logging, get_logger
44
47
 
45
48
  # Configure logging
@@ -438,6 +441,7 @@ def batch_command(
438
441
 
439
442
  # Validate theme
440
443
  validate_theme_or_exit(effective_theme)
444
+ validate_model_or_exit(effective_model)
441
445
 
442
446
  # Get passphrase
443
447
  resolved_passphrase = resolve_passphrase(
@@ -449,6 +453,10 @@ def batch_command(
449
453
  # Initialize database if needed
450
454
  ensure_database(effective_db_path, resolved_passphrase, console)
451
455
 
456
+ # Open it once before any document: a pre-10.7 file is converted here,
457
+ # once, before any worker opens it (Story 10.7, AC3)
458
+ _convert_database_before_workers(effective_db_path, resolved_passphrase)
459
+
452
460
  # Determine processing mode based on workers parameter
453
461
  actual_workers = min(cpu_count(), effective_workers, 8)
454
462
  actual_workers = max(1, actual_workers)
@@ -712,6 +720,35 @@ def batch_command(
712
720
  sys.exit(2)
713
721
 
714
722
 
723
+ def _convert_database_before_workers(db_path: str, passphrase: str) -> None:
724
+ """Open the database once before any document is processed (Story 10.7).
725
+
726
+ A database made before Story 10.7 is converted by its first open, so it
727
+ is converted here, once, before the workers start. A conversion that
728
+ cannot be made stops the run before any document, with its message. Any
729
+ other open error is left to the per-document processing, as before.
730
+
731
+ Args:
732
+ db_path: Path to the database (created by ensure_database if missing)
733
+ passphrase: Database passphrase
734
+ """
735
+ if not Path(db_path).exists():
736
+ return
737
+ try:
738
+ with open_database(db_path, passphrase):
739
+ pass
740
+ except DatabaseConversionError as e:
741
+ format_error_message(
742
+ "Database Conversion Failed",
743
+ str(e),
744
+ "Make the database file and its folder writable, free some disk "
745
+ "space or close other programs using it, then run the command again.",
746
+ )
747
+ sys.exit(1)
748
+ except Exception as e:
749
+ logger.info("batch_database_precheck_skipped", error_type=type(e).__name__)
750
+
751
+
715
752
  def _display_batch_summary(result: BatchResult) -> None:
716
753
  """Display batch processing summary report.
717
754