mloda-pii 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mloda-pii might be problematic. Click here for more details.
- mloda_pii-0.3.0/LICENSE +72 -0
- mloda_pii-0.3.0/PKG-INFO +143 -0
- mloda_pii-0.3.0/README.md +111 -0
- mloda_pii-0.3.0/mloda_pii/__init__.py +0 -0
- mloda_pii-0.3.0/mloda_pii/feature_groups/__init__.py +1 -0
- mloda_pii-0.3.0/mloda_pii/feature_groups/anonymizer_rs/__init__.py +13 -0
- mloda_pii-0.3.0/mloda_pii/feature_groups/anonymizer_rs/base.py +68 -0
- mloda_pii-0.3.0/mloda_pii/feature_groups/anonymizer_rs/binary.py +124 -0
- mloda_pii-0.3.0/mloda_pii/feature_groups/anonymizer_rs/pyarrow_anonymizer_rs.py +36 -0
- mloda_pii-0.3.0/mloda_pii/feature_groups/manifest.py +8 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/__init__.py +0 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/cli.py +53 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/__init__.py +4 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/blocking.py +46 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/claude_hook.py +63 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/cli_request.py +150 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/config.py +73 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/detect.py +164 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/handlers.py +292 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/onboarding.py +404 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/pipeline.py +148 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/server.py +114 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/settings_file.py +145 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/tokens.py +45 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/hook/vault.py +92 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/invitation_code/__init__.py +0 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/invitation_code/license.py +223 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/invitation_code/licensed_group.py +22 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/invitation_code/store.py +99 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/stats/__init__.py +0 -0
- mloda_pii-0.3.0/mloda_pii/user_interface/stats/stats.py +68 -0
- mloda_pii-0.3.0/mloda_pii.egg-info/PKG-INFO +143 -0
- mloda_pii-0.3.0/mloda_pii.egg-info/SOURCES.txt +41 -0
- mloda_pii-0.3.0/mloda_pii.egg-info/dependency_links.txt +1 -0
- mloda_pii-0.3.0/mloda_pii.egg-info/entry_points.txt +5 -0
- mloda_pii-0.3.0/mloda_pii.egg-info/requires.txt +18 -0
- mloda_pii-0.3.0/mloda_pii.egg-info/top_level.txt +1 -0
- mloda_pii-0.3.0/pyproject.toml +86 -0
- mloda_pii-0.3.0/setup.cfg +4 -0
- mloda_pii-0.3.0/tests/test_entry_points.py +35 -0
- mloda_pii-0.3.0/tests/test_mloda_imports.py +23 -0
- mloda_pii-0.3.0/tests/test_package_config.py +112 -0
- mloda_pii-0.3.0/tests/test_reserved_namespace.py +103 -0
mloda_pii-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
mloda-pii License
|
|
2
|
+
|
|
3
|
+
Copyright 2026-present Tom Kaltofen (mloda)
|
|
4
|
+
|
|
5
|
+
Licensor: Tom Kaltofen, trading as mloda, Berlin, Germany (contact details at
|
|
6
|
+
https://mloda.ai/impressum), and any successor entity to which this license is
|
|
7
|
+
assigned.
|
|
8
|
+
|
|
9
|
+
This license covers the software it accompanies (the "Software"): mloda-pii
|
|
10
|
+
(the code of the mloda-pii repository and the packages built from it). It
|
|
11
|
+
follows the mloda Enterprise License. The Software is source-available. It is
|
|
12
|
+
not open source.
|
|
13
|
+
|
|
14
|
+
Three things are called a license here. This text is the license. A license
|
|
15
|
+
key is the signed file or token the Software verifies before its licensed mode
|
|
16
|
+
runs; an invite issued by the Licensor is one. A license agreement is a written
|
|
17
|
+
contract with the Licensor.
|
|
18
|
+
|
|
19
|
+
PERMITTED WITHOUT A LICENSE KEY OR LICENSE AGREEMENT
|
|
20
|
+
|
|
21
|
+
You may:
|
|
22
|
+
- view, read, copy and modify the Software for development, testing and
|
|
23
|
+
evaluation;
|
|
24
|
+
- install it from PyPI, run its test suites and run automated security scans
|
|
25
|
+
on it;
|
|
26
|
+
- build and test your own code against it;
|
|
27
|
+
- keep and pass on unmodified copies as part of a mirror of a package index,
|
|
28
|
+
with the notices kept.
|
|
29
|
+
|
|
30
|
+
PRODUCTION USE
|
|
31
|
+
|
|
32
|
+
Production use means operating the Software for your business, for example
|
|
33
|
+
processing live data with it or using it in your daily work. Production use is
|
|
34
|
+
permitted while you hold a valid license key issued by the Licensor, including
|
|
35
|
+
one issued free of charge, and for the key's validity period and grace window,
|
|
36
|
+
in the local mode with an anonymizer binary the Licensor provided to you, or
|
|
37
|
+
under a license agreement. The Licensor may decide at any time whether to
|
|
38
|
+
issue or renew license keys. The licensed mode runs only with a key the
|
|
39
|
+
Licensor issues and stops after the key's grace window.
|
|
40
|
+
|
|
41
|
+
LIMITATIONS
|
|
42
|
+
|
|
43
|
+
You may not:
|
|
44
|
+
- distribute, sublicense or sell the Software or a modified copy, or ship it as
|
|
45
|
+
part of your own product, except as permitted above or by a license agreement;
|
|
46
|
+
- provide the Software to third parties as a hosted or managed service, except
|
|
47
|
+
as a license agreement permits;
|
|
48
|
+
- share a license key issued to you with others;
|
|
49
|
+
- move, change, disable or circumvent the license key functionality of the
|
|
50
|
+
Software or of a binary it uses, or remove or obscure functionality it
|
|
51
|
+
protects;
|
|
52
|
+
- alter, remove or obscure any licensing, copyright or other notice of the
|
|
53
|
+
Licensor;
|
|
54
|
+
- decompile or reverse engineer a shipped binary, except where mandatory law
|
|
55
|
+
permits it.
|
|
56
|
+
|
|
57
|
+
A modification you make remains subject to this license.
|
|
58
|
+
|
|
59
|
+
THIRD-PARTY COMPONENTS
|
|
60
|
+
|
|
61
|
+
Third-party components used by the Software remain under their own licenses.
|
|
62
|
+
The binary wheels the Software installs carry their own license and notices.
|
|
63
|
+
|
|
64
|
+
NO WARRANTY
|
|
65
|
+
|
|
66
|
+
Outside a license agreement, the Software is provided free of charge and "as
|
|
67
|
+
is", without warranty of any kind. The Software does not guarantee that all
|
|
68
|
+
personal data is detected or withheld; you remain responsible for your
|
|
69
|
+
processing of personal data. The Licensor is liable only for intent and gross
|
|
70
|
+
negligence; liability for injury to life, body or health and under the Product
|
|
71
|
+
Liability Act remains unaffected. Warranty, liability, support and term under a
|
|
72
|
+
license agreement are governed by that agreement. German law applies.
|
mloda_pii-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mloda-pii
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Claude Code hook and mloda FeatureGroup that pseudonymize PII before the model sees it
|
|
5
|
+
Author-email: Tom Kaltofen <info@mloda.ai>
|
|
6
|
+
License-Expression: LicenseRef-Proprietary
|
|
7
|
+
Project-URL: Homepage, https://mloda.ai
|
|
8
|
+
Keywords: mloda,mloda-plugin,pii,privacy,redaction,anonymization,claude-code,hook
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: mloda[pyarrow]<0.18.0,>=0.17.0
|
|
18
|
+
Requires-Dist: tomli>=1.0.0; python_version < "3.11"
|
|
19
|
+
Provides-Extra: hook
|
|
20
|
+
Requires-Dist: mloda-enterprise[anonymizer]>=0.11.0; extra == "hook"
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: mloda-testing[binary-model]>=0.11.0; extra == "dev"
|
|
23
|
+
Requires-Dist: mloda-enterprise>=0.11.0; extra == "dev"
|
|
24
|
+
Requires-Dist: tox; extra == "dev"
|
|
25
|
+
Requires-Dist: tox-uv; extra == "dev"
|
|
26
|
+
Requires-Dist: uv>=0.13.4; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest; extra == "dev"
|
|
28
|
+
Requires-Dist: ruff; extra == "dev"
|
|
29
|
+
Requires-Dist: mypy; extra == "dev"
|
|
30
|
+
Requires-Dist: bandit; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
[](https://pypi.org/project/mloda-pii/)
|
|
34
|
+
[](https://www.python.org/)
|
|
35
|
+

|
|
36
|
+
[](https://github.com/mloda-ai/mloda)
|
|
37
|
+
|
|
38
|
+
# mloda-pii
|
|
39
|
+
|
|
40
|
+
mloda-pii is a Claude Code hook that replaces
|
|
41
|
+
identifiers (emails, phone numbers, IBANs, IP addresses, your own dictionary entries) in tool results with tokens such
|
|
42
|
+
as `PII_EMAIL_...` before the model sees them. Your local tools and Claude's replies on your screen keep the real
|
|
43
|
+
values. Every token is a keyed HMAC-SHA256, computed through [mloda](https://github.com/mloda-ai/mloda). It guards
|
|
44
|
+
against accidental exposure and does not see everything the model gets; see Limits below.
|
|
45
|
+
|
|
46
|
+
The package also ships a mloda FeatureGroup that tokenizes a string column the same way, without a license key.
|
|
47
|
+
|
|
48
|
+
## Setup
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
uv tool install "mloda-pii[hook]" # or: pip install "mloda-pii[hook]" into a virtualenv that stays in place
|
|
52
|
+
cd your-project
|
|
53
|
+
mloda-pii init invite.txt # the invite file, '-' for stdin, or the token itself
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Then restart Claude Code. `init` creates the master key, stores and verifies the license, runs the hook once on a test
|
|
57
|
+
value, and only then registers it in `.claude/settings.local.json`. It also creates `.mloda-pii/dictionary.txt` for
|
|
58
|
+
names and company-specific IDs to detect, kept out of git. Running it again is safe.
|
|
59
|
+
|
|
60
|
+
| Option | Effect |
|
|
61
|
+
| --- | --- |
|
|
62
|
+
| `--user` | Register in `~/.claude/settings.json`, for every project. |
|
|
63
|
+
| `--project` | Register in the committed `.claude/settings.json`; every teammate installs mloda-pii and runs `init --project`. |
|
|
64
|
+
| `--mode local` | No license key, see below. |
|
|
65
|
+
| `--key-file <path>` | Import an existing master key instead of creating one. |
|
|
66
|
+
|
|
67
|
+
Ask for an invite at info@mloda.ai or via [mloda.ai](https://mloda.ai).
|
|
68
|
+
|
|
69
|
+
## Modes
|
|
70
|
+
|
|
71
|
+
| Mode | License key | Needs |
|
|
72
|
+
| --- | --- | --- |
|
|
73
|
+
| `licensed` (default) | Required. Without a valid license every gated event blocks: tool results are withheld, tool calls denied, prompts blocked. | An invite. |
|
|
74
|
+
| `local` | None. | An anonymizer-rs `anonymizer` binary on `PATH` or in `ANONYMIZER_BINARY`; ask at info@mloda.ai. |
|
|
75
|
+
|
|
76
|
+
Both modes give the same tokens from the same master key; there is no fallback between them.
|
|
77
|
+
|
|
78
|
+
## Back up your master key
|
|
79
|
+
|
|
80
|
+
The master key lives in `~/.config/mloda-pii/master.key`. Back it up: every token depends on it, and a lost key
|
|
81
|
+
changes all of them. `init` never overwrites an existing key.
|
|
82
|
+
|
|
83
|
+
## Everyday commands
|
|
84
|
+
|
|
85
|
+
- `mloda-pii status`: the key, the license and its expiry, where the hook is registered, the last events. Exits 1 when
|
|
86
|
+
something is missing.
|
|
87
|
+
- `mloda-pii uninstall`: removes the hook entries (same `--user` / `--project` flags). `--purge` also deletes the
|
|
88
|
+
master key and the stored license, after asking (`--yes` skips it).
|
|
89
|
+
- `mloda-pii license install | status | remove`: manage the license alone.
|
|
90
|
+
- `mloda-pii stats`: per-day counts from the audit log, which never holds values.
|
|
91
|
+
|
|
92
|
+
A prompt that contains raw identifiers is blocked and a pseudonymized copy is shown to resubmit.
|
|
93
|
+
|
|
94
|
+
| Variable | Effect |
|
|
95
|
+
| --- | --- |
|
|
96
|
+
| `MLODA_PII_KEY_FILE` | Master key file (default `~/.config/mloda-pii/master.key`). |
|
|
97
|
+
| `MLODA_PII_DICTIONARY` | Literals and `re:` patterns to detect (default `.mloda-pii/dictionary.txt`). |
|
|
98
|
+
| `MLODA_PII_NAME_HEURISTIC` | `1` enables a capitalised-bigram name detector; otherwise only dictionary names are detected. |
|
|
99
|
+
| `MLODA_PII_WRAP_BASH` | `1` routes failed Bash output through the hook too. |
|
|
100
|
+
| `MLODA_PII_TOKEN_STYLE` | `hex` (default, `PII_EMAIL_<32 hex>`) or `readable` (`PII_EMAIL_1`, numbered per project). |
|
|
101
|
+
| `MLODA_PII_FAIL_CLOSED` | On by default. `0` lets an action pass when the hook itself fails; a license block always blocks. |
|
|
102
|
+
| `MLODA_PII_REVERSE_MCP` | `1` also puts real values back into MCP tool input; a remote MCP server then receives them. |
|
|
103
|
+
| `MLODA_PII_AUDIT_LOG` | Audit log path (default `~/.local/state/mloda-pii/audit.jsonl`); `off` disables it. |
|
|
104
|
+
| `MLODA_LICENSE_FILE`, `MLODA_LICENSE_KEY` | Take precedence over the stored license. |
|
|
105
|
+
|
|
106
|
+
Command options: `mloda-pii <command> --help`.
|
|
107
|
+
|
|
108
|
+
## The FeatureGroup
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from mloda.user import Feature, Options
|
|
112
|
+
|
|
113
|
+
feature = Feature("email__pii_anonymized", Options(context={"pii_key_file_env": "KEY_FILE"}))
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
`<column>__pii_anonymized` returns the 64-hex keyed HMAC-SHA256 token of a string column. `pii_key_file_env` names the
|
|
117
|
+
environment variable that holds the path to the master key file. It needs the anonymizer-rs binary, like the local
|
|
118
|
+
mode.
|
|
119
|
+
|
|
120
|
+
## Limits
|
|
121
|
+
|
|
122
|
+
- The hook guards against accidental exposure, not a model that tries to recover values: a Bash command gets the real
|
|
123
|
+
value and can return it disguised, which detection misses.
|
|
124
|
+
- Hooks never see `@file` mentions, CLAUDE.md and memory files, the client's own reminders, images or compaction
|
|
125
|
+
summaries; those reach the model unchanged.
|
|
126
|
+
- A crashed or timed-out hook on a tool result lets the original output through.
|
|
127
|
+
- The values of one event sit in a private temporary Parquet file (under `$XDG_RUNTIME_DIR`, else `/dev/shm`, else the
|
|
128
|
+
temp dir) for the duration of the call.
|
|
129
|
+
- The license gate is a speed bump: the local mode makes the same tokens without one, by design.
|
|
130
|
+
- Tokens are still personal data under GDPR; whoever holds the key can recompute them.
|
|
131
|
+
- The local vault under `~/.local/state/mloda-pii/` (or `$XDG_STATE_HOME`) maps tokens back to raw values; nothing expires it, and
|
|
132
|
+
`uninstall --purge` leaves it in place.
|
|
133
|
+
|
|
134
|
+
## License
|
|
135
|
+
|
|
136
|
+
mloda-pii is proprietary and source-available, under the mloda-pii License in the `LICENSE` file of the package. Use
|
|
137
|
+
in your daily work is permitted with a license key from mloda, also when the key is free, or in the local mode with an
|
|
138
|
+
anonymizer binary mloda provided. The license text is binding over this summary.
|
|
139
|
+
|
|
140
|
+
## Related
|
|
141
|
+
|
|
142
|
+
- [mloda](https://github.com/mloda-ai/mloda): declarative data and feature access, the framework mloda-pii runs on.
|
|
143
|
+
- [mloda-registry](https://github.com/mloda-ai/mloda-registry): plugins and guides for mloda.
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
[](https://pypi.org/project/mloda-pii/)
|
|
2
|
+
[](https://www.python.org/)
|
|
3
|
+

|
|
4
|
+
[](https://github.com/mloda-ai/mloda)
|
|
5
|
+
|
|
6
|
+
# mloda-pii
|
|
7
|
+
|
|
8
|
+
mloda-pii is a Claude Code hook that replaces
|
|
9
|
+
identifiers (emails, phone numbers, IBANs, IP addresses, your own dictionary entries) in tool results with tokens such
|
|
10
|
+
as `PII_EMAIL_...` before the model sees them. Your local tools and Claude's replies on your screen keep the real
|
|
11
|
+
values. Every token is a keyed HMAC-SHA256, computed through [mloda](https://github.com/mloda-ai/mloda). It guards
|
|
12
|
+
against accidental exposure and does not see everything the model gets; see Limits below.
|
|
13
|
+
|
|
14
|
+
The package also ships a mloda FeatureGroup that tokenizes a string column the same way, without a license key.
|
|
15
|
+
|
|
16
|
+
## Setup
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
uv tool install "mloda-pii[hook]" # or: pip install "mloda-pii[hook]" into a virtualenv that stays in place
|
|
20
|
+
cd your-project
|
|
21
|
+
mloda-pii init invite.txt # the invite file, '-' for stdin, or the token itself
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Then restart Claude Code. `init` creates the master key, stores and verifies the license, runs the hook once on a test
|
|
25
|
+
value, and only then registers it in `.claude/settings.local.json`. It also creates `.mloda-pii/dictionary.txt` for
|
|
26
|
+
names and company-specific IDs to detect, kept out of git. Running it again is safe.
|
|
27
|
+
|
|
28
|
+
| Option | Effect |
|
|
29
|
+
| --- | --- |
|
|
30
|
+
| `--user` | Register in `~/.claude/settings.json`, for every project. |
|
|
31
|
+
| `--project` | Register in the committed `.claude/settings.json`; every teammate installs mloda-pii and runs `init --project`. |
|
|
32
|
+
| `--mode local` | No license key, see below. |
|
|
33
|
+
| `--key-file <path>` | Import an existing master key instead of creating one. |
|
|
34
|
+
|
|
35
|
+
Ask for an invite at info@mloda.ai or via [mloda.ai](https://mloda.ai).
|
|
36
|
+
|
|
37
|
+
## Modes
|
|
38
|
+
|
|
39
|
+
| Mode | License key | Needs |
|
|
40
|
+
| --- | --- | --- |
|
|
41
|
+
| `licensed` (default) | Required. Without a valid license every gated event blocks: tool results are withheld, tool calls denied, prompts blocked. | An invite. |
|
|
42
|
+
| `local` | None. | An anonymizer-rs `anonymizer` binary on `PATH` or in `ANONYMIZER_BINARY`; ask at info@mloda.ai. |
|
|
43
|
+
|
|
44
|
+
Both modes give the same tokens from the same master key; there is no fallback between them.
|
|
45
|
+
|
|
46
|
+
## Back up your master key
|
|
47
|
+
|
|
48
|
+
The master key lives in `~/.config/mloda-pii/master.key`. Back it up: every token depends on it, and a lost key
|
|
49
|
+
changes all of them. `init` never overwrites an existing key.
|
|
50
|
+
|
|
51
|
+
## Everyday commands
|
|
52
|
+
|
|
53
|
+
- `mloda-pii status`: the key, the license and its expiry, where the hook is registered, the last events. Exits 1 when
|
|
54
|
+
something is missing.
|
|
55
|
+
- `mloda-pii uninstall`: removes the hook entries (same `--user` / `--project` flags). `--purge` also deletes the
|
|
56
|
+
master key and the stored license, after asking (`--yes` skips it).
|
|
57
|
+
- `mloda-pii license install | status | remove`: manage the license alone.
|
|
58
|
+
- `mloda-pii stats`: per-day counts from the audit log, which never holds values.
|
|
59
|
+
|
|
60
|
+
A prompt that contains raw identifiers is blocked and a pseudonymized copy is shown to resubmit.
|
|
61
|
+
|
|
62
|
+
| Variable | Effect |
|
|
63
|
+
| --- | --- |
|
|
64
|
+
| `MLODA_PII_KEY_FILE` | Master key file (default `~/.config/mloda-pii/master.key`). |
|
|
65
|
+
| `MLODA_PII_DICTIONARY` | Literals and `re:` patterns to detect (default `.mloda-pii/dictionary.txt`). |
|
|
66
|
+
| `MLODA_PII_NAME_HEURISTIC` | `1` enables a capitalised-bigram name detector; otherwise only dictionary names are detected. |
|
|
67
|
+
| `MLODA_PII_WRAP_BASH` | `1` routes failed Bash output through the hook too. |
|
|
68
|
+
| `MLODA_PII_TOKEN_STYLE` | `hex` (default, `PII_EMAIL_<32 hex>`) or `readable` (`PII_EMAIL_1`, numbered per project). |
|
|
69
|
+
| `MLODA_PII_FAIL_CLOSED` | On by default. `0` lets an action pass when the hook itself fails; a license block always blocks. |
|
|
70
|
+
| `MLODA_PII_REVERSE_MCP` | `1` also puts real values back into MCP tool input; a remote MCP server then receives them. |
|
|
71
|
+
| `MLODA_PII_AUDIT_LOG` | Audit log path (default `~/.local/state/mloda-pii/audit.jsonl`); `off` disables it. |
|
|
72
|
+
| `MLODA_LICENSE_FILE`, `MLODA_LICENSE_KEY` | Take precedence over the stored license. |
|
|
73
|
+
|
|
74
|
+
Command options: `mloda-pii <command> --help`.
|
|
75
|
+
|
|
76
|
+
## The FeatureGroup
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from mloda.user import Feature, Options
|
|
80
|
+
|
|
81
|
+
feature = Feature("email__pii_anonymized", Options(context={"pii_key_file_env": "KEY_FILE"}))
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`<column>__pii_anonymized` returns the 64-hex keyed HMAC-SHA256 token of a string column. `pii_key_file_env` names the
|
|
85
|
+
environment variable that holds the path to the master key file. It needs the anonymizer-rs binary, like the local
|
|
86
|
+
mode.
|
|
87
|
+
|
|
88
|
+
## Limits
|
|
89
|
+
|
|
90
|
+
- The hook guards against accidental exposure, not a model that tries to recover values: a Bash command gets the real
|
|
91
|
+
value and can return it disguised, which detection misses.
|
|
92
|
+
- Hooks never see `@file` mentions, CLAUDE.md and memory files, the client's own reminders, images or compaction
|
|
93
|
+
summaries; those reach the model unchanged.
|
|
94
|
+
- A crashed or timed-out hook on a tool result lets the original output through.
|
|
95
|
+
- The values of one event sit in a private temporary Parquet file (under `$XDG_RUNTIME_DIR`, else `/dev/shm`, else the
|
|
96
|
+
temp dir) for the duration of the call.
|
|
97
|
+
- The license gate is a speed bump: the local mode makes the same tokens without one, by design.
|
|
98
|
+
- Tokens are still personal data under GDPR; whoever holds the key can recompute them.
|
|
99
|
+
- The local vault under `~/.local/state/mloda-pii/` (or `$XDG_STATE_HOME`) maps tokens back to raw values; nothing expires it, and
|
|
100
|
+
`uninstall --purge` leaves it in place.
|
|
101
|
+
|
|
102
|
+
## License
|
|
103
|
+
|
|
104
|
+
mloda-pii is proprietary and source-available, under the mloda-pii License in the `LICENSE` file of the package. Use
|
|
105
|
+
in your daily work is permitted with a license key from mloda, also when the key is free, or in the local mode with an
|
|
106
|
+
anonymizer binary mloda provided. The license text is binding over this summary.
|
|
107
|
+
|
|
108
|
+
## Related
|
|
109
|
+
|
|
110
|
+
- [mloda](https://github.com/mloda-ai/mloda): declarative data and feature access, the framework mloda-pii runs on.
|
|
111
|
+
- [mloda-registry](https://github.com/mloda-ai/mloda-registry): plugins and guides for mloda.
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Feature groups namespace package."""
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Local, license-free FeatureGroup over the anonymizer-rs binary."""
|
|
2
|
+
|
|
3
|
+
from mloda_pii.feature_groups.anonymizer_rs.base import PII_KEY_FILE_ENV, LocalAnonymizerFeatureGroup
|
|
4
|
+
from mloda_pii.feature_groups.anonymizer_rs.binary import AnonymizerRsError, derive_anon_key
|
|
5
|
+
from mloda_pii.feature_groups.anonymizer_rs.pyarrow_anonymizer_rs import PyArrowLocalAnonymizerFeatureGroup
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"PII_KEY_FILE_ENV",
|
|
9
|
+
"AnonymizerRsError",
|
|
10
|
+
"LocalAnonymizerFeatureGroup",
|
|
11
|
+
"PyArrowLocalAnonymizerFeatureGroup",
|
|
12
|
+
"derive_anon_key",
|
|
13
|
+
]
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Framework-independent API of the local, license-free anonymizer-rs FeatureGroup."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from abc import abstractmethod
|
|
7
|
+
from typing import Any, ClassVar
|
|
8
|
+
|
|
9
|
+
from mloda.provider import DefaultOptionKeys, FeatureChainParserMixin, FeatureGroup, FeatureSet, property_spec
|
|
10
|
+
from mloda.user import DataType, Feature
|
|
11
|
+
|
|
12
|
+
from mloda_pii.feature_groups.anonymizer_rs.binary import AnonymizerRsError
|
|
13
|
+
|
|
14
|
+
PII_KEY_FILE_ENV = "pii_key_file_env"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class LocalAnonymizerFeatureGroup(FeatureChainParserMixin, FeatureGroup):
|
|
18
|
+
"""Keyed HMAC-SHA256 token (64 hex) of one string column by the anonymizer-rs binary; no license check.
|
|
19
|
+
|
|
20
|
+
String form: ``Feature("email__pii_anonymized", Options(context={"pii_key_file_env": "KEY_FILE"}))``.
|
|
21
|
+
``pii_key_file_env`` names the env var holding the path to the master key file (at least 32 bytes).
|
|
22
|
+
Same tokens as the licensed ``__hmac_sha256_pseudonymized`` group under the key derived by ``derive_anon_key``.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
PREFIX_PATTERN = r".+__pii_anonymized$"
|
|
26
|
+
RECOGNITION_ONLY_PATTERN = True
|
|
27
|
+
MIN_IN_FEATURES = 1
|
|
28
|
+
MAX_IN_FEATURES = 1
|
|
29
|
+
BINARY_COMMAND_OVERRIDE: ClassVar[list[str] | None] = None
|
|
30
|
+
BINARY_TIMEOUT_SECONDS: ClassVar[float | None] = 60.0
|
|
31
|
+
|
|
32
|
+
PROPERTY_MAPPING: ClassVar = {
|
|
33
|
+
# Required and not strict, so no validator can echo it.
|
|
34
|
+
PII_KEY_FILE_ENV: property_spec("Name of the env var holding the path to the master key file"),
|
|
35
|
+
DefaultOptionKeys.in_features: property_spec("Source column to anonymize"),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
@classmethod
|
|
39
|
+
def _validate_string_match(cls, _feature_name: str, _operation_config: str, _in_feature: str) -> bool:
|
|
40
|
+
return cls.IN_FEATURE_SEPARATOR not in _in_feature
|
|
41
|
+
|
|
42
|
+
@classmethod
|
|
43
|
+
def return_data_type_rule(cls, feature: Feature) -> DataType | None:
|
|
44
|
+
return DataType.STRING
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def calculate_feature(cls, data: Any, features: FeatureSet) -> Any:
|
|
48
|
+
by_key_file: dict[str, list[tuple[str, str]]] = {}
|
|
49
|
+
for feature in features.features:
|
|
50
|
+
key_file = cls._key_file(feature.options.get(PII_KEY_FILE_ENV))
|
|
51
|
+
source = cls._extract_single_source_feature(feature)
|
|
52
|
+
by_key_file.setdefault(key_file, []).append((source, str(feature.name)))
|
|
53
|
+
for key_file, mappings in by_key_file.items():
|
|
54
|
+
data = cls._anonymize(data, mappings, key_file)
|
|
55
|
+
return data
|
|
56
|
+
|
|
57
|
+
@classmethod
|
|
58
|
+
def _key_file(cls, env_name: object) -> str:
|
|
59
|
+
"""The key file path from the named variable; errors never echo the name's value."""
|
|
60
|
+
path = os.environ.get(env_name, "").strip() if isinstance(env_name, str) and env_name else ""
|
|
61
|
+
if not path:
|
|
62
|
+
raise AnonymizerRsError(f"{PII_KEY_FILE_ENV} must name a set env var holding the master key file path")
|
|
63
|
+
return path
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
@abstractmethod
|
|
67
|
+
def _anonymize(cls, data: Any, mappings: list[tuple[str, str]], key_file: str) -> Any:
|
|
68
|
+
"""Append one token column per (source, output) pair; abstract so mloda never selects the base."""
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Runs the anonymizer-rs ``anonymizer`` binary over Arrow IPC; no mloda imports and no license check."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import hmac
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import shutil
|
|
10
|
+
import subprocess # nosec B404
|
|
11
|
+
import sys
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import pyarrow as pa
|
|
16
|
+
|
|
17
|
+
BINARY_ENV = "ANONYMIZER_BINARY"
|
|
18
|
+
BINARY_NAME = "anonymizer"
|
|
19
|
+
MIN_MASTER_KEY_BYTES = 32
|
|
20
|
+
_ANON_INFO = b"anonymizer-rs-anon"
|
|
21
|
+
_MESSAGE_LIMIT = 1024
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class AnonymizerRsError(ValueError):
|
|
25
|
+
"""The binary was missing, failed, or broke its output contract; never carries cell values."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, message: str, code: int | None = None) -> None:
|
|
28
|
+
super().__init__(message)
|
|
29
|
+
self.code = code
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def derive_anon_key(master_key: bytes) -> str:
|
|
33
|
+
"""The 64-hex HMAC key anonymizer-rs derives from a master key (HKDF-SHA256, no salt)."""
|
|
34
|
+
if len(master_key) < MIN_MASTER_KEY_BYTES:
|
|
35
|
+
raise AnonymizerRsError(f"the master key must be at least {MIN_MASTER_KEY_BYTES} bytes, got {len(master_key)}")
|
|
36
|
+
prk = hmac.new(bytes(32), master_key, hashlib.sha256).digest()
|
|
37
|
+
return hmac.new(prk, _ANON_INFO + b"\x01", hashlib.sha256).hexdigest()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def find_binary(override: Sequence[str] | None = None) -> list[str]:
|
|
41
|
+
"""argv prefix: ``override``, else ``$ANONYMIZER_BINARY``, else ``anonymizer`` on PATH."""
|
|
42
|
+
if override:
|
|
43
|
+
return list(override)
|
|
44
|
+
configured = os.environ.get(BINARY_ENV, "")
|
|
45
|
+
if configured:
|
|
46
|
+
path = Path(configured)
|
|
47
|
+
if not (path.is_absolute() and path.is_file() and os.access(path, os.X_OK)):
|
|
48
|
+
raise AnonymizerRsError(f"{BINARY_ENV} must be an absolute path to an executable file")
|
|
49
|
+
return [configured]
|
|
50
|
+
found = shutil.which(BINARY_NAME)
|
|
51
|
+
if found is None:
|
|
52
|
+
raise AnonymizerRsError(
|
|
53
|
+
f"anonymizer-rs binary not found: set {BINARY_ENV} or put '{BINARY_NAME}' on PATH "
|
|
54
|
+
"(bin/build-anonymizer-rs.sh builds it)"
|
|
55
|
+
)
|
|
56
|
+
return [found]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _environment(key_file: str, columns: str) -> dict[str, str]:
|
|
60
|
+
env = {"PATH": os.environ.get("PATH", ""), "ANONYMIZER_KEY_FILE": key_file, "ANONYMIZER_COLUMNS": columns}
|
|
61
|
+
if sys.platform == "win32" and "SYSTEMROOT" in os.environ:
|
|
62
|
+
env["SYSTEMROOT"] = os.environ["SYSTEMROOT"]
|
|
63
|
+
return env
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _ipc_bytes(table: pa.Table) -> bytes:
|
|
67
|
+
sink = pa.BufferOutputStream()
|
|
68
|
+
with pa.ipc.new_stream(sink, table.schema) as writer:
|
|
69
|
+
writer.write_table(table)
|
|
70
|
+
return bytes(sink.getvalue())
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _binary_message(stderr: bytes) -> str:
|
|
74
|
+
"""The ``error`` field of the binary's JSON stderr line, bounded; raw stderr is never echoed."""
|
|
75
|
+
for line in reversed(stderr.decode("utf-8", errors="replace").strip().splitlines()):
|
|
76
|
+
try:
|
|
77
|
+
parsed = json.loads(line)
|
|
78
|
+
except ValueError:
|
|
79
|
+
continue
|
|
80
|
+
error = parsed.get("error") if isinstance(parsed, dict) else None
|
|
81
|
+
if isinstance(error, str):
|
|
82
|
+
return error[:_MESSAGE_LIMIT]
|
|
83
|
+
return "no error message"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def anonymize_columns(
|
|
87
|
+
argv: Sequence[str], columns: Sequence[pa.ChunkedArray], key_file: str, timeout: float | None
|
|
88
|
+
) -> list[pa.ChunkedArray]:
|
|
89
|
+
"""One token column per input column, in one binary call; columns travel under positional names."""
|
|
90
|
+
if not columns:
|
|
91
|
+
return []
|
|
92
|
+
rows = len(columns[0])
|
|
93
|
+
if rows == 0:
|
|
94
|
+
return [pa.chunked_array([], pa.string()) for _ in columns]
|
|
95
|
+
inputs = [f"c{i}" for i in range(len(columns))]
|
|
96
|
+
outputs = [f"o{i}" for i in range(len(columns))]
|
|
97
|
+
table = pa.table(dict(zip(inputs, columns, strict=True)))
|
|
98
|
+
spec = ",".join(f"{i}:{o}" for i, o in zip(inputs, outputs, strict=True))
|
|
99
|
+
try:
|
|
100
|
+
proc = subprocess.run( # nosec B603
|
|
101
|
+
list(argv),
|
|
102
|
+
input=_ipc_bytes(table),
|
|
103
|
+
capture_output=True,
|
|
104
|
+
env=_environment(key_file, spec),
|
|
105
|
+
timeout=timeout,
|
|
106
|
+
check=False,
|
|
107
|
+
)
|
|
108
|
+
except subprocess.TimeoutExpired:
|
|
109
|
+
raise AnonymizerRsError(f"anonymizer-rs binary timed out after {timeout} s") from None
|
|
110
|
+
except OSError as exc:
|
|
111
|
+
raise AnonymizerRsError(f"anonymizer-rs binary could not start: {exc.strerror}") from None
|
|
112
|
+
if proc.returncode != 0:
|
|
113
|
+
message = _binary_message(proc.stderr)
|
|
114
|
+
raise AnonymizerRsError(f"anonymizer-rs binary exited {proc.returncode}: {message}", proc.returncode)
|
|
115
|
+
try:
|
|
116
|
+
result = pa.ipc.open_stream(proc.stdout).read_all()
|
|
117
|
+
except (pa.ArrowException, OSError) as exc:
|
|
118
|
+
raise AnonymizerRsError(f"anonymizer-rs output is not an Arrow IPC stream: {type(exc).__name__}") from None
|
|
119
|
+
if sorted(result.column_names) != sorted(outputs) or result.num_rows != rows:
|
|
120
|
+
raise AnonymizerRsError(
|
|
121
|
+
f"anonymizer-rs output has columns {sorted(result.column_names)} and {result.num_rows} rows, "
|
|
122
|
+
f"expected {sorted(outputs)} and {rows}"
|
|
123
|
+
)
|
|
124
|
+
return [result.column(name).cast(pa.string()) for name in outputs]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""PyArrow implementation of the local anonymizer-rs FeatureGroup."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import pyarrow as pa
|
|
8
|
+
from mloda.provider import ComputeFramework
|
|
9
|
+
from mloda_plugins.compute_framework.base_implementations.pyarrow.table import PyArrowTable
|
|
10
|
+
|
|
11
|
+
from mloda_pii.feature_groups.anonymizer_rs.base import LocalAnonymizerFeatureGroup
|
|
12
|
+
from mloda_pii.feature_groups.anonymizer_rs.binary import AnonymizerRsError, anonymize_columns, find_binary
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PyArrowLocalAnonymizerFeatureGroup(LocalAnonymizerFeatureGroup):
|
|
16
|
+
@classmethod
|
|
17
|
+
def compute_framework_rule(cls) -> set[type[ComputeFramework]]:
|
|
18
|
+
return {PyArrowTable}
|
|
19
|
+
|
|
20
|
+
@classmethod
|
|
21
|
+
def _anonymize(cls, data: Any, mappings: list[tuple[str, str]], key_file: str) -> Any:
|
|
22
|
+
columns = []
|
|
23
|
+
for source, output in mappings:
|
|
24
|
+
if source not in data.column_names:
|
|
25
|
+
raise AnonymizerRsError(f"Source column {source!r} not found; available: {sorted(data.column_names)}.")
|
|
26
|
+
if output in data.column_names:
|
|
27
|
+
raise AnonymizerRsError(f"Output column {output!r} already exists.")
|
|
28
|
+
column = data.column(source)
|
|
29
|
+
if not (pa.types.is_string(column.type) or pa.types.is_large_string(column.type)):
|
|
30
|
+
raise AnonymizerRsError(f"Source column {source!r} must be a string column, got {column.type}.")
|
|
31
|
+
columns.append(column.cast(pa.string()))
|
|
32
|
+
argv = find_binary(cls.BINARY_COMMAND_OVERRIDE)
|
|
33
|
+
tokens = anonymize_columns(argv, columns, key_file, cls.BINARY_TIMEOUT_SECONDS)
|
|
34
|
+
for (_, output), token_column in zip(mappings, tokens, strict=True):
|
|
35
|
+
data = data.append_column(output, token_column)
|
|
36
|
+
return data
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Entry-point manifest: concrete FeatureGroup classes this package exports to mloda discovery."""
|
|
2
|
+
|
|
3
|
+
from mloda.provider import FeatureGroup
|
|
4
|
+
|
|
5
|
+
from mloda_pii.feature_groups.anonymizer_rs import PyArrowLocalAnonymizerFeatureGroup
|
|
6
|
+
|
|
7
|
+
# Referenced by the "mloda.feature_groups" entry point in pyproject.toml. Concrete classes only.
|
|
8
|
+
FEATURE_GROUPS: list[type[FeatureGroup]] = [PyArrowLocalAnonymizerFeatureGroup]
|
|
File without changes
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""The ``mloda-pii`` command. Subcommands are imported on use, so ``hook`` loads nothing it does not need."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
ONBOARDING = "mloda_pii.user_interface.hook.onboarding"
|
|
11
|
+
COMMANDS = { # name: (module, entry function, help)
|
|
12
|
+
"init": (ONBOARDING, "init_main", "Set up the hook: key, license, self-test, Claude Code settings."),
|
|
13
|
+
"status": (ONBOARDING, "status_main", "Show the install, key, license, registered hooks and recent events."),
|
|
14
|
+
"uninstall": (ONBOARDING, "uninstall_main", "Remove the hook from Claude Code settings."),
|
|
15
|
+
"hook": (
|
|
16
|
+
"mloda_pii.user_interface.hook.claude_hook",
|
|
17
|
+
"main",
|
|
18
|
+
"Command hook: event JSON on stdin, decision on stdout.",
|
|
19
|
+
),
|
|
20
|
+
"serve": ("mloda_pii.user_interface.hook.server", "main", "Warm http-hook server."),
|
|
21
|
+
"license": ("mloda_pii.user_interface.invitation_code.license", "main", "Install, check or remove the license."),
|
|
22
|
+
"stats": ("mloda_pii.user_interface.stats.stats", "main", "Per-day counts from the audit log."),
|
|
23
|
+
}
|
|
24
|
+
BLOCKING_EXIT = 2 # also the usage-error exit, as in argparse
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def usage() -> str:
|
|
28
|
+
rows = [f" {name:<10} {text}" for name, (_, _, text) in COMMANDS.items()]
|
|
29
|
+
return "\n".join([f"usage: mloda-pii {{{','.join(COMMANDS)}}} ...", "", *rows]) + "\n"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
33
|
+
args = list(sys.argv[1:] if argv is None else argv)
|
|
34
|
+
if args[:1] in (["-h"], ["--help"]):
|
|
35
|
+
sys.stdout.write(usage())
|
|
36
|
+
return 0
|
|
37
|
+
name, rest = (args[0], args[1:]) if args else ("", [])
|
|
38
|
+
if name not in COMMANDS or (name == "hook" and rest): # hook takes none: it would wait on stdin
|
|
39
|
+
sys.stderr.write(usage())
|
|
40
|
+
return BLOCKING_EXIT
|
|
41
|
+
try:
|
|
42
|
+
module = importlib.import_module(COMMANDS[name][0])
|
|
43
|
+
except Exception as exc: # a broken install must not let hook traffic through
|
|
44
|
+
if name != "hook":
|
|
45
|
+
raise
|
|
46
|
+
print(f"mloda-pii hook failed to load: {type(exc).__name__}", file=sys.stderr)
|
|
47
|
+
return BLOCKING_EXIT if os.environ.get("MLODA_PII_FAIL_CLOSED") != "0" else 1
|
|
48
|
+
entry = getattr(module, COMMANDS[name][1])
|
|
49
|
+
return int(entry() if name == "hook" else entry(rest))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__":
|
|
53
|
+
sys.exit(main())
|