ztds 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ztds-1.1.0/.gitignore +7 -0
- ztds-1.1.0/LICENSE +21 -0
- ztds-1.1.0/PKG-INFO +213 -0
- ztds-1.1.0/README.md +171 -0
- ztds-1.1.0/pyproject.toml +80 -0
- ztds-1.1.0/src/ztds/__init__.py +38 -0
- ztds-1.1.0/src/ztds/__main__.py +9 -0
- ztds-1.1.0/src/ztds/cli.py +147 -0
- ztds-1.1.0/src/ztds/conformance.py +120 -0
- ztds-1.1.0/src/ztds/core.py +239 -0
- ztds-1.1.0/src/ztds/integrations/__init__.py +16 -0
- ztds-1.1.0/src/ztds/integrations/crewai.py +68 -0
- ztds-1.1.0/src/ztds/integrations/langchain.py +82 -0
- ztds-1.1.0/src/ztds/integrations/litellm.py +114 -0
- ztds-1.1.0/src/ztds/integrations/llamaindex.py +72 -0
- ztds-1.1.0/tests/test_conformance.py +19 -0
- ztds-1.1.0/tests/test_core.py +76 -0
- ztds-1.1.0/tests/test_integrations.py +111 -0
ztds-1.1.0/.gitignore
ADDED
ztds-1.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ilya Sibiryakov & ZTDS AI Consortium (https://ztds.ai)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ztds-1.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ztds
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Zero-Trust Data Sanitization (ZTDS) - Canonical Python Reference Implementation (RFC v1.0, draft-sibiryakov-ztds-protocol)
|
|
5
|
+
Project-URL: Homepage, https://ztds.ai
|
|
6
|
+
Project-URL: Documentation, https://ztds.ai/standard/
|
|
7
|
+
Project-URL: Specification, https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/
|
|
8
|
+
Project-URL: Repository, https://github.com/moxno/ztds.ai
|
|
9
|
+
Project-URL: Issues, https://github.com/moxno/ztds.ai/issues
|
|
10
|
+
Author-email: Ilya Sibiryakov <ilya@ztds.ai>, ZTDS AI Consortium <standard@ztds.ai>
|
|
11
|
+
License: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: crewai,ietf,langchain,litellm,llamaindex,pii,privacy,redaction,rfc,sanitizer,security,zero-trust,ztds
|
|
14
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Information Technology
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
26
|
+
Classifier: Topic :: Security
|
|
27
|
+
Requires-Python: >=3.8
|
|
28
|
+
Provides-Extra: all
|
|
29
|
+
Requires-Dist: crewai>=0.28.0; extra == 'all'
|
|
30
|
+
Requires-Dist: langchain-core>=0.1.0; extra == 'all'
|
|
31
|
+
Requires-Dist: litellm>=1.0.0; extra == 'all'
|
|
32
|
+
Requires-Dist: llama-index-core>=0.10.0; extra == 'all'
|
|
33
|
+
Provides-Extra: crewai
|
|
34
|
+
Requires-Dist: crewai>=0.28.0; extra == 'crewai'
|
|
35
|
+
Provides-Extra: langchain
|
|
36
|
+
Requires-Dist: langchain-core>=0.1.0; extra == 'langchain'
|
|
37
|
+
Provides-Extra: litellm
|
|
38
|
+
Requires-Dist: litellm>=1.0.0; extra == 'litellm'
|
|
39
|
+
Provides-Extra: llamaindex
|
|
40
|
+
Requires-Dist: llama-index-core>=0.10.0; extra == 'llamaindex'
|
|
41
|
+
Description-Content-Type: text/markdown
|
|
42
|
+
|
|
43
|
+
# ZTDS (Zero-Trust Data Sanitization)
|
|
44
|
+
|
|
45
|
+
[](https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/)
|
|
46
|
+
[](https://pypi.org/project/ztds/)
|
|
47
|
+
[](https://opensource.org/licenses/MIT)
|
|
48
|
+
[](https://ztds.ai/registry/)
|
|
49
|
+
|
|
50
|
+
Canonical Python Reference Implementation of the **Zero-Trust Data Sanitization (ZTDS) Protocol** ([IETF `draft-sibiryakov-ztds-protocol-02`](https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/), RFC v1.0).
|
|
51
|
+
|
|
52
|
+
ZTDS physically prevents PII, PHI, API tokens, and corporate secrets from crossing WAN sockets unmasked. It operates **100% in local process RAM** with zero external network egress, sub-millisecond execution (<0.05ms), and verifiable memory zeroization.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## 🏛️ Invariants Enforced (RFC v1.0)
|
|
57
|
+
|
|
58
|
+
1. **Invariant 1: Zero External Egress Prior to Sanitization**
|
|
59
|
+
Every byte of prompt data is inspected and sanitized locally in volatile memory. Zero unmasked data touches network interfaces.
|
|
60
|
+
2. **Invariant 2: Deterministic Reversible Bijective Tokenization**
|
|
61
|
+
Syntactic bracketed surrogates (`[NAME_1]`, `[EMAIL_1]`, `[API_SECRET_1]`) preserve entity coreference and LLM reasoning semantics while enabling exact bijective restoration ($f^{-1}(f(x)) = x$).
|
|
62
|
+
3. **Invariant 3: Ephemeral In-RAM Session Isolation & Zeroization (Theorem 2)**
|
|
63
|
+
Cryptographic mapping tables reside strictly in volatile RAM, isolated per session, and are destructively overwritten upon session completion.
|
|
64
|
+
4. **Invariant 4: Zero Subprocessors & Zero Telemetry**
|
|
65
|
+
No SaaS calls, no cloud APIs, and no telemetry side-channels. Eliminates GDPR Article 28 data processing agreements and vendor risk reviews.
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## ⚡ Quickstart
|
|
70
|
+
|
|
71
|
+
Install the zero-dependency Python package:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install ztds
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### 1. In-Memory Sanitization & Restoration
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
import ztds
|
|
81
|
+
|
|
82
|
+
prompt = "Deploy microservice with key sk-ant-api03-0123456789abcdef0123456789abcdef and email dev@corp.internal"
|
|
83
|
+
|
|
84
|
+
# Use context manager for automatic Invariant 3 RAM zeroization
|
|
85
|
+
with ztds.session() as s:
|
|
86
|
+
# Phase 1: Local In-Memory Sanitization (<0.05ms)
|
|
87
|
+
sanitized, minted_tokens = s.sanitize(prompt)
|
|
88
|
+
print(sanitized)
|
|
89
|
+
# Output: "Deploy microservice with key [API_SECRET_1] and email [EMAIL_1]"
|
|
90
|
+
|
|
91
|
+
# Phase 2: Send sanitized string safely to any frontier model
|
|
92
|
+
# ai_raw_response = client.chat.completions.create(messages=[{"role": "user", "content": sanitized}])
|
|
93
|
+
ai_raw_response = "Confirmed. Credentials for [EMAIL_1] are stored."
|
|
94
|
+
|
|
95
|
+
# Phase 3: Bijective Restoration in Local RAM
|
|
96
|
+
clean_response = s.reveal(ai_raw_response)
|
|
97
|
+
print(clean_response)
|
|
98
|
+
# Output: "Confirmed. Credentials for dev@corp.internal are stored."
|
|
99
|
+
|
|
100
|
+
# Memory mapping is completely wiped upon context exit (Theorem 2)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## 🔌 Framework Integrations
|
|
106
|
+
|
|
107
|
+
### LiteLLM Proxy / SDK
|
|
108
|
+
|
|
109
|
+
Install with LiteLLM extras:
|
|
110
|
+
```bash
|
|
111
|
+
pip install "ztds[litellm]"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
import litellm
|
|
116
|
+
from ztds.integrations import ZTDSGuardrail
|
|
117
|
+
|
|
118
|
+
# Intercept all calls locally before WAN socket dispatch
|
|
119
|
+
litellm.input_callbacks = [ZTDSGuardrail()]
|
|
120
|
+
litellm.output_callbacks = [ZTDSGuardrail()]
|
|
121
|
+
|
|
122
|
+
response = litellm.completion(
|
|
123
|
+
model="gpt-4o",
|
|
124
|
+
messages=[{"role": "user", "content": "My AWS key is AKIAIOSFODNN7EXAMPLE"}]
|
|
125
|
+
)
|
|
126
|
+
# Upstream OpenAI receives only "[API_SECRET_1]"
|
|
127
|
+
# Your application receives restored output automatically
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### LangChain
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from langchain_openai import ChatOpenAI
|
|
134
|
+
from ztds.integrations import ZTDSSanitizingCallbackHandler
|
|
135
|
+
|
|
136
|
+
llm = ChatOpenAI(
|
|
137
|
+
callbacks=[ZTDSSanitizingCallbackHandler()]
|
|
138
|
+
)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### CrewAI Multi-Agent Workflows
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
from crewai import Agent
|
|
145
|
+
from ztds.integrations import ZTDSSanitizerTool
|
|
146
|
+
|
|
147
|
+
security_agent = Agent(
|
|
148
|
+
role="Data Officer",
|
|
149
|
+
goal="Sanitize customer logs before delegation",
|
|
150
|
+
tools=[ZTDSSanitizerTool()]
|
|
151
|
+
)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### LlamaIndex RAG Node Postprocessor
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from llama_index.core import VectorStoreIndex
|
|
158
|
+
from ztds.integrations import ZTDSNodePostprocessor
|
|
159
|
+
|
|
160
|
+
query_engine = index.as_query_engine(
|
|
161
|
+
node_postprocessors=[ZTDSNodePostprocessor()]
|
|
162
|
+
)
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
## 🛠️ CLI Tools
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
# Verify compliance with RFC v1.0 canonical test vectors
|
|
171
|
+
ztds conformance
|
|
172
|
+
|
|
173
|
+
# Output JSON for CI/CD pipelines
|
|
174
|
+
ztds conformance --json
|
|
175
|
+
|
|
176
|
+
# Audit local codebase for unmasked secrets and telemetry side-channels
|
|
177
|
+
ztds audit --dir ./src --strict
|
|
178
|
+
|
|
179
|
+
# Quick in-memory sanitization from command line
|
|
180
|
+
ztds sanitize "Contact alice@example.com with key sk-proj-12345678901234567890"
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## 📊 Conformance & Performance
|
|
186
|
+
|
|
187
|
+
All 8 canonical RFC v1.0 test vectors pass with 100% Invariant fidelity:
|
|
188
|
+
|
|
189
|
+
| Test Vector ID | Category | Status | Latency |
|
|
190
|
+
| :--- | :--- | :--- | :--- |
|
|
191
|
+
| `VEC-CORE-001` | Universal Consumer PII | PASS | 0.04ms |
|
|
192
|
+
| `VEC-CORE-002` | Coreference Determinism | PASS | 0.02ms |
|
|
193
|
+
| `VEC-HEALTH-001` | Healthcare PHI & HIPAA Safe Harbor | PASS | 0.04ms |
|
|
194
|
+
| `VEC-FIN-001` | Financial & Banking (ISO 20022) | PASS | 0.03ms |
|
|
195
|
+
| `VEC-SEC-001` | Developer Secrets & Cloud Tokens | PASS | 0.03ms |
|
|
196
|
+
| `VEC-MULTI-001` | Multi-Lingual Global Entities | PASS | 0.04ms |
|
|
197
|
+
| `VEC-REDOS-001` | ReDoS Stress & Catastrophic Backtracking | PASS | 0.03ms |
|
|
198
|
+
| `VEC-UNICODE-001` | Unicode Normalization & Homoglyphs | PASS | 0.03ms |
|
|
199
|
+
|
|
200
|
+
Average latency per prompt: **~0.031 ms** (31 microseconds).
|
|
201
|
+
|
|
202
|
+
---
|
|
203
|
+
|
|
204
|
+
## 📜 Intellectual Property & Standards
|
|
205
|
+
|
|
206
|
+
- **IETF Specification**: [draft-sibiryakov-ztds-protocol-02](https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/)
|
|
207
|
+
- **Consortium & RFC Registry**: [https://ztds.ai](https://ztds.ai)
|
|
208
|
+
- **IPR Disclosure**: Filed under IETF RFC 8179 rules (Royalty-Free grant for Open-Source implementations with Defensive Termination).
|
|
209
|
+
- **Commercial Certified Reference Implementation**: [PrivacyScrubber](https://privacyscrubber.com) (30 specialized industry profiles, HIPAA/FRE 502/SOC 2 compliance packs, enterprise air-gapped on-premise engines).
|
|
210
|
+
|
|
211
|
+
## 📄 License
|
|
212
|
+
|
|
213
|
+
MIT License. Copyright (c) 2026 Ilya Sibiryakov & ZTDS AI Consortium.
|
ztds-1.1.0/README.md
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# ZTDS (Zero-Trust Data Sanitization)
|
|
2
|
+
|
|
3
|
+
[](https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/)
|
|
4
|
+
[](https://pypi.org/project/ztds/)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
[](https://ztds.ai/registry/)
|
|
7
|
+
|
|
8
|
+
Canonical Python Reference Implementation of the **Zero-Trust Data Sanitization (ZTDS) Protocol** ([IETF `draft-sibiryakov-ztds-protocol-02`](https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/), RFC v1.0).
|
|
9
|
+
|
|
10
|
+
ZTDS physically prevents PII, PHI, API tokens, and corporate secrets from crossing WAN sockets unmasked. It operates **100% in local process RAM** with zero external network egress, sub-millisecond execution (<0.05ms), and verifiable memory zeroization.
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## 🏛️ Invariants Enforced (RFC v1.0)
|
|
15
|
+
|
|
16
|
+
1. **Invariant 1: Zero External Egress Prior to Sanitization**
|
|
17
|
+
Every byte of prompt data is inspected and sanitized locally in volatile memory. Zero unmasked data touches network interfaces.
|
|
18
|
+
2. **Invariant 2: Deterministic Reversible Bijective Tokenization**
|
|
19
|
+
Syntactic bracketed surrogates (`[NAME_1]`, `[EMAIL_1]`, `[API_SECRET_1]`) preserve entity coreference and LLM reasoning semantics while enabling exact bijective restoration ($f^{-1}(f(x)) = x$).
|
|
20
|
+
3. **Invariant 3: Ephemeral In-RAM Session Isolation & Zeroization (Theorem 2)**
|
|
21
|
+
Cryptographic mapping tables reside strictly in volatile RAM, isolated per session, and are destructively overwritten upon session completion.
|
|
22
|
+
4. **Invariant 4: Zero Subprocessors & Zero Telemetry**
|
|
23
|
+
No SaaS calls, no cloud APIs, and no telemetry side-channels. Eliminates GDPR Article 28 data processing agreements and vendor risk reviews.
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## ⚡ Quickstart
|
|
28
|
+
|
|
29
|
+
Install the zero-dependency Python package:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install ztds
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### 1. In-Memory Sanitization & Restoration
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import ztds
|
|
39
|
+
|
|
40
|
+
prompt = "Deploy microservice with key sk-ant-api03-0123456789abcdef0123456789abcdef and email dev@corp.internal"
|
|
41
|
+
|
|
42
|
+
# Use context manager for automatic Invariant 3 RAM zeroization
|
|
43
|
+
with ztds.session() as s:
|
|
44
|
+
# Phase 1: Local In-Memory Sanitization (<0.05ms)
|
|
45
|
+
sanitized, minted_tokens = s.sanitize(prompt)
|
|
46
|
+
print(sanitized)
|
|
47
|
+
# Output: "Deploy microservice with key [API_SECRET_1] and email [EMAIL_1]"
|
|
48
|
+
|
|
49
|
+
# Phase 2: Send sanitized string safely to any frontier model
|
|
50
|
+
# ai_raw_response = client.chat.completions.create(messages=[{"role": "user", "content": sanitized}])
|
|
51
|
+
ai_raw_response = "Confirmed. Credentials for [EMAIL_1] are stored."
|
|
52
|
+
|
|
53
|
+
# Phase 3: Bijective Restoration in Local RAM
|
|
54
|
+
clean_response = s.reveal(ai_raw_response)
|
|
55
|
+
print(clean_response)
|
|
56
|
+
# Output: "Confirmed. Credentials for dev@corp.internal are stored."
|
|
57
|
+
|
|
58
|
+
# Memory mapping is completely wiped upon context exit (Theorem 2)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## 🔌 Framework Integrations
|
|
64
|
+
|
|
65
|
+
### LiteLLM Proxy / SDK
|
|
66
|
+
|
|
67
|
+
Install with LiteLLM extras:
|
|
68
|
+
```bash
|
|
69
|
+
pip install "ztds[litellm]"
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
import litellm
|
|
74
|
+
from ztds.integrations import ZTDSGuardrail
|
|
75
|
+
|
|
76
|
+
# Intercept all calls locally before WAN socket dispatch
|
|
77
|
+
litellm.input_callbacks = [ZTDSGuardrail()]
|
|
78
|
+
litellm.output_callbacks = [ZTDSGuardrail()]
|
|
79
|
+
|
|
80
|
+
response = litellm.completion(
|
|
81
|
+
model="gpt-4o",
|
|
82
|
+
messages=[{"role": "user", "content": "My AWS key is AKIAIOSFODNN7EXAMPLE"}]
|
|
83
|
+
)
|
|
84
|
+
# Upstream OpenAI receives only "[API_SECRET_1]"
|
|
85
|
+
# Your application receives restored output automatically
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### LangChain
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from langchain_openai import ChatOpenAI
|
|
92
|
+
from ztds.integrations import ZTDSSanitizingCallbackHandler
|
|
93
|
+
|
|
94
|
+
llm = ChatOpenAI(
|
|
95
|
+
callbacks=[ZTDSSanitizingCallbackHandler()]
|
|
96
|
+
)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
### CrewAI Multi-Agent Workflows
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from crewai import Agent
|
|
103
|
+
from ztds.integrations import ZTDSSanitizerTool
|
|
104
|
+
|
|
105
|
+
security_agent = Agent(
|
|
106
|
+
role="Data Officer",
|
|
107
|
+
goal="Sanitize customer logs before delegation",
|
|
108
|
+
tools=[ZTDSSanitizerTool()]
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### LlamaIndex RAG Node Postprocessor
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
from llama_index.core import VectorStoreIndex
|
|
116
|
+
from ztds.integrations import ZTDSNodePostprocessor
|
|
117
|
+
|
|
118
|
+
query_engine = index.as_query_engine(
|
|
119
|
+
node_postprocessors=[ZTDSNodePostprocessor()]
|
|
120
|
+
)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## 🛠️ CLI Tools
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
# Verify compliance with RFC v1.0 canonical test vectors
|
|
129
|
+
ztds conformance
|
|
130
|
+
|
|
131
|
+
# Output JSON for CI/CD pipelines
|
|
132
|
+
ztds conformance --json
|
|
133
|
+
|
|
134
|
+
# Audit local codebase for unmasked secrets and telemetry side-channels
|
|
135
|
+
ztds audit --dir ./src --strict
|
|
136
|
+
|
|
137
|
+
# Quick in-memory sanitization from command line
|
|
138
|
+
ztds sanitize "Contact alice@example.com with key sk-proj-12345678901234567890"
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
---
|
|
142
|
+
|
|
143
|
+
## 📊 Conformance & Performance
|
|
144
|
+
|
|
145
|
+
All 8 canonical RFC v1.0 test vectors pass with 100% Invariant fidelity:
|
|
146
|
+
|
|
147
|
+
| Test Vector ID | Category | Status | Latency |
|
|
148
|
+
| :--- | :--- | :--- | :--- |
|
|
149
|
+
| `VEC-CORE-001` | Universal Consumer PII | PASS | 0.04ms |
|
|
150
|
+
| `VEC-CORE-002` | Coreference Determinism | PASS | 0.02ms |
|
|
151
|
+
| `VEC-HEALTH-001` | Healthcare PHI & HIPAA Safe Harbor | PASS | 0.04ms |
|
|
152
|
+
| `VEC-FIN-001` | Financial & Banking (ISO 20022) | PASS | 0.03ms |
|
|
153
|
+
| `VEC-SEC-001` | Developer Secrets & Cloud Tokens | PASS | 0.03ms |
|
|
154
|
+
| `VEC-MULTI-001` | Multi-Lingual Global Entities | PASS | 0.04ms |
|
|
155
|
+
| `VEC-REDOS-001` | ReDoS Stress & Catastrophic Backtracking | PASS | 0.03ms |
|
|
156
|
+
| `VEC-UNICODE-001` | Unicode Normalization & Homoglyphs | PASS | 0.03ms |
|
|
157
|
+
|
|
158
|
+
Average latency per prompt: **~0.031 ms** (31 microseconds).
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
## 📜 Intellectual Property & Standards
|
|
163
|
+
|
|
164
|
+
- **IETF Specification**: [draft-sibiryakov-ztds-protocol-02](https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/)
|
|
165
|
+
- **Consortium & RFC Registry**: [https://ztds.ai](https://ztds.ai)
|
|
166
|
+
- **IPR Disclosure**: Filed under IETF RFC 8179 rules (Royalty-Free grant for Open-Source implementations with Defensive Termination).
|
|
167
|
+
- **Commercial Certified Reference Implementation**: [PrivacyScrubber](https://privacyscrubber.com) (30 specialized industry profiles, HIPAA/FRE 502/SOC 2 compliance packs, enterprise air-gapped on-premise engines).
|
|
168
|
+
|
|
169
|
+
## 📄 License
|
|
170
|
+
|
|
171
|
+
MIT License. Copyright (c) 2026 Ilya Sibiryakov & ZTDS AI Consortium.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ztds"
|
|
7
|
+
version = "1.1.0"
|
|
8
|
+
description = "Zero-Trust Data Sanitization (ZTDS) - Canonical Python Reference Implementation (RFC v1.0, draft-sibiryakov-ztds-protocol)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{ name = "Ilya Sibiryakov", email = "ilya@ztds.ai" },
|
|
12
|
+
{ name = "ZTDS AI Consortium", email = "standard@ztds.ai" }
|
|
13
|
+
]
|
|
14
|
+
license = { text = "MIT" }
|
|
15
|
+
requires-python = ">=3.8"
|
|
16
|
+
dependencies = []
|
|
17
|
+
keywords = [
|
|
18
|
+
"ztds",
|
|
19
|
+
"pii",
|
|
20
|
+
"sanitizer",
|
|
21
|
+
"zero-trust",
|
|
22
|
+
"rfc",
|
|
23
|
+
"ietf",
|
|
24
|
+
"privacy",
|
|
25
|
+
"redaction",
|
|
26
|
+
"security",
|
|
27
|
+
"litellm",
|
|
28
|
+
"langchain",
|
|
29
|
+
"crewai",
|
|
30
|
+
"llamaindex"
|
|
31
|
+
]
|
|
32
|
+
classifiers = [
|
|
33
|
+
"Development Status :: 5 - Production/Stable",
|
|
34
|
+
"Intended Audience :: Developers",
|
|
35
|
+
"Intended Audience :: Information Technology",
|
|
36
|
+
"License :: OSI Approved :: MIT License",
|
|
37
|
+
"Programming Language :: Python :: 3",
|
|
38
|
+
"Programming Language :: Python :: 3.8",
|
|
39
|
+
"Programming Language :: Python :: 3.9",
|
|
40
|
+
"Programming Language :: Python :: 3.10",
|
|
41
|
+
"Programming Language :: Python :: 3.11",
|
|
42
|
+
"Programming Language :: Python :: 3.12",
|
|
43
|
+
"Programming Language :: Python :: 3.13",
|
|
44
|
+
"Topic :: Security",
|
|
45
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence"
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
[project.urls]
|
|
49
|
+
Homepage = "https://ztds.ai"
|
|
50
|
+
Documentation = "https://ztds.ai/standard/"
|
|
51
|
+
Specification = "https://datatracker.ietf.org/doc/draft-sibiryakov-ztds-protocol/"
|
|
52
|
+
Repository = "https://github.com/moxno/ztds.ai"
|
|
53
|
+
Issues = "https://github.com/moxno/ztds.ai/issues"
|
|
54
|
+
|
|
55
|
+
[project.scripts]
|
|
56
|
+
ztds = "ztds.cli:main"
|
|
57
|
+
|
|
58
|
+
[project.optional-dependencies]
|
|
59
|
+
litellm = ["litellm>=1.0.0"]
|
|
60
|
+
langchain = ["langchain-core>=0.1.0"]
|
|
61
|
+
crewai = ["crewai>=0.28.0"]
|
|
62
|
+
llamaindex = ["llama-index-core>=0.10.0"]
|
|
63
|
+
all = [
|
|
64
|
+
"litellm>=1.0.0",
|
|
65
|
+
"langchain-core>=0.1.0",
|
|
66
|
+
"crewai>=0.28.0",
|
|
67
|
+
"llama-index-core>=0.10.0"
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
[tool.hatch.build.targets.wheel]
|
|
71
|
+
packages = ["src/ztds"]
|
|
72
|
+
|
|
73
|
+
[tool.hatch.build.targets.sdist]
|
|
74
|
+
include = [
|
|
75
|
+
"/src",
|
|
76
|
+
"/tests",
|
|
77
|
+
"/README.md",
|
|
78
|
+
"/LICENSE",
|
|
79
|
+
"/pyproject.toml"
|
|
80
|
+
]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ZTDS (Zero-Trust Data Sanitization) - Canonical Python Reference Implementation
|
|
3
|
+
Specification: IETF draft-sibiryakov-ztds-protocol-02 / RFC v1.0
|
|
4
|
+
Standards Authority: ZTDS AI Consortium (https://ztds.ai)
|
|
5
|
+
|
|
6
|
+
Invariants Formally Enforced:
|
|
7
|
+
- Invariant 1: Zero External Egress Prior to Sanitization (100% in-memory local RAM).
|
|
8
|
+
- Invariant 2: Deterministic Reversible Bijective Tokenization.
|
|
9
|
+
- Invariant 3: Ephemeral RAM Isolation & Theorem 2 Zeroization.
|
|
10
|
+
- Invariant 4: Zero Subprocessors and Zero Third-Party Telemetry.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .conformance import run_conformance_suite
|
|
14
|
+
from .core import (
|
|
15
|
+
NativeZtdsEngine,
|
|
16
|
+
SessionContext,
|
|
17
|
+
reveal,
|
|
18
|
+
restore,
|
|
19
|
+
sanitize,
|
|
20
|
+
session,
|
|
21
|
+
zeroize,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__version__ = "1.1.0"
|
|
25
|
+
__author__ = "Ilya Sibiryakov & ZTDS AI Consortium"
|
|
26
|
+
__license__ = "MIT"
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"NativeZtdsEngine",
|
|
30
|
+
"SessionContext",
|
|
31
|
+
"sanitize",
|
|
32
|
+
"reveal",
|
|
33
|
+
"restore",
|
|
34
|
+
"zeroize",
|
|
35
|
+
"session",
|
|
36
|
+
"run_conformance_suite",
|
|
37
|
+
"__version__",
|
|
38
|
+
]
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ZTDS (Zero-Trust Data Sanitization) - Python CLI
|
|
3
|
+
Command-line interface for Zero-Trust sanitization, invariant auditing, and conformance verification.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from typing import List, Optional
|
|
13
|
+
|
|
14
|
+
from . import __version__
|
|
15
|
+
from .conformance import run_conformance_suite
|
|
16
|
+
from .core import NativeZtdsEngine
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def audit_directory(target_dir: str, strict: bool = False, json_output: bool = False) -> int:
|
|
20
|
+
"""
|
|
21
|
+
Perform local in-memory audit of codebase files for sensitive patterns and WAN egress telemetry.
|
|
22
|
+
Equivalent to npx ztds-audit.
|
|
23
|
+
"""
|
|
24
|
+
from .core import BASELINE_PATTERNS
|
|
25
|
+
|
|
26
|
+
ignored_dirs = {
|
|
27
|
+
"node_modules", ".git", ".next", "dist", "build", ".vercel",
|
|
28
|
+
"coverage", ".cache", "public/badge", "vendor", "fixtures",
|
|
29
|
+
"benchmarks", "conformance", "__pycache__", ".pytest_cache", ".ruff_cache"
|
|
30
|
+
}
|
|
31
|
+
allowed_exts = {
|
|
32
|
+
".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".json",
|
|
33
|
+
".py", ".go", ".rs", ".java", ".yaml", ".yml", ".env"
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
files_to_scan = []
|
|
37
|
+
for root, dirs, files in os.walk(target_dir):
|
|
38
|
+
dirs[:] = [d for d in dirs if d not in ignored_dirs]
|
|
39
|
+
for f in files:
|
|
40
|
+
ext = os.path.splitext(f)[1].lower()
|
|
41
|
+
if ext in allowed_exts or f.startswith(".env"):
|
|
42
|
+
files_to_scan.append(os.path.join(root, f))
|
|
43
|
+
|
|
44
|
+
findings = []
|
|
45
|
+
for filepath in files_to_scan:
|
|
46
|
+
try:
|
|
47
|
+
with open(filepath, "r", encoding="utf-8", errors="ignore") as fh:
|
|
48
|
+
for line_idx, line in enumerate(fh, 1):
|
|
49
|
+
for entity_type, pat in BASELINE_PATTERNS.items():
|
|
50
|
+
if entity_type in ("API_SECRET", "CREDIT_CARD", "SSN"):
|
|
51
|
+
matches = pat.findall(line)
|
|
52
|
+
for m in matches:
|
|
53
|
+
findings.append({
|
|
54
|
+
"file": os.path.relpath(filepath, target_dir),
|
|
55
|
+
"line": line_idx,
|
|
56
|
+
"type": entity_type,
|
|
57
|
+
"sample": m[:12] + "..." if len(m) > 12 else m
|
|
58
|
+
})
|
|
59
|
+
except Exception:
|
|
60
|
+
continue
|
|
61
|
+
|
|
62
|
+
passed = len(findings) == 0
|
|
63
|
+
|
|
64
|
+
if json_output:
|
|
65
|
+
print(json.dumps({
|
|
66
|
+
"audit_passed": passed,
|
|
67
|
+
"target_dir": target_dir,
|
|
68
|
+
"files_scanned": len(files_to_scan),
|
|
69
|
+
"findings_count": len(findings),
|
|
70
|
+
"findings": findings
|
|
71
|
+
}, indent=2))
|
|
72
|
+
return 0 if passed else 1
|
|
73
|
+
|
|
74
|
+
print("\n\033[1m\033[36m[ZTDS™]\033[0m \033[1mPython In-Memory Codebase Auditor\033[0m")
|
|
75
|
+
print("-" * 74)
|
|
76
|
+
print(f"Directory: {target_dir}")
|
|
77
|
+
print(f"Files Scanned: {len(files_to_scan)}")
|
|
78
|
+
print("-" * 74)
|
|
79
|
+
|
|
80
|
+
if passed:
|
|
81
|
+
print("\033[1m\033[32m[PASS] CONFORMANCE CONFIRMED: 0 INVARIANT VIOLATIONS DETECTED\033[0m")
|
|
82
|
+
print("All scanned files comply with ZTDS Invariant 1 (Zero-Egress) and Invariant 3 (RAM Isolation).\n")
|
|
83
|
+
return 0
|
|
84
|
+
else:
|
|
85
|
+
print(f"\033[1m\033[31m[FAIL] AUDIT FAILED: {len(findings)} VIOLATION(S) DETECTED\033[0m")
|
|
86
|
+
for f in findings[:15]:
|
|
87
|
+
print(f" \033[31m[{f['type']}]\033[0m {f['file']}:{f['line']} -> {f['sample']}")
|
|
88
|
+
if len(findings) > 15:
|
|
89
|
+
print(f" ... and {len(findings) - 15} more findings.")
|
|
90
|
+
print("")
|
|
91
|
+
return 1
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def main(args: Optional[List[str]] = None) -> int:
|
|
95
|
+
parser = argparse.ArgumentParser(
|
|
96
|
+
prog="ztds",
|
|
97
|
+
description="Zero-Trust Data Sanitization (ZTDS) - Official Python CLI"
|
|
98
|
+
)
|
|
99
|
+
parser.add_argument("--version", "-v", action="version", version=f"ztds {__version__}")
|
|
100
|
+
|
|
101
|
+
subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
|
|
102
|
+
|
|
103
|
+
# Command: sanitize
|
|
104
|
+
sanitize_parser = subparsers.add_parser("sanitize", help="Sanitize a text prompt in local RAM")
|
|
105
|
+
sanitize_parser.add_argument("text", type=str, help="Text to sanitize")
|
|
106
|
+
sanitize_parser.add_argument("--session-id", type=str, default=None, help="Optional session ID")
|
|
107
|
+
|
|
108
|
+
# Command: conformance
|
|
109
|
+
conformance_parser = subparsers.add_parser("conformance", help="Run canonical RFC test vectors")
|
|
110
|
+
conformance_parser.add_argument("--json", action="store_true", help="Output machine-readable JSON")
|
|
111
|
+
|
|
112
|
+
# Command: audit
|
|
113
|
+
audit_parser = subparsers.add_parser("audit", help="Audit local codebase for WAN egress and raw secrets")
|
|
114
|
+
audit_parser.add_argument("--dir", "-d", type=str, default=".", help="Directory to audit")
|
|
115
|
+
audit_parser.add_argument("--strict", action="store_true", help="Fail on any findings")
|
|
116
|
+
audit_parser.add_argument("--json", action="store_true", help="Output machine-readable JSON")
|
|
117
|
+
|
|
118
|
+
parsed_args = parser.parse_args(args)
|
|
119
|
+
|
|
120
|
+
if not parsed_args.command:
|
|
121
|
+
parser.print_help()
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
if parsed_args.command == "sanitize":
|
|
125
|
+
engine = NativeZtdsEngine()
|
|
126
|
+
sanitized, minted = engine.sanitize(parsed_args.text, parsed_args.session_id)
|
|
127
|
+
print(sanitized)
|
|
128
|
+
return 0
|
|
129
|
+
|
|
130
|
+
elif parsed_args.command == "conformance":
|
|
131
|
+
res = run_conformance_suite(verbose=not parsed_args.json)
|
|
132
|
+
if parsed_args.json:
|
|
133
|
+
print(json.dumps(res, indent=2))
|
|
134
|
+
return 0 if res["conformance_passed"] else 1
|
|
135
|
+
|
|
136
|
+
elif parsed_args.command == "audit":
|
|
137
|
+
return audit_directory(
|
|
138
|
+
target_dir=os.path.abspath(parsed_args.dir),
|
|
139
|
+
strict=parsed_args.strict,
|
|
140
|
+
json_output=parsed_args.json
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
return 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
if __name__ == "__main__":
|
|
147
|
+
sys.exit(main())
|