datafog-core 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datafog_core-0.3.0 → datafog_core-0.3.1}/Cargo.lock +4 -4
- {datafog_core-0.3.0 → datafog_core-0.3.1}/PKG-INFO +1 -1
- {datafog_core-0.3.0 → datafog_core-0.3.1}/bindings/python/Cargo.toml +1 -1
- datafog_core-0.3.1/bindings/python/tests/requirements-typing.txt +3 -0
- datafog_core-0.3.1/bindings/python/tests/test_typing.py +260 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/crates/core/Cargo.toml +1 -1
- {datafog_core-0.3.0 → datafog_core-0.3.1}/pyproject.toml +3 -2
- datafog_core-0.3.1/python/datafog_core/__init__.py +57 -0
- datafog_core-0.3.1/python/datafog_core/__init__.pyi +381 -0
- datafog_core-0.3.1/python/datafog_core/py.typed +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/Cargo.toml +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/README.md +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/bindings/python/src/lib.rs +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/bindings/python/tests/test_installed.py +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/crates/core/README.md +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/crates/core/examples/scan_benchmark.rs +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/crates/core/src/lib.rs +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/crates/core/src/offsets.rs +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/crates/core/src/selection_tests.rs +0 -0
- {datafog_core-0.3.0 → datafog_core-0.3.1}/crates/core/src/structured.rs +0 -0
|
@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
|
|
|
80
80
|
|
|
81
81
|
[[package]]
|
|
82
82
|
name = "datafog-core"
|
|
83
|
-
version = "0.3.
|
|
83
|
+
version = "0.3.1"
|
|
84
84
|
dependencies = [
|
|
85
85
|
"base64",
|
|
86
86
|
"futures",
|
|
@@ -94,7 +94,7 @@ dependencies = [
|
|
|
94
94
|
|
|
95
95
|
[[package]]
|
|
96
96
|
name = "datafog-core-python"
|
|
97
|
-
version = "0.3.
|
|
97
|
+
version = "0.3.1"
|
|
98
98
|
dependencies = [
|
|
99
99
|
"datafog-core",
|
|
100
100
|
"pyo3",
|
|
@@ -104,7 +104,7 @@ dependencies = [
|
|
|
104
104
|
|
|
105
105
|
[[package]]
|
|
106
106
|
name = "datafog-node"
|
|
107
|
-
version = "0.3.
|
|
107
|
+
version = "0.3.1"
|
|
108
108
|
dependencies = [
|
|
109
109
|
"datafog-core",
|
|
110
110
|
"napi",
|
|
@@ -115,7 +115,7 @@ dependencies = [
|
|
|
115
115
|
|
|
116
116
|
[[package]]
|
|
117
117
|
name = "datafog-wasm"
|
|
118
|
-
version = "0.3.
|
|
118
|
+
version = "0.3.1"
|
|
119
119
|
dependencies = [
|
|
120
120
|
"datafog-core",
|
|
121
121
|
"serde",
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Check typing metadata and downstream code against the installed wheel."""
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
import importlib.metadata
|
|
5
|
+
import inspect
|
|
6
|
+
import json
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import datafog_core
|
|
13
|
+
|
|
14
|
+
VALID = """
|
|
15
|
+
from typing import TYPE_CHECKING
|
|
16
|
+
from typing_extensions import assert_type
|
|
17
|
+
from datafog_core import (
|
|
18
|
+
DataFogConfigurationError, DataFogFindingError, DataFogInternalError,
|
|
19
|
+
DataFogKeyProviderError, TextRange, Finding, Transformation, TransformResult,
|
|
20
|
+
Restoration, RestoreResult, FieldMapping, StructuredFinding,
|
|
21
|
+
StructuredScanResult, StructuredTransformation, StructuredTransformResult,
|
|
22
|
+
StructuredRestoration, StructuredRestoreResult, PrivacyManager,
|
|
23
|
+
scan, transform, scan_and_transform, discover_fields, scan_structured,
|
|
24
|
+
transform_structured, scan_and_transform_structured,
|
|
25
|
+
)
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from datafog_core import (
|
|
28
|
+
_TransformationConfig, _JsonDocument, _TokenizeItem, _TokenizeResult,
|
|
29
|
+
_RestoreItem, _RestoreResult, _ResolvedKey,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
config: _TransformationConfig = {
|
|
33
|
+
"default": {"strategy": "mask", "character": "*", "reveal": {"direction": "last", "count": 4}},
|
|
34
|
+
"entities": ["EMAIL"],
|
|
35
|
+
"overrides": {
|
|
36
|
+
"PHONE": {"strategy": "remove"},
|
|
37
|
+
"PERSON": {"strategy": "pseudonymize", "key_ref": "names", "key_version": "1"},
|
|
38
|
+
},
|
|
39
|
+
"allow": {"exact": {"EMAIL": ["safe@example.com"]}, "regex": {"EMAIL": [{"pattern": "safe", "case_sensitive": False}]}},
|
|
40
|
+
}
|
|
41
|
+
findings = scan("jane@example.com", {"locale": "en-US"})
|
|
42
|
+
assert_type(findings, list[Finding])
|
|
43
|
+
finding = findings[0]
|
|
44
|
+
assert_type(finding.entity_type, str)
|
|
45
|
+
assert_type(finding.matched_text, str)
|
|
46
|
+
assert_type(finding.byte_range, TextRange)
|
|
47
|
+
assert_type(finding.byte_range.start, int)
|
|
48
|
+
assert_type(finding.codepoint_range.end, int)
|
|
49
|
+
assert_type(finding.confidence, float | None)
|
|
50
|
+
assert_type(finding.detector_name, str)
|
|
51
|
+
assert_type(finding.detector_version, str | None)
|
|
52
|
+
manual = Finding("EMAIL", "jane@example.com", TextRange(0, 16), TextRange(0, 16), "custom", confidence=0.9)
|
|
53
|
+
assert_type(transform("jane@example.com", (manual,), config), TransformResult)
|
|
54
|
+
result = scan_and_transform("jane@example.com", {"scan": {"locale": "en-US"}, "transform": config})
|
|
55
|
+
assert_type(result.text, str)
|
|
56
|
+
assert_type(result.transformations, list[Transformation])
|
|
57
|
+
assert_type(result.transformations[0].source_byte_range, TextRange)
|
|
58
|
+
assert_type(result.transformations[0].output_codepoint_range, TextRange)
|
|
59
|
+
assert_type(result.transformations[0].resolved_key_version, str | None)
|
|
60
|
+
assert_type(result.transformations[0].resolved_token_version, str | None)
|
|
61
|
+
assert_type(scan("", None), list[Finding])
|
|
62
|
+
data: _JsonDocument = {"name": "Jane", "users": [None, True, 1, 1.5, {"email": "jane@example.com"}]}
|
|
63
|
+
assert_type(discover_fields(data, {"discover_person": False, "mappings": {"/name": "PERSON"}, "exclude": ["/users"]}), list[FieldMapping])
|
|
64
|
+
structured = scan_structured(data)
|
|
65
|
+
assert_type(structured, StructuredScanResult)
|
|
66
|
+
assert_type(structured.mappings, list[FieldMapping])
|
|
67
|
+
assert_type(structured.mappings[0].rule, str)
|
|
68
|
+
assert_type(structured.findings, list[StructuredFinding])
|
|
69
|
+
assert_type(structured.findings[0].finding, Finding)
|
|
70
|
+
assert_type(structured.findings[0].path, str)
|
|
71
|
+
located = StructuredFinding("/email", manual)
|
|
72
|
+
assert_type(transform_structured(data, [located], config), StructuredTransformResult)
|
|
73
|
+
structured_result = scan_and_transform_structured(data, {"transform": config})
|
|
74
|
+
assert_type(structured_result.data, _JsonDocument)
|
|
75
|
+
assert_type(structured_result.transformations, list[StructuredTransformation])
|
|
76
|
+
assert_type(structured_result.transformations[0].transformation, Transformation)
|
|
77
|
+
|
|
78
|
+
class KeyProvider:
|
|
79
|
+
async def resolve_key(self, ref: str, version: str | None) -> _ResolvedKey:
|
|
80
|
+
return {"key": bytes(range(32)), "resolved_version": version or "1"}
|
|
81
|
+
|
|
82
|
+
class AttributeKey:
|
|
83
|
+
key: bytes = bytes(range(32))
|
|
84
|
+
resolved_version: str = "1"
|
|
85
|
+
|
|
86
|
+
class AttributeKeyProvider:
|
|
87
|
+
async def resolve_key(self, ref: str, version: str | None) -> AttributeKey:
|
|
88
|
+
return AttributeKey()
|
|
89
|
+
|
|
90
|
+
class TokenProvider:
|
|
91
|
+
async def tokenize_batch(self, scope: str, items: list[_TokenizeItem]) -> list[_TokenizeResult]:
|
|
92
|
+
return [{"id": item["id"], "payload": b"token", "resolved_version": "1"} for item in items]
|
|
93
|
+
async def restore_batch(self, scope: str, items: list[_RestoreItem]) -> list[_RestoreResult]:
|
|
94
|
+
return [{"id": item["id"], "value": "restored"} for item in items]
|
|
95
|
+
|
|
96
|
+
PrivacyManager(AttributeKeyProvider())
|
|
97
|
+
async def use_manager() -> None:
|
|
98
|
+
manager = PrivacyManager(provider=KeyProvider(), token_provider=TokenProvider())
|
|
99
|
+
assert_type(await manager.transform("text", findings, config), TransformResult)
|
|
100
|
+
assert_type(await manager.scan_and_transform("text", {"transform": config}), TransformResult)
|
|
101
|
+
restored = await manager.restore("text", {"scope": "tenant"})
|
|
102
|
+
assert_type(restored, RestoreResult)
|
|
103
|
+
assert_type(restored.restorations, list[Restoration])
|
|
104
|
+
assert_type(restored.restorations[0].token_ref, str)
|
|
105
|
+
assert_type(restored.restorations[0].output_byte_range, TextRange)
|
|
106
|
+
assert_type(await manager.transform_structured(data, structured.findings, config), StructuredTransformResult)
|
|
107
|
+
assert_type(await manager.scan_and_transform_structured(data, {"transform": {"default": {"strategy": "tokenize", "token_ref": "names"}}}, {"scope": "tenant"}), StructuredTransformResult)
|
|
108
|
+
restored_data = await manager.restore_structured(data, {"scope": "tenant"})
|
|
109
|
+
assert_type(restored_data, StructuredRestoreResult)
|
|
110
|
+
assert_type(restored_data.data, _JsonDocument)
|
|
111
|
+
assert_type(restored_data.restorations, list[StructuredRestoration])
|
|
112
|
+
assert_type(restored_data.restorations[0].restoration, Restoration)
|
|
113
|
+
|
|
114
|
+
for error in (DataFogConfigurationError(), DataFogFindingError(), DataFogInternalError(), DataFogKeyProviderError()):
|
|
115
|
+
assert_type(error.code, str)
|
|
116
|
+
assert_type(error.reason, str | None)
|
|
117
|
+
assert_type(error.path, str | None)
|
|
118
|
+
assert_type(error.finding_index, int | None)
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
# Each line must produce a diagnostic; a missing stub or Any cannot satisfy this.
|
|
122
|
+
INVALID = """
|
|
123
|
+
from datafog_core import scan, transform, scan_structured, PrivacyManager, TransformResult
|
|
124
|
+
scan(123) # error
|
|
125
|
+
scan("text", {"locale": 123}) # error
|
|
126
|
+
scan("text", {"max_bytes": 100}) # error
|
|
127
|
+
scan("text")[0].entity_type = "OTHER" # error
|
|
128
|
+
scan("text")[0].byte_range.start = 1 # error
|
|
129
|
+
transform("text", ["not a finding"], {"default": {"strategy": "redact"}}) # error
|
|
130
|
+
transform("text", [], {"default": {"strategy": "invalid"}}) # error
|
|
131
|
+
transform("text", [], {"default": {"strategy": "pseudonymize"}}) # error
|
|
132
|
+
transform("text", [], {"default": {"strategy": "mask", "reveal": {"direction": "middle", "count": 4}}}) # error
|
|
133
|
+
scan_structured({"name": "Jane"}, {"mappings": {"/name": "EMAIL"}}) # error
|
|
134
|
+
scan_structured("not a document") # error
|
|
135
|
+
PrivacyManager(object()) # error
|
|
136
|
+
PrivacyManager().restore("text") # error
|
|
137
|
+
PrivacyManager().restore("text", {"scope": 12}) # error
|
|
138
|
+
TransformResult() # error
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def verify_metadata() -> None:
|
|
143
|
+
distribution = importlib.metadata.distribution("datafog-core")
|
|
144
|
+
files = {str(path) for path in distribution.files or []}
|
|
145
|
+
assert "datafog_core/py.typed" in files, files
|
|
146
|
+
assert "datafog_core/__init__.pyi" in files, files
|
|
147
|
+
stub = Path(distribution.locate_file("datafog_core/__init__.pyi"))
|
|
148
|
+
tree = ast.parse(stub.read_text())
|
|
149
|
+
declarations = {
|
|
150
|
+
node.name: node
|
|
151
|
+
for node in tree.body
|
|
152
|
+
if isinstance(node, (ast.ClassDef, ast.FunctionDef))
|
|
153
|
+
and not node.name.startswith("_")
|
|
154
|
+
}
|
|
155
|
+
native_module = importlib.import_module("datafog_core.datafog_core")
|
|
156
|
+
exports = set(native_module.__all__)
|
|
157
|
+
assert set(datafog_core.__all__) == exports
|
|
158
|
+
assert set(declarations) == exports, (
|
|
159
|
+
set(declarations) - exports,
|
|
160
|
+
exports - set(declarations),
|
|
161
|
+
)
|
|
162
|
+
for name, declaration in declarations.items():
|
|
163
|
+
value = getattr(datafog_core, name)
|
|
164
|
+
if isinstance(declaration, ast.ClassDef):
|
|
165
|
+
# Every native read-only field needs a corresponding typed property.
|
|
166
|
+
properties = {
|
|
167
|
+
member.name
|
|
168
|
+
for member in declaration.body
|
|
169
|
+
if isinstance(member, ast.FunctionDef)
|
|
170
|
+
and any(
|
|
171
|
+
isinstance(d, ast.Name) and d.id == "property"
|
|
172
|
+
for d in member.decorator_list
|
|
173
|
+
)
|
|
174
|
+
}
|
|
175
|
+
native = {
|
|
176
|
+
key
|
|
177
|
+
for key, member in vars(value).items()
|
|
178
|
+
if (
|
|
179
|
+
inspect.isgetsetdescriptor(member)
|
|
180
|
+
or inspect.ismemberdescriptor(member)
|
|
181
|
+
)
|
|
182
|
+
and not key.startswith("_")
|
|
183
|
+
}
|
|
184
|
+
assert properties == native, (name, properties, native)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def main() -> None:
|
|
188
|
+
verify_metadata()
|
|
189
|
+
with tempfile.TemporaryDirectory(prefix="datafog-typing-") as temporary:
|
|
190
|
+
root = Path(temporary)
|
|
191
|
+
(root / "valid.py").write_text(VALID)
|
|
192
|
+
(root / "invalid.py").write_text(INVALID)
|
|
193
|
+
(root / "pyrightconfig.json").write_text(
|
|
194
|
+
json.dumps(
|
|
195
|
+
{
|
|
196
|
+
"typeCheckingMode": "strict",
|
|
197
|
+
"pythonVersion": "3.10",
|
|
198
|
+
"reportPrivateUsage": False,
|
|
199
|
+
}
|
|
200
|
+
)
|
|
201
|
+
)
|
|
202
|
+
for checker in ("mypy", "pyright"):
|
|
203
|
+
command = [sys.executable, "-m", checker]
|
|
204
|
+
if checker == "mypy":
|
|
205
|
+
command += [
|
|
206
|
+
"--strict",
|
|
207
|
+
"--python-version",
|
|
208
|
+
"3.10",
|
|
209
|
+
"--no-incremental",
|
|
210
|
+
"--show-column-numbers",
|
|
211
|
+
]
|
|
212
|
+
else:
|
|
213
|
+
command += ["--pythonpath", sys.executable, "--outputjson"]
|
|
214
|
+
valid = subprocess.run(
|
|
215
|
+
command + ["valid.py"],
|
|
216
|
+
cwd=root,
|
|
217
|
+
capture_output=True,
|
|
218
|
+
text=True,
|
|
219
|
+
check=False,
|
|
220
|
+
)
|
|
221
|
+
assert valid.returncode == 0, (
|
|
222
|
+
f"{checker} valid usage failed:\n{valid.stdout}\n{valid.stderr}"
|
|
223
|
+
)
|
|
224
|
+
invalid = subprocess.run(
|
|
225
|
+
command + ["invalid.py"],
|
|
226
|
+
cwd=root,
|
|
227
|
+
capture_output=True,
|
|
228
|
+
text=True,
|
|
229
|
+
check=False,
|
|
230
|
+
)
|
|
231
|
+
assert invalid.returncode == 1, (
|
|
232
|
+
f"{checker} expected type errors:\n{invalid.stdout}\n{invalid.stderr}"
|
|
233
|
+
)
|
|
234
|
+
expected = {
|
|
235
|
+
number
|
|
236
|
+
for number, line in enumerate(INVALID.splitlines(), 1)
|
|
237
|
+
if "# error" in line
|
|
238
|
+
}
|
|
239
|
+
if checker == "pyright":
|
|
240
|
+
diagnostics = json.loads(invalid.stdout)["generalDiagnostics"]
|
|
241
|
+
actual = {
|
|
242
|
+
item["range"]["start"]["line"] + 1
|
|
243
|
+
for item in diagnostics
|
|
244
|
+
if item["severity"] == "error"
|
|
245
|
+
}
|
|
246
|
+
else:
|
|
247
|
+
actual = {
|
|
248
|
+
int(line.split(":")[1])
|
|
249
|
+
for line in invalid.stdout.splitlines()
|
|
250
|
+
if line.startswith("invalid.py:") and ": error:" in line
|
|
251
|
+
}
|
|
252
|
+
assert actual == expected, (
|
|
253
|
+
f"{checker}: expected lines {expected}, got {actual}\n{invalid.stdout}"
|
|
254
|
+
)
|
|
255
|
+
print(f"{checker}: valid usage passes; all invalid cases rejected.")
|
|
256
|
+
print("Installed typing metadata covers every runtime export and native property.")
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
if __name__ == "__main__":
|
|
260
|
+
main()
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "datafog-core"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
description = "Fast structured PII detection implemented in Rust"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -30,5 +30,6 @@ Repository = "https://github.com/DataFog/datafog-core"
|
|
|
30
30
|
Issues = "https://github.com/DataFog/datafog-core/issues"
|
|
31
31
|
|
|
32
32
|
[tool.maturin]
|
|
33
|
-
module-name = "datafog_core"
|
|
33
|
+
module-name = "datafog_core.datafog_core"
|
|
34
34
|
manifest-path = "bindings/python/Cargo.toml"
|
|
35
|
+
python-source = "python"
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Fast structured PII detection implemented in Rust."""
|
|
2
|
+
|
|
3
|
+
from .datafog_core import (
|
|
4
|
+
DataFogConfigurationError as DataFogConfigurationError,
|
|
5
|
+
DataFogFindingError as DataFogFindingError,
|
|
6
|
+
DataFogInternalError as DataFogInternalError,
|
|
7
|
+
DataFogKeyProviderError as DataFogKeyProviderError,
|
|
8
|
+
FieldMapping as FieldMapping,
|
|
9
|
+
Finding as Finding,
|
|
10
|
+
PrivacyManager as PrivacyManager,
|
|
11
|
+
Restoration as Restoration,
|
|
12
|
+
RestoreResult as RestoreResult,
|
|
13
|
+
StructuredFinding as StructuredFinding,
|
|
14
|
+
StructuredRestoration as StructuredRestoration,
|
|
15
|
+
StructuredRestoreResult as StructuredRestoreResult,
|
|
16
|
+
StructuredScanResult as StructuredScanResult,
|
|
17
|
+
StructuredTransformation as StructuredTransformation,
|
|
18
|
+
StructuredTransformResult as StructuredTransformResult,
|
|
19
|
+
TextRange as TextRange,
|
|
20
|
+
Transformation as Transformation,
|
|
21
|
+
TransformResult as TransformResult,
|
|
22
|
+
discover_fields as discover_fields,
|
|
23
|
+
scan as scan,
|
|
24
|
+
scan_and_transform as scan_and_transform,
|
|
25
|
+
scan_and_transform_structured as scan_and_transform_structured,
|
|
26
|
+
scan_structured as scan_structured,
|
|
27
|
+
transform as transform,
|
|
28
|
+
transform_structured as transform_structured,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"DataFogConfigurationError",
|
|
33
|
+
"DataFogFindingError",
|
|
34
|
+
"DataFogInternalError",
|
|
35
|
+
"DataFogKeyProviderError",
|
|
36
|
+
"TextRange",
|
|
37
|
+
"Finding",
|
|
38
|
+
"Transformation",
|
|
39
|
+
"TransformResult",
|
|
40
|
+
"Restoration",
|
|
41
|
+
"RestoreResult",
|
|
42
|
+
"FieldMapping",
|
|
43
|
+
"StructuredFinding",
|
|
44
|
+
"StructuredScanResult",
|
|
45
|
+
"StructuredTransformation",
|
|
46
|
+
"StructuredTransformResult",
|
|
47
|
+
"StructuredRestoration",
|
|
48
|
+
"StructuredRestoreResult",
|
|
49
|
+
"PrivacyManager",
|
|
50
|
+
"scan",
|
|
51
|
+
"transform",
|
|
52
|
+
"scan_and_transform",
|
|
53
|
+
"discover_fields",
|
|
54
|
+
"scan_structured",
|
|
55
|
+
"transform_structured",
|
|
56
|
+
"scan_and_transform_structured",
|
|
57
|
+
]
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
"""Types for the compiled API. Underscored helpers exist only for type checking."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Awaitable, Iterable, Sequence
|
|
4
|
+
from typing import Literal, NoReturn, Protocol, TypeAlias, TypedDict, final
|
|
5
|
+
|
|
6
|
+
_JsonValue: TypeAlias = (
|
|
7
|
+
None | bool | int | float | str | list["_JsonValue"] | dict[str, "_JsonValue"]
|
|
8
|
+
)
|
|
9
|
+
_JsonDocument: TypeAlias = list[_JsonValue] | dict[str, _JsonValue]
|
|
10
|
+
|
|
11
|
+
class _ScanConfig(TypedDict, total=False):
|
|
12
|
+
locale: str
|
|
13
|
+
|
|
14
|
+
class _StructuredScanConfig(_ScanConfig, total=False):
|
|
15
|
+
discover_person: bool
|
|
16
|
+
mappings: dict[str, Literal["PERSON"]]
|
|
17
|
+
exclude: list[str] | tuple[str, ...]
|
|
18
|
+
|
|
19
|
+
class _Reveal(TypedDict):
|
|
20
|
+
direction: Literal["first", "last"]
|
|
21
|
+
count: int
|
|
22
|
+
|
|
23
|
+
class _Redact(TypedDict):
|
|
24
|
+
strategy: Literal["redact"]
|
|
25
|
+
|
|
26
|
+
class _Remove(TypedDict):
|
|
27
|
+
strategy: Literal["remove"]
|
|
28
|
+
|
|
29
|
+
class _MaskRequired(TypedDict):
|
|
30
|
+
strategy: Literal["mask"]
|
|
31
|
+
|
|
32
|
+
class _Mask(_MaskRequired, total=False):
|
|
33
|
+
character: str
|
|
34
|
+
reveal: _Reveal
|
|
35
|
+
|
|
36
|
+
class _PseudonymizeRequired(TypedDict):
|
|
37
|
+
strategy: Literal["pseudonymize"]
|
|
38
|
+
key_ref: str
|
|
39
|
+
|
|
40
|
+
class _Pseudonymize(_PseudonymizeRequired, total=False):
|
|
41
|
+
key_version: str
|
|
42
|
+
|
|
43
|
+
class _Tokenize(TypedDict):
|
|
44
|
+
strategy: Literal["tokenize"]
|
|
45
|
+
token_ref: str
|
|
46
|
+
|
|
47
|
+
_StrategyConfig: TypeAlias = _Redact | _Remove | _Mask | _Pseudonymize | _Tokenize
|
|
48
|
+
|
|
49
|
+
class _RegexRequired(TypedDict):
|
|
50
|
+
pattern: str
|
|
51
|
+
|
|
52
|
+
class _Regex(_RegexRequired, total=False):
|
|
53
|
+
case_sensitive: bool
|
|
54
|
+
|
|
55
|
+
class _Allow(TypedDict, total=False):
|
|
56
|
+
exact: dict[str, list[str] | tuple[str, ...]]
|
|
57
|
+
regex: dict[str, list[_Regex] | tuple[_Regex, ...]]
|
|
58
|
+
|
|
59
|
+
class _TransformRequired(TypedDict):
|
|
60
|
+
default: _StrategyConfig
|
|
61
|
+
|
|
62
|
+
class _TransformationConfig(_TransformRequired, total=False):
|
|
63
|
+
entities: list[str] | tuple[str, ...]
|
|
64
|
+
overrides: dict[str, _StrategyConfig]
|
|
65
|
+
allow: _Allow
|
|
66
|
+
|
|
67
|
+
class _ScanAndTransformRequired(TypedDict):
|
|
68
|
+
transform: _TransformationConfig
|
|
69
|
+
|
|
70
|
+
class _ScanAndTransformConfig(_ScanAndTransformRequired, total=False):
|
|
71
|
+
scan: _ScanConfig
|
|
72
|
+
|
|
73
|
+
class _StructuredScanAndTransformConfig(_ScanAndTransformRequired, total=False):
|
|
74
|
+
scan: _StructuredScanConfig
|
|
75
|
+
|
|
76
|
+
class _PrivacyContext(TypedDict):
|
|
77
|
+
scope: str
|
|
78
|
+
|
|
79
|
+
class _ResolvedKey(TypedDict):
|
|
80
|
+
key: bytes | Sequence[int]
|
|
81
|
+
resolved_version: str
|
|
82
|
+
|
|
83
|
+
class _ResolvedKeyObject(Protocol):
|
|
84
|
+
@property
|
|
85
|
+
def key(self) -> bytes | Sequence[int]: ...
|
|
86
|
+
@property
|
|
87
|
+
def resolved_version(self) -> str: ...
|
|
88
|
+
|
|
89
|
+
class _KeyProvider(Protocol):
|
|
90
|
+
def resolve_key(
|
|
91
|
+
self, key_ref: str, key_version: str | None, /
|
|
92
|
+
) -> Awaitable[_ResolvedKey | _ResolvedKeyObject]: ...
|
|
93
|
+
|
|
94
|
+
class _TokenizeItem(TypedDict):
|
|
95
|
+
id: str
|
|
96
|
+
exact_value: str
|
|
97
|
+
token_ref: str
|
|
98
|
+
|
|
99
|
+
class _TokenizeResult(TypedDict):
|
|
100
|
+
id: str
|
|
101
|
+
payload: bytes | Sequence[int]
|
|
102
|
+
resolved_version: str
|
|
103
|
+
|
|
104
|
+
class _TokenizeResultObject(Protocol):
|
|
105
|
+
@property
|
|
106
|
+
def id(self) -> str: ...
|
|
107
|
+
@property
|
|
108
|
+
def payload(self) -> bytes | Sequence[int]: ...
|
|
109
|
+
@property
|
|
110
|
+
def resolved_version(self) -> str: ...
|
|
111
|
+
|
|
112
|
+
class _RestoreItem(TypedDict):
|
|
113
|
+
id: str
|
|
114
|
+
token_ref: str
|
|
115
|
+
resolved_version: str
|
|
116
|
+
payload: bytes
|
|
117
|
+
|
|
118
|
+
class _RestoreResult(TypedDict):
|
|
119
|
+
id: str
|
|
120
|
+
value: str
|
|
121
|
+
|
|
122
|
+
class _RestoreResultObject(Protocol):
|
|
123
|
+
@property
|
|
124
|
+
def id(self) -> str: ...
|
|
125
|
+
@property
|
|
126
|
+
def value(self) -> str: ...
|
|
127
|
+
|
|
128
|
+
class _TokenProvider(Protocol):
|
|
129
|
+
def tokenize_batch(
|
|
130
|
+
self, scope: str, items: list[_TokenizeItem], /
|
|
131
|
+
) -> Awaitable[Iterable[_TokenizeResult | _TokenizeResultObject]]: ...
|
|
132
|
+
def restore_batch(
|
|
133
|
+
self, scope: str, items: list[_RestoreItem], /
|
|
134
|
+
) -> Awaitable[Iterable[_RestoreResult | _RestoreResultObject]]: ...
|
|
135
|
+
|
|
136
|
+
class DataFogConfigurationError(ValueError):
|
|
137
|
+
code: str
|
|
138
|
+
reason: str | None
|
|
139
|
+
path: str | None
|
|
140
|
+
finding_index: int | None
|
|
141
|
+
|
|
142
|
+
class DataFogFindingError(ValueError):
|
|
143
|
+
code: str
|
|
144
|
+
reason: str | None
|
|
145
|
+
path: str | None
|
|
146
|
+
finding_index: int | None
|
|
147
|
+
|
|
148
|
+
class DataFogInternalError(RuntimeError):
|
|
149
|
+
code: str
|
|
150
|
+
reason: str | None
|
|
151
|
+
path: str | None
|
|
152
|
+
finding_index: int | None
|
|
153
|
+
|
|
154
|
+
class DataFogKeyProviderError(RuntimeError):
|
|
155
|
+
code: str
|
|
156
|
+
reason: str | None
|
|
157
|
+
path: str | None
|
|
158
|
+
finding_index: int | None
|
|
159
|
+
|
|
160
|
+
@final
|
|
161
|
+
class TextRange:
|
|
162
|
+
def __init__(self, start: int, end: int) -> None: ...
|
|
163
|
+
@property
|
|
164
|
+
def start(self) -> int: ...
|
|
165
|
+
@property
|
|
166
|
+
def end(self) -> int: ...
|
|
167
|
+
|
|
168
|
+
@final
|
|
169
|
+
class Finding:
|
|
170
|
+
def __init__(
|
|
171
|
+
self,
|
|
172
|
+
entity_type: str,
|
|
173
|
+
matched_text: str,
|
|
174
|
+
byte_range: TextRange,
|
|
175
|
+
codepoint_range: TextRange,
|
|
176
|
+
detector_name: str,
|
|
177
|
+
confidence: float | None = None,
|
|
178
|
+
detector_version: str | None = None,
|
|
179
|
+
) -> None: ...
|
|
180
|
+
@property
|
|
181
|
+
def entity_type(self) -> str: ...
|
|
182
|
+
@property
|
|
183
|
+
def matched_text(self) -> str: ...
|
|
184
|
+
@property
|
|
185
|
+
def byte_range(self) -> TextRange: ...
|
|
186
|
+
@property
|
|
187
|
+
def codepoint_range(self) -> TextRange: ...
|
|
188
|
+
@property
|
|
189
|
+
def confidence(self) -> float | None: ...
|
|
190
|
+
@property
|
|
191
|
+
def detector_name(self) -> str: ...
|
|
192
|
+
@property
|
|
193
|
+
def detector_version(self) -> str | None: ...
|
|
194
|
+
|
|
195
|
+
# NoReturn prevents construction of native result-only classes on Python 3.10
|
|
196
|
+
# without requiring typing_extensions.Never.
|
|
197
|
+
@final
|
|
198
|
+
class Transformation:
|
|
199
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
200
|
+
@property
|
|
201
|
+
def entity_type(self) -> str: ...
|
|
202
|
+
@property
|
|
203
|
+
def source_byte_range(self) -> TextRange: ...
|
|
204
|
+
@property
|
|
205
|
+
def source_codepoint_range(self) -> TextRange: ...
|
|
206
|
+
@property
|
|
207
|
+
def confidence(self) -> float | None: ...
|
|
208
|
+
@property
|
|
209
|
+
def detector_name(self) -> str: ...
|
|
210
|
+
@property
|
|
211
|
+
def detector_version(self) -> str | None: ...
|
|
212
|
+
@property
|
|
213
|
+
def strategy(self) -> str: ...
|
|
214
|
+
@property
|
|
215
|
+
def replacement(self) -> str: ...
|
|
216
|
+
@property
|
|
217
|
+
def output_byte_range(self) -> TextRange: ...
|
|
218
|
+
@property
|
|
219
|
+
def output_codepoint_range(self) -> TextRange: ...
|
|
220
|
+
@property
|
|
221
|
+
def key_ref(self) -> str | None: ...
|
|
222
|
+
@property
|
|
223
|
+
def resolved_key_version(self) -> str | None: ...
|
|
224
|
+
@property
|
|
225
|
+
def token_ref(self) -> str | None: ...
|
|
226
|
+
@property
|
|
227
|
+
def resolved_token_version(self) -> str | None: ...
|
|
228
|
+
|
|
229
|
+
@final
|
|
230
|
+
class TransformResult:
|
|
231
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
232
|
+
@property
|
|
233
|
+
def text(self) -> str: ...
|
|
234
|
+
@property
|
|
235
|
+
def transformations(self) -> list[Transformation]: ...
|
|
236
|
+
|
|
237
|
+
@final
|
|
238
|
+
class Restoration:
|
|
239
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
240
|
+
@property
|
|
241
|
+
def source_byte_range(self) -> TextRange: ...
|
|
242
|
+
@property
|
|
243
|
+
def source_codepoint_range(self) -> TextRange: ...
|
|
244
|
+
@property
|
|
245
|
+
def output_byte_range(self) -> TextRange: ...
|
|
246
|
+
@property
|
|
247
|
+
def output_codepoint_range(self) -> TextRange: ...
|
|
248
|
+
@property
|
|
249
|
+
def token_ref(self) -> str: ...
|
|
250
|
+
@property
|
|
251
|
+
def resolved_token_version(self) -> str: ...
|
|
252
|
+
|
|
253
|
+
@final
|
|
254
|
+
class RestoreResult:
|
|
255
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
256
|
+
@property
|
|
257
|
+
def text(self) -> str: ...
|
|
258
|
+
@property
|
|
259
|
+
def restorations(self) -> list[Restoration]: ...
|
|
260
|
+
|
|
261
|
+
@final
|
|
262
|
+
class FieldMapping:
|
|
263
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
264
|
+
@property
|
|
265
|
+
def path(self) -> str: ...
|
|
266
|
+
@property
|
|
267
|
+
def entity_type(self) -> str: ...
|
|
268
|
+
@property
|
|
269
|
+
def source(self) -> str: ...
|
|
270
|
+
@property
|
|
271
|
+
def rule(self) -> str: ...
|
|
272
|
+
|
|
273
|
+
@final
|
|
274
|
+
class StructuredFinding:
|
|
275
|
+
def __init__(self, path: str, finding: Finding) -> None: ...
|
|
276
|
+
@property
|
|
277
|
+
def path(self) -> str: ...
|
|
278
|
+
@property
|
|
279
|
+
def finding(self) -> Finding: ...
|
|
280
|
+
|
|
281
|
+
@final
|
|
282
|
+
class StructuredScanResult:
|
|
283
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
284
|
+
@property
|
|
285
|
+
def mappings(self) -> list[FieldMapping]: ...
|
|
286
|
+
@property
|
|
287
|
+
def findings(self) -> list[StructuredFinding]: ...
|
|
288
|
+
|
|
289
|
+
@final
|
|
290
|
+
class StructuredTransformation:
|
|
291
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
292
|
+
@property
|
|
293
|
+
def path(self) -> str: ...
|
|
294
|
+
@property
|
|
295
|
+
def transformation(self) -> Transformation: ...
|
|
296
|
+
|
|
297
|
+
@final
|
|
298
|
+
class StructuredTransformResult:
|
|
299
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
300
|
+
@property
|
|
301
|
+
def data(self) -> _JsonDocument: ...
|
|
302
|
+
@property
|
|
303
|
+
def transformations(self) -> list[StructuredTransformation]: ...
|
|
304
|
+
|
|
305
|
+
@final
|
|
306
|
+
class StructuredRestoration:
|
|
307
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
308
|
+
@property
|
|
309
|
+
def path(self) -> str: ...
|
|
310
|
+
@property
|
|
311
|
+
def restoration(self) -> Restoration: ...
|
|
312
|
+
|
|
313
|
+
@final
|
|
314
|
+
class StructuredRestoreResult:
|
|
315
|
+
def __init__(self, _unconstructible: NoReturn) -> None: ...
|
|
316
|
+
@property
|
|
317
|
+
def data(self) -> _JsonDocument: ...
|
|
318
|
+
@property
|
|
319
|
+
def restorations(self) -> list[StructuredRestoration]: ...
|
|
320
|
+
|
|
321
|
+
@final
|
|
322
|
+
class PrivacyManager:
|
|
323
|
+
def __init__(
|
|
324
|
+
self,
|
|
325
|
+
provider: _KeyProvider | None = None,
|
|
326
|
+
token_provider: _TokenProvider | None = None,
|
|
327
|
+
) -> None: ...
|
|
328
|
+
def transform(
|
|
329
|
+
self,
|
|
330
|
+
text: str,
|
|
331
|
+
findings: Sequence[Finding],
|
|
332
|
+
config: _TransformationConfig,
|
|
333
|
+
context: _PrivacyContext | None = None,
|
|
334
|
+
) -> Awaitable[TransformResult]: ...
|
|
335
|
+
def scan_and_transform(
|
|
336
|
+
self,
|
|
337
|
+
text: str,
|
|
338
|
+
config: _ScanAndTransformConfig,
|
|
339
|
+
context: _PrivacyContext | None = None,
|
|
340
|
+
) -> Awaitable[TransformResult]: ...
|
|
341
|
+
def restore(
|
|
342
|
+
self, text: str, context: _PrivacyContext
|
|
343
|
+
) -> Awaitable[RestoreResult]: ...
|
|
344
|
+
def transform_structured(
|
|
345
|
+
self,
|
|
346
|
+
data: _JsonDocument,
|
|
347
|
+
findings: Sequence[StructuredFinding],
|
|
348
|
+
config: _TransformationConfig,
|
|
349
|
+
context: _PrivacyContext | None = None,
|
|
350
|
+
) -> Awaitable[StructuredTransformResult]: ...
|
|
351
|
+
def scan_and_transform_structured(
|
|
352
|
+
self,
|
|
353
|
+
data: _JsonDocument,
|
|
354
|
+
config: _StructuredScanAndTransformConfig,
|
|
355
|
+
context: _PrivacyContext | None = None,
|
|
356
|
+
) -> Awaitable[StructuredTransformResult]: ...
|
|
357
|
+
def restore_structured(
|
|
358
|
+
self, data: _JsonDocument, context: _PrivacyContext
|
|
359
|
+
) -> Awaitable[StructuredRestoreResult]: ...
|
|
360
|
+
|
|
361
|
+
def scan(text: str, config: _ScanConfig | None = None) -> list[Finding]: ...
|
|
362
|
+
def transform(
|
|
363
|
+
text: str, findings: Sequence[Finding], config: _TransformationConfig
|
|
364
|
+
) -> TransformResult: ...
|
|
365
|
+
def scan_and_transform(
|
|
366
|
+
text: str, config: _ScanAndTransformConfig
|
|
367
|
+
) -> TransformResult: ...
|
|
368
|
+
def discover_fields(
|
|
369
|
+
data: _JsonDocument, config: _StructuredScanConfig | None = None
|
|
370
|
+
) -> list[FieldMapping]: ...
|
|
371
|
+
def scan_structured(
|
|
372
|
+
data: _JsonDocument, config: _StructuredScanConfig | None = None
|
|
373
|
+
) -> StructuredScanResult: ...
|
|
374
|
+
def transform_structured(
|
|
375
|
+
data: _JsonDocument,
|
|
376
|
+
findings: Sequence[StructuredFinding],
|
|
377
|
+
config: _TransformationConfig,
|
|
378
|
+
) -> StructuredTransformResult: ...
|
|
379
|
+
def scan_and_transform_structured(
|
|
380
|
+
data: _JsonDocument, config: _StructuredScanAndTransformConfig
|
|
381
|
+
) -> StructuredTransformResult: ...
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|