datafog-core 0.5.0b2__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/Cargo.lock +8 -6
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/PKG-INFO +39 -33
- {datafog_core-0.5.0b2/crates/core → datafog_core-1.0.0}/README.md +38 -32
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/Cargo.toml +4 -2
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/src/lib.rs +9 -0
- datafog_core-1.0.0/bindings/python/src/workspace/conversation.rs +166 -0
- datafog_core-1.0.0/bindings/python/src/workspace/operations.rs +84 -0
- datafog_core-1.0.0/bindings/python/src/workspace/stream.rs +346 -0
- datafog_core-1.0.0/bindings/python/src/workspace.rs +1235 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/test_installed.py +10 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/test_typing.py +143 -1
- datafog_core-1.0.0/bindings/python/tests/workspace_backup_operation_conformance.py +56 -0
- datafog_core-1.0.0/bindings/python/tests/workspace_conformance.py +280 -0
- datafog_core-1.0.0/bindings/python/tests/workspace_conversation_conformance.py +149 -0
- datafog_core-1.0.0/bindings/python/tests/workspace_operation_conformance.py +241 -0
- datafog_core-1.0.0/bindings/python/tests/workspace_streaming_conformance.py +148 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/Cargo.toml +1 -1
- datafog_core-1.0.0/crates/core/LICENSE +21 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0/crates/core}/README.md +38 -32
- datafog_core-1.0.0/crates/core/examples/workspace_backup_operation_fields.rs +581 -0
- datafog_core-1.0.0/crates/core/examples/workspace_contract/conversation.rs +330 -0
- datafog_core-1.0.0/crates/core/examples/workspace_contract.rs +661 -0
- datafog_core-1.0.0/crates/core/examples/workspace_conversation_capacity.rs +286 -0
- datafog_core-1.0.0/crates/core/examples/workspace_operation_capacity.rs +337 -0
- datafog_core-1.0.0/crates/core/examples/workspace_operation_cost.rs +173 -0
- datafog_core-1.0.0/crates/core/examples/workspace_streaming_contract.rs +250 -0
- datafog_core-1.0.0/crates/core/src/workspace/buffered_restore.rs +307 -0
- datafog_core-1.0.0/crates/core/src/workspace/codec.rs +1029 -0
- datafog_core-1.0.0/crates/core/src/workspace/conversation.rs +1498 -0
- datafog_core-1.0.0/crates/core/src/workspace/operations.rs +357 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/workspace/policy.rs +12 -2
- datafog_core-1.0.0/crates/core/src/workspace/private_bytes.rs +22 -0
- datafog_core-1.0.0/crates/core/src/workspace/progressive_restore.rs +561 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/workspace.rs +220 -66
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/workspace.rs +1 -1
- datafog_core-1.0.0/crates/core/tests/workspace_backup_fields.rs +173 -0
- datafog_core-1.0.0/crates/core/tests/workspace_backup_operation_fields.rs +333 -0
- datafog_core-1.0.0/crates/core/tests/workspace_buffered_restore.rs +194 -0
- datafog_core-1.0.0/crates/core/tests/workspace_conversation_dense.rs +118 -0
- datafog_core-1.0.0/crates/core/tests/workspace_conversations.rs +377 -0
- datafog_core-1.0.0/crates/core/tests/workspace_operations.rs +534 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/workspace_policy.rs +3 -3
- datafog_core-1.0.0/crates/core/tests/workspace_progressive_restore.rs +537 -0
- datafog_core-1.0.0/crates/core/tests/workspace_snapshot_buffer_cleanup.rs +280 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/pyproject.toml +1 -1
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/python/datafog_core/__init__.py +17 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/python/datafog_core/__init__.pyi +283 -0
- datafog_core-0.5.0b2/crates/core/src/workspace/codec.rs +0 -272
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/Cargo.toml +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/src/models.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/api_key_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/bearer_token_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/capabilities_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/credential_overlap_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/credential_uri_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/email_context_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/german_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/jwt_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/npi_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/private_key_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/requirements-typing.txt +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/synthetic_fixtures.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/test_python49_integration.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/us_routing_number_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/bindings/python/tests/uuid_conformance.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/examples/german_benchmark.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/examples/scan_benchmark.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/account_number.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/api_key.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/bearer_token.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/capabilities.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/coverage.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/credential_uri.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/email_context.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/german.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/jwt.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/lib.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/model_scan.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/models/gliner2-lock.json +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/models/gliner2.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/models/process.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/models/tests.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/models.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/npi.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/offsets.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/private_key.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/scan_selection_tests.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/selection_tests.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/street_address.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/structured.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/us_routing_number.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/uuid.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/workspace/recall.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/src/workspace/review.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/account_number.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/api_key.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/bearer_token.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/capabilities.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/credential_overlaps.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/credential_uri.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/email_context.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/german.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/jwt.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/model_datafog.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/model_gliner2_entity_count.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/npi.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/private_key.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/street_address.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/support/synthetic_fixtures.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/us_routing_number.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/uuid.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/workspace_recall.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/crates/core/tests/workspace_review.rs +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/python/datafog_core/_gliner2_worker.py +0 -0
- {datafog_core-0.5.0b2 → datafog_core-1.0.0}/python/datafog_core/py.typed +0 -0
|
@@ -122,7 +122,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
|
|
|
122
122
|
|
|
123
123
|
[[package]]
|
|
124
124
|
name = "datafog-core"
|
|
125
|
-
version = "0.
|
|
125
|
+
version = "1.0.0"
|
|
126
126
|
dependencies = [
|
|
127
127
|
"base64",
|
|
128
128
|
"futures",
|
|
@@ -137,29 +137,31 @@ dependencies = [
|
|
|
137
137
|
|
|
138
138
|
[[package]]
|
|
139
139
|
name = "datafog-core-python"
|
|
140
|
-
version = "0.
|
|
140
|
+
version = "1.0.0"
|
|
141
141
|
dependencies = [
|
|
142
142
|
"datafog-core",
|
|
143
143
|
"pyo3",
|
|
144
144
|
"pyo3-async-runtimes",
|
|
145
145
|
"serde",
|
|
146
146
|
"serde_json",
|
|
147
|
+
"zeroize",
|
|
147
148
|
]
|
|
148
149
|
|
|
149
150
|
[[package]]
|
|
150
151
|
name = "datafog-node"
|
|
151
|
-
version = "0.
|
|
152
|
+
version = "1.0.0"
|
|
152
153
|
dependencies = [
|
|
153
154
|
"datafog-core",
|
|
154
155
|
"napi",
|
|
155
156
|
"napi-build",
|
|
156
157
|
"napi-derive",
|
|
158
|
+
"serde",
|
|
157
159
|
"serde_json",
|
|
158
160
|
]
|
|
159
161
|
|
|
160
162
|
[[package]]
|
|
161
163
|
name = "datafog-wasm"
|
|
162
|
-
version = "0.
|
|
164
|
+
version = "1.0.0"
|
|
163
165
|
dependencies = [
|
|
164
166
|
"datafog-core",
|
|
165
167
|
"serde",
|
|
@@ -715,9 +717,9 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
715
717
|
|
|
716
718
|
[[package]]
|
|
717
719
|
name = "tokio"
|
|
718
|
-
version = "1.53.
|
|
720
|
+
version = "1.53.3"
|
|
719
721
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
720
|
-
checksum = "
|
|
722
|
+
checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e"
|
|
721
723
|
dependencies = [
|
|
722
724
|
"pin-project-lite",
|
|
723
725
|
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datafog-core
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.0
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: License :: OSI Approved :: MIT License
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -32,19 +32,19 @@ Project-URL: Issues, https://github.com/DataFog/datafog-core/issues
|
|
|
32
32
|
Project-URL: Repository, https://github.com/DataFog/datafog-core
|
|
33
33
|
|
|
34
34
|
<p align="center">
|
|
35
|
-
<a href="https://datafog.ai"><img src="docs/favicon.svg" width="72" height="72" alt="
|
|
35
|
+
<a href="https://datafog.ai"><img src="docs/favicon.svg" width="72" height="72" alt="datafog"></a>
|
|
36
36
|
</p>
|
|
37
37
|
|
|
38
|
-
<h1 align="center">
|
|
38
|
+
<h1 align="center">datafog core</h1>
|
|
39
39
|
|
|
40
40
|
<p align="center"><strong>Find and transform sensitive data in your application.</strong></p>
|
|
41
41
|
|
|
42
42
|
<p align="center">
|
|
43
|
-
<a href="https://pypi.org/project/datafog-core/"><img src="https://img.shields.io/pypi/v/datafog-core?style=flat-square&color=
|
|
44
|
-
<a href="https://www.npmjs.com/package/@datafog/node"><img src="https://img.shields.io/npm/v/%40datafog%2Fnode?style=flat-square&color=
|
|
45
|
-
<a href="https://crates.io/crates/datafog-core"><img src="https://img.shields.io/crates/v/datafog-core?style=flat-square&color=
|
|
43
|
+
<a href="https://pypi.org/project/datafog-core/"><img src="https://img.shields.io/pypi/v/datafog-core?style=flat-square&color=C0362C" alt="PyPI version"></a>
|
|
44
|
+
<a href="https://www.npmjs.com/package/@datafog/node"><img src="https://img.shields.io/npm/v/%40datafog%2Fnode?style=flat-square&color=C0362C" alt="npm version"></a>
|
|
45
|
+
<a href="https://crates.io/crates/datafog-core"><img src="https://img.shields.io/crates/v/datafog-core?style=flat-square&color=C0362C" alt="crates.io version"></a>
|
|
46
46
|
<a href="https://github.com/DataFog/datafog-core/actions/workflows/rust-quality.yml"><img src="https://github.com/DataFog/datafog-core/actions/workflows/rust-quality.yml/badge.svg?branch=main" alt="Rust CI status"></a>
|
|
47
|
-
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-
|
|
47
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-C0362C?style=flat-square" alt="MIT license"></a>
|
|
48
48
|
</p>
|
|
49
49
|
|
|
50
50
|
<p align="center">
|
|
@@ -55,22 +55,24 @@ Project-URL: Repository, https://github.com/DataFog/datafog-core
|
|
|
55
55
|
<a href="#documentation-and-development">Documentation</a>
|
|
56
56
|
</p>
|
|
57
57
|
|
|
58
|
-
|
|
58
|
+
datafog core is a Rust library with Python, Node.js, and browser WebAssembly bindings. Scan text or JSON for personal data and credentials, then redact, mask, remove, pseudonymize, or tokenize matched values. Findings use a consistent format across the bindings.
|
|
59
59
|
|
|
60
|
-
Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add
|
|
60
|
+
Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add datafog and GLiNER2 support in Rust, Python, and Node.js.
|
|
61
61
|
|
|
62
|
-
**
|
|
62
|
+
**1.0.0 release:** [release notes](docs/releases/1-0-0.mdx) · Publication underway; the coordinated release is complete only after public-registry installation checks pass.
|
|
63
|
+
|
|
64
|
+
Core 1.0 freezes Workspace contract 1 across Rust, Python, Node and browser WASM, with optional native/browser [FogVault integrations](docs/integrations/fogvault.md). Snapshot v3 writes and v1/v2/v3 reads keep their existing semantics. Exact published [0.5.0-beta.3](docs/releases/0-5-0-beta-3.mdx) interchange remains tested; FogVault is independently versioned 0.1.0.
|
|
63
65
|
|
|
64
66
|
## Quickstart
|
|
65
67
|
|
|
66
|
-
|
|
68
|
+
Once publication completes, choose the exact 1.0.0 package for your application:
|
|
67
69
|
|
|
68
|
-
| SDK | Install
|
|
70
|
+
| SDK | Install 1.0.0 | Requirements |
|
|
69
71
|
| --- | --- | --- |
|
|
70
|
-
| [Python](https://pypi.org/project/datafog-core/) | `python -m pip install datafog-core` | Python 3.10+ |
|
|
71
|
-
| [Node.js](https://www.npmjs.com/package/@datafog/node) | `npm install @datafog/node` | Node.js 24
|
|
72
|
-
| [Rust](https://crates.io/crates/datafog-core) | `cargo add datafog-core` | Rust 1.88+ |
|
|
73
|
-
| [Browser / WASM](https://www.npmjs.com/package/@datafog/wasm) | `npm install @datafog/wasm` | Browser with WebAssembly support |
|
|
72
|
+
| [Python](https://pypi.org/project/datafog-core/) | `python -m pip install datafog-core==1.0.0` | Python 3.10+ |
|
|
73
|
+
| [Node.js](https://www.npmjs.com/package/@datafog/node) | `npm install --save-exact @datafog/node@1.0.0` | Node.js 24+ |
|
|
74
|
+
| [Rust](https://crates.io/crates/datafog-core) | `cargo add datafog-core@=1.0.0` | Rust 1.88+ |
|
|
75
|
+
| [Browser / WASM](https://www.npmjs.com/package/@datafog/wasm) | `npm install --save-exact @datafog/wasm@1.0.0` | Browser with WebAssembly support |
|
|
74
76
|
|
|
75
77
|
### Python
|
|
76
78
|
|
|
@@ -169,7 +171,7 @@ Use [structured scans](docs/guides/person-discovery.mdx) to find names by JSON f
|
|
|
169
171
|
|
|
170
172
|
Each finding includes its entity type, matched text, byte and code-point ranges, optional confidence, and detector name/version. Ranges are zero-based and end-exclusive. Node.js and WASM also provide UTF-16 ranges for JavaScript `slice`. Rule-based findings have no confidence score. [Findings and ranges →](docs/concepts/findings-and-ranges.mdx)
|
|
171
173
|
|
|
172
|
-
For scanning files from AI agents, use [
|
|
174
|
+
For scanning files from AI agents, use [datafog MCP](https://github.com/DataFog/datafog-mcp).
|
|
173
175
|
|
|
174
176
|
<details>
|
|
175
177
|
<summary><strong>Detection settings</strong></summary>
|
|
@@ -198,42 +200,46 @@ Use `transform` with existing findings, or `scan_and_transform` / `scanAndTransf
|
|
|
198
200
|
|
|
199
201
|
Keep scan settings under `scan` and transformation policy under `transform`. A default strategy is required; entity selection, per-type overrides, and allowlists are optional. [Configuration guide →](docs/guides/configuration.mdx) · [Strategy details →](docs/concepts/privacy-transformations.mdx)
|
|
200
202
|
|
|
201
|
-
##
|
|
203
|
+
## Browser workspace and FogVault integration
|
|
204
|
+
|
|
205
|
+
Core 1.0 provides persistent `Workspace` pseudonyms, explicit approved saves,
|
|
206
|
+
strict/progressive restoration, operation history and conversation-scoped recall.
|
|
207
|
+
The optional browser adapter uses the maintained FogVault Worker/WASM owner and
|
|
208
|
+
IndexedDB storage. Native Rust, Node and Python owners use the same portable
|
|
209
|
+
vault core through the registered native adapter.
|
|
202
210
|
|
|
203
|
-
|
|
204
|
-
save-before-handoff, strict restoration, follow-ups and host-owned encrypted
|
|
205
|
-
backup/recovery, plus optional English GLiNER2.5 Small browser sessions. Use
|
|
206
|
-
synthetic data while the snapshot format and vault integration remain provisional.
|
|
211
|
+
After the coordinated publication completes:
|
|
207
212
|
|
|
208
213
|
```bash
|
|
209
|
-
npm install --save-exact @datafog/wasm@0.
|
|
214
|
+
npm install --save-exact @datafog/wasm@1.0.0 @datafog/fogvault-browser@0.1.0 @datafog/core-fogvault-browser@1.0.0
|
|
210
215
|
```
|
|
211
216
|
|
|
212
|
-
[Integration
|
|
217
|
+
[Integration guide and supported profiles →](docs/integrations/fogvault.md)
|
|
218
|
+
· [Stable Workspace contract →](docs/contracts/core-v1.md)
|
|
213
219
|
· [Model assets and runtime setup →](docs/browser-model.md)
|
|
214
|
-
· [
|
|
220
|
+
· [SDK release scope and separate extension acceptance →](docs/core-v1-release-scope.md)
|
|
215
221
|
|
|
216
222
|
## Local models (beta)
|
|
217
223
|
|
|
218
|
-
|
|
224
|
+
Core 1.0 includes optional local sessions for the datafog PII model and GLiNER2 in Rust, Python and Node.js. Model files are provisioned separately; model quality measurements remain separate from SDK correctness.
|
|
219
225
|
|
|
220
|
-
|
|
226
|
+
After publication completes, install the exact 1.0.0 SDK:
|
|
221
227
|
|
|
222
228
|
```bash
|
|
223
|
-
python -m pip install 'datafog-core==0.
|
|
224
|
-
npm install --save-exact @datafog/node@0.
|
|
225
|
-
cargo add datafog-core
|
|
229
|
+
python -m pip install 'datafog-core==1.0.0'
|
|
230
|
+
npm install --save-exact @datafog/node@1.0.0
|
|
231
|
+
cargo add datafog-core@=1.0.0 --features models
|
|
226
232
|
```
|
|
227
233
|
|
|
228
234
|
For GLiNER2, install its optional dependencies in the Python environment used by the runner:
|
|
229
235
|
|
|
230
236
|
```bash
|
|
231
|
-
python -m pip install 'datafog-core[gliner2]==0.
|
|
237
|
+
python -m pip install 'datafog-core[gliner2]==1.0.0'
|
|
232
238
|
```
|
|
233
239
|
|
|
234
240
|
GLiNER2 supports the PII checkpoint and compatible general-purpose span models with caller-defined labels. The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
|
|
235
241
|
|
|
236
|
-
[
|
|
242
|
+
[datafog setup and session API →](docs/model-sessions.md) · [GLiNER2 setup →](docs/gliner2-model.md)
|
|
237
243
|
|
|
238
244
|
## Documentation and development
|
|
239
245
|
|
|
@@ -261,5 +267,5 @@ See the [development guide](docs/development.mdx) for the repository layout and
|
|
|
261
267
|
|
|
262
268
|
## License
|
|
263
269
|
|
|
264
|
-
[MIT](LICENSE). Built by [
|
|
270
|
+
[MIT](LICENSE). Built by [datafog](https://datafog.ai).
|
|
265
271
|
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<a href="https://datafog.ai"><img src="docs/favicon.svg" width="72" height="72" alt="
|
|
2
|
+
<a href="https://datafog.ai"><img src="docs/favicon.svg" width="72" height="72" alt="datafog"></a>
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
|
-
<h1 align="center">
|
|
5
|
+
<h1 align="center">datafog core</h1>
|
|
6
6
|
|
|
7
7
|
<p align="center"><strong>Find and transform sensitive data in your application.</strong></p>
|
|
8
8
|
|
|
9
9
|
<p align="center">
|
|
10
|
-
<a href="https://pypi.org/project/datafog-core/"><img src="https://img.shields.io/pypi/v/datafog-core?style=flat-square&color=
|
|
11
|
-
<a href="https://www.npmjs.com/package/@datafog/node"><img src="https://img.shields.io/npm/v/%40datafog%2Fnode?style=flat-square&color=
|
|
12
|
-
<a href="https://crates.io/crates/datafog-core"><img src="https://img.shields.io/crates/v/datafog-core?style=flat-square&color=
|
|
10
|
+
<a href="https://pypi.org/project/datafog-core/"><img src="https://img.shields.io/pypi/v/datafog-core?style=flat-square&color=C0362C" alt="PyPI version"></a>
|
|
11
|
+
<a href="https://www.npmjs.com/package/@datafog/node"><img src="https://img.shields.io/npm/v/%40datafog%2Fnode?style=flat-square&color=C0362C" alt="npm version"></a>
|
|
12
|
+
<a href="https://crates.io/crates/datafog-core"><img src="https://img.shields.io/crates/v/datafog-core?style=flat-square&color=C0362C" alt="crates.io version"></a>
|
|
13
13
|
<a href="https://github.com/DataFog/datafog-core/actions/workflows/rust-quality.yml"><img src="https://github.com/DataFog/datafog-core/actions/workflows/rust-quality.yml/badge.svg?branch=main" alt="Rust CI status"></a>
|
|
14
|
-
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-
|
|
14
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-C0362C?style=flat-square" alt="MIT license"></a>
|
|
15
15
|
</p>
|
|
16
16
|
|
|
17
17
|
<p align="center">
|
|
@@ -22,22 +22,24 @@
|
|
|
22
22
|
<a href="#documentation-and-development">Documentation</a>
|
|
23
23
|
</p>
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
datafog core is a Rust library with Python, Node.js, and browser WebAssembly bindings. Scan text or JSON for personal data and credentials, then redact, mask, remove, pseudonymize, or tokenize matched values. Findings use a consistent format across the bindings.
|
|
26
26
|
|
|
27
|
-
Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add
|
|
27
|
+
Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add datafog and GLiNER2 support in Rust, Python, and Node.js.
|
|
28
28
|
|
|
29
|
-
**
|
|
29
|
+
**1.0.0 release:** [release notes](docs/releases/1-0-0.mdx) · Publication underway; the coordinated release is complete only after public-registry installation checks pass.
|
|
30
|
+
|
|
31
|
+
Core 1.0 freezes Workspace contract 1 across Rust, Python, Node and browser WASM, with optional native/browser [FogVault integrations](docs/integrations/fogvault.md). Snapshot v3 writes and v1/v2/v3 reads keep their existing semantics. Exact published [0.5.0-beta.3](docs/releases/0-5-0-beta-3.mdx) interchange remains tested; FogVault is independently versioned 0.1.0.
|
|
30
32
|
|
|
31
33
|
## Quickstart
|
|
32
34
|
|
|
33
|
-
|
|
35
|
+
Once publication completes, choose the exact 1.0.0 package for your application:
|
|
34
36
|
|
|
35
|
-
| SDK | Install
|
|
37
|
+
| SDK | Install 1.0.0 | Requirements |
|
|
36
38
|
| --- | --- | --- |
|
|
37
|
-
| [Python](https://pypi.org/project/datafog-core/) | `python -m pip install datafog-core` | Python 3.10+ |
|
|
38
|
-
| [Node.js](https://www.npmjs.com/package/@datafog/node) | `npm install @datafog/node` | Node.js 24
|
|
39
|
-
| [Rust](https://crates.io/crates/datafog-core) | `cargo add datafog-core` | Rust 1.88+ |
|
|
40
|
-
| [Browser / WASM](https://www.npmjs.com/package/@datafog/wasm) | `npm install @datafog/wasm` | Browser with WebAssembly support |
|
|
39
|
+
| [Python](https://pypi.org/project/datafog-core/) | `python -m pip install datafog-core==1.0.0` | Python 3.10+ |
|
|
40
|
+
| [Node.js](https://www.npmjs.com/package/@datafog/node) | `npm install --save-exact @datafog/node@1.0.0` | Node.js 24+ |
|
|
41
|
+
| [Rust](https://crates.io/crates/datafog-core) | `cargo add datafog-core@=1.0.0` | Rust 1.88+ |
|
|
42
|
+
| [Browser / WASM](https://www.npmjs.com/package/@datafog/wasm) | `npm install --save-exact @datafog/wasm@1.0.0` | Browser with WebAssembly support |
|
|
41
43
|
|
|
42
44
|
### Python
|
|
43
45
|
|
|
@@ -136,7 +138,7 @@ Use [structured scans](docs/guides/person-discovery.mdx) to find names by JSON f
|
|
|
136
138
|
|
|
137
139
|
Each finding includes its entity type, matched text, byte and code-point ranges, optional confidence, and detector name/version. Ranges are zero-based and end-exclusive. Node.js and WASM also provide UTF-16 ranges for JavaScript `slice`. Rule-based findings have no confidence score. [Findings and ranges →](docs/concepts/findings-and-ranges.mdx)
|
|
138
140
|
|
|
139
|
-
For scanning files from AI agents, use [
|
|
141
|
+
For scanning files from AI agents, use [datafog MCP](https://github.com/DataFog/datafog-mcp).
|
|
140
142
|
|
|
141
143
|
<details>
|
|
142
144
|
<summary><strong>Detection settings</strong></summary>
|
|
@@ -165,42 +167,46 @@ Use `transform` with existing findings, or `scan_and_transform` / `scanAndTransf
|
|
|
165
167
|
|
|
166
168
|
Keep scan settings under `scan` and transformation policy under `transform`. A default strategy is required; entity selection, per-type overrides, and allowlists are optional. [Configuration guide →](docs/guides/configuration.mdx) · [Strategy details →](docs/concepts/privacy-transformations.mdx)
|
|
167
169
|
|
|
168
|
-
##
|
|
170
|
+
## Browser workspace and FogVault integration
|
|
171
|
+
|
|
172
|
+
Core 1.0 provides persistent `Workspace` pseudonyms, explicit approved saves,
|
|
173
|
+
strict/progressive restoration, operation history and conversation-scoped recall.
|
|
174
|
+
The optional browser adapter uses the maintained FogVault Worker/WASM owner and
|
|
175
|
+
IndexedDB storage. Native Rust, Node and Python owners use the same portable
|
|
176
|
+
vault core through the registered native adapter.
|
|
169
177
|
|
|
170
|
-
|
|
171
|
-
save-before-handoff, strict restoration, follow-ups and host-owned encrypted
|
|
172
|
-
backup/recovery, plus optional English GLiNER2.5 Small browser sessions. Use
|
|
173
|
-
synthetic data while the snapshot format and vault integration remain provisional.
|
|
178
|
+
After the coordinated publication completes:
|
|
174
179
|
|
|
175
180
|
```bash
|
|
176
|
-
npm install --save-exact @datafog/wasm@0.
|
|
181
|
+
npm install --save-exact @datafog/wasm@1.0.0 @datafog/fogvault-browser@0.1.0 @datafog/core-fogvault-browser@1.0.0
|
|
177
182
|
```
|
|
178
183
|
|
|
179
|
-
[Integration
|
|
184
|
+
[Integration guide and supported profiles →](docs/integrations/fogvault.md)
|
|
185
|
+
· [Stable Workspace contract →](docs/contracts/core-v1.md)
|
|
180
186
|
· [Model assets and runtime setup →](docs/browser-model.md)
|
|
181
|
-
· [
|
|
187
|
+
· [SDK release scope and separate extension acceptance →](docs/core-v1-release-scope.md)
|
|
182
188
|
|
|
183
189
|
## Local models (beta)
|
|
184
190
|
|
|
185
|
-
|
|
191
|
+
Core 1.0 includes optional local sessions for the datafog PII model and GLiNER2 in Rust, Python and Node.js. Model files are provisioned separately; model quality measurements remain separate from SDK correctness.
|
|
186
192
|
|
|
187
|
-
|
|
193
|
+
After publication completes, install the exact 1.0.0 SDK:
|
|
188
194
|
|
|
189
195
|
```bash
|
|
190
|
-
python -m pip install 'datafog-core==0.
|
|
191
|
-
npm install --save-exact @datafog/node@0.
|
|
192
|
-
cargo add datafog-core
|
|
196
|
+
python -m pip install 'datafog-core==1.0.0'
|
|
197
|
+
npm install --save-exact @datafog/node@1.0.0
|
|
198
|
+
cargo add datafog-core@=1.0.0 --features models
|
|
193
199
|
```
|
|
194
200
|
|
|
195
201
|
For GLiNER2, install its optional dependencies in the Python environment used by the runner:
|
|
196
202
|
|
|
197
203
|
```bash
|
|
198
|
-
python -m pip install 'datafog-core[gliner2]==0.
|
|
204
|
+
python -m pip install 'datafog-core[gliner2]==1.0.0'
|
|
199
205
|
```
|
|
200
206
|
|
|
201
207
|
GLiNER2 supports the PII checkpoint and compatible general-purpose span models with caller-defined labels. The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
|
|
202
208
|
|
|
203
|
-
[
|
|
209
|
+
[datafog setup and session API →](docs/model-sessions.md) · [GLiNER2 setup →](docs/gliner2-model.md)
|
|
204
210
|
|
|
205
211
|
## Documentation and development
|
|
206
212
|
|
|
@@ -228,4 +234,4 @@ See the [development guide](docs/development.mdx) for the repository layout and
|
|
|
228
234
|
|
|
229
235
|
## License
|
|
230
236
|
|
|
231
|
-
[MIT](LICENSE). Built by [
|
|
237
|
+
[MIT](LICENSE). Built by [datafog](https://datafog.ai).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "datafog-core-python"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "1.0.0"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
rust-version = "1.88"
|
|
6
6
|
description = "Python bindings for datafog-core"
|
|
@@ -12,8 +12,10 @@ name = "datafog_core"
|
|
|
12
12
|
crate-type = ["cdylib"]
|
|
13
13
|
|
|
14
14
|
[dependencies]
|
|
15
|
-
datafog-core = { version = "0.
|
|
15
|
+
datafog-core = { version = "1.0.0", path = "../../crates/core", features = ["models"] }
|
|
16
16
|
pyo3 = { version = "0.29", features = ["abi3-py310"] }
|
|
17
17
|
pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] }
|
|
18
18
|
serde_json = "1"
|
|
19
19
|
serde = "1"
|
|
20
|
+
# Reuse Core's wipe primitive for transient copied UTF-16 transport buffers.
|
|
21
|
+
zeroize = "1"
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
use ::datafog_core as core;
|
|
2
2
|
mod models;
|
|
3
|
+
mod workspace;
|
|
3
4
|
use pyo3::create_exception;
|
|
4
5
|
use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError};
|
|
5
6
|
use pyo3::prelude::*;
|
|
@@ -1271,9 +1272,17 @@ fn scan_and_transform_structured(
|
|
|
1271
1272
|
structured_transform_result(py, result)
|
|
1272
1273
|
}
|
|
1273
1274
|
|
|
1275
|
+
/// Workspace API contract implemented by this compiled Core.
|
|
1276
|
+
#[pyfunction]
|
|
1277
|
+
fn workspace_contract_version() -> u32 {
|
|
1278
|
+
core::workspace::WORKSPACE_CONTRACT_VERSION
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1274
1281
|
#[pymodule]
|
|
1275
1282
|
fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
1276
1283
|
models::register(module)?;
|
|
1284
|
+
workspace::register(module)?;
|
|
1285
|
+
module.add_function(wrap_pyfunction!(workspace_contract_version, module)?)?;
|
|
1277
1286
|
module.add(
|
|
1278
1287
|
"DataFogConfigurationError",
|
|
1279
1288
|
module.py().get_type::<DataFogConfigurationError>(),
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
//! Exact Python transport values; conversation/admission decisions stay in Core.
|
|
2
|
+
use super::*;
|
|
3
|
+
use pyo3::types::{PyInt, PyString};
|
|
4
|
+
|
|
5
|
+
fn object<'py>(
|
|
6
|
+
py: Python<'py>,
|
|
7
|
+
value: &Bound<'py, PyAny>,
|
|
8
|
+
names: &[&str],
|
|
9
|
+
code: w::ErrorCode,
|
|
10
|
+
) -> PyResult<Bound<'py, PyDict>> {
|
|
11
|
+
let invalid = || failure(py, code);
|
|
12
|
+
let object = value.cast_exact::<PyDict>().map_err(|_| invalid())?;
|
|
13
|
+
if object.len() != names.len()
|
|
14
|
+
|| object
|
|
15
|
+
.keys()
|
|
16
|
+
.iter()
|
|
17
|
+
.any(|key| !key.is_exact_instance_of::<PyString>())
|
|
18
|
+
{
|
|
19
|
+
return Err(invalid());
|
|
20
|
+
}
|
|
21
|
+
for name in names {
|
|
22
|
+
if !object.contains(*name)? {
|
|
23
|
+
return Err(invalid());
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
Ok(object.clone())
|
|
27
|
+
}
|
|
28
|
+
fn required<'py>(
|
|
29
|
+
py: Python<'py>,
|
|
30
|
+
object: &Bound<'py, PyDict>,
|
|
31
|
+
name: &str,
|
|
32
|
+
code: w::ErrorCode,
|
|
33
|
+
) -> PyResult<Bound<'py, PyAny>> {
|
|
34
|
+
object.get_item(name)?.ok_or_else(|| failure(py, code))
|
|
35
|
+
}
|
|
36
|
+
fn public_id(
|
|
37
|
+
py: Python<'_>,
|
|
38
|
+
value: &Bound<'_, PyAny>,
|
|
39
|
+
code: w::ErrorCode,
|
|
40
|
+
) -> PyResult<w::PublicId> {
|
|
41
|
+
let invalid = || failure(py, code);
|
|
42
|
+
if let Ok(bytes) = value.cast_exact::<PyBytes>() {
|
|
43
|
+
return bytes.as_bytes().try_into().map_err(|_| invalid());
|
|
44
|
+
}
|
|
45
|
+
let values = value.cast_exact::<PyList>().map_err(|_| invalid())?;
|
|
46
|
+
if values.len() != 16 {
|
|
47
|
+
return Err(invalid());
|
|
48
|
+
}
|
|
49
|
+
let mut result = [0; 16];
|
|
50
|
+
for (index, item) in values.iter().enumerate() {
|
|
51
|
+
if !item.is_exact_instance_of::<PyInt>() {
|
|
52
|
+
return Err(invalid());
|
|
53
|
+
}
|
|
54
|
+
result[index] = item.extract::<u8>().map_err(|_| invalid())?;
|
|
55
|
+
}
|
|
56
|
+
Ok(result)
|
|
57
|
+
}
|
|
58
|
+
// Direct restoration borrows the caller string; saved context fields clone explicitly.
|
|
59
|
+
pub(super) fn text<'a>(
|
|
60
|
+
py: Python<'_>,
|
|
61
|
+
value: &'a Bound<'_, PyAny>,
|
|
62
|
+
code: w::ErrorCode,
|
|
63
|
+
) -> PyResult<&'a str> {
|
|
64
|
+
let value = value
|
|
65
|
+
.cast_exact::<PyString>()
|
|
66
|
+
.map_err(|_| failure(py, code))?;
|
|
67
|
+
let text = value.to_str().map_err(|_| failure(py, code))?;
|
|
68
|
+
material_limit(py, text.len())?;
|
|
69
|
+
Ok(text)
|
|
70
|
+
}
|
|
71
|
+
pub(super) fn selector(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<w::PublicId> {
|
|
72
|
+
let bytes = value
|
|
73
|
+
.cast_exact::<PyBytes>()
|
|
74
|
+
.map_err(|_| failure(py, w::ErrorCode::InvalidId))?;
|
|
75
|
+
id(py, bytes.as_bytes())
|
|
76
|
+
}
|
|
77
|
+
pub(super) fn intent(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<w::SendIntentId> {
|
|
78
|
+
let code = w::ErrorCode::InvalidSendIntent;
|
|
79
|
+
let object = object(py, value, &["preparationId", "inputIndex"], code)?;
|
|
80
|
+
let preparation_id = public_id(py, &required(py, &object, "preparationId", code)?, code)?;
|
|
81
|
+
let index = required(py, &object, "inputIndex", code)?;
|
|
82
|
+
if !index.is_exact_instance_of::<PyInt>() {
|
|
83
|
+
return Err(failure(py, code));
|
|
84
|
+
}
|
|
85
|
+
let input_index = index.extract::<usize>().map_err(|_| failure(py, code))?;
|
|
86
|
+
if input_index >= w::MAX_INPUTS {
|
|
87
|
+
return Err(failure(py, code));
|
|
88
|
+
}
|
|
89
|
+
Ok(w::SendIntentId {
|
|
90
|
+
preparation_id,
|
|
91
|
+
input_index,
|
|
92
|
+
})
|
|
93
|
+
}
|
|
94
|
+
pub(super) fn inputs(
|
|
95
|
+
py: Python<'_>,
|
|
96
|
+
value: &Bound<'_, PyAny>,
|
|
97
|
+
) -> PyResult<Vec<(ReviewOwned, w::Conversation)>> {
|
|
98
|
+
let code = w::ErrorCode::InvalidConversation;
|
|
99
|
+
let values = value
|
|
100
|
+
.cast_exact::<PyList>()
|
|
101
|
+
.map_err(|_| failure(py, w::ErrorCode::InvalidBatch))?;
|
|
102
|
+
if values.len() > w::MAX_INPUTS {
|
|
103
|
+
return Err(failure(py, w::ErrorCode::ResourceLimit));
|
|
104
|
+
}
|
|
105
|
+
let mut result = Vec::with_capacity(values.len());
|
|
106
|
+
let mut bytes = 0usize;
|
|
107
|
+
for (index, value) in values.iter().enumerate() {
|
|
108
|
+
let row = (|| -> PyResult<(ReviewOwned, w::Conversation)> {
|
|
109
|
+
let item = object(py, &value, &["input", "conversation"], code)?;
|
|
110
|
+
let input = required(py, &item, "input", code)?;
|
|
111
|
+
let input = input
|
|
112
|
+
.extract::<PyRef<'_, WorkspaceReviewInput>>()
|
|
113
|
+
.map_err(|_| failure(py, code))?;
|
|
114
|
+
bytes = bytes.saturating_add(input.text.len());
|
|
115
|
+
for finding in input.scanner.iter().chain(&input.user) {
|
|
116
|
+
bytes = bytes.saturating_add(finding_bytes(finding));
|
|
117
|
+
}
|
|
118
|
+
for correction in &input.corrections {
|
|
119
|
+
bytes = bytes
|
|
120
|
+
.saturating_add(finding_bytes(&correction.replacement))
|
|
121
|
+
.saturating_add(correction.candidate_key.len());
|
|
122
|
+
}
|
|
123
|
+
material_limit(py, bytes)?;
|
|
124
|
+
let context = object(
|
|
125
|
+
py,
|
|
126
|
+
&required(py, &item, "conversation", code)?,
|
|
127
|
+
&["id", "surface", "siteId", "siteIdByteBudget"],
|
|
128
|
+
code,
|
|
129
|
+
)?;
|
|
130
|
+
let id = public_id(py, &required(py, &context, "id", code)?, code)?;
|
|
131
|
+
let surface = text(py, &required(py, &context, "surface", code)?, code)?.to_owned();
|
|
132
|
+
let site = required(py, &context, "siteId", code)?;
|
|
133
|
+
let site_id = if site.is_none() {
|
|
134
|
+
None
|
|
135
|
+
} else {
|
|
136
|
+
Some(text(py, &site, code)?.to_owned())
|
|
137
|
+
};
|
|
138
|
+
bytes = bytes
|
|
139
|
+
.saturating_add(surface.len())
|
|
140
|
+
.saturating_add(site_id.as_ref().map_or(0, String::len));
|
|
141
|
+
material_limit(py, bytes)?;
|
|
142
|
+
let budget = required(py, &context, "siteIdByteBudget", code)?;
|
|
143
|
+
if !budget.is_exact_instance_of::<PyInt>() {
|
|
144
|
+
return Err(failure(py, code));
|
|
145
|
+
}
|
|
146
|
+
let site_id_byte_budget = budget.extract::<usize>().map_err(|_| failure(py, code))?;
|
|
147
|
+
Ok((
|
|
148
|
+
input.owned(),
|
|
149
|
+
w::Conversation {
|
|
150
|
+
id,
|
|
151
|
+
surface,
|
|
152
|
+
site_id,
|
|
153
|
+
site_id_byte_budget,
|
|
154
|
+
},
|
|
155
|
+
))
|
|
156
|
+
})();
|
|
157
|
+
match row {
|
|
158
|
+
Ok(row) => result.push(row),
|
|
159
|
+
Err(error) => {
|
|
160
|
+
error.value(py).setattr("input_index", index)?;
|
|
161
|
+
return Err(error);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
Ok(result)
|
|
166
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
//! Transport-only adapters for the shared content-free operation contract.
|
|
2
|
+
use super::*;
|
|
3
|
+
use pyo3::types::{PyInt, PyString};
|
|
4
|
+
|
|
5
|
+
pub(super) fn context(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<w::OperationContext> {
|
|
6
|
+
let invalid = || failure(py, w::ErrorCode::InvalidOperation);
|
|
7
|
+
let object = value.cast_exact::<PyDict>().map_err(|_| invalid())?;
|
|
8
|
+
if object.len() != 2
|
|
9
|
+
|| object
|
|
10
|
+
.keys()
|
|
11
|
+
.iter()
|
|
12
|
+
.any(|key| !key.is_exact_instance_of::<PyString>())
|
|
13
|
+
{
|
|
14
|
+
return Err(invalid());
|
|
15
|
+
}
|
|
16
|
+
let bytes = object.get_item("id")?.ok_or_else(invalid)?;
|
|
17
|
+
let bytes = bytes
|
|
18
|
+
.cast_exact::<PyBytes>()
|
|
19
|
+
.map_err(|_| failure(py, w::ErrorCode::InvalidId))?;
|
|
20
|
+
let operation_id = id(py, bytes.as_bytes())?;
|
|
21
|
+
let time = object.get_item("occurredAtMs")?.ok_or_else(invalid)?;
|
|
22
|
+
if !time.is_exact_instance_of::<PyInt>() {
|
|
23
|
+
return Err(invalid());
|
|
24
|
+
}
|
|
25
|
+
let occurred_at_ms = time.extract::<u64>().map_err(|_| invalid())?;
|
|
26
|
+
if occurred_at_ms > w::MAX_OPERATION_TIME_MS {
|
|
27
|
+
return Err(invalid());
|
|
28
|
+
}
|
|
29
|
+
Ok(w::OperationContext {
|
|
30
|
+
id: operation_id,
|
|
31
|
+
occurred_at_ms,
|
|
32
|
+
})
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
pub(super) fn outcome(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<w::OperationOutcome> {
|
|
36
|
+
let invalid = || failure(py, w::ErrorCode::InvalidOperation);
|
|
37
|
+
let object = value.cast_exact::<PyDict>().map_err(|_| invalid())?;
|
|
38
|
+
if object.len() != 2
|
|
39
|
+
|| object
|
|
40
|
+
.keys()
|
|
41
|
+
.iter()
|
|
42
|
+
.any(|key| !key.is_exact_instance_of::<PyString>())
|
|
43
|
+
{
|
|
44
|
+
return Err(invalid());
|
|
45
|
+
}
|
|
46
|
+
let kind = object.get_item("kind")?.ok_or_else(invalid)?;
|
|
47
|
+
let kind = kind.cast_exact::<PyString>().map_err(|_| invalid())?;
|
|
48
|
+
let kind = match kind.to_str().map_err(|_| invalid())? {
|
|
49
|
+
"copy_confirmed" => w::OutcomeKind::CopyConfirmed,
|
|
50
|
+
"user_cancelled" => w::OutcomeKind::UserCancelled,
|
|
51
|
+
"blocked_input" => w::OutcomeKind::BlockedInput,
|
|
52
|
+
"restore_blocked" => w::OutcomeKind::RestoreBlocked,
|
|
53
|
+
_ => return Err(invalid()),
|
|
54
|
+
};
|
|
55
|
+
let count = object.get_item("issueCount")?.ok_or_else(invalid)?;
|
|
56
|
+
if !count.is_exact_instance_of::<PyInt>() {
|
|
57
|
+
return Err(invalid());
|
|
58
|
+
}
|
|
59
|
+
let issue_count = count.extract::<usize>().map_err(|_| invalid())?;
|
|
60
|
+
if issue_count > w::MAX_FINDINGS {
|
|
61
|
+
return Err(invalid());
|
|
62
|
+
}
|
|
63
|
+
Ok(w::OperationOutcome { kind, issue_count })
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
pub(super) fn record(value: &w::OperationRecord) -> Value {
|
|
67
|
+
json!({"id":value.id,"kind":value.kind.as_str(),"occurredAtMs":value.occurred_at_ms,
|
|
68
|
+
"preparedInputs":value.prepared_inputs,"transformedSpans":value.transformed_spans})
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
pub(super) fn counts(value: w::OperationCounts) -> Value {
|
|
72
|
+
json!({"operations":value.operations,"preparedInputs":value.prepared_inputs,
|
|
73
|
+
"transformedSpans":value.transformed_spans,"confirmedHandoffs":value.confirmed_handoffs,
|
|
74
|
+
"reportedBlockedAttempts":value.reported_blocked_attempts})
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Bounds stay Core-owned; reject Python coercion before entering the native call.
|
|
78
|
+
pub(super) fn page_index(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<usize> {
|
|
79
|
+
let invalid = || failure(py, w::ErrorCode::ResourceLimit);
|
|
80
|
+
if !value.is_exact_instance_of::<PyInt>() {
|
|
81
|
+
return Err(invalid());
|
|
82
|
+
}
|
|
83
|
+
value.extract::<usize>().map_err(|_| invalid())
|
|
84
|
+
}
|