datafog-core 0.4.4__tar.gz → 0.4.5b2__tar.gz

{datafog_core-0.4.4 → datafog_core-0.4.5b2}/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.4.4"
+version = "0.4.5-beta.2"
 dependencies = [
  "base64",
  "futures",
@@ -94,17 +94,18 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.4.4"
+version = "0.4.5-beta.2"
 dependencies = [
  "datafog-core",
  "pyo3",
  "pyo3-async-runtimes",
+ "serde",
  "serde_json",
 ]
 
 [[package]]
 name = "datafog-node"
-version = "0.4.4"
+version = "0.4.5-beta.2"
 dependencies = [
  "datafog-core",
  "napi",
@@ -115,7 +116,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.4.4"
+version = "0.4.5-beta.2"
 dependencies = [
  "datafog-core",
  "serde",

datafog_core-0.4.5b2/PKG-INFO

@@ -0,0 +1,246 @@
+Metadata-Version: 2.4
+Name: datafog-core
+Version: 0.4.5b2
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Rust
+Requires-Dist: gliner2[local]==2.1.0 ; extra == 'gliner2'
+Requires-Dist: transformers==5.8.0 ; (platform_machine != 'x86_64' and extra == 'gliner2') or (sys_platform != 'darwin' and extra == 'gliner2')
+Requires-Dist: torch==2.12.0 ; (platform_machine != 'x86_64' and extra == 'gliner2') or (sys_platform != 'darwin' and extra == 'gliner2')
+Requires-Dist: peft==0.21.2 ; (platform_machine != 'x86_64' and extra == 'gliner2') or (sys_platform != 'darwin' and extra == 'gliner2')
+Requires-Dist: transformers==4.57.6 ; platform_machine == 'x86_64' and sys_platform == 'darwin' and extra == 'gliner2'
+Requires-Dist: torch==2.2.2 ; platform_machine == 'x86_64' and sys_platform == 'darwin' and extra == 'gliner2'
+Requires-Dist: protobuf==6.33.6 ; platform_machine == 'x86_64' and sys_platform == 'darwin' and extra == 'gliner2'
+Requires-Dist: numpy==1.26.4 ; platform_machine == 'x86_64' and sys_platform == 'darwin' and extra == 'gliner2'
+Requires-Dist: peft==0.18.1 ; platform_machine == 'x86_64' and sys_platform == 'darwin' and extra == 'gliner2'
+Provides-Extra: gliner2
+Summary: Fast structured PII detection implemented in Rust
+Keywords: pii,privacy,detection,rust
+Author: DataFog
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/DataFog/datafog-core
+Project-URL: Issues, https://github.com/DataFog/datafog-core/issues
+Project-URL: Repository, https://github.com/DataFog/datafog-core
+
+<p align="center">
+  <a href="https://datafog.ai"><img src="docs/favicon.svg" width="72" height="72" alt="DataFog"></a>
+</p>
+
+<h1 align="center">DataFog Core</h1>
+
+<p align="center"><strong>Find and transform sensitive data in your application.</strong></p>
+
+<p align="center">
+  <a href="https://pypi.org/project/datafog-core/"><img src="https://img.shields.io/pypi/v/datafog-core?style=flat-square&amp;color=E8A84C" alt="PyPI version"></a>
+  <a href="https://www.npmjs.com/package/@datafog/node"><img src="https://img.shields.io/npm/v/%40datafog%2Fnode?style=flat-square&amp;color=E8A84C" alt="npm version"></a>
+  <a href="https://crates.io/crates/datafog-core"><img src="https://img.shields.io/crates/v/datafog-core?style=flat-square&amp;color=E8A84C" alt="crates.io version"></a>
+  <a href="https://github.com/DataFog/datafog-core/actions/workflows/rust-quality.yml"><img src="https://github.com/DataFog/datafog-core/actions/workflows/rust-quality.yml/badge.svg?branch=main" alt="Rust CI status"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-E8A84C?style=flat-square" alt="MIT license"></a>
+</p>
+
+<p align="center">
+  <a href="#quickstart">Quickstart</a> ·
+  <a href="#detection-coverage">Detection</a> ·
+  <a href="#transformations">Transformations</a> ·
+  <a href="#local-models-beta">Local models</a> ·
+  <a href="#documentation-and-development">Documentation</a>
+</p>
+
+DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bindings. Scan text or JSON for personal data and credentials, then redact, mask, remove, pseudonymize, or tokenize matched values. Findings use a consistent format across the bindings.
+
+Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
+
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1)
+
+## Quickstart
+
+Choose the package for your application:
+
+| SDK | Install stable | Requirements |
+| --- | --- | --- |
+| [Python](https://pypi.org/project/datafog-core/) | `python -m pip install datafog-core` | Python 3.10+ |
+| [Node.js](https://www.npmjs.com/package/@datafog/node) | `npm install @datafog/node` | Node.js 24.x |
+| [Rust](https://crates.io/crates/datafog-core) | `cargo add datafog-core` | Rust 1.88+ |
+| [Browser / WASM](https://www.npmjs.com/package/@datafog/wasm) | `npm install @datafog/wasm` | Browser with WebAssembly support |
+
+### Python
+
+```python
+from datafog_core import scan, scan_and_transform
+
+findings = scan("Email jane@example.com")
+print(findings[0].entity_type)   # EMAIL
+print(findings[0].matched_text)  # jane@example.com
+
+result = scan_and_transform(
+    "Email jane@example.com",
+    {"transform": {"default": {"strategy": "redact"}}},
+)
+print(result.text)  # Email [EMAIL]
+```
+
+<details>
+<summary><strong>Node.js example</strong></summary>
+
+```js
+import { scan, scanAndTransform } from "@datafog/node";
+
+const findings = scan("Email jane@example.com");
+console.log(findings[0].entityType); // EMAIL
+console.log(findings[0].matchedText); // jane@example.com
+
+const result = scanAndTransform("Email jane@example.com", {
+  transform: { default: { strategy: "redact" } },
+});
+console.log(result.text); // Email [EMAIL]
+```
+
+</details>
+
+<details>
+<summary><strong>Rust example</strong></summary>
+
+```rust
+use datafog_core::{
+    scan, scan_and_transform, PrivacyError, ScanAndTransformConfig,
+    TransformationConfig, TransformationStrategy,
+};
+
+fn main() -> Result<(), PrivacyError> {
+    let findings = scan("Email jane@example.com");
+    println!("{}", findings[0].entity_type); // EMAIL
+    println!("{}", findings[0].matched_text); // jane@example.com
+
+    let result = scan_and_transform(
+        "Email jane@example.com",
+        &ScanAndTransformConfig::new(TransformationConfig::new(
+            TransformationStrategy::Redact,
+        )),
+    )?;
+    println!("{}", result.text); // Email [EMAIL]
+    Ok(())
+}
+```
+
+</details>
+
+<details>
+<summary><strong>Browser / WASM example</strong></summary>
+
+```js
+import { init, scan, scanAndTransform } from "@datafog/wasm";
+
+await init();
+console.log(scan("Email jane@example.com"));
+
+const result = scanAndTransform("Email jane@example.com", {
+  transform: { default: { strategy: "redact" } },
+});
+console.log(result.text); // Email [EMAIL]
+```
+
+</details>
+
+Python and Node.js packages include prebuilt native binaries for Linux x64 and ARM64 (GNU), macOS Intel and Apple Silicon, and Windows x64 (MSVC).
+
+## Detection coverage
+
+| Coverage | Entity types | Activation |
+| --- | --- | --- |
+| Personal and financial data | `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, `ZIP_CODE` | On by default |
+| Credentials | [`API_KEY`](docs/reference/api-keys.mdx), [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), [`JWT`](docs/reference/jwt.mdx), [`PRIVATE_KEY`](docs/reference/private-keys.mdx), [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx) | On by default |
+| Labeled US identifiers | [`US_ROUTING_NUMBER`](docs/reference/us-routing-number.mdx), [`NPI`](docs/reference/npi.mdx) | On by default; requires a supported text label |
+| German identifiers | IBAN, VAT ID, tax ID, social security number, postal code, passport number, residence permit number | Explicit [German locale](docs/reference/german-entities.mdx) |
+| UUIDs | [`UUID`](docs/reference/uuid.mdx) | Explicit `detect_uuid` setting |
+| Names in JSON fields | `PERSON` | [Structured scans](docs/guides/person-discovery.mdx), using name-field aliases or explicit JSON Pointer mappings |
+
+Use [structured scans](docs/guides/person-discovery.mdx) to find names by JSON field, or [local models](#local-models-beta) to detect names and street addresses in prose.
+
+Each finding includes its entity type, matched text, byte and code-point ranges, optional confidence, and detector name/version. Ranges are zero-based and end-exclusive. Node.js and WASM also provide UTF-16 ranges for JavaScript `slice`. Rule-based findings have no confidence score. [Findings and ranges →](docs/concepts/findings-and-ranges.mdx)
+
+For scanning files from AI agents, use [DataFog MCP](https://github.com/DataFog/datafog-mcp).
+
+<details>
+<summary><strong>Detection settings</strong></summary>
+
+- **Select checks:** `scan(text, {"entities": ["EMAIL", "SSN"]})` in Python runs only the selected text detectors. Omit `entities` to use all enabled detectors. Selections must be nonempty, unique, exact entity names. Selecting a German type or UUID does not activate it; set `locale` or `detect_uuid` too.
+- **German locale:** `{"locale": "de"}` enables the seven German detectors. These use format/context rules; IBAN checksums and account existence are not verified. See the [German reference](docs/reference/german-entities.mdx) for detector rules.
+- **ENV and SQL:** choose `format: "env"` or `format: "sql"` to handle email boundaries in assignments and quoted strings. [Source-context rules →](docs/guides/email-boundaries.mdx)
+- **Phone numbers:** numbers without `+` use North American checks: ten digits, optionally preceded by `1`, with area and exchange codes starting with `2`–`9`. Numbers starting with `+` use a permissive 7–15 digit check, including shortened examples such as `+1-555-1002`. Country codes and assigned numbers are not validated; numeric identifiers can still match.
+- **Decimal numbers and CSV:** contiguous card digits after a digit and `.` or `,` are suppressed, so `3.4111111111111111` and `3,4111111111111111` do not produce card findings. Cards containing spaces or hyphens still undergo normal checks. Parse CSV/TSV into cells before scanning so delimiters and adjacent fields do not affect matching.
+
+Use [`capabilities()`](docs/reference/capabilities.mdx) in Rust or Python to discover supported entities and their activation rules.
+
+</details>
+
+## Transformations
+
+| Strategy | Result | Application provider | Browser / WASM |
+| --- | --- | --- | --- |
+| `redact` | Replace with a label such as `[EMAIL]` | None | Supported |
+| `mask` | Replace characters, optionally revealing the first or last characters | None | Supported |
+| `remove` | Delete the matched span | None | Supported |
+| [`pseudonymize`](docs/guides/pseudonymization.mdx) | Create a stable keyed HMAC-SHA-256 pseudonym | Key provider | Unsupported |
+| [`tokenize`](docs/guides/tokenization-and-restoration.mdx) | Issue an opaque token that can be restored under the same scope | Token provider | Unsupported |
+
+Use `transform` with existing findings, or `scan_and_transform` / `scanAndTransform` to scan and transform together. Transformation records include source/output ranges and replacement metadata, without the original matched text.
+
+Keep scan settings under `scan` and transformation policy under `transform`. A default strategy is required; entity selection, per-type overrides, and allowlists are optional. [Configuration guide →](docs/guides/configuration.mdx) · [Strategy details →](docs/concepts/privacy-transformations.mdx)
+
+## Local models (beta)
+
+[0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
+
+Install the beta for your runtime:
+
+```bash
+python -m pip install 'datafog-core==0.4.5b1'
+npm install @datafog/node@beta
+cargo add datafog-core@0.4.5-beta.1 --features models
+```
+
+For GLiNER2, install its optional dependencies in the Python environment used by the runner:
+
+```bash
+python -m pip install 'datafog-core[gliner2]==0.4.5b1'
+```
+
+The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
+
+[DataFog setup and session API →](docs/model-sessions.md) · [GLiNER2 setup →](docs/gliner2-model.md)
+
+## Documentation and development
+
+- [Installation and runtime requirements](docs/get-started/installation.mdx)
+- [Python](docs/reference/python.mdx) · [Node.js](docs/reference/node.mdx) · [Rust](docs/reference/rust.mdx) · [Browser / WASM](docs/reference/browser-wasm.mdx) API references
+- [Structured JSON scans and name-field discovery](docs/guides/person-discovery.mdx)
+- [Configuration and allowlists](docs/guides/configuration.mdx)
+- [Errors](docs/reference/errors.mdx) and [0.4.x compatibility](docs/reference/compatibility.mdx)
+- [Build, test, and contribute](docs/development.mdx)
+- [Release notes](https://github.com/DataFog/datafog-core/releases)
+
+For existing `datafog` Python applications, follow the [migration guide](docs/guides/migrating-from-datafog-python.mdx) for changes to imports, results, and transformations.
+
+For Rust development, run:
+
+```bash
+cargo fmt --all --check
+cargo clippy --workspace --all-targets --all-features -- -D warnings
+cargo test --workspace --all-features
+```
+
+See the [development guide](docs/development.mdx) for the repository layout and Python, Node.js, and WASM package checks.
+
+## License
+
+[MIT](LICENSE). Built by [DataFog](https://datafog.ai).
+

datafog_core-0.4.5b2/README.md

@@ -0,0 +1,212 @@
+<p align="center">
+  <a href="https://datafog.ai"><img src="docs/favicon.svg" width="72" height="72" alt="DataFog"></a>
+</p>
+
+<h1 align="center">DataFog Core</h1>
+
+<p align="center"><strong>Find and transform sensitive data in your application.</strong></p>
+
+<p align="center">
+  <a href="https://pypi.org/project/datafog-core/"><img src="https://img.shields.io/pypi/v/datafog-core?style=flat-square&amp;color=E8A84C" alt="PyPI version"></a>
+  <a href="https://www.npmjs.com/package/@datafog/node"><img src="https://img.shields.io/npm/v/%40datafog%2Fnode?style=flat-square&amp;color=E8A84C" alt="npm version"></a>
+  <a href="https://crates.io/crates/datafog-core"><img src="https://img.shields.io/crates/v/datafog-core?style=flat-square&amp;color=E8A84C" alt="crates.io version"></a>
+  <a href="https://github.com/DataFog/datafog-core/actions/workflows/rust-quality.yml"><img src="https://github.com/DataFog/datafog-core/actions/workflows/rust-quality.yml/badge.svg?branch=main" alt="Rust CI status"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-E8A84C?style=flat-square" alt="MIT license"></a>
+</p>
+
+<p align="center">
+  <a href="#quickstart">Quickstart</a> ·
+  <a href="#detection-coverage">Detection</a> ·
+  <a href="#transformations">Transformations</a> ·
+  <a href="#local-models-beta">Local models</a> ·
+  <a href="#documentation-and-development">Documentation</a>
+</p>
+
+DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bindings. Scan text or JSON for personal data and credentials, then redact, mask, remove, pseudonymize, or tokenize matched values. Findings use a consistent format across the bindings.
+
+Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
+
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1)
+
+## Quickstart
+
+Choose the package for your application:
+
+| SDK | Install stable | Requirements |
+| --- | --- | --- |
+| [Python](https://pypi.org/project/datafog-core/) | `python -m pip install datafog-core` | Python 3.10+ |
+| [Node.js](https://www.npmjs.com/package/@datafog/node) | `npm install @datafog/node` | Node.js 24.x |
+| [Rust](https://crates.io/crates/datafog-core) | `cargo add datafog-core` | Rust 1.88+ |
+| [Browser / WASM](https://www.npmjs.com/package/@datafog/wasm) | `npm install @datafog/wasm` | Browser with WebAssembly support |
+
+### Python
+
+```python
+from datafog_core import scan, scan_and_transform
+
+findings = scan("Email jane@example.com")
+print(findings[0].entity_type)   # EMAIL
+print(findings[0].matched_text)  # jane@example.com
+
+result = scan_and_transform(
+    "Email jane@example.com",
+    {"transform": {"default": {"strategy": "redact"}}},
+)
+print(result.text)  # Email [EMAIL]
+```
+
+<details>
+<summary><strong>Node.js example</strong></summary>
+
+```js
+import { scan, scanAndTransform } from "@datafog/node";
+
+const findings = scan("Email jane@example.com");
+console.log(findings[0].entityType); // EMAIL
+console.log(findings[0].matchedText); // jane@example.com
+
+const result = scanAndTransform("Email jane@example.com", {
+  transform: { default: { strategy: "redact" } },
+});
+console.log(result.text); // Email [EMAIL]
+```
+
+</details>
+
+<details>
+<summary><strong>Rust example</strong></summary>
+
+```rust
+use datafog_core::{
+    scan, scan_and_transform, PrivacyError, ScanAndTransformConfig,
+    TransformationConfig, TransformationStrategy,
+};
+
+fn main() -> Result<(), PrivacyError> {
+    let findings = scan("Email jane@example.com");
+    println!("{}", findings[0].entity_type); // EMAIL
+    println!("{}", findings[0].matched_text); // jane@example.com
+
+    let result = scan_and_transform(
+        "Email jane@example.com",
+        &ScanAndTransformConfig::new(TransformationConfig::new(
+            TransformationStrategy::Redact,
+        )),
+    )?;
+    println!("{}", result.text); // Email [EMAIL]
+    Ok(())
+}
+```
+
+</details>
+
+<details>
+<summary><strong>Browser / WASM example</strong></summary>
+
+```js
+import { init, scan, scanAndTransform } from "@datafog/wasm";
+
+await init();
+console.log(scan("Email jane@example.com"));
+
+const result = scanAndTransform("Email jane@example.com", {
+  transform: { default: { strategy: "redact" } },
+});
+console.log(result.text); // Email [EMAIL]
+```
+
+</details>
+
+Python and Node.js packages include prebuilt native binaries for Linux x64 and ARM64 (GNU), macOS Intel and Apple Silicon, and Windows x64 (MSVC).
+
+## Detection coverage
+
+| Coverage | Entity types | Activation |
+| --- | --- | --- |
+| Personal and financial data | `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, `ZIP_CODE` | On by default |
+| Credentials | [`API_KEY`](docs/reference/api-keys.mdx), [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), [`JWT`](docs/reference/jwt.mdx), [`PRIVATE_KEY`](docs/reference/private-keys.mdx), [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx) | On by default |
+| Labeled US identifiers | [`US_ROUTING_NUMBER`](docs/reference/us-routing-number.mdx), [`NPI`](docs/reference/npi.mdx) | On by default; requires a supported text label |
+| German identifiers | IBAN, VAT ID, tax ID, social security number, postal code, passport number, residence permit number | Explicit [German locale](docs/reference/german-entities.mdx) |
+| UUIDs | [`UUID`](docs/reference/uuid.mdx) | Explicit `detect_uuid` setting |
+| Names in JSON fields | `PERSON` | [Structured scans](docs/guides/person-discovery.mdx), using name-field aliases or explicit JSON Pointer mappings |
+
+Use [structured scans](docs/guides/person-discovery.mdx) to find names by JSON field, or [local models](#local-models-beta) to detect names and street addresses in prose.
+
+Each finding includes its entity type, matched text, byte and code-point ranges, optional confidence, and detector name/version. Ranges are zero-based and end-exclusive. Node.js and WASM also provide UTF-16 ranges for JavaScript `slice`. Rule-based findings have no confidence score. [Findings and ranges →](docs/concepts/findings-and-ranges.mdx)
+
+For scanning files from AI agents, use [DataFog MCP](https://github.com/DataFog/datafog-mcp).
+
+<details>
+<summary><strong>Detection settings</strong></summary>
+
+- **Select checks:** `scan(text, {"entities": ["EMAIL", "SSN"]})` in Python runs only the selected text detectors. Omit `entities` to use all enabled detectors. Selections must be nonempty, unique, exact entity names. Selecting a German type or UUID does not activate it; set `locale` or `detect_uuid` too.
+- **German locale:** `{"locale": "de"}` enables the seven German detectors. These use format/context rules; IBAN checksums and account existence are not verified. See the [German reference](docs/reference/german-entities.mdx) for detector rules.
+- **ENV and SQL:** choose `format: "env"` or `format: "sql"` to handle email boundaries in assignments and quoted strings. [Source-context rules →](docs/guides/email-boundaries.mdx)
+- **Phone numbers:** numbers without `+` use North American checks: ten digits, optionally preceded by `1`, with area and exchange codes starting with `2`–`9`. Numbers starting with `+` use a permissive 7–15 digit check, including shortened examples such as `+1-555-1002`. Country codes and assigned numbers are not validated; numeric identifiers can still match.
+- **Decimal numbers and CSV:** contiguous card digits after a digit and `.` or `,` are suppressed, so `3.4111111111111111` and `3,4111111111111111` do not produce card findings. Cards containing spaces or hyphens still undergo normal checks. Parse CSV/TSV into cells before scanning so delimiters and adjacent fields do not affect matching.
+
+Use [`capabilities()`](docs/reference/capabilities.mdx) in Rust or Python to discover supported entities and their activation rules.
+
+</details>
+
+## Transformations
+
+| Strategy | Result | Application provider | Browser / WASM |
+| --- | --- | --- | --- |
+| `redact` | Replace with a label such as `[EMAIL]` | None | Supported |
+| `mask` | Replace characters, optionally revealing the first or last characters | None | Supported |
+| `remove` | Delete the matched span | None | Supported |
+| [`pseudonymize`](docs/guides/pseudonymization.mdx) | Create a stable keyed HMAC-SHA-256 pseudonym | Key provider | Unsupported |
+| [`tokenize`](docs/guides/tokenization-and-restoration.mdx) | Issue an opaque token that can be restored under the same scope | Token provider | Unsupported |
+
+Use `transform` with existing findings, or `scan_and_transform` / `scanAndTransform` to scan and transform together. Transformation records include source/output ranges and replacement metadata, without the original matched text.
+
+Keep scan settings under `scan` and transformation policy under `transform`. A default strategy is required; entity selection, per-type overrides, and allowlists are optional. [Configuration guide →](docs/guides/configuration.mdx) · [Strategy details →](docs/concepts/privacy-transformations.mdx)
+
+## Local models (beta)
+
+[0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
+
+Install the beta for your runtime:
+
+```bash
+python -m pip install 'datafog-core==0.4.5b1'
+npm install @datafog/node@beta
+cargo add datafog-core@0.4.5-beta.1 --features models
+```
+
+For GLiNER2, install its optional dependencies in the Python environment used by the runner:
+
+```bash
+python -m pip install 'datafog-core[gliner2]==0.4.5b1'
+```
+
+The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
+
+[DataFog setup and session API →](docs/model-sessions.md) · [GLiNER2 setup →](docs/gliner2-model.md)
+
+## Documentation and development
+
+- [Installation and runtime requirements](docs/get-started/installation.mdx)
+- [Python](docs/reference/python.mdx) · [Node.js](docs/reference/node.mdx) · [Rust](docs/reference/rust.mdx) · [Browser / WASM](docs/reference/browser-wasm.mdx) API references
+- [Structured JSON scans and name-field discovery](docs/guides/person-discovery.mdx)
+- [Configuration and allowlists](docs/guides/configuration.mdx)
+- [Errors](docs/reference/errors.mdx) and [0.4.x compatibility](docs/reference/compatibility.mdx)
+- [Build, test, and contribute](docs/development.mdx)
+- [Release notes](https://github.com/DataFog/datafog-core/releases)
+
+For existing `datafog` Python applications, follow the [migration guide](docs/guides/migrating-from-datafog-python.mdx) for changes to imports, results, and transformations.
+
+For Rust development, run:
+
+```bash
+cargo fmt --all --check
+cargo clippy --workspace --all-targets --all-features -- -D warnings
+cargo test --workspace --all-features
+```
+
+See the [development guide](docs/development.mdx) for the repository layout and Python, Node.js, and WASM package checks.
+
+## License
+
+[MIT](LICENSE). Built by [DataFog](https://datafog.ai).

{datafog_core-0.4.4 → datafog_core-0.4.5b2}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.4.4"
+version = "0.4.5-beta.2"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"
@@ -12,7 +12,8 @@ name = "datafog_core"
 crate-type = ["cdylib"]
 
 [dependencies]
-datafog-core = { path = "../../crates/core" }
+datafog-core = { version = "0.4.5-beta.2", path = "../../crates/core", features = ["models"] }
 pyo3 = { version = "0.29", features = ["abi3-py310"] }
 pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] }
 serde_json = "1"
+serde = "1"

{datafog_core-0.4.4 → datafog_core-0.4.5b2}/bindings/python/src/lib.rs

@@ -1,4 +1,5 @@
 use ::datafog_core as core;
+mod models;
 use pyo3::create_exception;
 use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError};
 use pyo3::prelude::*;
@@ -974,13 +975,11 @@ fn scan(py: Python<'_>, text: &str, config: Option<&Bound<'_, PyAny>>) -> PyResu
     } else {
         core::ScanConfig::default()
     };
-    std::panic::catch_unwind(|| {
-        core::scan_with_config(text, &config)
-            .into_iter()
-            .map(Finding::from)
-            .collect()
-    })
-    .map_err(|_| internal_error(py, "unexpected Rust scan failure"))
+    // Python strings are immutable, so other threads may run during the scan.
+    let findings = py
+        .detach(|| std::panic::catch_unwind(|| core::scan_with_config(text, &config)))
+        .map_err(|_| internal_error(py, "unexpected Rust scan failure"))?;
+    Ok(findings.into_iter().map(Finding::from).collect())
 }
 
 /// Transform explicit findings without scanning implicitly.
@@ -1274,6 +1273,7 @@ fn scan_and_transform_structured(
 
 #[pymodule]
 fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    models::register(module)?;
     module.add(
         "DataFogConfigurationError",
         module.py().get_type::<DataFogConfigurationError>(),