datafog-core 0.4.5b4__tar.gz → 0.5.0b2__tar.gz

{datafog_core-0.4.5b4 → datafog_core-0.5.0b2}/Cargo.lock

@@ -11,6 +11,21 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "android_system_properties"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
 [[package]]
 name = "base64"
 version = "0.22.1"
@@ -38,12 +53,33 @@ version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
+[[package]]
+name = "cc"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
+[[package]]
+name = "chrono"
+version = "0.4.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
+dependencies = [
+ "iana-time-zone",
+ "num-traits",
+ "windows-link",
+]
+
 [[package]]
 name = "convert_case"
 version = "0.11.0"
@@ -53,6 +89,12 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.17"
@@ -80,11 +122,12 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.4.5-beta.4"
+version = "0.5.0-beta.2"
 dependencies = [
  "base64",
  "futures",
  "hmac",
+ "minicbor",
  "regex",
  "serde",
  "serde_json",
@@ -94,7 +137,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.4.5-beta.4"
+version = "0.5.0-beta.2"
 dependencies = [
  "datafog-core",
  "pyo3",
@@ -105,7 +148,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-node"
-version = "0.4.5-beta.4"
+version = "0.5.0-beta.2"
 dependencies = [
  "datafog-core",
  "napi",
@@ -116,7 +159,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.4.5-beta.4"
+version = "0.5.0-beta.2"
 dependencies = [
  "datafog-core",
  "serde",
@@ -136,6 +179,12 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
+
 [[package]]
 name = "futures"
 version = "0.3.34"
@@ -249,6 +298,30 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "log",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -282,12 +355,24 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "log"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
+
 [[package]]
 name = "memchr"
 version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
 
+[[package]]
+name = "minicbor"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c12b4033ffaa92fbf9df03df38d19324f52bad130dd223f811734a8006dd2d69"
+
 [[package]]
 name = "napi"
 version = "3.12.2"
@@ -354,6 +439,15 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
 
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.21.4"
@@ -573,6 +667,12 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -691,12 +791,75 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "windows-core"
+version = "0.62.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "windows-link"
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
 
+[[package]]
+name = "windows-result"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "xtask"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "serde",
+ "serde_json",
+ "sha2",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.9.0"

{datafog_core-0.4.5b4 → datafog_core-0.5.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog-core
-Version: 0.4.5b4
+Version: 0.5.0b2
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -59,7 +59,7 @@ DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bin
 
 Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
 
-**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4)
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Prerelease:** [0.5.0-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.5.0-beta.1)
 
 ## Quickstart
 
@@ -158,12 +158,14 @@ Python and Node.js packages include prebuilt native binaries for Linux x64 and A
 | --- | --- | --- |
 | Personal and financial data | `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, `ZIP_CODE` | On by default |
 | Credentials | [`API_KEY`](docs/reference/api-keys.mdx), [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), [`JWT`](docs/reference/jwt.mdx), [`PRIVATE_KEY`](docs/reference/private-keys.mdx), [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx) | On by default |
+| Labeled street addresses | [`STREET_ADDRESS`](docs/reference/street-address.mdx) | On by default; bounded English street-line rule |
+| Labeled accounts | [`ACCOUNT_NUMBER`](docs/reference/account-number.mdx) | On by default; bounded lexical rule |
 | Labeled US identifiers | [`US_ROUTING_NUMBER`](docs/reference/us-routing-number.mdx), [`NPI`](docs/reference/npi.mdx) | On by default; requires a supported text label |
 | German identifiers | IBAN, VAT ID, tax ID, social security number, postal code, passport number, residence permit number | Explicit [German locale](docs/reference/german-entities.mdx) |
 | UUIDs | [`UUID`](docs/reference/uuid.mdx) | Explicit `detect_uuid` setting |
 | Names in JSON fields | `PERSON` | [Structured scans](docs/guides/person-discovery.mdx), using name-field aliases or explicit JSON Pointer mappings |
 
-Use [structured scans](docs/guides/person-discovery.mdx) to find names by JSON field, or [local models](#local-models-beta) to detect names and street addresses in prose.
+Use [structured scans](docs/guides/person-discovery.mdx) to find names by JSON field, or [local models](#local-models-beta) to detect names and unlabelled street addresses in prose.
 
 Each finding includes its entity type, matched text, byte and code-point ranges, optional confidence, and detector name/version. Ranges are zero-based and end-exclusive. Node.js and WASM also provide UTF-16 ranges for JavaScript `slice`. Rule-based findings have no confidence score. [Findings and ranges →](docs/concepts/findings-and-ranges.mdx)
 
@@ -196,22 +198,37 @@ Use `transform` with existing findings, or `scan_and_transform` / `scanAndTransf
 
 Keep scan settings under `scan` and transformation policy under `transform`. A default strategy is required; entity selection, per-type overrides, and allowlists are optional. [Configuration guide →](docs/guides/configuration.mdx) · [Strategy details →](docs/concepts/privacy-transformations.mdx)
 
+## Chrome extension integration in 0.5.0-beta.1
+
+The developer prerelease adds persistent `Workspace` pseudonyms, approved
+save-before-handoff, strict restoration, follow-ups and host-owned encrypted
+backup/recovery, plus optional English GLiNER2.5 Small browser sessions. Use
+synthetic data while the snapshot format and vault integration remain provisional.
+
+```bash
+npm install --save-exact @datafog/wasm@0.5.0-beta.1
+```
+
+[Integration lifecycle, all browser calls and errors →](docs/extension-integration-0.5.0-beta.1.md)
+· [Model assets and runtime setup →](docs/browser-model.md)
+· [Release scope and verification →](https://github.com/DataFog/datafog-core/issues/132)
+
 ## Local models (beta)
 
 [0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
 
-Install the beta for your runtime:
+Install `0.5.0-beta.1` (Python `0.5.0b1`) for the current native prerelease:
 
 ```bash
-python -m pip install 'datafog-core==0.4.5b4'
-npm install @datafog/node@beta
-cargo add datafog-core@0.4.5-beta.4 --features models
+python -m pip install 'datafog-core==0.5.0b1'
+npm install --save-exact @datafog/node@0.5.0-beta.1
+cargo add datafog-core@0.5.0-beta.1 --features models
 ```
 
 For GLiNER2, install its optional dependencies in the Python environment used by the runner:
 
 ```bash
-python -m pip install 'datafog-core[gliner2]==0.4.5b4'
+python -m pip install 'datafog-core[gliner2]==0.5.0b1'
 ```
 
 GLiNER2 supports the PII checkpoint and compatible general-purpose span models with caller-defined labels. The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
@@ -220,6 +237,8 @@ GLiNER2 supports the PII checkpoint and compatible general-purpose span models w
 
 ## Documentation and development
 
+- [Roadmap](ROADMAP.md)
+- [Dev environment setup](CONTRIBUTING.md#dev-environment-setup) — devcontainer or local install, `cargo dfv doctor`
 - [Installation and runtime requirements](docs/get-started/installation.mdx)
 - [Python](docs/reference/python.mdx) · [Node.js](docs/reference/node.mdx) · [Rust](docs/reference/rust.mdx) · [Browser / WASM](docs/reference/browser-wasm.mdx) API references
 - [Structured JSON scans and name-field discovery](docs/guides/person-discovery.mdx)

{datafog_core-0.4.5b4/crates/core → datafog_core-0.5.0b2}/README.md

@@ -26,7 +26,7 @@ DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bin
 
 Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
 
-**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4)
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Prerelease:** [0.5.0-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.5.0-beta.1)
 
 ## Quickstart
 
@@ -125,12 +125,14 @@ Python and Node.js packages include prebuilt native binaries for Linux x64 and A
 | --- | --- | --- |
 | Personal and financial data | `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, `ZIP_CODE` | On by default |
 | Credentials | [`API_KEY`](docs/reference/api-keys.mdx), [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), [`JWT`](docs/reference/jwt.mdx), [`PRIVATE_KEY`](docs/reference/private-keys.mdx), [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx) | On by default |
+| Labeled street addresses | [`STREET_ADDRESS`](docs/reference/street-address.mdx) | On by default; bounded English street-line rule |
+| Labeled accounts | [`ACCOUNT_NUMBER`](docs/reference/account-number.mdx) | On by default; bounded lexical rule |
 | Labeled US identifiers | [`US_ROUTING_NUMBER`](docs/reference/us-routing-number.mdx), [`NPI`](docs/reference/npi.mdx) | On by default; requires a supported text label |
 | German identifiers | IBAN, VAT ID, tax ID, social security number, postal code, passport number, residence permit number | Explicit [German locale](docs/reference/german-entities.mdx) |
 | UUIDs | [`UUID`](docs/reference/uuid.mdx) | Explicit `detect_uuid` setting |
 | Names in JSON fields | `PERSON` | [Structured scans](docs/guides/person-discovery.mdx), using name-field aliases or explicit JSON Pointer mappings |
 
-Use [structured scans](docs/guides/person-discovery.mdx) to find names by JSON field, or [local models](#local-models-beta) to detect names and street addresses in prose.
+Use [structured scans](docs/guides/person-discovery.mdx) to find names by JSON field, or [local models](#local-models-beta) to detect names and unlabelled street addresses in prose.
 
 Each finding includes its entity type, matched text, byte and code-point ranges, optional confidence, and detector name/version. Ranges are zero-based and end-exclusive. Node.js and WASM also provide UTF-16 ranges for JavaScript `slice`. Rule-based findings have no confidence score. [Findings and ranges →](docs/concepts/findings-and-ranges.mdx)
 
@@ -163,22 +165,37 @@ Use `transform` with existing findings, or `scan_and_transform` / `scanAndTransf
 
 Keep scan settings under `scan` and transformation policy under `transform`. A default strategy is required; entity selection, per-type overrides, and allowlists are optional. [Configuration guide →](docs/guides/configuration.mdx) · [Strategy details →](docs/concepts/privacy-transformations.mdx)
 
+## Chrome extension integration in 0.5.0-beta.1
+
+The developer prerelease adds persistent `Workspace` pseudonyms, approved
+save-before-handoff, strict restoration, follow-ups and host-owned encrypted
+backup/recovery, plus optional English GLiNER2.5 Small browser sessions. Use
+synthetic data while the snapshot format and vault integration remain provisional.
+
+```bash
+npm install --save-exact @datafog/wasm@0.5.0-beta.1
+```
+
+[Integration lifecycle, all browser calls and errors →](docs/extension-integration-0.5.0-beta.1.md)
+· [Model assets and runtime setup →](docs/browser-model.md)
+· [Release scope and verification →](https://github.com/DataFog/datafog-core/issues/132)
+
 ## Local models (beta)
 
 [0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
 
-Install the beta for your runtime:
+Install `0.5.0-beta.1` (Python `0.5.0b1`) for the current native prerelease:
 
 ```bash
-python -m pip install 'datafog-core==0.4.5b4'
-npm install @datafog/node@beta
-cargo add datafog-core@0.4.5-beta.4 --features models
+python -m pip install 'datafog-core==0.5.0b1'
+npm install --save-exact @datafog/node@0.5.0-beta.1
+cargo add datafog-core@0.5.0-beta.1 --features models
 ```
 
 For GLiNER2, install its optional dependencies in the Python environment used by the runner:
 
 ```bash
-python -m pip install 'datafog-core[gliner2]==0.4.5b4'
+python -m pip install 'datafog-core[gliner2]==0.5.0b1'
 ```
 
 GLiNER2 supports the PII checkpoint and compatible general-purpose span models with caller-defined labels. The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
@@ -187,6 +204,8 @@ GLiNER2 supports the PII checkpoint and compatible general-purpose span models w
 
 ## Documentation and development
 
+- [Roadmap](ROADMAP.md)
+- [Dev environment setup](CONTRIBUTING.md#dev-environment-setup) — devcontainer or local install, `cargo dfv doctor`
 - [Installation and runtime requirements](docs/get-started/installation.mdx)
 - [Python](docs/reference/python.mdx) · [Node.js](docs/reference/node.mdx) · [Rust](docs/reference/rust.mdx) · [Browser / WASM](docs/reference/browser-wasm.mdx) API references
 - [Structured JSON scans and name-field discovery](docs/guides/person-discovery.mdx)

{datafog_core-0.4.5b4 → datafog_core-0.5.0b2}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.4.5-beta.4"
+version = "0.5.0-beta.2"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"
@@ -12,7 +12,7 @@ name = "datafog_core"
 crate-type = ["cdylib"]
 
 [dependencies]
-datafog-core = { version = "0.4.5-beta.4", path = "../../crates/core", features = ["models"] }
+datafog-core = { version = "0.5.0-beta.2", path = "../../crates/core", features = ["models"] }
 pyo3 = { version = "0.29", features = ["abi3-py310"] }
 pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] }
 serde_json = "1"

{datafog_core-0.4.5b4 → datafog_core-0.5.0b2}/bindings/python/src/lib.rs

@@ -921,7 +921,7 @@ fn privacy_error(py: Python<'_>, error: core::PrivacyError) -> PyErr {
         core::PrivacyErrorCode::InvalidConfiguration => {
             PyErr::new::<DataFogConfigurationError, _>(error.to_string())
         }
-        core::PrivacyErrorCode::InvalidFinding => {
+        core::PrivacyErrorCode::InvalidFinding | core::PrivacyErrorCode::ReviewConflict => {
             PyErr::new::<DataFogFindingError, _>(error.to_string())
         }
         core::PrivacyErrorCode::KeyProviderRequired

{datafog_core-0.4.5b4 → datafog_core-0.5.0b2}/bindings/python/src/models.rs

@@ -82,6 +82,10 @@ pub(super) struct ModelSession {
 }
 #[pymethods]
 impl ModelSession {
+    fn describe(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
+        json_object(py, &self.session.describe())
+    }
+
     #[staticmethod]
     fn open(py: Python<'_>, config: &Bound<'_, PyAny>) -> PyResult<Self> {
         let started = std::time::Instant::now();

{datafog_core-0.4.5b4 → datafog_core-0.5.0b2}/bindings/python/tests/capabilities_conformance.py

@@ -5,6 +5,7 @@ from pathlib import Path
 import datafog_core as api
 
 SUPPORTED = sorted([
+    "ACCOUNT_NUMBER", "STREET_ADDRESS",
     "BEARER_TOKEN", "CREDIT_CARD", "DATE", "DE_IBAN", "DE_PASSPORT_NUMBER", "DE_POSTAL_CODE", "API_KEY", "CREDENTIAL_URI",
     "DE_RESIDENCE_PERMIT_NUMBER", "DE_SOCIAL_SECURITY_NUMBER", "DE_TAX_ID",
     "DE_VAT_ID", "EMAIL", "IP_ADDRESS", "JWT", "NPI", "PERSON", "PHONE",

{datafog_core-0.4.5b4 → datafog_core-0.5.0b2}/bindings/python/tests/test_installed.py

@@ -23,6 +23,8 @@ from datafog_core import (
     DataFogFindingError,
     DataFogKeyProviderError,
     Finding,
+    StructuredFinding,
+    ModelSession,
     PrivacyManager,
     TextRange,
     scan,
@@ -38,6 +40,120 @@ from datafog_core import (
 ROOT = Path(__file__).resolve().parents[3]
 
 
+def verify_account_number() -> None:
+    for line in ((ROOT / "fixtures" / "account-number.jsonl").read_text().splitlines()
+                 + (ROOT / "fixtures" / "masked-account-number.jsonl").read_text().splitlines()):
+        row = json.loads(line)
+        text = row["text"]
+        def project(findings):
+            result = []
+            for finding in findings:
+                if finding.entity_type != "ACCOUNT_NUMBER":
+                    continue
+                assert text.encode()[finding.byte_range.start:finding.byte_range.end].decode() == finding.matched_text
+                assert text[finding.codepoint_range.start:finding.codepoint_range.end] == finding.matched_text
+                assert finding.detector_name == "datafog-core/account-number"
+                assert finding.confidence is None
+                result.append(dict(label=finding.entity_type, text=finding.matched_text, start=finding.codepoint_range.start, end=finding.codepoint_range.end))
+            return result
+        assert project(scan(text)) == row["entities"], row["id"]
+        assert project(scan(text, {"entities":["ACCOUNT_NUMBER"]})) == row["entities"], row["id"]
+        assert project([f.finding for f in scan_structured({"note":text}).findings]) == row["entities"]
+        cases = row.get("transforms", [dict(config={"default":{"strategy":"redact"},"entities":["ACCOUNT_NUMBER"]},text=text)])
+        for case in cases:
+            config = {"transform":case["config"]}
+            assert scan_and_transform(text, config).text == case["text"], row["id"]
+            assert scan_and_transform_structured({"note":text}, config).data == {"note":case["text"]}
+    assert not any(f.finding.entity_type == "ACCOUNT_NUMBER" for f in scan_structured({"account number":"12345678"}).findings)
+
+
+def verify_street_address() -> None:
+    for line in (ROOT / "fixtures" / "street-address.jsonl").read_text().splitlines():
+        row = json.loads(line)
+        text = row["text"]
+        def project(findings):
+            result = []
+            for finding in findings:
+                if finding.entity_type != "STREET_ADDRESS":
+                    continue
+                assert text.encode()[finding.byte_range.start:finding.byte_range.end].decode() == finding.matched_text
+                assert text[finding.codepoint_range.start:finding.codepoint_range.end] == finding.matched_text
+                assert finding.detector_name == "datafog-core/street-address"
+                assert finding.confidence is None
+                result.append(dict(label=finding.entity_type, text=finding.matched_text, start=finding.codepoint_range.start, end=finding.codepoint_range.end))
+            return result
+        assert project(scan(text)) == row["entities"], row["id"]
+        assert project(scan(text, {"entities":["STREET_ADDRESS"]})) == row["entities"], row["id"]
+        assert project([f.finding for f in scan_structured({"note":text}).findings]) == row["entities"]
+        cases = row.get("transforms") or [dict(config={"default":{"strategy":"redact"},"entities":["STREET_ADDRESS"]},text=text)]
+        for case in cases:
+            config = {"transform":case["config"]}
+            assert scan_and_transform(text, config).text == case["text"], row["id"]
+            assert scan_and_transform_structured({"note":text}, config).data == {"note":case["text"]}
+    assert not any(f.finding.entity_type == "STREET_ADDRESS" for f in scan_structured({"street address":"12 Willow Road"}).findings)
+
+
+def verify_coverage_conflicts() -> None:
+    text = "👋 abcdef"
+    first = Finding("PERSON", "abcd", TextRange(5, 9), TextRange(2, 6), "synthetic")
+    second = Finding("CUSTOM", "cdef", TextRange(7, 11), TextRange(4, 8), "synthetic")
+    config = {"default": {"strategy": "redact"}}
+    for inputs, index in [([first, second], 1), ([second, first], 0)]:
+        for operation in [
+            lambda: transform(text, inputs, config),
+            lambda: transform_structured({"z": text}, [StructuredFinding("/z", f) for f in inputs], config),
+        ]:
+            try:
+                operation()
+            except DataFogFindingError as error:
+                assert error.code == "review_conflict" and error.finding_index == index
+                assert "abcdef" not in str(error)
+            else:
+                raise AssertionError("crossing findings returned partial output")
+    assert transform(text, [first, second], {**config, "entities": ["PERSON"]}).text == "👋 [PERSON]ef"
+    assert transform(text, [first, second], {**config, "allow": {"exact": {"PERSON": ["abcd"]}}}).text == "👋 ab[CUSTOM]"
+    class UnusedProvider:
+        def __init__(self): self.calls = 0
+        async def resolve_key(self, *args): self.calls += 1; raise AssertionError("provider must not run")
+        async def tokenize_batch(self, *args): self.calls += 1; raise AssertionError("provider must not run")
+        async def restore_batch(self, *args): self.calls += 1; raise AssertionError("provider must not run")
+    provider = UnusedProvider()
+    async def verify_providers():
+        manager = PrivacyManager(provider, provider)
+        for strategy in [{"strategy":"pseudonymize", "key_ref":"unused"}, {"strategy":"tokenize", "token_ref":"unused"}]:
+            for operation in [
+                lambda: manager.transform(text, [first, second], {"default":strategy}, {"scope":"synthetic"}),
+                lambda: manager.transform_structured({"z":text}, [StructuredFinding("/z", f) for f in [first, second]], {"default":strategy}, {"scope":"synthetic"}),
+            ]:
+                try: await operation()
+                except DataFogFindingError as error: assert error.code == "review_conflict"
+                else: raise AssertionError("provider-backed conflict returned output")
+    asyncio.run(verify_providers())
+    assert provider.calls == 0
+
+
+def verify_model_engine_selection() -> None:
+    config = {
+        "bundle": str(ROOT / "target" / "missing-engine-bundle"),
+        "manifest_sha256": "a" * 64,
+        "limits": dict(max_input_bytes=1000, max_response_bytes=10000,
+                       max_findings=10, timeout_ms=1000, cleanup_timeout_ms=1000),
+    }
+    for engine, path, reason in [
+        ("private-engine", "/engine", "invalid_value"),
+        (True, "/engine", "invalid_type"),
+        (None, "/engine", "invalid_type"),
+        ("gliner2", "/gliner2", "missing_field"),
+    ]:
+        try:
+            ModelSession.open(dict(config, engine=engine))
+        except DataFogConfigurationError as error:
+            assert (error.path, error.reason) == (path, reason)
+            assert "private-engine" not in str(error)
+        else:
+            raise AssertionError("invalid engine selection accepted")
+
+
 def expected_entities(record: dict[str, object]) -> list[dict[str, object]]:
     return record["entities"]  # type: ignore[return-value]
 
@@ -163,7 +279,10 @@ def verify_text_selection() -> None:
 
 
 def main() -> None:
+    verify_coverage_conflicts()
     verify_text_selection()
+    verify_account_number()
+    verify_street_address()
     capabilities_conformance.verify()
     german_conformance.verify()
     credential_overlap_conformance.verify()
@@ -505,5 +624,25 @@ def main() -> None:
     print("Installed datafog_core wheel matches fixtures and transform contracts.")
 
 
+def verify_ssn_person_priority() -> None:
+    text = "👋 José: 123-45-6789"
+    ssn = next(finding for finding in scan(text) if finding.entity_type == "SSN")
+    person = Finding(
+        "PERSON", ssn.matched_text, ssn.byte_range, ssn.codepoint_range,
+        "semantic-person", 1.0,
+    )
+    for findings in ([ssn, person], [person, ssn]):
+        protected = transform(text, findings, {"default": {"strategy": "redact"}})
+        assert protected.text == "👋 José: [SSN]"
+        assert protected.transformations[0].entity_type == "SSN"
+        allowed = transform(text, findings, {
+            "default": {"strategy": "redact"},
+            "allow": {"exact": {"SSN": [ssn.matched_text]}},
+        })
+        assert allowed.text == "👋 José: [PERSON]"
+
+
 if __name__ == "__main__":
+    verify_model_engine_selection()
+    verify_ssn_person_priority()
     main()

{datafog_core-0.4.5b4 → datafog_core-0.5.0b2}/bindings/python/tests/test_python49_integration.py

@@ -21,8 +21,8 @@ def verify_integration():
     rows = [expand_fixture(json.loads(line)) for line in FIXTURES.read_text().splitlines()]
     capabilities = datafog_core.capabilities()
     assert capabilities["contract_version"] == 1
-    assert len(capabilities["supported_entities"]) == 23
-    assert len(capabilities["default_entities"]) == 14
+    assert len(capabilities["supported_entities"]) == 25
+    assert len(capabilities["default_entities"]) == 16
     assert {"API_KEY", "BEARER_TOKEN", "CREDENTIAL_URI"} <= set(
         capabilities["default_entities"]
     )