datafog-core 0.4.5b2__tar.gz → 0.4.5b4__tar.gz

{datafog_core-0.4.5b2 → datafog_core-0.4.5b4}/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.4"
 dependencies = [
  "base64",
  "futures",
@@ -94,7 +94,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.4"
 dependencies = [
  "datafog-core",
  "pyo3",
@@ -105,7 +105,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-node"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.4"
 dependencies = [
  "datafog-core",
  "napi",
@@ -116,7 +116,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.4"
 dependencies = [
  "datafog-core",
  "serde",

{datafog_core-0.4.5b2 → datafog_core-0.4.5b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog-core
-Version: 0.4.5b2
+Version: 0.4.5b4
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -59,7 +59,7 @@ DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bin
 
 Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
 
-**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1)
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4)
 
 ## Quickstart
 
@@ -198,23 +198,23 @@ Keep scan settings under `scan` and transformation policy under `transform`. A d
 
 ## Local models (beta)
 
-[0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
+[0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
 
 Install the beta for your runtime:
 
 ```bash
-python -m pip install 'datafog-core==0.4.5b1'
+python -m pip install 'datafog-core==0.4.5b4'
 npm install @datafog/node@beta
-cargo add datafog-core@0.4.5-beta.1 --features models
+cargo add datafog-core@0.4.5-beta.4 --features models
 ```
 
 For GLiNER2, install its optional dependencies in the Python environment used by the runner:
 
 ```bash
-python -m pip install 'datafog-core[gliner2]==0.4.5b1'
+python -m pip install 'datafog-core[gliner2]==0.4.5b4'
 ```
 
-The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
+GLiNER2 supports the PII checkpoint and compatible general-purpose span models with caller-defined labels. The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
 
 [DataFog setup and session API →](docs/model-sessions.md) · [GLiNER2 setup →](docs/gliner2-model.md)
 

{datafog_core-0.4.5b2/crates/core → datafog_core-0.4.5b4}/README.md

@@ -26,7 +26,7 @@ DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bin
 
 Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
 
-**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1)
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4)
 
 ## Quickstart
 
@@ -165,23 +165,23 @@ Keep scan settings under `scan` and transformation policy under `transform`. A d
 
 ## Local models (beta)
 
-[0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
+[0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
 
 Install the beta for your runtime:
 
 ```bash
-python -m pip install 'datafog-core==0.4.5b1'
+python -m pip install 'datafog-core==0.4.5b4'
 npm install @datafog/node@beta
-cargo add datafog-core@0.4.5-beta.1 --features models
+cargo add datafog-core@0.4.5-beta.4 --features models
 ```
 
 For GLiNER2, install its optional dependencies in the Python environment used by the runner:
 
 ```bash
-python -m pip install 'datafog-core[gliner2]==0.4.5b1'
+python -m pip install 'datafog-core[gliner2]==0.4.5b4'
 ```
 
-The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
+GLiNER2 supports the PII checkpoint and compatible general-purpose span models with caller-defined labels. The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
 
 [DataFog setup and session API →](docs/model-sessions.md) · [GLiNER2 setup →](docs/gliner2-model.md)
 

{datafog_core-0.4.5b2 → datafog_core-0.4.5b4}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.4"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"
@@ -12,7 +12,7 @@ name = "datafog_core"
 crate-type = ["cdylib"]
 
 [dependencies]
-datafog-core = { version = "0.4.5-beta.2", path = "../../crates/core", features = ["models"] }
+datafog-core = { version = "0.4.5-beta.4", path = "../../crates/core", features = ["models"] }
 pyo3 = { version = "0.29", features = ["abi3-py310"] }
 pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] }
 serde_json = "1"

{datafog_core-0.4.5b2 → datafog_core-0.4.5b4}/bindings/python/src/models.rs

@@ -91,8 +91,49 @@ impl ModelSession {
             .map(|session| Self { session })
             .map_err(|error| model_error(py, error))
     }
-    #[pyo3(signature = (text, *, cancellation=None))]
+    /// Rule detectors handle the entity types they support and the model runs
+    /// only for requested types it alone provides. Omitting `config` equals
+    /// `{}`: every enabled rule detector plus the model's remaining labels.
+    #[pyo3(signature = (text, config=None, *, cancellation=None))]
     fn scan(
+        &self,
+        py: Python<'_>,
+        text: String,
+        config: Option<&Bound<'_, PyAny>>,
+        cancellation: Option<PyRef<'_, ModelCancellation>>,
+    ) -> PyResult<ModelScanResult> {
+        let routed = match config {
+            Some(config) => {
+                let value = py_to_json(py, config, "")?;
+                self.session
+                    .routed_config(&value)
+                    .map_err(|error| model_error(py, error))?
+            }
+            None => self.session.default_routed_config(),
+        };
+        // Rule-only routing never reserves the model worker.
+        let request = if routed.needs_model() {
+            Some(
+                self.session
+                    .prepare_request()
+                    .map_err(|error| model_error(py, error))?,
+            )
+        } else {
+            None
+        };
+        let cancellation = cancellation
+            .map(|v| Arc::clone(&v.cancellation))
+            .unwrap_or_default();
+        py.detach(|| {
+            self.session
+                .scan_routed_request(request.as_ref(), &text, &routed, &cancellation)
+        })
+        .map(ModelScanResult::from)
+        .map_err(|error| model_error(py, error))
+    }
+    /// Raw model output: every model label, with no rule detectors.
+    #[pyo3(signature = (text, *, cancellation=None))]
+    fn scan_model(
         &self,
         py: Python<'_>,
         text: String,
@@ -105,9 +146,12 @@ impl ModelSession {
         let cancellation = cancellation
             .map(|v| Arc::clone(&v.cancellation))
             .unwrap_or_default();
-        py.detach(|| self.session.scan_request(&request, &text, &cancellation))
-            .map(ModelScanResult::from)
-            .map_err(|error| model_error(py, error))
+        py.detach(|| {
+            self.session
+                .scan_model_request(&request, &text, &cancellation)
+        })
+        .map(ModelScanResult::from)
+        .map_err(|error| model_error(py, error))
     }
     fn close(&self, py: Python<'_>) -> PyResult<()> {
         py.detach(|| self.session.close())

{datafog_core-0.4.5b2 → datafog_core-0.4.5b4}/crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.4"
 edition = "2024"
 rust-version = "1.88"
 description = "Fast structured PII detection library"

{datafog_core-0.4.5b2 → datafog_core-0.4.5b4/crates/core}/README.md

@@ -26,7 +26,7 @@ DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bin
 
 Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
 
-**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1)
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4)
 
 ## Quickstart
 
@@ -165,23 +165,23 @@ Keep scan settings under `scan` and transformation policy under `transform`. A d
 
 ## Local models (beta)
 
-[0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
+[0.4.5-beta.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.4) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
 
 Install the beta for your runtime:
 
 ```bash
-python -m pip install 'datafog-core==0.4.5b1'
+python -m pip install 'datafog-core==0.4.5b4'
 npm install @datafog/node@beta
-cargo add datafog-core@0.4.5-beta.1 --features models
+cargo add datafog-core@0.4.5-beta.4 --features models
 ```
 
 For GLiNER2, install its optional dependencies in the Python environment used by the runner:
 
 ```bash
-python -m pip install 'datafog-core[gliner2]==0.4.5b1'
+python -m pip install 'datafog-core[gliner2]==0.4.5b4'
 ```
 
-The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
+GLiNER2 supports the PII checkpoint and compatible general-purpose span models with caller-defined labels. The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.
 
 [DataFog setup and session API →](docs/model-sessions.md) · [GLiNER2 setup →](docs/gliner2-model.md)
 

{datafog_core-0.4.5b2 → datafog_core-0.4.5b4}/crates/core/src/models/gliner2.rs

@@ -1,11 +1,10 @@
-//! Pinned official GLiNER2 span runtime, executed in an explicitly installed Python environment.
+//! Official GLiNER2 span runtime, executed in an explicitly installed Python environment.
 use super::*;
 use std::process::Command;
 
 pub const MODEL: &str = "fastino/gliner2-privacy-filter-PII-multi";
 const ADAPTER: &str = "gliner2-worker-v1";
 const SCALE: &str = "normalized_span_sigmoid";
-const LABELS: &str = "person full_name first_name middle_name last_name date_of_birth email phone_number address street_address city state_or_region postal_code country government_id national_id_number passport_number drivers_license_number license_number tax_id tax_number bank_account account_number routing_number iban payment_card card_number card_expiry card_cvv username ip_address account_id sensitive_account_id password secret api_key access_token recovery_code sensitive_date document_date expiration_date transaction_date";
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -64,9 +63,13 @@ fn validate_config(config: &Gliner2Config) -> Result<(), ModelError> {
         return Err(ModelError::config("empty_value", "/gliner2/labels"));
     }
     let mut seen = std::collections::BTreeSet::new();
+    let mut entity_types = std::collections::BTreeSet::new();
     for (index, label) in config.labels.iter().enumerate() {
         let path = format!("/gliner2/labels/{index}");
-        if !LABELS.split_whitespace().any(|name| name == label.label) || !seen.insert(&label.label)
+        if label.label.trim().is_empty()
+            || label.label.trim() != label.label
+            || !seen.insert(&label.label)
+            || !entity_types.insert(mapping(&label.label))
         {
             return Err(ModelError::config(
                 "invalid_value",
@@ -142,24 +145,50 @@ pub(super) fn open(
         return Err(invalid());
     }
     let manifest: BundleManifest = serde_json::from_slice(&bytes).map_err(|_| invalid())?;
-    let lock: serde_json::Value =
-        serde_json::from_str(include_str!("gliner2-lock.json")).map_err(|_| invalid())?;
-    let expected: BTreeMap<String, AssetIdentity> =
-        serde_json::from_value(lock["files"].clone()).map_err(|_| invalid())?;
-    if manifest.platform != "portable-cpu" || manifest.files.len() != expected.len() {
+    let identity = validate_model_identity(&manifest, "gliner2")?;
+    let model_name = identity.map_or_else(|| MODEL.to_owned(), |model| model.id.clone());
+    let adapter_version = identity.map_or_else(
+        || ADAPTER.to_owned(),
+        |model| format!("{ADAPTER}:{}", model.revision),
+    );
+    if manifest.platform != "portable-cpu" || manifest.files.is_empty() {
         return Err(invalid());
     }
-    for (name, expected) in expected {
-        let identity = manifest.files.get(&name).ok_or_else(invalid)?;
-        if identity.sha256 != expected.sha256 || identity.bytes != expected.bytes {
+    if identity.is_none() {
+        let lock: serde_json::Value =
+            serde_json::from_str(include_str!("gliner2-lock.json")).map_err(|_| invalid())?;
+        let expected: BTreeMap<String, AssetIdentity> =
+            serde_json::from_value(lock["files"].clone()).map_err(|_| invalid())?;
+        if manifest.files.len() != expected.len() {
             return Err(invalid());
         }
+        for (name, expected) in expected {
+            let asset = manifest.files.get(&name).ok_or_else(invalid)?;
+            if asset.sha256 != expected.sha256 || asset.bytes != expected.bytes {
+                return Err(invalid());
+            }
+        }
+    }
+    // The existing worker loads this local, single-file GLiNER2 span-model layout.
+    for required in [
+        "config.json",
+        "encoder_config/config.json",
+        "tokenizer.json",
+        "tokenizer_config.json",
+        "model.safetensors",
+    ] {
+        if !manifest.files.contains_key(required) {
+            return Err(invalid());
+        }
+    }
+    for (name, asset) in &manifest.files {
+        validate_asset_name(name)?;
         let path = bundle.join(name).canonicalize().map_err(|_| invalid())?;
         if !path.starts_with(&bundle) || !path.is_file() {
             return Err(invalid());
         }
         let (hash, size) = file_digest(&path, deadline)?;
-        if hash != expected.sha256 || size != expected.bytes {
+        if hash != asset.sha256 || size != asset.bytes {
             return Err(invalid());
         }
     }
@@ -178,8 +207,8 @@ pub(super) fn open(
     let session = ModelSession {
         descriptor: ModelSessionDescriptor {
             backend: "gliner2_worker".into(),
-            model: MODEL.into(),
-            adapter_version: ADAPTER.into(),
+            model: model_name,
+            adapter_version,
             labels: options
                 .labels
                 .iter()
@@ -255,7 +284,7 @@ pub(super) fn decode(bytes: &[u8], max_findings: usize) -> Result<BackendResult,
         serde_json::from_value(value).map_err(|_| ModelError::invalid_output())?;
     let expected = runtime();
     if output.runtime != expected
-        || output.worker_version != "0.4.5b2"
+        || output.worker_version != "0.4.5b4"
         || output.findings.len() > max_findings
     {
         return Err(ModelError::invalid_output());
@@ -303,13 +332,71 @@ mod tests {
         bad.labels.push(bad.labels[0].clone());
         assert!(validate_config(&bad).is_err());
         let mut bad = parsed.clone();
-        bad.labels[0].label = "not_a_checkpoint_label".into();
-        assert!(validate_config(&bad).is_err());
+        bad.labels[0].label = "current_employer".into();
+        bad.labels[0].description = Some("The caller's current employer".into());
+        assert!(validate_config(&bad).is_ok());
+        for label in ["", " ", " employer", "employer "] {
+            bad.labels[0].label = label.into();
+            assert!(validate_config(&bad).is_err());
+        }
         let mut bad = parsed;
         bad.chunk_overlap = bad.chunk_size;
         assert!(validate_config(&bad).is_err());
     }
     #[test]
+    fn labels_must_have_distinct_output_types() {
+        for (first, second) in [("employer", "EMPLOYER"), ("phone_number", "phone")] {
+            let config = Gliner2Config {
+                python: std::env::current_exe().unwrap(),
+                labels: [first, second]
+                    .into_iter()
+                    .map(|label| Gliner2Label {
+                        label: label.into(),
+                        threshold: 0.5,
+                        description: None,
+                    })
+                    .collect(),
+                intra_threads: 1,
+                chunk_size: 384,
+                chunk_overlap: 64,
+            };
+            assert!(validate_config(&config).is_err());
+        }
+    }
+
+    #[test]
+    fn custom_model_labels_keep_types_offsets_and_transformation_support() {
+        let text = "👋 Cedar Labs";
+        let mut descriptor = super::super::tests::descriptor();
+        descriptor.backend = "gliner2_worker".into();
+        descriptor.model = "fastino/gliner2-base-v1".into();
+        descriptor.labels = [("current_employer".into(), "CURRENT_EMPLOYER".into())].into();
+        descriptor.thresholds = [("current_employer".into(), 0.5)].into();
+        descriptor.score_scale = SCALE.into();
+        let response = serde_json::json!({
+            "worker_version": "0.4.5b4", "runtime": runtime(),
+            "findings": [{"label": "current_employer", "start": 2, "end": 12,
+                          "text": "Cedar Labs", "raw_score": 0.9}]
+        })
+        .to_string();
+        let entries = convert(text, decode(response.as_bytes(), 1).unwrap(), &descriptor).unwrap();
+        let findings: Vec<_> = entries.into_iter().map(|(finding, _)| finding).collect();
+        assert_eq!(findings[0].entity_type, "CURRENT_EMPLOYER");
+        assert_eq!(findings[0].byte_range, TextRange { start: 5, end: 15 });
+        assert_eq!(findings[0].codepoint_range, TextRange { start: 2, end: 12 });
+        assert_eq!(findings[0].confidence, None);
+        let transformed = crate::transform(
+            text,
+            &findings,
+            &crate::TransformationConfig::new(crate::TransformationStrategy::Redact),
+        )
+        .unwrap();
+        assert_eq!(transformed.text, "👋 [CURRENT_EMPLOYER]");
+        let mut unexpected = descriptor;
+        unexpected.labels.clear();
+        assert!(convert(text, decode(response.as_bytes(), 1).unwrap(), &unexpected).is_err());
+    }
+    #[test]
     fn uncalibrated_scores_are_validated_before_thresholding_and_rebased() {
         let mut descriptor = super::super::tests::descriptor();
         descriptor.backend = "gliner2_worker".into();
@@ -318,7 +405,7 @@ mod tests {
         descriptor.thresholds = [("street_address".into(), 0.5)].into();
         descriptor.score_scale = SCALE.into();
         let response = |score: f64| {
-            serde_json::json!({"worker_version":"0.4.5b2","runtime":runtime(),
+            serde_json::json!({"worker_version":"0.4.5b4","runtime":runtime(),
             "findings":[{"label":"street_address","start":2,"end":17,"text":"123 Main Street","raw_score":score}]}).to_string()
         };
         let valid = decode(response(0.9).as_bytes(), 100).unwrap();
@@ -346,7 +433,7 @@ mod tests {
             .is_empty()
         );
         let corrupt =
-            serde_json::json!({"worker_version":"0.4.5b2","runtime":{},"findings":[]}).to_string();
+            serde_json::json!({"worker_version":"0.4.5b4","runtime":{},"findings":[]}).to_string();
         assert!(decode(corrupt.as_bytes(), 100).is_err());
     }
 }