datafog-core 0.4.5b2__tar.gz → 0.4.5b3__tar.gz

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.3"
 dependencies = [
  "base64",
  "futures",
@@ -94,7 +94,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.3"
 dependencies = [
  "datafog-core",
  "pyo3",
@@ -105,7 +105,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-node"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.3"
 dependencies = [
  "datafog-core",
  "napi",
@@ -116,7 +116,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.3"
 dependencies = [
  "datafog-core",
  "serde",

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog-core
-Version: 0.4.5b2
+Version: 0.4.5b3
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -59,7 +59,7 @@ DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bin
 
 Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
 
-**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1)
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.3](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.3)
 
 ## Quickstart
 
@@ -198,20 +198,20 @@ Keep scan settings under `scan` and transformation policy under `transform`. A d
 
 ## Local models (beta)
 
-[0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
+[0.4.5-beta.3](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.3) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
 
 Install the beta for your runtime:
 
 ```bash
-python -m pip install 'datafog-core==0.4.5b1'
+python -m pip install 'datafog-core==0.4.5b3'
 npm install @datafog/node@beta
-cargo add datafog-core@0.4.5-beta.1 --features models
+cargo add datafog-core@0.4.5-beta.3 --features models
 ```
 
 For GLiNER2, install its optional dependencies in the Python environment used by the runner:
 
 ```bash
-python -m pip install 'datafog-core[gliner2]==0.4.5b1'
+python -m pip install 'datafog-core[gliner2]==0.4.5b3'
 ```
 
 The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.

{datafog_core-0.4.5b2/crates/core → datafog_core-0.4.5b3}/README.md

@@ -26,7 +26,7 @@ DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bin
 
 Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
 
-**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1)
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.3](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.3)
 
 ## Quickstart
 
@@ -165,20 +165,20 @@ Keep scan settings under `scan` and transformation policy under `transform`. A d
 
 ## Local models (beta)
 
-[0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
+[0.4.5-beta.3](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.3) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
 
 Install the beta for your runtime:
 
 ```bash
-python -m pip install 'datafog-core==0.4.5b1'
+python -m pip install 'datafog-core==0.4.5b3'
 npm install @datafog/node@beta
-cargo add datafog-core@0.4.5-beta.1 --features models
+cargo add datafog-core@0.4.5-beta.3 --features models
 ```
 
 For GLiNER2, install its optional dependencies in the Python environment used by the runner:
 
 ```bash
-python -m pip install 'datafog-core[gliner2]==0.4.5b1'
+python -m pip install 'datafog-core[gliner2]==0.4.5b3'
 ```
 
 The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.3"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"
@@ -12,7 +12,7 @@ name = "datafog_core"
 crate-type = ["cdylib"]
 
 [dependencies]
-datafog-core = { version = "0.4.5-beta.2", path = "../../crates/core", features = ["models"] }
+datafog-core = { version = "0.4.5-beta.3", path = "../../crates/core", features = ["models"] }
 pyo3 = { version = "0.29", features = ["abi3-py310"] }
 pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] }
 serde_json = "1"

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/bindings/python/src/models.rs

@@ -91,8 +91,49 @@ impl ModelSession {
             .map(|session| Self { session })
             .map_err(|error| model_error(py, error))
     }
-    #[pyo3(signature = (text, *, cancellation=None))]
+    /// Rule detectors handle the entity types they support and the model runs
+    /// only for requested types it alone provides. Omitting `config` equals
+    /// `{}`: every enabled rule detector plus the model's remaining labels.
+    #[pyo3(signature = (text, config=None, *, cancellation=None))]
     fn scan(
+        &self,
+        py: Python<'_>,
+        text: String,
+        config: Option<&Bound<'_, PyAny>>,
+        cancellation: Option<PyRef<'_, ModelCancellation>>,
+    ) -> PyResult<ModelScanResult> {
+        let routed = match config {
+            Some(config) => {
+                let value = py_to_json(py, config, "")?;
+                self.session
+                    .routed_config(&value)
+                    .map_err(|error| model_error(py, error))?
+            }
+            None => self.session.default_routed_config(),
+        };
+        // Rule-only routing never reserves the model worker.
+        let request = if routed.needs_model() {
+            Some(
+                self.session
+                    .prepare_request()
+                    .map_err(|error| model_error(py, error))?,
+            )
+        } else {
+            None
+        };
+        let cancellation = cancellation
+            .map(|v| Arc::clone(&v.cancellation))
+            .unwrap_or_default();
+        py.detach(|| {
+            self.session
+                .scan_routed_request(request.as_ref(), &text, &routed, &cancellation)
+        })
+        .map(ModelScanResult::from)
+        .map_err(|error| model_error(py, error))
+    }
+    /// Raw model output: every model label, with no rule detectors.
+    #[pyo3(signature = (text, *, cancellation=None))]
+    fn scan_model(
         &self,
         py: Python<'_>,
         text: String,
@@ -105,9 +146,12 @@ impl ModelSession {
         let cancellation = cancellation
             .map(|v| Arc::clone(&v.cancellation))
             .unwrap_or_default();
-        py.detach(|| self.session.scan_request(&request, &text, &cancellation))
-            .map(ModelScanResult::from)
-            .map_err(|error| model_error(py, error))
+        py.detach(|| {
+            self.session
+                .scan_model_request(&request, &text, &cancellation)
+        })
+        .map(ModelScanResult::from)
+        .map_err(|error| model_error(py, error))
     }
     fn close(&self, py: Python<'_>) -> PyResult<()> {
         py.detach(|| self.session.close())

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core"
-version = "0.4.5-beta.2"
+version = "0.4.5-beta.3"
 edition = "2024"
 rust-version = "1.88"
 description = "Fast structured PII detection library"

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3/crates/core}/README.md

@@ -26,7 +26,7 @@ DataFog Core is a Rust library with Python, Node.js, and browser WebAssembly bin
 
 Rule-based scanning runs locally without model weights or downloads. Optional local model sessions add DataFog and GLiNER2 support in Rust, Python, and Node.js.
 
-**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1)
+**Stable:** [0.4.4](https://github.com/DataFog/datafog-core/releases/tag/v0.4.4) · **Model beta:** [0.4.5-beta.3](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.3)
 
 ## Quickstart
 
@@ -165,20 +165,20 @@ Keep scan settings under `scan` and transformation policy under `transform`. A d
 
 ## Local models (beta)
 
-[0.4.5-beta.1](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.1) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
+[0.4.5-beta.3](https://github.com/DataFog/datafog-core/releases/tag/v0.4.5-beta.3) adds local sessions for the DataFog PII model and GLiNER2 in Rust, Python, and Node.js on all five native platforms.
 
 Install the beta for your runtime:
 
 ```bash
-python -m pip install 'datafog-core==0.4.5b1'
+python -m pip install 'datafog-core==0.4.5b3'
 npm install @datafog/node@beta
-cargo add datafog-core@0.4.5-beta.1 --features models
+cargo add datafog-core@0.4.5-beta.3 --features models
 ```
 
 For GLiNER2, install its optional dependencies in the Python environment used by the runner:
 
 ```bash
-python -m pip install 'datafog-core[gliner2]==0.4.5b1'
+python -m pip install 'datafog-core[gliner2]==0.4.5b3'
 ```
 
 The GLiNER2 runner ships in the Python package and also serves Node.js and Rust callers. Download and verify model files separately, then use `ModelSession.open`, text `scan`, and `close`. Reuse sessions across calls. Both backends return consistent results and errors, with cancellation and cleanup support.

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/crates/core/src/models/gliner2.rs

@@ -255,7 +255,7 @@ pub(super) fn decode(bytes: &[u8], max_findings: usize) -> Result<BackendResult,
         serde_json::from_value(value).map_err(|_| ModelError::invalid_output())?;
     let expected = runtime();
     if output.runtime != expected
-        || output.worker_version != "0.4.5b2"
+        || output.worker_version != "0.4.5b3"
         || output.findings.len() > max_findings
     {
         return Err(ModelError::invalid_output());
@@ -318,7 +318,7 @@ mod tests {
         descriptor.thresholds = [("street_address".into(), 0.5)].into();
         descriptor.score_scale = SCALE.into();
         let response = |score: f64| {
-            serde_json::json!({"worker_version":"0.4.5b2","runtime":runtime(),
+            serde_json::json!({"worker_version":"0.4.5b3","runtime":runtime(),
             "findings":[{"label":"street_address","start":2,"end":17,"text":"123 Main Street","raw_score":score}]}).to_string()
         };
         let valid = decode(response(0.9).as_bytes(), 100).unwrap();
@@ -346,7 +346,7 @@ mod tests {
             .is_empty()
         );
         let corrupt =
-            serde_json::json!({"worker_version":"0.4.5b2","runtime":{},"findings":[]}).to_string();
+            serde_json::json!({"worker_version":"0.4.5b3","runtime":{},"findings":[]}).to_string();
         assert!(decode(corrupt.as_bytes(), 100).is_err());
     }
 }

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/crates/core/src/models/tests.rs

@@ -277,7 +277,7 @@ fn admission_precedes_scheduling_and_expired_or_cancelled_requests_do_not_infer(
     request.deadline = Instant::now();
     assert_eq!(
         session
-            .scan_request(&request, "secret", &Cancellation::default())
+            .scan_model_request(&request, "secret", &Cancellation::default())
             .unwrap_err()
             .code,
         "model_timeout"
@@ -290,7 +290,7 @@ fn admission_precedes_scheduling_and_expired_or_cancelled_requests_do_not_infer(
     cancellation.cancel();
     assert_eq!(
         session
-            .scan_request(&request, "secret", &cancellation)
+            .scan_model_request(&request, "secret", &cancellation)
             .unwrap_err()
             .code,
         "model_cancelled"
@@ -300,7 +300,7 @@ fn admission_precedes_scheduling_and_expired_or_cancelled_requests_do_not_infer(
     session.request_close();
     assert_eq!(
         session
-            .scan_request(&next, "", &Cancellation::default())
+            .scan_model_request(&next, "", &Cancellation::default())
             .unwrap_err()
             .code,
         "model_cancelled"
@@ -334,3 +334,201 @@ fn native_seven_label_mapping_preserves_names_and_identifier_meaning() {
         assert_eq!(result[0].1.native_label.as_deref(), Some(native));
     }
 }
+
+fn idle_session() -> ModelSession {
+    ModelSession {
+        descriptor: descriptor(),
+        inner: Mutex::new(SessionInner { worker: None }),
+        closing: AtomicBool::new(false),
+        state: AtomicU8::new(1),
+        cleanup_failed: Arc::new(AtomicBool::new(false)),
+        admitted: Arc::new(AtomicBool::new(false)),
+    }
+}
+
+#[test]
+fn routed_rule_entities_never_touch_the_model_worker() {
+    let session = idle_session();
+    let config = session
+        .routed_config(&serde_json::json!({"entities": ["CREDIT_CARD", "EMAIL"]}))
+        .unwrap();
+    assert!(!config.needs_model());
+    let text = "Card 4111 1111 1111 1111, mail may@example.test, call (212) 555-0100.";
+    // Another request holds the worker and the admission flag; rule-only
+    // routing neither waits for them nor reports `model_busy`.
+    let held = session.inner.lock().unwrap();
+    let request = session.prepare_request().unwrap();
+    let result = session
+        .scan_routed(text, &config, &Cancellation::default())
+        .unwrap();
+    let found: Vec<_> = result
+        .findings
+        .iter()
+        .map(|finding| (finding.entity_type.as_str(), finding.matched_text.as_str()))
+        .collect();
+    assert_eq!(
+        found,
+        [
+            ("CREDIT_CARD", "4111 1111 1111 1111"),
+            ("EMAIL", "may@example.test")
+        ]
+    );
+    assert_eq!(result.evidence.len(), 2);
+    for (index, evidence) in result.evidence.iter().enumerate() {
+        assert_eq!(evidence.finding_index, index);
+        assert!(
+            evidence.sources[0]
+                .detector_name
+                .starts_with("datafog-core/")
+        );
+        assert!(evidence.sources[0].raw_score.is_none());
+    }
+    assert_eq!(result.model, "DataFog/pii-en-65m");
+    drop(request);
+    drop(held);
+}
+
+#[test]
+fn routed_scans_keep_the_session_lifecycle_and_limits() {
+    let session = idle_session();
+    let config = session
+        .routed_config(&serde_json::json!({"entities": ["EMAIL"]}))
+        .unwrap();
+    let cancelled = Cancellation::default();
+    cancelled.cancel();
+    assert_eq!(
+        session
+            .scan_routed("may@example.test", &config, &cancelled)
+            .unwrap_err()
+            .code,
+        "model_cancelled"
+    );
+    let oversized = "x".repeat(session.descriptor.limits.max_input_bytes + 1);
+    assert_eq!(
+        session
+            .scan_routed(&oversized, &config, &Cancellation::default())
+            .unwrap_err()
+            .reason,
+        "limit_exceeded"
+    );
+    session.state.store(2, Ordering::Release);
+    assert_eq!(
+        session
+            .scan_routed("may@example.test", &config, &Cancellation::default())
+            .unwrap_err()
+            .code,
+        "model_session_failed"
+    );
+    let session = idle_session();
+    session.close().unwrap();
+    assert_eq!(
+        session
+            .scan_routed("may@example.test", &config, &Cancellation::default())
+            .unwrap_err()
+            .code,
+        "model_session_closed"
+    );
+    // A model-routed configuration must arrive with its reserved request.
+    let session = idle_session();
+    let model = session
+        .routed_config(&serde_json::json!({"entities": ["FIRST_NAME"]}))
+        .unwrap();
+    assert_eq!(
+        session
+            .scan_routed_request(None, "Jane", &model, &Cancellation::default())
+            .unwrap_err()
+            .path,
+        "/request"
+    );
+}
+
+#[test]
+fn routed_configuration_splits_entities_between_rules_and_model() {
+    let session = idle_session();
+    let mixed = session
+        .routed_config(&serde_json::json!({"entities": ["FIRST_NAME", "EMAIL", "CREDIT_CARD"]}))
+        .unwrap();
+    assert!(mixed.needs_model());
+    assert_eq!(
+        mixed.model_entities,
+        BTreeSet::from(["FIRST_NAME".to_owned()])
+    );
+    assert_eq!(
+        mixed.rules.as_ref().and_then(ScanConfig::entities),
+        Some(&BTreeSet::from([
+            "CREDIT_CARD".to_owned(),
+            "EMAIL".to_owned()
+        ]))
+    );
+    // Shared labels such as EMAIL and PHONE go to rules; without a selection
+    // the model returns only the labels rules cannot produce.
+    let everything = session.routed_config(&serde_json::json!({})).unwrap();
+    assert!(everything.rules.as_ref().unwrap().entities().is_none());
+    assert_eq!(
+        everything.model_entities,
+        BTreeSet::from(
+            [
+                "ACCOUNT_NUMBER",
+                "CUSTOMER_ID",
+                "FIRST_NAME",
+                "LAST_NAME",
+                "STREET_ADDRESS"
+            ]
+            .map(String::from)
+        )
+    );
+    // Omitting the configuration routes exactly like an empty configuration.
+    let default = session.default_routed_config();
+    assert_eq!(default.model_entities, everything.model_entities);
+    assert_eq!(default.rules, everything.rules);
+    let model_only = session
+        .routed_config(&serde_json::json!({"entities": ["STREET_ADDRESS"], "locale": "de"}))
+        .unwrap();
+    assert!(model_only.rules.is_none());
+    for (config, reason, path) in [
+        (serde_json::json!([]), "invalid_type", ""),
+        (
+            serde_json::json!({"entities": []}),
+            "empty_value",
+            "/entities",
+        ),
+        (
+            serde_json::json!({"entities": "EMAIL"}),
+            "invalid_type",
+            "/entities",
+        ),
+        (
+            serde_json::json!({"entities": ["EMAIL", 1]}),
+            "invalid_type",
+            "/entities/1",
+        ),
+        (
+            serde_json::json!({"entities": ["EMAIL", "EMAIL"]}),
+            "duplicate_value",
+            "/entities/1",
+        ),
+        (
+            serde_json::json!({"entities": ["EMAIL", "CARD_NUMBER"]}),
+            "invalid_value",
+            "/entities/1",
+        ),
+        (
+            serde_json::json!({"entities": ["STREET_ADDRESS"], "locale": "xx"}),
+            "invalid_value",
+            "/locale",
+        ),
+        (
+            serde_json::json!({"entities": ["EMAIL"], "unknown": true}),
+            "unknown_field",
+            "/unknown",
+        ),
+    ] {
+        let error = session.routed_config(&config).unwrap_err();
+        assert_eq!(error.code, "invalid_configuration", "{config}");
+        assert_eq!(
+            (error.reason.as_str(), error.path.as_str()),
+            (reason, path),
+            "{config}"
+        );
+    }
+}

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/crates/core/src/models.rs

@@ -6,10 +6,10 @@ pub use gliner2::{Gliner2Config, Gliner2Label};
 #[cfg(test)]
 mod tests;
 
-use crate::{Finding, PrivacyError, TextIndex, TextRange};
+use crate::{Finding, PrivacyError, PrivacyErrorReason, ScanConfig, TextIndex, TextRange};
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::fs::File;
 use std::io::Read;
 use std::path::{Component, Path, PathBuf};
@@ -233,6 +233,42 @@ pub struct FindingEvidence {
     pub sources: Vec<SourceEvidence>,
 }
 
+/// Entity routing for a model session scan. Rule detectors handle every
+/// requested entity type they support; the model runs only for requested
+/// types that it alone provides. Validated against one session's labels.
+#[derive(Clone, Debug)]
+pub struct RoutedScanConfig {
+    rules: Option<ScanConfig>,
+    model_entities: BTreeSet<String>,
+}
+
+impl RoutedScanConfig {
+    /// Whether a scan with this configuration runs model inference.
+    pub fn needs_model(&self) -> bool {
+        !self.model_entities.is_empty()
+    }
+}
+
+fn rule_entry(finding: Finding) -> (Finding, SourceEvidence) {
+    let source = SourceEvidence {
+        detector_name: finding.detector_name.clone(),
+        detector_version: finding.detector_version.clone(),
+        native_label: None,
+        raw_score: None,
+        calibrated_confidence: finding.confidence.map(f64::from),
+    };
+    (finding, source)
+}
+
+fn rule_entries(text: &str, config: &RoutedScanConfig) -> Vec<(Finding, SourceEvidence)> {
+    config.rules.as_ref().map_or_else(Vec::new, |rules| {
+        crate::scan_with_config(text, rules)
+            .into_iter()
+            .map(rule_entry)
+            .collect()
+    })
+}
+
 #[derive(Debug, Serialize)]
 pub struct ModelScanResult {
     pub contract_version: u32,
@@ -472,16 +508,28 @@ impl ModelSession {
         Ok(request.deadline)
     }
 
+    /// Scan with default routing: every enabled rule detector, plus the model
+    /// for the labels rules do not produce. Equivalent to an empty routed
+    /// configuration, as an omitted rule scan configuration equals `{}`.
     pub fn scan(
         &self,
         text: &str,
         cancellation: &Cancellation,
+    ) -> Result<ModelScanResult, ModelError> {
+        self.scan_routed(text, &self.default_routed_config(), cancellation)
+    }
+
+    /// Raw model output: every model label, with no rule detectors.
+    pub fn scan_model(
+        &self,
+        text: &str,
+        cancellation: &Cancellation,
     ) -> Result<ModelScanResult, ModelError> {
         let request = self.prepare_request()?;
-        self.scan_request(&request, text, cancellation)
+        self.scan_model_request(&request, text, cancellation)
     }
 
-    pub fn scan_request(
+    pub fn scan_model_request(
         &self,
         request: &ModelRequest,
         text: &str,
@@ -490,25 +538,188 @@ impl ModelSession {
         let deadline = self.request_deadline(request)?;
         self.check(text, deadline, cancellation)?;
         let mut inner = self.lock()?;
-        self.scan_locked(&mut inner, text, deadline, cancellation)
+        self.scan_locked(&mut inner, text, None, deadline, cancellation)
+    }
+
+    /// Validate a routed scan configuration. It accepts the rule scan fields
+    /// (`entities`, `locale`, `detect_uuid`, `format`); `entities` may also
+    /// name this session's model labels. Without `entities`, every enabled
+    /// rule detector runs and the model returns the labels rules do not cover.
+    pub fn routed_config(&self, value: &serde_json::Value) -> Result<RoutedScanConfig, ModelError> {
+        let invalid = |reason, path: &str, message: &str| {
+            ModelError::from(PrivacyError::invalid_configuration(reason, path, message))
+        };
+        let object = value.as_object().ok_or_else(|| {
+            invalid(
+                PrivacyErrorReason::InvalidType,
+                "",
+                "scan configuration must be an object",
+            )
+        })?;
+        let model_labels: BTreeSet<&str> = self
+            .descriptor
+            .labels
+            .values()
+            .map(String::as_str)
+            .collect();
+        let mut rule_object = object.clone();
+        let (rule_entities, model_entities) = match object.get("entities") {
+            None => (None, self.model_only_labels()),
+            Some(entities) => {
+                let entities = entities.as_array().ok_or_else(|| {
+                    invalid(
+                        PrivacyErrorReason::InvalidType,
+                        "/entities",
+                        "entities must be an array",
+                    )
+                })?;
+                if entities.is_empty() {
+                    return Err(invalid(
+                        PrivacyErrorReason::EmptyValue,
+                        "/entities",
+                        "entities must contain at least one entity type",
+                    ));
+                }
+                let mut seen = BTreeSet::new();
+                let mut rules = Vec::new();
+                let mut model = BTreeSet::new();
+                for (index, entity) in entities.iter().enumerate() {
+                    let path = format!("/entities/{index}");
+                    let entity = entity.as_str().ok_or_else(|| {
+                        invalid(
+                            PrivacyErrorReason::InvalidType,
+                            &path,
+                            "entity type must be a string",
+                        )
+                    })?;
+                    if !seen.insert(entity) {
+                        return Err(invalid(
+                            PrivacyErrorReason::DuplicateValue,
+                            &path,
+                            "entity selection contains a duplicate entity type",
+                        ));
+                    }
+                    if crate::capabilities::text_entity_supported(entity) {
+                        rules.push(serde_json::Value::String(entity.to_owned()));
+                    } else if model_labels.contains(entity) {
+                        model.insert(entity.to_owned());
+                    } else {
+                        return Err(invalid(
+                            PrivacyErrorReason::InvalidValue,
+                            &path,
+                            "entity type is not supported by rules or this model",
+                        ));
+                    }
+                }
+                (Some(rules), model)
+            }
+        };
+        // Rule-routed entities were validated above, so remaining errors name
+        // the shared locale, UUID and format fields at their own paths.
+        let run_rules = match rule_entities {
+            None => true,
+            Some(rules) if rules.is_empty() => {
+                rule_object.remove("entities");
+                false
+            }
+            Some(rules) => {
+                rule_object.insert("entities".into(), serde_json::Value::Array(rules));
+                true
+            }
+        };
+        let rules = crate::parse_scan_config(&serde_json::Value::Object(rule_object))?;
+        Ok(RoutedScanConfig {
+            rules: run_rules.then_some(rules),
+            model_entities,
+        })
+    }
+
+    /// The routing used without a configuration: all enabled rule detectors
+    /// and the model labels that rules do not produce.
+    pub fn default_routed_config(&self) -> RoutedScanConfig {
+        RoutedScanConfig {
+            rules: Some(ScanConfig::new()),
+            model_entities: self.model_only_labels(),
+        }
+    }
+
+    fn model_only_labels(&self) -> BTreeSet<String> {
+        self.descriptor
+            .labels
+            .values()
+            .filter(|label| !crate::capabilities::text_entity_supported(label))
+            .cloned()
+            .collect()
+    }
+
+    /// Scan with entity routing. A configuration that needs no model labels
+    /// runs only rule detectors, without reserving or waiting for the worker.
+    pub fn scan_routed(
+        &self,
+        text: &str,
+        config: &RoutedScanConfig,
+        cancellation: &Cancellation,
+    ) -> Result<ModelScanResult, ModelError> {
+        let request = if config.needs_model() {
+            Some(self.prepare_request()?)
+        } else {
+            None
+        };
+        self.scan_routed_request(request.as_ref(), text, config, cancellation)
+    }
+
+    /// Bindings reserve the session before dispatch only when the routed
+    /// configuration needs the model; otherwise `request` is `None`.
+    pub fn scan_routed_request(
+        &self,
+        request: Option<&ModelRequest>,
+        text: &str,
+        config: &RoutedScanConfig,
+        cancellation: &Cancellation,
+    ) -> Result<ModelScanResult, ModelError> {
+        if !config.needs_model() {
+            // Lifecycle, input limits and cancellation apply without the worker.
+            let deadline = deadline(self.descriptor.limits.timeout_ms)?;
+            self.check(text, deadline, cancellation)?;
+            if self.state.load(Ordering::Acquire) == 2 {
+                return Err(ModelError::new(
+                    "model_session_failed",
+                    "failed",
+                    "/backend",
+                ));
+            }
+            let result = self.compose(rule_entries(text, config))?;
+            bounded_json(&result, self.descriptor.limits.max_response_bytes, "/data")?;
+            return Ok(result);
+        }
+        let request = request.ok_or_else(|| ModelError::config("invalid_value", "/request"))?;
+        let deadline = self.request_deadline(request)?;
+        self.check(text, deadline, cancellation)?;
+        let mut inner = self.lock()?;
+        self.scan_locked(&mut inner, text, Some(config), deadline, cancellation)
     }
 
     fn scan_locked(
         &self,
         inner: &mut SessionInner,
         text: &str,
+        routing: Option<&RoutedScanConfig>,
         deadline: Instant,
         cancellation: &Cancellation,
     ) -> Result<ModelScanResult, ModelError> {
         let native = self.native(inner, text, deadline, cancellation)?;
         let converted = convert(text, native, &self.descriptor);
-        let entries = match converted {
+        let mut entries = match converted {
             Ok(entries) => entries,
             Err(error) => {
                 self.retire(inner)?;
                 return Err(error);
             }
         };
+        if let Some(config) = routing {
+            entries.retain(|(finding, _)| config.model_entities.contains(&finding.entity_type));
+            entries.splice(0..0, rule_entries(text, config));
+        }
         self.active_check(inner, text, deadline, cancellation)?;
         let result = self.compose(entries)?;
         bounded_json(&result, self.descriptor.limits.max_response_bytes, "/data")?;

datafog_core-0.4.5b3/crates/core/tests/model_datafog.rs

@@ -0,0 +1,118 @@
+#![cfg(all(feature = "models", not(target_arch = "wasm32")))]
+use datafog_core::models::{Cancellation, ModelSession, ModelSessionConfig};
+
+/// Explicit real-model integration. Missing config or assets fail the test.
+#[test]
+#[ignore = "requires explicitly provisioned local model"]
+fn pinned_model_execution() {
+    let config = std::env::var("DATAFOG_MODEL_CONFIG")
+        .expect("DATAFOG_MODEL_CONFIG must name a verified local session config");
+    let config: ModelSessionConfig =
+        serde_json::from_reader(std::fs::File::open(config).unwrap()).unwrap();
+    let session = ModelSession::open(config).unwrap();
+    let text = "👋 John Smith lives at 123 Main Street. Email john@example.com.";
+    let result = session.scan_model(text, &Cancellation::default()).unwrap();
+    assert!(!result.findings.is_empty());
+    assert_eq!(result.contract_version, 1);
+    assert_eq!(result.evidence.len(), result.findings.len());
+    for finding in &result.findings {
+        assert_eq!(
+            &text[finding.byte_range.start..finding.byte_range.end],
+            finding.matched_text
+        );
+        assert_eq!(
+            text.chars()
+                .skip(finding.codepoint_range.start)
+                .take(finding.codepoint_range.end - finding.codepoint_range.start)
+                .collect::<String>(),
+            finding.matched_text
+        );
+    }
+    assert!(
+        session
+            .scan("", &Cancellation::default())
+            .unwrap()
+            .findings
+            .is_empty()
+    );
+
+    // Routing: rule entities come from rule detectors; model entities are the
+    // plain model scan's findings for those types, and nothing else is returned.
+    let text =
+        "John Smith lives at 123 Main Street. Card 4111 1111 1111 1111, email john@example.com.";
+    let rules_only = session
+        .routed_config(&serde_json::json!({"entities": ["CREDIT_CARD", "EMAIL"]}))
+        .unwrap();
+    assert!(!rules_only.needs_model());
+    let routed = session
+        .scan_routed(text, &rules_only, &Cancellation::default())
+        .unwrap();
+    let rule_findings = datafog_core::scan_with_config(
+        text,
+        &datafog_core::ScanConfig::new()
+            .with_entities(vec!["CREDIT_CARD".into(), "EMAIL".into()])
+            .unwrap(),
+    );
+    assert_eq!(routed.findings, rule_findings);
+    let mixed = session
+        .routed_config(&serde_json::json!({"entities": ["STREET_ADDRESS", "EMAIL"]}))
+        .unwrap();
+    assert!(mixed.needs_model());
+    let routed = session
+        .scan_routed(text, &mixed, &Cancellation::default())
+        .unwrap();
+    let model_addresses: Vec<_> = session
+        .scan_model(text, &Cancellation::default())
+        .unwrap()
+        .findings
+        .into_iter()
+        .filter(|finding| finding.entity_type == "STREET_ADDRESS")
+        .collect();
+    let routed_addresses: Vec<_> = routed
+        .findings
+        .iter()
+        .filter(|finding| finding.entity_type == "STREET_ADDRESS")
+        .cloned()
+        .collect();
+    assert_eq!(routed_addresses, model_addresses);
+    let emails: Vec<_> = routed
+        .findings
+        .iter()
+        .filter(|finding| finding.entity_type == "EMAIL")
+        .collect();
+    assert_eq!(emails.len(), 1);
+    assert_eq!(emails[0].detector_name, "datafog-core/email");
+    assert!(
+        routed
+            .findings
+            .iter()
+            .all(|finding| matches!(finding.entity_type.as_str(), "STREET_ADDRESS" | "EMAIL"))
+    );
+    assert_eq!(routed.evidence.len(), routed.findings.len());
+
+    // Without a configuration, scan routes like `{}`: rule findings as from
+    // the rule scanner, plus the model's labels that rules do not produce.
+    let default = session.scan(text, &Cancellation::default()).unwrap();
+    let empty = session
+        .scan_routed(
+            text,
+            &session.routed_config(&serde_json::json!({})).unwrap(),
+            &Cancellation::default(),
+        )
+        .unwrap();
+    assert_eq!(default.findings, empty.findings);
+    let rules: Vec<_> = default
+        .findings
+        .iter()
+        .filter(|finding| !finding.detector_name.starts_with("datafog-core/model/"))
+        .cloned()
+        .collect();
+    assert_eq!(rules, datafog_core::scan(text));
+
+    session.close().unwrap();
+    session.close().unwrap();
+    assert_eq!(
+        session.scan("", &Cancellation::default()).unwrap_err().code,
+        "model_session_closed"
+    );
+}

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "datafog-core"
-version = "0.4.5b2"
+version = "0.4.5b3"
 description = "Fast structured PII detection implemented in Rust"
 readme = "README.md"
 license = "MIT"

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/python/datafog_core/__init__.pyi

@@ -63,7 +63,8 @@ class ModelCancellation:
 class ModelSession:
     @staticmethod
     def open(config: _ModelConfig) -> ModelSession: ...
-    def scan(self, text: str, *, cancellation: ModelCancellation | None = None) -> ModelScanResult: ...
+    def scan(self, text: str, config: _ScanConfig | None = None, *, cancellation: ModelCancellation | None = None) -> ModelScanResult: ...
+    def scan_model(self, text: str, *, cancellation: ModelCancellation | None = None) -> ModelScanResult: ...
     def close(self) -> None: ...
     def __enter__(self) -> ModelSession: ...
     def __exit__(self, kind: object, value: object, traceback: object) -> bool: ...

{datafog_core-0.4.5b2 → datafog_core-0.4.5b3}/python/datafog_core/_gliner2_worker.py

@@ -11,7 +11,7 @@ import platform
 from pathlib import Path
 import sys
 
-__version__ = "0.4.5b2"
+__version__ = "0.4.5b3"
 RUNTIME = {"gliner2": "2.1.0", "transformers": "5.8.0", "torch": "2.12.0"}
 if sys.platform == "darwin" and platform.machine() == "x86_64":
     RUNTIME.update(transformers="4.57.6", torch="2.2.2")

datafog_core-0.4.5b2/crates/core/tests/model_datafog.rs

@@ -1,44 +0,0 @@
-#![cfg(all(feature = "models", not(target_arch = "wasm32")))]
-use datafog_core::models::{Cancellation, ModelSession, ModelSessionConfig};
-
-/// Explicit real-model integration. Missing config or assets fail the test.
-#[test]
-#[ignore = "requires explicitly provisioned local model"]
-fn pinned_model_execution() {
-    let config = std::env::var("DATAFOG_MODEL_CONFIG")
-        .expect("DATAFOG_MODEL_CONFIG must name a verified local session config");
-    let config: ModelSessionConfig =
-        serde_json::from_reader(std::fs::File::open(config).unwrap()).unwrap();
-    let session = ModelSession::open(config).unwrap();
-    let text = "👋 John Smith lives at 123 Main Street. Email john@example.com.";
-    let result = session.scan(text, &Cancellation::default()).unwrap();
-    assert!(!result.findings.is_empty());
-    assert_eq!(result.contract_version, 1);
-    assert_eq!(result.evidence.len(), result.findings.len());
-    for finding in &result.findings {
-        assert_eq!(
-            &text[finding.byte_range.start..finding.byte_range.end],
-            finding.matched_text
-        );
-        assert_eq!(
-            text.chars()
-                .skip(finding.codepoint_range.start)
-                .take(finding.codepoint_range.end - finding.codepoint_range.start)
-                .collect::<String>(),
-            finding.matched_text
-        );
-    }
-    assert!(
-        session
-            .scan("", &Cancellation::default())
-            .unwrap()
-            .findings
-            .is_empty()
-    );
-    session.close().unwrap();
-    session.close().unwrap();
-    assert_eq!(
-        session.scan("", &Cancellation::default()).unwrap_err().code,
-        "model_session_closed"
-    );
-}