datafog-core 0.2.0__tar.gz → 0.3.0__tar.gz

{datafog_core-0.2.0 → datafog_core-0.3.0}/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
  "base64",
  "futures",
@@ -94,7 +94,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
  "datafog-core",
  "pyo3",
@@ -104,7 +104,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-node"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
  "datafog-core",
  "napi",
@@ -115,7 +115,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
  "datafog-core",
  "serde",

{datafog_core-0.2.0 → datafog_core-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog-core
-Version: 0.2.0
+Version: 0.3.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -32,6 +32,12 @@ entity type, matched text, byte range, code-point range,
 optional confidence, detector name, optional detector version
 ```
 
+Structured JSON scanning additionally discovers `PERSON` from documented name-field
+aliases or explicit JSON Pointer mappings. It scans every string value with the
+existing detectors and returns field paths plus string-local findings. No model
+or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
+for `scan_structured` / `scanStructured` and structured transformation APIs.
+
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score. Node.js and browser WASM also
@@ -85,6 +91,28 @@ detection settings remain separate from transformation policy.
 | Node.js | `@datafog/node` | `@datafog/node` | Published |
 | Browser/WASM | `@datafog/wasm` | `@datafog/wasm` | Published |
 
+## Migrating from DataFog Python
+
+DataFog Core is a separate distribution and canonical API, not a drop-in
+replacement for the established `datafog` Python package.
+
+| DataFog Python 4.8.x | DataFog Core 0.3.x |
+| --- | --- |
+| `pip install datafog` | `pip install datafog-core` |
+| `from datafog.engine import ...` | `from datafog_core import ...` |
+| `scan(...).entities` | `scan(...)` returns `list[Finding]` |
+| `scan_and_redact(...)` | `scan_and_transform(...)` |
+| `result.redacted_text` | `result.text` |
+| `Entity.type`, `.text`, `.start`, `.end` | `Finding.entity_type`, `.matched_text`, `.byte_range`, `.codepoint_range` |
+
+Do not mechanically rename legacy `token` to Core `tokenize`: Core tokenization
+is provider-backed and reversible. For non-reversible output, use `redact`,
+`mask`, or `remove`; use keyed `pseudonymize` when stable linkage is required.
+
+See the dedicated [DataFog Python migration
+guide](docs/guides/migrating-from-datafog-python.mdx) for the API mapping,
+strategy differences, entity names, range semantics, and migration checklist.
+
 ## Quick start
 
 ### Rust

{datafog_core-0.2.0/crates/core → datafog_core-0.3.0}/README.md

@@ -9,6 +9,12 @@ entity type, matched text, byte range, code-point range,
 optional confidence, detector name, optional detector version
 ```
 
+Structured JSON scanning additionally discovers `PERSON` from documented name-field
+aliases or explicit JSON Pointer mappings. It scans every string value with the
+existing detectors and returns field paths plus string-local findings. No model
+or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
+for `scan_structured` / `scanStructured` and structured transformation APIs.
+
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score. Node.js and browser WASM also
@@ -62,6 +68,28 @@ detection settings remain separate from transformation policy.
 | Node.js | `@datafog/node` | `@datafog/node` | Published |
 | Browser/WASM | `@datafog/wasm` | `@datafog/wasm` | Published |
 
+## Migrating from DataFog Python
+
+DataFog Core is a separate distribution and canonical API, not a drop-in
+replacement for the established `datafog` Python package.
+
+| DataFog Python 4.8.x | DataFog Core 0.3.x |
+| --- | --- |
+| `pip install datafog` | `pip install datafog-core` |
+| `from datafog.engine import ...` | `from datafog_core import ...` |
+| `scan(...).entities` | `scan(...)` returns `list[Finding]` |
+| `scan_and_redact(...)` | `scan_and_transform(...)` |
+| `result.redacted_text` | `result.text` |
+| `Entity.type`, `.text`, `.start`, `.end` | `Finding.entity_type`, `.matched_text`, `.byte_range`, `.codepoint_range` |
+
+Do not mechanically rename legacy `token` to Core `tokenize`: Core tokenization
+is provider-backed and reversible. For non-reversible output, use `redact`,
+`mask`, or `remove`; use keyed `pseudonymize` when stable linkage is required.
+
+See the dedicated [DataFog Python migration
+guide](docs/guides/migrating-from-datafog-python.mdx) for the API mapping,
+strategy differences, entity names, range semantics, and migration checklist.
+
 ## Quick start
 
 ### Rust

{datafog_core-0.2.0 → datafog_core-0.3.0}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"

{datafog_core-0.2.0 → datafog_core-0.3.0}/bindings/python/src/lib.rs

@@ -320,14 +320,7 @@ impl From<core::RestoreResult> for RestoreResult {
             restorations: result
                 .restorations
                 .into_iter()
-                .map(|record| Restoration {
-                    source_byte_range: record.source_byte_range.into(),
-                    source_codepoint_range: record.source_codepoint_range.into(),
-                    output_byte_range: record.output_byte_range.into(),
-                    output_codepoint_range: record.output_codepoint_range.into(),
-                    token_ref: record.token_ref,
-                    resolved_token_version: record.resolved_token_version,
-                })
+                .map(Restoration::from)
                 .collect(),
         }
     }
@@ -571,6 +564,124 @@ struct PrivacyManager {
 
 #[pymethods]
 impl PrivacyManager {
+    #[pyo3(signature = (data, findings, config, context=None))]
+    fn transform_structured<'py>(
+        &self,
+        py: Python<'py>,
+        data: Py<PyAny>,
+        findings: Vec<Py<StructuredFinding>>,
+        config: Py<PyAny>,
+        context: Option<Py<PyAny>>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let data = structured_data(py, data.bind(py))?;
+        let config_value = structured_options(py, config.bind(py), "")?;
+        let config = core::parse_transformation_config(&config_value)
+            .map_err(|error| privacy_error(py, error))?;
+        let findings = findings
+            .iter()
+            .map(|finding| finding.bind(py).borrow().to_core())
+            .collect::<Vec<_>>();
+        let context = context
+            .map(|context| structured_options(py, context.bind(py), ""))
+            .transpose()?
+            .map(|value| core::parse_privacy_context(&value))
+            .transpose()
+            .map_err(|error| privacy_error(py, error))?;
+        let key_provider = self
+            .key_provider
+            .as_ref()
+            .map(|provider| provider.clone_ref(py));
+        let token_provider = self
+            .token_provider
+            .as_ref()
+            .map(|provider| provider.clone_ref(py));
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let manager = core::PrivacyManager::new(PythonKeyProvider {
+                provider: key_provider,
+            })
+            .with_token_provider(PythonTokenProvider {
+                provider: token_provider,
+            });
+            let result = manager
+                .transform_structured(&data, &findings, &config, context.as_ref())
+                .await
+                .map_err(|error| Python::attach(|py| privacy_error(py, error)))?;
+            Python::attach(|py| Py::new(py, structured_transform_result(py, result)?))
+        })
+    }
+    #[pyo3(signature = (data, config, context=None))]
+    fn scan_and_transform_structured<'py>(
+        &self,
+        py: Python<'py>,
+        data: Py<PyAny>,
+        config: Py<PyAny>,
+        context: Option<Py<PyAny>>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let data = structured_data(py, data.bind(py))?;
+        let config_value = structured_options(py, config.bind(py), "")?;
+        let config = core::structured::parse_scan_and_transform_config(&config_value)
+            .map_err(|error| privacy_error(py, error))?;
+        let context = context
+            .map(|context| structured_options(py, context.bind(py), ""))
+            .transpose()?
+            .map(|value| core::parse_privacy_context(&value))
+            .transpose()
+            .map_err(|error| privacy_error(py, error))?;
+        let key_provider = self
+            .key_provider
+            .as_ref()
+            .map(|provider| provider.clone_ref(py));
+        let token_provider = self
+            .token_provider
+            .as_ref()
+            .map(|provider| provider.clone_ref(py));
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let manager = core::PrivacyManager::new(PythonKeyProvider {
+                provider: key_provider,
+            })
+            .with_token_provider(PythonTokenProvider {
+                provider: token_provider,
+            });
+            let result = manager
+                .scan_and_transform_structured(&data, &config, context.as_ref())
+                .await
+                .map_err(|error| Python::attach(|py| privacy_error(py, error)))?;
+            Python::attach(|py| Py::new(py, structured_transform_result(py, result)?))
+        })
+    }
+    fn restore_structured<'py>(
+        &self,
+        py: Python<'py>,
+        data: Py<PyAny>,
+        context: Py<PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let data = structured_data(py, data.bind(py))?;
+        let context = structured_options(py, context.bind(py), "")?;
+        let context =
+            core::parse_privacy_context(&context).map_err(|error| privacy_error(py, error))?;
+        let key_provider = self
+            .key_provider
+            .as_ref()
+            .map(|provider| provider.clone_ref(py));
+        let token_provider = self
+            .token_provider
+            .as_ref()
+            .map(|provider| provider.clone_ref(py));
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            let manager = core::PrivacyManager::new(PythonKeyProvider {
+                provider: key_provider,
+            })
+            .with_token_provider(PythonTokenProvider {
+                provider: token_provider,
+            });
+            let result = manager
+                .restore_structured(&data, &context)
+                .await
+                .map_err(|error| Python::attach(|py| privacy_error(py, error)))?;
+            Python::attach(|py| Py::new(py, structured_restore_result(py, result)?))
+        })
+    }
+
     #[new]
     #[pyo3(signature = (provider=None, token_provider=None))]
     fn new(
@@ -899,6 +1010,260 @@ fn scan_and_transform(
         .map_err(|error| privacy_error(py, error))
 }
 
+#[pyclass(frozen, skip_from_py_object)]
+#[derive(Clone)]
+struct FieldMapping {
+    #[pyo3(get)]
+    path: String,
+    #[pyo3(get)]
+    entity_type: String,
+    #[pyo3(get)]
+    source: String,
+    #[pyo3(get)]
+    rule: String,
+}
+impl From<core::structured::FieldMapping> for FieldMapping {
+    fn from(mapping: core::structured::FieldMapping) -> Self {
+        Self {
+            path: mapping.path,
+            entity_type: mapping.entity_type,
+            source: mapping.source,
+            rule: mapping.rule,
+        }
+    }
+}
+
+#[pyclass(frozen, skip_from_py_object)]
+#[derive(Clone)]
+struct StructuredFinding {
+    #[pyo3(get)]
+    path: String,
+    #[pyo3(get)]
+    finding: Finding,
+}
+
+#[pyclass(frozen, skip_from_py_object)]
+struct StructuredScanResult {
+    #[pyo3(get)]
+    mappings: Vec<FieldMapping>,
+    #[pyo3(get)]
+    findings: Vec<StructuredFinding>,
+}
+
+fn structured_data(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<serde_json::Value> {
+    // The stdlib encoder rejects cycles/deep recursion safely. Check dictionaries
+    // and sequences first so it cannot silently coerce keys or tuple values.
+    let mut pending = vec![value.clone()];
+    let mut seen = std::collections::BTreeSet::new();
+    while let Some(value) = pending.pop() {
+        if value.is_none()
+            || value.is_instance_of::<PyBool>()
+            || value.is_instance_of::<pyo3::types::PyString>()
+            || value.is_instance_of::<pyo3::types::PyInt>()
+            || value.is_instance_of::<pyo3::types::PyFloat>()
+        {
+            continue;
+        }
+        if !seen.insert(value.as_ptr() as usize) {
+            // Repeated containers are valid JSON trees after serialization;
+            // the encoder below distinguishes shared references from cycles.
+            continue;
+        }
+        if let Ok(object) = value.cast::<PyDict>() {
+            for (key, child) in object.iter() {
+                if !key.is_instance_of::<pyo3::types::PyString>() {
+                    return Err(privacy_error(py, core::structured::invalid_data()));
+                }
+                pending.push(child);
+            }
+        } else if let Ok(array) = value.cast::<PyList>() {
+            pending.extend(array.iter());
+        } else {
+            return Err(privacy_error(py, core::structured::invalid_data()));
+        }
+    }
+    let kwargs = PyDict::new(py);
+    kwargs.set_item("allow_nan", false)?;
+    let json: String = py
+        .import("json")?
+        .call_method("dumps", (value,), Some(&kwargs))
+        .and_then(|value| value.extract())
+        .map_err(|_| privacy_error(py, core::structured::invalid_data()))?;
+    core::structured::parse_document_json(&json).map_err(|error| privacy_error(py, error))
+}
+
+fn structured_config(
+    py: Python<'_>,
+    config: Option<&Bound<'_, PyAny>>,
+) -> PyResult<core::structured::StructuredScanConfig> {
+    match config {
+        Some(config) => core::structured::parse_scan_config(&structured_options(py, config, "")?)
+            .map_err(|error| privacy_error(py, error)),
+        None => Ok(core::structured::StructuredScanConfig::default()),
+    }
+}
+
+#[pyfunction]
+#[pyo3(signature = (data, config=None))]
+fn discover_fields(
+    py: Python<'_>,
+    data: &Bound<'_, PyAny>,
+    config: Option<&Bound<'_, PyAny>>,
+) -> PyResult<Vec<FieldMapping>> {
+    core::structured::discover_fields(&structured_data(py, data)?, &structured_config(py, config)?)
+        .map(|mappings| mappings.into_iter().map(FieldMapping::from).collect())
+        .map_err(|error| privacy_error(py, error))
+}
+
+#[pyfunction]
+#[pyo3(signature = (data, config=None))]
+fn scan_structured(
+    py: Python<'_>,
+    data: &Bound<'_, PyAny>,
+    config: Option<&Bound<'_, PyAny>>,
+) -> PyResult<StructuredScanResult> {
+    let result =
+        core::structured::scan(&structured_data(py, data)?, &structured_config(py, config)?)
+            .map_err(|error| privacy_error(py, error))?;
+    Ok(StructuredScanResult {
+        mappings: result
+            .mappings
+            .into_iter()
+            .map(FieldMapping::from)
+            .collect(),
+        findings: result
+            .findings
+            .into_iter()
+            .map(|located| StructuredFinding {
+                path: located.path,
+                finding: Finding::from(located.finding),
+            })
+            .collect(),
+    })
+}
+
+#[pymethods]
+impl StructuredFinding {
+    #[new]
+    fn new(path: String, finding: PyRef<'_, Finding>) -> Self {
+        Self {
+            path,
+            finding: finding.clone(),
+        }
+    }
+}
+impl StructuredFinding {
+    fn to_core(&self) -> core::structured::StructuredFinding {
+        core::structured::StructuredFinding {
+            path: self.path.clone(),
+            finding: self.finding.to_core(),
+        }
+    }
+}
+
+#[pyclass(frozen, skip_from_py_object)]
+#[derive(Clone)]
+struct StructuredTransformation {
+    #[pyo3(get)]
+    path: String,
+    #[pyo3(get)]
+    transformation: Transformation,
+}
+#[pyclass(frozen, skip_from_py_object)]
+struct StructuredTransformResult {
+    #[pyo3(get)]
+    data: Py<PyAny>,
+    #[pyo3(get)]
+    transformations: Vec<StructuredTransformation>,
+}
+#[pyclass(frozen, skip_from_py_object)]
+#[derive(Clone)]
+struct StructuredRestoration {
+    #[pyo3(get)]
+    path: String,
+    #[pyo3(get)]
+    restoration: Restoration,
+}
+#[pyclass(frozen, skip_from_py_object)]
+struct StructuredRestoreResult {
+    #[pyo3(get)]
+    data: Py<PyAny>,
+    #[pyo3(get)]
+    restorations: Vec<StructuredRestoration>,
+}
+fn structured_transform_result(
+    py: Python<'_>,
+    result: core::structured::StructuredTransformResult,
+) -> PyResult<StructuredTransformResult> {
+    Ok(StructuredTransformResult {
+        data: py
+            .import("json")?
+            .call_method1("loads", (result.data.to_string(),))?
+            .unbind(),
+        transformations: result
+            .transformations
+            .into_iter()
+            .map(|record| StructuredTransformation {
+                path: record.path,
+                transformation: Transformation::from(record.transformation),
+            })
+            .collect(),
+    })
+}
+fn structured_restore_result(
+    py: Python<'_>,
+    result: core::structured::StructuredRestoreResult,
+) -> PyResult<StructuredRestoreResult> {
+    Ok(StructuredRestoreResult {
+        data: py
+            .import("json")?
+            .call_method1("loads", (result.data.to_string(),))?
+            .unbind(),
+        restorations: result
+            .restorations
+            .into_iter()
+            .map(|record| StructuredRestoration {
+                path: record.path,
+                restoration: Restoration::from(record.restoration),
+            })
+            .collect(),
+    })
+}
+
+#[pyfunction]
+fn transform_structured(
+    py: Python<'_>,
+    data: &Bound<'_, PyAny>,
+    findings: Vec<Py<StructuredFinding>>,
+    config: &Bound<'_, PyAny>,
+) -> PyResult<StructuredTransformResult> {
+    let data = structured_data(py, data)?;
+    let config = core::parse_transformation_config(&structured_options(py, config, "")?)
+        .map_err(|error| privacy_error(py, error))?;
+    let findings: Vec<_> = findings
+        .iter()
+        .map(|finding| finding.bind(py).borrow().to_core())
+        .collect();
+    let result = core::structured::transform(&data, &findings, &config)
+        .map_err(|error| privacy_error(py, error))?;
+    structured_transform_result(py, result)
+}
+
+#[pyfunction]
+fn scan_and_transform_structured(
+    py: Python<'_>,
+    data: &Bound<'_, PyAny>,
+    config: &Bound<'_, PyAny>,
+) -> PyResult<StructuredTransformResult> {
+    let data = structured_data(py, data)?;
+    let config =
+        core::structured::parse_scan_and_transform_config(&structured_options(py, config, "")?)
+            .map_err(|error| privacy_error(py, error))?;
+    let result = core::structured::scan_and_transform(&data, &config)
+        .map_err(|error| privacy_error(py, error))?;
+    structured_transform_result(py, result)
+}
+
 #[pymodule]
 fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add(
@@ -917,6 +1282,17 @@ fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> {
         "DataFogKeyProviderError",
         module.py().get_type::<DataFogKeyProviderError>(),
     )?;
+    module.add_class::<FieldMapping>()?;
+    module.add_class::<StructuredFinding>()?;
+    module.add_class::<StructuredScanResult>()?;
+    module.add_class::<StructuredTransformation>()?;
+    module.add_class::<StructuredTransformResult>()?;
+    module.add_class::<StructuredRestoration>()?;
+    module.add_class::<StructuredRestoreResult>()?;
+    module.add_function(wrap_pyfunction!(transform_structured, module)?)?;
+    module.add_function(wrap_pyfunction!(scan_and_transform_structured, module)?)?;
+    module.add_function(wrap_pyfunction!(discover_fields, module)?)?;
+    module.add_function(wrap_pyfunction!(scan_structured, module)?)?;
     module.add_class::<TextRange>()?;
     module.add_class::<Finding>()?;
     module.add_class::<Transformation>()?;
@@ -929,3 +1305,30 @@ fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(scan_and_transform, module)?)?;
     Ok(())
 }
+
+impl From<core::Restoration> for Restoration {
+    fn from(record: core::Restoration) -> Self {
+        Restoration {
+            source_byte_range: record.source_byte_range.into(),
+            source_codepoint_range: record.source_codepoint_range.into(),
+            output_byte_range: record.output_byte_range.into(),
+            output_codepoint_range: record.output_codepoint_range.into(),
+            token_ref: record.token_ref,
+            resolved_token_version: record.resolved_token_version,
+        }
+    }
+}
+
+fn structured_options(
+    py: Python<'_>,
+    value: &Bound<'_, PyAny>,
+    path: &str,
+) -> PyResult<serde_json::Value> {
+    structured_data(py, value).map_err(|_| {
+        configuration_conversion_error(
+            py,
+            path,
+            "structured request options must be JSON-compatible",
+        )
+    })
+}

{datafog_core-0.2.0 → datafog_core-0.3.0}/bindings/python/tests/test_installed.py

@@ -14,6 +14,10 @@ from datafog_core import (
     PrivacyManager,
     TextRange,
     scan,
+    scan_structured,
+    transform_structured,
+    scan_and_transform_structured,
+    discover_fields,
     scan_and_transform,
     transform,
 )
@@ -64,7 +68,54 @@ def verify_fixture(name: str) -> None:
         verify_contract(record["text"])
 
 
+def verify_structured() -> None:
+    for line in (ROOT / "fixtures" / "structured.jsonl").read_text().splitlines():
+        record = json.loads(line)
+        result = scan_structured(record["data"], record.get("config"))
+        mappings = [dict(path=m.path, entity_type=m.entity_type, source=m.source, rule=m.rule) for m in result.mappings]
+        assert mappings == record["mappings"], record["id"]
+        discovered = discover_fields(record["data"], record.get("config"))
+        assert [m.path for m in discovered] == [m.path for m in result.mappings]
+        actual = []
+        for located in result.findings:
+            text = record["data"]
+            for part in located.path[1:].split("/"):
+                key = part.replace("~1", "/").replace("~0", "~")
+                text = text[int(key)] if isinstance(text, list) else text[key]
+            f = located.finding
+            assert text.encode()[f.byte_range.start:f.byte_range.end].decode() == f.matched_text
+            assert text[f.codepoint_range.start:f.codepoint_range.end] == f.matched_text
+            assert f.confidence is None
+            actual.append(dict(path=located.path, label=f.entity_type, text=f.matched_text, start=f.codepoint_range.start, end=f.codepoint_range.end))
+        assert actual == record["findings"], record["id"]
+    for line in (ROOT / "fixtures" / "structured-transform.jsonl").read_text().splitlines():
+        record = json.loads(line)
+        result = scan_and_transform_structured(record["data"],record["config"])
+        explicit = transform_structured(record["data"],scan_structured(record["data"]).findings,record["config"]["transform"])
+        assert result.data == record["expected_data"], record["id"]
+        assert explicit.data == result.data
+        assert all(not hasattr(r.transformation,"matched_text") for r in result.transformations)
+    cycle = {}
+    cycle["cycle"] = cycle
+    try:
+        scan_structured({}, cycle)
+    except DataFogConfigurationError:
+        pass
+    else:
+        raise AssertionError("cyclic options accepted")
+    for data in [None, "secret-value", {1: "name"}, {"n": 2**100}, {"n": float("nan")}, {"n": float("inf")}, {"tuple": (1, 2)}, cycle]:
+        try:
+            scan_structured(data)
+        except DataFogConfigurationError as error:
+            assert error.code == "invalid_configuration"
+            assert error.path == "/data"
+            assert "secret-value" not in str(error)
+        else:
+            raise AssertionError("invalid structured input accepted")
+
+
 def main() -> None:
+    verify_structured()
     verify_fixture("development.jsonl")
     verify_fixture("final.jsonl")
     emoji_finding = scan("👋 jane@example.com")[0]
@@ -307,6 +358,27 @@ def main() -> None:
                 results.append({"id": item["id"], "value": record[3]})
             return results
 
+    async def structured_round_trip():
+        original = {"users":[{"first_name":"👋 José"},{"full_name":"May"}],"count":2}
+        provider = Provider()
+        pseudonyms = await PrivacyManager(provider).scan_and_transform_structured(original,{"transform":pseudonym_config})
+        assert len(provider.calls) == 1
+        assert pseudonyms.data != original
+        manager = PrivacyManager(None,TokenProvider())
+        context = {"scope":"tenant/α"}
+        config = {"transform":{"default":{"strategy":"tokenize","token_ref":"names"}}}
+        tokens = await manager.scan_and_transform_structured(original,config,context)
+        restored = await manager.restore_structured(tokens.data,context)
+        assert restored.data == original
+        assert len(restored.restorations) == 2
+        try:
+            await manager.restore_structured(tokens.data,{"scope":"wrong"})
+        except DataFogKeyProviderError as error:
+            assert error.code == "token_access_denied"
+        else:
+            raise AssertionError("wrong scope was accepted")
+    asyncio.run(structured_round_trip())
+
     async def token_round_trip():
         manager = PrivacyManager(None, TokenProvider())
         context = {"scope": "tenant/α"}

{datafog_core-0.2.0 → datafog_core-0.3.0}/crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2024"
 rust-version = "1.88"
 description = "Fast structured PII detection library"