PyReverseETL 3.1.7__tar.gz → 3.3.0__tar.gz

{pyreverseetl-3.1.7 → pyreverseetl-3.3.0}/Cargo.lock

@@ -3199,7 +3199,7 @@ dependencies = [
 
 [[package]]
 name = "pyreverseetl"
-version = "3.1.7"
+version = "3.3.0"
 dependencies = [
  "chrono",
  "pyo3",
@@ -3210,7 +3210,7 @@ dependencies = [
 
 [[package]]
 name = "pyreverseetl-core"
-version = "3.1.7"
+version = "3.3.0"
 dependencies = [
  "async-trait",
  "aws-config",

{pyreverseetl-3.1.7 → pyreverseetl-3.3.0}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["core", "python"]
 resolver = "2"
 
 [workspace.package]
-version = "3.1.7"
+version = "3.3.0"
 authors = ["Georgi Mammen Mullassery <mullassery@gmail.com>"]
 license = "Apache-2.0"
 repository = "https://github.com/Mullassery/PyReverseETL"

{pyreverseetl-3.1.7 → pyreverseetl-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyReverseETL
-Version: 3.1.7
+Version: 3.3.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -249,6 +249,21 @@ Python CLI / API  →  pyreverseetl._core (PyO3 bindings)  →  pyreverseetl_cor
 - **`python/pyreverseetl/`** — the installed Python package: `cli.py` (the `pyreverseetl`
   command) and `server.py` (an optional Flask REST wrapper around the same engine).
 
+## Validation gate
+
+```bash
+# File-based report
+pyreverseetl execute-activation my-activation --validation-report statguardian-report.json
+
+# Live StatGuardian service: validate a sample before the real sync
+pyreverseetl execute-activation my-activation --statguardian-url http://127.0.0.1:8000
+```
+
+A file report must have `"passed": true`; anything else blocks the sync. The
+live `--statguardian-url` gate dry-runs a sample and calls the service's
+`/validate`; a failure or network error blocks the real sync. It is a sample
+gate, not a full-table scan.
+
 ## Documentation
 
 - [ROADMAP_HONEST.md](ROADMAP_HONEST.md) — what's tested vs. untested vs. broken vs. not

{pyreverseetl-3.1.7 → pyreverseetl-3.3.0}/README.md

@@ -226,6 +226,21 @@ Python CLI / API  →  pyreverseetl._core (PyO3 bindings)  →  pyreverseetl_cor
 - **`python/pyreverseetl/`** — the installed Python package: `cli.py` (the `pyreverseetl`
   command) and `server.py` (an optional Flask REST wrapper around the same engine).
 
+## Validation gate
+
+```bash
+# File-based report
+pyreverseetl execute-activation my-activation --validation-report statguardian-report.json
+
+# Live StatGuardian service: validate a sample before the real sync
+pyreverseetl execute-activation my-activation --statguardian-url http://127.0.0.1:8000
+```
+
+A file report must have `"passed": true`; anything else blocks the sync. The
+live `--statguardian-url` gate dry-runs a sample and calls the service's
+`/validate`; a failure or network error blocks the real sync. It is a sample
+gate, not a full-table scan.
+
 ## Documentation
 
 - [ROADMAP_HONEST.md](ROADMAP_HONEST.md) — what's tested vs. untested vs. broken vs. not

{pyreverseetl-3.1.7 → pyreverseetl-3.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "PyReverseETL"
-version = "3.1.7"
+version = "3.3.0"
 description = "The Open Operational Data Activation Runtime"
 readme = "README.md"
 license = "Apache-2.0"

{pyreverseetl-3.1.7 → pyreverseetl-3.3.0}/python/pyreverseetl/cli.py

@@ -27,10 +27,71 @@ from __future__ import annotations
 import json
 import os
 import sys
+import urllib.error
+import urllib.request
 
 DEFAULT_STATE_PATH = os.path.join(os.path.expanduser("~"), ".pyreverseetl", "state.json")
 
 
+def validate_activation_rows(
+    rows: list[dict],
+    service_url: str,
+    *,
+    limit: int = 100,
+    timeout_seconds: float = 10.0,
+) -> tuple[bool, dict]:
+    """POST sample rows to StatGuardian's ``/validate`` endpoint.
+
+    Fails closed on network/HTTP/parse errors. Only the first ``limit`` rows
+    are checked; this is a pre-activation sample gate, not a full-table scan.
+    """
+    url = service_url.rstrip("/") + "/validate"
+    checked = 0
+    for row in rows[: max(1, limit)]:
+        checked += 1
+        body = json.dumps({"entity": {"attributes": row}}).encode("utf-8")
+        request = urllib.request.Request(
+            url,
+            data=body,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
+                payload = json.loads(response.read().decode("utf-8"))
+        except (urllib.error.URLError, OSError, ValueError) as exc:
+            return False, {"reason": f"StatGuardian request failed: {exc}", "checked": checked}
+        if payload.get("passed") is not True:
+            return False, {
+                "reason": "StatGuardian rejected a sample row",
+                "checked": checked,
+                "issues": payload.get("issues", []),
+            }
+    return True, {"checked": checked}
+
+
+def validation_report_allows(path: str) -> tuple[bool, dict]:
+    """Read a StatGuardian-style JSON report and decide whether sync may run.
+
+    Expected shape: ``{"passed": bool, "violations": [...], ...}``. A missing
+    or malformed file fails closed. This is a file-based gate until
+    StatGuardian exposes a service endpoint; it does not call StatGuardian.
+    """
+    try:
+        with open(path, encoding="utf-8") as handle:
+            report = json.load(handle)
+    except (OSError, ValueError) as exc:
+        return False, {"reason": f"cannot read validation report: {exc}"}
+    if not isinstance(report, dict) or "passed" not in report:
+        return False, {"reason": "validation report must be a JSON object with a 'passed' field"}
+    if report.get("passed") is not True:
+        return False, {
+            "reason": "validation report did not pass",
+            "violations": report.get("violations", []),
+        }
+    return True, {}
+
+
 class CLIInterface:
     """Command-line interface for PyReverseETL workflow integration."""
 
@@ -162,6 +223,9 @@ class CLIInterface:
         schema_store_path: str | None = None,
         idempotency_store_path: str | None = None,
         dlq_store_path: str | None = None,
+        validation_report_path: str | None = None,
+        statguardian_url: str | None = None,
+        statguardian_limit: int = 100,
     ) -> dict:
         """Execute an activation: run a real sync through the Rust engine.
 
@@ -200,6 +264,16 @@ class CLIInterface:
                 changed is still sent. Only applies to the adapter-based
                 destinations (webhook/salesforce/hubspot/marketo). Omit to
                 disable idempotency checking entirely (default).
+            validation_report_path: Optional path to a JSON validation report
+                (``{"passed": bool, ...}``). If supplied and not passing, the
+                sync is blocked before any source read or destination write.
+                This is a file-based StatGuardian gate.
+            statguardian_url: Optional base URL of a running StatGuardian
+                service. A dry-run reads a sample of source rows and POSTs them
+                to ``/validate``; any failure (or network error) blocks the real
+                sync. This doubles the source read for the sample pass.
+            statguardian_limit: Maximum sample rows sent to StatGuardian
+                (default 100).
             dlq_store_path: Optional path to a SQLite file used as a
                 dead-letter queue (quarantine) for records the destination
                 rejects. Each failed record is stored with its run id, the
@@ -220,6 +294,15 @@ class CLIInterface:
             }
 
         activation = self.activations[activation_id]
+        if validation_report_path:
+            allowed, detail = validation_report_allows(validation_report_path)
+            if not allowed:
+                return {
+                    "status": "error",
+                    "activation_id": activation_id,
+                    "message": "Blocked by validation gate",
+                    "validation": detail,
+                }
         workflow = self.workflows.get(activation["workflow_id"])
         if workflow is None:
             return {
@@ -239,6 +322,38 @@ class CLIInterface:
         source_config.setdefault("table", workflow.get("table"))
         destination_config = activation.get("destination_config") or {}
 
+        if statguardian_url:
+            try:
+                preview = _core.run_sync(
+                    source_type=workflow["source"],
+                    source_config=json.dumps(source_config),
+                    destination_type=activation["destination"],
+                    destination_config=json.dumps(destination_config),
+                    limit=limit,
+                    compliance_rules=json.dumps(compliance_rules) if compliance_rules else None,
+                    dry_run=True,
+                    schema_store_path=None,
+                    idempotency_store_path=None,
+                    dlq_store_path=None,
+                )
+            except Exception as exc:  # noqa: BLE001
+                return {
+                    "status": "error",
+                    "activation_id": activation_id,
+                    "message": f"StatGuardian sample read failed: {exc}",
+                }
+            sample_rows = [json.loads(row) for row in preview.dry_run_preview]
+            allowed, detail = validate_activation_rows(
+                sample_rows, statguardian_url, limit=statguardian_limit
+            )
+            if not allowed:
+                return {
+                    "status": "error",
+                    "activation_id": activation_id,
+                    "message": "Blocked by StatGuardian sample validation",
+                    "validation": detail,
+                }
+
         try:
             result = _core.run_sync(
                 source_type=workflow["source"],
@@ -501,6 +616,29 @@ def main():
                     sys.exit(1)
                 dlq_store_path = sys.argv[idx + 1]
 
+            validation_report_path = None
+            if "--validation-report" in sys.argv:
+                idx = sys.argv.index("--validation-report")
+                if idx + 1 >= len(sys.argv):
+                    print(json.dumps({"error": "--validation-report requires a file path"}))
+                    sys.exit(1)
+                validation_report_path = sys.argv[idx + 1]
+
+            statguardian_url = None
+            if "--statguardian-url" in sys.argv:
+                idx = sys.argv.index("--statguardian-url")
+                if idx + 1 >= len(sys.argv):
+                    print(json.dumps({"error": "--statguardian-url requires a URL"}))
+                    sys.exit(1)
+                statguardian_url = sys.argv[idx + 1]
+            statguardian_limit = 100
+            if "--statguardian-limit" in sys.argv:
+                idx = sys.argv.index("--statguardian-limit")
+                if idx + 1 >= len(sys.argv):
+                    print(json.dumps({"error": "--statguardian-limit requires a number"}))
+                    sys.exit(1)
+                statguardian_limit = int(sys.argv[idx + 1])
+
             result = cli.execute_activation(
                 activation_id,
                 limit,
@@ -509,6 +647,9 @@ def main():
                 schema_store_path,
                 idempotency_store_path,
                 dlq_store_path,
+                validation_report_path,
+                statguardian_url,
+                statguardian_limit,
             )
             print(json.dumps(result))