PyReverseETL 3.1.7__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/Cargo.lock +2 -2
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/Cargo.toml +1 -1
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/PKG-INFO +11 -1
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/README.md +10 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/pyproject.toml +1 -1
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/python/pyreverseetl/cli.py +46 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/LICENSE +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/Cargo.toml +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/benches/load_test.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/bin/pyreverseetl_dashboard.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/activation.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/alert_compat.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/error.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/http_client.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/hubspot.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/mapping.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/marketo.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/oauth_manager.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/retry_policy.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/salesforce.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/schema_detect.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/adapters/webhook.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/cdc/change_detector.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/cdc/changelog.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/cdc/checkpoint.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/cdc/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/config.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/connectors_db.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/database_advanced.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/destination.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/error.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/hdfs.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/mysql.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/object_storage.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/postgres.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/rate_limiting.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/registry.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/saas.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/source.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/connectors/web.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/destination.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/dlq.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/entity.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/error.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/executor.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/governance/cached_gate.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/governance/compliance_rules.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/governance/credentials.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/governance/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/governance/quality_gate.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/governance/retry_policy.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/governance/schema_evolution.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/governance/statguardian_client.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/idempotency.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/lib.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/lineage.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/observability/dashboard_launcher.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/observability/logs.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/observability/metrics.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/observability/metrics_server.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/observability/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/observability/traces.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/pipeline/activation_pipeline.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/pipeline/backpressure.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/pipeline/latency_tracker.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/pipeline/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/sources/kafka.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/sources/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/sources/polling.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/sources/sync_config.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/statguardian.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/storage/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/storage/repository.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/storage/schema.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/streaming/event_processor.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/streaming/event_schema.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/streaming/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/streampdf.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/streamxl.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/sync.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/testing/connector_test.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/testing/harness.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/testing/metrics.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/testing/mock_http.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/testing/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/testing/test_data.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/transformers/mod.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/transformers/spark.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/core/src/workflow.rs +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/python/Cargo.toml +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/python/pyreverseetl/__init__.py +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/python/pyreverseetl/server.py +0 -0
- {pyreverseetl-3.1.7 → pyreverseetl-3.2.0}/python/src/lib.rs +0 -0
|
@@ -3199,7 +3199,7 @@ dependencies = [
|
|
|
3199
3199
|
|
|
3200
3200
|
[[package]]
|
|
3201
3201
|
name = "pyreverseetl"
|
|
3202
|
-
version = "3.
|
|
3202
|
+
version = "3.2.0"
|
|
3203
3203
|
dependencies = [
|
|
3204
3204
|
"chrono",
|
|
3205
3205
|
"pyo3",
|
|
@@ -3210,7 +3210,7 @@ dependencies = [
|
|
|
3210
3210
|
|
|
3211
3211
|
[[package]]
|
|
3212
3212
|
name = "pyreverseetl-core"
|
|
3213
|
-
version = "3.
|
|
3213
|
+
version = "3.2.0"
|
|
3214
3214
|
dependencies = [
|
|
3215
3215
|
"async-trait",
|
|
3216
3216
|
"aws-config",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: PyReverseETL
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -249,6 +249,16 @@ Python CLI / API → pyreverseetl._core (PyO3 bindings) → pyreverseetl_cor
|
|
|
249
249
|
- **`python/pyreverseetl/`** — the installed Python package: `cli.py` (the `pyreverseetl`
|
|
250
250
|
command) and `server.py` (an optional Flask REST wrapper around the same engine).
|
|
251
251
|
|
|
252
|
+
## Validation gate
|
|
253
|
+
|
|
254
|
+
```bash
|
|
255
|
+
pyreverseetl execute-activation my-activation --validation-report statguardian-report.json
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
The report must be a JSON object with `"passed": true`; anything else blocks
|
|
259
|
+
the sync before reading or writing data. This is a file-based gate until a live
|
|
260
|
+
StatGuardian service endpoint is available.
|
|
261
|
+
|
|
252
262
|
## Documentation
|
|
253
263
|
|
|
254
264
|
- [ROADMAP_HONEST.md](ROADMAP_HONEST.md) — what's tested vs. untested vs. broken vs. not
|
|
@@ -226,6 +226,16 @@ Python CLI / API → pyreverseetl._core (PyO3 bindings) → pyreverseetl_cor
|
|
|
226
226
|
- **`python/pyreverseetl/`** — the installed Python package: `cli.py` (the `pyreverseetl`
|
|
227
227
|
command) and `server.py` (an optional Flask REST wrapper around the same engine).
|
|
228
228
|
|
|
229
|
+
## Validation gate
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
pyreverseetl execute-activation my-activation --validation-report statguardian-report.json
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
The report must be a JSON object with `"passed": true`; anything else blocks
|
|
236
|
+
the sync before reading or writing data. This is a file-based gate until a live
|
|
237
|
+
StatGuardian service endpoint is available.
|
|
238
|
+
|
|
229
239
|
## Documentation
|
|
230
240
|
|
|
231
241
|
- [ROADMAP_HONEST.md](ROADMAP_HONEST.md) — what's tested vs. untested vs. broken vs. not
|
|
@@ -31,6 +31,28 @@ import sys
|
|
|
31
31
|
DEFAULT_STATE_PATH = os.path.join(os.path.expanduser("~"), ".pyreverseetl", "state.json")
|
|
32
32
|
|
|
33
33
|
|
|
34
|
+
def validation_report_allows(path: str) -> tuple[bool, dict]:
|
|
35
|
+
"""Read a StatGuardian-style JSON report and decide whether sync may run.
|
|
36
|
+
|
|
37
|
+
Expected shape: ``{"passed": bool, "violations": [...], ...}``. A missing
|
|
38
|
+
or malformed file fails closed. This is a file-based gate until
|
|
39
|
+
StatGuardian exposes a service endpoint; it does not call StatGuardian.
|
|
40
|
+
"""
|
|
41
|
+
try:
|
|
42
|
+
with open(path, encoding="utf-8") as handle:
|
|
43
|
+
report = json.load(handle)
|
|
44
|
+
except (OSError, ValueError) as exc:
|
|
45
|
+
return False, {"reason": f"cannot read validation report: {exc}"}
|
|
46
|
+
if not isinstance(report, dict) or "passed" not in report:
|
|
47
|
+
return False, {"reason": "validation report must be a JSON object with a 'passed' field"}
|
|
48
|
+
if report.get("passed") is not True:
|
|
49
|
+
return False, {
|
|
50
|
+
"reason": "validation report did not pass",
|
|
51
|
+
"violations": report.get("violations", []),
|
|
52
|
+
}
|
|
53
|
+
return True, {}
|
|
54
|
+
|
|
55
|
+
|
|
34
56
|
class CLIInterface:
|
|
35
57
|
"""Command-line interface for PyReverseETL workflow integration."""
|
|
36
58
|
|
|
@@ -162,6 +184,7 @@ class CLIInterface:
|
|
|
162
184
|
schema_store_path: str | None = None,
|
|
163
185
|
idempotency_store_path: str | None = None,
|
|
164
186
|
dlq_store_path: str | None = None,
|
|
187
|
+
validation_report_path: str | None = None,
|
|
165
188
|
) -> dict:
|
|
166
189
|
"""Execute an activation: run a real sync through the Rust engine.
|
|
167
190
|
|
|
@@ -200,6 +223,11 @@ class CLIInterface:
|
|
|
200
223
|
changed is still sent. Only applies to the adapter-based
|
|
201
224
|
destinations (webhook/salesforce/hubspot/marketo). Omit to
|
|
202
225
|
disable idempotency checking entirely (default).
|
|
226
|
+
validation_report_path: Optional path to a JSON validation report
|
|
227
|
+
(``{"passed": bool, ...}``). If supplied and not passing, the
|
|
228
|
+
sync is blocked before any source read or destination write.
|
|
229
|
+
This is a file-based StatGuardian gate; it does not call the
|
|
230
|
+
StatGuardian service.
|
|
203
231
|
dlq_store_path: Optional path to a SQLite file used as a
|
|
204
232
|
dead-letter queue (quarantine) for records the destination
|
|
205
233
|
rejects. Each failed record is stored with its run id, the
|
|
@@ -220,6 +248,15 @@ class CLIInterface:
|
|
|
220
248
|
}
|
|
221
249
|
|
|
222
250
|
activation = self.activations[activation_id]
|
|
251
|
+
if validation_report_path:
|
|
252
|
+
allowed, detail = validation_report_allows(validation_report_path)
|
|
253
|
+
if not allowed:
|
|
254
|
+
return {
|
|
255
|
+
"status": "error",
|
|
256
|
+
"activation_id": activation_id,
|
|
257
|
+
"message": "Blocked by validation gate",
|
|
258
|
+
"validation": detail,
|
|
259
|
+
}
|
|
223
260
|
workflow = self.workflows.get(activation["workflow_id"])
|
|
224
261
|
if workflow is None:
|
|
225
262
|
return {
|
|
@@ -501,6 +538,14 @@ def main():
|
|
|
501
538
|
sys.exit(1)
|
|
502
539
|
dlq_store_path = sys.argv[idx + 1]
|
|
503
540
|
|
|
541
|
+
validation_report_path = None
|
|
542
|
+
if "--validation-report" in sys.argv:
|
|
543
|
+
idx = sys.argv.index("--validation-report")
|
|
544
|
+
if idx + 1 >= len(sys.argv):
|
|
545
|
+
print(json.dumps({"error": "--validation-report requires a file path"}))
|
|
546
|
+
sys.exit(1)
|
|
547
|
+
validation_report_path = sys.argv[idx + 1]
|
|
548
|
+
|
|
504
549
|
result = cli.execute_activation(
|
|
505
550
|
activation_id,
|
|
506
551
|
limit,
|
|
@@ -509,6 +554,7 @@ def main():
|
|
|
509
554
|
schema_store_path,
|
|
510
555
|
idempotency_store_path,
|
|
511
556
|
dlq_store_path,
|
|
557
|
+
validation_report_path,
|
|
512
558
|
)
|
|
513
559
|
print(json.dumps(result))
|
|
514
560
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|