PyReverseETL 3.1.4__tar.gz → 3.1.6__tar.gz

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/Cargo.lock

@@ -3199,7 +3199,7 @@ dependencies = [
 
 [[package]]
 name = "pyreverseetl"
-version = "3.1.4"
+version = "3.1.6"
 dependencies = [
  "chrono",
  "pyo3",
@@ -3210,7 +3210,7 @@ dependencies = [
 
 [[package]]
 name = "pyreverseetl-core"
-version = "3.1.4"
+version = "3.1.6"
 dependencies = [
  "async-trait",
  "aws-config",

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["core", "python"]
 resolver = "2"
 
 [workspace.package]
-version = "3.1.4"
+version = "3.1.6"
 authors = ["Georgi Mammen Mullassery <mullassery@gmail.com>"]
 license = "Apache-2.0"
 repository = "https://github.com/Mullassery/PyReverseETL"

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyReverseETL
-Version: 3.1.4
+Version: 3.1.6
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -103,7 +103,7 @@ repeated here to avoid implying vendor endorsement.
 
 ## Project status
 
-**Beta.** Version 3.1.4. The Postgres/MySQL/S3/webhook paths are real and verified against
+**Beta.** Version 3.1.6. The Postgres/MySQL/S3/webhook paths are real and verified against
 real services; the `web` source connector (MudFish-backed site crawling) is implemented
 and unit-tested but not yet exercised end-to-end against a live site; the CRM/marketing
 adapters are real but mock-verified only; several

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/README.md

@@ -80,7 +80,7 @@ repeated here to avoid implying vendor endorsement.
 
 ## Project status
 
-**Beta.** Version 3.1.4. The Postgres/MySQL/S3/webhook paths are real and verified against
+**Beta.** Version 3.1.6. The Postgres/MySQL/S3/webhook paths are real and verified against
 real services; the `web` source connector (MudFish-backed site crawling) is implemented
 and unit-tested but not yet exercised end-to-end against a live site; the CRM/marketing
 adapters are real but mock-verified only; several

pyreverseetl-3.1.6/core/src/dlq.rs

@@ -0,0 +1,161 @@
+//! Local dead-letter queue (quarantine) for records that failed to write.
+//!
+//! Per the verified audit gap (RECHECK-018): a sync that partially fails left
+//! the operator with only an aggregate `rows_failed` count -- there was no
+//! record of *which* records failed, why, or their payload, so re-driving a
+//! failed batch after a transient destination error (a rate limit, a 5xx, a
+//! single malformed record) meant re-reading the whole source and guessing.
+//!
+//! `DeadLetterQueue` is a real, persisted (SQLite) quarantine: every record
+//! that a destination rejected is stored with its run id, destination key,
+//! the reason for rejection, and its exact payload, so a failed record can be
+//! inspected and replayed deliberately instead of re-running the entire sync.
+
+use rusqlite::{params, Connection};
+use serde::Serialize;
+use std::path::Path;
+use std::sync::Mutex;
+
+/// One quarantined record: what failed, where it was headed, and why.
+#[derive(Debug, Clone, Serialize)]
+pub struct DeadLetter {
+    pub run_id: String,
+    pub destination_key: String,
+    pub record_id: String,
+    pub reason: String,
+    /// The exact record payload (compact JSON) that was rejected.
+    pub payload: String,
+    pub failed_at: String,
+}
+
+pub struct DeadLetterQueue {
+    conn: Mutex<Connection>,
+}
+
+impl DeadLetterQueue {
+    /// Open (creating if absent) a real SQLite file as the quarantine store.
+    pub fn open(path: &Path) -> crate::Result<Self> {
+        let conn = Connection::open(path)?;
+        Self::from_connection(conn)
+    }
+
+    /// SQLite-backed but in-memory -- exercises the real SQL code path without
+    /// touching disk (tests).
+    #[cfg(test)]
+    pub fn in_memory() -> crate::Result<Self> {
+        let conn = Connection::open_in_memory()?;
+        Self::from_connection(conn)
+    }
+
+    fn from_connection(conn: Connection) -> crate::Result<Self> {
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS dead_letters (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                run_id TEXT NOT NULL,
+                destination_key TEXT NOT NULL,
+                record_id TEXT NOT NULL,
+                reason TEXT NOT NULL,
+                payload TEXT NOT NULL,
+                failed_at TEXT NOT NULL
+            )",
+            [],
+        )?;
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_dead_letters_run ON dead_letters(run_id)",
+            [],
+        )?;
+        Ok(Self {
+            conn: Mutex::new(conn),
+        })
+    }
+
+    /// Quarantine one failed record. Appends a new row every time (a record
+    /// that fails across several runs leaves a row per run), so the history of
+    /// repeated failures is preserved rather than overwritten.
+    #[allow(clippy::too_many_arguments)]
+    pub fn record(
+        &self,
+        run_id: &str,
+        destination_key: &str,
+        record_id: &str,
+        reason: &str,
+        payload: &str,
+    ) -> crate::Result<()> {
+        let failed_at = chrono::Utc::now().to_rfc3339();
+        let conn = self.conn.lock().unwrap();
+        conn.execute(
+            "INSERT INTO dead_letters
+                (run_id, destination_key, record_id, reason, payload, failed_at)
+             VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
+            params![run_id, destination_key, record_id, reason, payload, failed_at],
+        )?;
+        Ok(())
+    }
+
+    /// Every record quarantined by a single run, in insertion order.
+    pub fn list_for_run(&self, run_id: &str) -> crate::Result<Vec<DeadLetter>> {
+        let conn = self.conn.lock().unwrap();
+        let mut stmt = conn.prepare(
+            "SELECT run_id, destination_key, record_id, reason, payload, failed_at
+             FROM dead_letters WHERE run_id = ?1 ORDER BY id ASC",
+        )?;
+        let rows = stmt.query_map(params![run_id], |row| {
+            Ok(DeadLetter {
+                run_id: row.get(0)?,
+                destination_key: row.get(1)?,
+                record_id: row.get(2)?,
+                reason: row.get(3)?,
+                payload: row.get(4)?,
+                failed_at: row.get(5)?,
+            })
+        })?;
+        let mut out = Vec::new();
+        for row in rows {
+            out.push(row?);
+        }
+        Ok(out)
+    }
+
+    /// Total number of quarantined records across every run.
+    pub fn count(&self) -> crate::Result<u64> {
+        let conn = self.conn.lock().unwrap();
+        let count: i64 = conn.query_row("SELECT COUNT(*) FROM dead_letters", [], |row| row.get(0))?;
+        Ok(count as u64)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn records_and_lists_failures_per_run() {
+        let dlq = DeadLetterQueue::in_memory().unwrap();
+        dlq.record("run-1", "webhook:x", "rec-1", "HTTP 429", "{\"a\":1}")
+            .unwrap();
+        dlq.record("run-1", "webhook:x", "rec-2", "HTTP 500", "{\"b\":2}")
+            .unwrap();
+        dlq.record("run-2", "webhook:x", "rec-3", "boom", "{\"c\":3}")
+            .unwrap();
+
+        let run1 = dlq.list_for_run("run-1").unwrap();
+        assert_eq!(run1.len(), 2);
+        assert_eq!(run1[0].record_id, "rec-1");
+        assert_eq!(run1[0].reason, "HTTP 429");
+        assert_eq!(run1[0].payload, "{\"a\":1}");
+        assert_eq!(run1[1].record_id, "rec-2");
+
+        assert_eq!(dlq.list_for_run("run-2").unwrap().len(), 1);
+        assert_eq!(dlq.count().unwrap(), 3);
+    }
+
+    #[test]
+    fn repeated_failures_are_appended_not_overwritten() {
+        let dlq = DeadLetterQueue::in_memory().unwrap();
+        dlq.record("run-1", "webhook:x", "rec-1", "HTTP 429", "{}")
+            .unwrap();
+        dlq.record("run-2", "webhook:x", "rec-1", "HTTP 429", "{}")
+            .unwrap();
+        assert_eq!(dlq.count().unwrap(), 2);
+    }
+}

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/core/src/executor.rs

@@ -17,6 +17,7 @@ use crate::connectors::{
     ObjectStorageDestination, ObjectStorageSource, PostgreSQLConfig, PostgreSQLConnector,
     Record as ConnRecord, SourceConnector, WebConfig, WebSource,
 };
+use crate::dlq::DeadLetterQueue;
 use crate::entity::{Entity, EntityType};
 use crate::governance::{
     ComplianceEngine, ComplianceRule, DefaultComplianceEngine, DefaultSchemaEvolution,
@@ -91,6 +92,14 @@ pub struct ExecuteOptions {
     /// adapter-based destinations (webhook/Salesforce/HubSpot/Marketo) --
     /// `None` disables idempotency checking (matches prior behavior).
     pub idempotency_store_path: Option<PathBuf>,
+    /// When set, enables a real dead-letter queue (quarantine): every record
+    /// the destination rejected is persisted to a SQLite file at this path
+    /// with its run id, reason, and exact payload, so a partial batch failure
+    /// can be inspected and re-driven deliberately instead of re-running the
+    /// whole sync. Only applies to the adapter-based destinations
+    /// (webhook/Salesforce/HubSpot/Marketo), where per-record success/failure
+    /// is observable. `None` disables quarantine (matches prior behavior).
+    pub dlq_store_path: Option<PathBuf>,
 }
 
 /// The real result of a sync: actual counts, actual timestamps -- never
@@ -119,6 +128,13 @@ pub struct ExecutionResult {
     /// content marked as synced to this destination. Always 0 when
     /// idempotency checking is disabled.
     pub rows_skipped_idempotent: u64,
+    /// Records quarantined into the dead-letter queue (see
+    /// `ExecuteOptions::dlq_store_path`) because the destination rejected
+    /// them. Always 0 when no DLQ path is configured. Equals `rows_failed`
+    /// for the adapter-based destinations when a DLQ path is set (every
+    /// observed failure is persisted); stays 0 for batch connectors, which
+    /// report only an aggregate count and cannot identify individual rows.
+    pub rows_dead_lettered: u64,
     /// Only populated when `ExecuteOptions::schema_store_path` is set: a
     /// human-readable description of every field-name/type change detected
     /// against the last-known shape for this source->destination pair.
@@ -206,21 +222,45 @@ async fn execute_with_records(
         Vec::new()
     };
 
-    let (rows_written, rows_failed, rows_skipped_idempotent, dry_run_preview, destination_node) =
+    let (rows_written, rows_failed, rows_skipped_idempotent, rows_dead_lettered, dry_run_preview, destination_node) =
         if options.dry_run {
             let preview = compliant_records
                 .iter()
                 .map(|r| r.data.to_string())
                 .collect();
-            (0, 0, 0, preview, None)
+            (0, 0, 0, 0, preview, None)
         } else {
-            let (written, failed, skipped, node) = write_to_destination(
+            let (written, failed_count, skipped, failed, node) = write_to_destination(
                 &destination,
                 &compliant_records,
                 options.idempotency_store_path.as_deref(),
             )
             .await?;
-            (written, failed, skipped, Vec::new(), Some(node))
+
+            // Quarantine real per-record failures for deliberate re-drive.
+            // Only records whose individual rejection was observed are
+            // persisted (the adapter path); batch connectors report an
+            // aggregate failure count but not which rows, so they don't
+            // contribute here.
+            let mut dead_lettered = 0u64;
+            if let Some(dlq_path) = &options.dlq_store_path {
+                if !failed.is_empty() {
+                    let dlq = DeadLetterQueue::open(dlq_path)?;
+                    let destination_key = destination_schema_key(&destination);
+                    for (record, reason) in &failed {
+                        dlq.record(
+                            &run_id,
+                            &destination_key,
+                            &record.id,
+                            reason,
+                            &record.data.to_string(),
+                        )?;
+                        dead_lettered += 1;
+                    }
+                }
+            }
+
+            (written, failed_count, skipped, dead_lettered, Vec::new(), Some(node))
         };
 
     let completed_at = Utc::now();
@@ -243,6 +283,7 @@ async fn execute_with_records(
         dry_run: options.dry_run,
         dry_run_preview,
         rows_skipped_idempotent,
+        rows_dead_lettered,
         schema_changes,
         compliance_violations,
         started_at,
@@ -424,7 +465,7 @@ async fn write_to_destination(
     destination: &DestinationSpec,
     records: &[ConnRecord],
     idempotency_store_path: Option<&Path>,
-) -> crate::Result<(u64, u64, u64, LineageNode)> {
+) -> crate::Result<(u64, u64, u64, Vec<(ConnRecord, String)>, LineageNode)> {
     match destination {
         DestinationSpec::Postgres(cfg) => {
             let connector = PostgreSQLConnector::new(cfg.clone());
@@ -439,6 +480,7 @@ async fn write_to_destination(
                 written,
                 (records.len() as u64).saturating_sub(written),
                 0,
+                Vec::new(),
                 node,
             ))
         }
@@ -455,6 +497,7 @@ async fn write_to_destination(
                 written,
                 (records.len() as u64).saturating_sub(written),
                 0,
+                Vec::new(),
                 node,
             ))
         }
@@ -473,11 +516,12 @@ async fn write_to_destination(
                 written,
                 (records.len() as u64).saturating_sub(written),
                 0,
+                Vec::new(),
                 node,
             ))
         }
         DestinationSpec::Webhook { config, auth } => {
-            let (written, failed, skipped) = write_via_adapter(
+            let (written, skipped, failed) = write_via_adapter(
                 "webhook",
                 config.clone(),
                 auth.clone(),
@@ -502,10 +546,10 @@ async fn write_to_destination(
                     .unwrap_or("webhook")
                     .to_string(),
             );
-            Ok((written, failed, skipped, node))
+            Ok((written, failed.len() as u64, skipped, failed, node))
         }
         DestinationSpec::Salesforce { config, auth } => {
-            let (written, failed, skipped) = write_via_adapter(
+            let (written, skipped, failed) = write_via_adapter(
                 "salesforce",
                 config.clone(),
                 auth.clone(),
@@ -531,10 +575,10 @@ async fn write_to_destination(
                 "salesforce",
                 object.to_string(),
             );
-            Ok((written, failed, skipped, node))
+            Ok((written, failed.len() as u64, skipped, failed, node))
         }
         DestinationSpec::HubSpot { config, auth } => {
-            let (written, failed, skipped) = write_via_adapter(
+            let (written, skipped, failed) = write_via_adapter(
                 "hubspot",
                 config.clone(),
                 auth.clone(),
@@ -553,10 +597,10 @@ async fn write_to_destination(
                 "hubspot",
                 object.to_string(),
             );
-            Ok((written, failed, skipped, node))
+            Ok((written, failed.len() as u64, skipped, failed, node))
         }
         DestinationSpec::Marketo { config, auth } => {
-            let (written, failed, skipped) = write_via_adapter(
+            let (written, skipped, failed) = write_via_adapter(
                 "marketo",
                 config.clone(),
                 auth.clone(),
@@ -577,7 +621,7 @@ async fn write_to_destination(
                 "marketo",
                 "leads".to_string(),
             );
-            Ok((written, failed, skipped, node))
+            Ok((written, failed.len() as u64, skipped, failed, node))
         }
     }
 }
@@ -599,8 +643,8 @@ async fn write_via_adapter(
     records: Vec<ConnRecord>,
     idempotency_store_path: Option<PathBuf>,
     destination_key: String,
-) -> crate::Result<(u64, u64, u64)> {
-    tokio::task::spawn_blocking(move || -> crate::Result<(u64, u64, u64)> {
+) -> crate::Result<(u64, u64, Vec<(ConnRecord, String)>)> {
+    tokio::task::spawn_blocking(move || -> crate::Result<(u64, u64, Vec<(ConnRecord, String)>)> {
         let adapter = AdapterFactory::create_adapter(adapter_type, &config, &auth)?;
         let ledger = idempotency_store_path
             .as_deref()
@@ -608,8 +652,10 @@ async fn write_via_adapter(
             .transpose()?;
 
         let mut written = 0u64;
-        let mut failed = 0u64;
         let mut skipped = 0u64;
+        // Records the destination rejected, with the reason, captured so a
+        // caller can quarantine them (DLQ) rather than only seeing a count.
+        let mut failed: Vec<(ConnRecord, String)> = Vec::new();
         for record in &records {
             if let Some(ledger) = &ledger {
                 let hash = idempotency::content_hash(&record.data);
@@ -623,17 +669,29 @@ async fn write_via_adapter(
                         ledger.mark_synced(&destination_key, &record.id, &hash)?;
                         written += 1;
                     }
-                    _ => failed += 1,
+                    Ok(result) => {
+                        let reason = result
+                            .error_message
+                            .unwrap_or_else(|| "destination rejected record".to_string());
+                        failed.push((record.clone(), reason));
+                    }
+                    Err(e) => failed.push((record.clone(), e.to_string())),
                 }
             } else {
                 let entity = record_to_entity(record);
                 match adapter.upsert(&entity, &[]) {
                     Ok(result) if result.success => written += 1,
-                    _ => failed += 1,
+                    Ok(result) => {
+                        let reason = result
+                            .error_message
+                            .unwrap_or_else(|| "destination rejected record".to_string());
+                        failed.push((record.clone(), reason));
+                    }
+                    Err(e) => failed.push((record.clone(), e.to_string())),
                 }
             }
         }
-        Ok((written, failed, skipped))
+        Ok((written, skipped, failed))
     })
     .await
     .map_err(|e| crate::Error::Internal(format!("destination adapter task panicked: {e}")))?
@@ -747,6 +805,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: None,
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -793,6 +852,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: None,
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -829,6 +889,7 @@ mod tests {
                 dry_run: true,
                 schema_store_path: None,
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -886,6 +947,7 @@ mod tests {
                 dry_run: true,
                 schema_store_path: None,
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -932,6 +994,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: Some(store_path.clone()),
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -950,6 +1013,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: Some(store_path.clone()),
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -974,6 +1038,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: Some(store_path.clone()),
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -1022,6 +1087,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: Some(store_path.clone()),
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -1042,6 +1108,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: Some(store_path.clone()),
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -1072,6 +1139,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: Some(store_path.clone()),
                 idempotency_store_path: None,
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -1117,6 +1185,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: None,
                 idempotency_store_path: Some(ledger_path.clone()),
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -1143,6 +1212,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: None,
                 idempotency_store_path: Some(ledger_path.clone()),
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -1190,6 +1260,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: None,
                 idempotency_store_path: Some(ledger_path.clone()),
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -1212,6 +1283,7 @@ mod tests {
                 dry_run: false,
                 schema_store_path: None,
                 idempotency_store_path: Some(ledger_path.clone()),
+                dlq_store_path: None,
             },
             &lineage,
         )
@@ -1268,4 +1340,85 @@ mod tests {
             "with no idempotency store configured, every run resends every record (prior behavior)"
         );
     }
+
+    #[tokio::test]
+    async fn failed_records_are_quarantined_in_dead_letter_queue() {
+        let dir = std::env::temp_dir().join(format!("pyreverseetl_dlq_{}", uuid::Uuid::new_v4()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let dlq_path = dir.join("dlq.db");
+
+        // Destination rejects every record with a 500.
+        let server = MockHttpServer::start(500, "{}");
+        let mut config = HashMap::new();
+        config.insert("url".to_string(), json!(server.base_url.clone()));
+        let destination = DestinationSpec::Webhook {
+            config,
+            auth: AdapterAuth::Bearer {
+                token: "tok".to_string(),
+            },
+        };
+        let lineage = LineageStore::new();
+
+        let result = execute_with_records(
+            sample_records(),
+            source_node(),
+            destination,
+            ExecuteOptions {
+                dlq_store_path: Some(dlq_path.clone()),
+                ..Default::default()
+            },
+            &lineage,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.rows_written, 0);
+        assert_eq!(result.rows_failed, 2);
+        assert_eq!(
+            result.rows_dead_lettered, 2,
+            "every observed per-record failure is quarantined"
+        );
+
+        let dlq = DeadLetterQueue::open(&dlq_path).unwrap();
+        let letters = dlq.list_for_run(&result.run_id).unwrap();
+        assert_eq!(letters.len(), 2);
+        assert!(
+            letters.iter().all(|l| !l.reason.is_empty()),
+            "each quarantined record carries a reason"
+        );
+        assert!(
+            letters.iter().all(|l| !l.payload.is_empty()),
+            "each quarantined record carries its payload for replay"
+        );
+    }
+
+    #[tokio::test]
+    async fn no_dlq_path_means_no_quarantine() {
+        let server = MockHttpServer::start(500, "{}");
+        let mut config = HashMap::new();
+        config.insert("url".to_string(), json!(server.base_url.clone()));
+        let destination = DestinationSpec::Webhook {
+            config,
+            auth: AdapterAuth::Bearer {
+                token: "tok".to_string(),
+            },
+        };
+        let lineage = LineageStore::new();
+
+        let result = execute_with_records(
+            sample_records(),
+            source_node(),
+            destination,
+            ExecuteOptions::default(),
+            &lineage,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(result.rows_failed, 2);
+        assert_eq!(
+            result.rows_dead_lettered, 0,
+            "no quarantine store configured -> nothing persisted (prior behavior)"
+        );
+    }
 }

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/core/src/lib.rs

@@ -3,6 +3,7 @@ pub mod adapters;
 pub mod cdc;
 pub mod connectors;
 pub mod destination;
+pub mod dlq;
 pub mod entity;
 pub mod error;
 pub mod executor;
@@ -28,6 +29,7 @@ pub use cdc::{
     Change, ChangeDetector, ChangeLog, ChangeLogEntry, ChangeType, Checkpoint, CheckpointManager,
 };
 pub use destination::Destination;
+pub use dlq::{DeadLetter, DeadLetterQueue};
 pub use entity::Entity;
 pub use error::{Error, Result};
 pub use executor::{execute_sync, DestinationSpec, ExecuteOptions, ExecutionResult, SourceSpec};

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "PyReverseETL"
-version = "3.1.4"
+version = "3.1.6"
 description = "The Open Operational Data Activation Runtime"
 readme = "README.md"
 license = "Apache-2.0"

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/python/pyreverseetl/cli.py

@@ -161,6 +161,7 @@ class CLIInterface:
         dry_run: bool = False,
         schema_store_path: str | None = None,
         idempotency_store_path: str | None = None,
+        dlq_store_path: str | None = None,
     ) -> dict:
         """Execute an activation: run a real sync through the Rust engine.
 
@@ -199,6 +200,14 @@ class CLIInterface:
                 changed is still sent. Only applies to the adapter-based
                 destinations (webhook/salesforce/hubspot/marketo). Omit to
                 disable idempotency checking entirely (default).
+            dlq_store_path: Optional path to a SQLite file used as a
+                dead-letter queue (quarantine) for records the destination
+                rejects. Each failed record is stored with its run id, the
+                reason for rejection, and its exact payload, so a partial
+                batch failure can be inspected and re-driven deliberately
+                instead of re-running the whole sync. Only applies to the
+                adapter-based destinations. Omit to disable quarantine
+                (default).
 
         Returns:
             JSON response with real execution details, or a real error if
@@ -241,6 +250,7 @@ class CLIInterface:
                 dry_run=dry_run,
                 schema_store_path=schema_store_path,
                 idempotency_store_path=idempotency_store_path,
+                dlq_store_path=dlq_store_path,
             )
         except Exception as e:  # noqa: BLE001 - surface real engine errors verbatim
             return {
@@ -258,6 +268,7 @@ class CLIInterface:
                 "dry_run": True,
                 "rows_read": result.rows_read,
                 "rows_would_write": len(result.dry_run_preview),
+                "rows_dead_lettered": result.rows_dead_lettered,
                 "dry_run_preview": [json.loads(p) for p in result.dry_run_preview],
                 "compliance_violations": result.compliance_violations,
                 "schema_changes": result.schema_changes,
@@ -277,6 +288,7 @@ class CLIInterface:
             "rows_read": result.rows_read,
             "rows_failed": result.rows_failed,
             "rows_skipped_idempotent": result.rows_skipped_idempotent,
+            "rows_dead_lettered": result.rows_dead_lettered,
             "compliance_violations": result.compliance_violations,
             "schema_changes": result.schema_changes,
             "destination": activation["destination"],
@@ -294,6 +306,7 @@ class CLIInterface:
             "rows_read": result.rows_read,
             "rows_failed": result.rows_failed,
             "rows_skipped_idempotent": result.rows_skipped_idempotent,
+            "rows_dead_lettered": result.rows_dead_lettered,
             "compliance_violations": result.compliance_violations,
             "schema_changes": result.schema_changes,
             "duration_ms": result.duration_ms,
@@ -480,6 +493,13 @@ def main():
                     print(json.dumps({"error": "--idempotency-store requires a file path"}))
                     sys.exit(1)
                 idempotency_store_path = sys.argv[idx + 1]
+            dlq_store_path = None
+            if "--dlq-store" in sys.argv:
+                idx = sys.argv.index("--dlq-store")
+                if idx + 1 >= len(sys.argv):
+                    print(json.dumps({"error": "--dlq-store requires a file path"}))
+                    sys.exit(1)
+                dlq_store_path = sys.argv[idx + 1]
 
             result = cli.execute_activation(
                 activation_id,
@@ -488,6 +508,7 @@ def main():
                 dry_run,
                 schema_store_path,
                 idempotency_store_path,
+                dlq_store_path,
             )
             print(json.dumps(result))
 
@@ -584,7 +605,7 @@ COMMANDS:
             pyreverseetl create-activation ltv_to_hook ltv_sync webhook \\
                 --dest-config '{"url":"https://example.com/hook","auth":{"type":"bearer","token":"secret"}}'
 
-    execute <activation_id> [limit] [--compliance-rules <json|@file>] [--dry-run] [--schema-store <path>] [--idempotency-store <path>]
+    execute <activation_id> [limit] [--compliance-rules <json|@file>] [--dry-run] [--schema-store <path>] [--idempotency-store <path>] [--dlq-store <path>]
         Execute a REAL data synchronization through the Rust engine
         - activation_id: Activation to execute (required)
         - limit: Max rows to read from the source (optional)
@@ -606,12 +627,19 @@ COMMANDS:
           re-sending them, while a record whose content genuinely changed
           is still sent. Only applies to webhook/salesforce/hubspot/marketo
           destinations.
+        - --dlq-store <path>: Persist every record the destination rejects to
+          a real SQLite dead-letter queue at <path>, with the run id, reason,
+          and exact payload (rows_dead_lettered), so a partial batch failure
+          can be inspected and re-driven deliberately instead of re-running
+          the whole sync. Only applies to webhook/salesforce/hubspot/marketo
+          destinations.
 
         Example:
             pyreverseetl execute ltv_to_hook 5000
             pyreverseetl execute ltv_to_hook --dry-run
             pyreverseetl execute ltv_to_hook --schema-store .pyreverseetl/schema.db
             pyreverseetl execute ltv_to_hook --idempotency-store .pyreverseetl/idempotency.db
+            pyreverseetl execute ltv_to_hook --dlq-store .pyreverseetl/dlq.db
 
     status <run_id>
         Get status of a sync run

{pyreverseetl-3.1.4 → pyreverseetl-3.1.6}/python/src/lib.rs

@@ -2,7 +2,7 @@ use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
 use pyreverseetl_core::adapters::AuthMethod as AdapterAuth;
 use pyreverseetl_core::connectors::{
-    MySQLConfig, ObjectStorageConfig, PostgreSQLConfig, TableFormat,
+    MySQLConfig, ObjectStorageConfig, PostgreSQLConfig, TableFormat, WebConfig,
 };
 use pyreverseetl_core::destination::DestinationType;
 use pyreverseetl_core::entity::EntityType;
@@ -301,8 +301,27 @@ fn build_source_spec(source_type: &str, config: &Value) -> PyResult<SourceSpec>
             Ok(SourceSpec::MySQL(cfg))
         }
         "s3" => Ok(SourceSpec::S3(build_object_storage_config(config)?)),
+        "web" => {
+            let mut cfg = WebConfig {
+                seed_url: get_str(config, "seed_url")?.to_string(),
+                ..Default::default()
+            };
+            if let Some(v) = config.get("max_depth").and_then(|v| v.as_u64()) {
+                cfg.max_depth = v as usize;
+            }
+            if let Some(v) = config.get("max_urls").and_then(|v| v.as_u64()) {
+                cfg.max_urls = v as usize;
+            }
+            if let Some(v) = config.get("same_domain").and_then(|v| v.as_bool()) {
+                cfg.same_domain = v;
+            }
+            if let Some(v) = config.get("render_js").and_then(|v| v.as_bool()) {
+                cfg.render_js = v;
+            }
+            Ok(SourceSpec::Web(cfg))
+        }
         other => Err(py_err(format!(
-            "unsupported source_type '{other}' (supported: postgres, mysql, s3)"
+            "unsupported source_type '{other}' (supported: postgres, mysql, s3, web)"
         ))),
     }
 }
@@ -484,12 +503,20 @@ pub struct PySyncResult {
     /// disabled.
     #[pyo3(get)]
     pub rows_skipped_idempotent: u64,
+    /// Records quarantined into the dead-letter queue (see `dlq_store_path`)
+    /// because the destination rejected them. Always 0 when no DLQ path is
+    /// configured; equals `rows_failed` for the adapter-based destinations
+    /// (webhook/salesforce/hubspot/marketo) when a DLQ path is set.
+    #[pyo3(get)]
+    pub rows_dead_lettered: u64,
 }
 
 /// Run a real sync: read from `source_type`/`source_config` (JSON string),
 /// apply `compliance_rules` (JSON string, optional), write to
 /// `destination_type`/`destination_config` (JSON string), and record a real
-/// lineage edge. Supported `source_type`: `postgres`, `mysql`, `s3`.
+/// lineage edge. Supported `source_type`: `postgres`, `mysql`, `s3`, `web`
+/// (MudFish-backed site crawl; `source_config` takes `seed_url` plus optional
+/// `max_depth`, `max_urls`, `same_domain`, `render_js`).
 /// Supported `destination_type`: all of those plus `webhook`, `salesforce`,
 /// `hubspot`, `marketo`. With `dry_run=True`, records are still read and run
 /// through the real compliance engine, but the destination is never
@@ -503,8 +530,14 @@ pub struct PySyncResult {
 /// records already synced instead of re-sending them -- see
 /// `PySyncResult.rows_skipped_idempotent`. Only applies to the
 /// adapter-based destinations (webhook/salesforce/hubspot/marketo).
+/// With `dlq_store_path` set, every record the destination rejects is
+/// persisted to a real SQLite dead-letter queue at that path (run id, reason,
+/// and exact payload), so a partial batch failure can be inspected and
+/// re-driven deliberately instead of re-running the whole sync -- see
+/// `PySyncResult.rows_dead_lettered`. Only applies to the adapter-based
+/// destinations.
 #[pyfunction]
-#[pyo3(signature = (source_type, source_config, destination_type, destination_config, limit=None, compliance_rules=None, dry_run=false, schema_store_path=None, idempotency_store_path=None))]
+#[pyo3(signature = (source_type, source_config, destination_type, destination_config, limit=None, compliance_rules=None, dry_run=false, schema_store_path=None, idempotency_store_path=None, dlq_store_path=None))]
 pub fn run_sync(
     source_type: &str,
     source_config: &str,
@@ -515,6 +548,7 @@ pub fn run_sync(
     dry_run: bool,
     schema_store_path: Option<&str>,
     idempotency_store_path: Option<&str>,
+    dlq_store_path: Option<&str>,
 ) -> PyResult<PySyncResult> {
     let source_cfg_value = parse_json(source_config)?;
     let dest_cfg_value = parse_json(destination_config)?;
@@ -529,6 +563,7 @@ pub fn run_sync(
         dry_run,
         schema_store_path: schema_store_path.map(std::path::PathBuf::from),
         idempotency_store_path: idempotency_store_path.map(std::path::PathBuf::from),
+        dlq_store_path: dlq_store_path.map(std::path::PathBuf::from),
     };
 
     let runtime = tokio::runtime::Runtime::new().map_err(py_err)?;
@@ -547,6 +582,7 @@ pub fn run_sync(
         duration_ms: result.duration_ms,
         schema_changes: result.schema_changes,
         rows_skipped_idempotent: result.rows_skipped_idempotent,
+        rows_dead_lettered: result.rows_dead_lettered,
         dry_run: result.dry_run,
         dry_run_preview: result.dry_run_preview,
     })