PyReverseETL 3.1.5__tar.gz → 3.1.7__tar.gz

{pyreverseetl-3.1.5 → pyreverseetl-3.1.7}/Cargo.lock

@@ -3199,7 +3199,7 @@ dependencies = [
 
 [[package]]
 name = "pyreverseetl"
-version = "3.1.5"
+version = "3.1.7"
 dependencies = [
  "chrono",
  "pyo3",
@@ -3210,7 +3210,7 @@ dependencies = [
 
 [[package]]
 name = "pyreverseetl-core"
-version = "3.1.5"
+version = "3.1.7"
 dependencies = [
  "async-trait",
  "aws-config",

{pyreverseetl-3.1.5 → pyreverseetl-3.1.7}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["core", "python"]
 resolver = "2"
 
 [workspace.package]
-version = "3.1.5"
+version = "3.1.7"
 authors = ["Georgi Mammen Mullassery <mullassery@gmail.com>"]
 license = "Apache-2.0"
 repository = "https://github.com/Mullassery/PyReverseETL"

{pyreverseetl-3.1.5 → pyreverseetl-3.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyReverseETL
-Version: 3.1.5
+Version: 3.1.7
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -103,7 +103,7 @@ repeated here to avoid implying vendor endorsement.
 
 ## Project status
 
-**Beta.** Version 3.1.5. The Postgres/MySQL/S3/webhook paths are real and verified against
+**Beta.** Version 3.1.6. The Postgres/MySQL/S3/webhook paths are real and verified against
 real services; the `web` source connector (MudFish-backed site crawling) is implemented
 and unit-tested but not yet exercised end-to-end against a live site; the CRM/marketing
 adapters are real but mock-verified only; several

{pyreverseetl-3.1.5 → pyreverseetl-3.1.7}/README.md

@@ -80,7 +80,7 @@ repeated here to avoid implying vendor endorsement.
 
 ## Project status
 
-**Beta.** Version 3.1.5. The Postgres/MySQL/S3/webhook paths are real and verified against
+**Beta.** Version 3.1.6. The Postgres/MySQL/S3/webhook paths are real and verified against
 real services; the `web` source connector (MudFish-backed site crawling) is implemented
 and unit-tested but not yet exercised end-to-end against a live site; the CRM/marketing
 adapters are real but mock-verified only; several

{pyreverseetl-3.1.5 → pyreverseetl-3.1.7}/core/src/connectors/database_advanced.rs

@@ -222,27 +222,29 @@ impl super::DestinationConnector for AdvancedDatabaseDestination {
     }
 
     async fn test_connection(&self) -> crate::Result<ConnectionTest> {
-        Ok(ConnectionTest::success(&format!(
-            "Connected to {} database",
-            self.config.database_type
-        )))
+        // Honest: this connector has no real driver, so it cannot connect.
+        Ok(ConnectionTest::failure(
+            &format!(
+                "{} destination is not implemented: no real connection was attempted",
+                self.config.database_type
+            ),
+            Some("advanced database connector is a stub (RECHECK-054)".to_string()),
+        ))
     }
 
     async fn write_record(&self, _record: &Record) -> crate::Result<()> {
-        // In production: execute INSERT/UPDATE based on strategy
-        Ok(())
+        Err(crate::Error::ConnectorError(format!(
+            "{} destination write is not implemented; refusing to report a write that did not happen",
+            self.config.database_type
+        )))
     }
 
-    async fn write_batch(&self, records: &[Record]) -> crate::Result<usize> {
-        // In production:
-        // 1. If schema defined: CREATE TABLE IF NOT EXISTS
-        // 2. Choose strategy:
-        //    - INSERT: simple insert all
-        //    - UPSERT: use ON CONFLICT or MERGE
-        //    - BULK_LOAD: write to S3/GCS, COPY into database
-        // 3. Handle partitioning for warehouses
-        // 4. Create clustering indexes for BigQuery
-        Ok(records.len())
+    async fn write_batch(&self, _records: &[Record]) -> crate::Result<usize> {
+        Err(crate::Error::ConnectorError(format!(
+            "{} destination batch write is not implemented; refusing to report {} rows written when none were",
+            self.config.database_type,
+            _records.len()
+        )))
     }
 
     async fn validate_records(&self, _records: &[Record]) -> crate::Result<()> {
@@ -283,38 +285,42 @@ impl super::SourceConnector for AdvancedDatabaseSource {
     }
 
     async fn test_connection(&self) -> crate::Result<ConnectionTest> {
-        Ok(ConnectionTest::success(&format!(
-            "Connected to {} database: {}",
-            self.config.database_type, self.config.table
-        )))
+        // Honest: this connector has no real driver, so it cannot connect.
+        Ok(ConnectionTest::failure(
+            &format!(
+                "{} source is not implemented: no real connection was attempted",
+                self.config.database_type
+            ),
+            Some("advanced database connector is a stub (RECHECK-054)".to_string()),
+        ))
     }
 
     async fn detect_schema(&self) -> crate::Result<super::Schema> {
-        // In production: query database metadata
-        // SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?
-        Ok(super::Schema {
-            fields: vec![],
-            sample_records: vec![],
-        })
+        Err(crate::Error::ConnectorError(format!(
+            "{} schema detection is not implemented",
+            self.config.database_type
+        )))
     }
 
     async fn read_all(&self) -> crate::Result<Vec<Record>> {
-        // In production: SELECT * FROM table
-        Ok(vec![])
+        Err(crate::Error::ConnectorError(format!(
+            "{} read is not implemented; refusing to report an empty result as a successful read",
+            self.config.database_type
+        )))
     }
 
     async fn read_batch(&self, _offset: u64, _limit: u64) -> crate::Result<Vec<Record>> {
-        // In production: SELECT * FROM table OFFSET ? LIMIT ?
-        Ok(vec![])
+        Err(crate::Error::ConnectorError(format!(
+            "{} read_batch is not implemented",
+            self.config.database_type
+        )))
     }
 
     async fn read_incremental(&self, _last_value: &str) -> crate::Result<Vec<Record>> {
-        // In production:
-        // If incremental_column is timestamp:
-        //   SELECT * FROM table WHERE incremental_column > ? ORDER BY incremental_column
-        // If incremental_column is id:
-        //   SELECT * FROM table WHERE id > ? ORDER BY id LIMIT ?
-        Ok(vec![])
+        Err(crate::Error::ConnectorError(format!(
+            "{} read_incremental is not implemented",
+            self.config.database_type
+        )))
     }
 
     fn capabilities(&self) -> Vec<Capability> {
@@ -370,8 +376,13 @@ mod tests {
         let dest = AdvancedDatabaseDestination { config };
         assert_eq!(dest.name(), "Snowflake");
 
+        // The connector is a stub: it must report failure, not fake success.
         let test = dest.test_connection().await.unwrap();
-        assert!(test.success);
+        assert!(!test.success);
+        assert!(test.message.contains("not implemented"));
+
+        // And a write must error rather than silently claim rows were written.
+        assert!(dest.write_batch(&[]).await.is_err());
     }
 
     #[tokio::test]

pyreverseetl-3.1.7/core/src/dlq.rs

@@ -0,0 +1,161 @@
+//! Local dead-letter queue (quarantine) for records that failed to write.
+//!
+//! Per the verified audit gap (RECHECK-018): a sync that partially fails left
+//! the operator with only an aggregate `rows_failed` count -- there was no
+//! record of *which* records failed, why, or their payload, so re-driving a
+//! failed batch after a transient destination error (a rate limit, a 5xx, a
+//! single malformed record) meant re-reading the whole source and guessing.
+//!
+//! `DeadLetterQueue` is a real, persisted (SQLite) quarantine: every record
+//! that a destination rejected is stored with its run id, destination key,
+//! the reason for rejection, and its exact payload, so a failed record can be
+//! inspected and replayed deliberately instead of re-running the entire sync.
+
+use rusqlite::{params, Connection};
+use serde::Serialize;
+use std::path::Path;
+use std::sync::Mutex;
+
+/// One quarantined record: what failed, where it was headed, and why.
+#[derive(Debug, Clone, Serialize)]
+pub struct DeadLetter {
+    pub run_id: String,
+    pub destination_key: String,
+    pub record_id: String,
+    pub reason: String,
+    /// The exact record payload (compact JSON) that was rejected.
+    pub payload: String,
+    pub failed_at: String,
+}
+
+pub struct DeadLetterQueue {
+    conn: Mutex<Connection>,
+}
+
+impl DeadLetterQueue {
+    /// Open (creating if absent) a real SQLite file as the quarantine store.
+    pub fn open(path: &Path) -> crate::Result<Self> {
+        let conn = Connection::open(path)?;
+        Self::from_connection(conn)
+    }
+
+    /// SQLite-backed but in-memory -- exercises the real SQL code path without
+    /// touching disk (tests).
+    #[cfg(test)]
+    pub fn in_memory() -> crate::Result<Self> {
+        let conn = Connection::open_in_memory()?;
+        Self::from_connection(conn)
+    }
+
+    fn from_connection(conn: Connection) -> crate::Result<Self> {
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS dead_letters (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                run_id TEXT NOT NULL,
+                destination_key TEXT NOT NULL,
+                record_id TEXT NOT NULL,
+                reason TEXT NOT NULL,
+                payload TEXT NOT NULL,
+                failed_at TEXT NOT NULL
+            )",
+            [],
+        )?;
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_dead_letters_run ON dead_letters(run_id)",
+            [],
+        )?;
+        Ok(Self {
+            conn: Mutex::new(conn),
+        })
+    }
+
+    /// Quarantine one failed record. Appends a new row every time (a record
+    /// that fails across several runs leaves a row per run), so the history of
+    /// repeated failures is preserved rather than overwritten.
+    #[allow(clippy::too_many_arguments)]
+    pub fn record(
+        &self,
+        run_id: &str,
+        destination_key: &str,
+        record_id: &str,
+        reason: &str,
+        payload: &str,
+    ) -> crate::Result<()> {
+        let failed_at = chrono::Utc::now().to_rfc3339();
+        let conn = self.conn.lock().unwrap();
+        conn.execute(
+            "INSERT INTO dead_letters
+                (run_id, destination_key, record_id, reason, payload, failed_at)
+             VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
+            params![run_id, destination_key, record_id, reason, payload, failed_at],
+        )?;
+        Ok(())
+    }
+
+    /// Every record quarantined by a single run, in insertion order.
+    pub fn list_for_run(&self, run_id: &str) -> crate::Result<Vec<DeadLetter>> {
+        let conn = self.conn.lock().unwrap();
+        let mut stmt = conn.prepare(
+            "SELECT run_id, destination_key, record_id, reason, payload, failed_at
+             FROM dead_letters WHERE run_id = ?1 ORDER BY id ASC",
+        )?;
+        let rows = stmt.query_map(params![run_id], |row| {
+            Ok(DeadLetter {
+                run_id: row.get(0)?,
+                destination_key: row.get(1)?,
+                record_id: row.get(2)?,
+                reason: row.get(3)?,
+                payload: row.get(4)?,
+                failed_at: row.get(5)?,
+            })
+        })?;
+        let mut out = Vec::new();
+        for row in rows {
+            out.push(row?);
+        }
+        Ok(out)
+    }
+
+    /// Total number of quarantined records across every run.
+    pub fn count(&self) -> crate::Result<u64> {
+        let conn = self.conn.lock().unwrap();
+        let count: i64 = conn.query_row("SELECT COUNT(*) FROM dead_letters", [], |row| row.get(0))?;
+        Ok(count as u64)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn records_and_lists_failures_per_run() {
+        let dlq = DeadLetterQueue::in_memory().unwrap();
+        dlq.record("run-1", "webhook:x", "rec-1", "HTTP 429", "{\"a\":1}")
+            .unwrap();
+        dlq.record("run-1", "webhook:x", "rec-2", "HTTP 500", "{\"b\":2}")
+            .unwrap();
+        dlq.record("run-2", "webhook:x", "rec-3", "boom", "{\"c\":3}")
+            .unwrap();
+
+        let run1 = dlq.list_for_run("run-1").unwrap();
+        assert_eq!(run1.len(), 2);
+        assert_eq!(run1[0].record_id, "rec-1");
+        assert_eq!(run1[0].reason, "HTTP 429");
+        assert_eq!(run1[0].payload, "{\"a\":1}");
+        assert_eq!(run1[1].record_id, "rec-2");
+
+        assert_eq!(dlq.list_for_run("run-2").unwrap().len(), 1);
+        assert_eq!(dlq.count().unwrap(), 3);
+    }
+
+    #[test]
+    fn repeated_failures_are_appended_not_overwritten() {
+        let dlq = DeadLetterQueue::in_memory().unwrap();
+        dlq.record("run-1", "webhook:x", "rec-1", "HTTP 429", "{}")
+            .unwrap();
+        dlq.record("run-2", "webhook:x", "rec-1", "HTTP 429", "{}")
+            .unwrap();
+        assert_eq!(dlq.count().unwrap(), 2);
+    }
+}