quickhouse 0.3.2__tar.gz → 0.3.4__tar.gz

{quickhouse-0.3.2 → quickhouse-0.3.4}/Cargo.lock

@@ -2634,7 +2634,7 @@ dependencies = [
 
 [[package]]
 name = "quickhouse-core"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "anyhow",
  "arrow",
@@ -2670,7 +2670,7 @@ dependencies = [
 
 [[package]]
 name = "quickhouse-py"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "pyo3",
  "quickhouse-core",
@@ -3038,9 +3038,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.15.0"
+version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
+checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
 dependencies = [
  "web-time",
  "zeroize",

{quickhouse-0.3.2 → quickhouse-0.3.4}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/quickhouse-core", "crates/quickhouse-py"]
 
 [workspace.package]
-version = "0.3.2"
+version = "0.3.4"
 edition = "2021"
 license = "MIT"
 authors = ["quickhouse contributors"]

{quickhouse-0.3.2 → quickhouse-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quickhouse
-Version: 0.3.2
+Version: 0.3.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -72,6 +72,16 @@ print(result)   # rows_read, rows_written, bytes_written, duration_secs, new_wat
   legacy quirks like MySQL zero-dates or out-of-range timestamps are coerced to
   `NULL` with a warning instead of aborting the run.
 
+- **It's gentle on a small production database.** Set `read_max_rows_per_sec`
+  and quickhouse paces the read to that aggregate rate across all partitions;
+  because `COPY`/streaming results only produce as fast as the client consumes,
+  the source scan itself backs off — you're throttling the database's work, not
+  just your own. Combine it with `parallelism=1` (one connection), incremental
+  mode (only new rows), and a `statement_timeout_secs`, point it at a read
+  replica, and a bulk export stops competing with your app. The Postgres
+  connection also shows up as `application_name = 'quickhouse'` in
+  `pg_stat_activity`, so a DBA can see and kill it.
+
 - **There's nothing to stand up.** `pip install quickhouse` and you're done —
   no JVM, no Spark cluster, no separate service. It's an ordinary Python
   dependency that runs wherever your jobs already run: cron, Airflow, Dagster,
@@ -180,6 +190,12 @@ deduplicated on `key` — via ClickHouse's `ReplacingMergeTree`, or a `MERGE`
 upsert on BigQuery (where `key` is therefore required). Re-running with no new
 data does nothing.
 
+Both modes stage through a per-run-unique table (`{dest}_quickhouse_tmp_<id>`)
+that's dropped when the run finishes, including on failure. The unique name is
+what makes rapid re-runs and whole-call retries safe on BigQuery, whose
+streaming ingestion rejects writes into a table recently recreated under the
+same name.
+
 For daily syncs that need to catch late-arriving or edited rows, set
 `lookback_seconds` to re-scan a trailing window (e.g. `3 * 86400` for the last
 three days) — the dedup above keeps that overlap from creating duplicates.
@@ -212,6 +228,7 @@ errors are surfaced verbatim rather than wrapped in something generic.
 | `parallelism` | Concurrent read streams |
 | `batch_rows` / `batch_bytes` | Per-batch size knobs (rows, or estimated bytes) |
 | `max_memory_bytes` | Hard ceiling on total in-flight memory; decoding blocks when hit (default 512 MiB, `0` = unbounded) |
+| `read_max_rows_per_sec` | Cap the aggregate source read rate to be gentle on a small DB; `None` = unlimited (default). Postgres/MySQL only |
 | `type_overrides` | Force a destination column type, e.g. `{"qty": "Decimal(18, 3)"}` |
 | `rename`, `include`, `exclude` | Column renames and allow/deny lists |
 | `on_progress` | Progress callback |
@@ -234,6 +251,13 @@ knowing:
   front, rather than silently falling back to `Float64`.
 - **`TIME`** columns transfer as canonical text into a `String` column
   (ClickHouse has no time-of-day type).
+- **MySQL `DATETIME`/`TIMESTAMP`** map to a UTC-aware timestamp (BigQuery
+  `TIMESTAMP`, ClickHouse `DateTime64(6, 'UTC')`) — the wall-clock value is read
+  as UTC, matching how a `TIMESTAMP` column expects it. To land a column as a
+  naive BigQuery `DATETIME` instead, opt out per-column with
+  `type_overrides={"col": "DATETIME"}` — that flips the actual encoding, not
+  just the declared type. (PostgreSQL keeps the distinction natively:
+  `timestamptz` → UTC-aware, `timestamp` → naive.)
 - **Out-of-range dates** (and MySQL zero-dates like `0000-00-00`) coerce to
   `NULL` with a warning rather than failing the transfer.
 - **Nullable** source columns stay nullable in the destination.

{quickhouse-0.3.2 → quickhouse-0.3.4}/README.md

@@ -40,6 +40,16 @@ print(result)   # rows_read, rows_written, bytes_written, duration_secs, new_wat
   legacy quirks like MySQL zero-dates or out-of-range timestamps are coerced to
   `NULL` with a warning instead of aborting the run.
 
+- **It's gentle on a small production database.** Set `read_max_rows_per_sec`
+  and quickhouse paces the read to that aggregate rate across all partitions;
+  because `COPY`/streaming results only produce as fast as the client consumes,
+  the source scan itself backs off — you're throttling the database's work, not
+  just your own. Combine it with `parallelism=1` (one connection), incremental
+  mode (only new rows), and a `statement_timeout_secs`, point it at a read
+  replica, and a bulk export stops competing with your app. The Postgres
+  connection also shows up as `application_name = 'quickhouse'` in
+  `pg_stat_activity`, so a DBA can see and kill it.
+
 - **There's nothing to stand up.** `pip install quickhouse` and you're done —
   no JVM, no Spark cluster, no separate service. It's an ordinary Python
   dependency that runs wherever your jobs already run: cron, Airflow, Dagster,
@@ -148,6 +158,12 @@ deduplicated on `key` — via ClickHouse's `ReplacingMergeTree`, or a `MERGE`
 upsert on BigQuery (where `key` is therefore required). Re-running with no new
 data does nothing.
 
+Both modes stage through a per-run-unique table (`{dest}_quickhouse_tmp_<id>`)
+that's dropped when the run finishes, including on failure. The unique name is
+what makes rapid re-runs and whole-call retries safe on BigQuery, whose
+streaming ingestion rejects writes into a table recently recreated under the
+same name.
+
 For daily syncs that need to catch late-arriving or edited rows, set
 `lookback_seconds` to re-scan a trailing window (e.g. `3 * 86400` for the last
 three days) — the dedup above keeps that overlap from creating duplicates.
@@ -180,6 +196,7 @@ errors are surfaced verbatim rather than wrapped in something generic.
 | `parallelism` | Concurrent read streams |
 | `batch_rows` / `batch_bytes` | Per-batch size knobs (rows, or estimated bytes) |
 | `max_memory_bytes` | Hard ceiling on total in-flight memory; decoding blocks when hit (default 512 MiB, `0` = unbounded) |
+| `read_max_rows_per_sec` | Cap the aggregate source read rate to be gentle on a small DB; `None` = unlimited (default). Postgres/MySQL only |
 | `type_overrides` | Force a destination column type, e.g. `{"qty": "Decimal(18, 3)"}` |
 | `rename`, `include`, `exclude` | Column renames and allow/deny lists |
 | `on_progress` | Progress callback |
@@ -202,6 +219,13 @@ knowing:
   front, rather than silently falling back to `Float64`.
 - **`TIME`** columns transfer as canonical text into a `String` column
   (ClickHouse has no time-of-day type).
+- **MySQL `DATETIME`/`TIMESTAMP`** map to a UTC-aware timestamp (BigQuery
+  `TIMESTAMP`, ClickHouse `DateTime64(6, 'UTC')`) — the wall-clock value is read
+  as UTC, matching how a `TIMESTAMP` column expects it. To land a column as a
+  naive BigQuery `DATETIME` instead, opt out per-column with
+  `type_overrides={"col": "DATETIME"}` — that flips the actual encoding, not
+  just the declared type. (PostgreSQL keeps the distinction natively:
+  `timestamptz` → UTC-aware, `timestamp` → naive.)
 - **Out-of-range dates** (and MySQL zero-dates like `0000-00-00`) coerce to
   `NULL` with a warning rather than failing the transfer.
 - **Nullable** source columns stay nullable in the destination.

{quickhouse-0.3.2 → quickhouse-0.3.4}/crates/quickhouse-core/src/config.rs

@@ -256,6 +256,17 @@ pub struct TransferConfig {
     /// Column used to split the table into parallel range partitions.
     /// Defaults to the first `key` column, else the sync falls back to a single stream.
     pub partition_column: Option<String>,
+    /// Optional cap on how many source rows are pulled **per second**, summed
+    /// across all parallel partitions (a global limiter, not per-connection).
+    /// Deliberately paces the read so a small/production database isn't
+    /// hammered by a bulk export: after each batch is read, the reader sleeps
+    /// long enough to hold the aggregate rate at this ceiling, which — because
+    /// `COPY TO STDOUT` streams only as fast as the client consumes — makes the
+    /// server-side scan itself back off (TCP backpressure), not just the
+    /// client. `None` (default) reads as fast as possible. Applies to the
+    /// PostgreSQL and MySQL sources; ignored for a BigQuery source (its read
+    /// path is a managed, separately-metered API).
+    pub read_max_rows_per_sec: Option<u64>,
 
     // ---- transforms ----
     /// Per-column destination type overrides (column name -> the
@@ -330,6 +341,11 @@ impl TransferConfig {
                 "max_memory_bytes must be 0 (unbounded) or >= 65536",
             ));
         }
+        if self.read_max_rows_per_sec == Some(0) {
+            return Err(EtlError::config(
+                "read_max_rows_per_sec must be None (unlimited) or >= 1",
+            ));
+        }
         Ok(())
     }
 }
@@ -367,6 +383,7 @@ mod tests {
             batch_bytes: 0,
             max_memory_bytes: 0,
             partition_column: None,
+            read_max_rows_per_sec: None,
             type_overrides: HashMap::new(),
             rename: HashMap::new(),
             include: vec![],
@@ -423,4 +440,21 @@ mod tests {
         c.lookback_seconds = 60;
         assert!(c.validate().is_ok());
     }
+
+    #[test]
+    fn validate_rejects_zero_read_rate() {
+        let mut c = cfg(SyncMode::Full, None);
+        c.read_max_rows_per_sec = Some(0);
+        let err = c.validate().unwrap_err().to_string();
+        assert!(err.contains("read_max_rows_per_sec"), "got: {err}");
+    }
+
+    #[test]
+    fn validate_accepts_none_or_positive_read_rate() {
+        let mut c = cfg(SyncMode::Full, None);
+        c.read_max_rows_per_sec = None;
+        assert!(c.validate().is_ok());
+        c.read_max_rows_per_sec = Some(10_000);
+        assert!(c.validate().is_ok());
+    }
 }

{quickhouse-0.3.2 → quickhouse-0.3.4}/crates/quickhouse-core/src/ddl.rs

@@ -147,6 +147,7 @@ mod tests {
             batch_bytes: 0,
             max_memory_bytes: 0,
             partition_column: None,
+            read_max_rows_per_sec: None,
             type_overrides: HashMap::new(),
             rename: HashMap::new(),
             include: vec![],

{quickhouse-0.3.2 → quickhouse-0.3.4}/crates/quickhouse-core/src/sink/bigquery.rs

@@ -945,6 +945,7 @@ mod tests {
             batch_bytes: 0,
             max_memory_bytes: 0,
             partition_column: None,
+            read_max_rows_per_sec: None,
             type_overrides: HashMap::new(),
             rename: HashMap::new(),
             include: vec![],

{quickhouse-0.3.2 → quickhouse-0.3.4}/crates/quickhouse-core/src/source/postgres.rs

@@ -92,14 +92,22 @@ impl PgSource {
                 tracing::warn!("postgres connection error: {e}");
             }
         });
+        // Session setup, run as one round-trip. `application_name` makes this
+        // export visible (and killable) in `pg_stat_activity` — exactly what a
+        // DBA wants when watching load from a bulk read. When a statement
+        // timeout is configured we also cap `idle_in_transaction_session_timeout`
+        // at the same value: defense-in-depth so a stalled connection can never
+        // sit inside an open transaction holding an MVCC snapshot (which would
+        // block vacuum and bloat the source). The bulk COPY itself runs in
+        // autocommit, so statement_timeout is the primary guard; this covers the
+        // edge/future transactional paths.
+        let mut setup = String::from("SET application_name = 'quickhouse'");
         if self.statement_timeout_secs > 0 {
-            client
-                .batch_execute(&format!(
-                    "SET statement_timeout = {}",
-                    self.statement_timeout_secs * 1000
-                ))
-                .await?;
+            let ms = self.statement_timeout_secs * 1000;
+            setup.push_str(&format!("; SET statement_timeout = {ms}"));
+            setup.push_str(&format!("; SET idle_in_transaction_session_timeout = {ms}"));
         }
+        client.batch_execute(&setup).await?;
         Ok(client)
     }
 
@@ -132,7 +140,7 @@ impl PgSource {
                 column: c.name().to_string(),
                 type_name: c.type_().name().to_string(),
             })?;
-            let nullable = !not_null.contains(&c.name().to_string());
+            let nullable = !not_null.contains(c.name());
             cols.push(ColumnType {
                 name: c.name().to_string(),
                 type_id: pg_oid,