quickhouse 0.3.4__tar.gz → 0.4.0__tar.gz

{quickhouse-0.3.4 → quickhouse-0.4.0}/Cargo.lock

@@ -555,9 +555,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "cc"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
+checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -949,9 +949,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
 
 [[package]]
 name = "either"
-version = "1.16.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
+checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
 
 [[package]]
 name = "encoding_rs"
@@ -2634,7 +2634,7 @@ dependencies = [
 
 [[package]]
 name = "quickhouse-core"
-version = "0.3.4"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "arrow",
@@ -2670,7 +2670,7 @@ dependencies = [
 
 [[package]]
 name = "quickhouse-py"
-version = "0.3.4"
+version = "0.4.0"
 dependencies = [
  "pyo3",
  "quickhouse-core",

{quickhouse-0.3.4 → quickhouse-0.4.0}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/quickhouse-core", "crates/quickhouse-py"]
 
 [workspace.package]
-version = "0.3.4"
+version = "0.4.0"
 edition = "2021"
 license = "MIT"
 authors = ["quickhouse contributors"]

{quickhouse-0.3.4 → quickhouse-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quickhouse
-Version: 0.3.4
+Version: 0.4.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -200,6 +200,35 @@ For daily syncs that need to catch late-arriving or edited rows, set
 `lookback_seconds` to re-scan a trailing window (e.g. `3 * 86400` for the last
 three days) — the dedup above keeps that overlap from creating duplicates.
 
+**Cursor control.** A few knobs make the incremental cursor robust in the real
+world:
+
+- `state_key` pins the cursor's identity. By default it's keyed by the source
+  table (or query text) + destination — so editing a `source_query`'s `WHERE`
+  would silently start a fresh full pull, and two syncs into one destination
+  tracking different `watermark` columns would clobber each other's cursor. Set
+  `state_key="orders:updated_at"` to give each a stable, distinct identity.
+- `skip_to_max=True` (or `seed_watermark="<value>"`) seeds the cursor on the
+  **first** run only, then self-retires. Use it when the destination already
+  holds complete data from a previous pipeline and a full first pull would be a
+  waste — it starts the cursor at the source's current max instead of scanning
+  everything.
+- `advance_watermark=False` reads and merges a window *without* moving the
+  cursor — for loading a historical backfill without rewinding your regular
+  schedule.
+
+**MERGE cost on large BigQuery tables.** By default an incremental `MERGE`
+scans the whole destination table (it joins on `key` only), so upserting a few
+delta rows into a huge partitioned table bills the whole table each run. Set
+`merge_prune_partition_by="create_date"` to bound the scan to the staging
+batch's range and let BigQuery prune partitions. **Only do this for an
+*immutable* column** — one whose value never changes for a given `key`, i.e. a
+`create_date`/inserted-at column that is also the partition column. Do **not**
+point it at a `write_date`/updated-at column: an updated row's new value lands
+in a different partition than the existing row, so pruning would miss it and
+insert a duplicate key instead of updating. quickhouse can't detect mutability,
+so this is a deliberate opt-in; the default full scan is always correct.
+
 ### Watching progress and diagnosing failures
 
 `on_progress` is a plain callback you can point at anything; `qh.progress_bar()`
@@ -222,6 +251,10 @@ errors are surfaced verbatim rather than wrapped in something generic.
 | `mode` | `"full"` or `"incremental"` |
 | `watermark` | Monotonic column for incremental (e.g. `updated_at`); ignored in full mode |
 | `lookback_seconds` | Re-scan a trailing window of the watermark to catch late/edited rows; `0` disables (default) |
+| `state_key` | Pin the incremental cursor's identity (stable across `source_query` edits; distinct per watermark column). Default derives it from source+dest |
+| `seed_watermark` / `skip_to_max` | Seed the cursor on the first run only (explicit floor, or the source's current max) — skips a doomed first full pull; mutually exclusive |
+| `advance_watermark` | `False` reads+merges a window without advancing the cursor (backfill without rewinding the schedule); default `True` |
+| `merge_prune_partition_by` | BigQuery incremental: prune the MERGE's destination scan to the staging range on this column. Only safe for an *immutable* partition column (e.g. `create_date`) — never a mutable `write_date` (would insert dup keys) |
 | `key` | Dedup key (required for BigQuery incremental) |
 | `create_if_missing` | Auto-create the destination table (default `True`) |
 | `engine`, `order_by`, `partition_by`, `primary_key` | DDL knobs, interpreted per destination |

{quickhouse-0.3.4 → quickhouse-0.4.0}/README.md

@@ -168,6 +168,35 @@ For daily syncs that need to catch late-arriving or edited rows, set
 `lookback_seconds` to re-scan a trailing window (e.g. `3 * 86400` for the last
 three days) — the dedup above keeps that overlap from creating duplicates.
 
+**Cursor control.** A few knobs make the incremental cursor robust in the real
+world:
+
+- `state_key` pins the cursor's identity. By default it's keyed by the source
+  table (or query text) + destination — so editing a `source_query`'s `WHERE`
+  would silently start a fresh full pull, and two syncs into one destination
+  tracking different `watermark` columns would clobber each other's cursor. Set
+  `state_key="orders:updated_at"` to give each a stable, distinct identity.
+- `skip_to_max=True` (or `seed_watermark="<value>"`) seeds the cursor on the
+  **first** run only, then self-retires. Use it when the destination already
+  holds complete data from a previous pipeline and a full first pull would be a
+  waste — it starts the cursor at the source's current max instead of scanning
+  everything.
+- `advance_watermark=False` reads and merges a window *without* moving the
+  cursor — for loading a historical backfill without rewinding your regular
+  schedule.
+
+**MERGE cost on large BigQuery tables.** By default an incremental `MERGE`
+scans the whole destination table (it joins on `key` only), so upserting a few
+delta rows into a huge partitioned table bills the whole table each run. Set
+`merge_prune_partition_by="create_date"` to bound the scan to the staging
+batch's range and let BigQuery prune partitions. **Only do this for an
+*immutable* column** — one whose value never changes for a given `key`, i.e. a
+`create_date`/inserted-at column that is also the partition column. Do **not**
+point it at a `write_date`/updated-at column: an updated row's new value lands
+in a different partition than the existing row, so pruning would miss it and
+insert a duplicate key instead of updating. quickhouse can't detect mutability,
+so this is a deliberate opt-in; the default full scan is always correct.
+
 ### Watching progress and diagnosing failures
 
 `on_progress` is a plain callback you can point at anything; `qh.progress_bar()`
@@ -190,6 +219,10 @@ errors are surfaced verbatim rather than wrapped in something generic.
 | `mode` | `"full"` or `"incremental"` |
 | `watermark` | Monotonic column for incremental (e.g. `updated_at`); ignored in full mode |
 | `lookback_seconds` | Re-scan a trailing window of the watermark to catch late/edited rows; `0` disables (default) |
+| `state_key` | Pin the incremental cursor's identity (stable across `source_query` edits; distinct per watermark column). Default derives it from source+dest |
+| `seed_watermark` / `skip_to_max` | Seed the cursor on the first run only (explicit floor, or the source's current max) — skips a doomed first full pull; mutually exclusive |
+| `advance_watermark` | `False` reads+merges a window without advancing the cursor (backfill without rewinding the schedule); default `True` |
+| `merge_prune_partition_by` | BigQuery incremental: prune the MERGE's destination scan to the staging range on this column. Only safe for an *immutable* partition column (e.g. `create_date`) — never a mutable `write_date` (would insert dup keys) |
 | `key` | Dedup key (required for BigQuery incremental) |
 | `create_if_missing` | Auto-create the destination table (default `True`) |
 | `engine`, `order_by`, `partition_by`, `primary_key` | DDL knobs, interpreted per destination |

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/config.rs

@@ -182,6 +182,25 @@ pub enum SyncMode {
     Incremental,
 }
 
+/// How to seed the incremental cursor on the **first** run for a state
+/// identity (i.e. when `_quickhouse_state` has no row yet). Once a real
+/// watermark has been persisted this is ignored, so it self-retires after the
+/// first successful run — a safe no-op thereafter.
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub enum WatermarkSeed {
+    /// First run reads the whole table (today's behavior).
+    #[default]
+    None,
+    /// First run reads only rows past this explicit watermark floor.
+    Value(String),
+    /// First run seeds the cursor to the source's current MAX(watermark),
+    /// reading (almost) nothing — for when the destination already holds
+    /// complete data from a prior/legacy pipeline and a full first pull would
+    /// be a doomed waste. Replaces the manual `INSERT INTO _quickhouse_state`
+    /// hack.
+    CurrentMax,
+}
+
 /// One table transfer.
 #[derive(Debug, Clone)]
 pub struct TransferConfig {
@@ -192,6 +211,16 @@ pub struct TransferConfig {
     /// Destination table name (a bare name within the ClickHouse `database` or
     /// BigQuery `dataset_id`).
     pub dest_table: String,
+    /// Stable identity for the persisted incremental cursor in
+    /// `_quickhouse_state`. `None` (default) derives it from `source_table`,
+    /// else the `source_query` text (byte-identical to pre-`state_key`
+    /// behavior — so existing state is never orphaned). Set it to (a) keep the
+    /// cursor stable when you edit a `source_query`'s WHERE/SELECT (whose text
+    /// would otherwise change the derived key and silently reset the cursor),
+    /// and (b) give two syncs that share a `dest_table` but track different
+    /// `watermark` columns distinct cursors (which otherwise collide on one
+    /// state row). See [`TransferConfig::effective_state_key`].
+    pub state_key: Option<String>,
 
     pub mode: SyncMode,
 
@@ -206,6 +235,17 @@ pub struct TransferConfig {
     /// duplicate them) and a `watermark` column that resolves to a date or
     /// timestamp type.
     pub lookback_seconds: u64,
+    /// How to seed the incremental cursor on the first run for this state
+    /// identity (default [`WatermarkSeed::None`] = read the whole table, as
+    /// before). Only meaningful in incremental mode; ignored once a real
+    /// watermark has been persisted. See [`WatermarkSeed`].
+    pub seed_watermark: WatermarkSeed,
+    /// Whether a successful incremental run persists (advances) the watermark.
+    /// `true` (default) is today's behavior. Set `false` to read+merge a
+    /// window WITHOUT moving the scheduled cursor — the primitive a bounded
+    /// backfill needs so it doesn't rewind the regular schedule. Incremental
+    /// mode only.
+    pub advance_watermark: bool,
     /// Business/dedup key. ClickHouse: contributes to `ORDER BY` when no
     /// explicit `order_by` is given. BigQuery: contributes to `Clustering`
     /// alongside `order_by` (see its docs) — BigQuery has no dedicated key
@@ -230,6 +270,24 @@ pub struct TransferConfig {
     /// resolve to one of those types.
     pub partition_by: Option<String>,
     pub primary_key: Vec<String>,
+    /// BigQuery-destination incremental only: prune the `MERGE`'s destination
+    /// scan to the staging batch's range on this column, so BigQuery reads only
+    /// the touched partitions instead of full-scanning the (possibly huge)
+    /// destination table on every merge. `None` (default) full-scans, as before.
+    ///
+    /// **Correctness contract — read before setting.** This is ONLY safe when
+    /// the named column is IMMUTABLE for a given merge `key` (its value never
+    /// changes across updates to the same row), and it should be the table's
+    /// partition column. A `create_date`/inserted-at column is safe: a row's
+    /// value stays put, so the existing target row always lives in the
+    /// partition the staging row implies. A `write_date`/updated-at column is
+    /// **NOT** safe: an updated row's new `write_date` points at a different
+    /// partition than where the old row lives, so pruning would miss it, fall
+    /// through to `WHEN NOT MATCHED`, and INSERT A DUPLICATE KEY instead of
+    /// updating. (This is the historical `merge_query_filter` duplicate-id bug;
+    /// do not "optimize" a mutable partition column into this field.) quickhouse
+    /// cannot detect mutability, so this is a deliberate per-table opt-in.
+    pub merge_prune_partition_by: Option<String>,
 
     // ---- parallelism / batching ----
     pub parallelism: usize,
@@ -301,9 +359,23 @@ impl TransferConfig {
     pub fn normalize(&mut self) {
         if self.mode == SyncMode::Full {
             self.watermark = None;
+            // The seed only meaningfully floors an incremental cursor; a
+            // full refresh has none, so clear it (mirrors `watermark`).
+            self.seed_watermark = WatermarkSeed::None;
         }
     }
 
+    /// Identity of the persisted watermark row in `_quickhouse_state`.
+    /// Overridable via `state_key`; otherwise the source table name, else the
+    /// query text (unchanged default — so existing state rows keep matching).
+    pub fn effective_state_key(&self) -> String {
+        self.state_key
+            .clone()
+            .or_else(|| self.source_table.clone())
+            .or_else(|| self.source_query.clone())
+            .unwrap_or_default()
+    }
+
     pub fn validate(&self) -> crate::error::Result<()> {
         use crate::error::EtlError;
         if self.source_table.is_none() && self.source_query.is_none() {
@@ -321,6 +393,16 @@ impl TransferConfig {
                 "lookback_seconds only applies to incremental mode",
             ));
         }
+        if self.seed_watermark != WatermarkSeed::None && self.mode != SyncMode::Incremental {
+            return Err(EtlError::config(
+                "seed_watermark only applies to incremental mode",
+            ));
+        }
+        if !self.advance_watermark && self.mode != SyncMode::Incremental {
+            return Err(EtlError::config(
+                "advance_watermark=false only applies to incremental mode",
+            ));
+        }
         if self.lookback_seconds > 0 && self.key.is_empty() && self.order_by.is_empty() {
             return Err(EtlError::config(
                 "lookback_seconds requires key or order_by (otherwise the re-synced \
@@ -369,15 +451,19 @@ mod tests {
             source_table: Some("t".into()),
             source_query: None,
             dest_table: "t".into(),
+            state_key: None,
             mode,
             watermark: watermark.map(str::to_string),
             lookback_seconds: 0,
+            seed_watermark: WatermarkSeed::None,
+            advance_watermark: true,
             key: vec!["id".into()],
             create_if_missing: true,
             engine: None,
             order_by: vec![],
             partition_by: None,
             primary_key: vec![],
+            merge_prune_partition_by: None,
             parallelism: 1,
             batch_rows: 1000,
             batch_bytes: 0,
@@ -457,4 +543,46 @@ mod tests {
         c.read_max_rows_per_sec = Some(10_000);
         assert!(c.validate().is_ok());
     }
+
+    #[test]
+    fn effective_state_key_prefers_override_then_table_then_query() {
+        let mut c = cfg(SyncMode::Incremental, Some("write_date"));
+        // Default: source_table.
+        assert_eq!(c.effective_state_key(), "t");
+        // source_query wins over nothing but loses to source_table.
+        c.source_table = None;
+        c.source_query = Some("SELECT * FROM t WHERE x".into());
+        assert_eq!(c.effective_state_key(), "SELECT * FROM t WHERE x");
+        // explicit override wins over both — and is stable across query edits.
+        c.state_key = Some("orders:write_date".into());
+        assert_eq!(c.effective_state_key(), "orders:write_date");
+        c.source_query = Some("SELECT * FROM t WHERE y /* edited */".into());
+        assert_eq!(c.effective_state_key(), "orders:write_date");
+    }
+
+    #[test]
+    fn validate_rejects_seed_and_freeze_outside_incremental() {
+        let mut c = cfg(SyncMode::Full, None);
+        c.seed_watermark = WatermarkSeed::CurrentMax;
+        assert!(c.validate().unwrap_err().to_string().contains("seed_watermark"));
+        c.seed_watermark = WatermarkSeed::None;
+        c.advance_watermark = false;
+        assert!(c.validate().unwrap_err().to_string().contains("advance_watermark"));
+    }
+
+    #[test]
+    fn validate_accepts_seed_and_freeze_in_incremental() {
+        let mut c = cfg(SyncMode::Incremental, Some("write_date"));
+        c.seed_watermark = WatermarkSeed::Value("2026-01-01".into());
+        c.advance_watermark = false;
+        assert!(c.validate().is_ok());
+    }
+
+    #[test]
+    fn normalize_clears_seed_in_full_mode() {
+        let mut c = cfg(SyncMode::Full, Some("write_date"));
+        c.seed_watermark = WatermarkSeed::CurrentMax;
+        c.normalize();
+        assert_eq!(c.seed_watermark, WatermarkSeed::None);
+    }
 }

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/ddl.rs

@@ -142,12 +142,16 @@ mod tests {
             order_by: vec![],
             partition_by: None,
             primary_key: vec![],
+            merge_prune_partition_by: None,
             parallelism: 4,
             batch_rows: 1000,
             batch_bytes: 0,
             max_memory_bytes: 0,
             partition_column: None,
             read_max_rows_per_sec: None,
+            state_key: None,
+            seed_watermark: crate::config::WatermarkSeed::None,
+            advance_watermark: true,
             type_overrides: HashMap::new(),
             rename: HashMap::new(),
             include: vec![],

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/decode.rs

@@ -613,6 +613,58 @@ mod tests {
         v
     }
 
+    /// Golden type matrix for the PostgreSQL source: for every OID `map_oid`
+    /// produces, assert the full type chain END-TO-END from the source type —
+    /// (1) mapping to Arrow + ClickHouse inner, (2) the decoder's output array
+    /// type carries the correct timezone, (3) the batch-vs-schema check
+    /// (`RecordBatch::try_new`, the literal `flush_batch` validation) passes,
+    /// and (4) the Arrow→BigQuery destination-type bridge. Starting from the
+    /// OID (not a hand-built Arrow type) is what makes this catch a mapping
+    /// revert — the exact class of regression that shipped in 0.3.4 for MySQL
+    /// (there was no equivalent Postgres tz coverage at all before this).
+    #[test]
+    fn postgres_type_golden_matrix() {
+        use crate::types::bigquery::arrow_to_bigquery_type as a2b;
+        use crate::types::{map_oid, oid};
+        use arrow_schema::{Field, Schema, TimeUnit};
+        use google_cloud_bigquery::http::table::TableFieldType as Bq;
+        use std::sync::Arc;
+
+        let ts = |tz: Option<&str>| DataType::Timestamp(TimeUnit::Microsecond, tz.map(Arc::from));
+        // (oid, expected Arrow, expected ClickHouse inner, expected BigQuery type)
+        let rows: Vec<(u32, DataType, &str, Bq)> = vec![
+            (oid::BOOL, DataType::Boolean, "Bool", Bq::Boolean),
+            (oid::INT2, DataType::Int16, "Int16", Bq::Integer),
+            (oid::INT4, DataType::Int32, "Int32", Bq::Integer),
+            (oid::INT8, DataType::Int64, "Int64", Bq::Integer),
+            (oid::OID, DataType::UInt32, "UInt32", Bq::Integer),
+            (oid::FLOAT4, DataType::Float32, "Float32", Bq::Float),
+            (oid::FLOAT8, DataType::Float64, "Float64", Bq::Float),
+            (oid::NUMERIC, DataType::Float64, "Float64", Bq::Float),
+            (oid::TEXT, DataType::Utf8, "String", Bq::String),
+            (oid::UUID, DataType::Utf8, "UUID", Bq::String),
+            (oid::BYTEA, DataType::Binary, "String", Bq::Bytes),
+            (oid::DATE, DataType::Date32, "Date32", Bq::Date),
+            (oid::TIMESTAMP, ts(None), "DateTime64(6)", Bq::Datetime),
+            (oid::TIMESTAMPTZ, ts(Some("UTC")), "DateTime64(6, 'UTC')", Bq::Timestamp),
+            (oid::TIME, DataType::Utf8, "String", Bq::String),
+        ];
+        for (o, arrow, ch, bq) in rows {
+            let (mapped_arrow, mapped_ch) = map_oid(o).unwrap_or_else(|| panic!("oid {o} unmapped"));
+            assert_eq!(mapped_arrow, arrow, "oid {o}: Arrow type");
+            assert_eq!(mapped_ch, ch, "oid {o}: ClickHouse inner");
+            // Decoder output type (incl. tz) must equal the resolved schema type.
+            let mut b = ColBuilder::new(&arrow).unwrap();
+            let out = b.finish();
+            assert_eq!(out.data_type(), &arrow, "oid {o}: decoder output type/tz");
+            // The literal flush_batch validation must accept it.
+            let schema = Arc::new(Schema::new(vec![Field::new("c", arrow.clone(), true)]));
+            RecordBatch::try_new(schema, vec![out]).unwrap_or_else(|e| panic!("oid {o}: batch: {e}"));
+            // Destination bridge (Arrow -> BigQuery column type).
+            assert_eq!(a2b(&arrow), Some(bq), "oid {o}: BigQuery type");
+        }
+    }
+
     #[test]
     fn decodes_rows_and_nulls() {
         let cols = vec![

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/decode_bigquery.rs

@@ -379,6 +379,50 @@ mod tests {
         Row::decode_arrow(&[arr], 0).unwrap()
     }
 
+    /// Golden type matrix for the BigQuery source: for every `TableFieldType`
+    /// `map_type` handles, assert the chain from the BQ type through (1) mapping
+    /// to (type_id, Arrow, ClickHouse inner), (2) the decoder's output array
+    /// type carrying the correct timezone — the TIMESTAMP(→UTC-aware) vs
+    /// DATETIME(→naive) split is the exact cell whose MySQL analogue broke in
+    /// 0.3.4 and had no BigQuery coverage — (3) the batch-vs-schema check, and
+    /// (4) the Arrow→BigQuery bridge (lossy for TIME/NUMERIC, asserted as such).
+    #[test]
+    fn bigquery_type_golden_matrix() {
+        use crate::types::bigquery::arrow_to_bigquery_type as a2b;
+        use crate::types::bigquery::{map_type, type_id as id};
+        use arrow_schema::{Field, Schema, TimeUnit};
+        use google_cloud_bigquery::http::table::TableFieldType as Bq;
+
+        let ts = |tz: Option<&str>| DataType::Timestamp(TimeUnit::Microsecond, tz.map(Arc::from));
+        // (input BQ type, expected type_id, expected Arrow, expected CH inner, expected a2b bridge)
+        let rows: Vec<(Bq, u32, DataType, &str, Bq)> = vec![
+            (Bq::String, id::STRING, DataType::Utf8, "String", Bq::String),
+            (Bq::Bytes, id::BYTES, DataType::Binary, "String", Bq::Bytes),
+            (Bq::Integer, id::INTEGER, DataType::Int64, "Int64", Bq::Integer),
+            (Bq::Float, id::FLOAT, DataType::Float64, "Float64", Bq::Float),
+            (Bq::Boolean, id::BOOLEAN, DataType::Boolean, "Bool", Bq::Boolean),
+            (Bq::Timestamp, id::TIMESTAMP, ts(Some("UTC")), "DateTime64(6, 'UTC')", Bq::Timestamp),
+            (Bq::Date, id::DATE, DataType::Date32, "Date32", Bq::Date),
+            (Bq::Time, id::TIME, DataType::Utf8, "String", Bq::String), // lossy: Time -> String
+            (Bq::Datetime, id::DATETIME, ts(None), "DateTime64(6)", Bq::Datetime),
+            (Bq::Numeric, id::NUMERIC, DataType::Float64, "Float64", Bq::Float), // lossy
+            (Bq::Bignumeric, id::BIGNUMERIC, DataType::Float64, "Float64", Bq::Float), // lossy
+        ];
+        for (input, tid, arrow, ch, bridge) in rows {
+            let (mapped_id, mapped_arrow, mapped_ch) =
+                map_type(&input).unwrap_or_else(|| panic!("{input:?} unmapped"));
+            assert_eq!(mapped_id, tid, "{input:?}: type_id");
+            assert_eq!(mapped_arrow, arrow, "{input:?}: Arrow type");
+            assert_eq!(mapped_ch, ch, "{input:?}: ClickHouse inner");
+            let mut b = ColBuilder::new(&arrow).unwrap();
+            let out = b.finish();
+            assert_eq!(out.data_type(), &arrow, "{input:?}: decoder output type/tz");
+            let schema = Arc::new(Schema::new(vec![Field::new("c", arrow.clone(), true)]));
+            RecordBatch::try_new(schema, vec![out]).unwrap_or_else(|e| panic!("{input:?}: batch: {e}"));
+            assert_eq!(a2b(&arrow), Some(bridge), "{input:?}: Arrow->BigQuery bridge");
+        }
+    }
+
     #[test]
     fn decimal_decodes_exact_text_value() {
         let mut b = ColBuilder::new(&DataType::Decimal128(10, 4)).unwrap();

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/decode_mysql.rs

@@ -59,8 +59,16 @@ impl ColBuilder {
             DataType::Utf8 => ColBuilder::Str(StringBuilder::new()),
             DataType::Binary => ColBuilder::Bin(BinaryBuilder::new()),
             DataType::Date32 => ColBuilder::Date(Date32Builder::new()),
-            DataType::Timestamp(TimeUnit::Microsecond, _) => {
-                ColBuilder::Ts(TimestampMicrosecondBuilder::new())
+            // Carry the resolved timezone onto the builder so `finish()`
+            // produces an array whose type matches the batch schema exactly.
+            // MySQL datetimes now resolve to `Timestamp(µs, Some("UTC"))` (see
+            // types::map_mysql_type); dropping the tz here (as this used to)
+            // built a naive array that failed the batch-vs-schema check with
+            // "expected Timestamp(.., Some(\"UTC\")) but found Timestamp(..,
+            // None)". The stored micros are unaffected — this only sets the
+            // array's tz metadata. Mirrors the Postgres decoder (decode.rs).
+            DataType::Timestamp(TimeUnit::Microsecond, tz) => {
+                ColBuilder::Ts(TimestampMicrosecondBuilder::new().with_timezone_opt(tz.clone()))
             }
             DataType::Decimal128(p, s) => ColBuilder::Decimal128(
                 Decimal128Builder::new().with_precision_and_scale(*p, *s)?,
@@ -446,6 +454,58 @@ mod tests {
         assert!(arr.is_null(2));
     }
 
+    /// End-to-end reproduction of the reported v0.3.4 failure, hermetic (no
+    /// live MySQL needed). The crash was a batch-vs-schema mismatch in
+    /// `RecordBatch::try_new` inside `flush_batch`; this rebuilds that exact
+    /// step — the schema field from the resolved Arrow type (as
+    /// `MySqlBatcher::with_batch_bytes` does via `Field::new(c.arrow, ...)`)
+    /// and the column array from `ColBuilder::finish()` — for a UTC-mapped
+    /// MySQL timestamp column. Before the decoder fix, `try_new` returned
+    /// "column types must match schema types, expected Timestamp(..,
+    /// Some(\"UTC\")) but found Timestamp(.., None)"; now the batch builds and
+    /// the column keeps its UTC type.
+    #[test]
+    fn utc_timestamp_column_builds_a_valid_batch() {
+        let arrow = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()));
+        let schema = Arc::new(Schema::new(vec![Field::new("created_date", arrow.clone(), true)]));
+        let mut b = ColBuilder::new(&arrow).unwrap();
+        b.append_value(Value::Date(2024, 5, 1, 10, 0, 0, 0)).unwrap();
+        let batch = RecordBatch::try_new(schema, vec![b.finish()])
+            .expect("v0.3.4 failed here: naive array vs UTC schema");
+        assert_eq!(batch.column(0).data_type(), &arrow);
+        assert_eq!(batch.num_rows(), 1);
+    }
+
+    /// Regression test for the 0.3.4 follow-up bug: the finished array's *type*
+    /// must carry the timezone from the resolved schema, not merely the correct
+    /// micros. MySQL datetimes resolve to `Timestamp(µs, Some("UTC"))`; a naive
+    /// array (tz `None`) built here failed the downstream batch-vs-schema check
+    /// with "expected Timestamp(.., Some(\"UTC\")) but found Timestamp(.., None)".
+    /// The earlier 0.3.4 tests asserted the mapping and the value, but not this
+    /// — the exact gap that let the regression ship.
+    #[test]
+    fn ts_builder_output_type_carries_timezone() {
+        // UTC-aware (the MySQL datetime default): finished array is tz-aware.
+        let mut utc =
+            ColBuilder::new(&DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()))).unwrap();
+        utc.append_value(Value::Date(2024, 5, 1, 10, 0, 0, 0)).unwrap();
+        assert_eq!(
+            utc.finish().data_type(),
+            &DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
+            "MySQL datetime array must carry the UTC timezone to match the resolved schema"
+        );
+        // Naive (the per-column `type_overrides={col: \"DATETIME\"}` opt-out)
+        // must still produce a naive array — the reverse mismatch.
+        let mut naive =
+            ColBuilder::new(&DataType::Timestamp(TimeUnit::Microsecond, None)).unwrap();
+        naive.append_value(Value::Date(2024, 5, 1, 10, 0, 0, 0)).unwrap();
+        assert_eq!(
+            naive.finish().data_type(),
+            &DataType::Timestamp(TimeUnit::Microsecond, None),
+            "a naive-mapped datetime column must stay naive"
+        );
+    }
+
     #[test]
     fn decimal_decodes_exact_text_value() {
         let mut b = ColBuilder::new(&DataType::Decimal128(10, 4)).unwrap();

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/lib.rs

@@ -28,7 +28,7 @@ pub mod types;
 pub use config::{
     BigQueryConfig, BigQueryDestConfig, BigQueryWriteMethod, ClickHouseConfig, Compression,
     DestinationConfig, MySqlConfig, ParquetCompression, PostgresConfig, S3ArchiveConfig,
-    SourceConfig, SyncMode, TransferConfig, TransferResult,
+    SourceConfig, SyncMode, TransferConfig, TransferResult, WatermarkSeed,
 };
 pub use error::{EtlError, Result};
 pub use sync::{run_transfer, Progress, ProgressCb};

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/sink/bigquery.rs

@@ -389,8 +389,15 @@ impl BigQuerySink {
     /// naturally idempotent: re-running the same merge (e.g. after a crash,
     /// before the watermark advances) re-applies the same key-matched rows
     /// rather than duplicating them.
-    pub async fn merge_into(&self, dest: &str, staging: &str, key: &[String], columns: &[ColumnType]) -> Result<()> {
-        let query = build_merge_sql(&self.project_id, &self.dataset_id, dest, staging, key, columns)?;
+    pub async fn merge_into(
+        &self,
+        dest: &str,
+        staging: &str,
+        key: &[String],
+        columns: &[ColumnType],
+        prune_partition: Option<&str>,
+    ) -> Result<()> {
+        let query = build_merge_sql(&self.project_id, &self.dataset_id, dest, staging, key, columns, prune_partition)?;
         let job = Job {
             job_reference: JobReference {
                 project_id: self.project_id.clone(),
@@ -466,12 +473,12 @@ impl BigQuerySink {
         Ok(())
     }
 
-    /// Read the last persisted watermark for this `(source, dest_table)` pair.
+    /// Read the last persisted watermark for this `(state_key, dest_table)` pair.
     pub async fn read_last_watermark(&self, cfg: &TransferConfig) -> Result<Option<String>> {
         if !self.table_exists(STATE_TABLE).await? {
             return Ok(None);
         }
-        let source_id = cfg.source_table.clone().or_else(|| cfg.source_query.clone()).unwrap_or_default();
+        let source_id = cfg.effective_state_key();
         let query = format!(
             "SELECT last_watermark FROM `{}`.`{}`.`{STATE_TABLE}` \
              WHERE source_table = '{}' AND dest_table = '{}' \
@@ -499,7 +506,7 @@ impl BigQuerySink {
     /// DML `INSERT` run as a query job (BigQuery executes DML through the
     /// same job mechanism as `SELECT`).
     pub async fn persist_watermark(&self, cfg: &TransferConfig, watermark: &str, rows: u64) -> Result<()> {
-        let source_id = cfg.source_table.clone().or_else(|| cfg.source_query.clone()).unwrap_or_default();
+        let source_id = cfg.effective_state_key();
         let query = build_persist_watermark_sql(&self.project_id, &self.dataset_id, &source_id, &cfg.dest_table, watermark, rows);
         let job = Job {
             job_reference: JobReference {
@@ -693,6 +700,7 @@ fn build_merge_sql(
     staging: &str,
     key: &[String],
     columns: &[ColumnType],
+    prune_partition: Option<&str>,
 ) -> Result<String> {
     if key.is_empty() {
         // Should already be caught by sync.rs's prepare_target validation
@@ -701,7 +709,21 @@ fn build_merge_sql(
             "build_merge_sql called with an empty key (should have been validated before staging began)",
         ));
     }
-    let on_clause = key.iter().map(|k| format!("T.`{k}` = S.`{k}`")).collect::<Vec<_>>().join(" AND ");
+    let mut on_clause = key.iter().map(|k| format!("T.`{k}` = S.`{k}`")).collect::<Vec<_>>().join(" AND ");
+    // Optional partition pruning: bound the destination scan to the staging
+    // batch's range on an IMMUTABLE partition column, so BigQuery reads only
+    // the touched partitions instead of the whole table. Scalar subqueries over
+    // the (small) staging table give BigQuery the pruning bounds; the caller
+    // guarantees the column is immutable-per-key (see the correctness contract
+    // on `TransferConfig::merge_prune_partition_by` — a mutable column here
+    // silently inserts duplicate keys).
+    if let Some(pcol) = prune_partition {
+        on_clause.push_str(&format!(
+            " AND T.`{pcol}` BETWEEN \
+             (SELECT MIN(`{pcol}`) FROM `{project_id}`.`{dataset_id}`.`{staging}`) \
+             AND (SELECT MAX(`{pcol}`) FROM `{project_id}`.`{dataset_id}`.`{staging}`)"
+        ));
+    }
 
     let all_cols: Vec<&str> = columns.iter().map(|c| c.name.as_str()).collect();
     let update_cols: Vec<&str> = all_cols.iter().copied().filter(|c| !key.iter().any(|k| k == c)).collect();
@@ -940,12 +962,16 @@ mod tests {
             order_by: vec![],
             partition_by: None,
             primary_key: vec![],
+            merge_prune_partition_by: None,
             parallelism: 1,
             batch_rows: 1000,
             batch_bytes: 0,
             max_memory_bytes: 0,
             partition_column: None,
             read_max_rows_per_sec: None,
+            state_key: None,
+            seed_watermark: crate::config::WatermarkSeed::None,
+            advance_watermark: true,
             type_overrides: HashMap::new(),
             rename: HashMap::new(),
             include: vec![],
@@ -1113,10 +1139,12 @@ mod tests {
             col("amount", DataType::Float64, true),
         ];
         let key = vec!["id".to_string()];
-        let sql = build_merge_sql("proj", "ds", "orders", "orders_quickhouse_tmp", &key, &cols).unwrap();
+        let sql = build_merge_sql("proj", "ds", "orders", "orders_quickhouse_tmp", &key, &cols, None).unwrap();
 
         assert!(sql.starts_with("MERGE INTO `proj`.`ds`.`orders` T USING `proj`.`ds`.`orders_quickhouse_tmp` S"));
         assert!(sql.contains("ON T.`id` = S.`id`"));
+        // No prune column -> no partition-bound predicate (full-scan, correct default).
+        assert!(!sql.contains("BETWEEN"), "unexpected prune predicate without an immutable column: {sql}");
         assert!(sql.contains("WHEN MATCHED THEN UPDATE SET `name` = S.`name`, `amount` = S.`amount`"));
         assert!(sql.contains("WHEN NOT MATCHED THEN INSERT (`id`, `name`, `amount`) VALUES (S.`id`, S.`name`, S.`amount`)"));
         // The key column must never appear in the UPDATE SET list.
@@ -1127,15 +1155,39 @@ mod tests {
     fn build_merge_sql_composite_key() {
         let cols = vec![col("a", DataType::Int64, false), col("b", DataType::Int64, false)];
         let key = vec!["a".to_string(), "b".to_string()];
-        let sql = build_merge_sql("p", "d", "t", "s", &key, &cols).unwrap();
+        let sql = build_merge_sql("p", "d", "t", "s", &key, &cols, None).unwrap();
         assert!(sql.contains("ON T.`a` = S.`a` AND T.`b` = S.`b`"));
     }
 
+    #[test]
+    fn build_merge_sql_prunes_on_immutable_partition_column() {
+        let cols = vec![
+            col("id", DataType::Int64, false),
+            col("create_date", DataType::Timestamp(arrow_schema::TimeUnit::Microsecond, Some("UTC".into())), false),
+            col("amount", DataType::Float64, true),
+        ];
+        let key = vec!["id".to_string()];
+        let sql = build_merge_sql("p", "d", "orders", "orders_tmp", &key, &cols, Some("create_date")).unwrap();
+        // The join still matches on the key, AND the destination is bounded to
+        // the staging batch's create_date range so BigQuery prunes partitions.
+        assert!(sql.contains("ON T.`id` = S.`id`"), "{sql}");
+        assert!(
+            sql.contains(
+                "AND T.`create_date` BETWEEN \
+                 (SELECT MIN(`create_date`) FROM `p`.`d`.`orders_tmp`) \
+                 AND (SELECT MAX(`create_date`) FROM `p`.`d`.`orders_tmp`)"
+            ),
+            "missing partition-prune predicate: {sql}"
+        );
+        // The pruned column is still upserted like any other non-key column.
+        assert!(sql.contains("`create_date` = S.`create_date`"), "{sql}");
+    }
+
     #[test]
     fn build_merge_sql_all_columns_are_key_becomes_insert_only() {
         let cols = vec![col("id", DataType::Int64, false)];
         let key = vec!["id".to_string()];
-        let sql = build_merge_sql("p", "d", "t", "s", &key, &cols).unwrap();
+        let sql = build_merge_sql("p", "d", "t", "s", &key, &cols, None).unwrap();
         assert!(!sql.contains("WHEN MATCHED"), "no columns left to update: {sql}");
         assert!(sql.contains("WHEN NOT MATCHED THEN INSERT (`id`) VALUES (S.`id`)"));
     }
@@ -1143,7 +1195,7 @@ mod tests {
     #[test]
     fn build_merge_sql_rejects_empty_key() {
         let cols = vec![col("id", DataType::Int64, false)];
-        let err = build_merge_sql("p", "d", "t", "s", &[], &cols).unwrap_err().to_string();
+        let err = build_merge_sql("p", "d", "t", "s", &[], &cols, None).unwrap_err().to_string();
         assert!(err.contains("empty key"), "{err}");
         assert!(err.contains("quickhouse bug"), "must be framed as internal, not a config error: {err}");
     }

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/sink/clickhouse.rs

@@ -85,17 +85,13 @@ impl ClickHouseSink {
         self.execute(&crate::ddl::create_state_table(&self.cfg.database)).await
     }
 
-    /// Read the last persisted watermark for this `(source, dest_table)` pair.
+    /// Read the last persisted watermark for this `(state_key, dest_table)` pair.
     pub async fn read_last_watermark(&self, cfg: &TransferConfig) -> Result<Option<String>> {
         // The state table may not exist yet on the very first incremental run.
         if !self.table_exists("_quickhouse_state").await? {
             return Ok(None);
         }
-        let source_id = cfg
-            .source_table
-            .clone()
-            .or_else(|| cfg.source_query.clone())
-            .unwrap_or_default();
+        let source_id = cfg.effective_state_key();
         let sql = format!(
             "SELECT last_watermark FROM {}.`_quickhouse_state` FINAL \
              WHERE source_table = '{}' AND dest_table = '{}' \
@@ -109,11 +105,7 @@ impl ClickHouseSink {
 
     /// Persist a new watermark after a successful incremental run.
     pub async fn persist_watermark(&self, cfg: &TransferConfig, watermark: &str, rows: u64) -> Result<()> {
-        let source_id = cfg
-            .source_table
-            .clone()
-            .or_else(|| cfg.source_query.clone())
-            .unwrap_or_default();
+        let source_id = cfg.effective_state_key();
         let sql = format!(
             "INSERT INTO {}.`_quickhouse_state` (source_table, dest_table, last_watermark, rows) \
              VALUES ('{}', '{}', '{}', {})",

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/sink/mod.rs

@@ -165,13 +165,14 @@ impl Sink {
         staging: &str,
         key: &[String],
         columns: &[ColumnType],
+        prune_partition: Option<&str>,
     ) -> Result<()> {
         match self {
             Sink::ClickHouse(_) => Err(EtlError::internal(
                 "merge_into called on a ClickHouse sink — unreachable, since ClickHouse never \
                  reports requires_staging_for_incremental()",
             )),
-            Sink::BigQuery(s) => s.merge_into(dest, staging, key, columns).await,
+            Sink::BigQuery(s) => s.merge_into(dest, staging, key, columns, prune_partition).await,
         }
     }
 }

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/sync.rs

@@ -19,7 +19,7 @@ use tokio::task::JoinSet;
 use crate::archive::{archive_object_key, build_s3_store, S3ArchiveWriter};
 use crate::config::{
     DestinationConfig, ParquetCompression, S3ArchiveConfig, SourceConfig, SyncMode, TransferConfig,
-    TransferResult,
+    TransferResult, WatermarkSeed,
 };
 use crate::decode::CopyDecoder;
 use crate::decode_bigquery::BigQueryBatcher;
@@ -368,6 +368,10 @@ async fn run_transfer_impl(
     let (extra_filter, new_watermark) = if cfg.mode == SyncMode::Incremental {
         let watermark = cfg.watermark.as_ref().unwrap();
         let last = sink.read_last_watermark(&cfg).await?;
+        // First run only (no persisted cursor): apply the configured seed so a
+        // fresh table can skip a doomed whole-table pull. Self-retires once a
+        // real watermark is persisted (`last` is then Some).
+        let last = last.or_else(|| seed_value(&cfg.seed_watermark, snapshot_max.as_deref()));
         tracing::info!(
             "incremental watermark on '{}': last synced={:?}, current source max={:?}",
             watermark,
@@ -492,13 +496,19 @@ async fn run_transfer_impl(
         if cfg.mode == SyncMode::Incremental {
             if sink.requires_staging_for_incremental() {
                 tracing::info!("merging staged incremental rows into '{}'", cfg.dest_table);
-                sink.merge_into(&cfg.dest_table, &staging, &cfg.key, &plan.dest_columns).await?;
+                sink.merge_into(&cfg.dest_table, &staging, &cfg.key, &plan.dest_columns, cfg.merge_prune_partition_by.as_deref()).await?;
                 sink.drop_table(&staging).await?;
             }
             if let Some(w) = &new_watermark {
-                tracing::info!("persisting new watermark: {w}");
-                sink.persist_watermark(&cfg, w, counters.rows_written.load(Ordering::Relaxed))
-                    .await?;
+                if cfg.advance_watermark {
+                    tracing::info!("persisting new watermark: {w}");
+                    sink.persist_watermark(&cfg, w, counters.rows_written.load(Ordering::Relaxed))
+                        .await?;
+                } else {
+                    tracing::info!(
+                        "advance_watermark=false: computed watermark {w} NOT persisted (cursor left unchanged)"
+                    );
+                }
             }
         }
 
@@ -571,6 +581,9 @@ async fn run_transfer_bigquery(
         let snapshot_max = source
             .max_watermark(&client, &project_id, &table_sql, watermark)
             .await?;
+        // First run only: apply the seed (needs snapshot_max, computed above,
+        // for the CurrentMax variant). Self-retires once a cursor is persisted.
+        let last = last.or_else(|| seed_value(&cfg.seed_watermark, snapshot_max.as_deref()));
         tracing::info!(
             "incremental watermark on '{}': last synced={:?}, current source max={:?}",
             watermark,
@@ -679,13 +692,19 @@ async fn run_transfer_bigquery(
         if cfg.mode == SyncMode::Incremental {
             if sink.requires_staging_for_incremental() {
                 tracing::info!("merging staged incremental rows into '{}'", cfg.dest_table);
-                sink.merge_into(&cfg.dest_table, &staging, &cfg.key, &plan.dest_columns).await?;
+                sink.merge_into(&cfg.dest_table, &staging, &cfg.key, &plan.dest_columns, cfg.merge_prune_partition_by.as_deref()).await?;
                 sink.drop_table(&staging).await?;
             }
             if let Some(w) = &new_watermark {
-                tracing::info!("persisting new watermark: {w}");
-                sink.persist_watermark(&cfg, w, counters.rows_written.load(Ordering::Relaxed))
-                    .await?;
+                if cfg.advance_watermark {
+                    tracing::info!("persisting new watermark: {w}");
+                    sink.persist_watermark(&cfg, w, counters.rows_written.load(Ordering::Relaxed))
+                        .await?;
+                } else {
+                    tracing::info!(
+                        "advance_watermark=false: computed watermark {w} NOT persisted (cursor left unchanged)"
+                    );
+                }
             }
         }
 
@@ -1253,6 +1272,19 @@ async fn cleanup_staging(sink: &Sink, staging: &str) {
     }
 }
 
+/// Resolve the first-run seed into the effective `last` watermark. Only ever
+/// consulted when no cursor is persisted yet (`read_last_watermark` returned
+/// `None`), so it self-retires after the first successful run. `CurrentMax`
+/// seeds to the source's current MAX (reading ~nothing — for a destination
+/// already backfilled by a prior pipeline).
+fn seed_value(seed: &WatermarkSeed, snapshot_max: Option<&str>) -> Option<String> {
+    match seed {
+        WatermarkSeed::None => None,
+        WatermarkSeed::Value(v) => Some(v.clone()),
+        WatermarkSeed::CurrentMax => snapshot_max.map(str::to_string),
+    }
+}
+
 fn build_watermark_filter_pg(
     watermark: &str,
     last: Option<&str>,
@@ -1453,6 +1485,24 @@ mod tests {
         assert_ne!(staging_name("orders", "1"), staging_name("orders", "2"));
     }
 
+    #[test]
+    fn seed_value_resolves_first_run_floor() {
+        // None seed -> no floor (whole-table first pull).
+        assert_eq!(seed_value(&WatermarkSeed::None, Some("2026-01-01")), None);
+        // Explicit floor is used verbatim.
+        assert_eq!(
+            seed_value(&WatermarkSeed::Value("2026-01-01".into()), Some("2026-06-01")),
+            Some("2026-01-01".to_string())
+        );
+        // CurrentMax seeds to the source's snapshot max (skip the first pull).
+        assert_eq!(
+            seed_value(&WatermarkSeed::CurrentMax, Some("2026-06-01")),
+            Some("2026-06-01".to_string())
+        );
+        // CurrentMax on an empty source (no max) is a safe no-op.
+        assert_eq!(seed_value(&WatermarkSeed::CurrentMax, None), None);
+    }
+
     #[test]
     fn throttle_reserve_zero_is_immediate() {
         let t = ReadThrottle::new(1000);

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-core/src/transform.rs

@@ -273,12 +273,16 @@ mod tests {
             order_by: vec![],
             partition_by: None,
             primary_key: vec![],
+            merge_prune_partition_by: None,
             parallelism: 1,
             batch_rows: 1,
             batch_bytes: 0,
             max_memory_bytes: 0,
             partition_column: None,
             read_max_rows_per_sec: None,
+            state_key: None,
+            seed_watermark: crate::config::WatermarkSeed::None,
+            advance_watermark: true,
             type_overrides: HashMap::new(),
             rename: HashMap::new(),
             include: vec![],

{quickhouse-0.3.4 → quickhouse-0.4.0}/crates/quickhouse-py/src/lib.rs

@@ -465,15 +465,20 @@ fn parse_parquet_compression(c: &str) -> PyResult<core::ParquetCompression> {
     *,
     source_table=None,
     source_query=None,
+    state_key=None,
     mode="full".to_string(),
     watermark=None,
     lookback_seconds=0,
+    seed_watermark=None,
+    skip_to_max=false,
+    advance_watermark=true,
     key=None,
     create_if_missing=true,
     engine=None,
     order_by=None,
     partition_by=None,
     primary_key=None,
+    merge_prune_partition_by=None,
     parallelism=4,
     batch_rows=100_000,
     batch_bytes=4_194_304,
@@ -494,15 +499,20 @@ fn sync(
     dest_table: String,
     source_table: Option<String>,
     source_query: Option<String>,
+    state_key: Option<String>,
     mode: String,
     watermark: Option<String>,
     lookback_seconds: u64,
+    seed_watermark: Option<String>,
+    skip_to_max: bool,
+    advance_watermark: bool,
     key: Option<Vec<String>>,
     create_if_missing: bool,
     engine: Option<String>,
     order_by: Option<Vec<String>>,
     partition_by: Option<String>,
     primary_key: Option<Vec<String>>,
+    merge_prune_partition_by: Option<String>,
     parallelism: usize,
     batch_rows: usize,
     batch_bytes: usize,
@@ -518,19 +528,35 @@ fn sync(
     init_logging();
     let source_cfg: core::SourceConfig = source.into();
     let dest_cfg = target.into_config()?;
+    // `seed_watermark` (an explicit floor) and `skip_to_max` (seed to the
+    // source's current MAX) are mutually exclusive ways to seed the first run.
+    let seed_watermark = match (seed_watermark, skip_to_max) {
+        (Some(_), true) => {
+            return Err(PyRuntimeError::new_err(
+                "seed_watermark and skip_to_max are mutually exclusive",
+            ))
+        }
+        (Some(v), false) => core::WatermarkSeed::Value(v),
+        (None, true) => core::WatermarkSeed::CurrentMax,
+        (None, false) => core::WatermarkSeed::None,
+    };
     let cfg = core::TransferConfig {
         source_table,
         source_query,
         dest_table,
+        state_key,
         mode: parse_mode(&mode)?,
         watermark,
         lookback_seconds,
+        seed_watermark,
+        advance_watermark,
         key: key.unwrap_or_default(),
         create_if_missing,
         engine,
         order_by: order_by.unwrap_or_default(),
         partition_by,
         primary_key: primary_key.unwrap_or_default(),
+        merge_prune_partition_by,
         parallelism,
         batch_rows,
         batch_bytes,

{quickhouse-0.3.4 → quickhouse-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "quickhouse"
-version = "0.3.4"
+version = "0.4.0"
 description = "Fast PostgreSQL/MySQL/BigQuery -> ClickHouse ETL with a Rust engine (parallel, bounded-memory, Arrow-based)."
 readme = "README.md"
 requires-python = ">=3.9"

{quickhouse-0.3.4 → quickhouse-0.4.0}/python/quickhouse/_quickhouse.pyi

@@ -218,15 +218,20 @@ def sync(
     *,
     source_table: Optional[str] = None,
     source_query: Optional[str] = None,
+    state_key: Optional[str] = None,
     mode: str = "full",
     watermark: Optional[str] = None,
     lookback_seconds: int = 0,
+    seed_watermark: Optional[str] = None,
+    skip_to_max: bool = False,
+    advance_watermark: bool = True,
     key: Optional[Sequence[str]] = None,
     create_if_missing: bool = True,
     engine: Optional[str] = None,
     order_by: Optional[Sequence[str]] = None,
     partition_by: Optional[str] = None,
     primary_key: Optional[Sequence[str]] = None,
+    merge_prune_partition_by: Optional[str] = None,
     parallelism: int = 4,
     batch_rows: int = 100_000,
     batch_bytes: int = 4_194_304,
@@ -265,6 +270,32 @@ def sync(
     a whole day. Default ``0`` disables lookback entirely (byte-identical to
     the plain watermark filter).
 
+    Incremental cursor control (all incremental-mode only):
+
+    - ``state_key`` pins the identity of the persisted cursor in the internal
+      ``_quickhouse_state`` table. By default the cursor is keyed by
+      ``source_table`` (or the ``source_query`` text) + ``dest_table``. Set
+      ``state_key`` to (a) keep the cursor stable when you edit a
+      ``source_query``'s WHERE/SELECT — whose changed text would otherwise
+      derive a new key and silently trigger a fresh full pull — and (b) give
+      two syncs that share a ``dest_table`` but track different ``watermark``
+      columns *distinct* cursors (they otherwise collide on one state row and
+      clobber each other). Default ``None`` reproduces the pre-existing key
+      exactly, so existing state is never orphaned.
+    - ``seed_watermark`` / ``skip_to_max`` seed the cursor on the **first** run
+      only (when no cursor is persisted yet), then self-retire. ``seed_watermark``
+      is an explicit floor — the first run reads only rows past it.
+      ``skip_to_max=True`` seeds to the source's current ``MAX(watermark)``,
+      reading (almost) nothing — for when the destination already holds
+      complete data from a prior/legacy pipeline and a full first pull would be
+      a doomed waste. The two are mutually exclusive. Once a real watermark is
+      persisted both are ignored, so they are safe to leave set.
+    - ``advance_watermark=False`` reads and merges a window WITHOUT persisting
+      (advancing) the cursor — the primitive a bounded backfill needs so it
+      doesn't rewind the regular schedule. The computed watermark is still
+      returned in ``TransferResult.new_watermark`` for observability but is not
+      written to ``_quickhouse_state``.
+
     ``engine``/``order_by``/``partition_by``/``primary_key``/``key`` are
     interpreted per destination: for ClickHouse they drive `MergeTree`-family
     DDL as before; for BigQuery, ``engine`` is ignored, ``partition_by`` must
@@ -282,6 +313,19 @@ def sync(
     incremental run re-applies the same key-matched rows rather than
     duplicating them.
 
+    By default that ``MERGE`` full-scans the destination table every run (it
+    joins on ``key`` only), which on a large partitioned table bills the whole
+    table to upsert a few delta rows. ``merge_prune_partition_by="<col>"``
+    bounds the destination to the staging batch's range on ``<col>`` so
+    BigQuery reads only the touched partitions. **Only safe when ``<col>`` is
+    immutable per ``key``** (its value never changes across updates to a row)
+    and it is the table's partition column — e.g. a ``create_date``/inserted-at
+    column. Do **not** use a ``write_date``/updated-at column: an updated row's
+    new value points at a different partition than the existing row, so pruning
+    would miss it and INSERT A DUPLICATE KEY instead of updating (the classic
+    merge-filter dup bug). quickhouse can't detect mutability — this is a
+    deliberate per-table opt-in. Default ``None`` keeps the safe full scan.
+
     Memory vs. batch sizing:
 
     - ``batch_rows`` / ``batch_bytes`` control how big each individual Arrow