quickhouse 0.2.3__tar.gz → 0.2.4__tar.gz

{quickhouse-0.2.3 → quickhouse-0.2.4}/Cargo.lock

@@ -65,9 +65,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.103"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
 
 [[package]]
 name = "arrayvec"
@@ -325,13 +325,13 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.89"
+version = "0.1.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.119",
+ "syn 3.0.1",
 ]
 
 [[package]]
@@ -357,9 +357,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.17.1"
+version = "1.17.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
+checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1"
 dependencies = [
  "aws-lc-sys",
  "zeroize",
@@ -367,9 +367,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.42.0"
+version = "0.43.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
+checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c"
 dependencies = [
  "cc",
  "cmake",
@@ -555,9 +555,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "cc"
-version = "1.2.67"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
+checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -965,9 +965,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
 
 [[package]]
 name = "fastrand"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
 
 [[package]]
 name = "find-msvc-tools"
@@ -1098,9 +1098,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
 
 [[package]]
 name = "futures"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
+checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1113,9 +1113,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
+checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -1123,15 +1123,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
+checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -1140,15 +1140,15 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
+checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
+checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1157,21 +1157,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"
+checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
 
 [[package]]
 name = "futures-task"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
 
 [[package]]
 name = "futures-util"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1500,7 +1500,7 @@ dependencies = [
  "tokio",
  "tokio-rustls",
  "tower-service",
- "webpki-roots 1.0.8",
+ "webpki-roots 1.0.9",
 ]
 
 [[package]]
@@ -2288,9 +2288,9 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
 [[package]]
 name = "portable-atomic"
-version = "1.13.1"
+version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
 
 [[package]]
 name = "postgres-protocol"
@@ -2382,9 +2382,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.106"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
 dependencies = [
  "unicode-ident",
 ]
@@ -2506,7 +2506,7 @@ dependencies = [
 
 [[package]]
 name = "quickhouse-core"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "anyhow",
  "arrow",
@@ -2536,7 +2536,7 @@ dependencies = [
 
 [[package]]
 name = "quickhouse-py"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "pyo3",
  "quickhouse-core",
@@ -2557,7 +2557,7 @@ dependencies = [
  "rustc-hash",
  "rustls",
  "socket2 0.6.5",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
  "tokio",
  "tracing",
  "web-time",
@@ -2579,7 +2579,7 @@ dependencies = [
  "rustls",
  "rustls-pki-types",
  "slab",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
  "tinyvec",
  "tracing",
  "web-time",
@@ -2601,9 +2601,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.46"
+version = "1.0.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
 dependencies = [
  "proc-macro2",
 ]
@@ -2770,7 +2770,7 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams",
  "web-sys",
- "webpki-roots 1.0.8",
+ "webpki-roots 1.0.9",
 ]
 
 [[package]]
@@ -2948,9 +2948,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
 
 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
  "serde_core",
  "serde_derive",
@@ -2958,22 +2958,22 @@ dependencies = [
 
 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.119",
+ "syn 3.0.1",
 ]
 
 [[package]]
@@ -3075,7 +3075,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
 dependencies = [
  "num-bigint",
  "num-traits",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
  "time",
 ]
 
@@ -3194,6 +3194,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "3.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5edbec4ed188954a10c12c038215f8ce7606b2d5c973cd8dc43e8795065c5f2f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "sync_wrapper"
 version = "1.0.2"
@@ -3246,11 +3257,11 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
+checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
 dependencies = [
- "thiserror-impl 2.0.18",
+ "thiserror-impl 2.0.19",
 ]
 
 [[package]]
@@ -3266,13 +3277,13 @@ dependencies = [
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
+checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.119",
+ "syn 3.0.1",
 ]
 
 [[package]]
@@ -3934,14 +3945,14 @@ version = "0.26.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
 dependencies = [
- "webpki-roots 1.0.8",
+ "webpki-roots 1.0.9",
 ]
 
 [[package]]
 name = "webpki-roots"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
+checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
 dependencies = [
  "rustls-pki-types",
 ]

{quickhouse-0.2.3 → quickhouse-0.2.4}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/quickhouse-core", "crates/quickhouse-py"]
 
 [workspace.package]
-version = "0.2.3"
+version = "0.2.4"
 edition = "2021"
 license = "MIT"
 authors = ["quickhouse contributors"]

quickhouse-0.2.4/PKG-INFO

@@ -0,0 +1,289 @@
+Metadata-Version: 2.4
+Name: quickhouse
+Version: 0.2.4
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: tqdm>=4.60 ; extra == 'progress'
+Requires-Dist: pytest>=7 ; extra == 'test'
+Requires-Dist: psycopg[binary]>=3.1 ; extra == 'test'
+Requires-Dist: pymysql>=1.1 ; extra == 'test'
+Requires-Dist: clickhouse-connect>=0.7 ; extra == 'test'
+Requires-Dist: tqdm>=4.60 ; extra == 'test'
+Provides-Extra: progress
+Provides-Extra: test
+License-File: LICENSE
+Summary: Fast PostgreSQL/MySQL/BigQuery -> ClickHouse ETL with a Rust engine (parallel, bounded-memory, Arrow-based).
+Keywords: etl,postgresql,mysql,bigquery,clickhouse,arrow,rust,data-engineering
+Author-email: M Mirza Fahmi <mr.sm1l3yz@gmail.com>
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/mmirzafahmi/quickhouse
+Project-URL: Issues, https://github.com/mmirzafahmi/quickhouse/issues
+Project-URL: Repository, https://github.com/mmirzafahmi/quickhouse
+
+# quickhouse
+
+Fast **PostgreSQL / MySQL / BigQuery → ClickHouse** ETL: a native **Rust engine**
+driven from a small, typed **Python** API.
+
+The hot path never materializes Python objects — each source's native wire
+protocol flows straight into Apache Arrow and out to ClickHouse's
+`FORMAT ArrowStream`, with parallel range-partitioned reads, bounded-memory
+streaming, automatic table creation, and both full-refresh and incremental
+(watermark) sync modes.
+
+```python
+import quickhouse
+
+src = quickhouse.Postgres("postgresql://user:pw@localhost:5432/odoo")
+dst = quickhouse.ClickHouse("http://localhost:8123", database="analytics")
+
+result = quickhouse.sync(src, dst, dest_table="account_move_line",
+                         source_table="account_move_line", key=["id"])
+print(result)   # rows_read, rows_written, bytes_written, duration_secs, new_watermark
+```
+
+## Key features
+
+- **Three sources, one API.** PostgreSQL, MySQL, and Google BigQuery — swap the
+  source object, and everything else about `sync()` is identical.
+- **Fast by construction.** Source rows are decoded straight into Arrow in Rust
+  (no per-row Python), tables are split into key ranges read in parallel, and
+  decoding overlaps uploading — each finished batch's insert is spawned as a
+  background task while the next batch is still being decoded.
+- **Bounded memory.** A single hard ceiling, `max_memory_bytes`, caps *total*
+  in-flight batch memory across every partition and every upload, measured
+  against each batch's real Arrow allocation. When it's reached, decoding blocks
+  (backpressure) — so peak RSS stays flat regardless of `parallelism`, row
+  width, or partition skew.
+- **Streaming compressed inserts.** Arrow IPC ingested by ClickHouse's native
+  `ArrowStream`, compressed on the fly with zstd (default), gzip, or none — the
+  body is produced incrementally, never buffered in full.
+- **Atomic full refresh.** Loads into a staging table, then `EXCHANGE TABLES` to
+  swap it into place; a crash mid-run never leaves the destination partial.
+- **Idempotent incremental.** Tracks a watermark in an internal ClickHouse state
+  table, copies only new rows, and dedupes via `ReplacingMergeTree`. Re-running
+  with no new data is a no-op.
+- **Resilient.** Transient insert failures (dropped connections, timeouts, HTTP
+  5xx/429) retry with exponential backoff (up to 4 attempts); deterministic 4xx
+  errors (bad SQL, auth) fail fast. Messy legacy data — zero-dates, out-of-range
+  dates — is coerced rather than crashing the run.
+- **Automatic DDL.** Generates the ClickHouse `CREATE TABLE` from the source
+  schema, with per-column type overrides, renames, and include/exclude lists.
+- **GIL-free.** The whole transfer runs inside `Python::allow_threads`; the GIL
+  is only re-acquired to fire your `on_progress` callback.
+
+## Installation
+
+Prebuilt wheels — no Rust toolchain required:
+
+```bash
+pip install quickhouse
+pip install "quickhouse[progress]"   # + a ready-made tqdm progress bar
+```
+
+Python 3.9+ on Linux, macOS (Intel + Apple Silicon), and Windows (x86_64).
+
+Building from source (for development, or to run an unreleased version) needs the
+Rust toolchain and maturin — see [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## How to use
+
+```python
+import quickhouse as qh
+
+src = qh.Postgres("postgresql://user:pw@localhost:5432/odoo")
+# or: src = qh.MySQL("mysql://user:pw@localhost:3306/odoo")
+# or: src = qh.BigQuery("my-gcp-project")   # source_table="dataset.table"
+dst = qh.ClickHouse("http://localhost:8123", database="analytics")
+
+result = qh.sync(
+    src, dst,
+    dest_table="account_move_line",
+    source_table="account_move_line",
+    mode="incremental",           # or "full"
+    watermark="write_date",       # required for incremental
+    key=["id"],                   # ORDER BY / dedup key
+    create_if_missing=True,       # auto-generate the ClickHouse table
+    parallelism=8,
+    batch_rows=100_000,
+    exclude=["display_name"],
+    rename={"amount": "amt"},
+    type_overrides={"amt": "Decimal(18, 2)"},
+    on_progress=lambda p: print(f"{p.rows_written:,} rows @ {p.rows_per_sec:,.0f}/s"),
+)
+print(result)   # rows_read, rows_written, bytes_written, duration_secs, new_watermark
+```
+
+### Choosing a source
+
+`sync()`'s first argument accepts any of:
+
+```python
+qh.Postgres("postgresql://user:pw@host:5432/db", statement_timeout_secs=0, ca_cert_file=None)
+qh.MySQL("mysql://user:pw@host:3306/db", require_tls=False, ca_cert_file=None)
+qh.BigQuery("my-gcp-project", credentials_file=None)   # None → Application Default Credentials
+```
+
+Read a whole table with `source_table=...`, or a custom `SELECT` with
+`source_query=...` (exactly one is required).
+
+### Sync modes
+
+- **`full`** — loads into a staging table, then `EXCHANGE TABLES` to swap it into
+  place atomically. A crash mid-run never leaves the destination empty/partial.
+- **`incremental`** — reads the last watermark from an internal
+  `_quickhouse_state` table in ClickHouse, copies only rows past it (snapshotting
+  the current max up front for consistency), and dedupes via
+  `ReplacingMergeTree(<watermark>)`. Re-running with no new data is a no-op.
+
+### Progress reporting
+
+`on_progress` is a plain callback, so you can wire up anything — a `print`, a
+logger, a custom UI. For a ready-made bar, `qh.progress_bar()` wraps
+[tqdm](https://github.com/tqdm/tqdm) (`pip install "quickhouse[progress]"`):
+
+```python
+with qh.progress_bar() as on_progress:
+    qh.sync(src, dst, dest_table="t", source_table="t", on_progress=on_progress)
+```
+
+Pass `total=<row count>` (e.g. from a prior `COUNT(*)`) for a percentage/ETA bar
+instead of a running count; other keyword arguments pass straight through to
+`tqdm.tqdm`. The bar closes automatically on exit, including when `sync()` raises.
+
+### Logging
+
+Every `sync()` call prints step-by-step progress to **stderr** via `tracing`:
+connecting, resolving columns/partitions, watermark resolution, DDL/staging
+creation, per-partition read start/completion, the full-refresh swap, watermark
+persistence, and a final summary (rows, duration, rows/sec). This complements
+`on_progress` (which only fires during the row-ingestion loop, never during
+connect/DDL/swap).
+
+Default level is `INFO` for `quickhouse_core` (dependency internals stay quiet).
+Override with the standard `RUST_LOG` environment variable:
+
+```bash
+RUST_LOG=quickhouse_core=debug python my_script.py   # + actual SQL/DDL text
+RUST_LOG=debug python my_script.py                   # everything, incl. deps
+```
+
+### `sync()` parameters
+
+| Parameter | Meaning |
+| --- | --- |
+| `source_table` / `source_query` | Read a whole table, or a custom `SELECT` (one is required) |
+| `dest_table` | Destination table in the ClickHouse database |
+| `mode` | `"full"` or `"incremental"` |
+| `watermark` | Monotonic column (e.g. `write_date`, `id`) — required for incremental; ignored (cleared to `None`) in full mode |
+| `key` | Business key → ClickHouse `ORDER BY` and dedup key |
+| `create_if_missing` | Auto-run generated `CREATE TABLE` when the destination is absent |
+| `engine`, `order_by`, `partition_by`, `primary_key` | DDL knobs (sensible defaults per mode) |
+| `parallelism` | Number of concurrent partition streams |
+| `batch_rows` | Max rows per Arrow batch / insert — a per-batch granularity knob, **not** the memory ceiling |
+| `batch_bytes` | Also cap each batch at this many estimated source bytes (default 4 MiB); `0` disables |
+| `max_memory_bytes` | **The memory ceiling.** Hard cap on total in-flight Arrow memory across all partitions and uploads; decoding blocks when reached. Default 512 MiB; `0` = unbounded |
+| `partition_column` | Integer column to range-split on (defaults to first `key`) |
+| `type_overrides` | Per-column ClickHouse type (e.g. `{"qty": "Decimal(18, 3)"}`) |
+| `rename` | Source → destination column renames |
+| `include` / `exclude` | Column allow/deny lists |
+| `on_progress` | Callback receiving a `Progress` (`rows_written`, `rows_per_sec`, …) |
+
+## Type mapping
+
+Nullable source columns become `Nullable(T)`. Arbitrary-precision numeric types
+(`numeric`/`DECIMAL`/`NUMERIC`/`BIGNUMERIC`, marked \*) map to `Float64` by
+default — precision isn't recoverable from the type alone; override to a
+`Decimal(P, S)` via `type_overrides` and ClickHouse converts on insert. `TIME`
+values are transferred as canonical `[-]HH:MM:SS[.ffffff]` **text** into a
+`String` column (ClickHouse has no time-of-day type; text preserves MySQL's
+negative / >24h durations losslessly). Dates outside ClickHouse's
+`[1900-01-01, 2299-12-31]` window — and MySQL zero-dates like `0000-00-00` — are
+coerced to `NULL` (with a per-partition warning) rather than aborting the run.
+
+### PostgreSQL
+
+| PostgreSQL | Arrow | ClickHouse |
+| --- | --- | --- |
+| `int2/4/8` | `Int16/32/64` | `Int16/32/64` |
+| `float4/8`, `numeric`\* | `Float32/64` | `Float32/64` |
+| `bool` | `Boolean` | `Bool` |
+| `text/varchar/json/jsonb` | `Utf8` | `String` |
+| `uuid` | `Utf8` | `UUID` |
+| `date` | `Date32` | `Date32` |
+| `time` | `Utf8` | `String` |
+| `timestamp[tz]` | `Timestamp(µs)` | `DateTime64(6[, tz])` |
+
+### MySQL
+
+| MySQL | Arrow | ClickHouse |
+| --- | --- | --- |
+| `TINYINT(1)` | `Boolean` | `Bool` |
+| `TINYINT` / `SMALLINT` / `INT` / `BIGINT` (± `UNSIGNED`) | `Int8..64` / `UInt8..64` | matching `Int*`/`UInt*` |
+| `FLOAT` | `Float32` | `Float32` |
+| `DOUBLE`, `DECIMAL`/`NUMERIC`\* | `Float64` | `Float64` |
+| `VARCHAR/TEXT/ENUM/SET/JSON` | `Utf8` | `String` |
+| `BLOB` family, `BIT` | `Binary` | `String` |
+| `DATE` | `Date32` | `Date32` |
+| `TIME` | `Utf8` | `String` |
+| `DATETIME`/`TIMESTAMP` | `Timestamp(µs)` | `DateTime64(6)` |
+
+`TINYINT(1)` follows MySQL's de facto boolean convention (matching most client
+libraries); other `TINYINT` widths map to `Int8`/`UInt8`. Column nullability
+comes directly from MySQL's wire-protocol metadata (`NOT_NULL_FLAG`), so it works
+even for `source_query`.
+
+### BigQuery
+
+| BigQuery | Arrow | ClickHouse |
+| --- | --- | --- |
+| `BOOLEAN`/`BOOL` | `Boolean` | `Bool` |
+| `INTEGER`/`INT64` | `Int64` | `Int64` |
+| `FLOAT`/`FLOAT64` | `Float64` | `Float64` |
+| `NUMERIC`/`BIGNUMERIC`/`DECIMAL`\* | `Float64` | `Float64` |
+| `STRING`, `JSON` | `Utf8` | `String` |
+| `BYTES` | `Binary` | `String` |
+| `DATE` | `Date32` | `Date32` |
+| `TIME` | `Utf8` | `String` |
+| `TIMESTAMP` | `Timestamp(µs, UTC)` | `DateTime64(6, 'UTC')` |
+| `DATETIME` | `Timestamp(µs)` | `DateTime64(6)` |
+
+`RECORD`/`STRUCT` and repeated (`ARRAY`) fields aren't supported in v1 — same
+scalar-only scope as the Postgres/MySQL sources.
+
+## Limitations / roadmap (v1)
+
+- TLS uses rustls, trusting the public CA roots plus, optionally, an extra CA
+  file via `ca_cert_file=...` on `Postgres`/`MySQL` (needed for providers like
+  AWS RDS with a private regional CA). PostgreSQL follows the libpq `sslmode` DSN
+  parameter (`disable` | `prefer` (default) | `require`); MySQL has no such
+  convention, so use `MySQL(..., require_tls=True)`. Client-certificate (mTLS)
+  auth isn't supported yet.
+- Array and composite (`RECORD`/`STRUCT`) types aren't supported; extend
+  `types.rs` + the relevant `decode*.rs`.
+- **BigQuery parallelism**: `parallelism` is passed as a stream-count hint
+  (server-side parallel preparation), but rows are consumed on a single
+  connection rather than fanned out across concurrent local tasks. Only single
+  `SELECT` statements are supported for `source_query` (not multi-statement
+  scripts).
+- No CLI yet — a config-driven CLI over the same engine is planned.
+- Logical-replication CDC and arbitrary transform callbacks are future work.
+
+## Contributing
+
+Bug reports, source/type-mapping additions, and PRs are welcome. Build steps,
+the test workflow, project layout, and the release process live in
+[CONTRIBUTING.md](CONTRIBUTING.md).
+
+## License
+
+MIT
+