quickhouse 0.2.4__tar.gz → 0.3.1__tar.gz

{quickhouse-0.2.4 → quickhouse-0.3.1}/Cargo.lock

@@ -331,7 +331,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.1",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -1235,9 +1235,9 @@ dependencies = [
 
 [[package]]
 name = "glob"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
 
 [[package]]
 name = "google-cloud-auth"
@@ -1468,9 +1468,9 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "1.10.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
+checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -1841,9 +1841,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "libloading"
@@ -2506,19 +2506,23 @@ dependencies = [
 
 [[package]]
 name = "quickhouse-core"
-version = "0.2.4"
+version = "0.3.1"
 dependencies = [
  "anyhow",
  "arrow",
  "arrow-array",
  "arrow-schema",
  "async-compression",
+ "base64 0.22.1",
  "bytes",
  "chrono",
  "futures",
  "google-cloud-bigquery",
+ "google-cloud-gax",
  "google-cloud-googleapis",
  "mysql_async",
+ "prost",
+ "prost-types",
  "reqwest",
  "rustls",
  "rustls-pemfile",
@@ -2536,7 +2540,7 @@ dependencies = [
 
 [[package]]
 name = "quickhouse-py"
-version = "0.2.4"
+version = "0.3.1"
 dependencies = [
  "pyo3",
  "quickhouse-core",
@@ -2973,14 +2977,14 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.1",
+ "syn 3.0.3",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
  "itoa",
  "memchr",
@@ -3196,9 +3200,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "3.0.1"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edbec4ed188954a10c12c038215f8ce7606b2d5c973cd8dc43e8795065c5f2f"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3283,7 +3287,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.1",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -3297,9 +3301,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.53"
+version = "0.3.54"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
+checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244"
 dependencies = [
  "deranged",
  "num-conv",
@@ -3317,9 +3321,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
 
 [[package]]
 name = "time-macros"
-version = "0.2.31"
+version = "0.2.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
+checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85"
 dependencies = [
  "num-conv",
  "time-core",
@@ -3382,9 +3386,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.53.0"
+version = "1.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
 dependencies = [
  "bytes",
  "libc",
@@ -3470,9 +3474,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
+checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -3481,13 +3485,14 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.18"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
 dependencies = [
  "bytes",
  "futures-core",
  "futures-sink",
+ "libc",
  "pin-project-lite",
  "tokio",
 ]
@@ -4209,18 +4214,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.54"
+version = "0.8.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
+checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.54"
+version = "0.8.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
+checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb"
 dependencies = [
  "proc-macro2",
  "quote",

{quickhouse-0.2.4 → quickhouse-0.3.1}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/quickhouse-core", "crates/quickhouse-py"]
 
 [workspace.package]
-version = "0.2.4"
+version = "0.3.1"
 edition = "2021"
 license = "MIT"
 authors = ["quickhouse contributors"]
@@ -38,7 +38,18 @@ mysql_async = { version = "0.34", default-features = false, features = ["default
 # that entirely, matching the rest of this workspace's pure-rustls stack.
 google-cloud-bigquery = { version = "0.15", default-features = false, features = ["auth", "rustls-tls"] }
 google-cloud-googleapis = { version = "0.16", features = ["bigquery"] }
+# gRPC transport types (Status/Code) for the Storage Write API error handling.
+google-cloud-gax = "0.19"
 time = { version = "0.3", features = ["macros"] }
+# Only used to encode Binary columns as BigQuery's BYTES JSON representation
+# (base64 text) for the insertAll destination path.
+base64 = "0.22"
+# For the opt-in BigQuery Storage Write API path: prost-types provides the
+# runtime protobuf `DescriptorProto`/`FieldDescriptorProto` we build from the
+# Arrow schema; prost is used in tests to decode our hand-encoded rows and
+# verify the wire format. Pinned to the versions the bigquery crate resolves.
+prost = "0.13"
+prost-types = "0.13"
 
 # Faster release builds for the shipped wheel.
 [profile.release]

quickhouse-0.3.1/PKG-INFO

@@ -0,0 +1,227 @@
+Metadata-Version: 2.4
+Name: quickhouse
+Version: 0.3.1
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: tqdm>=4.60 ; extra == 'progress'
+Requires-Dist: pytest>=7 ; extra == 'test'
+Requires-Dist: psycopg[binary]>=3.1 ; extra == 'test'
+Requires-Dist: pymysql>=1.1 ; extra == 'test'
+Requires-Dist: clickhouse-connect>=0.7 ; extra == 'test'
+Requires-Dist: tqdm>=4.60 ; extra == 'test'
+Provides-Extra: progress
+Provides-Extra: test
+License-File: LICENSE
+Summary: Fast PostgreSQL/MySQL/BigQuery -> ClickHouse ETL with a Rust engine (parallel, bounded-memory, Arrow-based).
+Keywords: etl,postgresql,mysql,bigquery,clickhouse,arrow,rust,data-engineering
+Author-email: M Mirza Fahmi <mr.sm1l3yz@gmail.com>
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/mmirzafahmi/quickhouse
+Project-URL: Issues, https://github.com/mmirzafahmi/quickhouse/issues
+Project-URL: Repository, https://github.com/mmirzafahmi/quickhouse
+
+# quickhouse
+
+**Move tables from PostgreSQL, MySQL, or BigQuery into ClickHouse or BigQuery — fast, in one function call.**
+
+quickhouse is a small, typed Python API on top of a native Rust engine. You
+hand it a source, a destination, and a table name; it figures out the schema,
+creates the destination table, streams the rows across in parallel, and keeps
+memory flat the whole way. The heavy lifting never touches Python objects —
+each database's native wire protocol flows straight into Apache Arrow and out
+the other side.
+
+```python
+import quickhouse
+
+src = quickhouse.Postgres("postgresql://user:pw@localhost:5432/shop")
+dst = quickhouse.ClickHouse("http://localhost:8123", database="analytics")
+
+result = quickhouse.sync(src, dst, dest_table="orders",
+                         source_table="orders", key=["id"])
+print(result)   # rows_read, rows_written, bytes_written, duration_secs, new_watermark
+```
+
+## Why quickhouse
+
+- **It's fast.** Rows are decoded straight off the wire into Arrow, in Rust —
+  no per-row Python, no intermediate DataFrame. Tables are split into ranges
+  and read in parallel, and decoding overlaps uploading. On a laptop-class box
+  a 1M-row, 20-column full refresh runs at **hundreds of thousands of rows per
+  second** while peak memory stays flat (under ~180 MB) no matter how much you
+  parallelize. Reproduce it with `python benchmarks/bench_transfer.py`.
+
+- **It's one function call.** `sync()` replaces the cursor loop, manual
+  batching, retry logic, and `CREATE TABLE` you'd otherwise write by hand.
+  Defaults handle table creation, type mapping, parallelism, and batching, and
+  a typed stub gives you autocomplete on every argument.
+
+- **It's safe with real, messy data.** Full refreshes swap in atomically, so a
+  crash never leaves a half-written table. Incremental syncs are idempotent —
+  safe to re-run or retry. Transient network blips retry automatically. And
+  legacy quirks like MySQL zero-dates or out-of-range timestamps are coerced to
+  `NULL` with a warning instead of aborting the run.
+
+- **There's nothing to stand up.** `pip install quickhouse` and you're done —
+  no JVM, no Spark cluster, no separate service. It's an ordinary Python
+  dependency that runs wherever your jobs already run: cron, Airflow, Dagster,
+  a Lambda, or a plain script.
+
+## Install
+
+```bash
+pip install quickhouse
+pip install "quickhouse[progress]"   # adds a ready-made tqdm progress bar
+```
+
+Prebuilt wheels ship for Python 3.9+ on Linux, macOS (Intel + Apple Silicon),
+and Windows (x86_64) — no Rust toolchain needed. Building from source is only
+for development; see [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## Using it
+
+A fuller call, with the options you'll reach for most:
+
+```python
+import quickhouse as qh
+
+src = qh.Postgres("postgresql://user:pw@localhost:5432/shop")
+dst = qh.ClickHouse("http://localhost:8123", database="analytics")
+
+qh.sync(
+    src, dst,
+    dest_table="orders",
+    source_table="orders",        # or source_query="SELECT ..."
+    mode="incremental",           # or "full"
+    watermark="updated_at",       # required for incremental
+    key=["id"],                   # dedup key / ORDER BY
+    parallelism=8,
+    exclude=["internal_notes"],
+    rename={"amount": "amt"},
+    on_progress=lambda p: print(f"{p.rows_written:,} rows @ {p.rows_per_sec:,.0f}/s"),
+)
+```
+
+### Sources and destinations
+
+Pick a source and a destination by constructing the matching object —
+everything else about `sync()` stays the same:
+
+```python
+# sources
+qh.Postgres("postgresql://user:pw@host:5432/db")
+qh.MySQL("mysql://user:pw@host:3306/db", require_tls=True)
+qh.BigQuery("my-gcp-project")                       # source_table="dataset.table"
+
+# destinations
+qh.ClickHouse("http://host:8123", database="analytics")
+qh.BigQuery("my-gcp-project", dataset_id="analytics")
+```
+
+BigQuery authenticates with a service-account key (`credentials_file=...`) or
+Application Default Credentials. As a **destination** it also takes
+`write_method`: the default `"insert_all"` (simple, proven) or the opt-in
+`"storage_write"` (the gRPC Storage Write API — free and higher-throughput).
+
+The DDL knobs (`engine`, `partition_by`, `order_by`, `primary_key`, `key`) are
+interpreted per destination — for ClickHouse they shape the `MergeTree`
+DDL; for BigQuery they map to partitioning and clustering. quickhouse creates
+the table for you (`create_if_missing=True` by default) with a sensible schema
+derived from the source.
+
+### Full vs. incremental
+
+**Full** reloads the whole table into a staging table, then swaps it into place
+atomically — a crash mid-run never leaves the destination partial. For a
+BigQuery destination that swap runs as a query (a billed scan of the staged
+data), not a free copy job — BigQuery's copy jobs can silently skip rows still
+sitting in a table's streaming buffer, so a real query is what keeps this
+correct rather than just fast.
+
+**Incremental** tracks a high-water mark (the `watermark` column) in a small
+state table in the destination and copies only newer rows. Updated rows are
+deduplicated on `key` — via ClickHouse's `ReplacingMergeTree`, or a `MERGE`
+upsert on BigQuery (where `key` is therefore required). Re-running with no new
+data does nothing.
+
+For daily syncs that need to catch late-arriving or edited rows, set
+`lookback_seconds` to re-scan a trailing window (e.g. `3 * 86400` for the last
+three days) — the dedup above keeps that overlap from creating duplicates.
+
+### Watching progress and diagnosing failures
+
+`on_progress` is a plain callback you can point at anything; `qh.progress_bar()`
+wraps [tqdm](https://github.com/tqdm/tqdm) for a ready-made bar. Every `sync()`
+also logs each step to stderr (`RUST_LOG=quickhouse_core=debug` for the actual
+SQL).
+
+When something goes wrong, `sync()` raises a `RuntimeError` written to be
+actionable on its own: it names the table involved, and for a bad config or an
+unmappable column it says exactly what's wrong and how to fix it (e.g.
+`exclude=` the column or cast it in a `source_query`). Underlying database
+errors are surfaced verbatim rather than wrapped in something generic.
+
+### Full parameter list
+
+| Parameter | Meaning |
+| --- | --- |
+| `source_table` / `source_query` | Read a whole table, or a custom `SELECT` (one required) |
+| `dest_table` | Destination table name |
+| `mode` | `"full"` or `"incremental"` |
+| `watermark` | Monotonic column for incremental (e.g. `updated_at`); ignored in full mode |
+| `lookback_seconds` | Re-scan a trailing window of the watermark to catch late/edited rows; `0` disables (default) |
+| `key` | Dedup key (required for BigQuery incremental) |
+| `create_if_missing` | Auto-create the destination table (default `True`) |
+| `engine`, `order_by`, `partition_by`, `primary_key` | DDL knobs, interpreted per destination |
+| `parallelism` | Concurrent read streams |
+| `batch_rows` / `batch_bytes` | Per-batch size knobs (rows, or estimated bytes) |
+| `max_memory_bytes` | Hard ceiling on total in-flight memory; decoding blocks when hit (default 512 MiB, `0` = unbounded) |
+| `type_overrides` | Force a destination column type, e.g. `{"qty": "Decimal(18, 3)"}` |
+| `rename`, `include`, `exclude` | Column renames and allow/deny lists |
+| `on_progress` | Progress callback |
+
+## How types are mapped
+
+quickhouse maps each source type to a sensible destination type automatically:
+integers to integers, floats to floats, text/JSON/UUID to strings, dates and
+timestamps across as-is, and booleans preserved. A few deliberate choices worth
+knowing:
+
+- **Arbitrary-precision decimals** (`numeric`/`DECIMAL`/`NUMERIC`) default to
+  `Float64`, since precision can't be recovered from the type alone. Pin an
+  exact type with `type_overrides` (e.g. `"Decimal(18, 2)"`).
+- **`TIME`** columns transfer as canonical text into a `String` column
+  (ClickHouse has no time-of-day type).
+- **Out-of-range dates** (and MySQL zero-dates like `0000-00-00`) coerce to
+  `NULL` with a warning rather than failing the transfer.
+- **Nullable** source columns stay nullable in the destination.
+
+Arrays and composite (`RECORD`/`STRUCT`) types aren't supported yet.
+
+## Limitations
+
+- **mTLS** (client-certificate auth) isn't supported; server TLS is, including
+  an extra CA file via `ca_cert_file=...` for providers like AWS RDS.
+- **Array / composite types** aren't mapped yet.
+- **BigQuery as a source** reads through a single connection — `parallelism`
+  becomes a server-side hint rather than true client-side fan-out (a limitation
+  of the underlying crate's read API).
+- **No CLI yet**, and CDC / custom transforms are future work.
+
+## Contributing
+
+Bug reports, new source/type mappings, and PRs are welcome — see
+[CONTRIBUTING.md](CONTRIBUTING.md) for build steps, tests, and layout.
+
+## License
+
+MIT
+

quickhouse-0.3.1/README.md

@@ -0,0 +1,196 @@
+# quickhouse
+
+**Move tables from PostgreSQL, MySQL, or BigQuery into ClickHouse or BigQuery — fast, in one function call.**
+
+quickhouse is a small, typed Python API on top of a native Rust engine. You
+hand it a source, a destination, and a table name; it figures out the schema,
+creates the destination table, streams the rows across in parallel, and keeps
+memory flat the whole way. The heavy lifting never touches Python objects —
+each database's native wire protocol flows straight into Apache Arrow and out
+the other side.
+
+```python
+import quickhouse
+
+src = quickhouse.Postgres("postgresql://user:pw@localhost:5432/shop")
+dst = quickhouse.ClickHouse("http://localhost:8123", database="analytics")
+
+result = quickhouse.sync(src, dst, dest_table="orders",
+                         source_table="orders", key=["id"])
+print(result)   # rows_read, rows_written, bytes_written, duration_secs, new_watermark
+```
+
+## Why quickhouse
+
+- **It's fast.** Rows are decoded straight off the wire into Arrow, in Rust —
+  no per-row Python, no intermediate DataFrame. Tables are split into ranges
+  and read in parallel, and decoding overlaps uploading. On a laptop-class box
+  a 1M-row, 20-column full refresh runs at **hundreds of thousands of rows per
+  second** while peak memory stays flat (under ~180 MB) no matter how much you
+  parallelize. Reproduce it with `python benchmarks/bench_transfer.py`.
+
+- **It's one function call.** `sync()` replaces the cursor loop, manual
+  batching, retry logic, and `CREATE TABLE` you'd otherwise write by hand.
+  Defaults handle table creation, type mapping, parallelism, and batching, and
+  a typed stub gives you autocomplete on every argument.
+
+- **It's safe with real, messy data.** Full refreshes swap in atomically, so a
+  crash never leaves a half-written table. Incremental syncs are idempotent —
+  safe to re-run or retry. Transient network blips retry automatically. And
+  legacy quirks like MySQL zero-dates or out-of-range timestamps are coerced to
+  `NULL` with a warning instead of aborting the run.
+
+- **There's nothing to stand up.** `pip install quickhouse` and you're done —
+  no JVM, no Spark cluster, no separate service. It's an ordinary Python
+  dependency that runs wherever your jobs already run: cron, Airflow, Dagster,
+  a Lambda, or a plain script.
+
+## Install
+
+```bash
+pip install quickhouse
+pip install "quickhouse[progress]"   # adds a ready-made tqdm progress bar
+```
+
+Prebuilt wheels ship for Python 3.9+ on Linux, macOS (Intel + Apple Silicon),
+and Windows (x86_64) — no Rust toolchain needed. Building from source is only
+for development; see [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## Using it
+
+A fuller call, with the options you'll reach for most:
+
+```python
+import quickhouse as qh
+
+src = qh.Postgres("postgresql://user:pw@localhost:5432/shop")
+dst = qh.ClickHouse("http://localhost:8123", database="analytics")
+
+qh.sync(
+    src, dst,
+    dest_table="orders",
+    source_table="orders",        # or source_query="SELECT ..."
+    mode="incremental",           # or "full"
+    watermark="updated_at",       # required for incremental
+    key=["id"],                   # dedup key / ORDER BY
+    parallelism=8,
+    exclude=["internal_notes"],
+    rename={"amount": "amt"},
+    on_progress=lambda p: print(f"{p.rows_written:,} rows @ {p.rows_per_sec:,.0f}/s"),
+)
+```
+
+### Sources and destinations
+
+Pick a source and a destination by constructing the matching object —
+everything else about `sync()` stays the same:
+
+```python
+# sources
+qh.Postgres("postgresql://user:pw@host:5432/db")
+qh.MySQL("mysql://user:pw@host:3306/db", require_tls=True)
+qh.BigQuery("my-gcp-project")                       # source_table="dataset.table"
+
+# destinations
+qh.ClickHouse("http://host:8123", database="analytics")
+qh.BigQuery("my-gcp-project", dataset_id="analytics")
+```
+
+BigQuery authenticates with a service-account key (`credentials_file=...`) or
+Application Default Credentials. As a **destination** it also takes
+`write_method`: the default `"insert_all"` (simple, proven) or the opt-in
+`"storage_write"` (the gRPC Storage Write API — free and higher-throughput).
+
+The DDL knobs (`engine`, `partition_by`, `order_by`, `primary_key`, `key`) are
+interpreted per destination — for ClickHouse they shape the `MergeTree`
+DDL; for BigQuery they map to partitioning and clustering. quickhouse creates
+the table for you (`create_if_missing=True` by default) with a sensible schema
+derived from the source.
+
+### Full vs. incremental
+
+**Full** reloads the whole table into a staging table, then swaps it into place
+atomically — a crash mid-run never leaves the destination partial. For a
+BigQuery destination that swap runs as a query (a billed scan of the staged
+data), not a free copy job — BigQuery's copy jobs can silently skip rows still
+sitting in a table's streaming buffer, so a real query is what keeps this
+correct rather than just fast.
+
+**Incremental** tracks a high-water mark (the `watermark` column) in a small
+state table in the destination and copies only newer rows. Updated rows are
+deduplicated on `key` — via ClickHouse's `ReplacingMergeTree`, or a `MERGE`
+upsert on BigQuery (where `key` is therefore required). Re-running with no new
+data does nothing.
+
+For daily syncs that need to catch late-arriving or edited rows, set
+`lookback_seconds` to re-scan a trailing window (e.g. `3 * 86400` for the last
+three days) — the dedup above keeps that overlap from creating duplicates.
+
+### Watching progress and diagnosing failures
+
+`on_progress` is a plain callback you can point at anything; `qh.progress_bar()`
+wraps [tqdm](https://github.com/tqdm/tqdm) for a ready-made bar. Every `sync()`
+also logs each step to stderr (`RUST_LOG=quickhouse_core=debug` for the actual
+SQL).
+
+When something goes wrong, `sync()` raises a `RuntimeError` written to be
+actionable on its own: it names the table involved, and for a bad config or an
+unmappable column it says exactly what's wrong and how to fix it (e.g.
+`exclude=` the column or cast it in a `source_query`). Underlying database
+errors are surfaced verbatim rather than wrapped in something generic.
+
+### Full parameter list
+
+| Parameter | Meaning |
+| --- | --- |
+| `source_table` / `source_query` | Read a whole table, or a custom `SELECT` (one required) |
+| `dest_table` | Destination table name |
+| `mode` | `"full"` or `"incremental"` |
+| `watermark` | Monotonic column for incremental (e.g. `updated_at`); ignored in full mode |
+| `lookback_seconds` | Re-scan a trailing window of the watermark to catch late/edited rows; `0` disables (default) |
+| `key` | Dedup key (required for BigQuery incremental) |
+| `create_if_missing` | Auto-create the destination table (default `True`) |
+| `engine`, `order_by`, `partition_by`, `primary_key` | DDL knobs, interpreted per destination |
+| `parallelism` | Concurrent read streams |
+| `batch_rows` / `batch_bytes` | Per-batch size knobs (rows, or estimated bytes) |
+| `max_memory_bytes` | Hard ceiling on total in-flight memory; decoding blocks when hit (default 512 MiB, `0` = unbounded) |
+| `type_overrides` | Force a destination column type, e.g. `{"qty": "Decimal(18, 3)"}` |
+| `rename`, `include`, `exclude` | Column renames and allow/deny lists |
+| `on_progress` | Progress callback |
+
+## How types are mapped
+
+quickhouse maps each source type to a sensible destination type automatically:
+integers to integers, floats to floats, text/JSON/UUID to strings, dates and
+timestamps across as-is, and booleans preserved. A few deliberate choices worth
+knowing:
+
+- **Arbitrary-precision decimals** (`numeric`/`DECIMAL`/`NUMERIC`) default to
+  `Float64`, since precision can't be recovered from the type alone. Pin an
+  exact type with `type_overrides` (e.g. `"Decimal(18, 2)"`).
+- **`TIME`** columns transfer as canonical text into a `String` column
+  (ClickHouse has no time-of-day type).
+- **Out-of-range dates** (and MySQL zero-dates like `0000-00-00`) coerce to
+  `NULL` with a warning rather than failing the transfer.
+- **Nullable** source columns stay nullable in the destination.
+
+Arrays and composite (`RECORD`/`STRUCT`) types aren't supported yet.
+
+## Limitations
+
+- **mTLS** (client-certificate auth) isn't supported; server TLS is, including
+  an extra CA file via `ca_cert_file=...` for providers like AWS RDS.
+- **Array / composite types** aren't mapped yet.
+- **BigQuery as a source** reads through a single connection — `parallelism`
+  becomes a server-side hint rather than true client-side fan-out (a limitation
+  of the underlying crate's read API).
+- **No CLI yet**, and CDC / custom transforms are future work.
+
+## Contributing
+
+Bug reports, new source/type mappings, and PRs are welcome — see
+[CONTRIBUTING.md](CONTRIBUTING.md) for build steps, tests, and layout.
+
+## License
+
+MIT

{quickhouse-0.2.4 → quickhouse-0.3.1}/crates/quickhouse-core/Cargo.toml

@@ -31,7 +31,11 @@ rustls-pemfile.workspace = true
 mysql_async.workspace = true
 google-cloud-bigquery.workspace = true
 google-cloud-googleapis.workspace = true
+google-cloud-gax.workspace = true
 time.workspace = true
+base64.workspace = true
+prost.workspace = true
+prost-types.workspace = true
 
 [dev-dependencies]
 anyhow.workspace = true