adbc-replicate 0.1.0__tar.gz

adbc_replicate-0.1.0/.dockerignore

@@ -0,0 +1,9 @@
+.venv
+.git
+.github
+tests
+charts
+dist
+**/__pycache__
+*.duckdb
+.env

adbc_replicate-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,218 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+  # release.yml runs this whole workflow against the tagged commit before publishing anything.
+  workflow_call:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.13'
+      - run: pip install ruff
+      - run: ruff check src tests
+
+  unit:
+    name: unit (${{ matrix.os }}, py${{ matrix.python }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python: ['3.11', '3.12', '3.13', '3.14']
+        include:
+          - os: macos-latest
+            python: '3.13'
+          - os: windows-latest
+            python: '3.13'
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python }}
+      - run: pip install -e ".[dev]"
+      - run: pytest tests/unit -v
+
+  lowest-dependencies:
+    # The oldest versions pyproject.toml allows must really work.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+      - run: uv venv --python 3.11
+      - run: uv pip install --resolution lowest-direct -e ".[all]" "pytest>=8"
+      - run: uv pip list
+      - run: .venv/bin/pytest tests/unit -v
+
+  integration:
+    name: integration (py${{ matrix.python }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.11', '3.14']
+    services: &services
+      gizmosql:
+        image: gizmodata/gizmosql:v1.41.0-slim
+        env:
+          GIZMOSQL_USERNAME: gizmosql
+          GIZMOSQL_PASSWORD: gizmosql
+          TLS_ENABLED: '0'
+          DATABASE_FILENAME: /tmp/target.duckdb
+        ports:
+          - 31377:31337
+      oracle:
+        image: gvenzl/oracle-free:23-slim-faststart
+        env:
+          ORACLE_PASSWORD: oracle
+          APP_USER: replicate
+          APP_USER_PASSWORD: replicate
+        ports:
+          - 15211:1521
+        options: >-
+          --health-cmd healthcheck.sh
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 30
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: source
+        ports:
+          - 15432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python }}
+      - run: pip install -e ".[dev]"
+      # An unreachable service FAILS these tests (never skips), and pytest exits 5 if nothing ran.
+      - run: pytest tests/integration -m integration -v
+
+  python-3-15:
+    # Python 3.15 is brand new: adbc-driver-manager and pyyaml build from source here, and DuckDB
+    # has no 3.15 wheels yet, so this job covers the Oracle / PostgreSQL → GizmoSQL path.
+    runs-on: ubuntu-latest
+    services: *services
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.15'
+          allow-prereleases: true
+      - run: pip install -e ".[oracle,postgresql,gizmosql]" "pytest>=8"
+      - run: pytest tests/unit --ignore=tests/unit/test_engine_duckdb.py -v
+      - run: pytest tests/integration -m integration -v
+
+  snowflake:
+    # Needs credentials, so it doesn't run for pull requests (fork PRs get no secrets). Everywhere
+    # else, including the release gate, missing secrets are a failure.
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    services:
+      gizmosql:
+        image: gizmodata/gizmosql:v1.41.0-slim
+        env:
+          GIZMOSQL_USERNAME: gizmosql
+          GIZMOSQL_PASSWORD: gizmosql
+          TLS_ENABLED: '0'
+          DATABASE_FILENAME: /tmp/target.duckdb
+        ports:
+          - 31377:31337
+    env:
+      ADBC_REPLICATE_TEST_SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+      ADBC_REPLICATE_TEST_SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}
+      ADBC_REPLICATE_TEST_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.13'
+      - run: pip install -e ".[dev]"
+      - run: pytest tests/integration -m snowflake -v
+
+  docker:
+    # Build the image and run a real PostgreSQL → GizmoSQL replication with it.
+    runs-on: ubuntu-latest
+    services:
+      gizmosql:
+        image: gizmodata/gizmosql:v1.41.0-slim
+        env:
+          GIZMOSQL_USERNAME: gizmosql
+          GIZMOSQL_PASSWORD: gizmosql
+          TLS_ENABLED: '0'
+          DATABASE_FILENAME: /tmp/target.duckdb
+        ports:
+          - 31377:31337
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: source
+        ports:
+          - 15432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    steps:
+      - uses: actions/checkout@v7
+      - run: docker build -t adbc-replicate:ci .
+      - run: docker run --rm adbc-replicate:ci --version
+      - name: Seed PostgreSQL
+        run: |
+          docker exec ${{ job.services.postgres.id }} psql -U postgres -d source -v ON_ERROR_STOP=1 -c "
+            CREATE SCHEMA app;
+            CREATE TABLE app.items AS SELECT i AS id, 'item ' || i AS name, i * 1.5::numeric(10,2) AS price
+            FROM generate_series(1, 1000) i;
+            CREATE VIEW app.cheap AS SELECT id, name FROM app.items WHERE price < 100;"
+      - name: Replicate with the image
+        run: |
+          cat > /tmp/replicate.yaml <<'EOF'
+          version: 1
+          source: {type: postgresql, uri: "postgresql://127.0.0.1:15432/source", username: postgres, password: "${PGPASSWORD}"}
+          target: {type: gizmosql, uri: "gizmosql://127.0.0.1:31377?transport=tcp", username: gizmosql, password: "${GZPASSWORD}"}
+          objects: [{db_schemas: [app]}]
+          EOF
+          for _ in 1 2 3 4 5 6; do
+            docker run --rm --network host -v /tmp/replicate.yaml:/config/replicate.yaml:ro \
+              -e PGPASSWORD=postgres -e GZPASSWORD=gizmosql adbc-replicate:ci test-connection --config /config/replicate.yaml && break
+            sleep 5
+          done
+          docker run --rm --network host -v /tmp/replicate.yaml:/config/replicate.yaml:ro \
+            -e PGPASSWORD=postgres -e GZPASSWORD=gizmosql adbc-replicate:ci run --config /config/replicate.yaml
+
+  helm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: azure/setup-helm@v5
+      - run: helm lint charts/adbc-replicate --set config.version=1
+      - run: |
+          helm template demo charts/adbc-replicate \
+            --set config.version=1 --set existingSecret=demo-env \
+            --set-string 'viewOverrides.HR\.V_X\.sql=SELECT 1' > /tmp/rendered.yaml
+          grep -q 'kind: CronJob' /tmp/rendered.yaml
+          grep -q 'concurrencyPolicy: Forbid' /tmp/rendered.yaml
+          grep -q 'karpenter.sh/do-not-disrupt' /tmp/rendered.yaml

adbc_replicate-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,147 @@
+name: release
+
+on:
+  push:
+    tags: ['v*']
+
+permissions:
+  contents: read
+
+jobs:
+  # The full suite (unit, lowest deps, integration against real GizmoSQL/Oracle/PostgreSQL,
+  # Python 3.15, Snowflake, Docker, Helm) on the tagged commit itself. Nothing publishes without it.
+  tests:
+    uses: ./.github/workflows/ci.yml
+    secrets: inherit
+    permissions:
+      contents: read
+
+  build:
+    needs: tests
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.13'
+      - id: version
+        name: Check the tag matches the package, chart and CHANGELOG
+        run: |
+          VERSION="${GITHUB_REF_NAME#v}"
+          [ -n "$VERSION" ] || { echo "empty version"; exit 1; }
+          PKG=$(python -c "import re,pathlib; print(re.search(r'__version__ = \"(.+)\"', pathlib.Path('src/adbc_replicate/__init__.py').read_text()).group(1))")
+          CHART=$(awk '/^version:/ {print $2}' charts/adbc-replicate/Chart.yaml)
+          APP=$(awk '/^appVersion:/ {gsub(/"/, "", $2); print $2}' charts/adbc-replicate/Chart.yaml)
+          echo "tag=$VERSION package=$PKG chart=$CHART appVersion=$APP"
+          [ "$VERSION" = "$PKG" ] && [ "$VERSION" = "$CHART" ] && [ "$VERSION" = "$APP" ] || { echo "version mismatch"; exit 1; }
+          grep -q "^## \[$VERSION\]" CHANGELOG.md || { echo "CHANGELOG.md has no [$VERSION] section"; exit 1; }
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+      - run: pip install build
+      - run: python -m build
+      - uses: azure/setup-helm@v5
+      - run: helm package charts/adbc-replicate --destination dist-chart
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+      - uses: actions/upload-artifact@v7
+        with:
+          name: chart
+          path: dist-chart/
+
+  pypi:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/adbc-replicate
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist
+      - name: Confirm PyPI serves the new version
+        run: |
+          for _ in $(seq 1 30); do
+            curl -fsS "https://pypi.org/pypi/adbc-replicate/${{ needs.build.outputs.version }}/json" >/dev/null && exit 0
+            sleep 10
+          done
+          echo "PyPI doesn't list adbc-replicate ${{ needs.build.outputs.version }}"; exit 1
+
+  docker:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v7
+      - uses: docker/setup-qemu-action@v4
+      - uses: docker/setup-buildx-action@v4
+      - uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ghcr.io/gizmodata/adbc-replicate:${{ needs.build.outputs.version }}
+            ghcr.io/gizmodata/adbc-replicate:latest
+      - name: Confirm the image is published for both architectures
+        run: |
+          docker buildx imagetools inspect ghcr.io/gizmodata/adbc-replicate:${{ needs.build.outputs.version }} | tee /tmp/inspect.txt
+          grep -q 'linux/amd64' /tmp/inspect.txt && grep -q 'linux/arm64' /tmp/inspect.txt
+
+  helm:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: chart
+          path: dist-chart
+      - uses: azure/setup-helm@v5
+      - run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io --username "${{ github.actor }}" --password-stdin
+      - run: helm push dist-chart/adbc-replicate-${{ needs.build.outputs.version }}.tgz oci://ghcr.io/gizmodata/charts
+      - run: helm show chart oci://ghcr.io/gizmodata/charts/adbc-replicate --version ${{ needs.build.outputs.version }}
+
+  github-release:
+    needs: [build, pypi, docker, helm]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/download-artifact@v8
+        with:
+          path: artifacts
+      - name: Extract release notes from CHANGELOG.md
+        run: |
+          VERSION="${{ needs.build.outputs.version }}"
+          awk -v ver="${VERSION}" '
+            $0 ~ "^## \\[" ver "\\]" { found=1; next }
+            found && /^## \[/ { exit }
+            found { print }
+          ' CHANGELOG.md > release_notes.md
+          test -s release_notes.md
+          cat release_notes.md
+      - uses: softprops/action-gh-release@v3
+        with:
+          body_path: release_notes.md
+          files: |
+            artifacts/dist/*
+            artifacts/chart/*.tgz

adbc_replicate-0.1.0/.gitignore

@@ -0,0 +1,15 @@
+.venv/
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.ruff_cache/
+*.duckdb
+*.duckdb.wal
+.env
+.env.*
+!.env.example
+*.tgz
+.idea/

adbc_replicate-0.1.0/CHANGELOG.md

@@ -0,0 +1,41 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-10-14
+
+### Added
+
+- First release: config-driven ADBC-to-ADBC replication into GizmoSQL (or DuckDB).
+- Sources: Oracle (adbc-driver-oracle), PostgreSQL, Snowflake and DuckDB, each a small
+  plugin for discovery, consistency, change detection, view dialect and type fixes.
+- A versioned YAML/JSON config (JSON Schema via `adbc-replicate schema`) with `${VAR}`
+  environment references, resolved after YAML parsing; unset variables are an error.
+- Selection by case-insensitive glob or `/regex/` on db_schemas, tables and views; excludes;
+  per-rule target db_schema templates; views materialized as tables; a copied table
+  replaces a same-named view, and references to it inside views are re-pointed.
+- Column exclude / allow-list and masks (`null`, salted SHA-256 `hash`, `redact`, source-SQL
+  `expression`) evaluated inside the source database. Exclude/include/mask patterns that
+  match nothing fail the run (`fail_on_unmatched_column_rules`).
+- Consistency modes: `serializable` (one transaction) and `snapshot` (parallel workers at
+  one point in time: Oracle `AS OF SCN`, PostgreSQL exported snapshots, Snowflake time travel).
+  Snapshot mode is refused at plan time on Snowflake shared (imported) databases, which have
+  no time travel.
+- Change detection: Snowflake `LAST_ALTERED` for base tables, otherwise a row count plus a
+  content fingerprint computed in the source; column-rule changes force a re-copy.
+- View translation with sqlglot: Oracle `(+)` joins to ANSI joins, `FROM DUAL`, `ROWNUM`
+  limits, Oracle view column lists, dependency-ordered creation, overrides directory and
+  `on_failure: materialize`.
+- Lossless numerics: unconstrained Oracle `NUMBER` and PostgreSQL `numeric` become
+  `DECIMAL(38,10)` (rounded to 10 fractional digits; integer overflow fails loudly).
+- Streaming with batch coalescing (65,536 rows / 12 MiB), parallel table copies, atomic
+  per-table swaps, `--row-limit` trial runs, `--tables` filters, `plan` and `test-connection`.
+- Sync state, run log and per-object results stored in the target (`_adbc_replicate`).
+- Emoji text output and `--log-format json`; exit codes 0/1/2/3.
+- Container image (`ghcr.io/gizmodata/adbc-replicate`) and a Helm chart running a CronJob.
+- Python 3.11 – 3.15.

adbc_replicate-0.1.0/CLAUDE.md

@@ -0,0 +1,68 @@
+# CLAUDE.md
+
+Project notes for Claude and other contributors.
+
+## What this is
+
+`adbc-replicate`: config-driven ADBC → ADBC replication into GizmoSQL (or DuckDB). Python;
+the data path is Arrow C streams between drivers (`fetch_record_batch` → `adbc_ingest`), so
+Python never touches rows. It generalizes per-customer jobs like
+the Snowflake → GizmoSQL loader behind the Vaden Automotive case study (its lessons are
+what the defaults here encode: 12 MiB batch coalescing, 32 MiB gRPC cap, one catalog snapshot
+instead of per-object queries, `abandoned` runs). Planned later: a GizmoData Cloud wizard
+that writes this config, so keep the config schema, `plan --format json` and the event
+stream stable and UI-friendly.
+
+## Layout
+
+```
+src/adbc_replicate/config.py      pydantic models = the config contract (+ JSON Schema)
+src/adbc_replicate/planner.py     rules + source catalog -> Plan (tables, views, name_map)
+src/adbc_replicate/engine.py      run: consistency, parallel copies, change detection, views
+src/adbc_replicate/sources/*.py   one plugin per database (base.py is the interface)
+src/adbc_replicate/targets/       DuckDB-family targets (GizmoSQL, DuckDB)
+src/adbc_replicate/views.py       sqlglot translation (Oracle rewrites, name re-pointing)
+src/adbc_replicate/state.py       sync state / run log tables in the target
+tests/unit                        no servers; DuckDB -> DuckDB engine tests, README YAML check
+tests/integration                 real GizmoSQL + Oracle + PostgreSQL (docker-compose.yml);
+                                  Snowflake tests are marker `snowflake` (need credentials)
+charts/adbc-replicate             Helm CronJob
+```
+
+## Conventions
+
+- ADBC vocabulary everywhere (catalog, db_schema, table) — never Oracle-specific terms.
+- Keyword arguments at call sites (also for pyarrow/ADBC calls that accept them).
+- Python floor 3.11; CI matrix 3.11–3.14 + a 3.15 job (DuckDB has no 3.15 wheels yet, so the
+  `all` extra skips it there). The lowest-dependencies CI job keeps pyproject floors honest.
+- Tests read only `ADBC_REPLICATE_TEST_*` variables: developer shells export real
+  `GIZMOSQL_*` credentials. Docker addresses are `127.0.0.1`, never `localhost`.
+- Integration tests FAIL (never skip) when a server is unreachable.
+- Every YAML block in README.md is validated by tests/unit/test_readme.py; quick-start
+  output in the README was captured from a real run — re-capture it if output changes.
+- Releases: bump `src/adbc_replicate/__init__.py`, `charts/adbc-replicate/Chart.yaml`
+  (version + appVersion) and CHANGELOG together; tag `vX.Y.Z`. release.yml re-runs ci.yml on
+  the tag and checks all three versions match before publishing to PyPI, GHCR (image + chart)
+  and creating the GitHub Release.
+
+## Gotchas found the hard way
+
+- Oracle `AS OF SCN` fails with ORA-01466 for ~2 s after a table is created/altered/truncated.
+  `TIMESTAMP_TO_SCN` is ~3 s coarse; an exact SCN needs `EXECUTE ON SYS.DBMS_FLASHBACK`
+  (only SYS can grant it in 23ai). The integration fixture waits until flashback reads work.
+- Oracle unconstrained `NUMBER` reads as float64 by default; we CAST to NUMBER(38,10).
+  Oracle computed columns (CASE, functions) lose precision metadata — but
+  `CASE WHEN 1 = 0 THEN col END` keeps it, which is how the `null` mask stays typed.
+- PostgreSQL's driver returns `numeric` as text (arrow.opaque); we round in SQL then strictly
+  cast to decimal128 client-side. `pg_get_viewdef` expands `SELECT *` into explicit columns.
+- sqlglot silently drops Oracle `(+)` when generating; always run `eliminate_join_marks` first.
+  sqlglot renamed `Select.args["from"]` to `"from_"` — use the `_FROM` constant in views.py.
+- DuckDB can't resolve `"x"."t"` when a catalog and a db_schema are both named `x`; target
+  DDL and translated views qualify with the catalog exactly in that case.
+- DuckDB's ADBC driver supports ingest mode `replace` only from 1.5.x (floor: 1.5.6).
+- The GizmoSQL driver's wheels need glibc >= 2.34 (Docker base: bookworm or newer).
+- Snowflake shared (imported) databases have no time travel: `AT(TIMESTAMP => …)` fails with
+  000707 on every table (found on SNOWFLAKE_SAMPLE_DATA), so the plan refuses snapshot mode
+  there (`INFORMATION_SCHEMA.DATABASES.TYPE = 'IMPORTED DATABASE'`).
+- The Snowflake CI user (`ADBC_REPLICATE_CI`) has only PUBLIC: it can read the sample share and
+  use SNOWFLAKE_LEARNING_WH, nothing else. Snowflake tests must stay read-only.

adbc_replicate-0.1.0/Dockerfile

@@ -0,0 +1,27 @@
+# adbc-replicate container image: the CLI plus every source/target driver.
+# Debian bookworm (glibc 2.36): the GizmoSQL driver's wheels need glibc >= 2.34.
+FROM python:3.13-slim-bookworm
+
+LABEL org.opencontainers.image.title="adbc-replicate" \
+      org.opencontainers.image.description="Config-driven ADBC-to-ADBC replication into GizmoSQL" \
+      org.opencontainers.image.source="https://github.com/gizmodata/adbc-replicate" \
+      org.opencontainers.image.vendor="GizmoData LLC" \
+      org.opencontainers.image.licenses="Apache-2.0"
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+WORKDIR /opt/adbc-replicate
+COPY pyproject.toml README.md LICENSE ./
+COPY src ./src
+RUN pip install ".[all]" && adbc-replicate --version
+
+# Run as an unprivileged user; configs are mounted read-only (e.g. /config/replicate.yaml).
+RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin replicate
+USER 10001
+WORKDIR /home/replicate
+
+ENTRYPOINT ["adbc-replicate"]
+CMD ["--help"]