odbc2deltalake2 0.1.0a0__tar.gz

odbc2deltalake2-0.1.0a0/.github/workflows/python-test.yml

@@ -0,0 +1,69 @@
+name: Python Test
+
+on:
+  push:
+    branches: ["main"]
+    paths-ignore: ["README.md", "docs", ".github"]
+  pull_request:
+    branches: ["main"]
+    paths-ignore: ["README.md", "docs", ".github"]
+
+jobs:
+  test_types:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install all dependencies
+        run: uv sync --all-extras --all-groups
+
+      - name: ty
+        run: uv run ty check odbc2deltalake2
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+        test_config: ["local", "spark", "azure"]
+        database: ["postgres", "mssql"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: Yarden-zamir/install-mssql-odbc@main
+        with:
+          ODBC_VERSION: 18
+        if: matrix.database == 'mssql'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install project dependencies incl Spark
+        if: matrix.test_config == 'spark'
+        run: uv sync --group test --group dev --group spark --extra ${{matrix.database}}
+      - name: Test beeing able to import
+        if: matrix.test_config == 'spark'
+        run: uv run ./test_spark_import.py
+      - name: Install project dependencies
+        if: matrix.test_config != 'spark'
+        run: uv sync --extra local --extra local_azure --group test --group dev  --extra ${{matrix.database}}
+      - name: Test with pytest
+        run: uv run -m pytest --maxfail=3 --cov=odbc2deltalake2 tests
+        env:
+          ODBCLAKE_TEST_CONFIGURATION: ${{ matrix.test_config }}
+          ODBCLAKE_TEST_SOURCE_SERVER: ${{matrix.database}}

odbc2deltalake2-0.1.0a0/.github/workflows/workflow.yml

@@ -0,0 +1,32 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/odbc2deltalake2
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv sync --all-extras
+      - name: Build package
+        run: uv build
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

odbc2deltalake2-0.1.0a0/.gitignore

@@ -0,0 +1,172 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+tests/_data
+tests/_db
+__azurite*
+__blob*
+debug.sql
+requirements.txt
+prof
+repo.py
+
+# claude / serena tooling state, not project source
+.serena/

odbc2deltalake2-0.1.0a0/.python-version

@@ -0,0 +1 @@
+3.11

odbc2deltalake2-0.1.0a0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Building Material Suisse
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

odbc2deltalake2-0.1.0a0/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.5
+Name: odbc2deltalake2
+Version: 0.1.0a0
+Summary: Faster fork of odbc2deltalake: ODBC/ADBC/Spark to SCD2 Delta tables
+Author-email: Adrian Ehrsam <adrian.ehrsam@bmsuisse.ch>, Dominik Peter <dominik.peter@bmsuisse.ch>
+License-File: LICENSE
+Requires-Python: ~=3.9
+Requires-Dist: azure-identity<2,>=1.15.0
+Requires-Dist: pydantic>=1.10.0
+Requires-Dist: sqlglot>=23.17.0
+Provides-Extra: adbc
+Requires-Dist: adbc-driver-manager>=1.7.0; extra == 'adbc'
+Provides-Extra: local
+Requires-Dist: arrow-odbc; extra == 'local'
+Requires-Dist: arrow-odbc>=5.0.0; extra == 'local'
+Requires-Dist: deltalake; extra == 'local'
+Requires-Dist: deltalake2db[deltalake]>=0.9.3; extra == 'local'
+Requires-Dist: deltalake>=0.17.1; extra == 'local'
+Requires-Dist: duckdb; extra == 'local'
+Requires-Dist: pyarrow>=16.1; extra == 'local'
+Requires-Dist: pyodbc<6,>=5.1.0; extra == 'local'
+Provides-Extra: local-azure
+Requires-Dist: adlfs; extra == 'local-azure'
+Requires-Dist: adlfs<2025,>=2024.2.0; extra == 'local-azure'
+Requires-Dist: arrow-odbc>=5.0.0; extra == 'local-azure'
+Requires-Dist: azure-identity; extra == 'local-azure'
+Requires-Dist: azure-storage-file-datalake; extra == 'local-azure'
+Requires-Dist: deltalake2db[deltalake]>=0.9.3; extra == 'local-azure'
+Requires-Dist: deltalake>=0.17.1; extra == 'local-azure'
+Requires-Dist: pyarrow>=16.1; extra == 'local-azure'
+Requires-Dist: pyodbc<6,>=5.1.0; extra == 'local-azure'
+Provides-Extra: mssql
+Requires-Dist: pyodbc<6,>=5.1.0; extra == 'mssql'
+Provides-Extra: postgres
+Requires-Dist: adbc-driver-postgresql>=1.7.0; extra == 'postgres'

odbc2deltalake2-0.1.0a0/docs/superpowers/specs/2026-09-09-odbc2deltalake2-design.md

@@ -0,0 +1,138 @@
+# odbc2deltalake2 — Design Spec
+
+## Goal
+
+Replace [bmsuisse/odbc2deltalake](https://github.com/bmsuisse/odbc2deltalake) with a faster,
+cleaner package (`odbc2deltalake2`), published to PyPI, drop-in for the production
+Databricks usage in Fabricks.Runtime (`extractors/sql_server/extract_table.ipynb`,
+`SparkReader` + `DatabricksDestination`, onetrade SQL Server extraction).
+
+"Faster" is decided by measurement, not assumption. No Rust/PyO3 rewrite — the
+production path is Spark/JDBC on Databricks; the heavy lifting already runs in the
+JVM, not Python. A prior finding confirmed `SparkReader`'s JDBC read has zero
+partitioning options (`partitionColumn`/`numPartitions`/`lowerBound`/`upperBound`),
+meaning every table reads through a single-threaded JDBC connection today — the
+leading suspect, to be confirmed by Phase 0.
+
+## Non-goals
+
+- Rust/PyO3 core rewrite (delta writes already go through native `delta-rs`; the
+  Spark production path bypasses Python compute entirely).
+- Touching the ODBC/ADBC (non-Databricks) reader path in 1.0 — left as-is,
+  correctness-only, ported and tested but not speed-tuned.
+- Any refactor of `db_to_delta.py` beyond what's needed to unblock Phase 3 speed work.
+
+## Phases
+
+### Phase 0 — Baseline measurement (no code changes)
+
+Run the *published* `odbc2deltalake` against 5 real onetrade tables on the
+Databricks premium workspace: 1 tiny (<100k rows), 2 mid (1-10M), 2 large (>50M,
+one routed through `linked_server_proxy`, one direct). 3 runs each, off-peak,
+using the production `WriteConfig` load modes actually in use (`simple_delta`,
+`force_full`, `append`).
+
+Instrument read / diff-merge / write stage timings separately (Spark UI stage
+timings + `time.perf_counter` around orchestrator stages). Record cluster config,
+DBR version, row counts, output bytes.
+
+**Exit:** results committed as CSV/Delta table + a short "where the time goes"
+writeup. This decides Phase 3's priority order — if the plan below turns out
+wrong once we have numbers, the plan changes, not the numbers.
+
+### Phase 1 — Repo bootstrap
+
+- Copy `odbc2deltalake` source in as the starting point (MIT licensed — keep
+  original LICENSE + copyright notice, credit in README).
+- Rename package to `odbc2deltalake2`. `uv`-managed `pyproject.toml`.
+- Apply [bmsuisse coding-guidelines-python](https://github.com/bmsuisse/skills/tree/main/skills/coding-guidelines-python):
+  full typing, `from __future__ import annotations`, `uv run ty check` clean,
+  dataclasses/`TypedDict` over raw dicts, `Literal`/`Enum` over string constants,
+  no mutable defaults, no module-level mutable state.
+- Set up [prek](https://github.com/bmsuisse/skills/tree/main/skills/prek):
+  ruff check+format, `check_files.py` line-limit guard, `prek install`.
+- Port the existing test suite (`tests/test_01`...`test_12`, `test_dbutils.py`,
+  MSSQL/Postgres test-server fixtures) — **must pass green, unmodified behavior,
+  before anything else.**
+- Publish `0.1.0a0` to PyPI immediately to prove the publish pipeline works.
+
+**Exit:** `pip install odbc2deltalake2` on a cluster reproduces Phase 0 numbers
+within noise (this is the correctness guardrail for every phase after).
+
+### Phase 2 — Architecture audit (time-boxed, 3 days)
+
+Apply the aihero-style audit methodology to `db_to_delta.py` and `write_init.py`:
+tag findings Strong / Worth-exploring / Speculative. Fix **only Strong findings
+that block Phase 3** (e.g. reader/writer seams needed to inject partition
+options or per-stage timing). Everything else goes to a backlog, not into scope.
+
+Ponytail applies throughout: no speculative abstractions, shortest diff that
+unblocks Phase 3.
+
+**Exit:** tests still green, Phase 0 numbers unchanged (refactor caused no
+regression or improvement — it's structural, not a speed change), audit doc
+committed, backlog list captured for later.
+
+### Phase 3 — Speed work
+
+Metrics per table: wall clock (primary), rows/sec, SQL-Server-side CPU/wait
+during the run (safety check on the source), DBU cost (secondary).
+
+Priority order, by confidence (from Phase 0 data, adjust as needed):
+
+1. **Strong** — Partitioned JDBC reads: auto-pick partition column (PK
+   int/identity/rowversion/date), one `MIN/MAX` query for bounds,
+   `numPartitions` = min(executor cores, configurable cap ~32) to avoid
+   overwhelming a shared SQL Server. Fallback to single-partition read when no
+   suitable column exists or bounds indicate skew.
+2. **Strong** — JDBC `fetchsize` tuning (default is tiny; test 10k-100k).
+3. **Strong** — Concurrent multi-table orchestration via `ThreadPool`, bounded
+   by a **global connection budget** shared across tables (not per-table
+   `numPartitions` × table count unbounded — that would multiply source load).
+4. **Worth-exploring** — SCD2 diff/merge optimization (avoid recomputing full
+   hashes, broadcast small sides, `MERGE` predicate pushdown) — only pursued if
+   Phase 0 shows diff/merge as a significant share of wall time.
+5. **Worth-exploring** — Delta write tuning (`optimizeWrite`, target file size,
+   dropping unneeded `count()` actions).
+6. **Speculative** — Lakehouse Federation as an alternate reader (spike only,
+   requires DBR ≥16.1 / Serverless SQL Warehouse).
+
+Each technique: A/B run against the Phase 0 baseline on the same 5 tables,
+plus a correctness check (row count + hash of SCD2 output matches baseline) —
+not just green pytest, since these changes touch load-mode logic directly.
+
+New tests required for every technique added here (partitioning column
+selection, connection budget enforcement, skew fallback), on top of the ported
+Phase 1 suite staying green throughout.
+
+**Exit:** each technique's A/B result recorded; net improvement vs. baseline
+established with real numbers, not estimates.
+
+### Phase 4 — Release
+
+- Publish `1.0.0`. README covers migration from `odbc2deltalake`, tuning knobs,
+  and results from Phase 0/3.
+- Switch Fabricks.Runtime's onetrade extraction to `odbc2deltalake2` behind a
+  flag; one week of shadow runs before full cutover.
+
+**Exit:** production onetrade extraction running on `odbc2deltalake2`, with
+Phase 0 numbers as the documented before/after.
+
+## Risks
+
+- **Production SQL Server load during benchmarking/tuning:** off-peak only,
+  hard partition-count cap, monitor `sys.dm_exec_requests`, coordinate timing
+  with DBA — the `linked_server_proxy` hop doubles the load surface (both
+  servers take the hit).
+- **Partition-column skew:** identity columns with large deleted ranges produce
+  empty/oversized partitions — needs a skew check and single-partition fallback.
+- **Benchmark noise:** shared cluster and shared source mean median-of-3 runs,
+  not single-run or mean.
+- **License:** original repo is MIT — no restriction on copying, keep the
+  notice.
+
+## Explicitly deferred / open
+
+- ODBC/ADBC reader path: untouched in 1.0, ported + tested for correctness only.
+- Correctness oracle beyond pytest for SCD2 output equivalence when diff/merge
+  logic changes in Phase 3 — needs defining before Phase 3.4 starts, not before.

odbc2deltalake2-0.1.0a0/odbc2deltalake2/__init__.py

@@ -0,0 +1,25 @@
+from .write_init import (
+    make_writer,
+    DBDeltaPathConfigs,
+    WriteConfig,
+    DEFAULT_DATA_TYPE_MAP,
+    WriteConfigAndInfos,
+)
+from .reader.reader import DataSourceReader
+from .destination.destination import Destination
+from pathlib import Path
+from typing import Union
+
+
+def write_db_to_delta(
+    source: Union[DataSourceReader, str],
+    table_or_query: tuple[str, str],
+    destination: Union[Destination, Path],
+    write_config: Union[WriteConfig, None] = None,
+):
+    return make_writer(
+        source=source,
+        table_or_query=table_or_query,
+        destination=destination,
+        write_config=write_config,
+    ).execute()

odbc2deltalake2-0.1.0a0/odbc2deltalake2/consistency.py

@@ -0,0 +1,56 @@
+from typing import Any
+from .write_init import WriteConfigAndInfos, DBDeltaPathConfigs
+from .write_utils.restore_pk import create_last_pk_version_view
+import sqlglot as sg
+import sqlglot.expressions as ex
+
+
+class InconsistentPrimaryKeyError(Exception):
+    """Exception raised when primary keys are inconsistent."""
+
+    def __init__(self, message, invalid_data: list[dict[str, Any]]):
+        self.message = message
+        self.invalid_data = invalid_data
+        super().__init__(self.message)
+
+
+def check_latest_pk(infos: WriteConfigAndInfos, raise_if_not_consistent=True):
+    if not infos.delta_col or not infos.pk_cols:
+        raise ValueError("Primary keys and delta column must be defined")
+    postfix = "_" + str(abs(hash(str(infos.destination))))
+    lpk_view = "lastest_pk_" + postfix
+    infos.source.local_register_update_view(
+        infos.destination / "delta_load" / DBDeltaPathConfigs.LATEST_PK_VERSION,
+        lpk_view,
+    )
+    _, view_name, success = create_last_pk_version_view(
+        infos, view_prefix="v_" + postfix
+    )
+    d_cols = list(infos.pk_cols) + [infos.delta_col]
+    col_names = [infos.write_config.get_target_name(p) for p in d_cols]
+    assert success
+    assert view_name is not None
+    query1 = sg.except_(
+        sg.from_(ex.table_(lpk_view, alias="lpk"))
+        .select(*[ex.column(c, "lpk", quoted=True) for c in col_names])
+        .select(ex.convert("added in persisted data"), append=True),
+        sg.from_(ex.table_(view_name, alias="rs"))
+        .select(*[ex.column(c, "rs", quoted=True) for c in col_names])
+        .select(ex.convert("added in persisted data"), append=True),
+    )
+    query2 = sg.except_(
+        sg.from_(ex.table_(view_name, alias="rs"))
+        .select(*[ex.column(c, "rs", quoted=True) for c in col_names])
+        .select(ex.convert("missing in persisted data"), append=True),
+        sg.from_(ex.table_(lpk_view, alias="lpk"))
+        .select(*[ex.column(c, "lpk", quoted=True) for c in col_names])
+        .select(ex.convert("missing in persisted data"), append=True),
+    )
+    result = infos.source.local_execute_sql_to_py(
+        ex.union(query1, query2, distinct=False)
+    )
+    if result:
+        print(result)
+        if raise_if_not_consistent:
+            raise InconsistentPrimaryKeyError("Primary keys are not consistent", result)
+    return result