sql2iceberg 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sql2iceberg-0.1.1/CHANGELOG.md +61 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/PKG-INFO +39 -16
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/README.md +38 -15
- sql2iceberg-0.1.1/docs/assets/logo.png +0 -0
- sql2iceberg-0.1.1/docs/assets/make_logo.py +38 -0
- sql2iceberg-0.1.1/docs/release-notes-0.1.0.md +33 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/pyproject.toml +1 -1
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/__init__.py +1 -1
- sql2iceberg-0.1.0/CHANGELOG.md +0 -12
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/.gitignore +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/CONTRIBUTING.md +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/LICENSE +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/environments.md +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/examples.md +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/iceberg-setup.md +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/known-issues.md +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/packages.md +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/roadmap.md +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/snowflake-setup.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/troubleshooting.md +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/app.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/dateadd_timestamp/case.json +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/dateadd_timestamp/expected.csv +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/dateadd_timestamp/orders.csv +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/qualify/case.json +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/qualify/expected.csv +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/qualify/orders.csv +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/app.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/ddl/schema.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/01_basic.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/02_dates.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/03_windows.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/04_semistructured.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/05_strings_regex.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/06_dml_pivot_generator.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/07_curated_orders.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/sample.sql +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/cli.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/compare.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/connectors/__init__.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/connectors/base.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/connectors/others.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/connectors/snowflake.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/convert.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/detect.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/diagnostics.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/generate.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/lineage.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/platforms.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/__init__.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/aws.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/base.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/gcp.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/metadata.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/snowflake.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/pyspark_gen.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/pytest_plugin.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/report.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/tracing.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/validate.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/verify.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_connectors.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_convert.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_detect.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_features.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_publish.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_pyspark_gen.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_spark.py +0 -0
- {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_trace_validate.py +0 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/),
|
|
4
|
+
and the project uses [Semantic Versioning](https://semver.org/). While the version is below 1.0, minor releases may change behaviour.
|
|
5
|
+
|
|
6
|
+
## [0.1.1] - 2026-11-09
|
|
7
|
+
|
|
8
|
+
Documentation-only release: improves the PyPI project page. No code changes.
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- Logo banner and status badges (PyPI version, Python versions, CI, license) at the top of the README.
|
|
12
|
+
- Table of contents in the README.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
- README links to the docs, changelog, contributing guide and license were relative, so they did not resolve on the PyPI
|
|
16
|
+
page. They are now absolute GitHub URLs.
|
|
17
|
+
|
|
18
|
+
## [0.1.0] - 2026-11-09
|
|
19
|
+
|
|
20
|
+
First public release (alpha). Published on PyPI as [`sql2iceberg`](https://pypi.org/project/sql2iceberg/).
|
|
21
|
+
|
|
22
|
+
### Added
|
|
23
|
+
- **Detection and conversion.** Find SQL in `.sql`, `.py` and `.ipynb` files; guess the source platform from driver imports,
|
|
24
|
+
connection strings, SQL syntax and environment variables; convert to Spark SQL with SQLGlot (Snowflake, Postgres, MySQL,
|
|
25
|
+
T-SQL, Oracle dialects) and flag output that cannot run or may behave differently (`converted`, `review`, `unsupported`).
|
|
26
|
+
- **Connectors.** Snowflake (tested live); Postgres, MySQL, SQL Server and Oracle (experimental, mock-tested only). Credentials come
|
|
27
|
+
from environment variables only.
|
|
28
|
+
- **PySpark job generator.** JDBC read of source tables, write to Iceberg, then the curation SQL with tables pointed at the
|
|
29
|
+
Iceberg catalog. Catalog types `hadoop`, `jdbc`, `glue`, `rest`, `hive`; S3 and GCS warehouses; Snowflake key-pair login.
|
|
30
|
+
Generated scripts never contain secrets.
|
|
31
|
+
- **Publishers.** Expose Iceberg tables to Snowflake (external Iceberg tables, reusing or creating the volume and catalog
|
|
32
|
+
integration), AWS (Glue registration, Athena counts) and GCP (BigQuery BigLake). Every publisher prints a plan first and only
|
|
33
|
+
changes an account with `--apply`.
|
|
34
|
+
- **Validation.** `sql2iceberg validate` compares row counts, per-column non-null counts and numeric sums across platforms in
|
|
35
|
+
one batched query per table.
|
|
36
|
+
- **Tracer.** `--trace FILE` writes timed, nested JSONL spans for every command, with secrets redacted.
|
|
37
|
+
- **Support tooling.** `sql2iceberg doctor` checks Java, Spark, packages and certificates; `sql2iceberg issue` opens a GitHub issue
|
|
38
|
+
pre-filled with environment names and versions (never values). Issue templates, troubleshooting guide, environment and
|
|
39
|
+
package lists, and worked examples from the live tests are in `docs/`.
|
|
40
|
+
- **pytest plugin** with Spark fixtures (`sql2iceberg[pytest]`).
|
|
41
|
+
|
|
42
|
+
### Changed
|
|
43
|
+
- Renamed from the working name `sf2iceberg` to `sql2iceberg` (package, CLI, plugin, docs). If you installed the old name,
|
|
44
|
+
`pip uninstall sf2iceberg` first (see troubleshooting).
|
|
45
|
+
- Default Iceberg format version is 2, the version verified on all three target platforms.
|
|
46
|
+
|
|
47
|
+
### Verified
|
|
48
|
+
Live run on 2026-11-09 with synthetic data: Snowflake source to Iceberg on S3 (JDBC, key-pair login) and on GCS, exposed to
|
|
49
|
+
Snowflake, AWS Athena and BigQuery. 38 validation checks across 4 tables and 4 platforms matched (20 customers, 100 orders,
|
|
50
|
+
100 order items, 75 curated orders). CI passes on Python 3.10, 3.12 and 3.13 (Linux).
|
|
51
|
+
|
|
52
|
+
### Known limitations
|
|
53
|
+
- Only Snowflake was exercised as a source. Postgres, MySQL, SQL Server and Oracle are untested.
|
|
54
|
+
- Tested on Iceberg format version 2; v3 support on Snowflake and BigQuery is unverified.
|
|
55
|
+
- BigQuery cannot read an S3 warehouse; the GCP path writes a second copy to GCS.
|
|
56
|
+
- Snowflake object-store tables and BigQuery tables point at one metadata file and need a refresh after new writes.
|
|
57
|
+
- Validation proves counts, non-null counts and sums, not row-by-row equality.
|
|
58
|
+
- Live tests ran on Windows 10 with Python 3.13 only.
|
|
59
|
+
|
|
60
|
+
[0.1.1]: https://github.com/jey1987-cmd/sql2iceberg/releases/tag/v0.1.1
|
|
61
|
+
[0.1.0]: https://github.com/jey1987-cmd/sql2iceberg/releases/tag/v0.1.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sql2iceberg
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Convert SQL from Snowflake and other databases to PySpark, persist it as Apache Iceberg tables, expose them to Snowflake, AWS and GCP, and validate the result.
|
|
5
5
|
Project-URL: Homepage, https://github.com/jey1987-cmd/sql2iceberg
|
|
6
6
|
Project-URL: Documentation, https://github.com/jey1987-cmd/sql2iceberg/tree/main/docs
|
|
@@ -56,13 +56,24 @@ Provides-Extra: spark
|
|
|
56
56
|
Requires-Dist: pyspark>=3.5; extra == 'spark'
|
|
57
57
|
Description-Content-Type: text/markdown
|
|
58
58
|
|
|
59
|
+
<p align="center">
|
|
60
|
+
<img src="https://raw.githubusercontent.com/jey1987-cmd/sql2iceberg/main/docs/assets/logo.png" alt="sql2iceberg: SQL to PySpark to Apache Iceberg" width="520">
|
|
61
|
+
</p>
|
|
62
|
+
|
|
63
|
+
<p align="center">
|
|
64
|
+
<a href="https://pypi.org/project/sql2iceberg/"><img alt="PyPI" src="https://img.shields.io/pypi/v/sql2iceberg"></a>
|
|
65
|
+
<a href="https://pypi.org/project/sql2iceberg/"><img alt="Python versions" src="https://img.shields.io/pypi/pyversions/sql2iceberg"></a>
|
|
66
|
+
<a href="https://github.com/jey1987-cmd/sql2iceberg/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/jey1987-cmd/sql2iceberg/actions/workflows/ci.yml/badge.svg"></a>
|
|
67
|
+
<a href="https://github.com/jey1987-cmd/sql2iceberg/blob/main/LICENSE"><img alt="License: MIT" src="https://img.shields.io/pypi/l/sql2iceberg"></a>
|
|
68
|
+
</p>
|
|
69
|
+
|
|
59
70
|
# sql2iceberg
|
|
60
71
|
|
|
61
72
|
Take the SQL in your code base, convert it to **PySpark**, persist the result as **Apache Iceberg** tables, expose those
|
|
62
73
|
tables to **Snowflake, AWS (Glue/Athena) and GCP (BigQuery)**, and **validate** the migration.
|
|
63
74
|
|
|
64
|
-
**Status: alpha
|
|
65
|
-
accounts; everything else is written but untested and labelled so. See [what was tested](docs/environments.md#what-was-tested)
|
|
75
|
+
**Status: alpha.** One source platform (Snowflake) and three target platforms were exercised against real
|
|
76
|
+
accounts; everything else is written but untested and labelled so. See [what was tested](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md#what-was-tested)
|
|
66
77
|
and [what is not supported yet](#limitations).
|
|
67
78
|
|
|
68
79
|
```
|
|
@@ -74,6 +85,18 @@ your code ──detect──▶ SQL ──convert──▶ Spark SQL ──gener
|
|
|
74
85
|
└─▶ BigQuery (BigLake)
|
|
75
86
|
```
|
|
76
87
|
|
|
88
|
+
## Table of contents
|
|
89
|
+
|
|
90
|
+
- [Core capabilities](#core-capabilities)
|
|
91
|
+
- [Install](#install)
|
|
92
|
+
- [Environment dependencies (summary)](#environment-dependencies-summary)
|
|
93
|
+
- [Quick start](#quick-start)
|
|
94
|
+
- [Python API](#python-api)
|
|
95
|
+
- [Documentation](#documentation)
|
|
96
|
+
- [Getting help and reporting problems](#getting-help-and-reporting-problems)
|
|
97
|
+
- [Limitations](#limitations)
|
|
98
|
+
- [License](#license)
|
|
99
|
+
|
|
77
100
|
## Core capabilities
|
|
78
101
|
|
|
79
102
|
| # | Capability | Command / API | Status |
|
|
@@ -115,10 +138,10 @@ Extras: `spark`, `snowflake`, `postgres`, `mysql`, `mssql`, `oracle`, `aws`, `gc
|
|
|
115
138
|
| Snowflake | account, role that can create external volumes, catalog integrations and Iceberg tables; a warehouse |
|
|
116
139
|
| AWS | an AWS profile or environment credentials with S3, Glue and (to count) Athena permissions |
|
|
117
140
|
| GCP | `gcloud auth login`, a project **with billing enabled**, a BigQuery connection to the bucket |
|
|
118
|
-
| behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](docs/troubleshooting.md) |
|
|
141
|
+
| behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) |
|
|
119
142
|
|
|
120
|
-
Full details, versions we tested and the list of every package and jar: [docs/environments.md](docs/environments.md) and
|
|
121
|
-
[docs/packages.md](docs/packages.md). Run `sql2iceberg doctor` to check your machine.
|
|
143
|
+
Full details, versions we tested and the list of every package and jar: [docs/environments.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md) and
|
|
144
|
+
[docs/packages.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md). Run `sql2iceberg doctor` to check your machine.
|
|
122
145
|
|
|
123
146
|
## Quick start
|
|
124
147
|
|
|
@@ -141,7 +164,7 @@ sql2iceberg validate validate.json --out out
|
|
|
141
164
|
```
|
|
142
165
|
|
|
143
166
|
Add `--trace trace.jsonl` to any command to record a timed trace of the run.
|
|
144
|
-
Worked examples from real runs (configs, plans, results): [docs/examples.md](docs/examples.md).
|
|
167
|
+
Worked examples from real runs (configs, plans, results): [docs/examples.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md).
|
|
145
168
|
|
|
146
169
|
## Python API
|
|
147
170
|
|
|
@@ -158,20 +181,20 @@ script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", w
|
|
|
158
181
|
|
|
159
182
|
## Documentation
|
|
160
183
|
|
|
161
|
-
- [Environments and what was tested](docs/environments.md)
|
|
162
|
-
- [Every package, jar and service used in this build](docs/packages.md)
|
|
163
|
-
- [Examples from the live tests](docs/examples.md)
|
|
164
|
-
- [Troubleshooting frequent problems](docs/troubleshooting.md)
|
|
165
|
-
- [Known conversion differences between Snowflake and Spark](docs/known-issues.md)
|
|
166
|
-
- [Roadmap](docs/roadmap.md)
|
|
184
|
+
- [Environments and what was tested](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md)
|
|
185
|
+
- [Every package, jar and service used in this build](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md)
|
|
186
|
+
- [Examples from the live tests](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md)
|
|
187
|
+
- [Troubleshooting frequent problems](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md)
|
|
188
|
+
- [Known conversion differences between Snowflake and Spark](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/known-issues.md)
|
|
189
|
+
- [Roadmap](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/roadmap.md)
|
|
167
190
|
|
|
168
191
|
## Getting help and reporting problems
|
|
169
192
|
|
|
170
193
|
- Run `sql2iceberg doctor`, then `sql2iceberg issue --title "short summary" --open` to open a GitHub issue pre-filled with
|
|
171
194
|
your environment (names and versions only, never values or credentials).
|
|
172
195
|
- Attach a trace (`--trace trace.jsonl`) after reading it. Secrets are redacted but check anyway.
|
|
173
|
-
- Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](docs/troubleshooting.md) first.
|
|
174
|
-
- Security problems: do not open a public issue; see [CONTRIBUTING.md](CONTRIBUTING.md#security).
|
|
196
|
+
- Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) first.
|
|
197
|
+
- Security problems: do not open a public issue; see [CONTRIBUTING.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/CONTRIBUTING.md#security).
|
|
175
198
|
|
|
176
199
|
## Limitations
|
|
177
200
|
|
|
@@ -183,7 +206,7 @@ script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", w
|
|
|
183
206
|
- Validation compares counts, non-null counts and sums. It does not compare every row.
|
|
184
207
|
- Tested on Windows 10 with Python 3.13. Linux and macOS should work; CI covers Linux but live tests were not run there.
|
|
185
208
|
- Conversion uses [SQLGlot](https://github.com/tobymao/sqlglot); the converter flags constructs that behave differently in
|
|
186
|
-
Spark (see [known issues](docs/known-issues.md)) but a clean result still needs your own verification.
|
|
209
|
+
Spark (see [known issues](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/known-issues.md)) but a clean result still needs your own verification.
|
|
187
210
|
|
|
188
211
|
## License
|
|
189
212
|
|
|
@@ -1,10 +1,21 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/jey1987-cmd/sql2iceberg/main/docs/assets/logo.png" alt="sql2iceberg: SQL to PySpark to Apache Iceberg" width="520">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://pypi.org/project/sql2iceberg/"><img alt="PyPI" src="https://img.shields.io/pypi/v/sql2iceberg"></a>
|
|
7
|
+
<a href="https://pypi.org/project/sql2iceberg/"><img alt="Python versions" src="https://img.shields.io/pypi/pyversions/sql2iceberg"></a>
|
|
8
|
+
<a href="https://github.com/jey1987-cmd/sql2iceberg/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/jey1987-cmd/sql2iceberg/actions/workflows/ci.yml/badge.svg"></a>
|
|
9
|
+
<a href="https://github.com/jey1987-cmd/sql2iceberg/blob/main/LICENSE"><img alt="License: MIT" src="https://img.shields.io/pypi/l/sql2iceberg"></a>
|
|
10
|
+
</p>
|
|
11
|
+
|
|
1
12
|
# sql2iceberg
|
|
2
13
|
|
|
3
14
|
Take the SQL in your code base, convert it to **PySpark**, persist the result as **Apache Iceberg** tables, expose those
|
|
4
15
|
tables to **Snowflake, AWS (Glue/Athena) and GCP (BigQuery)**, and **validate** the migration.
|
|
5
16
|
|
|
6
|
-
**Status: alpha
|
|
7
|
-
accounts; everything else is written but untested and labelled so. See [what was tested](docs/environments.md#what-was-tested)
|
|
17
|
+
**Status: alpha.** One source platform (Snowflake) and three target platforms were exercised against real
|
|
18
|
+
accounts; everything else is written but untested and labelled so. See [what was tested](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md#what-was-tested)
|
|
8
19
|
and [what is not supported yet](#limitations).
|
|
9
20
|
|
|
10
21
|
```
|
|
@@ -16,6 +27,18 @@ your code ──detect──▶ SQL ──convert──▶ Spark SQL ──gener
|
|
|
16
27
|
└─▶ BigQuery (BigLake)
|
|
17
28
|
```
|
|
18
29
|
|
|
30
|
+
## Table of contents
|
|
31
|
+
|
|
32
|
+
- [Core capabilities](#core-capabilities)
|
|
33
|
+
- [Install](#install)
|
|
34
|
+
- [Environment dependencies (summary)](#environment-dependencies-summary)
|
|
35
|
+
- [Quick start](#quick-start)
|
|
36
|
+
- [Python API](#python-api)
|
|
37
|
+
- [Documentation](#documentation)
|
|
38
|
+
- [Getting help and reporting problems](#getting-help-and-reporting-problems)
|
|
39
|
+
- [Limitations](#limitations)
|
|
40
|
+
- [License](#license)
|
|
41
|
+
|
|
19
42
|
## Core capabilities
|
|
20
43
|
|
|
21
44
|
| # | Capability | Command / API | Status |
|
|
@@ -57,10 +80,10 @@ Extras: `spark`, `snowflake`, `postgres`, `mysql`, `mssql`, `oracle`, `aws`, `gc
|
|
|
57
80
|
| Snowflake | account, role that can create external volumes, catalog integrations and Iceberg tables; a warehouse |
|
|
58
81
|
| AWS | an AWS profile or environment credentials with S3, Glue and (to count) Athena permissions |
|
|
59
82
|
| GCP | `gcloud auth login`, a project **with billing enabled**, a BigQuery connection to the bucket |
|
|
60
|
-
| behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](docs/troubleshooting.md) |
|
|
83
|
+
| behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) |
|
|
61
84
|
|
|
62
|
-
Full details, versions we tested and the list of every package and jar: [docs/environments.md](docs/environments.md) and
|
|
63
|
-
[docs/packages.md](docs/packages.md). Run `sql2iceberg doctor` to check your machine.
|
|
85
|
+
Full details, versions we tested and the list of every package and jar: [docs/environments.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md) and
|
|
86
|
+
[docs/packages.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md). Run `sql2iceberg doctor` to check your machine.
|
|
64
87
|
|
|
65
88
|
## Quick start
|
|
66
89
|
|
|
@@ -83,7 +106,7 @@ sql2iceberg validate validate.json --out out
|
|
|
83
106
|
```
|
|
84
107
|
|
|
85
108
|
Add `--trace trace.jsonl` to any command to record a timed trace of the run.
|
|
86
|
-
Worked examples from real runs (configs, plans, results): [docs/examples.md](docs/examples.md).
|
|
109
|
+
Worked examples from real runs (configs, plans, results): [docs/examples.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md).
|
|
87
110
|
|
|
88
111
|
## Python API
|
|
89
112
|
|
|
@@ -100,20 +123,20 @@ script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", w
|
|
|
100
123
|
|
|
101
124
|
## Documentation
|
|
102
125
|
|
|
103
|
-
- [Environments and what was tested](docs/environments.md)
|
|
104
|
-
- [Every package, jar and service used in this build](docs/packages.md)
|
|
105
|
-
- [Examples from the live tests](docs/examples.md)
|
|
106
|
-
- [Troubleshooting frequent problems](docs/troubleshooting.md)
|
|
107
|
-
- [Known conversion differences between Snowflake and Spark](docs/known-issues.md)
|
|
108
|
-
- [Roadmap](docs/roadmap.md)
|
|
126
|
+
- [Environments and what was tested](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md)
|
|
127
|
+
- [Every package, jar and service used in this build](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md)
|
|
128
|
+
- [Examples from the live tests](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md)
|
|
129
|
+
- [Troubleshooting frequent problems](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md)
|
|
130
|
+
- [Known conversion differences between Snowflake and Spark](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/known-issues.md)
|
|
131
|
+
- [Roadmap](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/roadmap.md)
|
|
109
132
|
|
|
110
133
|
## Getting help and reporting problems
|
|
111
134
|
|
|
112
135
|
- Run `sql2iceberg doctor`, then `sql2iceberg issue --title "short summary" --open` to open a GitHub issue pre-filled with
|
|
113
136
|
your environment (names and versions only, never values or credentials).
|
|
114
137
|
- Attach a trace (`--trace trace.jsonl`) after reading it. Secrets are redacted but check anyway.
|
|
115
|
-
- Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](docs/troubleshooting.md) first.
|
|
116
|
-
- Security problems: do not open a public issue; see [CONTRIBUTING.md](CONTRIBUTING.md#security).
|
|
138
|
+
- Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) first.
|
|
139
|
+
- Security problems: do not open a public issue; see [CONTRIBUTING.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/CONTRIBUTING.md#security).
|
|
117
140
|
|
|
118
141
|
## Limitations
|
|
119
142
|
|
|
@@ -125,7 +148,7 @@ script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", w
|
|
|
125
148
|
- Validation compares counts, non-null counts and sums. It does not compare every row.
|
|
126
149
|
- Tested on Windows 10 with Python 3.13. Linux and macOS should work; CI covers Linux but live tests were not run there.
|
|
127
150
|
- Conversion uses [SQLGlot](https://github.com/tobymao/sqlglot); the converter flags constructs that behave differently in
|
|
128
|
-
Spark (see [known issues](docs/known-issues.md)) but a clean result still needs your own verification.
|
|
151
|
+
Spark (see [known issues](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/known-issues.md)) but a clean result still needs your own verification.
|
|
129
152
|
|
|
130
153
|
## License
|
|
131
154
|
|
|
Binary file
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Regenerate docs/assets/logo.png (the banner used in the README and on PyPI).
|
|
2
|
+
|
|
3
|
+
python docs/assets/make_logo.py [path/to/DejaVuSans-Bold.ttf path/to/DejaVuSans.ttf]
|
|
4
|
+
|
|
5
|
+
Needs Pillow. Uses DejaVu Sans (free license); matplotlib ships it in mpl-data/fonts/ttf.
|
|
6
|
+
"""
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from PIL import Image, ImageDraw, ImageFont
|
|
11
|
+
|
|
12
|
+
BOLD = sys.argv[1] if len(sys.argv) > 2 else "DejaVuSans-Bold.ttf"
|
|
13
|
+
REGULAR = sys.argv[2] if len(sys.argv) > 2 else "DejaVuSans.ttf"
|
|
14
|
+
W, H = 2560, 720 # drawn at 2x, saved at half size for smooth edges
|
|
15
|
+
NAVY, WATER, ICE, ICE_SHADE, SUBMERGED = "#0B2545", "#13315C", "#EAF4FF", "#BCD9F5", "#3A86C8"
|
|
16
|
+
ACCENT, TAGLINE = "#5EC8F2", "#9DB8D6"
|
|
17
|
+
|
|
18
|
+
img = Image.new("RGBA", (W, H), (0, 0, 0, 0))
|
|
19
|
+
d = ImageDraw.Draw(img)
|
|
20
|
+
d.rounded_rectangle((0, 0, W - 1, H - 1), radius=90, fill=NAVY)
|
|
21
|
+
d.rectangle((0, 430, 620, H - 1), fill=WATER) # water
|
|
22
|
+
d.rounded_rectangle((0, 430, 620, H - 1), radius=90, fill=WATER, corners=(False, False, False, True))
|
|
23
|
+
d.polygon([(110, 430), (480, 430), (530, 530), (410, 650), (240, 670), (125, 570)], fill=SUBMERGED)
|
|
24
|
+
d.polygon([(150, 430), (230, 262), (290, 305), (345, 150), (435, 330), (485, 430)], fill=ICE)
|
|
25
|
+
d.polygon([(345, 150), (435, 330), (485, 430), (365, 430)], fill=ICE_SHADE)
|
|
26
|
+
d.line((0, 430, 620, 430), fill="#6FA8DC", width=6)
|
|
27
|
+
|
|
28
|
+
big = ImageFont.truetype(BOLD, 215)
|
|
29
|
+
small = ImageFont.truetype(REGULAR, 76)
|
|
30
|
+
x = 700
|
|
31
|
+
for text, color in (("sql", "#FFFFFF"), ("2", ACCENT), ("iceberg", "#FFFFFF")):
|
|
32
|
+
d.text((x, 150), text, font=big, fill=color)
|
|
33
|
+
x += int(d.textlength(text, font=big))
|
|
34
|
+
d.text((706, 440), "SQL \u2192 PySpark \u2192 Apache Iceberg", font=small, fill=TAGLINE)
|
|
35
|
+
|
|
36
|
+
out = Path(__file__).with_name("logo.png")
|
|
37
|
+
img.resize((W // 2, H // 2), Image.LANCZOS).save(out, optimize=True)
|
|
38
|
+
print("wrote", out)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
## sql2iceberg 0.1.0 (alpha)
|
|
2
|
+
|
|
3
|
+
Convert the SQL in your code base to PySpark, persist it as Apache Iceberg, expose the tables to Snowflake, AWS and GCP, and validate the result.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install sql2iceberg # core
|
|
7
|
+
pip install "sql2iceberg[spark,snowflake,aws,gcp]"
|
|
8
|
+
sql2iceberg doctor # check your environment
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
### Highlights
|
|
12
|
+
- **Detect and convert** SQL from `.sql`, `.py` and `.ipynb` files to Spark SQL, with `converted` / `review` / `unsupported` findings.
|
|
13
|
+
- **Generate a PySpark job** that reads the source over JDBC, writes Iceberg to S3 or GCS, and runs the curation SQL.
|
|
14
|
+
- **Publish** to Snowflake (external Iceberg tables), AWS Glue/Athena and BigQuery. Plan first, `--apply` to change anything.
|
|
15
|
+
- **Validate** counts, non-null counts and sums across platforms in one query per table.
|
|
16
|
+
- **Trace** any run with `--trace`, **diagnose** with `doctor`, **report** with `issue`.
|
|
17
|
+
|
|
18
|
+
### Verified live (synthetic data)
|
|
19
|
+
Snowflake source → Iceberg on S3 and GCS → Snowflake, AWS Athena and BigQuery: 38 checks on 4 tables across 4 platforms all matched.
|
|
20
|
+
CI is green on Python 3.10, 3.12 and 3.13.
|
|
21
|
+
|
|
22
|
+
### Not verified yet
|
|
23
|
+
Postgres, MySQL, SQL Server and Oracle connectors (mock-tested only); Iceberg format v3 on the targets; Linux and macOS live runs;
|
|
24
|
+
Spark 3.5 live runs. See [environments](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md).
|
|
25
|
+
|
|
26
|
+
### Docs
|
|
27
|
+
[Environments](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md) ·
|
|
28
|
+
[Packages](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md) ·
|
|
29
|
+
[Examples](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md) ·
|
|
30
|
+
[Troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) ·
|
|
31
|
+
[Full changelog](https://github.com/jey1987-cmd/sql2iceberg/blob/main/CHANGELOG.md)
|
|
32
|
+
|
|
33
|
+
Found a problem? `sql2iceberg issue --open` pre-fills a GitHub issue with your environment.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "sql2iceberg"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.1"
|
|
8
8
|
description = "Convert SQL from Snowflake and other databases to PySpark, persist it as Apache Iceberg tables, expose them to Snowflake, AWS and GCP, and validate the result."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
sql2iceberg-0.1.0/CHANGELOG.md
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
|
-
## 0.1.0 (first release, alpha)
|
|
4
|
-
|
|
5
|
-
- Renamed from the working name `sf2iceberg` to `sql2iceberg`.
|
|
6
|
-
- Detect SQL platform and SQL in `.sql`, `.py`, `.ipynb`; dialect-aware conversion to Spark SQL with safety findings.
|
|
7
|
-
- Connectors: Snowflake (tested live); Postgres, MySQL, SQL Server, Oracle (experimental, mock-tested).
|
|
8
|
-
- PySpark job generator: JDBC ingest to Iceberg, curation SQL, hadoop/jdbc/glue/rest/hive catalogs, S3 and GCS warehouses.
|
|
9
|
-
- Publishers: Snowflake external Iceberg tables, AWS Glue/Athena, BigQuery BigLake. Plan first, `--apply` to change accounts.
|
|
10
|
-
- `validate`: counts, non-null counts and sums across Snowflake, Athena and BigQuery.
|
|
11
|
-
- `--trace` JSONL tracer with secret redaction; `doctor` and `issue` commands.
|
|
12
|
-
- Live-tested on Snowflake, AWS and GCP with Iceberg format-version 2 (see docs/examples.md).
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/06_dml_pivot_generator.sql
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|