sql2iceberg 0.1.0__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. sql2iceberg-0.1.1/CHANGELOG.md +61 -0
  2. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/PKG-INFO +39 -16
  3. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/README.md +38 -15
  4. sql2iceberg-0.1.1/docs/assets/logo.png +0 -0
  5. sql2iceberg-0.1.1/docs/assets/make_logo.py +38 -0
  6. sql2iceberg-0.1.1/docs/release-notes-0.1.0.md +33 -0
  7. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/pyproject.toml +1 -1
  8. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/__init__.py +1 -1
  9. sql2iceberg-0.1.0/CHANGELOG.md +0 -12
  10. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/.gitignore +0 -0
  11. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/CONTRIBUTING.md +0 -0
  12. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/LICENSE +0 -0
  13. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/environments.md +0 -0
  14. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/examples.md +0 -0
  15. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/iceberg-setup.md +0 -0
  16. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/known-issues.md +0 -0
  17. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/packages.md +0 -0
  18. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/roadmap.md +0 -0
  19. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/snowflake-setup.sql +0 -0
  20. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/docs/troubleshooting.md +0 -0
  21. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/app.py +0 -0
  22. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/dateadd_timestamp/case.json +0 -0
  23. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/dateadd_timestamp/expected.csv +0 -0
  24. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/dateadd_timestamp/orders.csv +0 -0
  25. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/qualify/case.json +0 -0
  26. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/qualify/expected.csv +0 -0
  27. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/cases/qualify/orders.csv +0 -0
  28. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/app.py +0 -0
  29. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/ddl/schema.sql +0 -0
  30. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/01_basic.sql +0 -0
  31. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/02_dates.sql +0 -0
  32. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/03_windows.sql +0 -0
  33. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/04_semistructured.sql +0 -0
  34. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/05_strings_regex.sql +0 -0
  35. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/06_dml_pivot_generator.sql +0 -0
  36. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/corpus/pipelines/07_curated_orders.sql +0 -0
  37. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/examples/sample.sql +0 -0
  38. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/cli.py +0 -0
  39. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/compare.py +0 -0
  40. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/connectors/__init__.py +0 -0
  41. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/connectors/base.py +0 -0
  42. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/connectors/others.py +0 -0
  43. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/connectors/snowflake.py +0 -0
  44. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/convert.py +0 -0
  45. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/detect.py +0 -0
  46. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/diagnostics.py +0 -0
  47. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/generate.py +0 -0
  48. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/lineage.py +0 -0
  49. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/platforms.py +0 -0
  50. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/__init__.py +0 -0
  51. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/aws.py +0 -0
  52. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/base.py +0 -0
  53. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/gcp.py +0 -0
  54. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/metadata.py +0 -0
  55. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/publish/snowflake.py +0 -0
  56. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/pyspark_gen.py +0 -0
  57. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/pytest_plugin.py +0 -0
  58. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/report.py +0 -0
  59. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/tracing.py +0 -0
  60. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/validate.py +0 -0
  61. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/src/sql2iceberg/verify.py +0 -0
  62. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_connectors.py +0 -0
  63. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_convert.py +0 -0
  64. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_detect.py +0 -0
  65. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_features.py +0 -0
  66. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_publish.py +0 -0
  67. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_pyspark_gen.py +0 -0
  68. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_spark.py +0 -0
  69. {sql2iceberg-0.1.0 → sql2iceberg-0.1.1}/tests/test_trace_validate.py +0 -0
@@ -0,0 +1,61 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/),
4
+ and the project uses [Semantic Versioning](https://semver.org/). While the version is below 1.0, minor releases may change behaviour.
5
+
6
+ ## [0.1.1] - 2026-11-09
7
+
8
+ Documentation-only release: improves the PyPI project page. No code changes.
9
+
10
+ ### Added
11
+ - Logo banner and status badges (PyPI version, Python versions, CI, license) at the top of the README.
12
+ - Table of contents in the README.
13
+
14
+ ### Fixed
15
+ - README links to the docs, changelog, contributing guide and license were relative, so they did not resolve on the PyPI
16
+ page. They are now absolute GitHub URLs.
17
+
18
+ ## [0.1.0] - 2026-11-09
19
+
20
+ First public release (alpha). Published on PyPI as [`sql2iceberg`](https://pypi.org/project/sql2iceberg/).
21
+
22
+ ### Added
23
+ - **Detection and conversion.** Find SQL in `.sql`, `.py` and `.ipynb` files; guess the source platform from driver imports,
24
+ connection strings, SQL syntax and environment variables; convert to Spark SQL with SQLGlot (Snowflake, Postgres, MySQL,
25
+ T-SQL, Oracle dialects) and flag output that cannot run or may behave differently (`converted`, `review`, `unsupported`).
26
+ - **Connectors.** Snowflake (tested live); Postgres, MySQL, SQL Server and Oracle (experimental, mock-tested only). Credentials come
27
+ from environment variables only.
28
+ - **PySpark job generator.** JDBC read of source tables, write to Iceberg, then the curation SQL with tables pointed at the
29
+ Iceberg catalog. Catalog types `hadoop`, `jdbc`, `glue`, `rest`, `hive`; S3 and GCS warehouses; Snowflake key-pair login.
30
+ Generated scripts never contain secrets.
31
+ - **Publishers.** Expose Iceberg tables to Snowflake (external Iceberg tables, reusing or creating the volume and catalog
32
+ integration), AWS (Glue registration, Athena counts) and GCP (BigQuery BigLake). Every publisher prints a plan first and only
33
+ changes an account with `--apply`.
34
+ - **Validation.** `sql2iceberg validate` compares row counts, per-column non-null counts and numeric sums across platforms in
35
+ one batched query per table.
36
+ - **Tracer.** `--trace FILE` writes timed, nested JSONL spans for every command, with secrets redacted.
37
+ - **Support tooling.** `sql2iceberg doctor` checks Java, Spark, packages and certificates; `sql2iceberg issue` opens a GitHub issue
38
+ pre-filled with environment names and versions (never values). Issue templates, troubleshooting guide, environment and
39
+ package lists, and worked examples from the live tests are in `docs/`.
40
+ - **pytest plugin** with Spark fixtures (`sql2iceberg[pytest]`).
41
+
42
+ ### Changed
43
+ - Renamed from the working name `sf2iceberg` to `sql2iceberg` (package, CLI, plugin, docs). If you installed the old name,
44
+ `pip uninstall sf2iceberg` first (see troubleshooting).
45
+ - Default Iceberg format version is 2, the version verified on all three target platforms.
46
+
47
+ ### Verified
48
+ Live run on 2026-11-09 with synthetic data: Snowflake source to Iceberg on S3 (JDBC, key-pair login) and on GCS, exposed to
49
+ Snowflake, AWS Athena and BigQuery. 38 validation checks across 4 tables and 4 platforms matched (20 customers, 100 orders,
50
+ 100 order items, 75 curated orders). CI passes on Python 3.10, 3.12 and 3.13 (Linux).
51
+
52
+ ### Known limitations
53
+ - Only Snowflake was exercised as a source. Postgres, MySQL, SQL Server and Oracle are untested.
54
+ - Tested on Iceberg format version 2; v3 support on Snowflake and BigQuery is unverified.
55
+ - BigQuery cannot read an S3 warehouse; the GCP path writes a second copy to GCS.
56
+ - Snowflake object-store tables and BigQuery tables point at one metadata file and need a refresh after new writes.
57
+ - Validation proves counts, non-null counts and sums, not row-by-row equality.
58
+ - Live tests ran on Windows 10 with Python 3.13 only.
59
+
60
+ [0.1.1]: https://github.com/jey1987-cmd/sql2iceberg/releases/tag/v0.1.1
61
+ [0.1.0]: https://github.com/jey1987-cmd/sql2iceberg/releases/tag/v0.1.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sql2iceberg
3
- Version: 0.1.0
3
+ Version: 0.1.1
4
4
  Summary: Convert SQL from Snowflake and other databases to PySpark, persist it as Apache Iceberg tables, expose them to Snowflake, AWS and GCP, and validate the result.
5
5
  Project-URL: Homepage, https://github.com/jey1987-cmd/sql2iceberg
6
6
  Project-URL: Documentation, https://github.com/jey1987-cmd/sql2iceberg/tree/main/docs
@@ -56,13 +56,24 @@ Provides-Extra: spark
56
56
  Requires-Dist: pyspark>=3.5; extra == 'spark'
57
57
  Description-Content-Type: text/markdown
58
58
 
59
+ <p align="center">
60
+ <img src="https://raw.githubusercontent.com/jey1987-cmd/sql2iceberg/main/docs/assets/logo.png" alt="sql2iceberg: SQL to PySpark to Apache Iceberg" width="520">
61
+ </p>
62
+
63
+ <p align="center">
64
+ <a href="https://pypi.org/project/sql2iceberg/"><img alt="PyPI" src="https://img.shields.io/pypi/v/sql2iceberg"></a>
65
+ <a href="https://pypi.org/project/sql2iceberg/"><img alt="Python versions" src="https://img.shields.io/pypi/pyversions/sql2iceberg"></a>
66
+ <a href="https://github.com/jey1987-cmd/sql2iceberg/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/jey1987-cmd/sql2iceberg/actions/workflows/ci.yml/badge.svg"></a>
67
+ <a href="https://github.com/jey1987-cmd/sql2iceberg/blob/main/LICENSE"><img alt="License: MIT" src="https://img.shields.io/pypi/l/sql2iceberg"></a>
68
+ </p>
69
+
59
70
  # sql2iceberg
60
71
 
61
72
  Take the SQL in your code base, convert it to **PySpark**, persist the result as **Apache Iceberg** tables, expose those
62
73
  tables to **Snowflake, AWS (Glue/Athena) and GCP (BigQuery)**, and **validate** the migration.
63
74
 
64
- **Status: alpha (0.1.0).** One source platform (Snowflake) and three target platforms were exercised against real
65
- accounts; everything else is written but untested and labelled so. See [what was tested](docs/environments.md#what-was-tested)
75
+ **Status: alpha.** One source platform (Snowflake) and three target platforms were exercised against real
76
+ accounts; everything else is written but untested and labelled so. See [what was tested](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md#what-was-tested)
66
77
  and [what is not supported yet](#limitations).
67
78
 
68
79
  ```
@@ -74,6 +85,18 @@ your code ──detect──▶ SQL ──convert──▶ Spark SQL ──gener
74
85
  └─▶ BigQuery (BigLake)
75
86
  ```
76
87
 
88
+ ## Table of contents
89
+
90
+ - [Core capabilities](#core-capabilities)
91
+ - [Install](#install)
92
+ - [Environment dependencies (summary)](#environment-dependencies-summary)
93
+ - [Quick start](#quick-start)
94
+ - [Python API](#python-api)
95
+ - [Documentation](#documentation)
96
+ - [Getting help and reporting problems](#getting-help-and-reporting-problems)
97
+ - [Limitations](#limitations)
98
+ - [License](#license)
99
+
77
100
  ## Core capabilities
78
101
 
79
102
  | # | Capability | Command / API | Status |
@@ -115,10 +138,10 @@ Extras: `spark`, `snowflake`, `postgres`, `mysql`, `mssql`, `oracle`, `aws`, `gc
115
138
  | Snowflake | account, role that can create external volumes, catalog integrations and Iceberg tables; a warehouse |
116
139
  | AWS | an AWS profile or environment credentials with S3, Glue and (to count) Athena permissions |
117
140
  | GCP | `gcloud auth login`, a project **with billing enabled**, a BigQuery connection to the bucket |
118
- | behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](docs/troubleshooting.md) |
141
+ | behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) |
119
142
 
120
- Full details, versions we tested and the list of every package and jar: [docs/environments.md](docs/environments.md) and
121
- [docs/packages.md](docs/packages.md). Run `sql2iceberg doctor` to check your machine.
143
+ Full details, versions we tested and the list of every package and jar: [docs/environments.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md) and
144
+ [docs/packages.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md). Run `sql2iceberg doctor` to check your machine.
122
145
 
123
146
  ## Quick start
124
147
 
@@ -141,7 +164,7 @@ sql2iceberg validate validate.json --out out
141
164
  ```
142
165
 
143
166
  Add `--trace trace.jsonl` to any command to record a timed trace of the run.
144
- Worked examples from real runs (configs, plans, results): [docs/examples.md](docs/examples.md).
167
+ Worked examples from real runs (configs, plans, results): [docs/examples.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md).
145
168
 
146
169
  ## Python API
147
170
 
@@ -158,20 +181,20 @@ script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", w
158
181
 
159
182
  ## Documentation
160
183
 
161
- - [Environments and what was tested](docs/environments.md)
162
- - [Every package, jar and service used in this build](docs/packages.md)
163
- - [Examples from the live tests](docs/examples.md)
164
- - [Troubleshooting frequent problems](docs/troubleshooting.md)
165
- - [Known conversion differences between Snowflake and Spark](docs/known-issues.md)
166
- - [Roadmap](docs/roadmap.md)
184
+ - [Environments and what was tested](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md)
185
+ - [Every package, jar and service used in this build](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md)
186
+ - [Examples from the live tests](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md)
187
+ - [Troubleshooting frequent problems](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md)
188
+ - [Known conversion differences between Snowflake and Spark](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/known-issues.md)
189
+ - [Roadmap](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/roadmap.md)
167
190
 
168
191
  ## Getting help and reporting problems
169
192
 
170
193
  - Run `sql2iceberg doctor`, then `sql2iceberg issue --title "short summary" --open` to open a GitHub issue pre-filled with
171
194
  your environment (names and versions only, never values or credentials).
172
195
  - Attach a trace (`--trace trace.jsonl`) after reading it. Secrets are redacted but check anyway.
173
- - Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](docs/troubleshooting.md) first.
174
- - Security problems: do not open a public issue; see [CONTRIBUTING.md](CONTRIBUTING.md#security).
196
+ - Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) first.
197
+ - Security problems: do not open a public issue; see [CONTRIBUTING.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/CONTRIBUTING.md#security).
175
198
 
176
199
  ## Limitations
177
200
 
@@ -183,7 +206,7 @@ script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", w
183
206
  - Validation compares counts, non-null counts and sums. It does not compare every row.
184
207
  - Tested on Windows 10 with Python 3.13. Linux and macOS should work; CI covers Linux but live tests were not run there.
185
208
  - Conversion uses [SQLGlot](https://github.com/tobymao/sqlglot); the converter flags constructs that behave differently in
186
- Spark (see [known issues](docs/known-issues.md)) but a clean result still needs your own verification.
209
+ Spark (see [known issues](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/known-issues.md)) but a clean result still needs your own verification.
187
210
 
188
211
  ## License
189
212
 
@@ -1,10 +1,21 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/jey1987-cmd/sql2iceberg/main/docs/assets/logo.png" alt="sql2iceberg: SQL to PySpark to Apache Iceberg" width="520">
3
+ </p>
4
+
5
+ <p align="center">
6
+ <a href="https://pypi.org/project/sql2iceberg/"><img alt="PyPI" src="https://img.shields.io/pypi/v/sql2iceberg"></a>
7
+ <a href="https://pypi.org/project/sql2iceberg/"><img alt="Python versions" src="https://img.shields.io/pypi/pyversions/sql2iceberg"></a>
8
+ <a href="https://github.com/jey1987-cmd/sql2iceberg/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/jey1987-cmd/sql2iceberg/actions/workflows/ci.yml/badge.svg"></a>
9
+ <a href="https://github.com/jey1987-cmd/sql2iceberg/blob/main/LICENSE"><img alt="License: MIT" src="https://img.shields.io/pypi/l/sql2iceberg"></a>
10
+ </p>
11
+
1
12
  # sql2iceberg
2
13
 
3
14
  Take the SQL in your code base, convert it to **PySpark**, persist the result as **Apache Iceberg** tables, expose those
4
15
  tables to **Snowflake, AWS (Glue/Athena) and GCP (BigQuery)**, and **validate** the migration.
5
16
 
6
- **Status: alpha (0.1.0).** One source platform (Snowflake) and three target platforms were exercised against real
7
- accounts; everything else is written but untested and labelled so. See [what was tested](docs/environments.md#what-was-tested)
17
+ **Status: alpha.** One source platform (Snowflake) and three target platforms were exercised against real
18
+ accounts; everything else is written but untested and labelled so. See [what was tested](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md#what-was-tested)
8
19
  and [what is not supported yet](#limitations).
9
20
 
10
21
  ```
@@ -16,6 +27,18 @@ your code ──detect──▶ SQL ──convert──▶ Spark SQL ──gener
16
27
  └─▶ BigQuery (BigLake)
17
28
  ```
18
29
 
30
+ ## Table of contents
31
+
32
+ - [Core capabilities](#core-capabilities)
33
+ - [Install](#install)
34
+ - [Environment dependencies (summary)](#environment-dependencies-summary)
35
+ - [Quick start](#quick-start)
36
+ - [Python API](#python-api)
37
+ - [Documentation](#documentation)
38
+ - [Getting help and reporting problems](#getting-help-and-reporting-problems)
39
+ - [Limitations](#limitations)
40
+ - [License](#license)
41
+
19
42
  ## Core capabilities
20
43
 
21
44
  | # | Capability | Command / API | Status |
@@ -57,10 +80,10 @@ Extras: `spark`, `snowflake`, `postgres`, `mysql`, `mssql`, `oracle`, `aws`, `gc
57
80
  | Snowflake | account, role that can create external volumes, catalog integrations and Iceberg tables; a warehouse |
58
81
  | AWS | an AWS profile or environment credentials with S3, Glue and (to count) Athena permissions |
59
82
  | GCP | `gcloud auth login`, a project **with billing enabled**, a BigQuery connection to the bucket |
60
- | behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](docs/troubleshooting.md) |
83
+ | behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) |
61
84
 
62
- Full details, versions we tested and the list of every package and jar: [docs/environments.md](docs/environments.md) and
63
- [docs/packages.md](docs/packages.md). Run `sql2iceberg doctor` to check your machine.
85
+ Full details, versions we tested and the list of every package and jar: [docs/environments.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md) and
86
+ [docs/packages.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md). Run `sql2iceberg doctor` to check your machine.
64
87
 
65
88
  ## Quick start
66
89
 
@@ -83,7 +106,7 @@ sql2iceberg validate validate.json --out out
83
106
  ```
84
107
 
85
108
  Add `--trace trace.jsonl` to any command to record a timed trace of the run.
86
- Worked examples from real runs (configs, plans, results): [docs/examples.md](docs/examples.md).
109
+ Worked examples from real runs (configs, plans, results): [docs/examples.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md).
87
110
 
88
111
  ## Python API
89
112
 
@@ -100,20 +123,20 @@ script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", w
100
123
 
101
124
  ## Documentation
102
125
 
103
- - [Environments and what was tested](docs/environments.md)
104
- - [Every package, jar and service used in this build](docs/packages.md)
105
- - [Examples from the live tests](docs/examples.md)
106
- - [Troubleshooting frequent problems](docs/troubleshooting.md)
107
- - [Known conversion differences between Snowflake and Spark](docs/known-issues.md)
108
- - [Roadmap](docs/roadmap.md)
126
+ - [Environments and what was tested](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md)
127
+ - [Every package, jar and service used in this build](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md)
128
+ - [Examples from the live tests](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md)
129
+ - [Troubleshooting frequent problems](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md)
130
+ - [Known conversion differences between Snowflake and Spark](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/known-issues.md)
131
+ - [Roadmap](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/roadmap.md)
109
132
 
110
133
  ## Getting help and reporting problems
111
134
 
112
135
  - Run `sql2iceberg doctor`, then `sql2iceberg issue --title "short summary" --open` to open a GitHub issue pre-filled with
113
136
  your environment (names and versions only, never values or credentials).
114
137
  - Attach a trace (`--trace trace.jsonl`) after reading it. Secrets are redacted but check anyway.
115
- - Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](docs/troubleshooting.md) first.
116
- - Security problems: do not open a public issue; see [CONTRIBUTING.md](CONTRIBUTING.md#security).
138
+ - Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) first.
139
+ - Security problems: do not open a public issue; see [CONTRIBUTING.md](https://github.com/jey1987-cmd/sql2iceberg/blob/main/CONTRIBUTING.md#security).
117
140
 
118
141
  ## Limitations
119
142
 
@@ -125,7 +148,7 @@ script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", w
125
148
  - Validation compares counts, non-null counts and sums. It does not compare every row.
126
149
  - Tested on Windows 10 with Python 3.13. Linux and macOS should work; CI covers Linux but live tests were not run there.
127
150
  - Conversion uses [SQLGlot](https://github.com/tobymao/sqlglot); the converter flags constructs that behave differently in
128
- Spark (see [known issues](docs/known-issues.md)) but a clean result still needs your own verification.
151
+ Spark (see [known issues](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/known-issues.md)) but a clean result still needs your own verification.
129
152
 
130
153
  ## License
131
154
 
Binary file
@@ -0,0 +1,38 @@
1
+ """Regenerate docs/assets/logo.png (the banner used in the README and on PyPI).
2
+
3
+ python docs/assets/make_logo.py [path/to/DejaVuSans-Bold.ttf path/to/DejaVuSans.ttf]
4
+
5
+ Needs Pillow. Uses DejaVu Sans (free license); matplotlib ships it in mpl-data/fonts/ttf.
6
+ """
7
+ import sys
8
+ from pathlib import Path
9
+
10
+ from PIL import Image, ImageDraw, ImageFont
11
+
12
+ BOLD = sys.argv[1] if len(sys.argv) > 2 else "DejaVuSans-Bold.ttf"
13
+ REGULAR = sys.argv[2] if len(sys.argv) > 2 else "DejaVuSans.ttf"
14
+ W, H = 2560, 720 # drawn at 2x, saved at half size for smooth edges
15
+ NAVY, WATER, ICE, ICE_SHADE, SUBMERGED = "#0B2545", "#13315C", "#EAF4FF", "#BCD9F5", "#3A86C8"
16
+ ACCENT, TAGLINE = "#5EC8F2", "#9DB8D6"
17
+
18
+ img = Image.new("RGBA", (W, H), (0, 0, 0, 0))
19
+ d = ImageDraw.Draw(img)
20
+ d.rounded_rectangle((0, 0, W - 1, H - 1), radius=90, fill=NAVY)
21
+ d.rectangle((0, 430, 620, H - 1), fill=WATER) # water
22
+ d.rounded_rectangle((0, 430, 620, H - 1), radius=90, fill=WATER, corners=(False, False, False, True))
23
+ d.polygon([(110, 430), (480, 430), (530, 530), (410, 650), (240, 670), (125, 570)], fill=SUBMERGED)
24
+ d.polygon([(150, 430), (230, 262), (290, 305), (345, 150), (435, 330), (485, 430)], fill=ICE)
25
+ d.polygon([(345, 150), (435, 330), (485, 430), (365, 430)], fill=ICE_SHADE)
26
+ d.line((0, 430, 620, 430), fill="#6FA8DC", width=6)
27
+
28
+ big = ImageFont.truetype(BOLD, 215)
29
+ small = ImageFont.truetype(REGULAR, 76)
30
+ x = 700
31
+ for text, color in (("sql", "#FFFFFF"), ("2", ACCENT), ("iceberg", "#FFFFFF")):
32
+ d.text((x, 150), text, font=big, fill=color)
33
+ x += int(d.textlength(text, font=big))
34
+ d.text((706, 440), "SQL \u2192 PySpark \u2192 Apache Iceberg", font=small, fill=TAGLINE)
35
+
36
+ out = Path(__file__).with_name("logo.png")
37
+ img.resize((W // 2, H // 2), Image.LANCZOS).save(out, optimize=True)
38
+ print("wrote", out)
@@ -0,0 +1,33 @@
1
+ ## sql2iceberg 0.1.0 (alpha)
2
+
3
+ Convert the SQL in your code base to PySpark, persist it as Apache Iceberg, expose the tables to Snowflake, AWS and GCP, and validate the result.
4
+
5
+ ```bash
6
+ pip install sql2iceberg # core
7
+ pip install "sql2iceberg[spark,snowflake,aws,gcp]"
8
+ sql2iceberg doctor # check your environment
9
+ ```
10
+
11
+ ### Highlights
12
+ - **Detect and convert** SQL from `.sql`, `.py` and `.ipynb` files to Spark SQL, with `converted` / `review` / `unsupported` findings.
13
+ - **Generate a PySpark job** that reads the source over JDBC, writes Iceberg to S3 or GCS, and runs the curation SQL.
14
+ - **Publish** to Snowflake (external Iceberg tables), AWS Glue/Athena and BigQuery. Plan first, `--apply` to change anything.
15
+ - **Validate** counts, non-null counts and sums across platforms in one query per table.
16
+ - **Trace** any run with `--trace`, **diagnose** with `doctor`, **report** with `issue`.
17
+
18
+ ### Verified live (synthetic data)
19
+ Snowflake source → Iceberg on S3 and GCS → Snowflake, AWS Athena and BigQuery: 38 checks on 4 tables across 4 platforms all matched.
20
+ CI is green on Python 3.10, 3.12 and 3.13.
21
+
22
+ ### Not verified yet
23
+ Postgres, MySQL, SQL Server and Oracle connectors (mock-tested only); Iceberg format v3 on the targets; Linux and macOS live runs;
24
+ Spark 3.5 live runs. See [environments](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md).
25
+
26
+ ### Docs
27
+ [Environments](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/environments.md) ·
28
+ [Packages](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/packages.md) ·
29
+ [Examples](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/examples.md) ·
30
+ [Troubleshooting](https://github.com/jey1987-cmd/sql2iceberg/blob/main/docs/troubleshooting.md) ·
31
+ [Full changelog](https://github.com/jey1987-cmd/sql2iceberg/blob/main/CHANGELOG.md)
32
+
33
+ Found a problem? `sql2iceberg issue --open` pre-fills a GitHub issue with your environment.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "sql2iceberg"
7
- version = "0.1.0"
7
+ version = "0.1.1"
8
8
  description = "Convert SQL from Snowflake and other databases to PySpark, persist it as Apache Iceberg tables, expose them to Snowflake, AWS and GCP, and validate the result."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -12,4 +12,4 @@ __all__ = [
12
12
  "detect_python",
13
13
  "detect_sql_text",
14
14
  ]
15
- __version__ = "0.1.0"
15
+ __version__ = "0.1.1"
@@ -1,12 +0,0 @@
1
- # Changelog
2
-
3
- ## 0.1.0 (first release, alpha)
4
-
5
- - Renamed from the working name `sf2iceberg` to `sql2iceberg`.
6
- - Detect SQL platform and SQL in `.sql`, `.py`, `.ipynb`; dialect-aware conversion to Spark SQL with safety findings.
7
- - Connectors: Snowflake (tested live); Postgres, MySQL, SQL Server, Oracle (experimental, mock-tested).
8
- - PySpark job generator: JDBC ingest to Iceberg, curation SQL, hadoop/jdbc/glue/rest/hive catalogs, S3 and GCS warehouses.
9
- - Publishers: Snowflake external Iceberg tables, AWS Glue/Athena, BigQuery BigLake. Plan first, `--apply` to change accounts.
10
- - `validate`: counts, non-null counts and sums across Snowflake, Athena and BigQuery.
11
- - `--trace` JSONL tracer with secret redaction; `doctor` and `issue` commands.
12
- - Live-tested on Snowflake, AWS and GCP with Iceberg format-version 2 (see docs/examples.md).
File without changes
File without changes
File without changes
File without changes
File without changes