sql2iceberg 0.1.0__py3-none-any.whl

sql2iceberg/validate.py

@@ -0,0 +1,186 @@
+"""Validate a migration: compare row counts, per-column non-null counts and numeric sums across platforms.
+
+Each platform is a ``Target``: a name, a function that runs SQL and returns rows, and a map from the logical
+table name to that platform's qualified name. One of the targets is the reference (usually the source system).
+Nothing here connects anywhere by itself; ``snowflake_runner``, ``athena_runner`` and ``bigquery_runner`` are
+thin adapters over clients you already have.
+
+What it proves: same row count, same count of non-null values per column, same sum of each numeric column
+(within a tolerance). What it does not prove: row-by-row equality. For that use ``sql2iceberg verify`` on a case
+folder, or compare a sample you export yourself.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+from .tracing import span
+
+#: how each platform spells a floating-point cast
+CASTS = {"snowflake": "FLOAT", "spark": "DOUBLE", "athena": "DOUBLE", "bigquery": "FLOAT64", "sqlite": "REAL"}
+
+
+@dataclass
+class Target:
+    name: str
+    run: Callable[[str], list[tuple]]
+    tables: dict[str, str]                 # logical name -> qualified name on this platform
+    cast: str = "DOUBLE"                   # floating-point type name, see CASTS
+
+
+@dataclass
+class ColumnSpec:
+    name: str
+    numeric: bool = False
+
+
+@dataclass
+class CheckResult:
+    table: str
+    check: str                             # row_count | non_null:<col> | sum:<col>
+    values: dict[str, float | int | None]  # platform -> value
+    ok: bool
+    note: str = ""
+
+
+@dataclass
+class ValidationReport:
+    reference: str
+    results: list[CheckResult] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return not self.errors and all(r.ok for r in self.results)
+
+    def to_dict(self) -> dict:
+        return {"reference": self.reference, "ok": self.ok, "errors": self.errors,
+                "results": [r.__dict__ for r in self.results]}
+
+    def to_markdown(self) -> str:
+        names = list(self.results[0].values) if self.results else []
+        head = ["table", "check", *names, "result"]
+        out = [f"# Validation (reference: {self.reference}): {'PASS' if self.ok else 'FAIL'}\n",
+               "| " + " | ".join(head) + " |", "|" + "---|" * len(head)]
+        for r in self.results:
+            vals = ["" if r.values.get(n) is None else str(r.values[n]) for n in names]
+            out.append("| " + " | ".join([r.table, r.check, *vals, "ok" if r.ok else f"MISMATCH {r.note}"]) + " |")
+        out += [f"\nERROR: {e}" for e in self.errors]
+        return "\n".join(out) + "\n"
+
+
+def _close(a, b, rel: float, abs_: float) -> bool:
+    if a is None or b is None:
+        return a is None and b is None
+    return math.isclose(float(a), float(b), rel_tol=rel, abs_tol=abs_)
+
+
+def validate(columns: dict[str, list[ColumnSpec]], reference: Target, others: list[Target], *,
+             rel_tol: float = 1e-9, abs_tol: float = 1e-6) -> ValidationReport:
+    """Compare every table in ``columns`` (logical name -> columns to check) across the targets."""
+    report = ValidationReport(reference.name)
+    targets = [reference, *others]
+    for table, cols in columns.items():
+        checks = [("row_count", "COUNT(*)", False)]
+        for c in cols:
+            checks.append((f"non_null:{c.name}", f"COUNT({c.name})", False))
+            if c.numeric:
+                checks.append((f"sum:{c.name}", f"SUM(CAST({c.name} AS {{cast}}))", True))
+        values: dict[str, dict[str, float | None]] = {t.name: {} for t in targets}
+        for t in targets:
+            qualified = t.tables.get(table)
+            if not qualified:
+                report.errors.append(f"{t.name}: no table mapped for {table}")
+                continue
+            # one query per table and platform: every check is a column of the same SELECT
+            # zero-padded aliases: clients that return columns sorted by name still give the right order
+            exprs = ", ".join(f"{expr.format(cast=t.cast)} AS c{i:03d}" for i, (_, expr, _) in enumerate(checks))
+            with span("validate.query", platform=t.name, table=table, checks=len(checks)):
+                try:
+                    rows = t.run(f"SELECT {exprs} FROM {qualified}")
+                    row = rows[0] if rows else ()
+                    if len(row) != len(checks):
+                        raise ValueError(f"expected {len(checks)} columns, got {len(row)}")
+                    for (label, _, _), v in zip(checks, row, strict=True):
+                        values[t.name][label] = float(v) if v is not None else None
+                except Exception as exc:  # noqa: BLE001  report per target, keep checking the others
+                    report.errors.append(f"{t.name} {table}: {type(exc).__name__}: {str(exc)[:200]}")
+        for label, _, is_sum in checks:
+            vals = {t.name: values[t.name].get(label) for t in targets}
+            ref = vals[reference.name]
+            bad = [t.name for t in others if t.name in vals and not _close(
+                ref, vals[t.name], rel_tol if is_sum else 0, abs_tol if is_sum else 0)]
+            report.results.append(CheckResult(table, label, vals, not bad, f"differs on {', '.join(bad)}" if bad else ""))
+    return report
+
+
+# ---- adapters ---------------------------------------------------------------------------------------------
+
+def snowflake_runner(connector) -> Callable[[str], list[tuple]]:
+    return connector.query
+
+
+def athena_runner(session, workgroup: str, output: str, database: str | None = None) -> Callable[[str], list[tuple]]:
+    import time
+
+    athena = session.client("athena")
+
+    def run(sql: str) -> list[tuple]:
+        kw = {"QueryExecutionContext": {"Database": database}} if database else {}
+        qid = athena.start_query_execution(QueryString=sql, WorkGroup=workgroup,
+                                           ResultConfiguration={"OutputLocation": output}, **kw)["QueryExecutionId"]
+        while True:
+            st = athena.get_query_execution(QueryExecutionId=qid)["QueryExecution"]["Status"]
+            if st["State"] in ("SUCCEEDED", "FAILED", "CANCELLED"):
+                break
+            time.sleep(1)
+        if st["State"] != "SUCCEEDED":
+            raise RuntimeError(f"Athena {st['State']}: {st.get('StateChangeReason', '')}")
+        rows = athena.get_query_results(QueryExecutionId=qid)["ResultSet"]["Rows"][1:]
+        return [tuple(c.get("VarCharValue") for c in r["Data"]) for r in rows]
+
+    return run
+
+
+def bigquery_runner(client) -> Callable[[str], list[tuple]]:
+    return lambda sql: [tuple(r) for r in client.query(sql).result()]
+
+
+# ---- config-file entry point ------------------------------------------------------------------------------
+
+def _target_from_config(cfg: dict) -> Target:
+    kind = cfg["kind"]
+    if kind == "snowflake":
+        from .connectors import get_connector
+
+        conn = get_connector("snowflake")
+        return Target(cfg.get("name", "snowflake"), snowflake_runner(conn), cfg["tables"], CASTS["snowflake"])
+    if kind == "athena":
+        import boto3
+
+        session = boto3.Session(profile_name=cfg.get("profile"), region_name=cfg.get("region"))
+        run = athena_runner(session, cfg.get("workgroup", "primary"), cfg["output"], cfg.get("database"))
+        return Target(cfg.get("name", "athena"), run, cfg["tables"], CASTS["athena"])
+    if kind == "bigquery":
+        from google.cloud import bigquery
+
+        client = bigquery.Client(project=cfg["project"], location=cfg.get("location", "US"))
+        return Target(cfg.get("name", "bigquery"), bigquery_runner(client), cfg["tables"], CASTS["bigquery"])
+    raise ValueError(f"unknown target kind {kind!r}: use snowflake, athena or bigquery")
+
+
+def report_from_config(cfg: dict) -> ValidationReport:
+    """Build targets from a plain dict (the CLI's JSON file) and run :func:`validate`.
+
+    {"tables": {"orders": ["order_id", {"name": "amount", "numeric": true}]},
+     "reference": {"kind": "snowflake", "name": "source", "tables": {"orders": "DB.SCH.ORDERS"}},
+     "targets": [{"kind": "athena", ...}, {"kind": "bigquery", ...}]}
+    """
+    columns = {t: [ColumnSpec(c) if isinstance(c, str) else ColumnSpec(c["name"], c.get("numeric", False))
+                   for c in cols] for t, cols in cfg["tables"].items()}
+    with span("validate", tables=len(columns)):
+        return validate(columns, _target_from_config(cfg["reference"]),
+                        [_target_from_config(t) for t in cfg["targets"]],
+                        rel_tol=cfg.get("rel_tol", 1e-9), abs_tol=cfg.get("abs_tol", 1e-6))

sql2iceberg/verify.py

@@ -0,0 +1,138 @@
+"""Offline verification of one converted statement.
+
+A case folder holds case.json, seed CSV files and an expected-results CSV. The expected CSV is what
+the original Snowflake query returned on the same seed data (export it from your own Snowflake
+account). The converted Spark SQL is run on the seed data and compared with it.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .convert import UNPARSED, UNSUPPORTED, convert_statement
+
+PASS, FAIL, BLOCKED = "pass", "fail", "blocked"
+
+
+@dataclass
+class Verification:
+    case: str
+    outcome: str                       # pass | fail | blocked
+    conversion_status: str
+    spark_sql: str | None = None
+    differences: list[str] = field(default_factory=list)
+    findings: list[dict] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return self.__dict__.copy()
+
+
+def make_spark(iceberg: bool = False, warehouse: str | None = None):
+    from pyspark.sql import SparkSession
+
+    # A stray SPARK_HOME points PySpark at a different Spark than the pip package (py4j "does not exist in the JVM").
+    os.environ.pop("SPARK_HOME", None)
+    builder = (
+        SparkSession.builder.master("local[2]")
+        .appName("sql2iceberg-verify")
+        .config("spark.ui.enabled", "false")
+        .config("spark.sql.shuffle.partitions", "2")
+    )
+    if iceberg:
+        pkg = os.environ.get(
+            "ICEBERG_PKG", "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.2"
+        )
+        # ICEBERG_JAR: a local runtime jar, for machines where Ivy cannot reach Maven Central (e.g. TLS proxy).
+        jar = os.environ.get("ICEBERG_JAR")
+        builder = (
+            builder.config("spark.jars", jar) if jar else builder.config("spark.jars.packages", pkg)
+        )
+        builder = (
+            builder.config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
+            .config("spark.sql.catalog.lake", "org.apache.iceberg.spark.SparkCatalog")
+            .config("spark.sql.catalog.lake.type", "hadoop")
+            .config("spark.sql.catalog.lake.warehouse", warehouse or "/tmp/sql2iceberg-warehouse")
+            .config("spark.sql.defaultCatalog", "lake")
+        )
+    spark = builder.getOrCreate()
+    spark.sparkContext.setLogLevel("ERROR")
+    if iceberg:
+        spark.sql("CREATE NAMESPACE IF NOT EXISTS lake.db")
+        spark.sql("USE lake.db")
+    return spark
+
+
+def _same(a, b, tol: float) -> bool:
+    if a is None or b is None:
+        return a is None and b is None
+    if isinstance(a, float) or isinstance(b, float):
+        try:
+            return math.isclose(float(a), float(b), rel_tol=tol, abs_tol=tol)
+        except (TypeError, ValueError):
+            return False
+    return type(a) is type(b) and a == b
+
+
+def _rows(df) -> list[tuple]:
+    return [tuple(r) for r in df.collect()]
+
+
+def _sort_key(row: tuple):
+    return tuple((v is None, str(type(v)), str(v)) for v in row)
+
+
+def verify_case(case_dir: str | Path, spark=None, iceberg: bool = False) -> Verification:
+    case_dir = Path(case_dir)
+    spec = json.loads((case_dir / "case.json").read_text(encoding="utf-8"))
+    name = spec.get("name", case_dir.name)
+    result = convert_statement(spec["sql"])
+    ver = Verification(
+        case=name,
+        outcome=BLOCKED,
+        conversion_status=result.status,
+        spark_sql=result.spark_sql,
+        findings=[f.__dict__ for f in result.findings],
+    )
+    if result.status in (UNSUPPORTED, UNPARSED):
+        ver.differences.append("not executed: conversion is " + result.status)
+        return ver
+
+    own_session = spark is None
+    spark = spark or make_spark(iceberg)
+    try:
+        for table, t in spec["tables"].items():
+            df = spark.read.csv(str(case_dir / t["csv"]), header=True, schema=t["schema"])
+            if iceberg:
+                spark.sql(f"DROP TABLE IF EXISTS {table}")
+                df.writeTo(table).using("iceberg").create()
+            else:
+                df.createOrReplaceTempView(table)  # no files left behind between runs
+        try:
+            got = _rows(spark.sql(result.spark_sql))
+        except Exception as exc:  # noqa: BLE001 - report any Spark failure as a failed verification
+            ver.outcome = FAIL
+            ver.differences.append(f"Spark failed to run the converted SQL: {str(exc).splitlines()[0][:200]}")
+            return ver
+        expected = _rows(
+            spark.read.csv(str(case_dir / spec["expected"]), header=True, schema=spec["expected_schema"])
+        )
+    finally:
+        if own_session:
+            spark.stop()
+
+    if not spec.get("ordered", False):
+        got, expected = sorted(got, key=_sort_key), sorted(expected, key=_sort_key)
+    tol = float(spec.get("float_tolerance", 1e-9))
+    if len(got) != len(expected):
+        ver.differences.append(f"row count: Spark {len(got)} vs expected {len(expected)}")
+    for i, (g, e) in enumerate(zip(got, expected)):
+        if len(g) != len(e) or not all(_same(x, y, tol) for x, y in zip(g, e)):
+            ver.differences.append(f"row {i}: Spark {g!r} vs expected {e!r}")
+            if len(ver.differences) >= 6:
+                break
+    ver.outcome = PASS if not ver.differences else FAIL
+    return ver

sql2iceberg-0.1.0.dist-info/METADATA

@@ -0,0 +1,190 @@
+Metadata-Version: 2.5
+Name: sql2iceberg
+Version: 0.1.0
+Summary: Convert SQL from Snowflake and other databases to PySpark, persist it as Apache Iceberg tables, expose them to Snowflake, AWS and GCP, and validate the result.
+Project-URL: Homepage, https://github.com/jey1987-cmd/sql2iceberg
+Project-URL: Documentation, https://github.com/jey1987-cmd/sql2iceberg/tree/main/docs
+Project-URL: Issues, https://github.com/jey1987-cmd/sql2iceberg/issues
+Project-URL: Changelog, https://github.com/jey1987-cmd/sql2iceberg/blob/main/CHANGELOG.md
+Author: Jeyaraman Ramalingam
+License-Expression: MIT
+License-File: LICENSE
+Keywords: bigquery,data-engineering,glue,iceberg,migration,pyspark,snowflake,sql,sqlglot
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Code Generators
+Requires-Python: >=3.10
+Requires-Dist: sqlglot>=30
+Provides-Extra: all
+Requires-Dist: boto3>=1.34; extra == 'all'
+Requires-Dist: google-cloud-bigquery>=3.20; extra == 'all'
+Requires-Dist: pyspark>=3.5; extra == 'all'
+Requires-Dist: snowflake-connector-python>=3.12; extra == 'all'
+Provides-Extra: aws
+Requires-Dist: boto3>=1.34; extra == 'aws'
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Provides-Extra: gcp
+Requires-Dist: google-cloud-bigquery>=3.20; extra == 'gcp'
+Provides-Extra: mssql
+Requires-Dist: pyodbc>=5; extra == 'mssql'
+Provides-Extra: mysql
+Requires-Dist: pymysql>=1.1; extra == 'mysql'
+Provides-Extra: oracle
+Requires-Dist: oracledb>=2; extra == 'oracle'
+Provides-Extra: postgres
+Requires-Dist: psycopg2-binary>=2.9; extra == 'postgres'
+Provides-Extra: pytest
+Requires-Dist: chispa>=0.10; extra == 'pytest'
+Requires-Dist: pyspark>=3.5; extra == 'pytest'
+Requires-Dist: pytest>=8; extra == 'pytest'
+Provides-Extra: snowflake
+Requires-Dist: snowflake-connector-python>=3.12; extra == 'snowflake'
+Provides-Extra: spark
+Requires-Dist: pyspark>=3.5; extra == 'spark'
+Description-Content-Type: text/markdown
+
+# sql2iceberg
+
+Take the SQL in your code base, convert it to **PySpark**, persist the result as **Apache Iceberg** tables, expose those
+tables to **Snowflake, AWS (Glue/Athena) and GCP (BigQuery)**, and **validate** the migration.
+
+**Status: alpha (0.1.0).** One source platform (Snowflake) and three target platforms were exercised against real
+accounts; everything else is written but untested and labelled so. See [what was tested](docs/environments.md#what-was-tested)
+and [what is not supported yet](#limitations).
+
+```
+your code ──detect──▶ SQL ──convert──▶ Spark SQL ──generate──▶ PySpark job ──run──▶ Iceberg on S3 / GCS
+                                                                                      │
+                          validate ◀──── counts, nulls, sums ────────────────────────┤
+                                                                                      ├─▶ Snowflake (external Iceberg table)
+                                                                                      ├─▶ AWS Glue / Athena
+                                                                                      └─▶ BigQuery (BigLake)
+```
+
+## Core capabilities
+
+| # | Capability | Command / API | Status |
+|---|---|---|---|
+| 1 | **Detect** the SQL platform and find SQL in `.sql`, `.py`, `.ipynb` | `detect`, `detect_platform()` | Snowflake tested; other dialects parse via SQLGlot |
+| 2 | **Connect** to the source (you set up authentication first) | `connectors.get_connector()` | Snowflake tested live; Postgres, MySQL, SQL Server, Oracle **experimental** (mock-tested only) |
+| 3 | **Convert** to Spark SQL and flag what cannot be trusted | `convert`, `convert_statement()` | tested |
+| 4 | **Generate a PySpark job**: JDBC read, write Iceberg, run the curation SQL | `pyspark`, `pyspark_pipeline()` | tested end to end (Snowflake to Iceberg on S3 and on GCS) |
+| 5 | **Expose** Iceberg tables to other platforms | `publish`, `publish.*Publisher` | Snowflake, AWS Glue/Athena, BigQuery all tested live |
+| 6 | **Validate** counts, non-null counts and column sums across platforms | `validate`, `validate.validate()` | tested live across four platforms |
+| 7 | **Report**: converted PySpark, plan of changes, validation results (Markdown and JSON) | all commands | tested |
+| 8 | **Trace** every step with timings, secrets redacted | `--trace trace.jsonl`, `tracing.span()` | tested |
+| 9 | **Diagnose** your environment; open a pre-filled GitHub issue | `doctor`, `issue` | tested |
+
+Safety defaults: `publish` prints a plan and changes nothing unless you pass `--apply`; credentials are only read from
+environment variables or your cloud profile; generated scripts never contain secrets.
+
+## Install
+
+```bash
+pip install sql2iceberg                       # core: detect + convert (needs only sqlglot)
+pip install "sql2iceberg[spark]"              # run the generated PySpark jobs locally
+pip install "sql2iceberg[snowflake,aws,gcp]"  # connectors and publishers you need
+pip install "sql2iceberg[all]"                # spark + snowflake + aws + gcp
+pip install "sql2iceberg[pytest]"             # pytest plugin and fixtures
+```
+
+Extras: `spark`, `snowflake`, `postgres`, `mysql`, `mssql`, `oracle`, `aws`, `gcp`, `pytest`, `all`, `dev`.
+
+## Environment dependencies (summary)
+
+| Needed for | Requirement |
+|---|---|
+| everything | Python 3.10 or newer |
+| running generated Spark jobs | **Java 17** (Java 21 should work), `pyspark` 3.5 or 4.x. Do not set `SPARK_HOME` to another Spark. |
+| Iceberg tables | Iceberg Spark runtime jar matching your Spark (we used `iceberg-spark-runtime-4.1_2.13-1.12.0`) |
+| Iceberg on S3 / GCS | `iceberg-aws-bundle` / `iceberg-gcp-bundle` jars |
+| reading the source over JDBC | the JDBC driver jar (we used `snowflake-jdbc-3.19.0`) |
+| Snowflake | account, role that can create external volumes, catalog integrations and Iceberg tables; a warehouse |
+| AWS | an AWS profile or environment credentials with S3, Glue and (to count) Athena permissions |
+| GCP | `gcloud auth login`, a project **with billing enabled**, a BigQuery connection to the bucket |
+| behind a TLS-inspecting proxy | a CA bundle for Python and a Java truststore: see [troubleshooting](docs/troubleshooting.md) |
+
+Full details, versions we tested and the list of every package and jar: [docs/environments.md](docs/environments.md) and
+[docs/packages.md](docs/packages.md). Run `sql2iceberg doctor` to check your machine.
+
+## Quick start
+
+```bash
+# 1. which platform is this code base for, and what SQL is in it?
+sql2iceberg detect path/to/code
+# 2. convert, and see what needs hand work
+sql2iceberg convert path/to/code --out out
+# 3. generate the PySpark job (reads the source over JDBC, writes Iceberg, runs the curation SQL)
+sql2iceberg pyspark path/to/code --out out --platform snowflake \
+    --catalog-type jdbc --catalog-uri jdbc:sqlite:catalog.db \
+    --warehouse s3://my-bucket/warehouse --namespace live --iceberg-version 2
+# 4. run it (set SNOWFLAKE_USER and SNOWFLAKE_PRIVATE_KEY_PATH or SNOWFLAKE_PASSWORD first)
+spark-submit out/pipeline_pyspark.py
+# 5. expose the tables: read the plan first, then --apply
+sql2iceberg publish publish.json
+sql2iceberg publish publish.json --apply
+# 6. validate across platforms
+sql2iceberg validate validate.json --out out
+```
+
+Add `--trace trace.jsonl` to any command to record a timed trace of the run.
+Worked examples from real runs (configs, plans, results): [docs/examples.md](docs/examples.md).
+
+## Python API
+
+```python
+from sql2iceberg import detect_path, convert_statement
+from sql2iceberg.pyspark_gen import MigrationConfig, pyspark_pipeline
+from sql2iceberg.publish import SnowflakePublisher, SnowflakeTarget, tables_from_jdbc_catalog
+from sql2iceberg.validate import validate, Target, ColumnSpec
+
+stmts = detect_path("pipelines/", dialect="snowflake")
+script, report = pyspark_pipeline(stmts, MigrationConfig(platform="snowflake", warehouse="s3://b/wh",
+                                                         catalog_type="jdbc", catalog_uri="jdbc:sqlite:catalog.db"))
+```
+
+## Documentation
+
+- [Environments and what was tested](docs/environments.md)
+- [Every package, jar and service used in this build](docs/packages.md)
+- [Examples from the live tests](docs/examples.md)
+- [Troubleshooting frequent problems](docs/troubleshooting.md)
+- [Known conversion differences between Snowflake and Spark](docs/known-issues.md)
+- [Roadmap](docs/roadmap.md)
+
+## Getting help and reporting problems
+
+- Run `sql2iceberg doctor`, then `sql2iceberg issue --title "short summary" --open` to open a GitHub issue pre-filled with
+  your environment (names and versions only, never values or credentials).
+- Attach a trace (`--trace trace.jsonl`) after reading it. Secrets are redacted but check anyway.
+- Issues: <https://github.com/jey1987-cmd/sql2iceberg/issues>. Please read [troubleshooting](docs/troubleshooting.md) first.
+- Security problems: do not open a public issue; see [CONTRIBUTING.md](CONTRIBUTING.md#security).
+
+## Limitations
+
+- Only **Snowflake as a source** and **Snowflake, AWS Glue/Athena, BigQuery as targets** were run against real systems.
+  Postgres, MySQL, SQL Server and Oracle connectors, REST/Hive catalogs, Spark 3.5, Azure and Databricks are untested.
+- Tested with Iceberg **format-version 2**. Whether Snowflake and BigQuery read v3 tables is unverified here.
+- BigQuery cannot read an S3 warehouse. The GCP path writes a second Iceberg copy to GCS.
+- Snowflake object-store tables and BigQuery tables point at one metadata file (a snapshot): refresh after new writes.
+- Validation compares counts, non-null counts and sums. It does not compare every row.
+- Tested on Windows 10 with Python 3.13. Linux and macOS should work; CI covers Linux but live tests were not run there.
+- Conversion uses [SQLGlot](https://github.com/tobymao/sqlglot); the converter flags constructs that behave differently in
+  Spark (see [known issues](docs/known-issues.md)) but a clean result still needs your own verification.
+
+## License
+
+MIT. Not affiliated with Snowflake, the Apache Software Foundation, Amazon or Google. "Iceberg" is a trademark of the ASF.

sql2iceberg-0.1.0.dist-info/RECORD

@@ -0,0 +1,30 @@
+sql2iceberg/__init__.py,sha256=stAfaZcxB9EhVvz6fuc8jLRwVBQZ-WR0_4vs_tQ_ncg,395
+sql2iceberg/cli.py,sha256=ruTmNvpwqsTilasCwba0ltQDtkMua3VWb1arJiA4u-E,12343
+sql2iceberg/compare.py,sha256=nXz3yLBNjhouICGVq8URBuZt1gMoVH5dQCw7DrjWni8,3361
+sql2iceberg/convert.py,sha256=HPuQjWu2DIyWq07oDqDE-OmKzt1ufh5ESLXCKedasxg,7587
+sql2iceberg/detect.py,sha256=R4UDXKngHU1O5d-Xw9UK1zdLHR3wuGVtkfCTnE47tZc,3666
+sql2iceberg/diagnostics.py,sha256=G3KSUEHAc5iCdt9F-aepXzfHnUG7nIO6YvCj_2HnUz0,5925
+sql2iceberg/generate.py,sha256=3jc-FB8-xM3QDGv9tEufOvQNubBysNy40kZ7PkX7WDk,5021
+sql2iceberg/lineage.py,sha256=LDeLaxG0QtG6P8CZxjsoe_8J7IhDnDekPYFGo6nq_5U,2643
+sql2iceberg/platforms.py,sha256=4S81gJ4naS0D-A3LZ2T_N6gPMj3RCjGkBCM8Eqt7pCI,2831
+sql2iceberg/pyspark_gen.py,sha256=SX3rRI2ANdypsCmSokrOw1OoVgvhA1qQTzeY8U6oU4s,11465
+sql2iceberg/pytest_plugin.py,sha256=7KbrrSLJ72fvEVFmXahWJOPx7pF7SX3CUTOAJoYKXhA,2767
+sql2iceberg/report.py,sha256=uQ9jxg-0JZkwUAGhe9e_Qb25DYbRKcrucrhYlDpPKDw,1884
+sql2iceberg/tracing.py,sha256=bxm7jnXrc249JAYYZYF1lbrHJmFl7Hb8vFUq9oPwu2o,4467
+sql2iceberg/validate.py,sha256=dRAasPsB84DFITSyDNdvHVKqLpoiwshmXa3xKWpJwQ8,8723
+sql2iceberg/verify.py,sha256=iToMmKL9lPajfJwwGHJn3K5rIr8J3d_-aB_RmFCpcu8,5312
+sql2iceberg/connectors/__init__.py,sha256=aw3Syhw-gEJT5JIJY9K9A2xwLPmD6cWv50m5hHGtptE,727
+sql2iceberg/connectors/base.py,sha256=biiQV8FLvX8Ll_CSE9Rqe5_DODNKbqVynG7Voi5hQbc,4165
+sql2iceberg/connectors/others.py,sha256=RMO82sToQBKEm-Rve5LyA93h0a2jY_Z40D-gQoWGLBg,4853
+sql2iceberg/connectors/snowflake.py,sha256=8sGeKDdwdIp5EzT57J33fS2W64KueQT5iKWHOM0V8rA,1960
+sql2iceberg/publish/__init__.py,sha256=SCSz_iZpfwCRD-HQ-rizo-iNAqSpKumyU3KZWfnqmDY,758
+sql2iceberg/publish/aws.py,sha256=3pWAvLXJat0LBBt8mknj3Dca1YS-7D_1l0cpf1iZjyo,5338
+sql2iceberg/publish/base.py,sha256=zS72f8jNx7pjFuWgkyQnA8Ntf3VsdS0E67gQediE91g,4744
+sql2iceberg/publish/gcp.py,sha256=4KE6ZKK6kWf4TllQvTKEX8j7sdd-WFpsZUWTtebsN0U,3544
+sql2iceberg/publish/metadata.py,sha256=SGCLEOxC3gmzgX2AB00ZR2xIobwHrrEf6qgvN5BI8go,2718
+sql2iceberg/publish/snowflake.py,sha256=uqTdurlDj0uB18Al0zG26xbMifmYG4XE-O5mKOuyJco,7622
+sql2iceberg-0.1.0.dist-info/METADATA,sha256=0qEpHUAZs2Rzz4GMETIJmwC2mPT4G24XE1L_evg2lCI,10776
+sql2iceberg-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+sql2iceberg-0.1.0.dist-info/entry_points.txt,sha256=3oOnpyo7QgckSJtXQOeCA3m-OK1T4Fwmy4oXNroly7w,105
+sql2iceberg-0.1.0.dist-info/licenses/LICENSE,sha256=rqA3WWMGK7lfkrqpbLaJYvH5FUfpnXbkiPakVC8AUOA,1077
+sql2iceberg-0.1.0.dist-info/RECORD,,

sql2iceberg-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

sql2iceberg-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+sql2iceberg = sql2iceberg.cli:main
+
+[pytest11]
+sql2iceberg = sql2iceberg.pytest_plugin

sql2iceberg-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jeyaraman Ramalingam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.