sql2iceberg 0.1.0__py3-none-any.whl

sql2iceberg/__init__.py

@@ -0,0 +1,15 @@
+"""sql2iceberg: detect Snowflake SQL, convert it for Spark/Iceberg, verify under pytest."""
+
+from .convert import Finding, Result, convert_statement
+from .detect import Statement, detect_path, detect_python, detect_sql_text
+
+__all__ = [
+    "Finding",
+    "Result",
+    "Statement",
+    "convert_statement",
+    "detect_path",
+    "detect_python",
+    "detect_sql_text",
+]
+__version__ = "0.1.0"

sql2iceberg/cli.py

@@ -0,0 +1,258 @@
+"""Command line: sql2iceberg detect | convert | report."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import report as rpt
+from .convert import UNPARSED, UNSUPPORTED, convert_statement
+from .detect import detect_path
+
+
+def _run(path: str) -> tuple[list, dict]:
+    statements = detect_path(path)
+    results = {
+        s.id: convert_statement(s.sql)
+        for s in statements
+        if not s.dynamic and not s.parse_error
+    }
+    return statements, results
+
+
+def _main(argv: list[str]) -> int:
+    parser = argparse.ArgumentParser(prog="sql2iceberg", description=__doc__)
+    parser.add_argument("--trace", metavar="FILE", help="write a JSONL trace of this run (secrets are redacted)")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("doctor", help="check Java, Spark, packages and certificates; print what to put in a bug report")
+    iss = sub.add_parser("issue", help="print (or open) a GitHub issue link pre-filled with doctor output")
+    iss.add_argument("--title", default="")
+    iss.add_argument("--open", action="store_true", help="open the link in your browser")
+    va = sub.add_parser("validate", help="compare counts, non-null counts and sums across platforms")
+    va.add_argument("config", help="JSON file: tables, reference and targets (see docs/examples.md)")
+    va.add_argument("--out", help="write validation.md and validation.json here")
+    v = sub.add_parser("verify", help="run one case folder: seed, execute converted SQL in Spark, compare")
+    v.add_argument("case", help="case folder with case.json, seed CSVs and expected CSV")
+    v.add_argument("--iceberg", action="store_true", help="use an Iceberg Hadoop catalog (needs the Iceberg runtime jar)")
+    m = sub.add_parser("map", help="map the tables each statement reads and writes")
+    m.add_argument("path")
+    m.add_argument("--out", help="write lineage.json here")
+    g = sub.add_parser("generate", help="write Iceberg v3 Spark DDL, Snowflake-managed Iceberg DDL and a PySpark script")
+    g.add_argument("path", help="code folder (statements to convert)")
+    g.add_argument("--ddl", help="SQL file or folder with CREATE TABLE statements (default: same path)")
+    g.add_argument("--out", required=True)
+    g.add_argument("--external-volume", help="Snowflake external volume name (optional)")
+    g.add_argument("--iceberg-version", type=int, default=2, choices=(2, 3))
+    pg = sub.add_parser("pyspark", help="generate a PySpark script: JDBC ingest to Iceberg, then curation SQL")
+    pg.add_argument("path", help="code folder or file (.sql, .py, .ipynb)")
+    pg.add_argument("--out", required=True, help="output folder")
+    pg.add_argument("--platform", help="source platform (default: detected from the code)")
+    pg.add_argument("--catalog", default="lake")
+    pg.add_argument("--namespace", default="db")
+    pg.add_argument("--catalog-type", default="hadoop", choices=("hadoop", "glue", "rest", "hive"))
+    pg.add_argument("--warehouse", default="./warehouse", help="path or s3://bucket/prefix")
+    pg.add_argument("--catalog-uri", default="")
+    pg.add_argument("--jdbc-url", default="", help="default: built from the platform's environment variables")
+    pg.add_argument("--source-schema", default="")
+    pg.add_argument("--iceberg-version", type=int, default=2, choices=(2, 3))
+    pu = sub.add_parser("publish", help="expose Iceberg tables to snowflake, aws (Glue) or gcp (BigQuery)")
+    pu.add_argument("config", help="JSON file: target, target settings and tables (see docs/publish.md)")
+    pu.add_argument("--apply", action="store_true", help="really change the target account (default: print the plan)")
+    pu.add_argument("--out", help="write publish_plan.md here")
+    c = sub.add_parser("compare", help="compare row counts and estimate cost")
+    c.add_argument("--snowflake-counts", required=True, help="CSV with table_name,row_count from Snowflake")
+    c.add_argument("--iceberg-counts", required=True, help="CSV with table_name,row_count from Iceberg/Spark")
+    c.add_argument("--snowflake-credits", type=float)
+    c.add_argument("--credit-price", type=float, help="your price per credit")
+    c.add_argument("--spark-seconds", type=float)
+    c.add_argument("--spark-hourly-rate", type=float, help="your cost per hour of the Spark cluster")
+    c.add_argument("--out")
+    q = sub.add_parser("sql", help="print SQL to run in Snowflake: row counts or warehouse credits")
+    q.add_argument("--counts", nargs="+", metavar="TABLE")
+    q.add_argument("--credits", nargs=3, metavar=("WAREHOUSE", "START", "END"))
+    for name in ("detect", "convert", "report"):
+        p = sub.add_parser(name)
+        p.add_argument("path", help="file or folder (.sql, .py, .ipynb)")
+        if name != "detect":
+            p.add_argument("--out", help="output folder for report files (default: print)")
+            p.add_argument("--fail-on-unsupported", action="store_true")
+    args = parser.parse_args(argv)
+
+    if args.cmd == "doctor":
+        from . import diagnostics
+
+        result = diagnostics.collect()
+        print(diagnostics.to_markdown(result))
+        return 1 if any(lvl == "FAIL" for lvl, _ in result["checks"]) else 0
+
+    if args.cmd == "issue":
+        from . import diagnostics
+
+        url = diagnostics.issue_url(args.title)
+        print(url)
+        if args.open:
+            import webbrowser
+
+            webbrowser.open(url)
+        return 0
+
+    if args.cmd == "validate":
+        import json
+
+        from .validate import report_from_config
+
+        rep = report_from_config(json.loads(Path(args.config).read_text(encoding="utf-8")))
+        print(rep.to_markdown())
+        if args.out:
+            Path(args.out).mkdir(parents=True, exist_ok=True)
+            (Path(args.out) / "validation.md").write_text(rep.to_markdown(), encoding="utf-8")
+            (Path(args.out) / "validation.json").write_text(json.dumps(rep.to_dict(), indent=2, default=str), encoding="utf-8")
+        return 0 if rep.ok else 1
+
+    if args.cmd == "verify":
+        import json
+
+        from .verify import verify_case
+
+        ver = verify_case(args.case, iceberg=args.iceberg)
+        print(json.dumps(ver.to_dict(), indent=2, default=str))
+        return 0 if ver.outcome == "pass" else 1
+
+    if args.cmd == "pyspark":
+        from .connectors import REGISTRY
+        from .platforms import detect_platform
+        from .pyspark_gen import MigrationConfig, pyspark_pipeline
+
+        platform = args.platform or detect_platform(args.path)["platform"]
+        if not platform:
+            print("could not detect the source platform: pass --platform", file=sys.stderr)
+            return 2
+        cfg = MigrationConfig(
+            platform=platform, catalog=args.catalog, namespace=args.namespace, catalog_type=args.catalog_type,
+            warehouse=args.warehouse, catalog_uri=args.catalog_uri, jdbc_url=args.jdbc_url,
+            source_schema=args.source_schema, format_version=args.iceberg_version,
+        )
+        stmts = detect_path(args.path, dialect=REGISTRY[platform].sqlglot_dialect)
+        script, rep = pyspark_pipeline(stmts, cfg)
+        out = Path(args.out)
+        out.mkdir(parents=True, exist_ok=True)
+        (out / "pipeline_pyspark.py").write_text(script, encoding="utf-8")
+        bad = [r for r in rep if r["status"] in ("unsupported", "unparsed", "skipped")]
+        print(f"platform={platform}; wrote {out / 'pipeline_pyspark.py'}; {len(rep)} statements, {len(bad)} need hand work")
+        return 0
+
+    if args.cmd == "publish":
+        import json
+
+        from .publish.base import publisher_from_config
+
+        pub, tables = publisher_from_config(json.loads(Path(args.config).read_text(encoding="utf-8")))
+        result = pub.apply(tables, confirm=True) if args.apply else pub.plan(tables)
+        md = result.to_markdown()
+        if args.out:
+            Path(args.out).mkdir(parents=True, exist_ok=True)
+            (Path(args.out) / "publish_plan.md").write_text(md, encoding="utf-8")
+        print(md)
+        return 0 if (not args.apply or result.ok) else 1
+
+    if args.cmd == "compare":
+        from . import compare as cmp
+
+        counts = cmp.compare_counts(cmp.read_counts(args.snowflake_counts), cmp.read_counts(args.iceberg_counts))
+        cost = None
+        if any(v is not None for v in (args.snowflake_credits, args.spark_seconds)):
+            cost = cmp.cost_summary(args.snowflake_credits, args.credit_price, args.spark_seconds, args.spark_hourly_rate)
+        md = cmp.to_markdown(counts, cost)
+        if args.out:
+            Path(args.out).mkdir(parents=True, exist_ok=True)
+            (Path(args.out) / "compare.md").write_text(md, encoding="utf-8")
+            print(f"wrote {Path(args.out) / 'compare.md'}")
+        else:
+            print(md)
+        return 0 if all(c["status"] == "match" for c in counts) else 1
+
+    if args.cmd == "sql":
+        from . import compare as cmp
+
+        if args.counts:
+            print(cmp.count_sql(args.counts))
+        if args.credits:
+            print(cmp.credits_sql(*args.credits))
+        return 0
+
+    if args.cmd == "generate":
+        from . import generate as gen
+
+        ddl_text = "\n".join(s.sql.rstrip(";") + ";" for s in detect_path(args.ddl or args.path)
+                             if not s.dynamic and not s.parse_error)
+        out = Path(args.out)
+        out.mkdir(parents=True, exist_ok=True)
+        spark_ddl, w1 = gen.spark_iceberg_ddl(ddl_text, format_version=args.iceberg_version)
+        sf_ddl, w2 = gen.snowflake_iceberg_ddl(ddl_text, args.external_volume, iceberg_version=args.iceberg_version)
+        (out / "iceberg_spark.sql").write_text(spark_ddl, encoding="utf-8")
+        (out / "snowflake_managed_iceberg.sql").write_text(sf_ddl, encoding="utf-8")
+        (out / "pipeline_pyspark.py").write_text(gen.pyspark_script(detect_path(args.path)), encoding="utf-8")
+        (out / "warnings.txt").write_text("\n".join(w1 + w2) + "\n", encoding="utf-8")
+        print(f"wrote 4 files to {out}; {len(w1) + len(w2)} type warnings to review")
+        return 0
+
+    statements, results = _run(args.path) if args.cmd != "map" else (detect_path(args.path), {})
+    if args.cmd == "map":
+        import json
+
+        from .lineage import map_statements
+
+        lin = map_statements(statements)
+        text = json.dumps(lin, indent=2)
+        if args.out:
+            Path(args.out).mkdir(parents=True, exist_ok=True)
+            (Path(args.out) / "lineage.json").write_text(text, encoding="utf-8")
+        print(f"sources: {lin['source_tables']}\ntargets: {lin['target_tables']}")
+        return 0
+
+    if args.cmd == "detect":
+        for s in statements:
+            tag = "DYNAMIC" if s.dynamic else ("UNPARSED" if s.parse_error else "SQL")
+            print(f"{s.id}\t{tag}\t{s.sql[:80].replace(chr(10), ' ')}")
+        return 0
+
+    report = rpt.build(statements, results)
+    if args.out:
+        out = Path(args.out)
+        out.mkdir(parents=True, exist_ok=True)
+        (out / "report.json").write_text(rpt.to_json(report), encoding="utf-8")
+        (out / "report.md").write_text(rpt.to_markdown(report), encoding="utf-8")
+        print(f"wrote {out/'report.json'} and {out/'report.md'}")
+    else:
+        print(rpt.to_markdown(report))
+    print(f"summary: {report['summary']}", file=sys.stderr)
+    bad = [i for i in report["statements"] if i["status"] in (UNSUPPORTED, UNPARSED)]
+    return 1 if (args.fail_on_unsupported and bad) else 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Entry point. ``--trace FILE`` is handled here so every command is wrapped in a span."""
+    from .tracing import Tracer, set_tracer, span
+
+    argv = list(sys.argv[1:] if argv is None else argv)
+    pre = argparse.ArgumentParser(add_help=False)
+    pre.add_argument("--trace")
+    known, rest = pre.parse_known_args(argv)
+    tracer = Tracer(known.trace) if known.trace else None
+    if tracer:
+        set_tracer(tracer)
+    cmd = next((a for a in rest if not a.startswith("-")), "help")
+    try:
+        with span(f"cli.{cmd}") as sp:
+            rc = _main(rest)
+            sp.set(exit_code=rc)
+    finally:
+        if tracer:
+            print(tracer.summary(), file=sys.stderr)
+            set_tracer(None)
+    return rc
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sql2iceberg/compare.py

@@ -0,0 +1,68 @@
+"""Compare row counts and cost between Snowflake and Iceberg runs.
+
+Counts: two CSVs (table,row_count) that you export. Cost: Snowflake credits come from your own
+ACCOUNT_USAGE views (up to ~3 hours late); Spark cost is runtime x a rate you supply. Costs are estimates.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+SNOWFLAKE_COUNT_SQL = "SELECT '{t}' AS table_name, COUNT(*) AS row_count FROM {t};"
+
+# Credits for one warehouse in a time window. ACCOUNT_USAGE data lags (up to 180 minutes; cloud services up to 6 h).
+SNOWFLAKE_CREDITS_SQL = """SELECT SUM(credits_used_compute) AS compute_credits,
+       SUM(credits_used_cloud_services) AS cloud_services_credits,
+       SUM(credits_used) AS credits_used   -- may be higher than billed (cloud services adjustment)
+FROM snowflake.account_usage.warehouse_metering_history
+WHERE warehouse_name = '{warehouse}'
+  AND start_time >= '{start}'::timestamp_ltz AND end_time <= '{end}'::timestamp_ltz;"""
+
+
+def count_sql(tables: list[str]) -> str:
+    return "\nUNION ALL\n".join(SNOWFLAKE_COUNT_SQL.format(t=t).rstrip(";") for t in tables) + ";"
+
+
+def credits_sql(warehouse: str, start: str, end: str) -> str:
+    return SNOWFLAKE_CREDITS_SQL.format(warehouse=warehouse, start=start, end=end)
+
+
+def read_counts(path: str | Path) -> dict[str, int]:
+    with open(path, newline="", encoding="utf-8") as fh:
+        rows = list(csv.DictReader(fh))
+    return {r["table_name"].strip().lower().split(".")[-1]: int(r["row_count"]) for r in rows}
+
+
+def spark_counts(spark, tables: list[str]) -> dict[str, int]:
+    return {t.lower().split(".")[-1]: spark.table(t).count() for t in tables}
+
+
+def compare_counts(snowflake: dict[str, int], iceberg: dict[str, int]) -> list[dict]:
+    out = []
+    for t in sorted(set(snowflake) | set(iceberg)):
+        a, b = snowflake.get(t), iceberg.get(t)
+        status = "missing" if a is None or b is None else ("match" if a == b else "mismatch")
+        out.append({"table": t, "snowflake": a, "iceberg": b, "status": status})
+    return out
+
+
+def cost_summary(snowflake_credits: float | None, credit_price: float | None,
+                 spark_seconds: float | None, spark_hourly_rate: float | None) -> dict:
+    sf = round(snowflake_credits * credit_price, 4) if snowflake_credits is not None and credit_price is not None else None
+    sp = round(spark_seconds / 3600 * spark_hourly_rate, 4) if spark_seconds is not None and spark_hourly_rate is not None else None
+    return {
+        "snowflake_credits": snowflake_credits, "snowflake_cost": sf,
+        "spark_seconds": spark_seconds, "spark_cost": sp,
+        "note": "estimate: Snowflake from credits x your price; Spark from runtime x your hourly rate; "
+                "storage, network and engineer time are not included",
+    }
+
+
+def to_markdown(counts: list[dict], cost: dict | None) -> str:
+    lines = ["# Count and cost comparison", "", "| table | snowflake | iceberg | status |", "|---|---|---|---|"]
+    lines += [f"| {c['table']} | {c['snowflake']} | {c['iceberg']} | {c['status']} |" for c in counts]
+    if cost:
+        lines += ["", "## Cost (estimate)", "", f"- Snowflake: {cost['snowflake_credits']} credits, cost {cost['snowflake_cost']}",
+                  f"- Spark: {cost['spark_seconds']} s, cost {cost['spark_cost']}", f"- {cost['note']}"]
+    return "\n".join(lines) + "\n"

sql2iceberg/connectors/__init__.py

@@ -0,0 +1,22 @@
+"""Source connectors."""
+
+from __future__ import annotations
+
+from .base import ColumnInfo, Connector, TableInfo
+from .others import MySQLConnector, OracleConnector, PostgresConnector, SQLServerConnector
+from .snowflake import SnowflakeConnector
+
+REGISTRY: dict[str, type[Connector]] = {
+    c.name: c
+    for c in (SnowflakeConnector, PostgresConnector, MySQLConnector, SQLServerConnector, OracleConnector)
+}
+
+
+def get_connector(name: str, **params: str) -> Connector:
+    try:
+        return REGISTRY[name.lower()](**params)
+    except KeyError:
+        raise ValueError(f"unknown platform {name!r}; choose from {sorted(REGISTRY)}") from None
+
+
+__all__ = ["REGISTRY", "ColumnInfo", "Connector", "TableInfo", "get_connector"]

sql2iceberg/connectors/base.py

@@ -0,0 +1,126 @@
+"""Connector interface: every source platform implements the same small surface.
+
+Credentials are never read from config files. Each connector reads them from the environment
+(or the platform's own profile/SSO mechanism) so the user sets up authentication before running.
+"""
+
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+
+
+@dataclass
+class ColumnInfo:
+    name: str
+    data_type: str
+    nullable: bool = True
+
+
+@dataclass
+class TableInfo:
+    schema: str
+    name: str
+    columns: list[ColumnInfo] = field(default_factory=list)
+
+    @property
+    def qualified(self) -> str:
+        return f"{self.schema}.{self.name}"
+
+
+class Connector(ABC):
+    """A live connection to a source system."""
+
+    #: registry key
+    name: str = ""
+    #: SQLGlot dialect used to parse this platform's SQL
+    sqlglot_dialect: str = ""
+    #: pip extra that installs the driver, e.g. ``pip install sql2iceberg[snowflake]``
+    extra: str = ""
+    #: only Snowflake is verified against a live system
+    tested: bool = False
+    #: env vars that must be set; checked for a clear error before connecting
+    required_env: tuple[str, ...] = ()
+    #: JDBC driver class so Spark can read the source in parallel
+    jdbc_driver: str = ""
+    #: env vars holding the JDBC login, and the Maven coordinate of the driver (review the version)
+    jdbc_user_env: str = ""
+    jdbc_password_env: str = ""
+    jdbc_package: str = ""
+    #: optional key-pair login: env var with the private-key file path, and the JDBC property that takes it
+    jdbc_key_env: str = ""
+    jdbc_key_option: str = ""
+
+    def __init__(self, **params: str) -> None:
+        self.params = params
+        self._conn = None
+
+    @classmethod
+    def missing_env(cls, env: dict[str, str] | None = None) -> list[str]:
+        env = os.environ if env is None else env
+        return [k for k in cls.required_env if not env.get(k)]
+
+    @abstractmethod
+    def connect(self):
+        """Open the DB-API connection, store it on ``self._conn`` and return it."""
+
+    @abstractmethod
+    def jdbc_url(self) -> str:
+        """JDBC URL for Spark reads (never contains credentials)."""
+
+    @abstractmethod
+    def _tables_sql(self, schema: str | None) -> tuple[str, tuple]:
+        """SQL returning (schema, table, column, type, nullable) rows, plus its parameters."""
+
+    def _count_sql(self, table: str) -> str:
+        return f"SELECT COUNT(*) FROM {table}"
+
+    def _require_driver(self, module: str):
+        import importlib
+
+        try:
+            return importlib.import_module(module)
+        except ImportError as exc:
+            raise ImportError(
+                f"{self.name} support needs its driver: pip install 'sql2iceberg[{self.extra}]'"
+            ) from exc
+
+    def _ensure(self):
+        if self._conn is None:
+            missing = self.missing_env()
+            if missing:
+                raise RuntimeError(f"Set these environment variables first: {', '.join(missing)}")
+            self.connect()
+        return self._conn
+
+    def query(self, sql: str, params: tuple = ()) -> list[tuple]:
+        cur = self._ensure().cursor()
+        try:
+            cur.execute(sql, params) if params else cur.execute(sql)
+            return list(cur.fetchall())
+        finally:
+            cur.close()
+
+    def list_tables(self, schema: str | None = None) -> list[TableInfo]:
+        sql, params = self._tables_sql(schema)
+        tables: dict[tuple[str, str], TableInfo] = {}
+        for sch, tbl, col, dtype, nullable in self.query(sql, params):
+            info = tables.setdefault((sch, tbl), TableInfo(sch, tbl))
+            info.columns.append(ColumnInfo(col, dtype, str(nullable).upper() in ("YES", "Y", "1", "TRUE")))
+        return list(tables.values())
+
+    def row_count(self, table: str) -> int:
+        return int(self.query(self._count_sql(table))[0][0])
+
+    def close(self) -> None:
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+    def __enter__(self):
+        self._ensure()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()

sql2iceberg/connectors/others.py

@@ -0,0 +1,133 @@
+"""Postgres, MySQL, SQL Server and Oracle connectors.
+
+EXPERIMENTAL: written to the same interface as Snowflake and covered by mock-based unit tests only.
+They have not been run against live servers.
+"""
+
+from __future__ import annotations
+
+import os
+
+from .base import Connector
+
+_INFO_SCHEMA = (
+    "SELECT table_schema, table_name, column_name, data_type, is_nullable "
+    "FROM information_schema.columns WHERE table_schema NOT IN ({excl})"
+)
+
+
+def _info_schema_query(excl: str, schema: str | None, marker: str = "%s"):
+    sql = _INFO_SCHEMA.format(excl=excl)
+    if schema:
+        return sql + f" AND table_schema = {marker} ORDER BY 1, 2, ordinal_position", (schema,)
+    return sql + " ORDER BY 1, 2, ordinal_position", ()
+
+
+class PostgresConnector(Connector):
+    name = "postgres"
+    sqlglot_dialect = "postgres"
+    extra = "postgres"
+    required_env = ("PGHOST", "PGUSER", "PGDATABASE")
+    jdbc_driver = "org.postgresql.Driver"
+    jdbc_user_env = "PGUSER"
+    jdbc_password_env = "PGPASSWORD"
+    jdbc_package = "org.postgresql:postgresql:42.7.4"
+
+    def connect(self):
+        pg = self._require_driver("psycopg2")
+        self._conn = pg.connect(
+            host=os.environ["PGHOST"], port=os.environ.get("PGPORT", "5432"),
+            user=os.environ["PGUSER"], password=os.environ.get("PGPASSWORD"),
+            dbname=os.environ["PGDATABASE"],
+        )
+        return self._conn
+
+    def jdbc_url(self) -> str:
+        e = os.environ
+        return f"jdbc:postgresql://{e['PGHOST']}:{e.get('PGPORT', '5432')}/{e['PGDATABASE']}"
+
+    def _tables_sql(self, schema):
+        return _info_schema_query("'pg_catalog','information_schema'", schema)
+
+
+class MySQLConnector(Connector):
+    name = "mysql"
+    sqlglot_dialect = "mysql"
+    extra = "mysql"
+    required_env = ("MYSQL_HOST", "MYSQL_USER", "MYSQL_DATABASE")
+    jdbc_driver = "com.mysql.cj.jdbc.Driver"
+    jdbc_user_env = "MYSQL_USER"
+    jdbc_password_env = "MYSQL_PASSWORD"
+    jdbc_package = "com.mysql:mysql-connector-j:8.4.0"
+
+    def connect(self):
+        my = self._require_driver("pymysql")
+        self._conn = my.connect(
+            host=os.environ["MYSQL_HOST"], port=int(os.environ.get("MYSQL_PORT", "3306")),
+            user=os.environ["MYSQL_USER"], password=os.environ.get("MYSQL_PASSWORD", ""),
+            database=os.environ["MYSQL_DATABASE"],
+        )
+        return self._conn
+
+    def jdbc_url(self) -> str:
+        e = os.environ
+        return f"jdbc:mysql://{e['MYSQL_HOST']}:{e.get('MYSQL_PORT', '3306')}/{e['MYSQL_DATABASE']}"
+
+    def _tables_sql(self, schema):
+        return _info_schema_query("'mysql','information_schema','performance_schema','sys'", schema)
+
+
+class SQLServerConnector(Connector):
+    name = "mssql"
+    sqlglot_dialect = "tsql"
+    extra = "mssql"
+    required_env = ("MSSQL_SERVER", "MSSQL_DATABASE")
+    jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+    jdbc_user_env = "MSSQL_USER"
+    jdbc_password_env = "MSSQL_PASSWORD"
+    jdbc_package = "com.microsoft.sqlserver:mssql-jdbc:12.8.1.jre11"
+
+    def connect(self):
+        py = self._require_driver("pyodbc")
+        e = os.environ
+        auth = ("Trusted_Connection=yes" if not e.get("MSSQL_USER")
+                else f"UID={e['MSSQL_USER']};PWD={e.get('MSSQL_PASSWORD', '')}")
+        driver = e.get("MSSQL_ODBC_DRIVER", "ODBC Driver 18 for SQL Server")
+        self._conn = py.connect(
+            f"DRIVER={{{driver}}};SERVER={e['MSSQL_SERVER']};DATABASE={e['MSSQL_DATABASE']};{auth}"
+        )
+        return self._conn
+
+    def jdbc_url(self) -> str:
+        e = os.environ
+        return f"jdbc:sqlserver://{e['MSSQL_SERVER']};databaseName={e['MSSQL_DATABASE']}"
+
+    def _tables_sql(self, schema):
+        return _info_schema_query("'sys','INFORMATION_SCHEMA'", schema, marker="?")
+
+
+class OracleConnector(Connector):
+    name = "oracle"
+    sqlglot_dialect = "oracle"
+    extra = "oracle"
+    required_env = ("ORACLE_DSN", "ORACLE_USER")
+    jdbc_driver = "oracle.jdbc.OracleDriver"
+    jdbc_user_env = "ORACLE_USER"
+    jdbc_password_env = "ORACLE_PASSWORD"
+    jdbc_package = "com.oracle.database.jdbc:ojdbc11:23.5.0.24.07"
+
+    def connect(self):
+        ora = self._require_driver("oracledb")
+        self._conn = ora.connect(user=os.environ["ORACLE_USER"],
+                                 password=os.environ.get("ORACLE_PASSWORD"), dsn=os.environ["ORACLE_DSN"])
+        return self._conn
+
+    def jdbc_url(self) -> str:
+        return f"jdbc:oracle:thin:@{os.environ['ORACLE_DSN']}"
+
+    def _tables_sql(self, schema):
+        sql = ("SELECT owner, table_name, column_name, data_type, nullable FROM all_tab_columns "
+               "WHERE owner NOT IN ('SYS','SYSTEM')")
+        if schema:
+            return sql + " AND owner = :1 ORDER BY 1, 2, column_id", (schema.upper(),)
+        return sql + " ORDER BY 1, 2, column_id", ()

sql2iceberg/connectors/snowflake.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import os
+
+from .base import Connector
+
+
+class SnowflakeConnector(Connector):
+    name = "snowflake"
+    sqlglot_dialect = "snowflake"
+    extra = "snowflake"
+    tested = True
+    required_env = ("SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER")
+    jdbc_driver = "net.snowflake.client.jdbc.SnowflakeDriver"
+    jdbc_user_env = "SNOWFLAKE_USER"
+    jdbc_password_env = "SNOWFLAKE_PASSWORD"
+    jdbc_package = "net.snowflake:snowflake-jdbc:3.19.0"
+    jdbc_key_env = "SNOWFLAKE_PRIVATE_KEY_PATH"
+    jdbc_key_option = "private_key_file"
+
+    def connect(self):
+        sf = self._require_driver("snowflake.connector")
+        p = {"account": os.environ["SNOWFLAKE_ACCOUNT"], "user": os.environ["SNOWFLAKE_USER"]}
+        if os.environ.get("SNOWFLAKE_ROLE"):
+            p["role"] = os.environ["SNOWFLAKE_ROLE"]
+        if os.environ.get("SNOWFLAKE_AUTHENTICATOR"):
+            p["authenticator"] = os.environ["SNOWFLAKE_AUTHENTICATOR"]
+        elif os.environ.get("SNOWFLAKE_PRIVATE_KEY_PATH"):
+            p["private_key_file"] = os.environ["SNOWFLAKE_PRIVATE_KEY_PATH"]
+        else:
+            p["password"] = os.environ.get("SNOWFLAKE_PASSWORD", "")
+        for key in ("warehouse", "database", "schema"):
+            value = self.params.get(key) or os.environ.get(f"SNOWFLAKE_{key.upper()}")
+            if value:
+                p[key] = value
+        self._conn = sf.connect(**p)
+        return self._conn
+
+    def jdbc_url(self) -> str:
+        return f"jdbc:snowflake://{os.environ['SNOWFLAKE_ACCOUNT']}.snowflakecomputing.com/"
+
+    def _tables_sql(self, schema):
+        sql = ("SELECT table_schema, table_name, column_name, data_type, is_nullable "
+               "FROM information_schema.columns WHERE table_schema <> 'INFORMATION_SCHEMA'")
+        if schema:
+            return sql + " AND table_schema = %s ORDER BY 1, 2, ordinal_position", (schema.upper(),)
+        return sql + " ORDER BY 1, 2, ordinal_position", ()