sql2iceberg 0.1.0__py3-none-any.whl

sql2iceberg/convert.py

@@ -0,0 +1,170 @@
+"""Convert Snowflake SQL to Spark SQL with SQLGlot, then flag what cannot be trusted.
+
+Findings come from checks against behavior observed with Spark 3.5.6 (see docs/known-issues.md).
+SQLGlot does not raise for these cases, so this module adds its own checks.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+
+import sqlglot
+from sqlglot import ErrorLevel, exp
+from sqlglot.errors import ParseError
+
+# status values
+CONVERTED = "converted"          # no known problems found (still needs verification)
+REVIEW = "review"                # converted, but semantics may differ from Snowflake
+UNSUPPORTED = "unsupported"      # output is known not to run in Spark
+UNPARSED = "unparsed"            # could not parse as Snowflake SQL
+
+
+@dataclass
+class Finding:
+    code: str
+    severity: str                # "unsupported" or "review"
+    message: str
+
+
+@dataclass
+class Result:
+    source_sql: str
+    spark_sql: str | None
+    status: str
+    findings: list[Finding] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "source_sql": self.source_sql,
+            "spark_sql": self.spark_sql,
+            "status": self.status,
+            "findings": [f.__dict__ for f in self.findings],
+        }
+
+
+# Patterns in the converted SQL that are known to fail in Spark 3.5 (confirmed by execution).
+_OUTPUT_RULES = [
+    (re.compile(r"EXPLODE\(\s*input\s*=>", re.IGNORECASE), "FLATTEN",
+     ("FLATTEN became EXPLODE(input => ...), which Spark rejects. Rewrite with explode/posexplode "
+     "on a real array column (parse JSON strings with from_json first).")),
+    (re.compile(r"\bWITHIN\s+GROUP\b", re.IGNORECASE), "WITHIN_GROUP",
+     ("WITHIN GROUP is not valid for collect_list or listagg in Spark 3.5. Order inside a subquery "
+     "or use array_sort/sort_array with array_join.")),
+    (re.compile(r"\bGET_PATH\s*\(", re.IGNORECASE), "GET_PATH",
+     ("VARIANT path access stayed as GET_PATH, which Spark does not have. Use get_json_object on a JSON string "
+      "or struct field access.")),
+    (re.compile(r"\bMONTHNAME\s*\(", re.IGNORECASE), "MONTHNAME",
+     "MONTHNAME is not a Spark function; use date_format(col, 'MMM')."),
+    (re.compile(r"\b(SEQ[1248]|GENERATOR)\s*\(", re.IGNORECASE), "GENERATOR",
+     "SEQ*/GENERATOR have no Spark equivalent; use range() or sequence()."),
+]
+
+_REVIEW_MESSAGES = {
+    "DATEADD": "Spark DATE_ADD returns a DATE, so the time of day is dropped on timestamps. "
+               "Use timestampadd or an interval for timestamp columns.",
+    "DIVIDE": "Division by zero returns NULL in Spark 3.5 (ANSI off) but raises in Snowflake; "
+              "Spark 4 defaults to ANSI on. Record spark.sql.ansi.enabled.",
+    "TO_DATE_FORMAT": "TO_DATE format argument was dropped. Confirm the format equals the default; "
+                      "bad strings give NULL in Spark but raise in Snowflake (use TRY_TO_DATE).",
+    "REGEXP_LIKE": "Snowflake REGEXP_LIKE matches the whole string; Spark RLIKE matches anywhere. "
+                   "Anchor the pattern (^...$) to keep the meaning.",
+    "VARIANT_PATH": "VARIANT path access became get_json_object, valid only if the column holds a "
+                    "JSON string, not a struct or map.",
+    "OBJECT_CONSTRUCT": "OBJECT_CONSTRUCT became STRUCT; null handling and dynamic keys differ.",
+}
+
+
+def _source_checks(expr: exp.Expression) -> list[Finding]:
+    found: list[Finding] = []
+    codes: set[str] = set()
+    for node in expr.walk():
+        if isinstance(node, exp.DateAdd):
+            codes.add("DATEADD")
+        elif isinstance(node, exp.Div):
+            codes.add("DIVIDE")
+        elif isinstance(node, exp.RegexpLike):
+            codes.add("REGEXP_LIKE")
+        elif isinstance(node, (exp.TsOrDsToDate, exp.StrToDate)) and node.args.get("format"):
+            codes.add("TO_DATE_FORMAT")
+        elif isinstance(node, (exp.JSONExtract, exp.JSONBExtract)):
+            codes.add("VARIANT_PATH")
+        elif isinstance(node, (exp.Struct,)) or (
+            isinstance(node, exp.Anonymous) and node.name.upper() == "OBJECT_CONSTRUCT"
+        ):
+            codes.add("OBJECT_CONSTRUCT")
+    for code in sorted(codes):
+        found.append(Finding(code, "review", _REVIEW_MESSAGES[code]))
+    return found
+
+
+def _keep_try_semantics(node: exp.Expression) -> exp.Expression:
+    """SQLGlot drops the TRY_ of TRY_TO_NUMBER/TRY_TO_DOUBLE/TRY_TO_DATE in Spark output. Under Spark 4 (ANSI on)
+    the plain CAST/TO_DATE then raises on bad input where Snowflake returns NULL, so keep the safe form."""
+    if node.args.get("safe") and not node.args.get("format"):
+        if isinstance(node, exp.ToNumber):
+            precision = node.args.get("precision") or exp.Literal.number(38)
+            scale = node.args.get("scale") or exp.Literal.number(0)
+            decimal = exp.DataType(
+                this=exp.DataType.Type.DECIMAL,
+                expressions=[exp.DataTypeParam(this=precision), exp.DataTypeParam(this=scale)],
+            )
+            return exp.TryCast(this=node.this, to=decimal, safe=True)
+        if isinstance(node, exp.ToDouble):
+            return exp.TryCast(this=node.this, to=exp.DataType.build("DOUBLE"), safe=True)
+    if isinstance(node, exp.TsOrDsToDate) and node.args.get("safe"):
+        plain = node.copy()
+        plain.set("safe", False)
+        text = plain.sql(dialect="spark", unsupported_level=ErrorLevel.IGNORE)
+        # try_to_date exists from Spark 4.0; earlier versions have no safe date parse
+        return sqlglot.parse_one(re.sub(r"^TO_DATE", "TRY_TO_DATE", text), read="spark")
+    return node
+
+
+class _Capture(logging.Handler):
+    """Collect SQLGlot's warnings about arguments it cannot express in the target dialect."""
+
+    def __init__(self) -> None:
+        super().__init__(level=logging.WARNING)
+        self.messages: list[str] = []
+
+    def emit(self, record: logging.LogRecord) -> None:
+        self.messages.append(record.getMessage())
+
+
+def convert_statement(sql: str, dialect: str = "snowflake") -> Result:
+    """Convert one statement from ``dialect`` to Spark SQL and report findings."""
+    try:
+        expr = sqlglot.parse_one(sql, read=dialect)
+    except ParseError as exc:
+        return Result(sql, None, UNPARSED, [Finding("PARSE", "unsupported", str(exc)[:200])])
+    if dialect == "snowflake":
+        expr = expr.transform(_keep_try_semantics)
+    cap = _Capture()
+    logger = logging.getLogger("sqlglot")
+    logger.addHandler(cap)
+    try:
+        spark = expr.sql(dialect="spark", unsupported_level=ErrorLevel.WARN, comments=False)
+    finally:
+        logger.removeHandler(cap)
+    findings: list[Finding] = []
+    for msg in dict.fromkeys(cap.messages):
+        findings.append(Finding("DROPPED", "review",
+                                f"SQLGlot could not express part of this in Spark and dropped it: {msg[:160]}. "
+                                "The result can differ from Snowflake."))
+    for pattern, code, message in _OUTPUT_RULES:
+        if pattern.search(spark):
+            findings.append(Finding(code, "unsupported", message))
+    findings.extend(_source_checks(expr))
+    try:
+        sqlglot.parse_one(spark, read="spark")
+    except ParseError as exc:
+        findings.append(Finding("SPARK_PARSE", "unsupported", f"output does not parse as Spark SQL: {str(exc)[:160]}"))
+    if any(f.severity == "unsupported" for f in findings):
+        status = UNSUPPORTED
+    elif findings:
+        status = REVIEW
+    else:
+        status = CONVERTED
+    return Result(sql, spark, status, findings)

sql2iceberg/detect.py

@@ -0,0 +1,95 @@
+"""Find SQL statements in .sql files, Python files and notebooks."""
+
+from __future__ import annotations
+
+import ast
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import sqlglot
+from sqlglot.errors import ParseError
+
+# Calls whose first argument is usually SQL text.
+SQL_CALLS = {"execute", "executemany", "execute_string", "sql", "read_sql", "read_sql_query"}
+
+
+@dataclass
+class Statement:
+    id: str
+    file: str
+    line: int
+    sql: str
+    dynamic: bool = False       # built from variables: needs parameters, not converted
+    parse_error: str | None = None
+
+
+def _split(sql: str, dialect: str = "snowflake") -> list[str]:
+    """Split a SQL script into statements with the source dialect parser."""
+    out = []
+    for expr in sqlglot.parse(sql, read=dialect):
+        if expr is not None:
+            out.append(expr.sql(dialect=dialect, comments=False))
+    return out
+
+
+def detect_sql_text(text: str, file: str = "<text>", first_line: int = 1,
+                    dialect: str = "snowflake") -> list[Statement]:
+    statements: list[Statement] = []
+    try:
+        parts = _split(text, dialect)
+    except ParseError as exc:
+        return [Statement(f"{file}:{first_line}", file, first_line, text.strip(), parse_error=str(exc)[:200])]
+    for i, sql in enumerate(parts, 1):
+        statements.append(Statement(f"{file}:{first_line}#{i}", file, first_line, sql))
+    return statements
+
+
+def detect_python(source: str, file: str = "<python>", dialect: str = "snowflake") -> list[Statement]:
+    """Find string literals passed to cursor.execute(), spark.sql() and similar calls."""
+    found: list[Statement] = []
+    try:
+        tree = ast.parse(source)
+    except SyntaxError as exc:
+        return [Statement(f"{file}:1", file, 1, "", parse_error=f"python syntax error: {exc.msg}")]
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call) or not node.args:
+            continue
+        func = node.func
+        name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "")
+        if name not in SQL_CALLS:
+            continue
+        arg = node.args[0]
+        if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
+            found.extend(detect_sql_text(arg.value, file, arg.lineno, dialect))
+        elif isinstance(arg, (ast.JoinedStr, ast.BinOp)):
+            found.append(Statement(f"{file}:{arg.lineno}", file, arg.lineno, "", dynamic=True))
+    return found
+
+
+def detect_notebook(path: Path, dialect: str = "snowflake") -> list[Statement]:
+    nb = json.loads(path.read_text(encoding="utf-8"))
+    found: list[Statement] = []
+    for n, cell in enumerate(nb.get("cells", []), 1):
+        if cell.get("cell_type") != "code":
+            continue
+        src = "".join(cell.get("source", []))
+        src = "\n".join(l for l in src.splitlines() if not l.lstrip().startswith(("%", "!")))
+        found.extend(detect_python(src, f"{path}#cell{n}", dialect))
+    return found
+
+
+def detect_path(path: str | Path, dialect: str = "snowflake") -> list[Statement]:
+    """Detect SQL in a file or in every supported file under a folder."""
+    p = Path(path)
+    files = sorted(f for f in p.rglob("*") if f.is_file()) if p.is_dir() else [p]
+    found: list[Statement] = []
+    for f in files:
+        suffix = f.suffix.lower()
+        if suffix == ".sql":
+            found.extend(detect_sql_text(f.read_text(encoding="utf-8"), str(f), dialect=dialect))
+        elif suffix == ".py":
+            found.extend(detect_python(f.read_text(encoding="utf-8"), str(f), dialect))
+        elif suffix == ".ipynb":
+            found.extend(detect_notebook(f, dialect))
+    return found

sql2iceberg/diagnostics.py

@@ -0,0 +1,124 @@
+"""``sql2iceberg doctor``: check the local environment, and build a pre-filled GitHub issue link.
+
+Only names and versions are collected: never values of environment variables, never credentials.
+"""
+
+from __future__ import annotations
+
+import importlib.metadata as md
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from urllib.parse import quote
+
+from . import __version__
+
+REPO = "jey1987-cmd/sql2iceberg"
+ISSUES_URL = f"https://github.com/{REPO}/issues"
+
+# (distribution name, needed for)
+PACKAGES = [
+    ("sqlglot", "core: SQL parsing and conversion"),
+    ("pyspark", "Spark jobs, verify, tests (extra: spark)"),
+    ("pytest", "pytest plugin (extra: pytest)"),
+    ("chispa", "DataFrame comparison in tests (extra: pytest)"),
+    ("snowflake-connector-python", "Snowflake connector (extra: snowflake)"),
+    ("psycopg2-binary", "Postgres connector, experimental (extra: postgres)"),
+    ("pymysql", "MySQL connector, experimental (extra: mysql)"),
+    ("pyodbc", "SQL Server connector, experimental (extra: mssql)"),
+    ("oracledb", "Oracle connector, experimental (extra: oracle)"),
+    ("boto3", "AWS Glue/Athena publisher (extra: aws)"),
+    ("google-cloud-bigquery", "BigQuery publisher (extra: gcp)"),
+    ("python-dotenv", "optional: load a .env file in your own scripts"),
+]
+ENV_NAMES = [
+    "SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER", "SNOWFLAKE_PASSWORD", "SNOWFLAKE_PRIVATE_KEY_PATH",
+    "SNOWFLAKE_AUTHENTICATOR", "SNOWFLAKE_WAREHOUSE", "AWS_PROFILE", "AWS_REGION", "AWS_CA_BUNDLE",
+    "GOOGLE_APPLICATION_CREDENTIALS", "JAVA_HOME", "SPARK_HOME", "JDK_JAVA_OPTIONS", "JAVA_TOOL_OPTIONS",
+    "SQL2ICEBERG_TRACE", "SQL2ICE_ICEBERG",
+]
+
+
+def _java_version(java: str) -> str | None:
+    try:
+        out = subprocess.run([java, "-version"], capture_output=True, text=True, timeout=15, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    m = re.search(r'version "([^"]+)"', out.stderr + out.stdout)
+    return m.group(1) if m else None
+
+
+def _major(v: str | None) -> int | None:
+    if not v:
+        return None
+    parts = v.split(".")
+    return int(parts[1]) if parts[0] == "1" and len(parts) > 1 else int(re.match(r"\d+", parts[0]).group())
+
+
+def collect() -> dict:
+    """Return {"info": {...}, "checks": [(level, message)]}. Level is OK, WARN or FAIL."""
+    checks: list[tuple[str, str]] = []
+    info: dict = {"sql2iceberg": __version__, "python": sys.version.split()[0],
+                  "os": f"{platform.system()} {platform.release()}"}
+    pkgs = {}
+    for dist, _ in PACKAGES:
+        try:
+            pkgs[dist] = md.version(dist)
+        except md.PackageNotFoundError:
+            pkgs[dist] = None
+    info["packages"] = pkgs
+    # Java and Spark
+    on_path = shutil.which("java")
+    home = os.environ.get("JAVA_HOME")
+    path_v = _java_version(on_path) if on_path else None
+    home_java = str(Path(home) / "bin" / "java") if home else None
+    home_v = _java_version(home_java) if home_java and Path(home_java + (".exe" if os.name == "nt" else "")).exists() else None
+    info["java_on_path"], info["java_home_version"] = path_v, home_v
+    if pkgs.get("pyspark"):
+        if not (path_v or home_v):
+            checks.append(("FAIL", "pyspark is installed but no Java was found. Install JDK 17 and set JAVA_HOME."))
+        for label, v in (("PATH", path_v), ("JAVA_HOME", home_v)):
+            m = _major(v)
+            if m is not None and m not in (17, 21) and pkgs["pyspark"].startswith("4"):
+                checks.append(("WARN", f"Java on {label} is {v}; PySpark 4 needs Java 17 or 21."))
+            elif m is not None and m not in (8, 11, 17) and pkgs["pyspark"].startswith("3.5"):
+                checks.append(("WARN", f"Java on {label} is {v}; PySpark 3.5 supports Java 8, 11 and 17."))
+        if path_v and home_v and _major(path_v) != _major(home_v):
+            checks.append(("WARN", (f"JAVA_HOME ({home_v}) differs from java on PATH ({path_v}). Spark uses JAVA_HOME. "
+                                    "See docs/troubleshooting.md: UnsupportedClassVersionError.")))
+    sh = os.environ.get("SPARK_HOME")
+    if sh and pkgs.get("pyspark") and "site-packages" not in sh.replace("\\", "/"):
+        checks.append(("WARN", ("SPARK_HOME points outside the pip-installed PySpark and will override it. "
+                                "Unset it (docs/troubleshooting.md: wrong Spark version).")))
+    # network trust
+    if os.environ.get("AWS_CA_BUNDLE") or os.environ.get("REQUESTS_CA_BUNDLE"):
+        checks.append(("OK", ("A custom CA bundle is set for Python/AWS. Java needs its own truststore "
+                              "(docs/troubleshooting.md: certificate_unknown).")))
+    # jars
+    jars = sorted(p.name for p in Path("jars").glob("*.jar")) if Path("jars").is_dir() else []
+    info["local_jars"] = jars
+    info["env_set"] = [n for n in ENV_NAMES if os.environ.get(n)]
+    if not checks:
+        checks.append(("OK", "No problems found"))
+    return {"info": info, "checks": checks}
+
+
+def to_markdown(result: dict) -> str:
+    i = result["info"]
+    lines = [f"sql2iceberg {i['sql2iceberg']} | Python {i['python']} | {i['os']}",
+             f"Java on PATH: {i['java_on_path']} | JAVA_HOME java: {i['java_home_version']}", "", "Packages:"]
+    lines += [f"- {k}: {v or 'not installed'}" for k, v in i["packages"].items()]
+    lines += ["", "Local jars: " + (", ".join(i["local_jars"]) or "none"),
+              "Environment variables set (names only): " + (", ".join(i["env_set"]) or "none"), "", "Checks:"]
+    lines += [f"- [{lvl}] {msg}" for lvl, msg in result["checks"]]
+    return "\n".join(lines) + "\n"
+
+
+def issue_url(title: str = "", template: str = "bug_report.yml") -> str:
+    body = "```\n" + to_markdown(collect()) + "```\n"
+    return (f"{ISSUES_URL}/new?template={quote(template)}&title={quote(title)}"
+            f"&environment={quote(body)}")

sql2iceberg/generate.py

@@ -0,0 +1,112 @@
+"""Generate Iceberg table DDL (Spark, format version 3) and Snowflake-managed Iceberg DDL.
+
+Snowflake syntax follows docs.snowflake.com "CREATE ICEBERG TABLE (Snowflake as the Iceberg catalog)".
+Type notes below are conservative guesses to review, not verified against a live account.
+"""
+
+from __future__ import annotations
+
+import sqlglot
+from sqlglot import exp
+
+from .convert import UNSUPPORTED, convert_statement
+
+# Spark 3.5 has no VARIANT/OBJECT type and rejects a bare ARRAY.
+_SPARK_TYPE = {
+    "VARIANT": "STRING", "OBJECT": "MAP<STRING,STRING>", "ARRAY": "ARRAY<STRING>",
+    "MAP": "MAP<STRING,STRING>",
+}
+_SNOW_REVIEW = {
+    "VARIANT": "VARIANT: check the Snowflake Iceberg data-types page; may need a structured type or VARCHAR",
+    "OBJECT": "OBJECT: Iceberg tables need a structured OBJECT(...) definition",
+    "ARRAY": "ARRAY: Iceberg tables need a typed ARRAY(<type>)",
+    "TIMESTAMP_TZ": "TIMESTAMP_TZ: check whether Iceberg tables accept it; TIMESTAMP_LTZ may be required",
+    "FLOAT": "FLOAT/DOUBLE: allowed as data, but not as a primary key",
+}
+
+
+def _create_tables(ddl_sql: str) -> list[exp.Create]:
+    out = []
+    for e in sqlglot.parse(ddl_sql, read="snowflake"):
+        if isinstance(e, exp.Create) and (e.args.get("kind") or "").upper() == "TABLE" and not e.expression:
+            out.append(e)
+    return out
+
+
+def _columns(create: exp.Create) -> tuple[str, list[exp.ColumnDef]]:
+    schema = create.this
+    table = schema.this
+    name = ".".join(p.name for p in (table.args.get("db"), table.this) if p)
+    return name, [c for c in schema.expressions if isinstance(c, exp.ColumnDef)]
+
+
+def _spark_type(col: exp.ColumnDef, warnings: list[str], table: str) -> str:
+    dt: exp.DataType = col.args["kind"]
+    key = dt.this.name if hasattr(dt.this, "name") else str(dt.this)
+    if key in _SPARK_TYPE and not dt.expressions:
+        warnings.append(f"{table}.{col.name}: {key} stored as {_SPARK_TYPE[key]} (Spark 3.5 has no {key})")
+        return _SPARK_TYPE[key]
+    return dt.sql(dialect="spark")
+
+
+def spark_iceberg_ddl(ddl_sql: str, catalog: str = "lake", namespace: str = "db",
+                      format_version: int = 3) -> tuple[str, list[str]]:
+    warnings: list[str] = []
+    stmts = []
+    for c in _create_tables(ddl_sql):
+        name, cols = _columns(c)
+        body = ",\n  ".join(f"{col.name} {_spark_type(col, warnings, name)}" for col in cols)
+        stmts.append(
+            f"CREATE TABLE IF NOT EXISTS {catalog}.{namespace}.{name.split('.')[-1]} (\n  {body}\n)\n"
+            f"USING iceberg\nTBLPROPERTIES ('format-version'='{format_version}');"
+        )
+    return "\n\n".join(stmts) + "\n", warnings
+
+
+def snowflake_iceberg_ddl(ddl_sql: str, external_volume: str | None = None, base_prefix: str = "",
+                          iceberg_version: int = 3) -> tuple[str, list[str]]:
+    """Snowflake-managed Iceberg tables (CATALOG = 'SNOWFLAKE'). Run in your own account."""
+    warnings: list[str] = []
+    stmts = []
+    for c in _create_tables(ddl_sql):
+        name, cols = _columns(c)
+        lines = []
+        for col in cols:
+            dt = col.args["kind"]
+            key = dt.this.name if hasattr(dt.this, "name") else str(dt.this)
+            if key in _SNOW_REVIEW:
+                warnings.append(f"{name}.{col.name}: {_SNOW_REVIEW[key]}")
+            lines.append(f"{col.name} {dt.sql(dialect='snowflake')}")
+        opts = []
+        if external_volume:
+            opts.append(f"EXTERNAL_VOLUME = '{external_volume}'")
+        opts += ["CATALOG = 'SNOWFLAKE'", f"BASE_LOCATION = '{base_prefix}{name.split('.')[-1].lower()}/'",
+                 f"ICEBERG_VERSION = {iceberg_version}"]
+        stmts.append(f"CREATE ICEBERG TABLE IF NOT EXISTS {name} (\n  " + ",\n  ".join(lines) + ")\n  "
+                     + "\n  ".join(opts) + ";")
+    return "\n\n".join(stmts) + "\n", warnings
+
+
+def pyspark_script(statements, catalog: str = "lake", namespace: str = "db") -> str:
+    """A PySpark script with each converted statement as spark.sql(...). Unsupported ones are commented out."""
+    out = [
+        '"""Generated by sql2iceberg. Review every statement; none has been run against your data."""',
+        "from pyspark.sql import SparkSession", "",
+        "spark = SparkSession.builder.getOrCreate()",
+        f'spark.sql("USE {catalog}.{namespace}")', "",
+    ]
+    for s in statements:
+        if s.dynamic or s.parse_error:
+            out.append(f"# {s.id}: skipped ({'built from variables' if s.dynamic else 'could not parse'})")
+            continue
+        r = convert_statement(s.sql)
+        out.append(f"# {s.id} [{r.status}]")
+        for f in r.findings:
+            out.append(f"#   {f.code}: {f.message}")
+        if r.status == UNSUPPORTED or r.spark_sql is None:
+            out.append("# NOT GENERATED: rewrite by hand")
+            out.append("# " + (r.spark_sql or s.sql).replace("\n", "\n# "))
+        else:
+            out.append(f'spark.sql("""{r.spark_sql}""")')
+        out.append("")
+    return "\n".join(out)

sql2iceberg/lineage.py

@@ -0,0 +1,58 @@
+"""Map the inputs and outputs of each SQL statement (which tables it reads and writes)."""
+
+from __future__ import annotations
+
+import sqlglot
+from sqlglot import exp
+from sqlglot.errors import ParseError
+
+_WRITERS = (exp.Create, exp.Insert, exp.Merge, exp.Delete, exp.Update)
+
+
+def _name(t: exp.Table) -> str:
+    return ".".join(p.name for p in (t.args.get("catalog"), t.args.get("db"), t.this) if p)
+
+
+def map_statement(sql: str) -> dict:
+    """Return {kind, outputs, inputs} for one statement. Table names are as written."""
+    try:
+        tree = sqlglot.parse_one(sql, read="snowflake")
+    except ParseError as exc:
+        return {"kind": "unparsed", "outputs": [], "inputs": [], "error": str(exc)[:160]}
+    kind = type(tree).__name__.upper()
+    outputs: list[str] = []
+    if isinstance(tree, _WRITERS):
+        target = tree.this.this if isinstance(tree, exp.Create) and isinstance(tree.this, exp.Schema) else tree.this
+        if isinstance(target, exp.Table):
+            outputs.append(_name(target))
+    ctes = {c.alias for c in tree.find_all(exp.CTE)}
+    target_ids = {id(tree.this)} | {id(tree.this.this)} if isinstance(tree, _WRITERS) else set()
+    inputs: list[str] = []
+    for t in tree.find_all(exp.Table):
+        n = _name(t)
+        if not n or n in ctes or id(t) in target_ids:
+            continue
+        if isinstance(t.this, exp.Anonymous) or not isinstance(t.this, exp.Identifier):
+            continue  # table functions such as GENERATOR(...)
+        if n not in inputs:
+            inputs.append(n)
+    return {"kind": kind, "outputs": outputs, "inputs": inputs}
+
+
+def map_statements(statements) -> dict:
+    """Build a lineage map over detected statements: per statement plus per table."""
+    items, tables = [], {}
+    for s in statements:
+        if s.dynamic or s.parse_error:
+            items.append({"id": s.id, "kind": "dynamic" if s.dynamic else "unparsed", "inputs": [], "outputs": []})
+            continue
+        m = map_statement(s.sql)
+        items.append({"id": s.id, **m})
+        for n in m["inputs"]:
+            tables.setdefault(n, {"read_by": [], "written_by": []})["read_by"].append(s.id)
+        for n in m["outputs"]:
+            tables.setdefault(n, {"read_by": [], "written_by": []})["written_by"].append(s.id)
+    # external = read but never written inside the scanned code: these need to exist before the run
+    sources = sorted(n for n, v in tables.items() if v["read_by"] and not v["written_by"])
+    targets = sorted(n for n, v in tables.items() if v["written_by"])
+    return {"statements": items, "tables": tables, "source_tables": sources, "target_tables": targets}

sql2iceberg/platforms.py

@@ -0,0 +1,63 @@
+"""Work out which SQL platform a code base targets, from imports, connection strings and env vars."""
+
+from __future__ import annotations
+
+import os
+import re
+from collections import Counter
+from pathlib import Path
+
+# platform -> (strong signals: drivers/URLs, weight 3; weak signals: SQL syntax, weight 1)
+_STRONG: dict[str, list[str]] = {
+    "snowflake": [r"snowflake\.connector", r"snowflake\.snowpark", r"snowflakecomputing\.com"],
+    "postgres": [r"psycopg2?", r"asyncpg", r"postgres(ql)?://"],
+    "mysql": [r"pymysql", r"mysql\.connector", r"mysqlclient", r"mysql://"],
+    "mssql": [r"pyodbc", r"pymssql", r"mssql\+"],
+    "oracle": [r"oracledb", r"cx_Oracle"],
+}
+_WEAK: dict[str, list[str]] = {
+    "snowflake": [r"\bQUALIFY\b", r"\bIFF\s*\(", r"\bLATERAL\s+FLATTEN\b", r"\bVARIANT\b"],
+    "postgres": [r"\bRETURNING\b", r"::\s*jsonb"],
+    "mysql": [r"\bAUTO_INCREMENT\b"],
+    "mssql": [r"\bNOLOCK\b", r"\bTOP\s+\d+\b", r"\bGETDATE\s*\("],
+    "oracle": [r"\bROWNUM\b", r"\bSYSDATE\b", r"\bNVL\s*\(", r"\bFROM\s+DUAL\b"],
+}
+_ENV_SIGNALS = {
+    "snowflake": "SNOWFLAKE_ACCOUNT", "postgres": "PGHOST", "mysql": "MYSQL_HOST",
+    "mssql": "MSSQL_SERVER", "oracle": "ORACLE_DSN",
+}
+_SUFFIXES = {".py", ".sql", ".ipynb", ".yml", ".yaml", ".toml", ".cfg", ".ini", ".json"}
+_SKIP_DIRS = {".venv", ".git", "node_modules", "__pycache__"}
+
+
+def detect_platform(path: str | Path, env: dict[str, str] | None = None) -> dict:
+    """Return ``{"platform": name | None, "scores": {...}, "evidence": [...]}``.
+
+    Driver imports and connection strings count 3, SQL-syntax hints 1, a set env var 2.
+    A tie or no signal gives ``platform=None``: the caller must then ask the user.
+    """
+    env = os.environ if env is None else env
+    p = Path(path)
+    files = [f for f in p.rglob("*") if f.is_file()] if p.is_dir() else [p]
+    scores: Counter = Counter()
+    evidence: list[str] = []
+    for f in files:
+        if f.suffix.lower() not in _SUFFIXES or _SKIP_DIRS & set(f.parts):
+            continue
+        try:
+            text = f.read_text(encoding="utf-8", errors="ignore")
+        except OSError:
+            continue
+        for table, weight in ((_STRONG, 3), (_WEAK, 1)):
+            for platform, patterns in table.items():
+                for pat in patterns:
+                    if re.search(pat, text, re.IGNORECASE):
+                        scores[platform] += weight
+                        evidence.append(f"{f.name}: {platform} ~ {pat}")
+    for platform, var in _ENV_SIGNALS.items():
+        if env.get(var):
+            scores[platform] += 2
+            evidence.append(f"env {var} is set")
+    ranked = scores.most_common(2)
+    best = ranked[0][0] if ranked and (len(ranked) == 1 or ranked[0][1] > ranked[1][1]) else None
+    return {"platform": best, "scores": dict(scores), "evidence": evidence[:20]}

sql2iceberg/publish/__init__.py

@@ -0,0 +1,18 @@
+"""Expose Iceberg tables to Snowflake, AWS Glue/Athena and BigQuery. Plans first; changes accounts only on apply."""
+
+from .aws import GluePublisher, GlueTarget
+from .base import Action, IcebergTable, Publisher, PublishResult
+from .gcp import BigQueryPublisher, BigQueryTarget
+from .metadata import (
+    latest_local_metadata,
+    metadata_from_spark,
+    tables_from_jdbc_catalog,
+    tables_from_spark,
+)
+from .snowflake import SnowflakePublisher, SnowflakeTarget
+
+__all__ = [
+    "Action", "BigQueryPublisher", "BigQueryTarget", "GluePublisher", "GlueTarget", "IcebergTable",
+    "PublishResult", "Publisher", "SnowflakePublisher", "SnowflakeTarget",
+    "latest_local_metadata", "metadata_from_spark", "tables_from_jdbc_catalog", "tables_from_spark",
+]