ripple-sql 0.1.0__py3-none-any.whl

ripple/engine/sql_script.py

@@ -0,0 +1,729 @@
+"""Multi-statement SQL scripts and dumps.
+
+Real non-dbt SQL is rarely one statement per file. A Chinook dump or a Postgres
+sample DB is a single file with hundreds to thousands of statements: DROP, CREATE
+TABLE, INSERT, CREATE VIEW, mixed with psql `\\` meta-commands that aren't SQL at
+all. This module splits such a file into statements resiliently (never raising),
+sniffs the dialect from content when there's no dbt adapter to ask, and turns the
+derivation-bearing statements into models with schema.
+
+Nothing here fabricates lineage. A statement that will not parse is skipped, and a
+CREATE TABLE with no SELECT contributes only its declared columns (the schema), not
+a made-up edge.
+"""
+
+from __future__ import annotations
+
+import re
+
+import sqlglot
+from sqlglot import exp
+from sqlglot.errors import ErrorLevel
+
+from ripple.engine.safe_gen import safe_sql
+
+# psql / mysql-cli meta-commands: whole lines starting with a backslash (\pset, \c,
+# \d, \.), which are client directives, not SQL. Stripped before parsing.
+_META_LINE = re.compile(r"^[ \t]*\\[a-zA-Z.].*$", re.M)
+
+# T-SQL's GO is the same class of client directive: a batch separator sqlcmd
+# understands but the server never sees. Left in place it lumps the next
+# batch into one unparseable Command ("GO\nCREATE VIEW ...", fhir_server,
+# holdout round 11). Rewritten to ";" so batch boundaries survive. The full
+# separator grammar: case-insensitive GO, optional count, optional trailing
+# line or block comment (cycle-12 review, F12). Applied per line by the
+# quote/comment-state scan below, never as a blind substitution, so a GO
+# inside a bracketed identifier or string stays part of the name (F1).
+_GO_LINE = re.compile(
+    r"^[ \t]*GO(?:[ \t]+\d+)?[ \t]*(?:--[^\r\n]*|/\*.*?\*/[ \t]*)?\r?$",
+    re.I,
+)
+
+
+def _rewrite_go_separators(sql: str) -> str:
+    """Replace batch-separator GO lines with ";", tracking quote and comment
+    state so quoted identifiers and comments are never rewritten."""
+    out: list[str] = []
+    state = ""  # "" | "'" | '"' | "`" | "[" | "/*"
+    for line in sql.split("\n"):
+        if not state and _GO_LINE.match(line):
+            out.append(";")
+            continue
+        i, n = 0, len(line)
+        while i < n:
+            c = line[i]
+            if state == "/*":
+                if c == "*" and line[i + 1 : i + 2] == "/":
+                    state = ""
+                    i += 2
+                    continue
+                i += 1
+                continue
+            if state == "[":
+                if c == "]":
+                    if line[i + 1 : i + 2] == "]":  # ]] escapes ] inside the name
+                        i += 2
+                        continue
+                    state = ""
+                i += 1
+                continue
+            if state:
+                if c == state:
+                    # a doubled quote reads as close-then-reopen; same state
+                    state = ""
+                i += 1
+                continue
+            if c == "-" and line[i + 1 : i + 2] == "-":
+                break  # the rest of the line is a comment
+            if c == "/" and line[i + 1 : i + 2] == "*":
+                state = "/*"
+                i += 2
+                continue
+            if c in "'\"`[":
+                state = c
+            i += 1
+        out.append(line)
+    return "\n".join(out)
+
+
+# Markers that only one dialect writes, weighted by how exclusive they are.
+# A backtick scores for both BigQuery and MySQL because both quote with it;
+# reading it as MySQL alone is what made filecoin-data-portal, a BigQuery
+# project, parse as the wrong dialect.
+_DIALECT_MARKERS: tuple[tuple[str, str, int, int], ...] = (
+    # (dialect, pattern, weight, re flags)
+    ("mysql", r"\bENGINE\s*=\s*\w+|\bAUTO_INCREMENT\b|\bUNSIGNED\b", 4, re.I),
+    ("mysql", r"\bTINYINT\b|\bLONGTEXT\b|\bDATETIME\(\d\)", 2, re.I),
+    ("bigquery", r"\bINFORMATION_SCHEMA\b", 3, 0),
+    ("bigquery", r"\bSTRUCT\s*<|\bARRAY\s*<", 3, re.I),
+    ("bigquery", r"\bSAFE[._]\w+", 3, re.I),
+    ("bigquery", r"`[\w-]+\.[\w-]+\.[\w-]+`", 3, 0),
+    # types and functions BigQuery alone spells this way
+    ("bigquery", r"\bFLOAT64\b|\bBIGNUMERIC\b|\bINT64\b", 3, re.I),
+    ("bigquery", r"\bTIMESTAMP_SECONDS\s*\(|\bCOUNTIF\s*\(|\bSAFE_DIVIDE\s*\(", 3, re.I),
+    ("bigquery", r"\bGENERATE_(DATE|TIMESTAMP)_ARRAY\s*\(|\bUNNEST\s*\(", 2, re.I),
+    # the GO alternative mirrors _GO_LINE's full separator grammar,
+    # case-insensitive via a scoped flag (cycle-12 review, F13)
+    (
+        "tsql",
+        r"\bIDENTITY\s*\(|\bNVARCHAR\b|\bGETDATE\s*\("
+        r"|(?i:^[ \t]*GO(?:[ \t]+\d+)?[ \t]*(?:--[^\r\n]*|/\*.*?\*/[ \t]*)?\r?$)",
+        4,
+        re.M,
+    ),
+    ("snowflake", r"\bVARIANT\b|\bLATERAL\s+FLATTEN\b|\bQUALIFY\b", 3, re.I),
+    # a notebook exported from Databricks announces itself on line one;
+    # dbdemos-notebooks was read as tsql without this
+    ("databricks", r"^-- Databricks notebook source\b", 6, re.M),
+    (
+        "databricks",
+        r"\bUSING\s+DELTA\b|\bTBLPROPERTIES\b|\bLIVE\s+TABLE\b|\bSTREAMING\s+TABLE\b",
+        3,
+        re.I,
+    ),
+    # stl_/svl_/svv_/stv_ system tables and DISTKEY/SORTKEY exist only on
+    # Redshift; dbt-labs/redshift has no profile and guessed snowflake
+    ("redshift", r"\b(stl|svl|svv|stv)_\w+", 3, re.I),
+    ("redshift", r"\bDISTKEY\b|\bSORTKEY\b|\bDISTSTYLE\b", 4, re.I),
+)
+
+# One backtick proves quoting style, not vendor: split evenly so a stronger
+# marker decides.
+_AMBIGUOUS_BACKTICK = ("bigquery", "mysql")
+
+_MIN_DIALECT_SCORE = 3
+
+
+def sniff_dialect(sql: str) -> str | None:
+    """Best-effort dialect from the SQL text itself, for repos with no dbt adapter
+    to declare one. Returns None when there's no strong signal (caller keeps its
+    default and says it assumed). Deliberately conservative: a wrong guess here is
+    what makes a real repo misparse.
+
+    Weighs every marker in the whole sample rather than returning on the first
+    hit. The caller assembles a slice of every file on purpose, so truncating
+    here threw that away: filecoin-data-portal's BigQuery markers sat past the
+    old 20,000 character cut and it was read as postgres.
+    """
+    if re.search(r"\\(pset|connect|echo|copy|d[itsv]?\b|c\b)", sql):
+        return "postgres"  # psql directives cannot appear in another dialect
+
+    scores: dict[str, int] = {}
+    for dialect, pattern, weight, flags in _DIALECT_MARKERS:
+        if re.search(pattern, sql, flags):
+            scores[dialect] = scores.get(dialect, 0) + weight
+    if "`" in sql:
+        for dialect in _AMBIGUOUS_BACKTICK:
+            scores[dialect] = scores.get(dialect, 0) + 1
+
+    if not scores:
+        return None
+    best = max(scores, key=lambda d: (scores[d], d))
+    if scores[best] < _MIN_DIALECT_SCORE:
+        return None
+    return best
+
+
+def _naive_split(sql: str):
+    """Split on semicolons that are not inside a string or comment. A fallback for
+    when sqlglot's own tokenizer refuses the whole blob."""
+    out, buf = [], []
+    i, n = 0, len(sql)
+    quote = None
+    while i < n:
+        c = sql[i]
+        if quote:
+            buf.append(c)
+            if c == quote:
+                quote = None
+            i += 1
+            continue
+        if c in ("'", '"', "`"):
+            quote = c
+            buf.append(c)
+        elif c == "-" and i + 1 < n and sql[i + 1] == "-":
+            j = sql.find("\n", i)
+            j = n if j == -1 else j
+            buf.append(sql[i:j])
+            i = j
+            continue
+        elif c == "/" and i + 1 < n and sql[i + 1] == "*":
+            j = sql.find("*/", i)
+            j = n if j == -1 else j + 2
+            buf.append(sql[i:j])
+            i = j
+            continue
+        elif c == ";":
+            out.append("".join(buf))
+            buf = []
+        else:
+            buf.append(c)
+        i += 1
+    if buf:
+        out.append("".join(buf))
+    return out
+
+
+def _flatten_blocks(
+    stmts: list[exp.Expression],
+    dialect: str | None,
+    _in_block: bool = False,
+    _conditional: bool = False,
+) -> list[exp.Expression]:
+    """Statements inside conditional/compound blocks are the file's real
+    content: the T-SQL deployment idiom IF NOT EXISTS (...) BEGIN CREATE
+    TABLE ... END guards deployment, not identity, and skipping it left
+    maintenance_solution with no models at all (holdout round 7). A
+    statement the guard parse left as a raw Command (often with the
+    block's stray END attached) gets one bounded re-parse.
+
+    Statements out of an IfBlock branch are marked ripple_conditional:
+    they may or may not run, so downstream reaching-definition analysis
+    must not let them kill or uniquely supply an earlier definition."""
+    out: list[exp.Expression] = []
+    for stmt in stmts:
+        block_cls = getattr(exp, "Block", None)
+        if_cls = getattr(exp, "IfBlock", None)
+        if if_cls is not None and isinstance(stmt, if_cls):
+            branches = [stmt.args.get("true"), stmt.args.get("false")]
+            inner = [b for b in branches if isinstance(b, exp.Expression)]
+            out.extend(_flatten_blocks(inner, dialect, _in_block=True, _conditional=True))
+        elif block_cls is not None and isinstance(stmt, block_cls):
+            out.extend(
+                _flatten_blocks(
+                    list(stmt.expressions or []),
+                    dialect,
+                    _in_block=True,
+                    _conditional=_conditional,
+                )
+            )
+        elif _in_block and isinstance(stmt, exp.Command):
+            raw = safe_sql(stmt, dialect)
+            if raw is None:
+                out.append(stmt)
+                continue
+            text = re.sub(r"\s*END\s*$", "", raw, flags=re.I)
+            try:
+                reparsed = sqlglot.parse_one(text, dialect=dialect, error_level=ErrorLevel.IGNORE)
+            except Exception:
+                reparsed = None
+            if reparsed is not None and not isinstance(reparsed, exp.Command):
+                out.append(reparsed)
+            else:
+                out.append(stmt)
+        else:
+            out.append(stmt)
+    if _conditional:
+        for stmt in out:
+            if stmt is not None:
+                stmt.meta["ripple_conditional"] = True
+    return out
+
+
+def split_statements(sql: str, dialect: str | None) -> list[exp.Expression]:
+    """Every parseable statement in the script, junk skipped, never raising."""
+    cleaned = _META_LINE.sub("", sql)
+    if (dialect or "").lower() == "tsql":
+        cleaned = _rewrite_go_separators(cleaned)
+    try:
+        parsed = sqlglot.parse(cleaned, dialect=dialect, error_level=ErrorLevel.IGNORE)
+        stmts = [p for p in parsed if p is not None]
+        if stmts:
+            return _flatten_blocks(stmts, dialect)
+    except Exception:
+        pass
+    out: list[exp.Expression] = []
+    for part in _naive_split(cleaned):
+        if not part.strip():
+            continue
+        try:
+            p = sqlglot.parse_one(part, dialect=dialect, error_level=ErrorLevel.IGNORE)
+            if p is not None:
+                out.append(p)
+        except Exception:
+            continue
+    return _flatten_blocks(out, dialect)
+
+
+def _table_name(node) -> str | None:
+    from ripple.engine.column_ref import temp_marked_name
+
+    if node is None:
+        return None
+    if isinstance(node, exp.Table):
+        return temp_marked_name(node)
+    if isinstance(node, (exp.Schema, exp.Identifier)):
+        return (
+            temp_marked_name(node) if isinstance(node, exp.Identifier) else _table_name(node.this)
+        )
+    name = getattr(node, "name", None)
+    return name or None
+
+
+_SEARCH_PATH_RE = re.compile(r"search_path\s*(?:=|to)\s*\"?([A-Za-z_][\w$]*)", re.I)
+_SEARCH_PATH_RESET_RE = re.compile(r"reset\s+search_path|search_path\s*(?:=|to)\s*default", re.I)
+
+
+def _qualified_target(node) -> str | None:
+    """The created/inserted relation's name WITH its qualifiers, when the SQL
+    writes any. The model keeps the bare name (round 0 proved renaming
+    regresses), but the qualified spelling must survive as an alias: openFEC's
+    migrations create disclosure.<view>, and holdout round 3 scored 12 correct
+    answers as misses because that spelling was recorded nowhere."""
+    if isinstance(node, exp.Schema):
+        node = node.this
+    if isinstance(node, exp.Table):
+        if not (node.catalog or node.db):
+            return None  # bare target: search_path (if any) supplies the schema
+        from ripple.engine.column_ref import qualified_table_name
+
+        qualified = qualified_table_name(node)
+        return qualified if qualified != node.name else None
+
+
+def _referenced_tables(select: exp.Expression) -> set[str]:
+    # temp/tablevar markers survive here too: parents recorded bare (a, b)
+    # while the models are #a/#b left star chains unresolvable (the
+    # review of cycle 7)
+    from ripple.engine.column_ref import temp_marked_name
+
+    return {temp_marked_name(t) for t in select.find_all(exp.Table) if t.name}
+
+
+def _create_columns(create: exp.Create) -> list[str]:
+    """Column names declared in a CREATE TABLE (...) definition, typed
+    (ColumnDef) or identifier-form as in CREATE TABLE x (a) AS ... (the
+    review of cycle 6: the VALUES CTAS lost its declared schema)."""
+    schema = create.this
+    cols: list[str] = []
+    if isinstance(schema, exp.Schema):
+        for c in schema.expressions:
+            if isinstance(c, (exp.ColumnDef, exp.Identifier, exp.Column)):
+                cols.append(c.name)
+    return cols
+
+
+def _is_schema_clone(inner) -> bool:
+    """A star-only SELECT that provably writes no rows: WHERE with no column
+    reference (0 = 1, false) or LIMIT 0. The loader idiom for cloning a
+    table's schema before INSERTing the real rows (usaspending's subaward
+    loader, holdout round 2). Only this exact shape may be replaced by a
+    later derivation for the same target; a plain star INSERT is real
+    lineage and must never be."""
+    if not isinstance(inner, exp.Select):
+        return False
+    selects = inner.selects
+    if not selects or not all(
+        isinstance(s, exp.Star) or (isinstance(s, exp.Column) and isinstance(s.this, exp.Star))
+        for s in selects
+    ):
+        return False
+    where = inner.args.get("where")
+    if where is not None and next(where.find_all(exp.Column), None) is None:
+        return True
+    limit = inner.args.get("limit")
+    if limit is not None:
+        lit = limit.expression
+        if isinstance(lit, exp.Literal) and lit.this == "0":
+            return True
+    return False
+
+
+def _realias_to_insert_columns(insert: exp.Expression, inner):
+    """INSERT INTO t (a, b) SELECT x, y: the table gains columns a and b, so
+    the derivation's projection is re-aliased to the target list. Pairwise,
+    only when the target list is all plain identifiers, the lengths match,
+    and no star is involved; otherwise the select is kept as written.
+    Identifier nodes are copied whole so quoting survives ("MixedCase").
+    usaspending's parent_award lost parent_award_id to the un-aliased
+    reading (holdout round 2). CREATE TABLE t (a, b) AS SELECT is the same
+    shape and takes the same rule (the cycle-6 review). Known
+    unhandled shapes, kept as written: UNION inners and postgres INSERT
+    INTO t AS alias (cols)."""
+    schema = insert.this
+    if not isinstance(schema, exp.Schema) or not isinstance(inner, exp.Select):
+        return inner
+    idents = schema.expressions
+    if not idents or not all(isinstance(c, exp.Identifier) for c in idents):
+        return inner
+    selects = inner.selects
+    if len(idents) != len(selects):
+        return inner
+    # only true star ITEMS (* or t.*) refuse the mapping; an aggregate whose
+    # argument is a star, COUNT_BIG(*), is an ordinary expression, and one of
+    # them disabled the whole #plan_creation column list (first_responder_kit,
+    # holdout round 6)
+    if any(
+        isinstance(s, exp.Star) or (isinstance(s, exp.Column) and isinstance(s.this, exp.Star))
+        for s in selects
+    ):
+        return inner
+    inner = inner.copy()
+    realiased = []
+    for item, ident in zip(inner.selects, idents, strict=True):
+        base = item.this if isinstance(item, exp.Alias) else item
+        realiased.append(exp.Alias(this=base, alias=ident.copy()))
+    inner.set("expressions", realiased)
+    return inner
+
+
+# One model discovered inside a script. `sql` is the derivation to analyze (a plain
+# SELECT), empty for a base table. `columns` are declared columns for a base table.
+class ScriptModel:
+    __slots__ = (
+        "name",
+        "sql",
+        "parents",
+        "columns",
+        "is_schema",
+        "schema_clone",
+        "qualified_name",
+        "self_read",
+        "is_function",
+    )
+
+    def __init__(
+        self,
+        name,
+        sql="",
+        parents=None,
+        columns=None,
+        is_schema=False,
+        schema_clone=False,
+        qualified_name=None,
+        self_read=False,
+        is_function=False,
+    ):
+        self.name = name
+        self.sql = sql
+        self.parents = parents or set()
+        self.columns = columns or []
+        self.is_schema = is_schema
+        self.schema_clone = schema_clone
+        self.qualified_name = qualified_name
+        # an UPDATE reads the table it writes; the self edge is prior
+        # state, not a CTE resolution loop
+        self.self_read = self_read
+        # minted from CREATE FUNCTION: a downstream call by this written
+        # name binds here instead of degrading as a relation collision
+        self.is_function = is_function
+
+
+def _update_models(
+    stmt: exp.Update, dialect: str | None, active_schema: str | None
+) -> list[ScriptModel]:
+    """UPDATE ... SET as derivations, one ScriptModel per written table.
+
+    nycdb builds every dataset as ALTER TABLE + UPDATE (holdout round 5).
+    The shapes come from the the review of PR #28: `UPDATE alias ... FROM
+    t alias` writes the aliased table, mysql's UPDATE t1, t2 writes each SET
+    qualifier's table, the statement's WITH and WHERE belong to the
+    synthesized derivation, and SET (a, b) = (subquery) maps by position.
+    """
+    if not isinstance(stmt.this, exp.Table):
+        return []
+
+    scope: dict[str, exp.Table] = {}
+    upd_from = stmt.args.get("from_") or stmt.args.get("from")
+    if upd_from is not None:
+        for t in upd_from.find_all(exp.Table):
+            scope.setdefault((t.alias or t.name).lower(), t)
+    for join in stmt.this.args.get("joins") or []:
+        for t in join.find_all(exp.Table):
+            scope.setdefault((t.alias or t.name).lower(), t)
+
+    primary = stmt.this
+    if not (primary.db or primary.catalog):
+        primary = scope.get(primary.name.lower(), primary)
+
+    def _positional_items(lhs: exp.Tuple, rhs: exp.Expression) -> list[exp.Alias]:
+        inner = rhs.this if isinstance(rhs, exp.Subquery) else None
+        if not isinstance(inner, exp.Select) or len(inner.selects) != len(lhs.expressions):
+            return []
+        items = []
+        for idx, target_col in enumerate(lhs.expressions):
+            if not isinstance(target_col, exp.Column):
+                continue
+            pruned = rhs.copy()
+            pruned.this.set("expressions", [inner.selects[idx].copy()])
+            items.append(exp.alias_(pruned, target_col.name))
+        return items
+
+    items_by_target: dict[str, tuple[exp.Table, list[exp.Alias]]] = {}
+
+    def _add(tnode: exp.Table, item: exp.Alias) -> None:
+        key = (tnode.alias or tnode.name).lower()
+        items_by_target.setdefault(key, (tnode, []))[1].append(item)
+
+    for eq in stmt.expressions:
+        if not isinstance(eq, exp.EQ) or eq.expression is None:
+            continue
+        lhs = eq.this
+        if isinstance(lhs, exp.Column):
+            tnode = scope.get(lhs.table.lower(), primary) if lhs.table else primary
+            _add(tnode, exp.alias_(eq.expression.copy(), lhs.name))
+        elif isinstance(lhs, exp.Tuple):
+            for item in _positional_items(lhs, eq.expression):
+                _add(primary, item)
+
+    base_tree = upd_from.this if upd_from is not None else stmt.this
+    models: list[ScriptModel] = []
+    for tnode, items in items_by_target.values():
+        target = _table_name(tnode)
+        if not target or not items:
+            continue
+        tree = base_tree.copy()
+        synthesized = exp.select(*items).from_(tree)
+        covered = {t.name.lower() for t in tree.find_all(exp.Table)}
+        if target.lower() not in covered:
+            bare = tnode.copy()
+            bare.set("joins", None)
+            synthesized = synthesized.join(bare)
+        where = stmt.args.get("where")
+        if where is not None:
+            synthesized.set("where", where.copy())
+        with_clause = stmt.args.get("with_") or stmt.args.get("with")
+        if with_clause is not None:
+            synthesized.set("with_", with_clause.copy())
+        synthesized_sql = safe_sql(synthesized, dialect)
+        if synthesized_sql is None:
+            continue
+        models.append(
+            ScriptModel(
+                name=target,
+                sql=synthesized_sql,
+                parents=_referenced_tables(synthesized),
+                qualified_name=_qualified_target(tnode)
+                or (f"{active_schema}.{target}" if active_schema else None),
+                self_read=True,
+            )
+        )
+    return models
+
+
+def expand_script(
+    file_stem: str, sql: str, dialect: str | None, warnings: list[str] | None = None
+) -> list[ScriptModel]:
+    """Turn one .sql file into the models it defines.
+
+    - CREATE VIEW / CREATE TABLE AS SELECT / INSERT INTO ... SELECT -> a model named
+      for the target, analyzed from its SELECT.
+    - CREATE TABLE (col defs) -> a base-table model carrying its declared columns
+      (the schema, so downstream SELECT * can resolve).
+    - a single bare SELECT with no target -> one model named after the file.
+    Everything else (DROP, SET, INSERT ... VALUES) contributes nothing.
+    `warnings`, when given, collects forfeits the caller should surface.
+    """
+    from ripple.engine.preprocess import prepare_sql_for_parse
+
+    # the same pre-parse encoding the statement path uses; parsing the raw
+    # text instead dropped or shredded jinja chain-path DML targets
+    # (cycle-13 review, F8)
+    sql, _ = prepare_sql_for_parse(sql, dialect or "")
+
+    models: list[ScriptModel] = []
+    if re.search(r"\bRETURNS\s+@[\w$]+\s+(?:as\s+)?TABLE\b", sql, re.I):
+        from ripple.engine.tsql_tvf import expand_table_functions
+
+        sql, tvf_models = expand_table_functions(sql, warnings=warnings)
+        models.extend(tvf_models)
+    statements = split_statements(sql, dialect)
+    bare_selects: list[exp.Expression] = []
+
+    tsql_scalars = "@" in sql and (dialect or "").lower() == "tsql"
+    if tsql_scalars:
+        from ripple.engine.tsql_scalar_vars import inline_scalar_params
+
+        inline_scalar_params(statements, dialect)
+
+    # SET search_path = disclosure, pg_catalog: everything created bare after
+    # it lives in that schema. openFEC's migrations create every view this
+    # way, and round 3 scored 12 correct answers as misses because the
+    # schema-qualified spelling existed nowhere on the model.
+    active_schema: str | None = None
+
+    for stmt in statements:
+        if isinstance(stmt, (exp.Set, exp.Command)):
+            # an ungenerable SET (tsql `SET @x -= 1`) forfeits only its own
+            # search_path sniff, never the rest of the script
+            text = safe_sql(stmt, dialect) or ""
+            if _SEARCH_PATH_RESET_RE.search(text):
+                # RESET search_path / SET search_path TO DEFAULT: a stale
+                # schema must not leak onto later creations
+                active_schema = None
+            else:
+                m = _SEARCH_PATH_RE.search(text)
+                if m:
+                    schema = m.group(1)
+                    active_schema = None if schema.lower() in ("pg_catalog", "public") else schema
+        if isinstance(stmt, exp.Create):
+            if "INDEX" in (stmt.kind or "").upper():
+                # an index is not a relation: minting a model for it made
+                # full_text.sql's stem five-way ambiguous (nycdb, holdout
+                # round 5 gap cycle). Substring match: tsql kinds read
+                # CLUSTERED INDEX / NONCLUSTERED INDEX (cycle-6 review)
+                continue
+            target = _table_name(stmt.this)
+            # CREATE TABLE x AS ( SELECT ... ): the paren wrapper must not
+            # reach the model's SQL, the statement layer refuses it as
+            # not-a-SELECT and the whole derivation vanishes (nycdb
+            # business_addrs, the real mechanism of round 5's class B)
+            inner = stmt.expression  # the SELECT for a view / CTAS, else None
+            while isinstance(inner, (exp.Subquery, exp.Paren)):
+                inner = inner.this
+            if target and isinstance(inner, (exp.Select, exp.Union)):
+                inner = _realias_to_insert_columns(stmt, inner)
+                inner_sql = safe_sql(inner, dialect)
+                if inner_sql is None:
+                    # the derivation exists but cannot be regenerated: the
+                    # model is forfeited outright. A schema-only stub here
+                    # would silently swallow downstream reads (cycle-12
+                    # review, F14).
+                    if warnings is not None:
+                        warnings.append(
+                            f"CREATE {target}: its SELECT body could not be "
+                            "regenerated; the model is forfeited"
+                        )
+                    continue
+                models.append(
+                    ScriptModel(
+                        name=target,
+                        sql=inner_sql,
+                        parents=_referenced_tables(inner),
+                        columns=_create_columns(stmt),
+                        schema_clone=_is_schema_clone(inner),
+                        qualified_name=_qualified_target(stmt.this)
+                        or (f"{active_schema}.{target}" if active_schema else None),
+                    )
+                )
+            elif target:
+                models.append(
+                    ScriptModel(
+                        name=target,
+                        columns=_create_columns(stmt),
+                        is_schema=True,
+                        qualified_name=_qualified_target(stmt.this)
+                        or (f"{active_schema}.{target}" if active_schema else None),
+                    )
+                )
+        elif isinstance(stmt, exp.Insert):
+            target = _table_name(stmt.this)
+            inner = stmt.expression
+            if target and isinstance(inner, (exp.Select, exp.Union)):
+                inner = _realias_to_insert_columns(stmt, inner)
+                inner_sql = safe_sql(inner, dialect)
+                if inner_sql is None:
+                    continue
+                models.append(
+                    ScriptModel(
+                        name=target,
+                        sql=inner_sql,
+                        parents=_referenced_tables(inner),
+                        schema_clone=_is_schema_clone(inner),
+                        qualified_name=_qualified_target(stmt.this)
+                        or (f"{active_schema}.{target}" if active_schema else None),
+                    )
+                )
+        elif isinstance(stmt, exp.Update):
+            # UPDATE t SET col = expr is a derivation of t: nycdb builds
+            # every dataset this way (ALTER ... ADD COLUMN + UPDATE), and
+            # round 5 scored all eleven of its cases as empty because no
+            # model existed. Synthesized as a SELECT so the ordinary
+            # engine walks the expressions.
+            models.extend(_update_models(stmt, dialect, active_schema))
+        elif isinstance(stmt, (exp.Select, exp.Union)):
+            # the INTO lives on the leftmost SELECT, including under a set
+            # operation (review: SELECT INTO ... UNION ALL ...)
+            head = stmt
+            while isinstance(head, exp.Union):
+                head = head.this
+            into = head.args.get("into") if isinstance(head, exp.Select) else None
+            target = _table_name(into.this) if into is not None else None
+            if target:
+                # SELECT ... INTO t is a creation of t (postgres); leaving it
+                # a bare select made the target read as a PARENT and left no
+                # model to resolve (nyc export.sql, holdout round 4)
+                body = stmt.copy()
+                body_head = body
+                while isinstance(body_head, exp.Union):
+                    body_head = body_head.this
+                body_head.set("into", None)
+                body_sql = safe_sql(body, dialect)
+                if body_sql is None:
+                    continue
+                models.append(
+                    ScriptModel(
+                        name=target,
+                        sql=body_sql,
+                        parents=_referenced_tables(body),
+                        schema_clone=_is_schema_clone(body),
+                        qualified_name=_qualified_target(into.this)
+                        or (f"{active_schema}.{target}" if active_schema else None),
+                    )
+                )
+            else:
+                bare_selects.append(stmt)
+
+    if tsql_scalars:
+        from ripple.engine.tsql_scalar_vars import scalar_value_insert_models
+
+        models.extend(scalar_value_insert_models(statements, dialect))
+
+    # a write into an @tablevar is procedure-local scratch, not a relation:
+    # minting models for them fabricated @loadedModules/@dm_os_memory_clerks
+    # and multiplied the file stem's owners until the column pick refused
+    # (sp_BlitzInMemoryOLTP, holdout round 6)
+    models = [m for m in models if not m.name.startswith("@")]
+    if not models and bare_selects:
+        # a plain query file: one model named for the file. The whole file is
+        # kept only when the query is the file's sole statement; a CREATE TEMP
+        # FUNCTION prelude (crux compliance-rates, holdout round 4) must not
+        # ride along, because multi-statement model SQL yields no edges.
+        body_sql = sql if len(statements) == 1 else safe_sql(bare_selects[-1], dialect)
+        if body_sql is not None:
+            models.append(
+                ScriptModel(
+                    name=file_stem, sql=body_sql, parents=_referenced_tables(bare_selects[-1])
+                )
+            )
+    return models