sql-lineage-extractor 0.1.1__py3-none-any.whl

sql_lineage/export/tables.py

@@ -0,0 +1,179 @@
+"""Derive the flat BI tables (as row dicts) from a manifest dict.
+
+Pure and backend-agnostic: this module never touches CSV / a database. It only
+turns the nested manifest into ordered lists of rows, so it is trivial to test
+and the same rows feed every backend.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from typing import Any, Callable
+
+# Ordered column layout per table. Backends rely on this order (CSV headers,
+# INSERT column lists) so the schema stays stable and diffable.
+TABLE_COLUMNS: dict[str, list[str]] = {
+    "dim_run": ["run_id", "generated_at", "schema_version", "n_nodes", "n_errors"],
+    "dim_object": ["run_id", "node_id", "origine", "dialecte", "layer", "n_columns"],
+    "dim_column": ["run_id", "node_id", "column_name", "expression_sql", "n_sources"],
+    "fact_column_edge": [
+        "run_id",
+        "target_node",
+        "target_column",
+        "source_table",
+        "source_column",
+        "expression_sql",
+    ],
+    "object_edge": ["run_id", "target_node", "depends_on_node"],
+    "closure": ["run_id", "ancestor_node", "descendant_node", "depth"],
+    "errors": ["run_id", "node_id", "origine", "message"],
+}
+
+# Columns that hold integers rather than text (matters for typed backends).
+INT_COLUMNS: set[str] = {"n_nodes", "n_errors", "n_columns", "n_sources", "depth"}
+
+# Medallion layers recognised by the default layer-derivation heuristic.
+KNOWN_LAYERS = ("raw", "bronze", "silver", "gold", "platinum")
+
+LayerFn = Callable[[str], str]
+
+
+def default_layer(node_id: str) -> str:
+    """Best-effort medallion layer from an id prefix; '' when unknown."""
+    head = node_id.split(".", 1)[0].split("_", 1)[0].lower()
+    return head if head in KNOWN_LAYERS else ""
+
+
+def build_export_tables(
+    manifest: dict[str, Any],
+    run_id: str | None = None,
+    layer_of: LayerFn = default_layer,
+) -> dict[str, list[dict[str, Any]]]:
+    """Return {table_name: [row, ...]} for the whole star schema.
+
+    ``run_id`` defaults to the manifest's ``generated_at`` so every row of a run
+    shares one stable, sortable key.
+    """
+    run = run_id or manifest.get("generated_at", "")
+    nodes: dict[str, Any] = manifest.get("nodes", {})
+    errors: list[Any] = manifest.get("errors", [])
+
+    dim_object: list[dict[str, Any]] = []
+    dim_column: list[dict[str, Any]] = []
+    fact_column_edge: list[dict[str, Any]] = []
+    object_edge: list[dict[str, Any]] = []
+
+    for node_id, node in nodes.items():
+        columns = node.get("columns", {})
+        dim_object.append(
+            {
+                "run_id": run,
+                "node_id": node_id,
+                "origine": node.get("origine", ""),
+                "dialecte": node.get("dialecte", ""),
+                "layer": layer_of(node_id),
+                "n_columns": len(columns),
+            }
+        )
+        for col_name, col in columns.items():
+            depends_on = col.get("depends_on", [])
+            dim_column.append(
+                {
+                    "run_id": run,
+                    "node_id": node_id,
+                    "column_name": col_name,
+                    "expression_sql": col.get("expression_sql", ""),
+                    "n_sources": len(depends_on),
+                }
+            )
+            for src in depends_on:
+                fact_column_edge.append(
+                    {
+                        "run_id": run,
+                        "target_node": node_id,
+                        "target_column": col_name,
+                        "source_table": src.get("table", ""),
+                        "source_column": src.get("column", ""),
+                        "expression_sql": col.get("expression_sql", ""),
+                    }
+                )
+        for dep in node.get("depends_on_nodes", []):
+            object_edge.append(
+                {"run_id": run, "target_node": node_id, "depends_on_node": dep}
+            )
+
+    closure = _closure_rows(run, object_edge)
+
+    dim_run = [
+        {
+            "run_id": run,
+            "generated_at": manifest.get("generated_at", run),
+            "schema_version": manifest.get("schema_version", ""),
+            "n_nodes": len(nodes),
+            "n_errors": len(errors),
+        }
+    ]
+
+    errors_rows = [
+        {
+            "run_id": run,
+            "node_id": e.get("id", ""),
+            "origine": e.get("origine", ""),
+            "message": e.get("message", ""),
+        }
+        for e in errors
+    ]
+
+    tables = {
+        "dim_run": dim_run,
+        "dim_object": dim_object,
+        "dim_column": dim_column,
+        "fact_column_edge": fact_column_edge,
+        "object_edge": object_edge,
+        "closure": closure,
+        "errors": errors_rows,
+    }
+    return {name: _sorted(name, rows) for name, rows in tables.items()}
+
+
+def _closure_rows(run: str, object_edge: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Transitive closure over object dependencies (BFS, cycle-safe).
+
+    An ``object_edge`` row says ``target_node`` depends on ``depends_on_node``,
+    so the dependency (upstream) is the *ancestor* and the consumer is the
+    *descendant*. We record every (ancestor, descendant, min-depth) pair.
+    """
+    down: dict[str, set[str]] = {}
+    for edge in object_edge:
+        ancestor = edge["depends_on_node"]
+        descendant = edge["target_node"]
+        down.setdefault(ancestor, set()).add(descendant)
+
+    rows: list[dict[str, Any]] = []
+    for ancestor in down:
+        best: dict[str, int] = {}
+        queue: deque[tuple[str, int]] = deque((d, 1) for d in down[ancestor])
+        while queue:
+            node, depth = queue.popleft()
+            if node in best and best[node] <= depth:
+                continue
+            best[node] = depth
+            for nxt in down.get(node, ()):  # noqa: SIM113
+                queue.append((nxt, depth + 1))
+        for descendant, depth in best.items():
+            if descendant == ancestor:
+                continue  # ignore self-loops from cyclic graphs
+            rows.append(
+                {
+                    "run_id": run,
+                    "ancestor_node": ancestor,
+                    "descendant_node": descendant,
+                    "depth": depth,
+                }
+            )
+    return rows
+
+
+def _sorted(name: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    cols = TABLE_COLUMNS[name]
+    return sorted(rows, key=lambda r: tuple(str(r[c]) for c in cols))

sql_lineage/graph/__init__.py

@@ -0,0 +1,3 @@
+from .builder import build_graph
+
+__all__ = ["build_graph"]

sql_lineage/graph/builder.py

@@ -0,0 +1,49 @@
+"""Global graph: link objects when one reads a table produced by another.
+
+Upstream tables are detected as qualified names (``bronze_navis.escale``) while
+object ids often come from file stems or notebook variables (``bronze_navis_escale``
+or ``escale``). Matching therefore normalizes dots to underscores and also
+allows a match on the table's final segment.
+"""
+
+from __future__ import annotations
+
+from ..models import ObjectNode
+
+
+def build_graph(nodes: list[ObjectNode]) -> list[ObjectNode]:
+    """Populate ``depends_on_nodes`` on each node in place, then return them."""
+    # canonical full form ("bronze_navis.escale" -> "bronze_navis_escale")
+    full_index: dict[str, str] = {}
+    # whole-id form, for matching a table's final segment against short ids
+    id_index: dict[str, str] = {}
+    for node in nodes:
+        full_index.setdefault(_canon(node.id), node.id)
+        id_index.setdefault(node.id.lower(), node.id)
+
+    for node in nodes:
+        deps: set[str] = set()
+        for table in node.source_tables:
+            target = _match(table, full_index, id_index)
+            if target is not None and target != node.id:
+                deps.add(target)
+        node.depends_on_nodes = sorted(deps)
+    return nodes
+
+
+def _canon(name: str) -> str:
+    return name.lower().replace(".", "_")
+
+
+def _match(
+    table: str, full_index: dict[str, str], id_index: dict[str, str]
+) -> str | None:
+    canon = _canon(table)
+    if canon in full_index:
+        return full_index[canon]
+    if table.lower() in id_index:
+        return id_index[table.lower()]
+    tail = table.rsplit(".", 1)[-1].lower()
+    if tail in id_index:
+        return id_index[tail]
+    return None

sql_lineage/manifest/__init__.py

@@ -0,0 +1,17 @@
+from .writer import (
+    SCHEMA_VERSION,
+    build_manifest,
+    dumps_manifest,
+    load_schema,
+    validate_manifest,
+    write_manifest,
+)
+
+__all__ = [
+    "SCHEMA_VERSION",
+    "build_manifest",
+    "dumps_manifest",
+    "load_schema",
+    "validate_manifest",
+    "write_manifest",
+]

sql_lineage/manifest/schema.json

@@ -0,0 +1,73 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://raw.githubusercontent.com/easytalents/sql-lineage-extractor/main/src/sql_lineage/manifest/schema.json",
+  "title": "sql-lineage manifest",
+  "description": "Contrat public du manifest de lineage. Le champ schema_version suit MAJOR.MINOR et évolue indépendamment de la version du package (voir docs/MANIFEST_SCHEMA.md).",
+  "type": "object",
+  "required": ["schema_version", "generated_at", "nodes", "errors"],
+  "additionalProperties": false,
+  "properties": {
+    "schema_version": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+$" },
+    "generated_at": { "type": "string" },
+    "nodes": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["origine", "columns", "depends_on_nodes"],
+        "additionalProperties": false,
+        "properties": {
+          "origine": { "type": "string" },
+          "dialecte": { "type": "string" },
+          "columns": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "object",
+              "required": ["expression_sql", "depends_on"],
+              "additionalProperties": false,
+              "properties": {
+                "expression_sql": { "type": "string" },
+                "depends_on": {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "required": ["table", "column"],
+                    "additionalProperties": false,
+                    "properties": {
+                      "table": { "type": "string" },
+                      "column": { "type": "string" }
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "depends_on_nodes": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "parametres_neutralises": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "avertissements": {
+            "type": "array",
+            "items": { "type": "string" }
+          }
+        }
+      }
+    },
+    "errors": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["id", "origine", "message"],
+        "additionalProperties": false,
+        "properties": {
+          "id": { "type": "string" },
+          "origine": { "type": "string" },
+          "message": { "type": "string" }
+        }
+      }
+    }
+  }
+}

sql_lineage/manifest/writer.py

@@ -0,0 +1,95 @@
+"""Serialize parsed objects + errors into the versioned manifest JSON.
+
+Determinism is a hard requirement: keys are sorted, lists are pre-sorted by the
+upstream layers, and the only non-reproducible field (``generated_at``) is
+injectable so callers/tests can pin it.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import json
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import jsonschema
+
+from ..models import ObjectNode, ParseError
+
+SCHEMA_VERSION = "1.0"
+
+_SCHEMA_PATH = Path(__file__).with_name("schema.json")
+
+
+@lru_cache(maxsize=1)
+def load_schema() -> dict[str, Any]:
+    return json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))
+
+
+def build_manifest(
+    nodes: list[ObjectNode],
+    errors: list[ParseError],
+    generated_at: str | None = None,
+) -> dict[str, Any]:
+    """Return the manifest as a plain dict, ready to be dumped or validated."""
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "generated_at": generated_at or _now_iso(),
+        "nodes": {node.id: _node_to_dict(node) for node in nodes},
+        "errors": [
+            {"id": e.id, "origine": e.origine, "message": e.message}
+            for e in sorted(errors, key=lambda e: (e.id, e.origine, e.message))
+        ],
+    }
+
+
+def _node_to_dict(node: ObjectNode) -> dict[str, Any]:
+    return {
+        "origine": node.origine,
+        "dialecte": node.dialecte,
+        "columns": {
+            col: {
+                "expression_sql": lin.expression_sql,
+                "depends_on": [
+                    {"table": s.table, "column": s.column} for s in lin.depends_on
+                ],
+            }
+            for col, lin in node.columns.items()
+        },
+        "depends_on_nodes": list(node.depends_on_nodes),
+        "parametres_neutralises": list(node.parametres_neutralises),
+        "avertissements": list(node.avertissements),
+    }
+
+
+def dumps_manifest(manifest: dict[str, Any]) -> str:
+    """Deterministic JSON text: sorted keys, stable spacing, trailing newline."""
+    return (
+        json.dumps(manifest, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
+    )
+
+
+def write_manifest(
+    path: str | Path,
+    nodes: list[ObjectNode],
+    errors: list[ParseError],
+    generated_at: str | None = None,
+) -> dict[str, Any]:
+    manifest = build_manifest(nodes, errors, generated_at=generated_at)
+    Path(path).write_text(dumps_manifest(manifest), encoding="utf-8")
+    return manifest
+
+
+def validate_manifest(manifest: dict[str, Any]) -> None:
+    """Raise jsonschema.ValidationError if the manifest is non-conforming."""
+    jsonschema.validate(instance=manifest, schema=load_schema())
+
+
+def _now_iso() -> str:
+    return (
+        _dt.datetime.now(tz=_dt.timezone.utc)
+        .replace(microsecond=0)
+        .isoformat()
+        .replace("+00:00", "Z")
+    )

sql_lineage/models.py

@@ -0,0 +1,71 @@
+"""Core data structures shared across every layer of the pipeline.
+
+These are intentionally plain dataclasses with no behaviour beyond a couple of
+serialization helpers, so each layer can be tested in isolation and the manifest
+writer stays the single source of truth for the JSON shape.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class SQLUnit:
+    """A single normalized SQL object, whatever its origin.
+
+    Every source (files, notebooks, live endpoint) produces these and nothing
+    else, so the parsing layer never needs to know where the SQL came from.
+    """
+
+    id: str
+    dialecte: str
+    sql_text: str
+    origine: str
+    # f-string / dynamic fragments that were replaced by a neutral token before
+    # handing the text to sqlglot. Kept for transparency in the manifest.
+    parametres_neutralises: list[str] = field(default_factory=list)
+    # Non-fatal warnings raised while building the unit (e.g. id fallback).
+    avertissements: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class ColumnSource:
+    """A single upstream (table, column) pair feeding a target column."""
+
+    table: str
+    column: str
+
+
+@dataclass
+class ColumnLineage:
+    """Lineage of one output column of an object."""
+
+    expression_sql: str
+    depends_on: list[ColumnSource] = field(default_factory=list)
+
+
+@dataclass
+class ObjectNode:
+    """A successfully parsed SQL object with its per-column lineage."""
+
+    id: str
+    origine: str
+    dialecte: str
+    columns: dict[str, ColumnLineage] = field(default_factory=dict)
+    parametres_neutralises: list[str] = field(default_factory=list)
+    avertissements: list[str] = field(default_factory=list)
+    # Filled by the graph builder: ids of other ObjectNodes this one reads from.
+    depends_on_nodes: list[str] = field(default_factory=list)
+    # Raw upstream table names detected while parsing (used by the graph builder
+    # to resolve depends_on_nodes). Not serialized directly.
+    source_tables: set[str] = field(default_factory=set)
+
+
+@dataclass
+class ParseError:
+    """A recoverable failure on a single object; never aborts the whole run."""
+
+    id: str
+    origine: str
+    message: str

sql_lineage/parsing/__init__.py

@@ -0,0 +1,4 @@
+from .engine import parse_unit
+from .lineage_extractor import extract_column_lineage
+
+__all__ = ["parse_unit", "extract_column_lineage"]

sql_lineage/parsing/engine.py

@@ -0,0 +1,166 @@
+"""Parsing engine: turn one SQLUnit into an ObjectNode or a ParseError.
+
+This layer never raises on malformed SQL — a failure on one object is captured
+and returned as a ParseError so the run keeps going for every other object.
+"""
+
+from __future__ import annotations
+
+import sqlglot
+from sqlglot import exp
+from sqlglot.errors import SqlglotError
+from sqlglot.optimizer.scope import Scope, build_scope
+
+from ..models import ColumnLineage, ObjectNode, ParseError, SQLUnit
+from .lineage_extractor import extract_column_lineage
+
+
+def parse_unit(unit: SQLUnit) -> ObjectNode | ParseError:
+    try:
+        return _parse_unit(unit)
+    except ParseUnitError as err:
+        return ParseError(id=unit.id, origine=unit.origine, message=str(err))
+    except SqlglotError as err:
+        return ParseError(id=unit.id, origine=unit.origine, message=f"Erreur sqlglot: {err}")
+    except Exception as err:  # defensive: never let one object abort the run
+        return ParseError(
+            id=unit.id,
+            origine=unit.origine,
+            message=f"Erreur inattendue ({type(err).__name__}): {err}",
+        )
+
+
+class ParseUnitError(Exception):
+    """Internal signal for an expected, recoverable parsing problem."""
+
+
+def _parse_unit(unit: SQLUnit) -> ObjectNode:
+    try:
+        tree = sqlglot.parse_one(unit.sql_text, dialect=unit.dialecte)
+    except SqlglotError as err:
+        raise ParseUnitError(f"SQL non analysable: {err}") from err
+
+    if tree is None:
+        raise ParseUnitError("SQL vide ou non analysable.")
+
+    select = tree if isinstance(tree, exp.Select) else tree.find(exp.Select)
+    if select is None:
+        raise ParseUnitError("Aucune instruction SELECT trouvée dans l'objet.")
+
+    target_columns = _target_columns(select)
+    if not target_columns:
+        raise ParseUnitError("Aucune colonne de sortie résoluble dans l'objet.")
+
+    node = ObjectNode(
+        id=unit.id,
+        origine=unit.origine,
+        dialecte=unit.dialecte,
+        parametres_neutralises=list(unit.parametres_neutralises),
+        avertissements=list(unit.avertissements),
+    )
+
+    for column in target_columns:
+        expression_sql, sources = extract_column_lineage(
+            column, unit.sql_text, unit.dialecte
+        )
+        node.columns[column] = ColumnLineage(
+            expression_sql=expression_sql,
+            depends_on=sources,
+        )
+        for src in sources:
+            node.source_tables.add(src.table)
+
+    return node
+
+
+def _target_columns(select: exp.Select) -> list[str]:
+    """Validate projections and return the list of resolvable output columns.
+
+    A ``SELECT *`` (or qualified ``t.*``) is expanded when its source columns are
+    statically knowable — i.e. the source is a CTE or a sub-query whose own
+    projection is enumerable (recursively). It is only rejected when the source is
+    a **physical table**, whose columns are unknown without a live schema. An
+    unaliased computed column is always rejected (no stable output name).
+    """
+    scope = _safe_build_scope(select)
+    columns: list[str] = []
+    for projection in select.expressions:
+        if _is_star(projection):
+            expanded = _expand_star(projection, scope) if scope is not None else None
+            if expanded is None:
+                raise ParseUnitError(
+                    "SELECT * non résoluble : la source est une table physique "
+                    "(colonnes inconnues en analyse statique) ; listez les colonnes "
+                    "explicitement, ou lisez depuis un CTE/sous-requête énumérable."
+                )
+            columns.extend(expanded)
+            continue
+        name = projection.output_name
+        if not name:
+            raise ParseUnitError(
+                "Colonne calculée sans alias détectée "
+                f"('{projection.sql()}') : un alias explicite est requis."
+            )
+        columns.append(name)
+    return columns
+
+
+def _is_star(projection: exp.Expression) -> bool:
+    if isinstance(projection, exp.Star):
+        return True
+    # qualified star such as t.*
+    return isinstance(projection, exp.Column) and isinstance(projection.this, exp.Star)
+
+
+def _safe_build_scope(select: exp.Select) -> Scope | None:
+    """Best-effort scope for star resolution; never raises."""
+    try:
+        return build_scope(select)
+    except Exception:  # pragma: no cover - defensive; falls back to rejecting stars
+        return None
+
+
+def _star_qualifier(projection: exp.Expression) -> str | None:
+    """Return the table qualifier of a ``t.*`` projection, or None for a bare ``*``."""
+    if isinstance(projection, exp.Column) and isinstance(projection.this, exp.Star):
+        return projection.table or None
+    return None
+
+
+def _expand_star(projection: exp.Expression, scope: Scope) -> list[str] | None:
+    """Column names a star expands to, or None if any source is unresolvable."""
+    qualifier = _star_qualifier(projection)
+    out: list[str] = []
+    matched = False
+    for alias, source in scope.sources.items():
+        if qualifier is not None and alias != qualifier:
+            continue
+        matched = True
+        if isinstance(source, Scope):
+            names = _scope_output_names(source)
+            if names is None:
+                return None
+            out.extend(names)
+        else:  # exp.Table -> physical table, columns unknown statically
+            return None
+    return out if matched else None
+
+
+def _scope_output_names(scope: Scope) -> list[str] | None:
+    """Output column names of a (CTE / sub-query) scope, expanding nested stars."""
+    expression = scope.expression
+    if not isinstance(expression, exp.Select):
+        return None  # set-operations / other shapes: not expanded
+    names: list[str] = []
+    for projection in expression.expressions:
+        if _is_star(projection):
+            expanded = _expand_star(projection, scope)
+            if expanded is None:
+                return None
+            names.extend(expanded)
+            continue
+        name = projection.output_name
+        if not name:
+            return None
+        names.append(name)
+    return names