sql-lineage-extractor 0.1.1__py3-none-any.whl

sql_lineage/__init__.py

@@ -0,0 +1,30 @@
+"""sql_lineage — static column-to-column SQL lineage extractor.
+
+Pipeline: sources -> parsing/lineage -> graph -> manifest (JSON).
+"""
+
+from __future__ import annotations
+
+from .config import LineageConfig
+from .models import ColumnLineage, ColumnSource, ObjectNode, ParseError, SQLUnit
+
+__all__ = [
+    "SQLUnit",
+    "ColumnSource",
+    "ColumnLineage",
+    "ObjectNode",
+    "ParseError",
+    "LineageConfig",
+]
+
+
+def _resolve_version() -> str:
+    from importlib.metadata import PackageNotFoundError, version
+
+    try:
+        return version("sql-lineage-extractor")
+    except PackageNotFoundError:  # pragma: no cover - source checkout without install
+        return "0.0.0+unknown"
+
+
+__version__ = _resolve_version()

sql_lineage/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.1'
+__version_tuple__ = version_tuple = (0, 1, 1)
+
+__commit_id__ = commit_id = None

sql_lineage/cli.py

@@ -0,0 +1,217 @@
+"""Command-line interface (click): ``sql-lineage extract`` and ``validate``."""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+import click
+
+from .config import LineageConfig
+from .export import CsvExporter, SqlDbExporter, SqliteExporter, build_export_tables
+from .manifest import validate_manifest, write_manifest
+from .pipeline import run_extraction
+from .sources import FileSQLSource, LiveSQLEndpointSource, NotebookSQLSource
+
+
+@click.group()
+@click.version_option(package_name="sql-lineage-extractor")
+def cli() -> None:
+    """Static column-to-column SQL lineage extractor."""
+
+
+@cli.command()
+@click.option(
+    "--source",
+    "source_kind",
+    type=click.Choice(["files", "notebooks", "live"]),
+    required=True,
+    help="Where the SQL comes from.",
+)
+@click.option("--path", type=click.Path(), default=None, help="Folder to scan (files/notebooks).")
+@click.option(
+    "--dialect",
+    default=None,
+    help="sqlglot dialect (défaut: tsql, ou default_dialect du fichier de config).",
+)
+@click.option("--out", "out_path", type=click.Path(), required=True, help="Manifest output path.")
+@click.option("--conn-string", default=None, help="DB-API connection string (live mode).")
+@click.option("--glob", "glob_pattern", default=None, help="Override the file glob pattern.")
+@click.option(
+    "--config",
+    "config_path",
+    type=click.Path(exists=True),
+    default=None,
+    help="Fichier de conventions (.toml/.json) ; les flags ci-dessous le surchargent.",
+)
+@click.option(
+    "--id-comment-pattern",
+    default=None,
+    help="Regex du commentaire d'id (1 groupe de capture). Défaut: '-- lineage:id=(...)'.",
+)
+@click.option(
+    "--var-prefix",
+    "var_prefixes",
+    multiple=True,
+    help="Préfixe de variable à retirer pour dériver l'id (répétable). Défaut: 'df_'.",
+)
+@click.option(
+    "--param-token",
+    default=None,
+    help="Jeton de neutralisation des interpolations f-string. Défaut: '__PARAM__'.",
+)
+def extract(
+    source_kind: str,
+    path: str | None,
+    dialect: str | None,
+    out_path: str,
+    conn_string: str | None,
+    glob_pattern: str | None,
+    config_path: str | None,
+    id_comment_pattern: str | None,
+    var_prefixes: tuple[str, ...],
+    param_token: str | None,
+) -> None:
+    """Extract lineage and write a manifest JSON."""
+    try:
+        config = _build_config(
+            config_path, id_comment_pattern, var_prefixes, param_token, dialect
+        )
+    except (ValueError, RuntimeError) as err:
+        raise click.UsageError(f"Configuration invalide: {err}") from err
+
+    source = _build_source(source_kind, path, conn_string, glob_pattern, config)
+    nodes, errors = run_extraction(source)
+    write_manifest(out_path, nodes, errors)
+
+    click.echo(
+        f"{len(nodes)} objet(s) analysé(s), {len(errors)} erreur(s) -> {out_path}"
+    )
+    for parse_err in errors:
+        click.echo(
+            f"  ! {parse_err.id} ({parse_err.origine}): {parse_err.message}", err=True
+        )
+
+
+@cli.command()
+@click.argument("manifest_path", type=click.Path(exists=True))
+def validate(manifest_path: str) -> None:
+    """Validate a manifest against the published JSON Schema."""
+    manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8"))
+    try:
+        validate_manifest(manifest)
+    except Exception as err:  # jsonschema.ValidationError and friends
+        click.echo(f"INVALIDE: {err}", err=True)
+        sys.exit(1)
+    click.echo("VALIDE: le manifest est conforme au schéma.")
+
+
+@cli.command()
+@click.argument("manifest_path", type=click.Path(exists=True))
+@click.option(
+    "--format",
+    "fmt",
+    type=click.Choice(["csv", "sqlite", "sqlserver"]),
+    required=True,
+    help="Backend cible pour les tables BI.",
+)
+@click.option(
+    "--out", "out_path", type=click.Path(), default=None,
+    help="Dossier (csv) ou fichier (sqlite).",
+)
+@click.option("--conn-string", default=None, help="Connexion DB-API (format sqlserver).")
+@click.option(
+    "--run-id", default=None,
+    help="Identifiant de run (défaut: generated_at du manifest).",
+)
+def export(
+    fmt: str,
+    manifest_path: str,
+    out_path: str | None,
+    conn_string: str | None,
+    run_id: str | None,
+) -> None:
+    """Dériver les tables BI (étoile) d'un manifest vers CSV / SQLite / SQL Server."""
+    manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8"))
+    tables = build_export_tables(manifest, run_id=run_id)
+
+    if fmt == "csv":
+        if not out_path:
+            raise click.UsageError("--out (dossier) est requis pour --format csv.")
+        CsvExporter(out_path).export(tables)
+        target = out_path
+    elif fmt == "sqlite":
+        if not out_path:
+            raise click.UsageError("--out (fichier .db) est requis pour --format sqlite.")
+        with SqliteExporter(out_path) as sqlite_exporter:
+            sqlite_exporter.export(tables)
+        target = out_path
+    else:  # sqlserver
+        if not conn_string:
+            raise click.UsageError("--conn-string est requis pour --format sqlserver.")
+        SqlDbExporter(_open_pyodbc(conn_string)).export(tables)
+        target = "<connexion SQL>"
+
+    counts = ", ".join(f"{name}={len(rows)}" for name, rows in tables.items())
+    click.echo(f"Tables exportées vers {target} : {counts}")
+
+
+def _build_config(
+    config_path: str | None,
+    id_comment_pattern: str | None,
+    var_prefixes: tuple[str, ...],
+    param_token: str | None,
+    dialect: str | None,
+) -> LineageConfig:
+    """Build the effective config: file (if any), then per-flag overrides."""
+    base = LineageConfig.from_file(config_path) if config_path else LineageConfig()
+    overrides: dict[str, Any] = {}
+    if id_comment_pattern is not None:
+        overrides["id_comment_pattern"] = id_comment_pattern
+    if var_prefixes:  # non-empty tuple => the flag was provided
+        overrides["var_prefixes"] = tuple(var_prefixes)
+    if param_token is not None:
+        overrides["param_token"] = param_token
+    if dialect is not None:
+        overrides["default_dialect"] = dialect
+    return dataclasses.replace(base, **overrides) if overrides else base
+
+
+def _build_source(
+    source_kind: str,
+    path: str | None,
+    conn_string: str | None,
+    glob_pattern: str | None,
+    config: LineageConfig,
+):
+    if source_kind == "files":
+        if not path:
+            raise click.UsageError("--path est requis pour --source files.")
+        return FileSQLSource(path, pattern=glob_pattern or "**/*.sql", config=config)
+    if source_kind == "notebooks":
+        if not path:
+            raise click.UsageError("--path est requis pour --source notebooks.")
+        return NotebookSQLSource(path, pattern=glob_pattern or "**/*.ipynb", config=config)
+    if source_kind == "live":
+        if not conn_string:
+            raise click.UsageError("--conn-string est requis pour --source live.")
+        return LiveSQLEndpointSource(_open_pyodbc(conn_string), config=config)
+    raise click.UsageError(f"Source inconnue: {source_kind}")
+
+
+def _open_pyodbc(conn_string: str):
+    try:
+        import pyodbc  # noqa: PLC0415 - optional dependency, imported lazily
+    except ImportError as err:  # pragma: no cover - depends on optional extra
+        raise click.UsageError(
+            "Le mode live nécessite l'extra 'live' (pip install "
+            "sql-lineage-extractor[live])."
+        ) from err
+    return pyodbc.connect(conn_string)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    cli()

sql_lineage/config.py

@@ -0,0 +1,145 @@
+"""Configuration of the *conventions* the extractor relies on.
+
+The library must stay **generic**: nothing project- or client-specific may be
+hard-coded. Every convention used to resolve a notebook object's id, to strip a
+dataframe-variable prefix, to neutralize f-string interpolations, or to pick a
+default SQL dialect lives here, in an immutable :class:`LineageConfig` that the
+caller can override — programmatically, via CLI flags, or from a TOML/JSON file.
+
+The defaults reproduce the historical behaviour exactly, so existing callers are
+unaffected.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Mapping
+
+# -- historical defaults (reproduce the previous hard-coded behaviour) ---------
+
+#: Convention comment naming a unit explicitly, e.g. ``-- lineage:id=silver_x``.
+DEFAULT_ID_COMMENT_PATTERN = r"--\s*lineage:id\s*=\s*([A-Za-z0-9_.]+)"
+#: Dataframe-variable prefixes stripped when deriving an id (``df_x`` -> ``x``).
+DEFAULT_VAR_PREFIXES: tuple[str, ...] = ("df_",)
+#: Token substituted for every f-string interpolation before parsing.
+DEFAULT_PARAM_TOKEN = "__PARAM__"
+#: Dialect used when a source is created without an explicit one.
+DEFAULT_DIALECT = "tsql"
+
+#: Keys accepted in a config mapping / file, mapped to the dataclass fields.
+_CONFIG_KEYS = frozenset(
+    {"id_comment_pattern", "var_prefixes", "param_token", "default_dialect"}
+)
+
+
+@dataclass(frozen=True)
+class LineageConfig:
+    """Immutable set of conventions driving id resolution and neutralization.
+
+    All fields are optional; each defaults to the historical behaviour.
+    """
+
+    id_comment_pattern: str = DEFAULT_ID_COMMENT_PATTERN
+    var_prefixes: tuple[str, ...] = DEFAULT_VAR_PREFIXES
+    param_token: str = DEFAULT_PARAM_TOKEN
+    default_dialect: str = DEFAULT_DIALECT
+    #: Compiled form of ``id_comment_pattern`` (built once, never part of eq/repr).
+    id_comment_re: "re.Pattern[str]" = field(init=False, compare=False, repr=False)
+
+    def __post_init__(self) -> None:
+        # Normalize a list/other iterable of prefixes to a tuple (hashable, frozen).
+        object.__setattr__(self, "var_prefixes", tuple(self.var_prefixes))
+
+        try:
+            compiled = re.compile(self.id_comment_pattern)
+        except re.error as err:
+            raise ValueError(f"id_comment_pattern invalide: {err}") from err
+        if compiled.groups < 1:
+            raise ValueError(
+                "id_comment_pattern doit contenir au moins un groupe de capture "
+                "délimitant l'id (ex. '-- lineage:id=(...)')."
+            )
+        object.__setattr__(self, "id_comment_re", compiled)
+
+        if not self.param_token:
+            raise ValueError("param_token ne peut pas être vide.")
+        if not self.default_dialect:
+            raise ValueError("default_dialect ne peut pas être vide.")
+
+    # -- id-resolution helpers (shared by notebook and file sources) ----------
+
+    def id_from_comment(self, text: str) -> str | None:
+        """Return the id declared by an ``-- lineage:id=`` comment, if any."""
+        match = self.id_comment_re.search(text)
+        return match.group(1) if match else None
+
+    def strip_var_prefix(self, var_name: str) -> str:
+        """Strip the first matching dataframe-variable prefix (``df_x`` -> ``x``)."""
+        for prefix in self.var_prefixes:
+            if prefix and var_name.startswith(prefix):
+                return var_name[len(prefix) :]
+        return var_name
+
+    # -- constructors ---------------------------------------------------------
+
+    @classmethod
+    def from_mapping(cls, data: Mapping[str, Any]) -> "LineageConfig":
+        """Build a config from a plain mapping, rejecting unknown keys."""
+        unknown = set(data) - _CONFIG_KEYS
+        if unknown:
+            raise ValueError(
+                f"Clé(s) de configuration inconnue(s): {sorted(unknown)}. "
+                f"Clés valides: {sorted(_CONFIG_KEYS)}."
+            )
+        kwargs: dict[str, Any] = dict(data)
+        if kwargs.get("var_prefixes") is not None:
+            kwargs["var_prefixes"] = tuple(kwargs["var_prefixes"])
+        return cls(**kwargs)
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> "LineageConfig":
+        """Load a config from a ``.toml`` or ``.json`` file.
+
+        The settings may sit at the top level, under a ``[sql_lineage]`` table,
+        or under ``[tool.sql_lineage]`` (so a shared ``pyproject.toml`` works).
+        """
+        path = Path(path)
+        text = path.read_text(encoding="utf-8")
+        suffix = path.suffix.lower()
+        if suffix == ".json":
+            import json
+
+            data: Any = json.loads(text)
+        elif suffix == ".toml":
+            data = _load_toml(text)
+        else:
+            raise ValueError(
+                f"Format de configuration non supporté: '{suffix}' "
+                "(attendu .toml ou .json)."
+            )
+
+        if not isinstance(data, dict):
+            raise ValueError("Le fichier de configuration doit contenir une table.")
+        # Unwrap the conventional nesting if present.
+        tool = data.get("tool")
+        if isinstance(tool, dict) and isinstance(tool.get("sql_lineage"), dict):
+            data = tool["sql_lineage"]
+        elif isinstance(data.get("sql_lineage"), dict):
+            data = data["sql_lineage"]
+        return cls.from_mapping(data)
+
+
+def _load_toml(text: str) -> dict[str, Any]:
+    try:
+        import tomllib  # Python >= 3.11
+    except ModuleNotFoundError:  # pragma: no cover - only on 3.10
+        try:
+            import tomli as tomllib  # type: ignore[no-redef]
+        except ModuleNotFoundError as err:  # pragma: no cover
+            raise RuntimeError(
+                "La lecture d'un fichier TOML requiert Python 3.11+ ou le paquet "
+                "'tomli' (installé automatiquement sur 3.10 via l'extra du package)."
+            ) from err
+    return tomllib.loads(text)

sql_lineage/export/__init__.py

@@ -0,0 +1,29 @@
+"""BI serving layer: derive flat star-schema tables from a manifest.
+
+The manifest JSON stays the canonical, diffable source of truth. This package
+*derives* tabular tables from it (a pure, replayable step) and writes them to a
+pluggable backend: CSV, SQLite, or any injected DB-API connection (SQL Server /
+Fabric Warehouse).
+
+Tables (all carry ``run_id`` so runs are appended, never overwritten):
+
+* ``dim_run``          one row per extraction run
+* ``dim_object``       one row per SQL object
+* ``dim_column``       one row per output column
+* ``fact_column_edge`` central fact: one row per source->target column edge
+* ``object_edge``      object-level direct dependency (1 hop)
+* ``closure``          transitive ancestor->descendant reachability (N hops)
+* ``errors``           one row per parse error
+"""
+
+from .backends import CsvExporter, SqlDbExporter, SqliteExporter
+from .tables import TABLE_COLUMNS, build_export_tables, default_layer
+
+__all__ = [
+    "build_export_tables",
+    "default_layer",
+    "TABLE_COLUMNS",
+    "CsvExporter",
+    "SqliteExporter",
+    "SqlDbExporter",
+]

sql_lineage/export/backends.py

@@ -0,0 +1,122 @@
+"""Pluggable export backends: CSV files, a SQLite file, or an injected DB-API
+connection (SQL Server / Fabric Warehouse).
+
+All backends are *append by run*: writing a run adds its rows. The database
+backends first delete any existing rows for the same ``run_id`` so re-running a
+given run is idempotent; the CSV backend appends (it never rewrites history).
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Any
+
+from .tables import INT_COLUMNS, TABLE_COLUMNS
+
+
+class CsvExporter:
+    """Write one ``<table>.csv`` per table under ``out_dir`` (append mode)."""
+
+    def __init__(self, out_dir: str | Path) -> None:
+        self.out_dir = Path(out_dir)
+
+    def export(self, tables: dict[str, list[dict[str, Any]]]) -> None:
+        self.out_dir.mkdir(parents=True, exist_ok=True)
+        for name, cols in TABLE_COLUMNS.items():
+            path = self.out_dir / f"{name}.csv"
+            write_header = not path.exists()
+            with path.open("a", newline="", encoding="utf-8") as fh:
+                writer = csv.DictWriter(fh, fieldnames=cols)
+                if write_header:
+                    writer.writeheader()
+                for row in tables.get(name, []):
+                    writer.writerow({c: row.get(c, "") for c in cols})
+
+
+class _DbApiExporter:
+    """Shared logic for any DB-API connection using ``?`` (qmark) parameters."""
+
+    # Backends override the column type used in CREATE TABLE.
+    TEXT_TYPE = "TEXT"
+    INT_TYPE = "INTEGER"
+
+    def __init__(self, connection: Any) -> None:
+        self.connection = connection
+
+    def export(self, tables: dict[str, list[dict[str, Any]]]) -> None:
+        cursor = self.connection.cursor()
+        try:
+            for name, cols in TABLE_COLUMNS.items():
+                self._create_table(cursor, name, cols)
+                rows = tables.get(name, [])
+                self._delete_runs(cursor, name, rows)
+                self._insert_rows(cursor, name, cols, rows)
+            self.connection.commit()
+        finally:
+            close = getattr(cursor, "close", None)
+            if callable(close):
+                close()
+
+    # -- overridable DDL ------------------------------------------------------
+
+    def _create_table(self, cursor: Any, name: str, cols: list[str]) -> None:
+        defs = ", ".join(f"{c} {self._col_type(c)}" for c in cols)
+        cursor.execute(f"CREATE TABLE IF NOT EXISTS {name} ({defs})")
+
+    def _col_type(self, col: str) -> str:
+        return self.INT_TYPE if col in INT_COLUMNS else self.TEXT_TYPE
+
+    # -- data -----------------------------------------------------------------
+
+    def _delete_runs(self, cursor: Any, name: str, rows: list[dict[str, Any]]) -> None:
+        run_ids = {r["run_id"] for r in rows}
+        for run_id in run_ids:
+            cursor.execute(f"DELETE FROM {name} WHERE run_id = ?", (run_id,))
+
+    def _insert_rows(
+        self, cursor: Any, name: str, cols: list[str], rows: list[dict[str, Any]]
+    ) -> None:
+        if not rows:
+            return
+        placeholders = ", ".join("?" for _ in cols)
+        sql = f"INSERT INTO {name} ({', '.join(cols)}) VALUES ({placeholders})"
+        cursor.executemany(sql, [tuple(r.get(c) for c in cols) for r in rows])
+
+
+class SqliteExporter(_DbApiExporter):
+    """Write to a single SQLite file (stdlib ``sqlite3``, no dependency)."""
+
+    def __init__(self, db_path: str | Path) -> None:
+        import sqlite3
+
+        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        super().__init__(sqlite3.connect(str(db_path)))
+
+    def close(self) -> None:
+        self.connection.close()
+
+    def __enter__(self) -> "SqliteExporter":
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+
+class SqlDbExporter(_DbApiExporter):
+    """Write to an injected DB-API connection (SQL Server / Fabric Warehouse).
+
+    The connection is provided by the caller (auth and driver are out of scope),
+    exactly like :class:`~sql_lineage.sources.LiveSQLEndpointSource`.
+    """
+
+    TEXT_TYPE = "NVARCHAR(4000)"
+    INT_TYPE = "INT"
+
+    def _create_table(self, cursor: Any, name: str, cols: list[str]) -> None:
+        # T-SQL has no CREATE TABLE IF NOT EXISTS before recent versions.
+        defs = ", ".join(f"{c} {self._col_type(c)}" for c in cols)
+        cursor.execute(
+            f"IF OBJECT_ID(N'{name}', N'U') IS NULL "
+            f"CREATE TABLE {name} ({defs})"
+        )