datalex-cli 0.1.1__py3-none-any.whl

datalex_core/dbt/profiles.py

@@ -0,0 +1,185 @@
+"""Parse dbt profiles.yml to pick a warehouse connection for sync.
+
+A dbt project has a `profile:` key in `dbt_project.yml`; that name indexes into
+a `profiles.yml` (either in the project dir or `~/.dbt/profiles.yml`). Each
+profile has a default `target:` and a map of named targets to connection
+config. This module flattens that into a simple `(dialect, config)` tuple that
+`datalex_core.dbt.warehouse.introspect_table()` can consume.
+
+We deliberately do NOT import dbt itself. Users who only want to *try* DataLex
+shouldn't need to install dbt just to read their manifest.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+import yaml
+
+
+class ProfileError(RuntimeError):
+    """Raised when a profile is missing, malformed, or lacks a usable target."""
+
+
+@dataclass
+class ProfileTarget:
+    """Resolved target: what `warehouse.introspect_table()` needs."""
+
+    profile_name: str
+    target_name: str
+    dialect: str
+    config: Dict[str, Any]
+    database: Optional[str]
+    schema: Optional[str]
+
+
+def find_profiles_yml(
+    dbt_project_dir: Optional[str] = None,
+    explicit_path: Optional[str] = None,
+) -> Path:
+    """Locate profiles.yml using dbt's own precedence:
+
+      1. --profiles-dir / `explicit_path` (if provided)
+      2. DBT_PROFILES_DIR env var
+      3. `<dbt_project_dir>/profiles.yml`
+      4. `~/.dbt/profiles.yml`
+    """
+    if explicit_path:
+        p = Path(explicit_path).expanduser()
+        if p.is_dir():
+            p = p / "profiles.yml"
+        if not p.exists():
+            raise ProfileError(f"profiles.yml not found at: {p}")
+        return p
+
+    env_dir = os.environ.get("DBT_PROFILES_DIR")
+    if env_dir:
+        p = Path(env_dir).expanduser() / "profiles.yml"
+        if p.exists():
+            return p
+
+    if dbt_project_dir:
+        p = Path(dbt_project_dir) / "profiles.yml"
+        if p.exists():
+            return p
+
+    home = Path.home() / ".dbt" / "profiles.yml"
+    if home.exists():
+        return home
+
+    raise ProfileError(
+        "Could not find profiles.yml. Looked in: "
+        "--profiles-dir, $DBT_PROFILES_DIR, <project>/profiles.yml, ~/.dbt/profiles.yml"
+    )
+
+
+def read_dbt_project_profile_name(dbt_project_dir: str) -> str:
+    """Return the `profile:` key from dbt_project.yml."""
+    p = Path(dbt_project_dir) / "dbt_project.yml"
+    if not p.exists():
+        raise ProfileError(f"dbt_project.yml not found in {dbt_project_dir}")
+    with p.open("r", encoding="utf-8") as f:
+        proj = yaml.safe_load(f) or {}
+    name = proj.get("profile")
+    if not name:
+        raise ProfileError(f"dbt_project.yml at {p} is missing a `profile:` key")
+    return str(name)
+
+
+def resolve_target(
+    profiles_yml: Path,
+    profile_name: str,
+    target_override: Optional[str] = None,
+    base_dir: Optional[Path] = None,
+) -> ProfileTarget:
+    """Load profiles.yml, pick the named profile, and flatten the chosen target.
+
+    `base_dir` anchors relative paths (e.g. DuckDB `path:`) — typically the dbt
+    project directory. Defaults to the profiles.yml parent.
+    """
+    with profiles_yml.open("r", encoding="utf-8") as f:
+        doc = yaml.safe_load(f) or {}
+
+    profile = doc.get(profile_name)
+    if not isinstance(profile, dict):
+        raise ProfileError(
+            f"profile '{profile_name}' not found in {profiles_yml}. "
+            f"Available: {sorted(k for k in doc.keys() if k != 'config')}"
+        )
+
+    outputs = profile.get("outputs") or {}
+    target_name = target_override or profile.get("target")
+    if not target_name:
+        raise ProfileError(
+            f"profile '{profile_name}' has no default `target:` and no --profile override"
+        )
+
+    target = outputs.get(target_name)
+    if not isinstance(target, dict):
+        raise ProfileError(
+            f"target '{target_name}' not found in profile '{profile_name}'. "
+            f"Available: {sorted(outputs.keys())}"
+        )
+
+    dialect = str(target.get("type", "")).lower()
+    if not dialect:
+        raise ProfileError(
+            f"target '{target_name}' in profile '{profile_name}' is missing `type:`"
+        )
+
+    config = dict(target)
+    anchor = base_dir or profiles_yml.parent
+    if dialect == "duckdb":
+        raw_path = config.get("path") or config.get("database")
+        if raw_path:
+            rp = Path(str(raw_path)).expanduser()
+            if not rp.is_absolute():
+                rp = (anchor / rp).resolve()
+            config["path"] = str(rp)
+
+    return ProfileTarget(
+        profile_name=profile_name,
+        target_name=target_name,
+        dialect=dialect,
+        config=config,
+        database=config.get("database") or config.get("dbname") or config.get("catalog"),
+        schema=config.get("schema") or config.get("dataset"),
+    )
+
+
+def resolve_for_dbt_project(
+    dbt_project_dir: str,
+    profiles_dir: Optional[str] = None,
+    target_override: Optional[str] = None,
+) -> ProfileTarget:
+    """High-level: given a dbt project dir, resolve its active target.
+
+    Reads `dbt_project.yml` to find the profile name, then consults
+    `profiles.yml` to flatten the target.
+    """
+    profile_name = read_dbt_project_profile_name(dbt_project_dir)
+    path = find_profiles_yml(dbt_project_dir=dbt_project_dir, explicit_path=profiles_dir)
+    return resolve_target(
+        path,
+        profile_name,
+        target_override=target_override,
+        base_dir=Path(dbt_project_dir).resolve(),
+    )
+
+
+def as_introspect_args(
+    target: ProfileTarget,
+    database: Optional[str] = None,
+    schema: Optional[str] = None,
+    table: Optional[str] = None,
+) -> Tuple[str, Dict[str, Any], str, str, str]:
+    """Pack a resolved target + (db, schema, table) into the positional args
+    accepted by `warehouse.introspect_table()`. Falls back to the target's
+    default database/schema when a caller doesn't pass overrides."""
+    db = database or target.database or ""
+    sc = schema or target.schema or ""
+    tb = table or ""
+    return target.dialect, target.config, db, sc, tb

datalex_core/dbt/sync.py

@@ -0,0 +1,279 @@
+"""`dbt sync` orchestrator — the adoption-shaped one-command flow.
+
+Given a dbt project directory, sync pulls:
+  1. `target/manifest.json` (dbt compiles it with `dbt parse`; we just read it)
+  2. The warehouse columns for every source + model, via the active profile
+
+And merges them into a DataLex project tree:
+  * user-authored fields (descriptions, tags, sensitivity, tests, etc.) are
+    preserved — manifest round-trip semantics from phase B
+  * `data_type` on every column comes from the warehouse when we can reach it,
+    otherwise from the manifest, otherwise left blank
+  * on re-sync, the `meta.datalex.dbt.unique_id` stable key means we never
+    duplicate entities
+
+The flow is offline-safe: if the warehouse is unreachable (or the table hasn't
+been built yet), we degrade to manifest-only columns and annotate a warning.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from datalex_core.dbt.manifest import import_manifest, write_import_result
+from datalex_core.dbt.profiles import (
+    ProfileError,
+    ProfileTarget,
+    resolve_for_dbt_project,
+)
+from datalex_core.dbt.warehouse import (
+    WarehouseColumn,
+    WarehouseError,
+    introspect_table,
+)
+
+
+# ------------------------ report ------------------------
+
+
+@dataclass
+class TableSyncRecord:
+    unique_id: str
+    kind: str  # 'source' | 'model'
+    database: Optional[str]
+    schema: Optional[str]
+    table: str
+    warehouse_reachable: bool
+    columns_from_warehouse: int = 0
+    columns_from_manifest: int = 0
+    error: Optional[str] = None
+
+
+@dataclass
+class SyncReport:
+    dbt_project: str
+    datalex_root: str
+    profile_name: Optional[str] = None
+    target_name: Optional[str] = None
+    dialect: Optional[str] = None
+    tables: List[TableSyncRecord] = field(default_factory=list)
+    files_written: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+
+    def summary(self) -> str:
+        reached = sum(1 for t in self.tables if t.warehouse_reachable)
+        lines = [
+            "dbt sync complete",
+            f"  dbt project: {self.dbt_project}",
+            f"  DataLex out: {self.datalex_root}",
+            f"  profile:     {self.profile_name} / {self.target_name} ({self.dialect})",
+            f"  tables:      {len(self.tables)} "
+            f"({reached} from warehouse, {len(self.tables) - reached} manifest-only)",
+            f"  files:       {len(self.files_written)} written",
+        ]
+        if self.warnings:
+            lines.append("  warnings:")
+            for w in self.warnings:
+                lines.append(f"    - {w}")
+        return "\n".join(lines)
+
+
+# ------------------------ public entry point ------------------------
+
+
+def sync_dbt_project(
+    dbt_project_dir: str,
+    datalex_root: str,
+    *,
+    profiles_dir: Optional[str] = None,
+    target_override: Optional[str] = None,
+    skip_warehouse: bool = False,
+    manifest_path: Optional[str] = None,
+) -> SyncReport:
+    """Run the full sync: manifest -> DataLex, enriched by live warehouse types.
+
+    Args:
+        dbt_project_dir: Directory containing `dbt_project.yml` and
+            `target/manifest.json`.
+        datalex_root: Where to write the DataLex source/model YAML tree.
+        profiles_dir: Override for profiles.yml search (default: dbt's rules).
+        target_override: Pick a non-default target from the profile.
+        skip_warehouse: Skip live introspection; rely on manifest `data_type`.
+        manifest_path: Override `<dbt_project>/target/manifest.json`.
+    """
+    dbt_dir = Path(dbt_project_dir)
+    out_root = Path(datalex_root)
+    manifest = Path(manifest_path) if manifest_path else dbt_dir / "target" / "manifest.json"
+
+    if not manifest.exists():
+        raise FileNotFoundError(
+            f"manifest.json not found at {manifest}. "
+            f"Run `dbt parse` (or `dbt compile`) in the dbt project first."
+        )
+
+    report = SyncReport(dbt_project=str(dbt_dir), datalex_root=str(out_root))
+
+    # Step 1: parse manifest (merge-preserving re-import)
+    imported = import_manifest(str(manifest), existing_project_root=str(out_root))
+
+    # Step 2: resolve warehouse target (optional)
+    target: Optional[ProfileTarget] = None
+    if not skip_warehouse:
+        try:
+            target = resolve_for_dbt_project(
+                str(dbt_dir),
+                profiles_dir=profiles_dir,
+                target_override=target_override,
+            )
+            report.profile_name = target.profile_name
+            report.target_name = target.target_name
+            report.dialect = target.dialect
+        except ProfileError as e:
+            report.warnings.append(f"profile lookup failed — manifest-only sync: {e}")
+
+    # Step 3: introspect each source/model and enrich columns
+    for source_doc in imported.sources.values():
+        for table_doc in source_doc.get("tables", []) or []:
+            rec = _enrich_table(
+                table_doc,
+                database=source_doc.get("database"),
+                schema=source_doc.get("schema"),
+                target=target,
+                kind="source",
+            )
+            report.tables.append(rec)
+
+    for model_doc in imported.models.values():
+        rec = _enrich_table(
+            model_doc,
+            database=model_doc.get("database"),
+            schema=model_doc.get("schema"),
+            target=target,
+            kind="model",
+        )
+        report.tables.append(rec)
+
+    # Step 4: write the DataLex tree
+    report.files_written = write_import_result(imported, str(out_root))
+
+    return report
+
+
+# ------------------------ per-table enrichment ------------------------
+
+
+def _enrich_table(
+    table_doc: Dict[str, Any],
+    *,
+    database: Optional[str],
+    schema: Optional[str],
+    target: Optional[ProfileTarget],
+    kind: str,
+) -> TableSyncRecord:
+    uid = (
+        (table_doc.get("meta") or {})
+        .get("datalex", {})
+        .get("dbt", {})
+        .get("unique_id", "")
+    )
+    table_name = table_doc.get("identifier") or table_doc.get("name") or ""
+
+    rec = TableSyncRecord(
+        unique_id=uid,
+        kind=kind,
+        database=database,
+        schema=schema,
+        table=table_name,
+        warehouse_reachable=False,
+    )
+
+    manifest_cols = list(table_doc.get("columns") or [])
+    rec.columns_from_manifest = sum(1 for c in manifest_cols if c.get("type"))
+
+    if target is None or not schema or not table_name:
+        return rec
+
+    db = database or target.database or ""
+    try:
+        wh_cols = introspect_table(
+            dialect=target.dialect,
+            config=target.config,
+            database=db,
+            schema=schema,
+            table=table_name,
+        )
+    except WarehouseError as e:
+        rec.error = str(e)
+        return rec
+    except Exception as e:
+        rec.error = f"{type(e).__name__}: {e}"
+        return rec
+
+    rec.warehouse_reachable = True
+    merged = _merge_warehouse_into_columns(manifest_cols, wh_cols)
+    if merged:
+        table_doc["columns"] = merged
+    rec.columns_from_warehouse = len(wh_cols)
+    return rec
+
+
+def _merge_warehouse_into_columns(
+    manifest_cols: List[Dict[str, Any]],
+    wh_cols: List[WarehouseColumn],
+) -> List[Dict[str, Any]]:
+    """Warehouse = authoritative for type + nullability + order.
+    Manifest/prior DataLex doc = authoritative for everything else
+    (description, sensitivity, tags, tests, constraints, etc.)."""
+    by_name = {c.get("name"): dict(c) for c in manifest_cols if c.get("name")}
+    out: List[Dict[str, Any]] = []
+    for wh in wh_cols:
+        existing = by_name.pop(wh.name, {"name": wh.name})
+        existing["type"] = wh.data_type
+        if wh.nullable is False:
+            existing["nullable"] = False
+        elif "nullable" in existing and existing["nullable"] is True:
+            existing.pop("nullable")
+        if wh.description and "description" not in existing:
+            existing["description"] = wh.description
+        out.append(existing)
+
+    # Any manifest-only columns (e.g. view not yet materialized) keep their
+    # place at the end so we don't drop user-authored metadata.
+    for leftover in by_name.values():
+        out.append(leftover)
+    return out
+
+
+# ------------------------ lightweight JSON view ------------------------
+
+
+def report_to_json(report: SyncReport) -> str:
+    return json.dumps(
+        {
+            "dbt_project": report.dbt_project,
+            "datalex_root": report.datalex_root,
+            "profile_name": report.profile_name,
+            "target_name": report.target_name,
+            "dialect": report.dialect,
+            "tables": [
+                {
+                    "unique_id": t.unique_id,
+                    "kind": t.kind,
+                    "database": t.database,
+                    "schema": t.schema,
+                    "table": t.table,
+                    "warehouse_reachable": t.warehouse_reachable,
+                    "columns_from_warehouse": t.columns_from_warehouse,
+                    "columns_from_manifest": t.columns_from_manifest,
+                    "error": t.error,
+                }
+                for t in report.tables
+            ],
+            "files_written": report.files_written,
+            "warnings": report.warnings,
+        },
+        indent=2,
+    )

datalex_core/dbt/warehouse.py

@@ -0,0 +1,215 @@
+"""Warehouse introspection for dbt sync.
+
+Given a dialect + connection config + a (database, schema, table) triple,
+return the column list the warehouse actually has. Kept narrow on purpose —
+the existing connectors in datalex_core/connectors/ do full schema discovery; for
+sync we only need per-table column introspection so we can backfill types
+into DataLex files.
+
+Supported dialects (v1):
+  * duckdb    — file-based, no setup (the zero-friction demo path)
+  * postgres  — information_schema.columns (psycopg2)
+
+Other dialects fall back to the existing full-pull connector and filter.
+The fallback is slower but means `dbt sync` works against any warehouse that
+already has a connector implementation — users don't have to wait for us to
+ship a bespoke path.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class WarehouseColumn:
+    name: str
+    data_type: str
+    nullable: bool = True
+    description: Optional[str] = None
+
+
+class WarehouseError(RuntimeError):
+    """Raised when warehouse introspection fails or is unsupported."""
+
+
+def introspect_table(
+    dialect: str,
+    config: Dict[str, Any],
+    database: str,
+    schema: str,
+    table: str,
+) -> List[WarehouseColumn]:
+    """Return the live column list for one table.
+
+    `config` is a dbt-profile-shaped dict — e.g. `{path: "/tmp/db.duckdb"}` for
+    duckdb, `{host, port, user, password, dbname}` for postgres. Per-dialect
+    functions know how to pick the keys they need.
+    """
+    dialect = dialect.lower()
+    if dialect == "duckdb":
+        return _introspect_duckdb(config, database, schema, table)
+    if dialect in ("postgres", "postgresql"):
+        return _introspect_postgres(config, database, schema, table)
+    raise WarehouseError(
+        f"dialect '{dialect}' is not supported yet for `dbt sync`. "
+        f"Supported: duckdb, postgres. "
+        f"Open an issue or contribute a driver under datalex_core/dbt/warehouse.py."
+    )
+
+
+# ------------------------ DuckDB ------------------------
+
+
+def _introspect_duckdb(
+    config: Dict[str, Any],
+    database: str,
+    schema: str,
+    table: str,
+) -> List[WarehouseColumn]:
+    try:
+        import duckdb  # type: ignore
+    except ImportError as e:
+        raise WarehouseError(
+            "DuckDB driver not installed. Run: pip install duckdb"
+        ) from e
+
+    path = config.get("path") or config.get("database")
+    if not path:
+        raise WarehouseError("DuckDB profile needs a `path:` pointing at the .duckdb file.")
+
+    conn = duckdb.connect(str(path), read_only=True)
+    try:
+        # duckdb_columns() is the stable introspection view
+        rows = conn.execute(
+            """
+            SELECT column_name, data_type, is_nullable
+            FROM information_schema.columns
+            WHERE table_schema = ? AND table_name = ?
+            ORDER BY ordinal_position
+            """,
+            [schema, table],
+        ).fetchall()
+    finally:
+        conn.close()
+
+    return [
+        WarehouseColumn(
+            name=r[0],
+            data_type=_normalize_type(str(r[1])),
+            nullable=(str(r[2]).upper() == "YES"),
+        )
+        for r in rows
+    ]
+
+
+# ------------------------ Postgres ------------------------
+
+
+def _introspect_postgres(
+    config: Dict[str, Any],
+    database: str,
+    schema: str,
+    table: str,
+) -> List[WarehouseColumn]:
+    try:
+        import psycopg2  # type: ignore
+    except ImportError as e:
+        raise WarehouseError(
+            "Postgres driver not installed. Run: pip install psycopg2-binary"
+        ) from e
+
+    # dbt uses `dbname` (sometimes `database`) + `host`/`port`/`user`/`password`.
+    conn = psycopg2.connect(
+        host=config.get("host", "localhost"),
+        port=int(config.get("port", 5432)),
+        user=config.get("user") or config.get("username") or "",
+        password=config.get("password", ""),
+        dbname=config.get("dbname") or config.get("database") or database,
+    )
+    try:
+        cur = conn.cursor()
+        cur.execute(
+            """
+            SELECT column_name, data_type, is_nullable, col_description(
+                ('"' || table_schema || '"."' || table_name || '"')::regclass,
+                ordinal_position
+            )
+            FROM information_schema.columns
+            WHERE table_schema = %s AND table_name = %s
+            ORDER BY ordinal_position
+            """,
+            (schema, table),
+        )
+        rows = cur.fetchall()
+        cur.close()
+    finally:
+        conn.close()
+
+    return [
+        WarehouseColumn(
+            name=r[0],
+            data_type=_normalize_type(str(r[1])),
+            nullable=(str(r[2]).upper() == "YES"),
+            description=r[3] if r[3] else None,
+        )
+        for r in rows
+    ]
+
+
+# ------------------------ type normalization ------------------------
+
+
+_TYPE_ALIASES = {
+    "character varying": "string",
+    "varchar": "string",
+    "text": "string",
+    "character": "string",
+    "char": "string",
+    "double precision": "double",
+    "double": "double",
+    "real": "float",
+    "numeric": "decimal",
+    "integer": "int",
+    "int4": "int",
+    "int8": "bigint",
+    "bigint": "bigint",
+    "smallint": "smallint",
+    "int2": "smallint",
+    "boolean": "boolean",
+    "bool": "boolean",
+    "timestamp without time zone": "timestamp",
+    "timestamp with time zone": "timestamp_tz",
+    "timestamp": "timestamp",
+    "date": "date",
+    "time": "time",
+    "uuid": "uuid",
+    "json": "json",
+    "jsonb": "json",
+    "bytea": "binary",
+    "blob": "binary",
+    "decimal": "decimal",
+    "hugeint": "bigint",
+    "utinyint": "smallint",
+    "usmallint": "int",
+    "uinteger": "bigint",
+    "ubigint": "bigint",
+}
+
+
+def _normalize_type(raw: str) -> str:
+    """Fold warehouse-specific type names to the DataLex canonical palette.
+
+    Unknown types pass through unchanged — the DataLex layer is permissive
+    about types at the physical layer.
+    """
+    raw_l = raw.lower().strip()
+    if raw_l in _TYPE_ALIASES:
+        return _TYPE_ALIASES[raw_l]
+    # Preserve parametric types: "varchar(255)" → "string(255)"
+    if "(" in raw_l:
+        head, tail = raw_l.split("(", 1)
+        base = _TYPE_ALIASES.get(head.strip(), head.strip())
+        return f"{base}({tail}"
+    return raw_l

datalex_core/dialects/__init__.py

@@ -0,0 +1,15 @@
+"""Dialect plugin registry.
+
+Each SQL/NoSQL target engine ships as a module under this package implementing the
+DialectPlugin protocol in `base.py`. The registry in `registry.py` is the single
+entry point for code that wants to emit DDL or type-map without knowing which
+dialect is in play.
+
+Ports in Phase A: postgres, snowflake. The legacy monolithic `generators.py`
+remains available as a fallback and continues to serve bigquery/databricks/mysql/
+sqlserver until those are ported (Phase A task 5).
+"""
+
+from datalex_core.dialects import base, registry, postgres, snowflake  # noqa: F401
+
+__all__ = ["base", "registry"]