datalex-cli 1.0.3__tar.gz → 1.0.4__tar.gz

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datalex-cli
-Version: 1.0.3
+Version: 1.0.4
 Summary: Git-native data modeling for dbt users
 Author-email: DuckCode AI Labs <hello@duckcode.ai>
 License: MIT

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/datalex_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datalex-cli
-Version: 1.0.3
+Version: 1.0.4
 Summary: Git-native data modeling for dbt users
 Author-email: DuckCode AI Labs <hello@duckcode.ai>
 License: MIT

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/datalex_cli.egg-info/SOURCES.txt

@@ -1113,6 +1113,7 @@ packages/core_engine/src/datalex_core/datalex/parse_cache.py
 packages/core_engine/src/datalex_core/datalex/project.py
 packages/core_engine/src/datalex_core/datalex/types.py
 packages/core_engine/src/datalex_core/dbt/__init__.py
+packages/core_engine/src/datalex_core/dbt/catalog.py
 packages/core_engine/src/datalex_core/dbt/emit.py
 packages/core_engine/src/datalex_core/dbt/manifest.py
 packages/core_engine/src/datalex_core/dbt/profiles.py

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/packages/cli/src/datalex_cli/datalex_cli.py

@@ -462,9 +462,17 @@ def _cmd_dbt_emit(args: argparse.Namespace) -> int:
 
 
 def _cmd_dbt_import(args: argparse.Namespace) -> int:
+    # Auto-discover catalog.json sibling to the manifest — `dbt docs generate`
+    # writes it to the same `target/` folder and it carries warehouse types
+    # even when manifest.data_type is empty.
+    from pathlib import Path as _Path
+    manifest_p = _Path(args.manifest)
+    catalog_candidate = manifest_p.parent / "catalog.json"
+    catalog_path = str(catalog_candidate) if catalog_candidate.exists() else None
     result = import_manifest(
         args.manifest,
         existing_project_root=args.merge_from or args.out_root,
+        catalog_path=catalog_path,
     )
     written = write_import_result(result, args.out_root)
     print(

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/packages/core_engine/src/datalex_core/_server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "datalex-api-server",
   "private": true,
-  "version": "1.0.3",
+  "version": "1.0.4",
   "type": "module",
   "scripts": {
     "start": "node index.js",

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/packages/core_engine/src/datalex_core/dbt/__init__.py

@@ -3,6 +3,7 @@
 from datalex_core.dbt.emit import emit_dbt, build_sources_yaml, build_models_yaml, EmitReport
 from datalex_core.dbt.manifest import import_manifest, write_import_result, ImportResult
 from datalex_core.dbt.sync import sync_dbt_project, SyncReport, TableSyncRecord
+from datalex_core.dbt.catalog import CatalogIndex, load_catalog, default_catalog_path
 
 __all__ = [
     "emit_dbt",
@@ -15,4 +16,7 @@ __all__ = [
     "sync_dbt_project",
     "SyncReport",
     "TableSyncRecord",
+    "CatalogIndex",
+    "load_catalog",
+    "default_catalog_path",
 ]

datalex_cli-1.0.4/packages/core_engine/src/datalex_core/dbt/catalog.py

@@ -0,0 +1,98 @@
+"""dbt catalog.json -> per-column data_type lookup.
+
+Populated by `dbt docs generate`, `catalog.json` contains the actual column
+types introspected from the warehouse — the same information `dbt compile`
+tries to fill into `manifest.json`'s `columns[].data_type`, but persisted
+even when dbt-compile didn't manage to populate it.
+
+We use this as a *fallback* type source after the manifest: if a column has
+`data_type` in the manifest we keep it (cheaper, and user has already opted
+into compile-time types), otherwise we look here before falling back to
+user-authored types or "unknown". This lets users who only run
+`dbt docs generate` (common in projects that rely on the dbt docs site) still
+get real types into DataLex without a live warehouse connection.
+
+The file layout is:
+    {
+      "nodes":   { "<unique_id>": { "columns": { "<col>": { "type": "..." } } } },
+      "sources": { "<unique_id>": { "columns": { "<col>": { "type": "..." } } } }
+    }
+
+Column lookups are case-insensitive on the column name, because dbt may
+lowercase/uppercase names depending on the warehouse (Snowflake uppercases
+identifiers; Postgres folds to lowercase). Looking up by the exact case the
+manifest uses wouldn't find the catalog entry on those warehouses.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Optional
+
+
+@dataclass
+class CatalogIndex:
+    """In-memory index mapping `unique_id -> {column_name_lower: data_type}`."""
+
+    by_unique_id: Dict[str, Dict[str, str]] = field(default_factory=dict)
+
+    def column_type(self, unique_id: str, column_name: str) -> Optional[str]:
+        if not unique_id or not column_name:
+            return None
+        cols = self.by_unique_id.get(unique_id)
+        if not cols:
+            return None
+        return cols.get(column_name.lower())
+
+    def is_empty(self) -> bool:
+        return not self.by_unique_id
+
+
+def load_catalog(catalog_path: str) -> CatalogIndex:
+    """Parse a dbt catalog.json into a CatalogIndex.
+
+    Returns an empty index if the file is missing or unparseable — callers
+    treat this as "no catalog available" and fall through to the next type
+    source rather than erroring.
+    """
+    p = Path(catalog_path)
+    if not p.exists():
+        return CatalogIndex()
+    try:
+        with p.open("r", encoding="utf-8") as f:
+            data = json.load(f) or {}
+    except Exception:
+        return CatalogIndex()
+
+    idx = CatalogIndex()
+    # `nodes` holds models/seeds/snapshots; `sources` holds source tables.
+    # Both share the same per-node shape, so index them identically.
+    for section in ("nodes", "sources"):
+        nodes = data.get(section) or {}
+        if not isinstance(nodes, dict):
+            continue
+        for uid, node in nodes.items():
+            if not isinstance(node, dict):
+                continue
+            cols = node.get("columns") or {}
+            if not isinstance(cols, dict):
+                continue
+            mapped: Dict[str, str] = {}
+            for col_name, col_info in cols.items():
+                if not isinstance(col_info, dict):
+                    continue
+                # dbt catalog stores this under "type" (not "data_type" as in
+                # manifest nodes). Normalize to a simple string.
+                dtype = col_info.get("type")
+                if isinstance(dtype, str) and dtype.strip():
+                    mapped[str(col_name).lower()] = dtype.strip()
+            if mapped and isinstance(uid, str):
+                idx.by_unique_id[uid] = mapped
+    return idx
+
+
+def default_catalog_path(dbt_project_dir: str) -> Path:
+    """The canonical location dbt writes catalog.json to."""
+    return Path(dbt_project_dir) / "target" / "catalog.json"

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/packages/core_engine/src/datalex_core/dbt/manifest.py

@@ -23,6 +23,8 @@ from typing import Any, Dict, List, Optional
 
 import yaml
 
+from datalex_core.dbt.catalog import CatalogIndex, load_catalog
+
 
 # ------------------------ public API ------------------------
 
@@ -45,16 +47,24 @@ class ImportResult:
 def import_manifest(
     manifest_path: str,
     existing_project_root: Optional[str] = None,
+    catalog_path: Optional[str] = None,
 ) -> ImportResult:
     """Parse a dbt manifest.json and return merged DataLex source/model docs.
 
     When `existing_project_root` is provided, documents with matching
     `meta.datalex.dbt.unique_id` are merged (user-authored fields preserved).
+
+    When `catalog_path` is provided (typically `<dbt_project>/target/catalog.json`,
+    populated by `dbt docs generate`), column types missing from the manifest
+    fall back to the catalog before defaulting to "unknown". This lets
+    projects that only run `dbt docs generate` (not `dbt compile`) still
+    import real column types.
     """
     with open(manifest_path, "r", encoding="utf-8") as f:
         manifest = json.load(f)
 
     existing = _load_existing_by_unique_id(existing_project_root) if existing_project_root else {}
+    catalog = load_catalog(catalog_path) if catalog_path else CatalogIndex()
 
     result = ImportResult()
 
@@ -68,7 +78,7 @@ def import_manifest(
         sources_grouped.setdefault(source_name, []).append(node)
 
     for source_name, tables in sources_grouped.items():
-        doc = _build_source_doc(source_name, tables, existing)
+        doc = _build_source_doc(source_name, tables, existing, catalog)
         result.sources[doc["name"]] = doc
         src_path = _common_source_path(tables)
         if src_path:
@@ -77,7 +87,7 @@ def import_manifest(
     for uid, node in nodes.items():
         if node.get("resource_type") != "model":
             continue
-        doc = _build_model_doc(node, existing)
+        doc = _build_model_doc(node, existing, catalog)
         result.models[doc["name"]] = doc
         model_path = node.get("original_file_path") or ""
         if model_path:
@@ -210,6 +220,7 @@ def _build_source_doc(
     source_name: str,
     tables: List[Dict[str, Any]],
     existing: Dict[str, Dict[str, Any]],
+    catalog: CatalogIndex,
 ) -> Dict[str, Any]:
     # Source-level attributes come from the first table (dbt stores them per-node; pick any).
     first = tables[0]
@@ -237,7 +248,7 @@ def _build_source_doc(
 
     table_docs: List[Dict[str, Any]] = []
     for t in tables:
-        table_docs.append(_build_source_table_doc(t, existing_doc))
+        table_docs.append(_build_source_table_doc(t, existing_doc, catalog))
     doc["tables"] = table_docs
 
     # meta.datalex.dbt.unique_id list, so re-import can find this doc even if one table is renamed.
@@ -250,6 +261,7 @@ def _build_source_doc(
 def _build_source_table_doc(
     t: Dict[str, Any],
     existing_source: Optional[Dict[str, Any]],
+    catalog: CatalogIndex,
 ) -> Dict[str, Any]:
     name = _safe_name(t.get("name", ""))
     table_doc: Dict[str, Any] = {"name": name}
@@ -277,8 +289,11 @@ def _build_source_table_doc(
     # columns
     cols_out: List[Dict[str, Any]] = []
     prior_cols = {c.get("name"): c for c in (prior_table.get("columns") or []) if c.get("name")}
+    tbl_uid = t.get("unique_id") or ""
     for c in t.get("columns", {}).values() if isinstance(t.get("columns"), dict) else (t.get("columns") or []):
-        cols_out.append(_build_source_column_doc(c, prior_cols.get(c.get("name"), {})))
+        cols_out.append(
+            _build_source_column_doc(c, prior_cols.get(c.get("name"), {}), catalog, tbl_uid)
+        )
     if cols_out:
         table_doc["columns"] = cols_out
 
@@ -291,18 +306,28 @@ def _build_source_table_doc(
     return table_doc
 
 
-def _build_source_column_doc(c: Dict[str, Any], prior: Dict[str, Any]) -> Dict[str, Any]:
+def _build_source_column_doc(
+    c: Dict[str, Any],
+    prior: Dict[str, Any],
+    catalog: CatalogIndex,
+    table_unique_id: str,
+) -> Dict[str, Any]:
     doc: Dict[str, Any] = {"name": c.get("name")}
-    # type: manifest owns it; prefer manifest value if present. Same
-    # "unknown" sentinel as _build_model_column_doc so the UI can flag
-    # columns that didn't pick up a real type (dbt manifest without
-    # `dbt compile`).
+    # Type precedence:
+    #   1. manifest `data_type` (from `dbt compile`) — freshest, already parsed
+    #   2. catalog.json `type` (from `dbt docs generate`) — real warehouse types
+    #   3. prior user-authored type (preserved across re-imports)
+    #   4. "unknown" sentinel — UI renders as "—" and importer warns
     if c.get("data_type"):
         doc["type"] = c["data_type"]
-    elif prior.get("type"):
-        doc["type"] = prior["type"]
     else:
-        doc["type"] = "unknown"
+        catalog_type = catalog.column_type(table_unique_id, c.get("name") or "")
+        if catalog_type:
+            doc["type"] = catalog_type
+        elif prior.get("type"):
+            doc["type"] = prior["type"]
+        else:
+            doc["type"] = "unknown"
 
     # user-authored: preserve
     for k in ("description", "sensitivity", "tags"):
@@ -318,6 +343,7 @@ def _build_source_column_doc(c: Dict[str, Any], prior: Dict[str, Any]) -> Dict[s
 def _build_model_doc(
     node: Dict[str, Any],
     existing: Dict[str, Dict[str, Any]],
+    catalog: CatalogIndex,
 ) -> Dict[str, Any]:
     name = _safe_name(node.get("name", ""))
     uid = node.get("unique_id")
@@ -361,7 +387,9 @@ def _build_model_doc(
     columns_raw = node.get("columns") or {}
     column_iter = columns_raw.values() if isinstance(columns_raw, dict) else columns_raw
     for c in column_iter:
-        cols_out.append(_build_model_column_doc(c, prior_cols.get(c.get("name"), {})))
+        cols_out.append(
+            _build_model_column_doc(c, prior_cols.get(c.get("name"), {}), catalog, uid or "")
+        )
     if cols_out:
         doc["columns"] = cols_out
 
@@ -371,20 +399,29 @@ def _build_model_doc(
     return doc
 
 
-def _build_model_column_doc(c: Dict[str, Any], prior: Dict[str, Any]) -> Dict[str, Any]:
+def _build_model_column_doc(
+    c: Dict[str, Any],
+    prior: Dict[str, Any],
+    catalog: CatalogIndex,
+    model_unique_id: str,
+) -> Dict[str, Any]:
     doc: Dict[str, Any] = {"name": c.get("name")}
-    # Type precedence: dbt `data_type` (populated by `dbt compile` / `dbt docs
-    # generate`), then any prior user-authored type, then an explicit
-    # "unknown" sentinel. Writing "unknown" (rather than omitting `type:`)
-    # makes uncompiled dbt projects surface visibly in the UI — the web-app
-    # renders "unknown" columns as "—" and the caller prints a one-line
-    # warning at import time so users know to run `dbt compile`.
+    # Type precedence:
+    #   1. manifest `data_type` (populated by `dbt compile`)
+    #   2. catalog.json `type` (populated by `dbt docs generate`)
+    #   3. prior user-authored type (preserved across re-imports)
+    #   4. "unknown" sentinel — UI renders as "—" and importer warns so
+    #      users know to run `dbt compile` or `dbt docs generate`.
     if c.get("data_type"):
         doc["type"] = c["data_type"]
-    elif prior.get("type"):
-        doc["type"] = prior["type"]
     else:
-        doc["type"] = "unknown"
+        catalog_type = catalog.column_type(model_unique_id, c.get("name") or "")
+        if catalog_type:
+            doc["type"] = catalog_type
+        elif prior.get("type"):
+            doc["type"] = prior["type"]
+        else:
+            doc["type"] = "unknown"
 
     for k in ("description", "sensitivity", "tags", "terms", "tests", "constraints"):
         if prior.get(k):

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/packages/core_engine/src/datalex_core/dbt/sync.py

@@ -116,8 +116,18 @@ def sync_dbt_project(
 
     report = SyncReport(dbt_project=str(dbt_dir), datalex_root=str(out_root))
 
-    # Step 1: parse manifest (merge-preserving re-import)
-    imported = import_manifest(str(manifest), existing_project_root=str(out_root))
+    # Step 1: parse manifest (merge-preserving re-import). Auto-discover a
+    # catalog.json alongside the manifest — `dbt docs generate` writes both
+    # into `target/`, so if the user has generated docs we pick up types
+    # that aren't yet in the manifest. Missing catalog.json is fine; the
+    # importer silently falls through.
+    catalog_candidate = manifest.parent / "catalog.json"
+    catalog_path = str(catalog_candidate) if catalog_candidate.exists() else None
+    imported = import_manifest(
+        str(manifest),
+        existing_project_root=str(out_root),
+        catalog_path=catalog_path,
+    )
     # Surface the manifest-level warnings (e.g. "N columns have no data type
     # — run `dbt compile`") up through the sync report so both the CLI and
     # the API server see them in a single place.

{datalex_cli-1.0.3 → datalex_cli-1.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datalex-cli"
-version = "1.0.3"
+version = "1.0.4"
 description = "Git-native data modeling for dbt users"
 readme = "README.md"
 requires-python = ">=3.9"