parrant 0.17.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parrant/artifacts/adapter_mapping.py +98 -0
- parrant/artifacts/catalog.py +91 -0
- parrant/artifacts/exceptions.py +15 -0
- parrant/artifacts/manifest.py +445 -0
- parrant/artifacts/registry.py +718 -0
- parrant/cli/main.py +1116 -0
- parrant/lineage/backtest.py +518 -0
- parrant/lineage/changeset.py +766 -0
- parrant/lineage/ci.py +269 -0
- parrant/lineage/display/__init__.py +5 -0
- parrant/lineage/display/backtest.py +115 -0
- parrant/lineage/display/base.py +50 -0
- parrant/lineage/display/dot.py +135 -0
- parrant/lineage/display/html/__init__.py +0 -0
- parrant/lineage/display/html/explore.py +1256 -0
- parrant/lineage/display/html/static/css/base.css +198 -0
- parrant/lineage/display/html/static/css/components.css +332 -0
- parrant/lineage/display/html/static/css/explore.css +837 -0
- parrant/lineage/display/html/static/css/graph.css +676 -0
- parrant/lineage/display/html/static/css/impact.css +2307 -0
- parrant/lineage/display/html/static/css/layout.css +87 -0
- parrant/lineage/display/html/static/css/model-details.css +337 -0
- parrant/lineage/display/html/static/css/policy.css +402 -0
- parrant/lineage/display/html/static/js/app.js +25 -0
- parrant/lineage/display/html/static/js/config.js +62 -0
- parrant/lineage/display/html/static/js/dataProcessor.js +641 -0
- parrant/lineage/display/html/static/js/explore.js +679 -0
- parrant/lineage/display/html/static/js/graph.js +1101 -0
- parrant/lineage/display/html/static/js/impact.js +1009 -0
- parrant/lineage/display/html/static/js/interactions.js +481 -0
- parrant/lineage/display/html/static/js/model-details.js +174 -0
- parrant/lineage/display/html/static/js/policy.js +247 -0
- parrant/lineage/display/html/static/js/renderer.js +1870 -0
- parrant/lineage/display/html/static/js/utils.js +398 -0
- parrant/lineage/display/html/static/vendor/d3.v7.min.js +2 -0
- parrant/lineage/display/html/templates/graph.html +255 -0
- parrant/lineage/display/json.py +97 -0
- parrant/lineage/display/markdown.py +660 -0
- parrant/lineage/display/text.py +73 -0
- parrant/lineage/policy.py +1046 -0
- parrant/lineage/policy_init.py +405 -0
- parrant/lineage/provider.py +198 -0
- parrant/lineage/semantic_diff.py +154 -0
- parrant/lineage/service.py +986 -0
- parrant/lineage/sqlglot_provider.py +87 -0
- parrant/lineage/verdict.py +366 -0
- parrant/metabase/__init__.py +17 -0
- parrant/metabase/artifact.py +68 -0
- parrant/metabase/cli.py +130 -0
- parrant/metabase/client.py +212 -0
- parrant/metabase/extract.py +194 -0
- parrant/metabase/join.py +77 -0
- parrant/metabase/reach.py +247 -0
- parrant/metabase/resolvers.py +463 -0
- parrant/metabase/warehouse_meta.py +158 -0
- parrant/models/__init__.py +0 -0
- parrant/models/schema.py +846 -0
- parrant/parser/__init__.py +3 -0
- parrant/parser/sql_parser.py +1005 -0
- parrant/parser/sql_parser_utils.py +125 -0
- parrant-0.17.1.dist-info/LICENSE +10 -0
- parrant-0.17.1.dist-info/METADATA +276 -0
- parrant-0.17.1.dist-info/RECORD +65 -0
- parrant-0.17.1.dist-info/WHEEL +4 -0
- parrant-0.17.1.dist-info/entry_points.txt +10 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Centralized adapter/dialect normalization for parrant.
|
|
3
|
+
|
|
4
|
+
This module defines a single source of truth for mapping raw dbt adapter
|
|
5
|
+
names to sqlglot dialect names. For example, dbt may report the adapter
|
|
6
|
+
"sqlserver" while sqlglot expects the dialect name "tsql".
|
|
7
|
+
|
|
8
|
+
Most common warehouse adapters (snowflake, bigquery, redshift, databricks,
|
|
9
|
+
postgres, duckdb, ...) already share their name with the corresponding
|
|
10
|
+
sqlglot dialect, so they resolve correctly without an explicit entry. The
|
|
11
|
+
mapping below covers the adapters whose dbt name differs from the sqlglot
|
|
12
|
+
dialect name, plus a few identity entries that pin well-supported adapters
|
|
13
|
+
to their verified dialect.
|
|
14
|
+
|
|
15
|
+
Extend ADAPTER_TO_DIALECT as needed to support additional adapters.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
from typing import Dict, Optional, Set
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
# Mapping from dbt adapter name (metadata.adapter_type) to sqlglot dialect.
|
|
24
|
+
#
|
|
25
|
+
# Only entries verified against sqlglot's supported dialects belong here. When
|
|
26
|
+
# the dbt adapter name already equals a valid sqlglot dialect (snowflake,
|
|
27
|
+
# bigquery, redshift, databricks, postgres, duckdb, spark, trino, presto,
|
|
28
|
+
# athena, clickhouse, ...) an explicit identity entry is optional -- the
|
|
29
|
+
# fallthrough in normalize_adapter returns the lowercased name unchanged.
|
|
30
|
+
ADAPTER_TO_DIALECT: Dict[str, str] = {
|
|
31
|
+
# --- adapters whose dbt name differs from the sqlglot dialect name ---
|
|
32
|
+
# The T-SQL family: dbt reports sqlserver/synapse/fabric, sqlglot uses "tsql".
|
|
33
|
+
"sqlserver": "tsql",
|
|
34
|
+
"synapse": "tsql",
|
|
35
|
+
"fabric": "tsql",
|
|
36
|
+
# --- identity pins for common, verified adapters (documented support) ---
|
|
37
|
+
"snowflake": "snowflake",
|
|
38
|
+
"bigquery": "bigquery",
|
|
39
|
+
"redshift": "redshift",
|
|
40
|
+
"databricks": "databricks",
|
|
41
|
+
"spark": "spark",
|
|
42
|
+
"trino": "trino",
|
|
43
|
+
"presto": "presto",
|
|
44
|
+
"athena": "athena",
|
|
45
|
+
"postgres": "postgres",
|
|
46
|
+
"duckdb": "duckdb",
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _known_sqlglot_dialects() -> Set[str]:
|
|
51
|
+
"""Return the set of dialect names sqlglot actually supports.
|
|
52
|
+
|
|
53
|
+
Derived from sqlglot at runtime so the check stays correct as sqlglot is
|
|
54
|
+
upgraded. Falls back to an empty set if the internal enum is unavailable,
|
|
55
|
+
in which case the unknown-dialect warning is simply skipped.
|
|
56
|
+
"""
|
|
57
|
+
try:
|
|
58
|
+
from sqlglot.dialects.dialect import Dialects
|
|
59
|
+
|
|
60
|
+
return {member.value for member in Dialects if member.value}
|
|
61
|
+
except Exception: # pragma: no cover - defensive, sqlglot API drift
|
|
62
|
+
return set()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
_KNOWN_DIALECTS: Set[str] = _known_sqlglot_dialects()
|
|
66
|
+
|
|
67
|
+
# Track adapters we have already warned about so the warning fires once per
|
|
68
|
+
# unresolved adapter instead of on every column parsed.
|
|
69
|
+
_warned_adapters: Set[str] = set()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def normalize_adapter(adapter_name: Optional[str]) -> Optional[str]:
|
|
73
|
+
"""Normalize a dbt adapter name to a sqlglot dialect name.
|
|
74
|
+
|
|
75
|
+
If adapter_name is None or empty, returns it unchanged.
|
|
76
|
+
If there is no mapping defined, returns the lowercased adapter_name.
|
|
77
|
+
|
|
78
|
+
When the resolved dialect is not a dialect sqlglot recognizes, a one-time
|
|
79
|
+
WARNING is emitted per adapter so the user gets a visible signal that SQL
|
|
80
|
+
parsing will fall back to sqlglot's default behavior instead of silently
|
|
81
|
+
degrading.
|
|
82
|
+
"""
|
|
83
|
+
if not adapter_name:
|
|
84
|
+
return adapter_name
|
|
85
|
+
lower = adapter_name.lower()
|
|
86
|
+
dialect = ADAPTER_TO_DIALECT.get(lower, lower)
|
|
87
|
+
|
|
88
|
+
if _KNOWN_DIALECTS and dialect not in _KNOWN_DIALECTS and lower not in _warned_adapters:
|
|
89
|
+
_warned_adapters.add(lower)
|
|
90
|
+
logger.warning(
|
|
91
|
+
"dbt adapter '%s' resolved to dialect '%s', which is not a known "
|
|
92
|
+
"sqlglot dialect; SQL parsing may be less accurate. Consider adding "
|
|
93
|
+
"a mapping in adapter_mapping.ADAPTER_TO_DIALECT.",
|
|
94
|
+
adapter_name,
|
|
95
|
+
dialect,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
return dialect
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Dict, Any
|
|
3
|
+
import json
|
|
4
|
+
from parrant.models.schema import Model
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CatalogReader:
|
|
8
|
+
def __init__(self, catalog_path: str):
|
|
9
|
+
self.catalog_path = Path(catalog_path)
|
|
10
|
+
self.catalog: Dict[str, Any] = {}
|
|
11
|
+
|
|
12
|
+
def load(self) -> None:
|
|
13
|
+
if not self.catalog_path.exists():
|
|
14
|
+
raise FileNotFoundError(f"Catalog file not found: {self.catalog_path}")
|
|
15
|
+
with open(self.catalog_path, "r") as f:
|
|
16
|
+
self.catalog = json.load(f)
|
|
17
|
+
|
|
18
|
+
def get_models_nodes(self) -> Dict[str, Model]:
|
|
19
|
+
models = {}
|
|
20
|
+
nodes = self.catalog.get("nodes", {})
|
|
21
|
+
sources = self.catalog.get("sources", {})
|
|
22
|
+
|
|
23
|
+
for node_id, model_data in nodes.items():
|
|
24
|
+
resource_type = node_id.split(".")[0]
|
|
25
|
+
# In dbt's catalog.json the schema/database/name live under `metadata`,
|
|
26
|
+
# not at the top level of the node.
|
|
27
|
+
metadata = model_data.get("metadata", {})
|
|
28
|
+
model_name = (
|
|
29
|
+
metadata.get("name") or model_data.get("name") or node_id.split(".")[-1]
|
|
30
|
+
).lower()
|
|
31
|
+
processed_data = {
|
|
32
|
+
"name": model_name,
|
|
33
|
+
"schema": metadata.get("schema") or model_data.get("schema") or "main",
|
|
34
|
+
"database": metadata.get("database") or model_data.get("database") or "main",
|
|
35
|
+
"columns": {},
|
|
36
|
+
"resource_type": resource_type,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
for col_name, col_data in model_data.get("columns", {}).items():
|
|
40
|
+
normalized_col_name = col_name.lower()
|
|
41
|
+
processed_data["columns"][normalized_col_name] = {
|
|
42
|
+
"name": normalized_col_name,
|
|
43
|
+
"model_name": model_name,
|
|
44
|
+
"description": col_data.get("description"),
|
|
45
|
+
"data_type": col_data.get("type") or col_data.get("data_type"),
|
|
46
|
+
"lineage": [],
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
model = Model(**processed_data)
|
|
50
|
+
models[model.name] = model
|
|
51
|
+
|
|
52
|
+
for source_id, source_data in sources.items():
|
|
53
|
+
metadata = source_data.get("metadata", {})
|
|
54
|
+
source_identifier = metadata.get("name")
|
|
55
|
+
normalized_source_identifier = source_identifier.lower() if source_identifier else None
|
|
56
|
+
|
|
57
|
+
table_name = (source_data.get("name") or source_id.split(".")[-1]).lower()
|
|
58
|
+
source_name = source_data.get("source_name")
|
|
59
|
+
if not source_name:
|
|
60
|
+
source_id_parts = source_id.split(".")
|
|
61
|
+
if len(source_id_parts) >= 4:
|
|
62
|
+
source_name = source_id_parts[2]
|
|
63
|
+
normalized_source_name = source_name.lower() if source_name else None
|
|
64
|
+
|
|
65
|
+
processed_data = {
|
|
66
|
+
"name": table_name,
|
|
67
|
+
"schema": metadata.get("schema") or source_data.get("schema") or "main",
|
|
68
|
+
"database": metadata.get("database") or source_data.get("database") or "main",
|
|
69
|
+
"columns": {},
|
|
70
|
+
"resource_type": "source",
|
|
71
|
+
"source_identifier": normalized_source_identifier,
|
|
72
|
+
"source_name": normalized_source_name,
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
for col_name, col_data in source_data.get("columns", {}).items():
|
|
76
|
+
normalized_col_name = col_name.lower()
|
|
77
|
+
processed_data["columns"][normalized_col_name] = {
|
|
78
|
+
"name": normalized_col_name,
|
|
79
|
+
"model_name": normalized_source_identifier or table_name,
|
|
80
|
+
"description": col_data.get("description"),
|
|
81
|
+
"data_type": col_data.get("type") or col_data.get("data_type"),
|
|
82
|
+
"lineage": [],
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
model = Model(**processed_data)
|
|
86
|
+
key = normalized_source_identifier or model.name
|
|
87
|
+
models[key] = model
|
|
88
|
+
for col in model.columns.values():
|
|
89
|
+
col.model_name = key
|
|
90
|
+
|
|
91
|
+
return models
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
class RegistryError(Exception):
|
|
2
|
+
"""Base exception for all registry-related errors."""
|
|
3
|
+
pass
|
|
4
|
+
|
|
5
|
+
class ModelNotFoundError(RegistryError):
|
|
6
|
+
"""Raised when a requested model is not found."""
|
|
7
|
+
pass
|
|
8
|
+
|
|
9
|
+
class RegistryNotLoadedError(RegistryError):
|
|
10
|
+
"""Raised when trying to access registry before loading data."""
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
class RegistryLoadError(Exception):
|
|
14
|
+
"""Base exception for registry loading errors."""
|
|
15
|
+
pass
|
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
from typing import Dict, List, Optional, Set, Any
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from parrant.artifacts.adapter_mapping import normalize_adapter
|
|
8
|
+
from parrant.models.schema import TestNode
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
# Matches the quoted name(s) inside a dbt ``ref(...)`` expression, e.g.
|
|
12
|
+
# ``ref('stg_accounts')`` or ``ref('my_pkg', 'stg_accounts')``. The *last* quoted
|
|
13
|
+
# token is the model name (the first, when present, is the package).
|
|
14
|
+
_REF_QUOTED_RE = re.compile(r"""['"]([^'"]+)['"]""")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _model_name_from_ref(ref_expr: Optional[str]) -> Optional[str]:
|
|
18
|
+
"""Extract the model name from a dbt ``ref(...)`` expression string.
|
|
19
|
+
|
|
20
|
+
Returns ``None`` when nothing quoted can be found (e.g. a ``source(...)`` target
|
|
21
|
+
or an unexpected shape) rather than guessing.
|
|
22
|
+
"""
|
|
23
|
+
if not ref_expr:
|
|
24
|
+
return None
|
|
25
|
+
matches = _REF_QUOTED_RE.findall(ref_expr)
|
|
26
|
+
if not matches:
|
|
27
|
+
return None
|
|
28
|
+
return matches[-1].lower()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _model_name_from_unique_id(unique_id: Optional[str]) -> Optional[str]:
|
|
32
|
+
"""Return the lowercased model name from a ``model.<pkg>.<name>`` unique_id."""
|
|
33
|
+
if not unique_id:
|
|
34
|
+
return None
|
|
35
|
+
parts = unique_id.split(".")
|
|
36
|
+
if parts[0] != "model":
|
|
37
|
+
return None
|
|
38
|
+
return parts[-1].lower()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ManifestReader:
|
|
42
|
+
def __init__(self, manifest_path: Optional[str] = None):
|
|
43
|
+
self.manifest_path = Path(manifest_path) if manifest_path else None
|
|
44
|
+
self.manifest: Dict[str, Any] = {}
|
|
45
|
+
# Lazily-built index of on-disk compiled SQL keyed by filename (e.g. ``orders.sql``),
|
|
46
|
+
# used to recover a model's compiled SQL when the manifest's ``original_file_path``
|
|
47
|
+
# has drifted from the ``target/compiled`` layout (a model moved between builds).
|
|
48
|
+
self._compiled_index: Optional[Dict[str, List[Path]]] = None
|
|
49
|
+
|
|
50
|
+
def load(self) -> None:
|
|
51
|
+
if not self.manifest_path or not self.manifest_path.exists():
|
|
52
|
+
raise FileNotFoundError(f"Manifest file not found: {self.manifest_path}")
|
|
53
|
+
with open(self.manifest_path, "r") as f:
|
|
54
|
+
self.manifest = json.load(f)
|
|
55
|
+
|
|
56
|
+
def get_adapter(self) -> Optional[str]:
|
|
57
|
+
adapter_name = self.manifest.get("metadata", {}).get("adapter_type")
|
|
58
|
+
return normalize_adapter(adapter_name)
|
|
59
|
+
|
|
60
|
+
def _find_node(self, model_name: str) -> Optional[Dict[str, Any]]:
|
|
61
|
+
"""Find a node in the manifest by model name."""
|
|
62
|
+
if not self.manifest:
|
|
63
|
+
return None
|
|
64
|
+
model_name_lower = model_name.lower()
|
|
65
|
+
for _, node in self.manifest.get("nodes", {}).items():
|
|
66
|
+
if node.get("name", "").lower() == model_name_lower:
|
|
67
|
+
return dict(node)
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
def get_model_dependencies(self) -> Dict[str, Set[str]]:
|
|
71
|
+
"""Return a dictionary of model dependencies with full model names.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
Dict[str, Set[str]]: Key is full model name, value is set of full dependency names
|
|
75
|
+
"""
|
|
76
|
+
dependencies = {}
|
|
77
|
+
for model_id, model_data in self.manifest.get("nodes", {}).items():
|
|
78
|
+
# `depends_on.nodes` is a list of unique_id strings (e.g. "model.pkg.name"),
|
|
79
|
+
# not a list of dicts, so index it directly.
|
|
80
|
+
depends_on = set(model_data.get("depends_on", {}).get("nodes", []))
|
|
81
|
+
dependencies[model_id] = depends_on
|
|
82
|
+
return dependencies
|
|
83
|
+
|
|
84
|
+
def get_model_upstream(self) -> Dict[str, Set[str]]:
|
|
85
|
+
"""Get upstream dependencies for each model."""
|
|
86
|
+
upstream: Dict[str, Set[str]] = {}
|
|
87
|
+
|
|
88
|
+
for _, node in self.manifest.get("nodes", {}).items():
|
|
89
|
+
resource_type = node.get("resource_type")
|
|
90
|
+
if resource_type in ("model", "snapshot"):
|
|
91
|
+
model_name = node.get("name")
|
|
92
|
+
if not model_name:
|
|
93
|
+
continue
|
|
94
|
+
|
|
95
|
+
model_name = model_name.lower()
|
|
96
|
+
upstream[model_name] = set()
|
|
97
|
+
|
|
98
|
+
depends_on = node.get("depends_on", {})
|
|
99
|
+
for dep_id in depends_on.get("nodes", []):
|
|
100
|
+
parts = dep_id.split(".")
|
|
101
|
+
if parts[0] == "model":
|
|
102
|
+
dep_name = parts[-1].lower()
|
|
103
|
+
upstream[model_name].add(dep_name)
|
|
104
|
+
elif parts[0] == "source":
|
|
105
|
+
source_node = self.manifest.get("sources", {}).get(dep_id, {})
|
|
106
|
+
source_identifier = source_node.get("identifier")
|
|
107
|
+
if source_identifier:
|
|
108
|
+
upstream[model_name].add(source_identifier.lower())
|
|
109
|
+
else:
|
|
110
|
+
# Fallback to source name if identifier not found
|
|
111
|
+
source_name = parts[-1].lower()
|
|
112
|
+
upstream[model_name].add(source_name)
|
|
113
|
+
elif parts[0] == "snapshot":
|
|
114
|
+
dep_name = parts[-1].lower()
|
|
115
|
+
upstream[model_name].add(dep_name)
|
|
116
|
+
|
|
117
|
+
return upstream
|
|
118
|
+
|
|
119
|
+
def get_model_downstream(self) -> Dict[str, Set[str]]:
|
|
120
|
+
"""Return a dictionary of model downstream dependencies."""
|
|
121
|
+
downstream: Dict[str, Set[str]] = {}
|
|
122
|
+
|
|
123
|
+
upstream_deps = self.get_model_upstream()
|
|
124
|
+
|
|
125
|
+
for model_name, upstream_models in upstream_deps.items():
|
|
126
|
+
for upstream_model in upstream_models:
|
|
127
|
+
if upstream_model not in downstream:
|
|
128
|
+
downstream[upstream_model] = set()
|
|
129
|
+
downstream[upstream_model].add(model_name)
|
|
130
|
+
|
|
131
|
+
return downstream
|
|
132
|
+
|
|
133
|
+
def _resolve_compiled_file(self, node: Dict[str, Any]) -> Optional[Path]:
|
|
134
|
+
"""Locate the on-disk compiled SQL file for a node.
|
|
135
|
+
|
|
136
|
+
Many real manifests are produced without embedded ``compiled_code`` (e.g.
|
|
137
|
+
``dbt parse`` or ``dbt docs generate`` without a compile step). In that case
|
|
138
|
+
the compiled SQL still lives under ``target/compiled/**`` on disk, so we
|
|
139
|
+
reconstruct its path from the manifest location and node metadata.
|
|
140
|
+
"""
|
|
141
|
+
if not self.manifest_path:
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
target_dir = self.manifest_path.parent
|
|
145
|
+
project_root = target_dir.parent
|
|
146
|
+
|
|
147
|
+
candidates = []
|
|
148
|
+
|
|
149
|
+
# dbt records ``compiled_path`` relative to the project root once compiled.
|
|
150
|
+
compiled_path = node.get("compiled_path")
|
|
151
|
+
if compiled_path:
|
|
152
|
+
candidates.append(project_root / compiled_path)
|
|
153
|
+
candidates.append(Path(compiled_path))
|
|
154
|
+
|
|
155
|
+
# dbt convention: <target>/compiled/<package_name>/<original_file_path>
|
|
156
|
+
package_name = node.get("package_name")
|
|
157
|
+
original_file_path = node.get("original_file_path")
|
|
158
|
+
if package_name and original_file_path:
|
|
159
|
+
candidates.append(target_dir / "compiled" / package_name / original_file_path)
|
|
160
|
+
|
|
161
|
+
for candidate in candidates:
|
|
162
|
+
try:
|
|
163
|
+
if candidate.is_file():
|
|
164
|
+
return candidate
|
|
165
|
+
except OSError:
|
|
166
|
+
continue
|
|
167
|
+
|
|
168
|
+
# Fallback: the exact path missed, but the compiled file may still be on disk under
|
|
169
|
+
# a different sub-path — the manifest's ``original_file_path`` can drift from the
|
|
170
|
+
# ``target/compiled`` layout when a model was moved/refactored between the build that
|
|
171
|
+
# produced the manifest and the one that produced ``compiled/``. Recover it by the
|
|
172
|
+
# compiled filename (dbt names it ``<model>.sql``), but ONLY when the match is
|
|
173
|
+
# unambiguous, so we never silently attach the wrong (or stale-duplicate) SQL.
|
|
174
|
+
if original_file_path:
|
|
175
|
+
return self._recover_compiled_by_name(Path(original_file_path).name, package_name)
|
|
176
|
+
return None
|
|
177
|
+
|
|
178
|
+
def _recover_compiled_by_name(
|
|
179
|
+
self, filename: str, package_name: Optional[str]
|
|
180
|
+
) -> Optional[Path]:
|
|
181
|
+
"""Find an on-disk compiled file by its ``<model>.sql`` name, unambiguously.
|
|
182
|
+
|
|
183
|
+
Prefers a single match under the model's own package dir; otherwise accepts a single
|
|
184
|
+
match anywhere under ``target/compiled``. Returns ``None`` on zero or multiple matches
|
|
185
|
+
(ambiguous → we decline to guess, keeping the model honestly unresolved).
|
|
186
|
+
"""
|
|
187
|
+
index = self._compiled_basename_index()
|
|
188
|
+
matches = index.get(filename, [])
|
|
189
|
+
if not matches:
|
|
190
|
+
return None
|
|
191
|
+
if package_name:
|
|
192
|
+
marker = f"{os.sep}compiled{os.sep}{package_name}{os.sep}"
|
|
193
|
+
scoped = [p for p in matches if marker in f"{os.sep}{p}{os.sep}"]
|
|
194
|
+
if len(scoped) == 1:
|
|
195
|
+
return scoped[0]
|
|
196
|
+
return matches[0] if len(matches) == 1 else None
|
|
197
|
+
|
|
198
|
+
def _compiled_basename_index(self) -> Dict[str, List[Path]]:
|
|
199
|
+
"""Lazily index ``target/compiled/**/*.sql`` by filename → list of paths."""
|
|
200
|
+
if self._compiled_index is not None:
|
|
201
|
+
return self._compiled_index
|
|
202
|
+
index: Dict[str, List[Path]] = {}
|
|
203
|
+
if self.manifest_path:
|
|
204
|
+
compiled_dir = self.manifest_path.parent / "compiled"
|
|
205
|
+
if compiled_dir.is_dir():
|
|
206
|
+
for path in compiled_dir.rglob("*.sql"):
|
|
207
|
+
index.setdefault(path.name, []).append(path)
|
|
208
|
+
self._compiled_index = index
|
|
209
|
+
return index
|
|
210
|
+
|
|
211
|
+
def get_compiled_sql(self, model_name: str) -> Optional[str]:
|
|
212
|
+
"""Get compiled SQL for a model.
|
|
213
|
+
|
|
214
|
+
Prefers SQL embedded in the manifest, falling back to the compiled file on
|
|
215
|
+
disk when the manifest was produced without embedded compiled code.
|
|
216
|
+
"""
|
|
217
|
+
node = self._find_node(model_name)
|
|
218
|
+
if not node:
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
embedded = node.get("compiled_sql") or node.get("compiled_code")
|
|
222
|
+
if embedded:
|
|
223
|
+
return embedded
|
|
224
|
+
|
|
225
|
+
compiled_file = self._resolve_compiled_file(node)
|
|
226
|
+
if compiled_file:
|
|
227
|
+
try:
|
|
228
|
+
return compiled_file.read_text()
|
|
229
|
+
except OSError:
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
return None
|
|
233
|
+
|
|
234
|
+
@staticmethod
|
|
235
|
+
def _merged_meta(
|
|
236
|
+
top_meta: Optional[Dict[str, Any]], config_meta: Optional[Dict[str, Any]]
|
|
237
|
+
) -> Dict[str, Any]:
|
|
238
|
+
"""Merge a node's two dbt meta locations, ``config.meta`` winning over top-level.
|
|
239
|
+
|
|
240
|
+
dbt exposes user-authored meta at both ``node.meta`` (legacy) and
|
|
241
|
+
``node.config.meta`` (canonical in dbt 1.x). When a key is present in both, the
|
|
242
|
+
``config`` value is authoritative (it is what dbt itself resolves). Neither
|
|
243
|
+
present yields an empty dict — meta is *absent*, never guessed.
|
|
244
|
+
"""
|
|
245
|
+
merged: Dict[str, Any] = {}
|
|
246
|
+
if isinstance(top_meta, dict):
|
|
247
|
+
merged.update(top_meta)
|
|
248
|
+
if isinstance(config_meta, dict):
|
|
249
|
+
merged.update(config_meta)
|
|
250
|
+
return merged
|
|
251
|
+
|
|
252
|
+
def get_model_meta(self, model_name: str) -> Dict[str, Any]:
|
|
253
|
+
"""Merged user-authored dbt ``meta`` for a model (``config.meta`` over ``meta``).
|
|
254
|
+
|
|
255
|
+
This is arbitrary consumer metadata — ANY key an author declared — captured
|
|
256
|
+
generically; no key is privileged. Returns an empty dict for an unknown model or
|
|
257
|
+
one with no meta.
|
|
258
|
+
"""
|
|
259
|
+
node = self._find_node(model_name)
|
|
260
|
+
if not node:
|
|
261
|
+
return {}
|
|
262
|
+
config = node.get("config") or {}
|
|
263
|
+
return self._merged_meta(node.get("meta"), config.get("meta"))
|
|
264
|
+
|
|
265
|
+
def get_column_meta(self, model_name: str) -> Dict[str, Dict[str, Any]]:
|
|
266
|
+
"""Per-column merged user meta for a model, keyed by lowercased column name.
|
|
267
|
+
|
|
268
|
+
Each column's meta merges ``columns.<c>.config.meta`` over ``columns.<c>.meta``
|
|
269
|
+
(config wins), mirroring :meth:`get_model_meta`. Column keys are lowercased to
|
|
270
|
+
match the codebase's case-insensitive column naming. Returns an empty dict for an
|
|
271
|
+
unknown model; a declared column with no meta maps to an empty dict.
|
|
272
|
+
"""
|
|
273
|
+
node = self._find_node(model_name)
|
|
274
|
+
if not node:
|
|
275
|
+
return {}
|
|
276
|
+
result: Dict[str, Dict[str, Any]] = {}
|
|
277
|
+
for col_name, col_data in (node.get("columns") or {}).items():
|
|
278
|
+
col_data = col_data or {}
|
|
279
|
+
col_config = col_data.get("config") or {}
|
|
280
|
+
result[col_name.lower()] = self._merged_meta(
|
|
281
|
+
col_data.get("meta"), col_config.get("meta")
|
|
282
|
+
)
|
|
283
|
+
return result
|
|
284
|
+
|
|
285
|
+
def get_model_path(self, model_name: str) -> Optional[str]:
|
|
286
|
+
"""Get the path to the model from the manifest."""
|
|
287
|
+
node = self._find_node(model_name)
|
|
288
|
+
if not node:
|
|
289
|
+
return None
|
|
290
|
+
|
|
291
|
+
return node.get("path")
|
|
292
|
+
|
|
293
|
+
def get_model_language(self, model_name: str) -> Optional[str]:
|
|
294
|
+
"""Get the language of a model from the manifest."""
|
|
295
|
+
node = self._find_node(model_name)
|
|
296
|
+
if not node:
|
|
297
|
+
return None
|
|
298
|
+
return node.get("language")
|
|
299
|
+
|
|
300
|
+
def get_model_resource_path(self, model_name: str) -> Optional[str]:
|
|
301
|
+
"""Get the original file path of a model from the manifest."""
|
|
302
|
+
node = self._find_node(model_name)
|
|
303
|
+
if not node:
|
|
304
|
+
return None
|
|
305
|
+
return node.get("original_file_path")
|
|
306
|
+
|
|
307
|
+
def get_node(self, node_id: str) -> Optional[Dict[str, Any]]:
|
|
308
|
+
node = self.manifest.get("nodes", {}).get(node_id)
|
|
309
|
+
if node is None:
|
|
310
|
+
return None
|
|
311
|
+
return dict(node)
|
|
312
|
+
|
|
313
|
+
def get_tests(self) -> List[TestNode]:
|
|
314
|
+
"""Read dbt test nodes (``resource_type == "test"``) from the manifest.
|
|
315
|
+
|
|
316
|
+
We never run the tests; we read what they *declare*. For each test we extract:
|
|
317
|
+
its ``unique_id``; the test kind (``test_metadata.name`` — not_null / unique /
|
|
318
|
+
relationships / ...); the target column (top-level ``column_name``, falling back
|
|
319
|
+
to ``test_metadata.kwargs.column_name``); the target model (from ``attached_node``,
|
|
320
|
+
falling back to the sole model in ``depends_on.nodes``); for ``relationships``
|
|
321
|
+
tests the referenced model/column (``kwargs.to`` / ``kwargs.field``); and the
|
|
322
|
+
``original_file_path``.
|
|
323
|
+
|
|
324
|
+
Tests whose target column or model cannot be attributed are kept with the
|
|
325
|
+
unknown field set to ``None`` (never guessed), so the reverse index can report
|
|
326
|
+
coverage honestly.
|
|
327
|
+
"""
|
|
328
|
+
tests: List[TestNode] = []
|
|
329
|
+
|
|
330
|
+
for node_id, node in self.manifest.get("nodes", {}).items():
|
|
331
|
+
if node.get("resource_type") != "test":
|
|
332
|
+
continue
|
|
333
|
+
|
|
334
|
+
test_metadata = node.get("test_metadata") or {}
|
|
335
|
+
test_name = test_metadata.get("name")
|
|
336
|
+
if not test_name:
|
|
337
|
+
# Singular / custom SQL tests carry no ``test_metadata`` and no declared
|
|
338
|
+
# column target. They CAN break on a column removal, but we can't know which
|
|
339
|
+
# columns they touch without parsing the test SQL, so they're out of scope
|
|
340
|
+
# for the column-level index — an unavoidable blind spot, not a safe one.
|
|
341
|
+
continue
|
|
342
|
+
|
|
343
|
+
kwargs = test_metadata.get("kwargs") or {}
|
|
344
|
+
|
|
345
|
+
target_column = node.get("column_name") or kwargs.get("column_name")
|
|
346
|
+
if isinstance(target_column, str):
|
|
347
|
+
target_column = target_column.lower()
|
|
348
|
+
else:
|
|
349
|
+
target_column = None
|
|
350
|
+
|
|
351
|
+
target_model = _model_name_from_unique_id(node.get("attached_node"))
|
|
352
|
+
if target_model is None:
|
|
353
|
+
model_deps = [
|
|
354
|
+
_model_name_from_unique_id(dep)
|
|
355
|
+
for dep in node.get("depends_on", {}).get("nodes", [])
|
|
356
|
+
]
|
|
357
|
+
model_deps = [m for m in model_deps if m is not None]
|
|
358
|
+
# Only attribute when unambiguous. A ``relationships`` test depends on
|
|
359
|
+
# two models, so without ``attached_node`` we cannot tell which side is
|
|
360
|
+
# the target — leave it unknown rather than guess.
|
|
361
|
+
if len(model_deps) == 1:
|
|
362
|
+
target_model = model_deps[0]
|
|
363
|
+
|
|
364
|
+
referenced_model: Optional[str] = None
|
|
365
|
+
referenced_column: Optional[str] = None
|
|
366
|
+
if test_name == "relationships":
|
|
367
|
+
referenced_model = _model_name_from_ref(kwargs.get("to"))
|
|
368
|
+
field = kwargs.get("field")
|
|
369
|
+
if isinstance(field, str):
|
|
370
|
+
referenced_column = field.lower()
|
|
371
|
+
|
|
372
|
+
tests.append(
|
|
373
|
+
TestNode(
|
|
374
|
+
unique_id=node.get("unique_id") or node_id,
|
|
375
|
+
test_name=test_name,
|
|
376
|
+
target_model=target_model,
|
|
377
|
+
target_column=target_column,
|
|
378
|
+
referenced_model=referenced_model,
|
|
379
|
+
referenced_column=referenced_column,
|
|
380
|
+
resource_path=node.get("original_file_path"),
|
|
381
|
+
)
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
return tests
|
|
385
|
+
|
|
386
|
+
def get_exposures(self) -> Dict[str, Dict[str, Any]]:
|
|
387
|
+
"""Get all exposures from the manifest.
|
|
388
|
+
|
|
389
|
+
Returns:
|
|
390
|
+
Dict[str, Dict[str, Any]]: Key is exposure unique_id, value is exposure data
|
|
391
|
+
"""
|
|
392
|
+
return self.manifest.get("exposures", {})
|
|
393
|
+
|
|
394
|
+
def get_exposure_dependencies(self) -> Dict[str, Set[str]]:
|
|
395
|
+
"""Get model dependencies for each exposure.
|
|
396
|
+
|
|
397
|
+
Returns:
|
|
398
|
+
Dict[str, Set[str]]: Key is exposure name, value is set of model names it depends on
|
|
399
|
+
"""
|
|
400
|
+
exposure_deps: Dict[str, Set[str]] = {}
|
|
401
|
+
|
|
402
|
+
for exposure_id, exposure_data in self.manifest.get("exposures", {}).items():
|
|
403
|
+
exposure_name = exposure_data.get("name")
|
|
404
|
+
if not exposure_name:
|
|
405
|
+
continue
|
|
406
|
+
|
|
407
|
+
exposure_deps[exposure_name] = set()
|
|
408
|
+
|
|
409
|
+
depends_on = exposure_data.get("depends_on", {})
|
|
410
|
+
for dep_id in depends_on.get("nodes", []):
|
|
411
|
+
parts = dep_id.split(".")
|
|
412
|
+
if parts[0] == "model":
|
|
413
|
+
dep_name = parts[-1].lower()
|
|
414
|
+
exposure_deps[exposure_name].add(dep_name)
|
|
415
|
+
elif parts[0] == "source":
|
|
416
|
+
source_node = self.manifest.get("sources", {}).get(dep_id, {})
|
|
417
|
+
source_identifier = source_node.get("identifier")
|
|
418
|
+
if source_identifier:
|
|
419
|
+
exposure_deps[exposure_name].add(source_identifier.lower())
|
|
420
|
+
else:
|
|
421
|
+
source_name = parts[-1].lower()
|
|
422
|
+
exposure_deps[exposure_name].add(source_name)
|
|
423
|
+
elif parts[0] == "snapshot":
|
|
424
|
+
dep_name = parts[-1].lower()
|
|
425
|
+
exposure_deps[exposure_name].add(dep_name)
|
|
426
|
+
|
|
427
|
+
return exposure_deps
|
|
428
|
+
|
|
429
|
+
def get_model_exposures(self) -> Dict[str, Set[str]]:
|
|
430
|
+
"""Get exposures that depend on each model.
|
|
431
|
+
|
|
432
|
+
Returns:
|
|
433
|
+
Dict[str, Set[str]]: Key is model name, value is set of exposure names that depend on it
|
|
434
|
+
"""
|
|
435
|
+
model_exposures: Dict[str, Set[str]] = {}
|
|
436
|
+
|
|
437
|
+
exposure_deps = self.get_exposure_dependencies()
|
|
438
|
+
|
|
439
|
+
for exposure_name, model_names in exposure_deps.items():
|
|
440
|
+
for model_name in model_names:
|
|
441
|
+
if model_name not in model_exposures:
|
|
442
|
+
model_exposures[model_name] = set()
|
|
443
|
+
model_exposures[model_name].add(exposure_name)
|
|
444
|
+
|
|
445
|
+
return model_exposures
|