tff-core 0.2.0__py3-none-any.whl

tff/core/checks/custom_exclusions.py

@@ -0,0 +1,174 @@
+"""Custom dependency exclusion rules for layer/domain boundaries."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+from tff.core.model import ModelRepresentation
+from tff.core.config import FitnessFunctionsConfig, resolve_project_path
+from tff.core.report import LintFinding
+from tff.core.utils.paths import get_layer_and_domain, model_path_relative
+
+logger = logging.getLogger(__name__)
+
+
+class CustomExclusionsChecker:
+    """Enforce custom exclusions for model dependencies between layers."""
+
+    def __init__(self, models: dict[str, ModelRepresentation], exclusions_path: Path):
+        self.models = models
+        self.exclusions_path = exclusions_path
+        self.exclusions = self._load_exclusions()
+
+    def _load_exclusions(self) -> dict:
+        if not self.exclusions_path.exists():
+            logger.warning(
+                "Config file %s not found. No exclusions will be enforced.",
+                self.exclusions_path,
+            )
+            return {}
+
+        try:
+            with open(self.exclusions_path, encoding="utf-8") as f:
+                return json.load(f)
+        except (json.JSONDecodeError, OSError) as e:
+            logger.warning(
+                "Could not load exclusions config from %s: %s",
+                self.exclusions_path,
+                e,
+            )
+            return {}
+
+    def _normalize_model_name(self, name: str) -> str:
+        parts = name.replace('"', "").split(".")
+        if len(parts) >= 2:
+            return f"{parts[-2]}.{parts[-1]}"
+        return name
+
+    def _is_allowed_exception(self, model_name: str, dependency_name: str) -> bool:
+        normalized_model = self._normalize_model_name(model_name)
+        normalized_dependency = self._normalize_model_name(dependency_name)
+
+        for exception in self.exclusions.get("allowed_exceptions", []):
+            if (
+                exception.get("model") == normalized_model
+                and exception.get("dependency") == normalized_dependency
+            ):
+                return True
+        return False
+
+    def _is_excluded_dependency(
+        self,
+        source_layer: str,
+        source_domain: str,
+        target_layer: str,
+        target_domain: str,
+        model_name: str | None = None,
+        dependency_name: str | None = None,
+    ) -> bool:
+        if model_name and dependency_name:
+            if self._is_allowed_exception(model_name, dependency_name):
+                return False
+
+        for exclusion in self.exclusions.get("exclusions", []):
+            source_match = True
+            if (
+                "source_layer" in exclusion
+                and exclusion["source_layer"] != source_layer
+            ):
+                source_match = False
+            if (
+                "source_domain" in exclusion
+                and exclusion["source_domain"] != source_domain
+            ):
+                source_match = False
+
+            target_match = True
+            if (
+                "target_layer" in exclusion
+                and exclusion["target_layer"] != target_layer
+            ):
+                target_match = False
+            if (
+                "target_domain" in exclusion
+                and exclusion["target_domain"] != target_domain
+            ):
+                target_match = False
+
+            if source_match and target_match:
+                return True
+
+        return False
+
+    def check_model(self, model: ModelRepresentation) -> list[str]:
+        if model.is_symbolic:
+            return []
+
+        violations = []
+        model_layer, model_domain = get_layer_and_domain(model.path)
+        if not model_layer:
+            return []
+
+        for dependency_name in model.depends_on:
+            try:
+                dependency_model = self.models.get(dependency_name)
+                if not dependency_model:
+                    continue
+
+                dep_layer, dep_domain = get_layer_and_domain(dependency_model.path)
+                if not dep_layer:
+                    continue
+
+                if self._is_excluded_dependency(
+                    source_layer=dep_layer,
+                    source_domain=dep_domain or "",
+                    target_layer=model_layer,
+                    target_domain=model_domain or "",
+                    model_name=str(model.name),
+                    dependency_name=str(dependency_name),
+                ):
+                    violations.append(
+                        f"Model '{model.name}' in layer '{model_layer}"
+                        f"{f'/{model_domain}' if model_domain else ''}' "
+                        f"depends on '{dependency_name}' in layer '{dep_layer}"
+                        f"{f'/{dep_domain}' if dep_domain else ''}', "
+                        f"which is not allowed by custom exclusions"
+                    )
+            except Exception as e:
+                logger.error(
+                    "Unexpected error checking dependency %s for model %s: %s",
+                    dependency_name,
+                    model.name,
+                    e,
+                    exc_info=True,
+                )
+                continue
+
+        return violations
+
+
+def collect_custom_exclusion_findings(
+    models: dict[str, ModelRepresentation], config: FitnessFunctionsConfig
+) -> list[LintFinding]:
+    exclusions_path = resolve_project_path(config, config.exclusions_path)
+    checker = CustomExclusionsChecker(models, exclusions_path)
+    findings: list[LintFinding] = []
+
+    for model_name, model in models.items():
+        if model.is_symbolic:
+            continue
+
+        for message in checker.check_model(model):
+            findings.append(
+                LintFinding(
+                    check="custom_exclusions",
+                    severity="error",
+                    model=str(model.name),
+                    path=model_path_relative(model),
+                    message=message.removeprefix(f"Model '{model.name}' ").strip(),
+                )
+            )
+
+    return findings

tff/core/checks/dependency_graph.py

@@ -0,0 +1,78 @@
+"""Dependency graph fan-in/fan-out metrics."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+
+from tff.core.model import ModelRepresentation
+from tff.core.config import FitnessFunctionsConfig
+from tff.core.report import LintFinding
+from tff.core.utils.paths import model_path_relative
+
+
+def collect_dependency_graph_findings(
+    models: dict[str, ModelRepresentation], config: FitnessFunctionsConfig
+) -> list[LintFinding]:
+    graph_config = config.checks.dependency_graph
+    reverse: dict[str, set[str]] = defaultdict(set)
+    for model_name, model in models.items():
+        if model.is_symbolic:
+            continue
+        for dependency in model.depends_on:
+            reverse[str(dependency)].add(str(model_name))
+
+    findings: list[LintFinding] = []
+    for model_name, model in models.items():
+        if model.is_symbolic:
+            continue
+
+        from tff.core.utils.paths import get_layer_from_path
+        layer = get_layer_from_path(model.path)
+        if not graph_config.should_run(layer):
+            continue
+
+        fan_in = len(model.depends_on)
+        fan_out = len(reverse.get(str(model_name), set()))
+
+        if fan_out > graph_config.fan_out_fail:
+            findings.append(
+                LintFinding(
+                    check="dependency_graph",
+                    severity="error",
+                    model=str(model_name),
+                    path=model_path_relative(model),
+                    message=(
+                        f"fan_out={fan_out} (fail>{graph_config.fan_out_fail}) — "
+                        "high blast-radius hub model"
+                    ),
+                )
+            )
+        elif fan_out > graph_config.fan_out_warn:
+            findings.append(
+                LintFinding(
+                    check="dependency_graph",
+                    severity="warning",
+                    model=str(model_name),
+                    path=model_path_relative(model),
+                    message=(
+                        f"fan_out={fan_out} (warn>{graph_config.fan_out_warn}) — "
+                        "run impact analysis before changing"
+                    ),
+                )
+            )
+
+        if fan_in > graph_config.fan_in_warn:
+            findings.append(
+                LintFinding(
+                    check="dependency_graph",
+                    severity="warning",
+                    model=str(model_name),
+                    path=model_path_relative(model),
+                    message=(
+                        f"fan_in={fan_in} (warn>{graph_config.fan_in_warn}) — "
+                        "consider decomposing upstream dependencies"
+                    ),
+                )
+            )
+
+    return findings

tff/core/checks/layer_integrity.py

@@ -0,0 +1,98 @@
+"""Layer integrity check — unidirectional flow and mart domain isolation."""
+
+from __future__ import annotations
+
+from tff.core.model import ModelRepresentation
+from tff.core.config import FitnessFunctionsConfig
+from tff.core.report import LintFinding, normalize_model_name
+from tff.core.utils.paths import (
+    get_layer_from_path,
+    get_marts_domain_from_path,
+    model_path_relative,
+)
+
+
+def _layer_index(
+    layer: str | None, dependency_model_kind: str, layer_order: list[str]
+) -> int | None:
+    layer_index = {name: idx for idx, name in enumerate(layer_order)}
+    if layer:
+        return layer_index.get(layer)
+    if dependency_model_kind == "EXTERNAL":
+        return -1
+    return None
+
+
+def collect_layer_integrity_findings(
+    models: dict[str, ModelRepresentation], config: FitnessFunctionsConfig
+) -> list[LintFinding]:
+    findings: list[LintFinding] = []
+    layer_order = config.layers.order
+    marts_layer = config.rules.mart_naming.layer_name
+
+    for model_name, model in models.items():
+        if model.is_external or model.is_symbolic:
+            continue
+
+        model_layer = get_layer_from_path(model.path, layer_order)
+        model_layer_index = _layer_index(model_layer, "EXTERNAL" if model.is_external else "STANDARD", layer_order)
+        model_marts_domain = (
+            get_marts_domain_from_path(model.path, marts_layer)
+            if model_layer == marts_layer
+            else None
+        )
+
+        for dependency in model.depends_on:
+            dependency_model = models.get(dependency)
+            if not dependency_model:
+                continue
+
+            dependency_layer = get_layer_from_path(dependency_model.path, layer_order)
+            dependency_layer_index = _layer_index(
+                dependency_layer,
+                "EXTERNAL" if dependency_model.is_external else "STANDARD",
+                layer_order,
+            )
+
+            if (
+                model_layer_index is not None
+                and dependency_layer_index is not None
+                and dependency_layer_index > model_layer_index
+            ):
+                findings.append(
+                    LintFinding(
+                        check="layer_integrity",
+                        severity="error",
+                        model=str(model.name),
+                        path=model_path_relative(model),
+                        message=(
+                            f"depends on {normalize_model_name(str(dependency))} "
+                            "in a downstream layer"
+                        ),
+                    )
+                )
+
+            if model_layer == marts_layer and dependency_layer == marts_layer:
+                dependency_marts_domain = get_marts_domain_from_path(
+                    dependency_model.path, marts_layer
+                )
+                if (
+                    model_marts_domain
+                    and dependency_marts_domain
+                    and model_marts_domain != dependency_marts_domain
+                ):
+                    findings.append(
+                        LintFinding(
+                            check="layer_integrity",
+                            severity="error",
+                            model=str(model.name),
+                            path=model_path_relative(model),
+                            message=(
+                                f"{marts_layer}/{model_marts_domain} depends on "
+                                f"{normalize_model_name(str(dependency))} "
+                                f"({marts_layer}/{dependency_marts_domain})"
+                            ),
+                        )
+                    )
+
+    return findings

tff/core/checks/schema_contracts.py

@@ -0,0 +1,106 @@
+"""Cross-model schema contract parity checks."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from tff.core.config import (
+    FitnessFunctionsConfig,
+    _ensure_under_root,
+    resolve_project_path,
+)
+from tff.core.report import LintFinding
+from tff.core.utils.schema_contract_utils import (
+    check_column_list_parity,
+    check_dimension_set_parity,
+    extract_final_select_columns,
+    normalize_columns,
+    read_model_sql,
+)
+
+
+def _resolve_path(project_root: Path, models_dir: str, filename: str) -> Path:
+    return _ensure_under_root(project_root / models_dir / filename, project_root)
+
+
+def _schema_contract_errors(project_root: Path, contract_config: dict) -> list[str]:
+    errors: list[str] = []
+    for group in contract_config.get("column_parity_groups", []):
+        models_dir = group["models_dir"]
+        reference_path = _resolve_path(project_root, models_dir, group["reference"])
+        if not reference_path.exists():
+            errors.append(f"{reference_path.name} not found")
+            continue
+
+        exclude = set(group.get("exclude_columns", []))
+        ref_substitutions = group.get("reference_substitutions", {})
+        reference_cols = normalize_columns(
+            extract_final_select_columns(read_model_sql(reference_path)),
+            substitutions=ref_substitutions,
+            exclude=exclude,
+        )
+
+        for member in group["members"]:
+            member_path = _resolve_path(project_root, models_dir, member["file"])
+            if not member_path.exists():
+                errors.append(f"{member_path.name} not found")
+                continue
+            member_cols = normalize_columns(
+                extract_final_select_columns(read_model_sql(member_path)),
+                substitutions=member.get("substitutions", {}),
+                exclude=exclude,
+            )
+            errors.extend(
+                check_column_list_parity(
+                    reference_cols,
+                    member_cols,
+                    group["reference"],
+                    member["file"],
+                )
+            )
+
+    for group in contract_config.get("dimension_parity_groups", []):
+        models_dir = group["models_dir"]
+        left_cfg = group["left"]
+        right_cfg = group["right"]
+        left_path = _resolve_path(project_root, models_dir, left_cfg["file"])
+        right_path = _resolve_path(project_root, models_dir, right_cfg["file"])
+        if not left_path.exists() or not right_path.exists():
+            continue
+
+        left_dims = set(extract_final_select_columns(read_model_sql(left_path))) - set(
+            left_cfg.get("exclude_columns", [])
+        )
+        right_dims = set(extract_final_select_columns(read_model_sql(right_path))) - set(
+            right_cfg.get("exclude_columns", [])
+        )
+        errors.extend(
+            check_dimension_set_parity(
+                left_dims,
+                right_dims,
+                left_cfg["file"],
+                right_cfg["file"],
+            )
+        )
+
+    return errors
+
+
+def collect_schema_contract_findings(
+    config: FitnessFunctionsConfig,
+) -> list[LintFinding]:
+    project_root: Path = getattr(config, "_project_root", Path.cwd())
+    contract_path = resolve_project_path(config, config.contract_groups_path)
+    if not contract_path.exists():
+        return []
+
+    contract_config = json.loads(contract_path.read_text(encoding="utf-8"))
+    return [
+        LintFinding(
+            check="schema_contracts",
+            severity="error",
+            message=error.replace("\n", " — "),
+        )
+        for error in _schema_contract_errors(project_root, contract_config)
+    ]

tff/core/config.py

@@ -0,0 +1,200 @@
+"""Pydantic models and loaders for fitness_functions.yaml."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class LayersConfig(BaseModel):
+    order: list[str] = Field(
+        default_factory=lambda: ["sources", "derived", "core", "marts", "export"]
+    )
+
+
+class CheckEnabled(BaseModel):
+    enabled: bool = True
+
+
+class LayerFilterConfig(BaseModel):
+    enabled: bool = True
+    skip_layers: list[str] = Field(default_factory=list)
+    only_layers: list[str] | None = None
+
+    def should_run(self, layer: str | None) -> bool:
+        if not self.enabled:
+            return False
+        if layer is None:
+            return True
+        if self.only_layers is not None:
+            return layer in self.only_layers
+        return layer not in self.skip_layers
+
+
+class DependencyGraphCheckConfig(LayerFilterConfig):
+    fan_out_warn: int = 15
+    fan_out_fail: int = 25
+    fan_in_warn: int = 10
+
+
+class ChecksConfig(BaseModel):
+    layer_integrity: CheckEnabled = Field(default_factory=CheckEnabled)
+    custom_exclusions: CheckEnabled = Field(default_factory=CheckEnabled)
+    schema_contracts: CheckEnabled = Field(default_factory=CheckEnabled)
+    dependency_graph: DependencyGraphCheckConfig = Field(
+        default_factory=DependencyGraphCheckConfig
+    )
+
+
+class ClassificationMacrosRuleConfig(LayerFilterConfig):
+    skip_layers: list[str] = Field(default_factory=lambda: ["sources"])
+    columns: dict[str, str] = Field(
+        default_factory=lambda: {
+            "product_type": r"@product_type\b|@PRODUCT_TYPE\b",
+            "billing_segment": r"@BILLING_SEGMENT\b|@billing_segment\b",
+            "industry": r"@INDUSTRY\b|@industry\b",
+        }
+    )
+
+
+class SqlComplexityRuleConfig(LayerFilterConfig):
+    warn_only: bool = True
+    thresholds: dict[str, list[int]] = Field(
+        default_factory=lambda: {
+            "decision_points": [15, 25],
+            "cte_count": [8, 12],
+            "join_count": [8, 12],
+            "line_count": [250, 400],
+        }
+    )
+
+
+class MartNamingRuleConfig(LayerFilterConfig):
+    layer_name: str = "marts"
+    rule: str = "prefix_with_subdirectory"
+
+
+class ColumnNamesRuleConfig(LayerFilterConfig):
+    replacements: dict[str, str] = Field(default_factory=dict)
+
+
+class ColumnTypeRuleEntry(BaseModel):
+    name: str
+    pattern: str
+    data_type: str
+
+
+class ColumnTypesRuleConfig(LayerFilterConfig):
+    rules: list[ColumnTypeRuleEntry] = Field(default_factory=list)
+    equivalent_types: dict[str, list[str]] = Field(
+        default_factory=lambda: {"text": ["text", "varchar"]}
+    )
+
+
+class MetadataRuleConfig(LayerFilterConfig):
+    owner: bool = True
+    description: bool = True
+    grain: bool = True
+    not_null: bool = True
+    unique_values: bool = True
+
+
+class FilenameEqualsModelnameRuleConfig(LayerFilterConfig):
+    pass
+
+
+class NoSelectStarRuleConfig(LayerFilterConfig):
+    skip_layers: list[str] = Field(default_factory=lambda: ["sources"])
+
+
+class NoPositionalGroupByOrOrderByRuleConfig(LayerFilterConfig):
+    skip_layers: list[str] = Field(default_factory=lambda: ["sources"])
+
+
+class RulesConfig(BaseModel):
+    classification_macros: ClassificationMacrosRuleConfig = Field(
+        default_factory=ClassificationMacrosRuleConfig
+    )
+    sql_complexity: SqlComplexityRuleConfig = Field(
+        default_factory=SqlComplexityRuleConfig
+    )
+    mart_naming: MartNamingRuleConfig = Field(default_factory=MartNamingRuleConfig)
+    column_names: ColumnNamesRuleConfig = Field(default_factory=ColumnNamesRuleConfig)
+    column_types: ColumnTypesRuleConfig = Field(default_factory=ColumnTypesRuleConfig)
+    metadata: MetadataRuleConfig = Field(default_factory=MetadataRuleConfig)
+    filename_equals_modelname: FilenameEqualsModelnameRuleConfig = Field(
+        default_factory=FilenameEqualsModelnameRuleConfig
+    )
+    no_select_star: NoSelectStarRuleConfig = Field(
+        default_factory=NoSelectStarRuleConfig
+    )
+    no_positional_group_by_or_order_by: NoPositionalGroupByOrOrderByRuleConfig = Field(
+        default_factory=NoPositionalGroupByOrOrderByRuleConfig
+    )
+
+
+class FitnessFunctionsConfig(BaseModel):
+    contract_groups_path: str = "linter_contract_groups.json"
+    exclusions_path: str = "linter_exclusions.json"
+    layers: LayersConfig = Field(default_factory=LayersConfig)
+    checks: ChecksConfig = Field(default_factory=ChecksConfig)
+    rules: RulesConfig = Field(default_factory=RulesConfig)
+
+
+def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    merged = dict(base)
+    for key, value in override.items():
+        if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
+            merged[key] = _deep_merge(merged[key], value)
+        else:
+            merged[key] = value
+    return merged
+
+
+def load_fitness_config(
+    project_root: Path,
+    config_path: str | Path | None = "fitness_functions.yaml",
+    overrides: dict[str, Any] | None = None,
+) -> FitnessFunctionsConfig:
+    """Load fitness config with defaults, yaml file, and optional overrides."""
+    data: dict[str, Any] = {}
+
+    if config_path is not None:
+        yaml_path = Path(config_path)
+        if not yaml_path.is_absolute():
+            yaml_path = project_root / yaml_path
+        if yaml_path.exists():
+            loaded = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) or {}
+            if not isinstance(loaded, dict):
+                raise ValueError(f"Expected mapping in {yaml_path}")
+            data = loaded
+
+    if overrides:
+        data = _deep_merge(data, overrides)
+
+    config = FitnessFunctionsConfig.model_validate(data)
+    config._project_root = project_root  # type: ignore[attr-defined]
+    return config
+
+
+def _ensure_under_root(path: Path, root: Path) -> Path:
+    resolved = path.resolve()
+    root_resolved = root.resolve()
+    try:
+        resolved.relative_to(root_resolved)
+    except ValueError:
+        raise ValueError(
+            f"Path {path} resolves outside project root {root}"
+        ) from None
+    return resolved
+
+
+def resolve_project_path(config: FitnessFunctionsConfig, relative: str) -> Path:
+    root: Path = getattr(config, "_project_root", Path.cwd())
+    path = Path(relative)
+    if path.is_absolute():
+        return _ensure_under_root(path, root)
+    return _ensure_under_root(root / path, root)

tff/core/context.py

@@ -0,0 +1,30 @@
+"""Thread-local fitness function configuration for SQLMesh rule classes."""
+
+from __future__ import annotations
+
+import threading
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from tff.core.config import FitnessFunctionsConfig
+
+_config_local = threading.local()
+
+
+def set_ff_config(config: FitnessFunctionsConfig) -> None:
+    _config_local.config = config
+
+
+def get_ff_config() -> FitnessFunctionsConfig:
+    from tff.core.config import FitnessFunctionsConfig
+
+    config = getattr(_config_local, "config", None)
+    if config is None:
+        config = FitnessFunctionsConfig()
+        _config_local.config = config
+    return config
+
+
+def clear_ff_config() -> None:
+    if hasattr(_config_local, "config"):
+        del _config_local.config