tff-sqlmesh 0.2.0__py3-none-any.whl

tff/sqlmesh/__init__.py

@@ -0,0 +1,9 @@
+"""SQLMesh adapter for Transformation Fitness Functions (tff)."""
+
+from tff.sqlmesh.loader import FitnessLoader
+from tff.sqlmesh.runner import run_all_checks
+
+__all__ = [
+    "FitnessLoader",
+    "run_all_checks",
+]

tff/sqlmesh/cli.py

@@ -0,0 +1,90 @@
+"""Command-line interface for tff-sqlmesh."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+from tff.core.config import load_fitness_config
+from tff.core.context import set_ff_config
+from tff.core.report import render_lint_report
+from tff.sqlmesh.runner import run_all_checks
+
+
+def _parse_checks(value: str | None) -> list[str] | None:
+    if not value:
+        return None
+    return [part.strip() for part in value.split(",") if part.strip()]
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="tff-sqlmesh",
+        description="Run SQLMesh Transformation Fitness Function (tff) checks",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    lint_parser = subparsers.add_parser("lint", help="Run all enabled fitness checks")
+    lint_parser.add_argument(
+        "--project",
+        type=Path,
+        default=Path.cwd(),
+        help="SQLMesh project root (default: current directory)",
+    )
+    lint_parser.add_argument(
+        "--config",
+        default="fitness_functions.yaml",
+        help="Path to fitness_functions.yaml (relative to project root)",
+    )
+    lint_parser.add_argument(
+        "--checks",
+        default=None,
+        help="Comma-separated checks to run (default: all enabled). "
+        "Use 'sqlmesh' for SQLMesh linter rules only.",
+    )
+    lint_parser.add_argument(
+        "--fail-level",
+        choices=["error", "warning"],
+        default="error",
+        help="Exit non-zero when findings at or above this severity exist",
+    )
+    lint_parser.add_argument(
+        "--group-by",
+        choices=["connascence", "model"],
+        default="connascence",
+        help="How to group violations in the report (default: connascence)",
+    )
+
+    args = parser.parse_args(argv)
+
+    if args.command == "lint":
+        logging.basicConfig(level=logging.ERROR)
+        project_root = args.project.resolve()
+        config = load_fitness_config(
+            project_root,
+            config_path=args.config,
+        )
+        set_ff_config(config)
+        checks = _parse_checks(args.checks)
+
+        findings, models_checked, executed_checks = run_all_checks(
+            project_root=project_root,
+            config=config,
+            checks=checks,
+        )
+        passed = render_lint_report(
+            findings,
+            models_checked=models_checked,
+            executed_checks=executed_checks,
+            fail_level=args.fail_level,  # type: ignore[arg-type]
+            group_by=args.group_by,  # type: ignore[arg-type]
+        )
+        return 0 if passed else 1
+
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tff/sqlmesh/loader.py

@@ -0,0 +1,107 @@
+"""SQLMesh loader that wraps tff-core package rules and registers them."""
+
+from __future__ import annotations
+
+import os
+import typing as t
+from pathlib import Path
+
+from sqlglot.helper import subclasses
+from sqlmesh.core import constants as c
+from sqlmesh.core.linter.definition import RuleSet
+from sqlmesh.core.linter.rule import Rule as SqlMeshRule, RuleViolation as SqlMeshRuleViolation
+from sqlmesh.core.model import Model as SqlMeshModel
+from sqlmesh.core.loader import SqlMeshLoader
+from sqlmesh.utils import UniqueKeyDict
+from sqlmesh.utils.metaprogramming import import_python_file
+
+from tff.core.config import load_fitness_config
+from tff.core.context import set_ff_config
+from tff.core.model import ModelRepresentation
+from tff.core.rules import ALL_RULES as CORE_RULES
+
+
+def map_sqlmesh_model(model: SqlMeshModel) -> ModelRepresentation:
+    columns_to_types = {
+        name: str(dtype)
+        for name, dtype in (model.columns_to_types or {}).items()
+    }
+    
+    audits = []
+    if model.audits:
+        for audit_name, audit_args in model.audits:
+            audits.append((audit_name, audit_args or {}))
+
+    return ModelRepresentation(
+        name=str(model.name),
+        path=str(model._path),
+        dialect=model.dialect or "bigquery",
+        is_symbolic=bool(model.kind.is_symbolic),
+        is_external=bool(model.kind.name == "EXTERNAL"),
+        columns_to_types=columns_to_types,
+        depends_on={str(dep) for dep in model.depends_on},
+        description=model.description,
+        owner=model.owner,
+        grains=[str(g) for g in (model.grains or [])],
+        audits=audits,
+    )
+
+
+def wrap_core_rule(core_rule_cls) -> type[SqlMeshRule]:
+    def check_model(self, model: SqlMeshModel) -> t.Optional[SqlMeshRuleViolation]:
+        rep = map_sqlmesh_model(model)
+        rule_instance = core_rule_cls()
+        violation = rule_instance.check_model(rep)
+        if violation:
+            return self.violation(violation.violation_msg)
+        return None
+
+    cls_name = core_rule_cls.__name__
+    attrs = {
+        "check_model": check_model,
+        "__doc__": core_rule_cls.__doc__,
+    }
+    return type(cls_name, (SqlMeshRule,), attrs)
+
+
+class FitnessLoader(SqlMeshLoader):
+    """Load core fitness rules adapted for SQLMesh, plus optional project-local rules."""
+
+    def __init__(self, context, path: Path, **loader_kwargs: t.Any) -> None:
+        super().__init__(context, path)
+        config_path = loader_kwargs.get("fitness_functions_config", "fitness_functions.yaml")
+        overrides = {
+            key: value
+            for key, value in loader_kwargs.items()
+            if key != "fitness_functions_config"
+        }
+        ff_config = load_fitness_config(
+            self.config_path,
+            config_path=config_path,
+            overrides=overrides or None,
+        )
+        set_ff_config(ff_config)
+        self._ff_config = ff_config
+
+    def _load_linting_rules(self) -> RuleSet:
+        user_rules: UniqueKeyDict[str, type[SqlMeshRule]] = UniqueKeyDict("rules")
+
+        # Dynamically wrap all tff-core rules to be SQLMesh-compatible
+        for core_rule_cls in CORE_RULES:
+            wrapped = wrap_core_rule(core_rule_cls)
+            user_rules[wrapped.name] = wrapped
+
+        for path in self._glob_paths(
+            self.config_path / c.LINTER,
+            ignore_patterns=self.config.ignore_patterns,
+            extension=".py",
+        ):
+            if os.path.getsize(path):
+                self._track_file(path)
+                module = import_python_file(path, self.config_path)
+                _rule_exclude: t.Set[t.Type[SqlMeshRule]] = {SqlMeshRule}  # type: ignore[type-abstract]
+                module_rules = subclasses(module.__name__, SqlMeshRule, exclude=_rule_exclude)
+                for user_rule in module_rules:
+                    user_rules[user_rule.name] = user_rule
+
+        return RuleSet(user_rules.values())

tff/sqlmesh/runner.py

@@ -0,0 +1,136 @@
+"""Unified lint runner orchestrating SQLMesh rules and architectural checks."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from sqlmesh.core.context import Context
+from sqlmesh.core.linter.definition import AnnotatedRuleViolation
+
+from tff.core.checks.custom_exclusions import collect_custom_exclusion_findings
+from tff.core.checks.dependency_graph import collect_dependency_graph_findings
+from tff.core.checks.layer_integrity import collect_layer_integrity_findings
+from tff.core.checks.schema_contracts import collect_schema_contract_findings
+from tff.core.config import FitnessFunctionsConfig, load_fitness_config
+from tff.core.context import set_ff_config
+from tff.core.report import LintFinding, format_message, normalize_model_name
+from tff.core.utils.paths import model_path_relative
+from tff.core.model import ModelRepresentation
+from tff.sqlmesh.loader import FitnessLoader, map_sqlmesh_model
+
+logger = logging.getLogger(__name__)
+
+CHECK_COLLECTORS = {
+    "layer_integrity": lambda models, cfg: collect_layer_integrity_findings(models, cfg),
+    "custom_exclusions": lambda models, cfg: collect_custom_exclusion_findings(models, cfg),
+    "schema_contracts": lambda _models, cfg: collect_schema_contract_findings(cfg),
+    "dependency_graph": lambda models, cfg: collect_dependency_graph_findings(models, cfg),
+}
+
+
+class _SilentLinterConsole:
+    def show_linter_violations(self, *args, **kwargs) -> None:
+        return None
+
+
+def collect_sqlmesh_findings(context: Context) -> list[LintFinding]:
+    findings: list[LintFinding] = []
+    silent_console = _SilentLinterConsole()
+
+    for model in context.models.values():
+        if model.kind.is_symbolic:
+            continue
+
+        linter = context._linters.get(model.project)
+        if not linter or not linter.enabled:
+            continue
+
+        _, violations = linter.lint_model(model, context, console=silent_console)
+        model_label = normalize_model_name(str(model.name))
+        for violation in violations:
+            if not isinstance(violation, AnnotatedRuleViolation):
+                continue
+
+            message = format_message(violation.violation_msg)
+            if message.startswith(f"{model_label}: "):
+                message = message[len(model_label) + 2 :]
+
+            messages = (
+                [part.strip() for part in message.split(";") if part.strip()]
+                if violation.rule.name == "sqlcomplexity"
+                else [message]
+            )
+
+            for part in messages:
+                findings.append(
+                    LintFinding(
+                        check=violation.rule.name,
+                        severity=violation.violation_type,
+                        model=str(model.name),
+                        path=model_path_relative(model),
+                        message=part,
+                    )
+                )
+
+    return findings
+
+
+def count_models_checked(context: Context) -> int:
+    return sum(
+        1 for model in context.models.values() if not model.kind.is_symbolic
+    )
+
+
+def _check_enabled(config: FitnessFunctionsConfig, check_name: str) -> bool:
+    check = getattr(config.checks, check_name, None)
+    return bool(getattr(check, "enabled", False))
+
+
+def map_sqlmesh_context_models(context: Context) -> dict[str, ModelRepresentation]:
+    mapped = {}
+    for model_name, model in context.models.items():
+        mapped[str(model_name)] = map_sqlmesh_model(model)
+    return mapped
+
+
+def run_all_checks(
+    project_root: Path | None = None,
+    context: Context | None = None,
+    config: FitnessFunctionsConfig | None = None,
+    checks: list[str] | None = None,
+) -> tuple[list[LintFinding], int, list[str]]:
+    project_root = project_root or Path.cwd()
+    if config is None:
+        config = load_fitness_config(project_root)
+    set_ff_config(config)
+
+    context = context or Context(
+        paths=[str(project_root)],
+        loader=FitnessLoader,
+    )
+
+    if checks is None:
+        selected = ["sqlmesh"] + [
+            name
+            for name in CHECK_COLLECTORS
+            if _check_enabled(config, name)
+        ]
+    else:
+        selected = checks
+
+    findings: list[LintFinding] = []
+
+    if "sqlmesh" in selected:
+        findings.extend(collect_sqlmesh_findings(context))
+
+    mapped_models = map_sqlmesh_context_models(context)
+
+    for check_name, collector in CHECK_COLLECTORS.items():
+        if check_name not in selected:
+            continue
+        if checks is None and not _check_enabled(config, check_name):
+            continue
+        findings.extend(collector(mapped_models, config))
+
+    return findings, count_models_checked(context), selected

tff_sqlmesh-0.2.0.dist-info/METADATA

@@ -0,0 +1,222 @@
+Metadata-Version: 2.4
+Name: tff-sqlmesh
+Version: 0.2.0
+Summary: SQLMesh adapter for Transformation Fitness Functions (tff)
+Author-email: Bart Schuijt <schuijt.bart@gmail.com>
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: sqlmesh>=0.234.0
+Requires-Dist: tff-core
+Description-Content-Type: text/markdown
+
+# sqlmesh-ff
+
+[![PyPI version](https://img.shields.io/pypi/v/sqlmesh-ff.svg)](https://pypi.org/project/sqlmesh-ff/)
+[![Python versions](https://img.shields.io/pypi/pyversions/sqlmesh-ff.svg)](https://pypi.org/project/sqlmesh-ff/)
+
+Configurable fitness functions plugin for [SQLMesh](https://sqlmesh.com) projects.
+
+Ships SQLMesh linter rules (classification macros, SQL complexity, metadata, naming) and architectural checks (layer integrity, custom exclusions, schema contracts, dependency graph) with a unified Rich lint report.
+
+## Installation
+
+```bash
+# Install from PyPI:
+uv add sqlmesh-ff
+
+# Or using pip:
+pip install sqlmesh-ff
+```
+
+## Quick start
+
+1. Add `fitness_functions.yaml` to your SQLMesh project root (see [Configuration](#configuration)).
+2. Add a small `config.py` bootstrap (see [Where configuration lives](#where-configuration-lives)) — SQLMesh requires the loader as a Python class and cannot load `config.py` and `config.yaml` in the same folder.
+3. Run lint:
+
+```bash
+sqlmesh-ff lint
+```
+
+## Where configuration lives
+
+There are three layers. Only the YAML/JSON files in your project are user-editable settings.
+
+| Layer | File | Role | You edit this? |
+|-------|------|------|----------------|
+| Plugin defaults | `sqlmesh_ff/config.py` (installed package) | Pydantic schema and built-in defaults (e.g. `fan_out_warn: 15`) | No — library code, never overwritten |
+| SQLMesh project | `settings.yaml` | Gateways, `linter.rules`, variables, CI/CD bot | Yes — normal SQLMesh config |
+| Fitness functions | `fitness_functions.yaml` | Thresholds, rule toggles, column naming/type rules, paths to JSON data | Yes — main FF config |
+| Loader bootstrap | `config.py` (project root) | Loads `settings.yaml` and registers `FitnessLoader` | Rarely — ~15 lines of wiring |
+| Contract data | `linter_contract_groups.json`, `linter_exclusions.json` | Repo-specific schema parity and dependency exclusions | Yes — project data |
+
+**Merge order for fitness settings:** plugin defaults → `fitness_functions.yaml` → optional `loader_kwargs` overrides in `config.py`. Your YAML always wins over plugin defaults. The project `config.py` does not hold fitness thresholds — it only points at `fitness_functions.yaml`.
+
+**Why `config.py` exists:** SQLMesh accepts `loader: FitnessLoader` only as a Python class, not as a YAML string. Because SQLMesh rejects having both `config.py` and `config.yaml` in one folder, projects use `settings.yaml` (SQLMesh settings) plus `config.py` (loader registration).
+
+Example bootstrap:
+
+```python
+from pathlib import Path
+
+from sqlmesh.core.config import Config
+from sqlmesh.utils.yaml import load as yaml_load
+from sqlmesh_ff.loader import FitnessLoader
+
+_settings = yaml_load(Path(__file__).parent / "settings.yaml")
+config = Config.parse_obj(_settings).update_with({
+    "loader": FitnessLoader,
+    "loader_kwargs": {"fitness_functions_config": "fitness_functions.yaml"},
+})
+```
+
+Enable individual SQLMesh rules in `settings.yaml` under `linter.rules` / `linter.warn_rules`.
+
+## Configuration
+
+Fitness function settings live in `fitness_functions.yaml` at the project root. Override the file path or individual keys via `loader_kwargs` in `config.py` (advanced — most projects only set `fitness_functions_config`).
+
+### Example `fitness_functions.yaml`
+
+```yaml
+contract_groups_path: linter_contract_groups.json
+exclusions_path: linter_exclusions.json
+
+layers:
+  order: [sources, derived, core, marts, export]
+
+checks:
+  layer_integrity: { enabled: true }
+  custom_exclusions: { enabled: true }
+  schema_contracts: { enabled: true }
+  dependency_graph:
+    enabled: true
+    fan_out_warn: 15
+    fan_out_fail: 25
+    fan_in_warn: 10
+
+rules:
+  classification_macros:
+    enabled: true
+    skip_layers: [sources]
+    columns:
+      product_type: "@product_type\\b|@PRODUCT_TYPE\\b"
+  sql_complexity:
+    enabled: true
+    thresholds:
+      decision_points: [15, 25]
+      cte_count: [8, 12]
+      join_count: [8, 12]
+      line_count: [250, 400]
+  mart_naming:
+    enabled: true
+    layer_name: marts
+    rule: prefix_with_subdirectory
+  column_names:
+    enabled: true
+    replacements: {}
+  column_types:
+    enabled: true
+    rules: []
+    equivalent_types:
+      text: [text, varchar]
+  metadata:
+    owner: true
+    description: true
+    grain: true
+  filename_equals_modelname:
+    enabled: true
+```
+
+### Project-specific JSON
+
+Keep repo-specific contract and exclusion data in your project:
+
+- `linter_contract_groups.json` — cross-model schema parity groups
+- `linter_exclusions.json` — blocked dependency patterns and allowed exceptions
+
+Reference their paths from `fitness_functions.yaml`. The plugin ships generic engines only; examples live in this README.
+
+### Rule name mapping
+
+SQLMesh uses lowercase class names in `linter.rules`:
+
+| Config key | SQLMesh rule name |
+|------------|-------------------|
+| `classification_macros` | `classificationmacros` |
+| `sql_complexity` | `sqlcomplexity` |
+| `mart_naming` | `martmodelnamingconvention` |
+| `column_names` | `columnnames` |
+| `column_types` | `columntypes` |
+| `metadata.owner` | `nomissingowner` |
+| `metadata.description` | `nomissingdescription` |
+| `metadata.grain` | `nomissinggrain` |
+| `filename_equals_modelname` | `filenameequalsmodelname` |
+
+## CLI
+
+```
+sqlmesh-ff lint [--project PATH] [--config PATH] [--checks CHECK,...] [--fail-level error|warning] [--group-by connascence|model]
+```
+
+- **Default:** all enabled checks plus SQLMesh linter rules
+- **`--checks layer_integrity,custom_exclusions`:** run subset (for pre-push hooks)
+- **`--fail-level warning`:** treat warnings as failures
+- **`--group-by connascence|model`:** change how violations are grouped in the report (default: `connascence`)
+
+## Integration example
+
+Example overrides. `api_request` should always be named `api_call`. `_id` columns should always be of type `text` and `is_` columns should always be of type `boolean`.
+
+```yaml
+column_names:
+  replacements:
+    api_request: api_call
+column_types:
+  rules:
+    - name: id_is_text
+      pattern: "_id$"
+      data_type: text
+    - name: boolean
+      pattern: "^is_"
+      data_type: boolean
+```
+
+## Examples
+
+A complete, runnable example project showcasing the configuration of `sqlmesh-ff` rules, exclusions, contracts, and a continuous integration workflow is located in the [examples/](file:///Users/bartschuijt/git/sqlmesh-ff/examples/) directory.
+
+To run the linter against the example project locally, run:
+```bash
+sqlmesh-ff lint --project examples/minimal-sqlmesh-project
+```
+
+See [examples/minimal-sqlmesh-project/fitness_functions.yaml](file:///Users/bartschuijt/git/sqlmesh-ff/examples/minimal-sqlmesh-project/fitness_functions.yaml) to inspect the configured rules.
+
+## Development
+
+Initialize your local environment and configure the Git pre-push hook:
+```bash
+make init
+```
+
+Run linter, tests, or check diff coverage:
+```bash
+make lint
+make test
+make coverage
+```
+
+### Releases and PR titles
+
+Releases are automated with [release-please](https://github.com/googleapis/release-please) on merges to `main`. Use [Conventional Commits](https://www.conventionalcommits.org/) in PR titles so changelog entries and semver bumps are correct.
+
+PR titles must start with a type prefix, for example:
+
+- `feat: add dependency graph fan-in check`
+- `fix: remove unused import in loader tests`
+- `docs: document fitness_functions.yaml merge order`
+- `ci: add release-please workflow`
+
+Supported types include `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, and `chore`. The PR title check in CI enforces this format.

tff_sqlmesh-0.2.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+tff/sqlmesh/__init__.py,sha256=AYeEn8DxjOf6tyuvdst2vqh8htimcFIrmsqOelp4O9U,216
+tff/sqlmesh/cli.py,sha256=4_7QuKiYTHhpOX-_bAnLox9PQScArreg6aerv4X34t4,2720
+tff/sqlmesh/loader.py,sha256=Ohg-SAACYChphe6hvdRTvlDzMu7vCfF03Tf3OID9sRg,3921
+tff/sqlmesh/runner.py,sha256=w5M_9WCLBVrDFJWPR9wcQcHW9ECv0IqiN2QTXljVas4,4681
+tff_sqlmesh-0.2.0.dist-info/METADATA,sha256=8mNHWXy4v0uUmT-PCcOXJnnTyUMhSAoOrk4L9eEnc38,7944
+tff_sqlmesh-0.2.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+tff_sqlmesh-0.2.0.dist-info/entry_points.txt,sha256=lGH_81NMPZjHJnPOex_2Vu74wIKi1Wq8aFKHmOLN4Wk,53
+tff_sqlmesh-0.2.0.dist-info/licenses/LICENSE,sha256=qJDdrCs67OvVJ8jlstmR3HCqsECKLxOJWsjys5K8tNg,1069
+tff_sqlmesh-0.2.0.dist-info/RECORD,,

tff_sqlmesh-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

tff_sqlmesh-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tff-sqlmesh = tff.sqlmesh.cli:main

tff_sqlmesh-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bart Schuijt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.