dbt-vitals 0.1.0__tar.gz

dbt_vitals-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shreeti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dbt_vitals-0.1.0/PKG-INFO

@@ -0,0 +1,221 @@
+Metadata-Version: 2.4
+Name: dbt-vitals
+Version: 0.1.0
+Summary: Static analysis and health scoring for dbt projects — lineage, test coverage, duplicate logic, and documentation checks.
+Author: Shreeti
+License: MIT
+Project-URL: Homepage, https://github.com/shivah12/dbt-vitals
+Project-URL: Issues, https://github.com/shivah12/dbt-vitals/issues
+Keywords: dbt,data-engineering,static-analysis,sql,linter,data-quality
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Database
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.1
+Requires-Dist: rich>=13.0
+Requires-Dist: networkx>=3.0
+Requires-Dist: sqlglot>=23.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: build>=1.0; extra == "dev"
+Requires-Dist: twine>=4.0; extra == "dev"
+Dynamic: license-file
+
+# dbt-vitals
+
+Static analysis and health scoring for dbt projects — the "cargo clippy for dbt."
+
+Unlike a SQL linter (indentation, keyword casing), dbt-vitals looks at your
+project's *structure*: dead models, missing tests, duplicated business logic,
+documentation coverage, and (given warehouse stats) incremental-model
+candidates. It parses `manifest.json` / `catalog.json` — the artifacts dbt
+already generates — so it needs no warehouse credentials of its own.
+
+```
+Overall Health
+76/100
+Warnings: 8   Critical: 0   Info: 4
+```
+
+## What it checks
+
+| Module | What it flags | Needs `catalog.json`? |
+|---|---|---|
+| Lineage | Dead/unused models, circular dependencies | No |
+| Testing | Models missing `unique`/`not_null` on their likely primary key | No |
+| Duplicate Logic | Repeated `CASE WHEN` blocks across models (candidates for a macro) | No |
+| Documentation | Model/column description coverage % | No |
+| Incremental Candidates | Large `table`-materialized models that could be `incremental` | Yes |
+
+## Local setup
+
+Requires Python 3.9+.
+
+```bash
+git clone https://github.com/YOUR_GITHUB_USERNAME/dbt-vitals.git
+cd dbt-vitals
+
+python -m venv .venv
+source .venv/bin/activate        # Windows: .venv\Scripts\activate
+
+pip install -e ".[dev]"
+```
+
+Run the test suite against the bundled synthetic example project:
+
+```bash
+pytest
+```
+
+Try the CLI against the bundled example (no real dbt project needed):
+
+```bash
+dbt-vitals analyze . --manifest examples/sample_manifest.json --catalog examples/sample_catalog.json
+```
+
+Run it against a real dbt project:
+
+```bash
+cd /path/to/your/dbt/project
+dbt compile               # or `dbt docs generate` to also get catalog.json
+dbt-vitals analyze .
+```
+
+### CLI options
+
+```bash
+dbt-vitals analyze <target_dir>          # target_dir defaults to "."
+  --manifest PATH        # override manifest.json location
+  --catalog PATH         # override catalog.json location
+  --json                 # machine-readable output
+  --ci-comment           # markdown summary for a PR comment
+  --fail-under N         # exit 1 if health score < N (CI gating)
+```
+
+## CI integration
+
+`.github/workflows/dbt-vitals.yml` is included — it runs `dbt-vitals` on every
+PR, posts the health score as a comment, and fails the build if the score
+drops below a threshold. Adjust the `dbt compile` step for your adapter/profile.
+
+## Publishing to PyPI
+
+This is a **Python** package, so it's published to [PyPI](https://pypi.org),
+not npm — `pip install dbt-vitals` is the equivalent of `npm install`.
+
+1. Create accounts at [pypi.org](https://pypi.org/account/register/) and
+   [test.pypi.org](https://test.pypi.org/account/register/) (the sandbox —
+   publish here first to make sure everything works).
+2. Generate an API token: PyPI account settings → API tokens → scope it to
+   this project (after the first upload) or "entire account" (for the first
+   upload, since the project doesn't exist yet).
+3. Update `pyproject.toml`: bump `version`, fix the `Homepage`/`Issues` URLs
+   to your actual GitHub repo, add a real `authors` email if you want one.
+4. Build and check the package:
+
+   ```bash
+   pip install build twine
+   python -m build                # creates dist/*.whl and dist/*.tar.gz
+   twine check dist/*
+   ```
+
+5. Upload to TestPyPI first:
+
+   ```bash
+   twine upload --repository testpypi dist/*
+   # username: __token__
+   # password: <your TestPyPI API token, including the pypi- prefix>
+   ```
+
+   Verify it installs cleanly:
+
+   ```bash
+   pip install --index-url https://test.pypi.org/simple/ --no-deps dbt-vitals
+   dbt-vitals --version
+   ```
+
+6. Once that works, upload for real:
+
+   ```bash
+   twine upload dist/*
+   # username: __token__
+   # password: <your PyPI API token>
+   ```
+
+7. From then on, anyone can install it with:
+
+   ```bash
+   pip install dbt-vitals
+   ```
+
+To ship a new version later: bump `version` in `pyproject.toml`, delete the
+old `dist/` folder, rebuild (`python -m build`), and `twine upload dist/*`
+again — PyPI rejects re-uploading an existing version number.
+
+## Pushing to GitHub
+
+```bash
+cd dbt-vitals
+git init
+git add .
+git commit -m "Initial commit: dbt-vitals MVP"
+
+# Create the repo on GitHub first (via github.com or `gh repo create`),
+# then:
+git remote add origin https://github.com/YOUR_GITHUB_USERNAME/dbt-vitals.git
+git branch -M main
+git push -u origin main
+```
+
+If you use the [GitHub CLI](https://cli.github.com/) instead of the website:
+
+```bash
+gh repo create dbt-vitals --public --source=. --remote=origin --push
+```
+
+## Project layout
+
+```
+dbt-vitals/
+  dbt_doctor/
+    cli.py              # click CLI entry point
+    parser.py           # manifest.json / catalog.json loading
+    graph.py            # dependency graph (networkx)
+    scoring.py           # health score calculation
+    report.py           # terminal + CI markdown rendering
+    findings.py          # shared Finding data structure
+    modules/
+      lineage.py         # dead models, circular deps
+      testing.py         # missing test coverage
+      duplicates.py       # repeated CASE-WHEN logic (via sqlglot AST)
+      documentation.py    # doc coverage
+      incremental.py       # incremental-model candidates
+  examples/               # synthetic manifest/catalog for demos & tests
+  tests/                  # pytest suite
+  .github/workflows/       # CI action
+```
+
+## Known limitations (by design, for v0.1)
+
+- **Duplicate detection** only looks at `CASE WHEN` expressions, not arbitrary
+  repeated subqueries or joins.
+- **Incremental candidates** use a fixed row-count threshold, not a real
+  cost/runtime estimate — that would require warehouse-specific query plans.
+- **Primary key inference** for the testing module is a naming heuristic
+  (`id`, `<model>_id`, or any `*_id` column) since dbt's manifest has no
+  first-class primary key concept.
+- No plugin system yet (warehouse-specific checks) — deliberately deferred,
+  see the original scoping notes for why.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

dbt_vitals-0.1.0/README.md

@@ -0,0 +1,189 @@
+# dbt-vitals
+
+Static analysis and health scoring for dbt projects — the "cargo clippy for dbt."
+
+Unlike a SQL linter (indentation, keyword casing), dbt-vitals looks at your
+project's *structure*: dead models, missing tests, duplicated business logic,
+documentation coverage, and (given warehouse stats) incremental-model
+candidates. It parses `manifest.json` / `catalog.json` — the artifacts dbt
+already generates — so it needs no warehouse credentials of its own.
+
+```
+Overall Health
+76/100
+Warnings: 8   Critical: 0   Info: 4
+```
+
+## What it checks
+
+| Module | What it flags | Needs `catalog.json`? |
+|---|---|---|
+| Lineage | Dead/unused models, circular dependencies | No |
+| Testing | Models missing `unique`/`not_null` on their likely primary key | No |
+| Duplicate Logic | Repeated `CASE WHEN` blocks across models (candidates for a macro) | No |
+| Documentation | Model/column description coverage % | No |
+| Incremental Candidates | Large `table`-materialized models that could be `incremental` | Yes |
+
+## Local setup
+
+Requires Python 3.9+.
+
+```bash
+git clone https://github.com/YOUR_GITHUB_USERNAME/dbt-vitals.git
+cd dbt-vitals
+
+python -m venv .venv
+source .venv/bin/activate        # Windows: .venv\Scripts\activate
+
+pip install -e ".[dev]"
+```
+
+Run the test suite against the bundled synthetic example project:
+
+```bash
+pytest
+```
+
+Try the CLI against the bundled example (no real dbt project needed):
+
+```bash
+dbt-vitals analyze . --manifest examples/sample_manifest.json --catalog examples/sample_catalog.json
+```
+
+Run it against a real dbt project:
+
+```bash
+cd /path/to/your/dbt/project
+dbt compile               # or `dbt docs generate` to also get catalog.json
+dbt-vitals analyze .
+```
+
+### CLI options
+
+```bash
+dbt-vitals analyze <target_dir>          # target_dir defaults to "."
+  --manifest PATH        # override manifest.json location
+  --catalog PATH         # override catalog.json location
+  --json                 # machine-readable output
+  --ci-comment           # markdown summary for a PR comment
+  --fail-under N         # exit 1 if health score < N (CI gating)
+```
+
+## CI integration
+
+`.github/workflows/dbt-vitals.yml` is included — it runs `dbt-vitals` on every
+PR, posts the health score as a comment, and fails the build if the score
+drops below a threshold. Adjust the `dbt compile` step for your adapter/profile.
+
+## Publishing to PyPI
+
+This is a **Python** package, so it's published to [PyPI](https://pypi.org),
+not npm — `pip install dbt-vitals` is the equivalent of `npm install`.
+
+1. Create accounts at [pypi.org](https://pypi.org/account/register/) and
+   [test.pypi.org](https://test.pypi.org/account/register/) (the sandbox —
+   publish here first to make sure everything works).
+2. Generate an API token: PyPI account settings → API tokens → scope it to
+   this project (after the first upload) or "entire account" (for the first
+   upload, since the project doesn't exist yet).
+3. Update `pyproject.toml`: bump `version`, fix the `Homepage`/`Issues` URLs
+   to your actual GitHub repo, add a real `authors` email if you want one.
+4. Build and check the package:
+
+   ```bash
+   pip install build twine
+   python -m build                # creates dist/*.whl and dist/*.tar.gz
+   twine check dist/*
+   ```
+
+5. Upload to TestPyPI first:
+
+   ```bash
+   twine upload --repository testpypi dist/*
+   # username: __token__
+   # password: <your TestPyPI API token, including the pypi- prefix>
+   ```
+
+   Verify it installs cleanly:
+
+   ```bash
+   pip install --index-url https://test.pypi.org/simple/ --no-deps dbt-vitals
+   dbt-vitals --version
+   ```
+
+6. Once that works, upload for real:
+
+   ```bash
+   twine upload dist/*
+   # username: __token__
+   # password: <your PyPI API token>
+   ```
+
+7. From then on, anyone can install it with:
+
+   ```bash
+   pip install dbt-vitals
+   ```
+
+To ship a new version later: bump `version` in `pyproject.toml`, delete the
+old `dist/` folder, rebuild (`python -m build`), and `twine upload dist/*`
+again — PyPI rejects re-uploading an existing version number.
+
+## Pushing to GitHub
+
+```bash
+cd dbt-vitals
+git init
+git add .
+git commit -m "Initial commit: dbt-vitals MVP"
+
+# Create the repo on GitHub first (via github.com or `gh repo create`),
+# then:
+git remote add origin https://github.com/YOUR_GITHUB_USERNAME/dbt-vitals.git
+git branch -M main
+git push -u origin main
+```
+
+If you use the [GitHub CLI](https://cli.github.com/) instead of the website:
+
+```bash
+gh repo create dbt-vitals --public --source=. --remote=origin --push
+```
+
+## Project layout
+
+```
+dbt-vitals/
+  dbt_doctor/
+    cli.py              # click CLI entry point
+    parser.py           # manifest.json / catalog.json loading
+    graph.py            # dependency graph (networkx)
+    scoring.py           # health score calculation
+    report.py           # terminal + CI markdown rendering
+    findings.py          # shared Finding data structure
+    modules/
+      lineage.py         # dead models, circular deps
+      testing.py         # missing test coverage
+      duplicates.py       # repeated CASE-WHEN logic (via sqlglot AST)
+      documentation.py    # doc coverage
+      incremental.py       # incremental-model candidates
+  examples/               # synthetic manifest/catalog for demos & tests
+  tests/                  # pytest suite
+  .github/workflows/       # CI action
+```
+
+## Known limitations (by design, for v0.1)
+
+- **Duplicate detection** only looks at `CASE WHEN` expressions, not arbitrary
+  repeated subqueries or joins.
+- **Incremental candidates** use a fixed row-count threshold, not a real
+  cost/runtime estimate — that would require warehouse-specific query plans.
+- **Primary key inference** for the testing module is a naming heuristic
+  (`id`, `<model>_id`, or any `*_id` column) since dbt's manifest has no
+  first-class primary key concept.
+- No plugin system yet (warehouse-specific checks) — deliberately deferred,
+  see the original scoping notes for why.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

dbt_vitals-0.1.0/dbt_vitals/__init__.py

@@ -0,0 +1,3 @@
+"""dbt-vitals: static analysis and health scoring for dbt projects."""
+
+__version__ = "0.1.0"

dbt_vitals-0.1.0/dbt_vitals/cli.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import click
+from rich.console import Console
+
+from dbt_vitals import __version__
+from dbt_vitals.graph import build_model_graph
+from dbt_vitals.modules import documentation, duplicates, incremental, lineage, testing
+from dbt_vitals.parser import ManifestNotFoundError, load_catalog, load_manifest
+from dbt_vitals.report import render, render_ci_comment
+from dbt_vitals.scoring import compute_score
+
+console = Console()
+
+
+@click.group()
+@click.version_option(__version__, prog_name="dbt-vitals")
+def cli() -> None:
+    """dbt-vitals: static analysis and health scoring for dbt projects."""
+
+
+@cli.command()
+@click.argument("target_dir", type=click.Path(exists=True, file_okay=False), default=".")
+@click.option(
+    "--manifest",
+    "manifest_override",
+    type=click.Path(exists=True, dir_okay=False),
+    help="Explicit path to manifest.json (default: <target_dir>/target/manifest.json)",
+)
+@click.option(
+    "--catalog",
+    "catalog_override",
+    type=click.Path(exists=True, dir_okay=False),
+    help="Explicit path to catalog.json (default: <target_dir>/target/catalog.json)",
+)
+@click.option("--json", "as_json", is_flag=True, help="Print raw JSON instead of the terminal report.")
+@click.option("--ci-comment", is_flag=True, help="Print a markdown summary suitable for a PR comment.")
+@click.option(
+    "--fail-under",
+    type=int,
+    default=None,
+    help="Exit with a non-zero status code if the health score is below this threshold (useful in CI).",
+)
+def analyze(
+    target_dir: str,
+    manifest_override: str | None,
+    catalog_override: str | None,
+    as_json: bool,
+    ci_comment: bool,
+    fail_under: int | None,
+) -> None:
+    """Analyze a dbt project and print its health report.
+
+    TARGET_DIR is the root of your dbt project (the directory containing
+    dbt_project.yml). Defaults to the current directory. Run `dbt compile`
+    or `dbt docs generate` first so target/manifest.json exists.
+    """
+    root = Path(target_dir)
+    manifest_path = Path(manifest_override) if manifest_override else root / "target" / "manifest.json"
+    catalog_path = Path(catalog_override) if catalog_override else root / "target" / "catalog.json"
+
+    try:
+        project = load_manifest(manifest_path)
+    except ManifestNotFoundError as e:
+        console.print(f"[bold red]Error:[/] {e}")
+        raise SystemExit(1)
+
+    catalog_loaded = load_catalog(catalog_path, project)
+
+    graph = build_model_graph(project)
+
+    findings = []
+    findings += lineage.analyze(project, graph)
+    findings += testing.analyze(project)
+    findings += duplicates.analyze(project)
+    doc_coverage, doc_findings = documentation.analyze(project)
+    findings += doc_findings
+    findings += incremental.analyze(project, catalog_loaded)
+
+    model_count = len(project.models)
+    score = compute_score(findings, model_count)
+
+    if as_json:
+        payload = {
+            "score": score.score,
+            "critical_count": score.critical_count,
+            "warning_count": score.warning_count,
+            "info_count": score.info_count,
+            "model_count": model_count,
+            "documentation": {
+                "model_coverage_pct": doc_coverage.model_coverage_pct,
+                "column_coverage_pct": doc_coverage.column_coverage_pct,
+            },
+            "findings": [
+                {
+                    "module": f.module,
+                    "severity": f.severity.value,
+                    "subject": f.subject,
+                    "message": f.message,
+                    "detail": f.detail,
+                    "suggestion": f.suggestion,
+                }
+                for f in findings
+            ],
+        }
+        click.echo(json.dumps(payload, indent=2))
+    elif ci_comment:
+        click.echo(render_ci_comment(score, findings))
+    else:
+        render(console, score, findings, doc_coverage, model_count)
+
+    if fail_under is not None and score.score < fail_under:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    cli()

dbt_vitals-0.1.0/dbt_vitals/findings.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class Severity(str, Enum):
+    CRITICAL = "critical"
+    WARNING = "warning"
+    INFO = "info"
+
+
+@dataclass
+class Finding:
+    module: str          # e.g. "lineage", "testing", "duplicates"
+    severity: Severity
+    subject: str          # model/macro name this finding is about
+    message: str          # short human-readable summary
+    detail: str = ""      # optional longer explanation / "why"
+    suggestion: str = ""  # optional recommended fix
+    weight: float = 1.0   # deduction weight used by the scoring engine
+    meta: dict = field(default_factory=dict)

dbt_vitals-0.1.0/dbt_vitals/graph.py

@@ -0,0 +1,33 @@
+"""Builds a directed dependency graph of models from a parsed DbtProject."""
+
+from __future__ import annotations
+
+import networkx as nx
+
+from dbt_vitals.parser import DbtProject
+
+
+def build_model_graph(project: DbtProject) -> nx.DiGraph:
+    """Nodes = model unique_ids. Edge A -> B means B depends on A
+    (A feeds into B), matching dbt's own lineage direction."""
+    graph = nx.DiGraph()
+
+    for uid, node in project.models.items():
+        graph.add_node(uid, name=node.name, path=node.path)
+
+    for uid, node in project.models.items():
+        for dep in node.depends_on:
+            if dep in project.models:
+                graph.add_edge(dep, uid)
+
+    return graph
+
+
+def models_referenced_by_tests_or_exposures(project: DbtProject) -> set[str]:
+    """Models that have at least one test attached to them - used as a
+    signal that a model is intentionally a checked, "real" asset."""
+    referenced = set()
+    for test in project.tests.values():
+        for dep in test.depends_on:
+            referenced.add(dep)
+    return referenced

dbt_vitals-0.1.0/dbt_vitals/modules/documentation.py

@@ -0,0 +1,65 @@
+"""Documentation coverage module. Reports coverage %, plus a finding per
+undocumented model so the report can list them."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from dbt_vitals.findings import Finding, Severity
+from dbt_vitals.parser import DbtProject
+
+
+@dataclass
+class DocCoverage:
+    model_coverage_pct: float
+    column_coverage_pct: float
+    total_models: int
+    documented_models: int
+    total_columns: int
+    documented_columns: int
+
+
+def compute_coverage(project: DbtProject) -> DocCoverage:
+    models = project.models
+    total_models = len(models)
+    documented_models = sum(1 for m in models.values() if m.description.strip())
+
+    total_columns = 0
+    documented_columns = 0
+    for m in models.values():
+        for col in m.columns.values():
+            total_columns += 1
+            if col.description.strip():
+                documented_columns += 1
+
+    return DocCoverage(
+        model_coverage_pct=_pct(documented_models, total_models),
+        column_coverage_pct=_pct(documented_columns, total_columns),
+        total_models=total_models,
+        documented_models=documented_models,
+        total_columns=total_columns,
+        documented_columns=documented_columns,
+    )
+
+
+def _pct(part: int, whole: int) -> float:
+    return round((part / whole) * 100, 1) if whole else 0.0
+
+
+def analyze(project: DbtProject) -> tuple[DocCoverage, list[Finding]]:
+    coverage = compute_coverage(project)
+    findings: list[Finding] = []
+
+    for uid, model in project.models.items():
+        if not model.description.strip():
+            findings.append(
+                Finding(
+                    module="documentation",
+                    severity=Severity.INFO,
+                    subject=model.name,
+                    message="Model has no description.",
+                    weight=0.25,
+                )
+            )
+
+    return coverage, findings