pipelineprobe 0.3.0__tar.gz

pipelineprobe-0.3.0/.dockerignore

@@ -0,0 +1,31 @@
+# Version control
+.git
+.gitignore
+
+# Environments
+.venv
+venv
+
+# Python bytecode and caches
+__pycache__
+*.py[cod]
+*.egg-info
+.ruff_cache
+.pytest_cache
+
+# Build artifacts
+dist
+build
+
+# Generated reports
+reports
+
+# Internal / non-runtime directories
+.superpowers
+video-work
+site
+
+# Tests, docs and lockfile are not needed to build the image
+tests
+docs
+uv.lock

pipelineprobe-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,63 @@
+name: CI
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+jobs:
+  test-and-lint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,cloud]"
+
+      - name: Lint with ruff
+        run: ruff check .
+
+      - name: Run tests
+        run: pytest
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Install wheel in a clean venv
+        run: |
+          python -m venv /tmp/pipelineprobe-venv
+          /tmp/pipelineprobe-venv/bin/python -m pip install --upgrade pip
+          /tmp/pipelineprobe-venv/bin/pip install dist/*.whl
+          /tmp/pipelineprobe-venv/bin/pipelineprobe --version
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        run: docker build -t pipelineprobe:test .

pipelineprobe-0.3.0/.github/workflows/pages.yml

@@ -0,0 +1,50 @@
+name: Pages
+
+# Pages hosting is deferred: the site/ directory is published manually with
+# "Run workflow" once Pages is enabled in the repository settings
+# (Settings -> Pages -> Source: GitHub Actions). Publishing on every push
+# would fail until that setting exists.
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Fetch demo video (optional)
+        run: |
+          mkdir -p site/assets
+          curl -fL --retry 3 --retry-delay 2 \
+            https://github.com/willowvibe/pipelineprobe/releases/download/v0.3.0/pipelineprobe-demo.mp4 \
+            -o site/assets/pipelineprobe-demo.mp4 \
+            || {
+              rm -f site/assets/pipelineprobe-demo.mp4
+              echo "::notice::pipelineprobe-demo.mp4 is not published yet; deploying the page with its poster and release fallback."
+            }
+
+      - name: Configure Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

pipelineprobe-0.3.0/.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Create GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release create "$GITHUB_REF_NAME" dist/* --generate-notes
+
+  publish-pypi:
+    # Trusted Publishing: PyPI verifies this workflow through GitHub OIDC, so
+    # no API token is stored. The pending publisher on PyPI must name this
+    # repository, the file `release.yml`, and the environment `pypi`.
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pipelineprobe
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/

pipelineprobe-0.3.0/.gitignore

@@ -0,0 +1,171 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+/target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Mac OS
+.DS_Store
+
+# PipelineProbe specific
+/reports/
+examples/quickstart/reports/*
+!examples/quickstart/reports/.gitkeep
+/pipelineprobe.yml
+uv.lock
+video-work/
+site/assets/*.mp4

pipelineprobe-0.3.0/.vscode/settings.json

@@ -0,0 +1 @@
+{}

pipelineprobe-0.3.0/CHANGELOG.md

@@ -0,0 +1,130 @@
+# Changelog
+
+All notable changes to PipelineProbe are documented here.
+
+Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+Versioning follows [Semantic Versioning](https://semver.org/).
+
+---
+
+## [0.3.0] — 2026-11-15
+
+### Added
+
+#### v0.3.0 — Quickstart, sample report, and install shape
+- **Docker quickstart** (`examples/quickstart/`) — a real Airflow 2.7.1, dbt and
+  Postgres stack with 8 seeded DAGs and deterministic failures;
+  `docker compose up --build` writes HTML and JSON reports with no cloud
+  accounts or existing pipelines required.
+- **Sample report** (`site/report/index.html`) — the quickstart audit committed
+  as one self-contained HTML file with working severity filters.
+- **Optional `cloud` extra** — `google-cloud-bigquery` and
+  `snowflake-connector-python` moved out of the default install. The base
+  package covers Airflow, dbt and Postgres; configuring a BigQuery or Snowflake
+  warehouse without the extra exits with an install hint instead of a
+  traceback.
+
+#### v0.2.0 — New Orchestrator Connectors
+- **Prefect connector** (`pipelineprobe/connectors/prefect.py`) — connects to a
+  Prefect 2/3 Server or Cloud workspace via the Prefect REST API.  Maps flows
+  and flow runs to the shared `Dag` / `DagRun` models so all existing rules
+  (high failure rate, stale pipelines, …) work out of the box.  Prefect Cloud
+  authentication is handled via `orchestrator.api_key` in config.
+- **Dagster connector** (`pipelineprobe/connectors/dagster.py`) — queries the
+  Dagster GraphQL API to fetch runs, groups them by job name, and exposes each
+  job as a `Dag` object.  Supports Dagster Cloud via `orchestrator.api_key`.
+- `orchestrator.type` config field now supports `"airflow"` (default),
+  `"prefect"`, and `"dagster"`.  The `doctor` command probes all three.
+- `orchestrator.api_key` config field (+ `PIPELINEPROBE_ORCHESTRATOR_API_KEY`
+  env var) for Prefect Cloud / Dagster Cloud bearer-token authentication.
+- `orchestrator_type` key added to the `summary.metadata` block in JSON/HTML
+  reports so downstream tooling knows which orchestrator was audited.
+
+#### v0.3.0 — Cost Insights
+- **BigQuery cost insights** — `BigQueryConnector.get_cost_insights_sync()`
+  queries `INFORMATION_SCHEMA.JOBS_BY_PROJECT` (last 30 days) and returns the
+  25 tables with the highest bytes billed.
+- **Snowflake credit insights** — `SnowflakeConnector.get_cost_insights_sync()`
+  queries `SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY` (last 30 days)
+  and returns credit consumption per virtual warehouse.
+- **`check_expensive_bq_tables`** rule — warning at ≥ 500 GiB billed / 30 days,
+  critical at ≥ 5 TiB.
+- **`check_snowflake_credit_spenders`** rule — warning at ≥ 500 credits / 30 days,
+  critical at ≥ 5 000 credits.
+- `report.include_cost_section: true` config flag enables cost-insight queries
+  during `pipelineprobe audit`; defaults to `false` (opt-in to avoid requiring
+  elevated IAM permissions by default).
+- `warehouse.bq_region` config field allows overriding the default BigQuery
+  multi-region (`region-us`) for teams using e.g. `region-eu`.
+- `doctor` command extended to probe cost-insight permissions when
+  `include_cost_section: true` is set.
+
+#### General
+- `pipelineprobe audit` now prints a one-line health summary
+  (`Health score: N/100 — X critical, Y warning, Z info`) before the
+  fail-threshold check.
+- `SUPPORTED_ORCHESTRATORS` constant exported from `pipelineprobe.config` for
+  validation use in the CLI.
+- `init` command now generates a config template documenting all new fields.
+
+### Added (previous Unreleased — UI / bug fixes)
+- **Report UI overhaul** — sticky topbar, animated SVG health-score ring, severity filter buttons (All / Critical / Warnings / Info), affected-resource tags on each finding card, subtle per-severity background tints, and responsive print styles.
+- **Info count card** — the summary grid now shows a fourth card for informational findings alongside Critical and Warnings.
+- `info_count` included in the JSON report summary for downstream tooling.
+- `ring_offset` computed in the renderer and injected into the Jinja2 template so the SVG ring accurately reflects the health score.
+- BigQuery connector querying `INFORMATION_SCHEMA.TABLE_STORAGE` and `COLUMNS` for real `has_timestamps` detection.
+- Snowflake connector using a CTE pattern to work around correlated subquery restrictions.
+- `python-dateutil` declared as an explicit dependency in `pyproject.toml`.
+
+### Fixed
+- `check_missing_slas` no longer reports false "no SLA configured" findings when the orchestrator API does not expose task SLA configuration (Airflow 2.7.1 never serializes `sla`).
+- `pipelineprobe audit --format` was shadowing Python's built-in `format()`. Renamed CLI parameter to `report_format` internally.
+- Added validation to `--format` — passing unsupported values (e.g. `csv`) now exits with a clear error.
+- `get_dags()` in the Airflow connector now paginates correctly. Previously, organisations with more than 100 DAGs would silently receive truncated results.
+- `check_stale_dags` now correctly flags active DAGs that have **zero recorded runs** (previously silently skipped).
+- Double-space indentation bug in `check_stale_dags` fixed.
+- `BigQueryConnector` was returning `has_timestamps = True` for every table (hardcoded). Now queries actual `INFORMATION_SCHEMA.COLUMNS`.
+- `SnowflakeConnector` was silently connecting with empty string credentials. Now validates and returns early with a log error.
+- `SnowflakeConnector` correlated subquery replaced with a CTE, which is supported by Snowflake's `INFORMATION_SCHEMA`.
+- `affected_resources` in warehouse rules could contain `None` when `tablename` was absent. Now falls back to `"<unknown>"`.
+- `renderer.render_json()` raised `TypeError` when `Task.sla` (`timedelta`) was serialized. Added `_json_default` handler.
+- `test_cli_init` was failing if a `pipelineprobe.yml` already existed in the project root. Test now uses a temporary directory.
+- All bare `print()` error statements replaced with `logging.getLogger(__name__)` across all modules.
+- PostgreSQL connector now uses `try/finally` to guarantee `conn.close()` is called even on errors.
+
+---
+
+## 0.1.0 — 2026-03-15
+
+### Added
+- **Phase 0** — Project skeleton, `pyproject.toml`, CI workflow (ruff + pytest), CLI entry point.
+- **Phase 1 — MVP connectors:**
+  - `AirflowConnector` — fetches DAGs, DAG runs, and task configurations via the Airflow REST API.
+  - `DbtConnector` — reads `manifest.json` and `run_results.json`; counts tests per model.
+  - `PostgresConnector` — queries `pg_stat_user_tables` and `information_schema.columns`.
+- **Phase 1 — Rules Engine:**
+  - `check_missing_retries` — warns on tasks with no retry configuration.
+  - `check_missing_slas` — informs on tasks with no SLA.
+  - `check_high_failure_rate` — critical alert for DAGs with >20% failure rate over ≥5 runs.
+  - `check_stale_dags` — warns on active DAGs with no successful run in 7+ days.
+  - `check_missing_tests` — warns on dbt models with zero tests.
+  - `check_failing_models` — critical alert for dbt models that failed their last run.
+  - `check_large_tables` — warns on tables with >10M rows.
+  - `check_missing_timestamps` — warns on tables >1M rows without `created_at`/`updated_at`.
+- **Phase 1 — Report Renderer** — Jinja2 HTML template; JSON output via `model_dump()`.
+- **Phase 2 — DX & CI:**
+  - `pipelineprobe init` command generating a default `pipelineprobe.yml`.
+  - `--fail-on-critical` and `--format` CLI overrides.
+  - `examples/github-actions/pipelineprobe.yml` GitHub Actions workflow.
+  - `docs/ci-integration.md`.
+  - `.gitignore` with standard Python exclusions.
+- **Phase 3 — Extended Integrations:**
+  - `BigQueryConnector`.
+  - `SnowflakeConnector`.
+  - `pipelineprobe doctor` command for connectivity validation.
+  - `pipelineprobe diff` command for regression detection between two JSON reports.
+
+---
+
+[Unreleased]: https://github.com/willowvibe/pipelineprobe/compare/v0.3.0...HEAD
+[0.3.0]: https://github.com/willowvibe/pipelineprobe/releases/tag/v0.3.0

pipelineprobe-0.3.0/CONTRIBUTING.md

@@ -0,0 +1,189 @@
+# Contributing to PipelineProbe
+
+Thank you for your interest in contributing! PipelineProbe is an open-source project by [WillowVibe](https://www.willowvibe.com) and we welcome contributions of all kinds — bug reports, documentation improvements, new rules, and new connectors.
+
+---
+
+## Development Setup
+
+### Prerequisites
+
+- Python 3.11+
+- `pip`
+- A virtual environment tool (`venv`, `hatch`, `pyenv`, etc.)
+
+### Clone and Install
+
+```bash
+git clone https://github.com/willowvibe/pipelineprobe.git
+cd pipelineprobe
+
+python -m venv .venv
+source .venv/bin/activate      # Linux / macOS
+# .venv\Scripts\activate       # Windows
+
+pip install -e ".[dev]"
+```
+
+---
+
+## Running Tests
+
+```bash
+pytest
+```
+
+Tests live in `tests/` and use `pytest`. Fixtures for mock data are in `tests/conftest.py`.
+
+To run a specific file or test:
+
+```bash
+pytest tests/test_rules.py
+pytest tests/test_renderer.py::test_render_html_no_issues
+```
+
+---
+
+## Linting & Formatting
+
+We use [ruff](https://docs.astral.sh/ruff/) for both linting and formatting:
+
+```bash
+ruff check .        # report lint errors
+ruff check --fix .  # auto-fix safe issues
+ruff format .       # format all files
+```
+
+CI fails if `ruff check .` reports any errors. Run both before opening a PR.
+
+---
+
+## Project Structure
+
+```
+pipelineprobe/
+├── pipelineprobe/
+│   ├── cli.py              # CLI entry points (audit, init, doctor, diff)
+│   ├── config.py           # YAML + env var config loader (Pydantic)
+│   ├── models.py           # Core domain models (Dag, Task, DbtModel, Issue)
+│   ├── renderer.py         # HTML + JSON report renderer (Jinja2)
+│   ├── connectors/
+│   │   ├── airflow.py      # Apache Airflow REST API connector
+│   │   ├── dbt.py          # dbt manifest / run_results reader
+│   │   ├── postgres.py     # PostgreSQL INFORMATION_SCHEMA queries
+│   │   ├── bigquery.py     # BigQuery INFORMATION_SCHEMA queries
+│   │   └── snowflake.py    # Snowflake INFORMATION_SCHEMA queries
+│   ├── rules/
+│   │   ├── engine.py       # Pluggable rule registration and execution
+│   │   ├── airflow_rules.py
+│   │   ├── dbt_rules.py
+│   │   └── postgres_rules.py
+│   └── templates/
+│       └── report.html     # Jinja2 HTML report template
+├── tests/
+│   ├── conftest.py         # Shared pytest fixtures
+│   ├── test_cli.py
+│   ├── test_config.py
+│   ├── test_connectors.py
+│   ├── test_renderer.py
+│   └── test_rules.py
+├── docs/
+├── examples/
+└── pyproject.toml
+```
+
+---
+
+## Adding a New Rule
+
+1. Add a function to the appropriate `rules/` module (e.g. `airflow_rules.py`):
+
+```python
+from typing import List
+from pipelineprobe.models import Issue
+
+def check_my_new_rule(context: dict) -> List[Issue]:
+    issues = []
+    for dag in context.get("airflow_dags", []):
+        if some_condition(dag):
+            issues.append(Issue(
+                severity="warning",
+                category="dag",
+                summary="Short description of the problem",
+                details="Longer explanation with context.",
+                recommendation="Concrete steps to fix it.",
+                affected_resources=[dag.id],
+            ))
+    return issues
+```
+
+2. Register it at the bottom of the same file:
+
+```python
+def register_airflow_rules(engine):
+    ...
+    engine.register_rule(check_my_new_rule)
+```
+
+3. Write tests in `tests/test_rules.py` — at minimum one happy-path and one edge-case test.
+
+4. If the rule is configurable (e.g. severity override), document it in `docs/configuration.md` under the `rules.severity_overrides` table.
+
+---
+
+## Adding a New Connector
+
+1. Create `pipelineprobe/connectors/my_connector.py`.
+2. Implement a class with a `get_stats_sync() -> List[Dict[str, Any]]` method returning rows with at least: `schemaname`, `tablename`, `row_count`, `has_timestamps`.
+3. Route `cfg.warehouse.type == "my_connector"` in the `audit()` function in `cli.py`.
+4. Add any required credentials to `WarehouseConfig` in `config.py`.
+5. Document the new connector and its config fields in `docs/configuration.md`.
+6. Add at least one connectivity test in `tests/test_connectors.py`.
+
+---
+
+## Working on the HTML Report
+
+The report template lives at `pipelineprobe/templates/report.html` and is rendered by `renderer.py` using Jinja2.
+
+Variables available in the template:
+
+| Variable | Type | Description |
+|---|---|---|
+| `issues` | `List[Issue]` | All findings from the rule engine |
+| `summary` | `dict` | Health score, counts, dag_count, metadata |
+| `top_actions` | `List[Issue]` | Top 3 critical/warning issues |
+| `metadata` | `dict` | orchestrator_url, warehouse_type, dbt_target |
+| `generated_at` | `str` | Formatted timestamp string |
+| `version` | `str` | PipelineProbe version |
+| `ring_offset` | `float` | SVG ring stroke-dashoffset (0 = full, 251.3 = empty) |
+
+To preview a rendered report, run a real or mock audit and open `./reports/pipelineprobe-report.html` in a browser.
+
+---
+
+## Pull Request Process
+
+1. Fork the repo and create your branch from `main`.
+2. Write tests for your changes.
+3. Ensure `ruff check .` and `pytest` both pass.
+4. Open a PR with a description covering:
+   - **What** changed
+   - **Why** it is needed
+   - **How** to test it manually
+
+---
+
+## Reporting Bugs
+
+Open a GitHub Issue with:
+- PipelineProbe version (`pip show pipelineprobe`)
+- Python version
+- A clear description of what happened vs. what you expected
+- Sanitized config (no credentials) and relevant log output
+
+---
+
+## Code of Conduct
+
+Be kind, constructive, and respectful. We follow the [Contributor Covenant](https://www.contributor-covenant.org/) Code of Conduct.