dagwick 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagwick-0.1.0/.gitignore +18 -0
- dagwick-0.1.0/CHANGELOG.md +21 -0
- dagwick-0.1.0/LICENSE +21 -0
- dagwick-0.1.0/PKG-INFO +80 -0
- dagwick-0.1.0/README.md +49 -0
- dagwick-0.1.0/SECURITY.md +37 -0
- dagwick-0.1.0/pyproject.toml +96 -0
- dagwick-0.1.0/src/dagwick/__init__.py +6 -0
- dagwick-0.1.0/src/dagwick/api.py +183 -0
- dagwick-0.1.0/src/dagwick/checks/__init__.py +5 -0
- dagwick-0.1.0/src/dagwick/checks/_util.py +32 -0
- dagwick-0.1.0/src/dagwick/checks/observability.py +40 -0
- dagwick-0.1.0/src/dagwick/checks/ownership.py +53 -0
- dagwick-0.1.0/src/dagwick/checks/reliability.py +57 -0
- dagwick-0.1.0/src/dagwick/checks/scheduling.py +75 -0
- dagwick-0.1.0/src/dagwick/checks/structure.py +40 -0
- dagwick-0.1.0/src/dagwick/core.py +178 -0
- dagwick-0.1.0/src/dagwick/plugin.py +33 -0
- dagwick-0.1.0/src/dagwick/py.typed +0 -0
- dagwick-0.1.0/src/dagwick/static/dagwick.js +328 -0
- dagwick-0.1.0/src/dagwick/static/icon.svg +1 -0
- dagwick-0.1.0/tests/conftest.py +42 -0
- dagwick-0.1.0/tests/test_api.py +105 -0
- dagwick-0.1.0/tests/test_checks.py +86 -0
- dagwick-0.1.0/tests/test_core.py +79 -0
- dagwick-0.1.0/ui/package-lock.json +1846 -0
- dagwick-0.1.0/ui/package.json +17 -0
- dagwick-0.1.0/ui/src/App.tsx +14 -0
- dagwick-0.1.0/ui/src/DagView.tsx +283 -0
- dagwick-0.1.0/ui/src/FleetView.tsx +289 -0
- dagwick-0.1.0/ui/src/Lamp.tsx +149 -0
- dagwick-0.1.0/ui/src/api.ts +51 -0
- dagwick-0.1.0/ui/src/brand.ts +6 -0
- dagwick-0.1.0/ui/src/components.tsx +176 -0
- dagwick-0.1.0/ui/src/main.tsx +14 -0
- dagwick-0.1.0/ui/src/styles.ts +330 -0
- dagwick-0.1.0/ui/src/types.ts +51 -0
- dagwick-0.1.0/ui/tsconfig.json +14 -0
- dagwick-0.1.0/ui/vite.config.ts +33 -0
dagwick-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
|
|
5
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.1.0] - 2026-11-08
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Dagwick tab** on every DAG page: a score from 0 to 100, a grade from A to F,
|
|
14
|
+
a lantern mascot, the next step to the next grade, the affected tasks, and a
|
|
15
|
+
fix with a copyable code example for every check that does not pass.
|
|
16
|
+
- **Fleet page** in the Airflow sidebar: every DAG the user can read, ranked by
|
|
17
|
+
score, with grade filters, search, sort, and the most common issues.
|
|
18
|
+
- 14 built-in checks covering reliability, observability, scheduling,
|
|
19
|
+
ownership and structure.
|
|
20
|
+
- `dagwick.checks` entry point so other packages can add or override checks.
|
|
21
|
+
- Scores reuse Airflow's own permission checks and are cached per DAG version.
|
dagwick-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The Dagwick Authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
dagwick-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dagwick
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: The wick that keeps your DAGs lit: DAG quality scores, a fleet view, and fixes you can act on, inside the Airflow 3 UI.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ardalan-jalal/dagwick
|
|
6
|
+
Project-URL: Repository, https://github.com/ardalan-jalal/dagwick
|
|
7
|
+
Project-URL: Issues, https://github.com/ardalan-jalal/dagwick/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/ardalan-jalal/dagwick/blob/main/CHANGELOG.md
|
|
9
|
+
Author: The Dagwick Authors
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: airflow,airflow-plugin,apache-airflow,dag,data-quality,linting
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Environment :: Plugins
|
|
15
|
+
Classifier: Environment :: Web Environment
|
|
16
|
+
Classifier: Framework :: Apache Airflow
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
27
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
28
|
+
Classifier: Typing :: Typed
|
|
29
|
+
Requires-Python: >=3.10
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# Dagwick
|
|
33
|
+
|
|
34
|
+

|
|
35
|
+
|
|
36
|
+
## Setup
|
|
37
|
+
|
|
38
|
+
**1. Install** it where Airflow runs:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install dagwick
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Using Docker? Add `dagwick` to your Airflow image's `requirements.txt`.
|
|
45
|
+
|
|
46
|
+
**2. Restart** the Airflow api-server.
|
|
47
|
+
|
|
48
|
+
**3. Open Airflow.** You now have:
|
|
49
|
+
|
|
50
|
+
- a **Dagwick** tab on every DAG page, and
|
|
51
|
+
- a **Dagwick** page in the sidebar that grades all your DAGs.
|
|
52
|
+
|
|
53
|
+
That's it. There is nothing to configure.
|
|
54
|
+
|
|
55
|
+

|
|
56
|
+
|
|
57
|
+

|
|
58
|
+
|
|
59
|
+
## Requirements
|
|
60
|
+
|
|
61
|
+
- Apache Airflow 3.1 or newer
|
|
62
|
+
- Python 3.10 – 3.15
|
|
63
|
+
|
|
64
|
+
Dagwick has no dependencies of its own, so installing it never changes your
|
|
65
|
+
other packages. It is read-only and only shows each user the DAGs they can
|
|
66
|
+
already see.
|
|
67
|
+
|
|
68
|
+
## Remove
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip uninstall dagwick
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Then restart the api-server.
|
|
75
|
+
|
|
76
|
+
## More
|
|
77
|
+
|
|
78
|
+
- [What it checks, and how to add your own checks](https://github.com/ardalan-jalal/dagwick/blob/main/docs/checks.md)
|
|
79
|
+
- [Security](https://github.com/ardalan-jalal/dagwick/blob/main/SECURITY.md) · [Contributing](https://github.com/ardalan-jalal/dagwick/blob/main/CONTRIBUTING.md) · [Changelog](https://github.com/ardalan-jalal/dagwick/blob/main/CHANGELOG.md)
|
|
80
|
+
- License: [MIT](https://github.com/ardalan-jalal/dagwick/blob/main/LICENSE)
|
dagwick-0.1.0/README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Dagwick
|
|
2
|
+
|
|
3
|
+

|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
**1. Install** it where Airflow runs:
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install dagwick
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Using Docker? Add `dagwick` to your Airflow image's `requirements.txt`.
|
|
14
|
+
|
|
15
|
+
**2. Restart** the Airflow api-server.
|
|
16
|
+
|
|
17
|
+
**3. Open Airflow.** You now have:
|
|
18
|
+
|
|
19
|
+
- a **Dagwick** tab on every DAG page, and
|
|
20
|
+
- a **Dagwick** page in the sidebar that grades all your DAGs.
|
|
21
|
+
|
|
22
|
+
That's it. There is nothing to configure.
|
|
23
|
+
|
|
24
|
+

|
|
25
|
+
|
|
26
|
+

|
|
27
|
+
|
|
28
|
+
## Requirements
|
|
29
|
+
|
|
30
|
+
- Apache Airflow 3.1 or newer
|
|
31
|
+
- Python 3.10 – 3.15
|
|
32
|
+
|
|
33
|
+
Dagwick has no dependencies of its own, so installing it never changes your
|
|
34
|
+
other packages. It is read-only and only shows each user the DAGs they can
|
|
35
|
+
already see.
|
|
36
|
+
|
|
37
|
+
## Remove
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip uninstall dagwick
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Then restart the api-server.
|
|
44
|
+
|
|
45
|
+
## More
|
|
46
|
+
|
|
47
|
+
- [What it checks, and how to add your own checks](https://github.com/ardalan-jalal/dagwick/blob/main/docs/checks.md)
|
|
48
|
+
- [Security](https://github.com/ardalan-jalal/dagwick/blob/main/SECURITY.md) · [Contributing](https://github.com/ardalan-jalal/dagwick/blob/main/CONTRIBUTING.md) · [Changelog](https://github.com/ardalan-jalal/dagwick/blob/main/CHANGELOG.md)
|
|
49
|
+
- License: [MIT](https://github.com/ardalan-jalal/dagwick/blob/main/LICENSE)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Security policy
|
|
2
|
+
|
|
3
|
+
## Reporting a vulnerability
|
|
4
|
+
|
|
5
|
+
Please **do not open a public issue** for security problems. Report them
|
|
6
|
+
privately through GitHub: open this repository's **Security** tab and choose
|
|
7
|
+
**Report a vulnerability**.
|
|
8
|
+
|
|
9
|
+
We aim to acknowledge reports within 3 working days and will keep you updated
|
|
10
|
+
until a fix is released.
|
|
11
|
+
|
|
12
|
+
## Supported versions
|
|
13
|
+
|
|
14
|
+
Only the latest released minor version receives security fixes.
|
|
15
|
+
|
|
16
|
+
## Security design
|
|
17
|
+
|
|
18
|
+
Dagwick runs inside the Airflow api-server. It is designed to add no new attack
|
|
19
|
+
surface beyond what Airflow already exposes:
|
|
20
|
+
|
|
21
|
+
- **Same permissions as Airflow.** Every endpoint uses Airflow's own security
|
|
22
|
+
dependencies (`requires_access_dag`, `get_authorized_dag_ids`). A user only
|
|
23
|
+
sees scores for DAGs they can already read in the Airflow UI.
|
|
24
|
+
- **Read-only.** Dagwick reads serialized DAGs from the metadata database. It
|
|
25
|
+
never writes to the database, never executes DAG code, and never makes network
|
|
26
|
+
calls.
|
|
27
|
+
- **No configuration and no secrets.** Dagwick reads no environment variables,
|
|
28
|
+
Airflow Variables, Connections or credentials.
|
|
29
|
+
- **Safe errors.** Error responses contain only a generic message and a
|
|
30
|
+
reference id. Details go to the server log, never to the browser.
|
|
31
|
+
- **Same origin only.** No CORS headers are added.
|
|
32
|
+
- **Static files from an allowlist.** The static route never builds file paths
|
|
33
|
+
from user input.
|
|
34
|
+
- **No third-party code in the browser.** The UI bundle uses the React that
|
|
35
|
+
Airflow already provides. It loads nothing from a CDN, uses no
|
|
36
|
+
`innerHTML`, and stores nothing in `localStorage`.
|
|
37
|
+
- **No runtime dependencies.** The package declares no dependencies of its own.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dagwick"
|
|
3
|
+
dynamic = ["version"]
|
|
4
|
+
description = "The wick that keeps your DAGs lit: DAG quality scores, a fleet view, and fixes you can act on, inside the Airflow 3 UI."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
authors = [{ name = "The Dagwick Authors" }]
|
|
10
|
+
keywords = ["airflow", "apache-airflow", "airflow-plugin", "dag", "data-quality", "linting"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 3 - Alpha",
|
|
13
|
+
"Environment :: Plugins",
|
|
14
|
+
"Environment :: Web Environment",
|
|
15
|
+
"Framework :: Apache Airflow",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Programming Language :: Python :: 3.15",
|
|
26
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
# Intentionally empty: Dagwick runs inside an existing Airflow 3.1+ installation and
|
|
30
|
+
# uses the Airflow (and FastAPI) that is already there. Installing it never pulls in,
|
|
31
|
+
# upgrades or downgrades Airflow.
|
|
32
|
+
dependencies = []
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/ardalan-jalal/dagwick"
|
|
36
|
+
Repository = "https://github.com/ardalan-jalal/dagwick"
|
|
37
|
+
Issues = "https://github.com/ardalan-jalal/dagwick/issues"
|
|
38
|
+
Changelog = "https://github.com/ardalan-jalal/dagwick/blob/main/CHANGELOG.md"
|
|
39
|
+
|
|
40
|
+
[project.entry-points."airflow.plugins"]
|
|
41
|
+
dagwick = "dagwick.plugin:DagwickPlugin"
|
|
42
|
+
|
|
43
|
+
# Development only (never published as a dependency): tests and the local sandbox.
|
|
44
|
+
[dependency-groups]
|
|
45
|
+
dev = [
|
|
46
|
+
"apache-airflow==3.3.2",
|
|
47
|
+
"pytest>=8",
|
|
48
|
+
"pytest-cov>=6",
|
|
49
|
+
"ruff>=0.14",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[tool.uv]
|
|
53
|
+
# Resolve the dev lockfile (Airflow + test tools) for Python 3.11+ only, so it can
|
|
54
|
+
# use current, patched versions (e.g. dnspython >= 2.9 needs 3.11). This does not
|
|
55
|
+
# change the published package, which has no dependencies and supports 3.10.
|
|
56
|
+
environments = ["python_version >= '3.11'"]
|
|
57
|
+
|
|
58
|
+
[build-system]
|
|
59
|
+
requires = ["hatchling>=1.27"]
|
|
60
|
+
build-backend = "hatchling.build"
|
|
61
|
+
|
|
62
|
+
[tool.hatch.version]
|
|
63
|
+
path = "src/dagwick/__init__.py"
|
|
64
|
+
|
|
65
|
+
[tool.hatch.build.targets.wheel]
|
|
66
|
+
packages = ["src/dagwick"]
|
|
67
|
+
|
|
68
|
+
[tool.hatch.build.targets.sdist]
|
|
69
|
+
include = [
|
|
70
|
+
"src/dagwick",
|
|
71
|
+
"ui/src",
|
|
72
|
+
"ui/package.json",
|
|
73
|
+
"ui/package-lock.json",
|
|
74
|
+
"ui/tsconfig.json",
|
|
75
|
+
"ui/vite.config.ts",
|
|
76
|
+
"tests",
|
|
77
|
+
"README.md",
|
|
78
|
+
"LICENSE",
|
|
79
|
+
"CHANGELOG.md",
|
|
80
|
+
"SECURITY.md",
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
[tool.pytest.ini_options]
|
|
84
|
+
testpaths = ["tests"]
|
|
85
|
+
addopts = "--cov=dagwick --cov-report=term-missing --cov-fail-under=70"
|
|
86
|
+
|
|
87
|
+
[tool.coverage.run]
|
|
88
|
+
branch = true
|
|
89
|
+
source = ["dagwick"]
|
|
90
|
+
|
|
91
|
+
[tool.ruff]
|
|
92
|
+
line-length = 100
|
|
93
|
+
target-version = "py310"
|
|
94
|
+
|
|
95
|
+
[tool.ruff.lint]
|
|
96
|
+
select = ["E", "F", "I", "B", "UP", "SIM"]
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""HTTP API served inside the Airflow api-server under ``/dagwick``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import threading
|
|
7
|
+
import uuid
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Annotated, Any
|
|
10
|
+
|
|
11
|
+
from fastapi import Depends, FastAPI, HTTPException, Query
|
|
12
|
+
from fastapi.responses import FileResponse, JSONResponse
|
|
13
|
+
|
|
14
|
+
from dagwick import __version__
|
|
15
|
+
from dagwick.core import registered_checks, score_dag
|
|
16
|
+
|
|
17
|
+
log = logging.getLogger(__name__)
|
|
18
|
+
STATIC_DIR = Path(__file__).parent / "static"
|
|
19
|
+
FLEET_LIMIT = 2000
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _require_dag_read() -> Any:
|
|
23
|
+
"""Reuse Airflow's own DAG permission check (same rules as the REST API)."""
|
|
24
|
+
from airflow.api_fastapi.core_api.security import requires_access_dag
|
|
25
|
+
|
|
26
|
+
return Depends(requires_access_dag(method="GET"))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _require_login() -> Any:
|
|
30
|
+
from airflow.api_fastapi.core_api.security import requires_authenticated
|
|
31
|
+
|
|
32
|
+
return Depends(requires_authenticated())
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _user() -> Any:
|
|
36
|
+
"""Airflow's own user resolver: Bearer header, OAuth token, or the UI cookie."""
|
|
37
|
+
from airflow.api_fastapi.core_api.security import get_user
|
|
38
|
+
|
|
39
|
+
return Depends(get_user)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _ScoreCache:
|
|
43
|
+
"""Scores keyed by (dag_id, dag_hash): a DAG is re-scored only when its code changes."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, max_entries: int = 5000) -> None:
|
|
46
|
+
self._data: dict[str, tuple[str, dict[str, Any]]] = {}
|
|
47
|
+
self._lock = threading.Lock()
|
|
48
|
+
self._max = max_entries
|
|
49
|
+
|
|
50
|
+
def get(self, dag_id: str, dag_hash: str) -> dict[str, Any] | None:
|
|
51
|
+
with self._lock:
|
|
52
|
+
hit = self._data.get(dag_id)
|
|
53
|
+
return hit[1] if hit and hit[0] == dag_hash else None
|
|
54
|
+
|
|
55
|
+
def put(self, dag_id: str, dag_hash: str, report: dict[str, Any]) -> None:
|
|
56
|
+
with self._lock:
|
|
57
|
+
if dag_id not in self._data and len(self._data) >= self._max:
|
|
58
|
+
self._data.pop(next(iter(self._data)))
|
|
59
|
+
self._data[dag_id] = (dag_hash, report)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_cache = _ScoreCache()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _score_by_id(dag_id: str) -> dict[str, Any] | None:
|
|
66
|
+
from airflow.models.serialized_dag import SerializedDagModel
|
|
67
|
+
|
|
68
|
+
sdm = SerializedDagModel.get(dag_id)
|
|
69
|
+
if sdm is None:
|
|
70
|
+
return None
|
|
71
|
+
cached = _cache.get(dag_id, sdm.dag_hash)
|
|
72
|
+
if cached is not None:
|
|
73
|
+
return cached
|
|
74
|
+
report = score_dag(sdm.dag)
|
|
75
|
+
_cache.put(dag_id, sdm.dag_hash, report)
|
|
76
|
+
return report
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _server_error(what: str, exc: BaseException) -> HTTPException:
|
|
80
|
+
ref = uuid.uuid4().hex[:12]
|
|
81
|
+
log.error("dagwick: %s (ref %s)", what, ref, exc_info=exc)
|
|
82
|
+
return HTTPException(status_code=500, detail=f"Something went wrong (ref {ref}).")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
app = FastAPI(title="Dagwick", version=__version__, docs_url=None, redoc_url=None, openapi_url=None)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
STATIC_FILES = {"dagwick.js": "application/javascript", "icon.svg": "image/svg+xml"}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@app.get("/static/{name}", include_in_schema=False)
|
|
92
|
+
def static(name: str) -> FileResponse:
|
|
93
|
+
media_type = STATIC_FILES.get(name) # allowlist: never build paths from user input
|
|
94
|
+
if media_type is None:
|
|
95
|
+
raise HTTPException(status_code=404)
|
|
96
|
+
return FileResponse(
|
|
97
|
+
STATIC_DIR / name, media_type=media_type, headers={"Cache-Control": "no-cache"}
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@app.get("/api/score", dependencies=[_require_dag_read()])
|
|
102
|
+
def score(dag_id: str = Query(..., min_length=1, max_length=250)) -> dict[str, Any]:
|
|
103
|
+
try:
|
|
104
|
+
report = _score_by_id(dag_id)
|
|
105
|
+
except Exception as exc:
|
|
106
|
+
raise _server_error(f"failed to score DAG {dag_id}", exc) from None
|
|
107
|
+
if report is None:
|
|
108
|
+
raise HTTPException(status_code=404, detail="DAG not found or not parsed yet.")
|
|
109
|
+
return report
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def fleet_entry(
|
|
113
|
+
dag_id: str, is_paused: bool, owners: str, report: dict[str, Any]
|
|
114
|
+
) -> dict[str, Any]:
|
|
115
|
+
"""Compact per-DAG summary for the fleet page; issues sorted worst first."""
|
|
116
|
+
issues = [
|
|
117
|
+
{
|
|
118
|
+
"id": c["id"],
|
|
119
|
+
"title": c["title"],
|
|
120
|
+
"status": c["status"],
|
|
121
|
+
"gain": round(c["weight"] - c["points"], 1),
|
|
122
|
+
}
|
|
123
|
+
for c in report["checks"]
|
|
124
|
+
if c["status"] in ("fail", "warn")
|
|
125
|
+
]
|
|
126
|
+
issues.sort(key=lambda i: (i["status"] != "fail", -i["gain"]))
|
|
127
|
+
return {
|
|
128
|
+
"dag_id": dag_id,
|
|
129
|
+
"score": report["score"],
|
|
130
|
+
"grade": report["grade"],
|
|
131
|
+
"counts": report["counts"],
|
|
132
|
+
"is_paused": bool(is_paused),
|
|
133
|
+
"owners": owners or "",
|
|
134
|
+
"issues": issues,
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@app.get("/api/fleet")
|
|
139
|
+
def fleet(user: Annotated[Any, _user()]) -> dict[str, Any]:
|
|
140
|
+
"""Scores for every active DAG the current user may read."""
|
|
141
|
+
from airflow.api_fastapi.app import get_auth_manager
|
|
142
|
+
from airflow.models.dag import DagModel
|
|
143
|
+
from airflow.utils.session import create_session
|
|
144
|
+
from sqlalchemy import select
|
|
145
|
+
|
|
146
|
+
try:
|
|
147
|
+
allowed = get_auth_manager().get_authorized_dag_ids(user=user, method="GET")
|
|
148
|
+
with create_session() as session:
|
|
149
|
+
rows = session.execute(
|
|
150
|
+
select(DagModel.dag_id, DagModel.is_paused, DagModel.owners)
|
|
151
|
+
.where(DagModel.is_stale.is_(False), DagModel.dag_id.in_(allowed))
|
|
152
|
+
.order_by(DagModel.dag_id)
|
|
153
|
+
.limit(FLEET_LIMIT)
|
|
154
|
+
).all()
|
|
155
|
+
dags = []
|
|
156
|
+
for dag_id, is_paused, owners in rows:
|
|
157
|
+
report = _score_by_id(dag_id)
|
|
158
|
+
if report is not None:
|
|
159
|
+
dags.append(fleet_entry(dag_id, is_paused, owners, report))
|
|
160
|
+
except Exception as exc:
|
|
161
|
+
raise _server_error("failed to build fleet view", exc) from None
|
|
162
|
+
return {"dags": dags, "truncated": len(rows) >= FLEET_LIMIT}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@app.get("/api/checks", dependencies=[_require_login()])
|
|
166
|
+
def checks() -> list[dict[str, Any]]:
|
|
167
|
+
return [
|
|
168
|
+
{
|
|
169
|
+
"id": c.id,
|
|
170
|
+
"title": c.title,
|
|
171
|
+
"category": c.category.value,
|
|
172
|
+
"weight": c.weight,
|
|
173
|
+
"fix": c.fix,
|
|
174
|
+
}
|
|
175
|
+
for c in registered_checks()
|
|
176
|
+
]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@app.exception_handler(Exception)
|
|
180
|
+
async def _unhandled(_request: Any, exc: Exception) -> JSONResponse:
|
|
181
|
+
ref = uuid.uuid4().hex[:12]
|
|
182
|
+
log.error("dagwick: unhandled error (ref %s)", ref, exc_info=exc)
|
|
183
|
+
return JSONResponse(status_code=500, content={"detail": f"Internal error (ref {ref})."})
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from dagwick.core import Finding, Status
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def coverage(
|
|
10
|
+
dag: Any,
|
|
11
|
+
predicate: Callable[[Any], bool],
|
|
12
|
+
*,
|
|
13
|
+
noun: str,
|
|
14
|
+
all_msg: str,
|
|
15
|
+
none_msg: str,
|
|
16
|
+
) -> Finding:
|
|
17
|
+
"""PASS if every task matches, WARN if some do, FAIL if none do."""
|
|
18
|
+
tasks = list(dag.tasks)
|
|
19
|
+
if not tasks:
|
|
20
|
+
return Finding(Status.SKIP, "The DAG has no tasks.")
|
|
21
|
+
missing = tuple(sorted(t.task_id for t in tasks if not predicate(t)))
|
|
22
|
+
if not missing:
|
|
23
|
+
return Finding(Status.PASS, all_msg)
|
|
24
|
+
if len(missing) == len(tasks):
|
|
25
|
+
return Finding(Status.FAIL, none_msg) # every task: listing them adds nothing
|
|
26
|
+
ok = len(tasks) - len(missing)
|
|
27
|
+
return Finding(Status.WARN, f"{ok} of {len(tasks)} tasks {noun}.", missing)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def has_schedule(dag: Any) -> bool:
|
|
31
|
+
timetable = getattr(dag, "timetable", None)
|
|
32
|
+
return timetable is not None and type(timetable).__name__ != "NullTimetable"
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from dagwick.checks._util import coverage
|
|
6
|
+
from dagwick.core import Category, Finding, Status, check
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@check(
|
|
10
|
+
id="failure_alerts",
|
|
11
|
+
title="Someone hears about failures",
|
|
12
|
+
category=Category.OBSERVABILITY,
|
|
13
|
+
weight=15,
|
|
14
|
+
example='default_args={"on_failure_callback": send_slack_alert}',
|
|
15
|
+
fix="Add on_failure_callback (e.g. a Slack notifier) in default_args.",
|
|
16
|
+
)
|
|
17
|
+
def failure_alerts(dag: Any) -> Finding:
|
|
18
|
+
if getattr(dag, "has_on_failure_callback", False):
|
|
19
|
+
return Finding(Status.PASS, "The DAG has an on_failure_callback.")
|
|
20
|
+
return coverage(
|
|
21
|
+
dag,
|
|
22
|
+
lambda t: bool(getattr(t, "has_on_failure_callback", False)),
|
|
23
|
+
noun="alert on failure",
|
|
24
|
+
all_msg="Every task alerts on failure.",
|
|
25
|
+
none_msg="Nothing alerts on failure; it will fail silently.",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@check(
|
|
30
|
+
id="retry_alerts",
|
|
31
|
+
title="Retries are visible",
|
|
32
|
+
category=Category.OBSERVABILITY,
|
|
33
|
+
weight=3,
|
|
34
|
+
example='default_args={"on_retry_callback": send_slack_alert}',
|
|
35
|
+
fix="Add on_retry_callback so flaky tasks surface before they start failing.",
|
|
36
|
+
)
|
|
37
|
+
def retry_alerts(dag: Any) -> Finding:
|
|
38
|
+
if any(getattr(t, "has_on_retry_callback", False) for t in dag.tasks):
|
|
39
|
+
return Finding(Status.PASS, "Retries send a notification.")
|
|
40
|
+
return Finding(Status.WARN, "Retries happen silently.")
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from dagwick.core import Category, Finding, Status, check
|
|
6
|
+
|
|
7
|
+
DEFAULT_OWNERS = {"", "airflow"}
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@check(
|
|
11
|
+
id="owner",
|
|
12
|
+
title="Has a real owner",
|
|
13
|
+
category=Category.OWNERSHIP,
|
|
14
|
+
weight=8,
|
|
15
|
+
example='default_args={"owner": "data-team"}',
|
|
16
|
+
fix="Set owner in default_args to the responsible team (e.g. 'data-team').",
|
|
17
|
+
)
|
|
18
|
+
def owner(dag: Any) -> Finding:
|
|
19
|
+
owners = {o.strip().lower() for o in (getattr(dag, "owner", "") or "").split(",")}
|
|
20
|
+
if owners - DEFAULT_OWNERS:
|
|
21
|
+
return Finding(Status.PASS, "An owner is set.")
|
|
22
|
+
return Finding(Status.FAIL, "Owner is the default 'airflow'; nobody is accountable.")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@check(
|
|
26
|
+
id="tags",
|
|
27
|
+
title="Tagged",
|
|
28
|
+
category=Category.OWNERSHIP,
|
|
29
|
+
weight=5,
|
|
30
|
+
example='DAG(..., tags=["sales", "ingest"])',
|
|
31
|
+
fix="Add tags (domain, layer) so the DAG is easy to find and filter.",
|
|
32
|
+
)
|
|
33
|
+
def tags(dag: Any) -> Finding:
|
|
34
|
+
found = sorted(getattr(dag, "tags", None) or [])
|
|
35
|
+
if found:
|
|
36
|
+
return Finding(Status.PASS, f"Tags: {', '.join(found)}.")
|
|
37
|
+
return Finding(Status.FAIL, "No tags.")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@check(
|
|
41
|
+
id="docs",
|
|
42
|
+
title="Documented",
|
|
43
|
+
category=Category.OWNERSHIP,
|
|
44
|
+
weight=8,
|
|
45
|
+
example="DAG(..., doc_md=__doc__) # reuse the module docstring",
|
|
46
|
+
fix="Add doc_md explaining what the DAG loads, from where, and who to call.",
|
|
47
|
+
)
|
|
48
|
+
def docs(dag: Any) -> Finding:
|
|
49
|
+
if str(getattr(dag, "doc_md", None) or "").strip():
|
|
50
|
+
return Finding(Status.PASS, "doc_md is present.")
|
|
51
|
+
if str(getattr(dag, "description", None) or "").strip():
|
|
52
|
+
return Finding(Status.WARN, "Only a one-line description; no doc_md.")
|
|
53
|
+
return Finding(Status.FAIL, "No documentation.")
|