airflow-embedash 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow_embedash-0.1.1/PKG-INFO +58 -0
- airflow_embedash-0.1.1/README.md +46 -0
- airflow_embedash-0.1.1/airflow_embedash/__init__.py +7 -0
- airflow_embedash-0.1.1/airflow_embedash/plugin/__init__.py +31 -0
- airflow_embedash-0.1.1/airflow_embedash/plugin/airflow2.py +219 -0
- airflow_embedash-0.1.1/airflow_embedash/plugin/airflow3.py +151 -0
- airflow_embedash-0.1.1/airflow_embedash/plugin/cluster_policy.py +45 -0
- airflow_embedash-0.1.1/airflow_embedash/plugin/snippets.py +24 -0
- airflow_embedash-0.1.1/airflow_embedash/plugin/storage.py +23 -0
- airflow_embedash-0.1.1/airflow_embedash.egg-info/PKG-INFO +58 -0
- airflow_embedash-0.1.1/airflow_embedash.egg-info/SOURCES.txt +14 -0
- airflow_embedash-0.1.1/airflow_embedash.egg-info/dependency_links.txt +1 -0
- airflow_embedash-0.1.1/airflow_embedash.egg-info/entry_points.txt +2 -0
- airflow_embedash-0.1.1/airflow_embedash.egg-info/top_level.txt +1 -0
- airflow_embedash-0.1.1/pyproject.toml +61 -0
- airflow_embedash-0.1.1/setup.cfg +4 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: airflow-embedash
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Add embeded dashboars to Airflow
|
|
5
|
+
Author-email: Rodrigo Carneiro <teoria@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/teoria/airflow-embedash
|
|
7
|
+
Project-URL: Documentation, https://github.com/teoria/airflow-embedash
|
|
8
|
+
Project-URL: Source code, https://github.com/teoria/airflow-embedash
|
|
9
|
+
Keywords: airflow,apache-airflow,metabase,grafana,datadog
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# Airflow Embedash
|
|
14
|
+
|
|
15
|
+
A Python package for embedding dashboards in Apache Airflow.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
Install the package using pip:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install airflow-embedash
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Or in development mode:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install -e .
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Usage
|
|
32
|
+
|
|
33
|
+
Create new dashboards
|
|
34
|
+
|
|
35
|
+
### Variable
|
|
36
|
+
|
|
37
|
+
The default menu label is "Dashboards" but can be changed setup `embeded_dashboards_menu_label`variable.
|
|
38
|
+
|
|
39
|
+
For private dashboars need metabase token `embeded_dashboards_metabase_token`
|
|
40
|
+
|
|
41
|
+
## Package Structure
|
|
42
|
+
|
|
43
|
+
- `src/` - Source code directory
|
|
44
|
+
- `__init__.py` - Package initialization
|
|
45
|
+
- `airflow_embeded_dashboards.py` - Main module with the main function
|
|
46
|
+
|
|
47
|
+
## Development
|
|
48
|
+
|
|
49
|
+
To contribute to this project:
|
|
50
|
+
|
|
51
|
+
1. Fork the repository
|
|
52
|
+
2. Create a feature branch
|
|
53
|
+
3. Make your changes
|
|
54
|
+
4. Submit a pull request
|
|
55
|
+
|
|
56
|
+
## License
|
|
57
|
+
|
|
58
|
+
This project is licensed under the MIT License.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Airflow Embedash
|
|
2
|
+
|
|
3
|
+
A Python package for embedding dashboards in Apache Airflow.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Install the package using pip:
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install airflow-embedash
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Or in development mode:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install -e .
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
Create new dashboards
|
|
22
|
+
|
|
23
|
+
### Variable
|
|
24
|
+
|
|
25
|
+
The default menu label is "Dashboards" but can be changed setup `embeded_dashboards_menu_label`variable.
|
|
26
|
+
|
|
27
|
+
For private dashboars need metabase token `embeded_dashboards_metabase_token`
|
|
28
|
+
|
|
29
|
+
## Package Structure
|
|
30
|
+
|
|
31
|
+
- `src/` - Source code directory
|
|
32
|
+
- `__init__.py` - Package initialization
|
|
33
|
+
- `airflow_embeded_dashboards.py` - Main module with the main function
|
|
34
|
+
|
|
35
|
+
## Development
|
|
36
|
+
|
|
37
|
+
To contribute to this project:
|
|
38
|
+
|
|
39
|
+
1. Fork the repository
|
|
40
|
+
2. Create a feature branch
|
|
41
|
+
3. Make your changes
|
|
42
|
+
4. Submit a pull request
|
|
43
|
+
|
|
44
|
+
## License
|
|
45
|
+
|
|
46
|
+
This project is licensed under the MIT License.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from airflow import __version__ as airflow_version
|
|
6
|
+
from packaging import version
|
|
7
|
+
|
|
8
|
+
_AIRFLOW3_MAJOR_VERSION = 3
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
11
|
+
from .airflow2 import EmbededDashPlugin as _EmbededDashPluginType
|
|
12
|
+
from .airflow3 import EmbededDashAF3Plugin as _EmbededDashAF3PluginType
|
|
13
|
+
|
|
14
|
+
_EmbededDashPlugin: _EmbededDashPluginType | _EmbededDashAF3PluginType | None = None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def __getattr__(name: str) :
|
|
18
|
+
if name == "EmbededDashPlugin":
|
|
19
|
+
global _EmbededDashPlugin
|
|
20
|
+
if _EmbededDashPlugin is None:
|
|
21
|
+
if version.parse(airflow_version).major < _AIRFLOW3_MAJOR_VERSION:
|
|
22
|
+
from .airflow2 import EmbededDashPlugin # type: ignore[assignment] # noqa: F401
|
|
23
|
+
|
|
24
|
+
_EmbededDashPlugin = EmbededDashPlugin # type: ignore[assignment]
|
|
25
|
+
else:
|
|
26
|
+
# _EmbededDashPlugin = EmbededDashPlugin # type: ignore[assignment]
|
|
27
|
+
from .airflow3 import EmbededDashAF3Plugin # type: ignore[assignment] # noqa: F401
|
|
28
|
+
|
|
29
|
+
_EmbededDashPlugin = EmbededDashAF3Plugin # type: ignore[assignment]
|
|
30
|
+
return _EmbededDashPlugin
|
|
31
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os.path as op
|
|
3
|
+
from typing import Any
|
|
4
|
+
from urllib.parse import urlsplit
|
|
5
|
+
from airflow.models import Variable
|
|
6
|
+
|
|
7
|
+
from airflow.configuration import conf
|
|
8
|
+
from airflow.plugins_manager import AirflowPlugin
|
|
9
|
+
from airflow.security import permissions
|
|
10
|
+
from airflow.www.auth import has_access
|
|
11
|
+
from airflow.www.views import AirflowBaseView
|
|
12
|
+
from flask import abort
|
|
13
|
+
from flask_appbuilder import AppBuilder, expose
|
|
14
|
+
from flask import request, redirect, url_for
|
|
15
|
+
import uuid
|
|
16
|
+
from airflow.www.app import csrf
|
|
17
|
+
|
|
18
|
+
MENU_ACCESS_PERMISSIONS = [
|
|
19
|
+
(permissions.ACTION_CAN_READ, permissions.RESOURCE_WEBSITE),
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class EmbededView(AirflowBaseView): # type: ignore
|
|
25
|
+
default_view = "settings"
|
|
26
|
+
route_base = "/embedash"
|
|
27
|
+
template_folder = op.join(op.dirname(__file__), "templates")
|
|
28
|
+
static_folder = op.join(op.dirname(__file__), "static")
|
|
29
|
+
|
|
30
|
+
plugin = None # type: ignore[assignment]
|
|
31
|
+
|
|
32
|
+
def create_blueprint(
|
|
33
|
+
self, appbuilder: AppBuilder, endpoint: str | None = None, static_folder: str | None = None
|
|
34
|
+
) -> None:
|
|
35
|
+
# Make sure the static folder is not overwritten, as we want to use it.
|
|
36
|
+
return super().create_blueprint(appbuilder, endpoint=endpoint, static_folder=self.static_folder) # type: ignore[no-any-return]
|
|
37
|
+
|
|
38
|
+
@expose("/settings") # type: ignore[untyped-decorator]
|
|
39
|
+
@has_access(MENU_ACCESS_PERMISSIONS) # type: ignore[untyped-decorator]
|
|
40
|
+
def settings(self) -> Any:
|
|
41
|
+
dashboards = Variable.get("embeded_dashboards", default_var=[], deserialize_json=True)
|
|
42
|
+
return self.render_template("settings.html", dashboards=dashboards) # type: ignore[no-any-return,no-untyped-call]
|
|
43
|
+
|
|
44
|
+
@expose("/add_dash", methods=['GET', 'POST']) # type: ignore[untyped-decorator]
|
|
45
|
+
@has_access(MENU_ACCESS_PERMISSIONS) # type: ignore[untyped-decorator]
|
|
46
|
+
@csrf.exempt
|
|
47
|
+
def DashboardSettings_add(self) -> Any:
|
|
48
|
+
if request.method == 'POST':
|
|
49
|
+
# Get data from the form fields using their 'name' attributes
|
|
50
|
+
name = request.form.get('name')
|
|
51
|
+
description = request.form.get('description')
|
|
52
|
+
url = request.form.get('url')
|
|
53
|
+
payload = request.form.get('payload',{})
|
|
54
|
+
if payload == "":
|
|
55
|
+
payload = {}
|
|
56
|
+
else:
|
|
57
|
+
payload = json.loads(payload)
|
|
58
|
+
# Perform some action with the data (e.g., save to database, validation)
|
|
59
|
+
if name and description and url:
|
|
60
|
+
# Create a new dashboard object
|
|
61
|
+
new_dashboard = {
|
|
62
|
+
"id": uuid.uuid4().hex, # Generate a unique ID for the dashboard
|
|
63
|
+
"name": name,
|
|
64
|
+
"description": description,
|
|
65
|
+
"url": url,
|
|
66
|
+
"type": "public" if len(payload.items()) == 0 else "private",
|
|
67
|
+
"payload": payload
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
dashboards = Variable.get("embeded_dashboards", default_var=[], deserialize_json=True)
|
|
71
|
+
dashboards.append(new_dashboard)
|
|
72
|
+
Variable.set(key="embeded_dashboards", value=dashboards, serialize_json=True)
|
|
73
|
+
return redirect(url_for('EmbededView.settings'))
|
|
74
|
+
|
|
75
|
+
else:
|
|
76
|
+
return "Invalid credentials, please try again."
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@expose("/adddashboard") # type: ignore[untyped-decorator]
|
|
81
|
+
@has_access(MENU_ACCESS_PERMISSIONS) # type: ignore[untyped-decorator]
|
|
82
|
+
def DashboardSettings_add_dash(self) -> Any:
|
|
83
|
+
return self.render_template("add_dashboard.html") # type: ignore[no-any-return,no-untyped-call]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@expose("/editdashboard/<id>") # type: ignore[untyped-decorator]
|
|
88
|
+
@has_access(MENU_ACCESS_PERMISSIONS) # type: ignore[untyped-decorator]
|
|
89
|
+
def DashboardSettings_edit(self, id: str) -> Any:
|
|
90
|
+
|
|
91
|
+
dashboards = Variable.get("embeded_dashboards", default_var=[], deserialize_json=True)
|
|
92
|
+
for dashboard in dashboards:
|
|
93
|
+
if dashboard.get("id") == id:
|
|
94
|
+
dash = dashboard
|
|
95
|
+
break
|
|
96
|
+
else:
|
|
97
|
+
dash = None
|
|
98
|
+
return self.render_template("edit_dashboard.html", dashboard=dash) # type: ignore[no-any-return,no-untyped-call]
|
|
99
|
+
|
|
100
|
+
@expose("/edit_dash/<id>", methods=['GET', 'POST']) # type: ignore[untyped-decorator]
|
|
101
|
+
@has_access(MENU_ACCESS_PERMISSIONS) # type: ignore[untyped-decorator]
|
|
102
|
+
@csrf.exempt
|
|
103
|
+
def DashboardSettings_edit_dash(self, id: str) -> Any:
|
|
104
|
+
if request.method == 'POST':
|
|
105
|
+
# Get data from the form fields using their 'name' attributes
|
|
106
|
+
|
|
107
|
+
name = request.form.get('name')
|
|
108
|
+
description = request.form.get('description')
|
|
109
|
+
url = request.form.get('url')
|
|
110
|
+
payload = request.form.get('payload',{})
|
|
111
|
+
if payload == "":
|
|
112
|
+
payload = {}
|
|
113
|
+
else:
|
|
114
|
+
payload = json.loads(payload)
|
|
115
|
+
# Perform some action with the data (e.g., save to database, validation)
|
|
116
|
+
if name and description and url:
|
|
117
|
+
# Create a new dashboard object
|
|
118
|
+
new_dashboard = {
|
|
119
|
+
"id": id,
|
|
120
|
+
"name": name,
|
|
121
|
+
"description": description,
|
|
122
|
+
"url": url,
|
|
123
|
+
"type": "public" if len(payload.items()) == 0 else "private",
|
|
124
|
+
"payload": payload
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
dashboards = Variable.get("embeded_dashboards", default_var=[], deserialize_json=True)
|
|
128
|
+
|
|
129
|
+
for i, dashboard in enumerate(dashboards):
|
|
130
|
+
if dashboard.get("id") == id:
|
|
131
|
+
dashboards[i] = new_dashboard # Update the existing dashboard with the new data
|
|
132
|
+
break
|
|
133
|
+
|
|
134
|
+
Variable.set(key="embeded_dashboards", value=dashboards, serialize_json=True)
|
|
135
|
+
return redirect(url_for('EmbededView.settings'))
|
|
136
|
+
|
|
137
|
+
else:
|
|
138
|
+
return "Invalid credentials, please try again."
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@expose("/deletedashboard/<id>") # type: ignore[untyped-decorator]
|
|
145
|
+
@has_access(MENU_ACCESS_PERMISSIONS) # type: ignore[untyped-decorator]
|
|
146
|
+
def DashboardSettings_delete(self, id: str) -> Any:
|
|
147
|
+
# Here you would add logic to delete the dashboard with the given ID from your database
|
|
148
|
+
# For now, we'll just redirect back to the settings page after "deleting"
|
|
149
|
+
|
|
150
|
+
dashboards = Variable.get("embeded_dashboards", default_var=[], deserialize_json=True)
|
|
151
|
+
new_dashboards = [dashboard for dashboard in dashboards if dashboard.get("id") != id]
|
|
152
|
+
Variable.set(key="embeded_dashboards", value=new_dashboards, serialize_json=True)
|
|
153
|
+
embeded_view.plugin.refresh_appbuilder_views()
|
|
154
|
+
return redirect(url_for('EmbededView.settings'))
|
|
155
|
+
|
|
156
|
+
@expose("/view/<id>") # type: ignore[untyped-decorator]
|
|
157
|
+
@has_access(MENU_ACCESS_PERMISSIONS) # type: ignore[untyped-decorator]
|
|
158
|
+
def DashboardSettings_view(self, id: str) -> Any:
|
|
159
|
+
|
|
160
|
+
dashboards = Variable.get("embeded_dashboards", default_var=[], deserialize_json=True)
|
|
161
|
+
for dashboard in dashboards:
|
|
162
|
+
if dashboard.get("id") == id:
|
|
163
|
+
dash = dashboard
|
|
164
|
+
break
|
|
165
|
+
else:
|
|
166
|
+
dash = None
|
|
167
|
+
|
|
168
|
+
if dashboard.get("type") == "private":
|
|
169
|
+
payload = dashboard.get("payload", {})
|
|
170
|
+
if payload:
|
|
171
|
+
|
|
172
|
+
import jwt
|
|
173
|
+
import time
|
|
174
|
+
|
|
175
|
+
METABASE_SITE_URL = dashboard.get("url", "")
|
|
176
|
+
METABASE_SECRET_KEY = Variable.get("embeded_dashboards_metabase_token")
|
|
177
|
+
|
|
178
|
+
payload["exp"] = round(time.time()) + (60 * 10) # 10 minute expiration
|
|
179
|
+
token = jwt.encode(payload, METABASE_SECRET_KEY, algorithm="HS256")
|
|
180
|
+
|
|
181
|
+
iframeUrl = METABASE_SITE_URL + "/embed/dashboard/" + token + "#bordered=true&titled=true"
|
|
182
|
+
dashboard["url"] = iframeUrl
|
|
183
|
+
|
|
184
|
+
return self.render_template("view.html", dashboard=dash) # type: ignore[no-any-return,no-untyped-call]
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
embeded_view = EmbededView()
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class EmbededDashPlugin(AirflowPlugin):
|
|
191
|
+
name = "embeded_dashboards"
|
|
192
|
+
|
|
193
|
+
def __init__(self) -> None:
|
|
194
|
+
embeded_view.plugin = self # type: ignore[assignment]
|
|
195
|
+
self.refresh_appbuilder_views()
|
|
196
|
+
|
|
197
|
+
def refresh_appbuilder_views(self) -> None:
|
|
198
|
+
# This method can be called to refresh the appbuilder views based on the current dashboards variable
|
|
199
|
+
menu_label = Variable.get("embeded_dashboards_menu_label", default_var="Dashboards")
|
|
200
|
+
dashboards = Variable.get("embeded_dashboards", default_var=[], deserialize_json=True)
|
|
201
|
+
items = []
|
|
202
|
+
for dashboard in dashboards:
|
|
203
|
+
items.append(
|
|
204
|
+
{
|
|
205
|
+
"name": dashboard.get("name", "Unnamed Dashboard"),
|
|
206
|
+
"category": menu_label,
|
|
207
|
+
"view": embeded_view,
|
|
208
|
+
"href": conf.get("webserver", "base_url") + "/embedash/view/"+dashboard.get("id", ""),
|
|
209
|
+
}
|
|
210
|
+
)
|
|
211
|
+
items.append(
|
|
212
|
+
{
|
|
213
|
+
"name": "Settings",
|
|
214
|
+
"category": menu_label,
|
|
215
|
+
"view": embeded_view,
|
|
216
|
+
"href": conf.get("webserver", "base_url") + "/embedash/settings",
|
|
217
|
+
}
|
|
218
|
+
)
|
|
219
|
+
self.appbuilder_views = items
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import html
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import os.path as op
|
|
8
|
+
from collections.abc import Generator
|
|
9
|
+
from contextlib import contextmanager
|
|
10
|
+
from typing import Any
|
|
11
|
+
from unittest.mock import patch
|
|
12
|
+
from urllib.parse import urlsplit
|
|
13
|
+
|
|
14
|
+
import airflow
|
|
15
|
+
from airflow.configuration import conf
|
|
16
|
+
from airflow.plugins_manager import AirflowPlugin
|
|
17
|
+
from fastapi import FastAPI
|
|
18
|
+
from fastapi.responses import HTMLResponse, JSONResponse, Response
|
|
19
|
+
from packaging.version import Version
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# Airflow version gating: External views feature for the plugins used here (CosmosAF3Plugin) exist only in >= 3.1
|
|
23
|
+
# Note: We compute AIRFLOW_VERSION locally here (not from constants) so that tests can patch airflow.__version__ and reload this module
|
|
24
|
+
AIRFLOW_VERSION = Version(airflow.__version__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def ensure_airflow_version_supported() -> None:
|
|
28
|
+
if AIRFLOW_VERSION < Version("3.1.0"):
|
|
29
|
+
raise RuntimeError(
|
|
30
|
+
"Cosmos AF3 plugin requires Airflow >= 3.1. External views are unavailable on earlier versions."
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
API_BASE = conf.get("api", "base_url", fallback="") # reads AIRFLOW__API__BASE_URL
|
|
35
|
+
API_BASE_PATH = urlsplit(API_BASE).path.rstrip("/")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# Note: Airflow 3.1.0 had a limitation where plugins could not resolve connections via the API server.
|
|
39
|
+
# The fix was shipped in Airflow 3.1.1. For 3.1.0, we temporarily expose the connection via env vars inside a context manager.
|
|
40
|
+
@contextmanager
|
|
41
|
+
def connection_env(conn_id: str | None = None) -> Generator[None, None, None]: # pragma: no cover
|
|
42
|
+
"""
|
|
43
|
+
Temporarily expose a connection as AIRFLOW_CONN_{CONN_ID} in the environment.
|
|
44
|
+
|
|
45
|
+
This allows hooks and SDK code resolving connections via the environment
|
|
46
|
+
variables backend to find the connection during the scope of the context.
|
|
47
|
+
"""
|
|
48
|
+
if conn_id is None:
|
|
49
|
+
yield
|
|
50
|
+
|
|
51
|
+
from airflow.models.connection import Connection as ORMConnection
|
|
52
|
+
|
|
53
|
+
conn = ORMConnection.get_connection_from_secrets(conn_id) # type: ignore[arg-type]
|
|
54
|
+
env_name = f"AIRFLOW_CONN_{conn_id.upper()}" # type: ignore[union-attr]
|
|
55
|
+
env_value = conn.get_uri()
|
|
56
|
+
with patch.dict(os.environ, {env_name: env_value}, clear=False):
|
|
57
|
+
yield
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _load_projects_from_conf() -> dict[str, dict[str, str | None]]:
|
|
63
|
+
"""
|
|
64
|
+
Load dbt docs projects configuration.
|
|
65
|
+
|
|
66
|
+
Supports either:
|
|
67
|
+
- [cosmos] dbt_docs_projects = JSON mapping of project slug to {"dir","conn_id","index"}
|
|
68
|
+
- Legacy single-project settings: dbt_docs_dir, dbt_docs_conn_id, dbt_docs_index_file_name
|
|
69
|
+
"""
|
|
70
|
+
projects_raw = conf.get("cosmos", "dbt_docs_projects", fallback=None)
|
|
71
|
+
projects: dict[str, dict[str, str | None]] = {}
|
|
72
|
+
if projects_raw:
|
|
73
|
+
parsed = None
|
|
74
|
+
try:
|
|
75
|
+
parsed = json.loads(projects_raw)
|
|
76
|
+
except json.JSONDecodeError:
|
|
77
|
+
logging.exception("Invalid JSON in [cosmos] dbt_docs_projects: %s", projects_raw)
|
|
78
|
+
raise
|
|
79
|
+
|
|
80
|
+
if isinstance(parsed, dict):
|
|
81
|
+
for key, value in parsed.items():
|
|
82
|
+
if not isinstance(value, dict): # pragma: no cover
|
|
83
|
+
continue
|
|
84
|
+
projects[str(key)] = {
|
|
85
|
+
"dir": value.get("dir"),
|
|
86
|
+
"conn_id": value.get("conn_id"),
|
|
87
|
+
"index": value.get("index", "index.html"),
|
|
88
|
+
"name": value.get("name"),
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return projects
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def create_embedash_fastapi_app() -> FastAPI: # noqa: C901
|
|
96
|
+
ensure_airflow_version_supported()
|
|
97
|
+
app = FastAPI()
|
|
98
|
+
|
|
99
|
+
projects = _load_projects_from_conf()
|
|
100
|
+
|
|
101
|
+
# Dynamic endpoints for each project
|
|
102
|
+
for slug, cfg in projects.items():
|
|
103
|
+
# Simple HTML wrapper to embed the dbt docs UI
|
|
104
|
+
@app.get(f"/{slug}/view", response_class=HTMLResponse)
|
|
105
|
+
def dbt_docs_view(slug_alias: str = slug) -> str: # type: ignore[no-redef]
|
|
106
|
+
cfg_local = projects.get(slug_alias, {})
|
|
107
|
+
if not cfg_local.get("dir"):
|
|
108
|
+
return "<div>dbt Docs are not configured.</div>"
|
|
109
|
+
iframe_src = f"/embedash/{slug_alias}/dbt_docs_index.html"
|
|
110
|
+
safe_iframe_src = html.escape(iframe_src, quote=True)
|
|
111
|
+
return (
|
|
112
|
+
'<div style="height:100%;display:flex;flex-direction:column;">'
|
|
113
|
+
f'<iframe src="{safe_iframe_src}" style="border:0;flex:1 1 auto;"></iframe>'
|
|
114
|
+
"</div>"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
return app
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class EmbededDashAF3Plugin(AirflowPlugin):
|
|
121
|
+
name = "embeded_dashboards"
|
|
122
|
+
|
|
123
|
+
# Mount our FastAPI sub-app under /cosmos (initialized in __init__ after version check)
|
|
124
|
+
fastapi_apps: list[dict[str, Any]] = []
|
|
125
|
+
|
|
126
|
+
# Register external views for navigation
|
|
127
|
+
external_views: list[dict[str, Any]] = []
|
|
128
|
+
|
|
129
|
+
listeners = []
|
|
130
|
+
|
|
131
|
+
def __init__(self) -> None:
|
|
132
|
+
super().__init__()
|
|
133
|
+
ensure_airflow_version_supported()
|
|
134
|
+
# Initialize FastAPI app only after version support is confirmed
|
|
135
|
+
self.fastapi_apps = [
|
|
136
|
+
{
|
|
137
|
+
"name": "embeded_dashboards",
|
|
138
|
+
"app": create_embedash_fastapi_app(),
|
|
139
|
+
"url_prefix": "/embedash",
|
|
140
|
+
}
|
|
141
|
+
]
|
|
142
|
+
projects = _load_projects_from_conf()
|
|
143
|
+
for slug, cfg in projects.items():
|
|
144
|
+
display_name = cfg.get("name") or f"dbt Docs ({slug})"
|
|
145
|
+
self.external_views.append(
|
|
146
|
+
{
|
|
147
|
+
"name": display_name,
|
|
148
|
+
"category": "Browse",
|
|
149
|
+
"href": f"{API_BASE_PATH}/embedash/settings",
|
|
150
|
+
}
|
|
151
|
+
)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# from logging import getLogger
|
|
2
|
+
|
|
3
|
+
# from airflow.models.taskinstance import TaskInstance
|
|
4
|
+
# from airflow.policies import hookimpl
|
|
5
|
+
# from packaging.version import Version
|
|
6
|
+
|
|
7
|
+
# AIRFLOW_VERSION = 3
|
|
8
|
+
|
|
9
|
+
# log = getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# def _is_watcher_sensor(task_instance: TaskInstance) -> bool:
|
|
13
|
+
# """
|
|
14
|
+
# Check if the task instance is a watcher sensor.
|
|
15
|
+
|
|
16
|
+
# In Airflow 3, task_instance.task is a SerializedBaseOperator, isinstance checks won't work.
|
|
17
|
+
# Instead, check the task's module path, which is preserved in serialization.
|
|
18
|
+
# """
|
|
19
|
+
# from cosmos.operators._watcher.base import BaseConsumerSensor
|
|
20
|
+
|
|
21
|
+
# task = task_instance.task
|
|
22
|
+
# # Get the module, using _task_module if available (serialized tasks) or __module__ as fallback
|
|
23
|
+
# module = getattr(task, "_task_module", None) or task.__class__.__module__
|
|
24
|
+
|
|
25
|
+
# # Check if it's from the watcher operators module
|
|
26
|
+
# return "cosmos.operators.watcher" in module or isinstance(task, BaseConsumerSensor)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# @hookimpl
|
|
30
|
+
# def task_instance_mutation_hook(task_instance: TaskInstance) -> None:
|
|
31
|
+
# from cosmos.settings import watcher_dbt_execution_queue
|
|
32
|
+
|
|
33
|
+
# # In Airflow 3.x the task_instance_mutation_hook try_number starts at None or 0
|
|
34
|
+
# # in Airflow 2.x it starts at 1
|
|
35
|
+
# if AIRFLOW_VERSION >= Version("3.0.0"):
|
|
36
|
+
# retry_number = 1
|
|
37
|
+
# else:
|
|
38
|
+
# retry_number = 2
|
|
39
|
+
|
|
40
|
+
# if watcher_dbt_execution_queue and task_instance.try_number and _is_watcher_sensor(task_instance):
|
|
41
|
+
# if task_instance.try_number >= retry_number:
|
|
42
|
+
# log.info(
|
|
43
|
+
# f"Setting task {task_instance.task_id} to use watcher dbt execution queue: {watcher_dbt_execution_queue}",
|
|
44
|
+
# )
|
|
45
|
+
# task_instance.queue = watcher_dbt_execution_queue
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
IFRAME_SCRIPT = """
|
|
2
|
+
<script>
|
|
3
|
+
// Prevent parent hash changes from sending a message back to the parent.
|
|
4
|
+
// This is necessary for making sure the browser back button works properly.
|
|
5
|
+
let hashChangeLock = true;
|
|
6
|
+
|
|
7
|
+
window.addEventListener('hashchange', function () {
|
|
8
|
+
if (!hashChangeLock) {
|
|
9
|
+
window.parent.postMessage(window.location.hash);
|
|
10
|
+
}
|
|
11
|
+
hashChangeLock = false;
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
window.addEventListener('message', function (event) {
|
|
15
|
+
let msgData = event.data;
|
|
16
|
+
if (typeof msgData === 'string' && msgData.startsWith('#!')) {
|
|
17
|
+
let updateUrl = new URL(window.location);
|
|
18
|
+
updateUrl.hash = msgData;
|
|
19
|
+
hashChangeLock = true;
|
|
20
|
+
history.replaceState(null, null, updateUrl);
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
</script>
|
|
24
|
+
"""
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Utility functions for Cosmos plugins."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def get_storage_type_from_path(path: str) -> str:
|
|
5
|
+
"""Determine the storage type from the path.
|
|
6
|
+
|
|
7
|
+
Args:
|
|
8
|
+
path: Storage path (e.g., 's3://bucket/path', '/local/path')
|
|
9
|
+
|
|
10
|
+
Returns:
|
|
11
|
+
Storage type identifier: 's3', 'gcs', 'azure', 'http', or 'local'
|
|
12
|
+
"""
|
|
13
|
+
path = path.strip()
|
|
14
|
+
if path.startswith("s3://"):
|
|
15
|
+
return "s3"
|
|
16
|
+
elif path.startswith("gs://"):
|
|
17
|
+
return "gcs"
|
|
18
|
+
elif path.startswith("wasb://"):
|
|
19
|
+
return "azure"
|
|
20
|
+
elif path.startswith("http://") or path.startswith("https://"):
|
|
21
|
+
return "http"
|
|
22
|
+
else:
|
|
23
|
+
return "local"
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: airflow-embedash
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Add embeded dashboars to Airflow
|
|
5
|
+
Author-email: Rodrigo Carneiro <teoria@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/teoria/airflow-embedash
|
|
7
|
+
Project-URL: Documentation, https://github.com/teoria/airflow-embedash
|
|
8
|
+
Project-URL: Source code, https://github.com/teoria/airflow-embedash
|
|
9
|
+
Keywords: airflow,apache-airflow,metabase,grafana,datadog
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# Airflow Embedash
|
|
14
|
+
|
|
15
|
+
A Python package for embedding dashboards in Apache Airflow.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
Install the package using pip:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install airflow-embedash
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Or in development mode:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install -e .
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Usage
|
|
32
|
+
|
|
33
|
+
Create new dashboards
|
|
34
|
+
|
|
35
|
+
### Variable
|
|
36
|
+
|
|
37
|
+
The default menu label is "Dashboards" but can be changed setup `embeded_dashboards_menu_label`variable.
|
|
38
|
+
|
|
39
|
+
For private dashboars need metabase token `embeded_dashboards_metabase_token`
|
|
40
|
+
|
|
41
|
+
## Package Structure
|
|
42
|
+
|
|
43
|
+
- `src/` - Source code directory
|
|
44
|
+
- `__init__.py` - Package initialization
|
|
45
|
+
- `airflow_embeded_dashboards.py` - Main module with the main function
|
|
46
|
+
|
|
47
|
+
## Development
|
|
48
|
+
|
|
49
|
+
To contribute to this project:
|
|
50
|
+
|
|
51
|
+
1. Fork the repository
|
|
52
|
+
2. Create a feature branch
|
|
53
|
+
3. Make your changes
|
|
54
|
+
4. Submit a pull request
|
|
55
|
+
|
|
56
|
+
## License
|
|
57
|
+
|
|
58
|
+
This project is licensed under the MIT License.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
airflow_embedash/__init__.py
|
|
4
|
+
airflow_embedash.egg-info/PKG-INFO
|
|
5
|
+
airflow_embedash.egg-info/SOURCES.txt
|
|
6
|
+
airflow_embedash.egg-info/dependency_links.txt
|
|
7
|
+
airflow_embedash.egg-info/entry_points.txt
|
|
8
|
+
airflow_embedash.egg-info/top_level.txt
|
|
9
|
+
airflow_embedash/plugin/__init__.py
|
|
10
|
+
airflow_embedash/plugin/airflow2.py
|
|
11
|
+
airflow_embedash/plugin/airflow3.py
|
|
12
|
+
airflow_embedash/plugin/cluster_policy.py
|
|
13
|
+
airflow_embedash/plugin/snippets.py
|
|
14
|
+
airflow_embedash/plugin/storage.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
airflow_embedash
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
|
|
2
|
+
[project]
|
|
3
|
+
name = "airflow-embedash"
|
|
4
|
+
version = "0.1.1"
|
|
5
|
+
description = "Add embeded dashboars to Airflow"
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
authors = [{ name = "Rodrigo Carneiro", email = "teoria@gmail.com" }]
|
|
8
|
+
keywords = ["airflow", "apache-airflow", "metabase", "grafana", "datadog"]
|
|
9
|
+
requires-python = ">=3.9"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
[[tool.uv.index]]
|
|
13
|
+
name = "testpypi"
|
|
14
|
+
url = "https://test.pypi.org/simple/"
|
|
15
|
+
publish-url = "https://test.pypi.org/legacy/"
|
|
16
|
+
explicit = true
|
|
17
|
+
|
|
18
|
+
[build-system]
|
|
19
|
+
requires = ["setuptools", "wheel"]
|
|
20
|
+
build-backend = "setuptools.build_meta"
|
|
21
|
+
|
|
22
|
+
[dependency-groups]
|
|
23
|
+
dev = [
|
|
24
|
+
"pytest>=8.3.5",
|
|
25
|
+
"ruff>=0.11.7",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Homepage = "https://github.com/teoria/airflow-embedash"
|
|
31
|
+
Documentation = "https://github.com/teoria/airflow-embedash"
|
|
32
|
+
"Source code" = "https://github.com/teoria/airflow-embedash"
|
|
33
|
+
|
|
34
|
+
[tool.hatch.version]
|
|
35
|
+
path = "airflow_embedash/__init__.py"
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.targets.sdist]
|
|
38
|
+
include = ["/airflow_embedash"]
|
|
39
|
+
|
|
40
|
+
[tool.hatch.build.targets.wheel]
|
|
41
|
+
packages = ["/airflow_embedash"]
|
|
42
|
+
|
|
43
|
+
[tool.pytest.ini_options]
|
|
44
|
+
#addopts = "-n 8" # run tests in parallel, you can disable parallel test execution with "pytest -n0" command
|
|
45
|
+
log_level = "INFO"
|
|
46
|
+
#log_cli = "true" # activate live logging, do not use with -n 8 xdist option for parallel test execution: https://github.com/pytest-dev/pytest-xdist/issues/402
|
|
47
|
+
log_cli_level = "INFO"
|
|
48
|
+
|
|
49
|
+
[tool.ruff]
|
|
50
|
+
line-length = 120
|
|
51
|
+
|
|
52
|
+
[tool.ruff.lint]
|
|
53
|
+
extend-select = [
|
|
54
|
+
"I", # re-order imports in alphabetic order
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
[project.entry-points."airflow.plugins"]
|
|
59
|
+
airflow-embedash = "airflow_embedash.plugin:EmbededDashPlugin"
|
|
60
|
+
|
|
61
|
+
|