apache-airflow-providers-edge3 1.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/edge3/LICENSE +201 -0
- airflow/providers/edge3/__init__.py +39 -0
- airflow/providers/edge3/cli/__init__.py +16 -0
- airflow/providers/edge3/cli/api_client.py +206 -0
- airflow/providers/edge3/cli/dataclasses.py +95 -0
- airflow/providers/edge3/cli/edge_command.py +689 -0
- airflow/providers/edge3/example_dags/__init__.py +16 -0
- airflow/providers/edge3/example_dags/integration_test.py +164 -0
- airflow/providers/edge3/example_dags/win_notepad.py +83 -0
- airflow/providers/edge3/example_dags/win_test.py +342 -0
- airflow/providers/edge3/executors/__init__.py +22 -0
- airflow/providers/edge3/executors/edge_executor.py +367 -0
- airflow/providers/edge3/get_provider_info.py +99 -0
- airflow/providers/edge3/models/__init__.py +16 -0
- airflow/providers/edge3/models/edge_job.py +94 -0
- airflow/providers/edge3/models/edge_logs.py +73 -0
- airflow/providers/edge3/models/edge_worker.py +230 -0
- airflow/providers/edge3/openapi/__init__.py +19 -0
- airflow/providers/edge3/openapi/edge_worker_api_v1.yaml +808 -0
- airflow/providers/edge3/plugins/__init__.py +16 -0
- airflow/providers/edge3/plugins/edge_executor_plugin.py +229 -0
- airflow/providers/edge3/plugins/templates/edge_worker_hosts.html +175 -0
- airflow/providers/edge3/plugins/templates/edge_worker_jobs.html +69 -0
- airflow/providers/edge3/version_compat.py +36 -0
- airflow/providers/edge3/worker_api/__init__.py +17 -0
- airflow/providers/edge3/worker_api/app.py +43 -0
- airflow/providers/edge3/worker_api/auth.py +135 -0
- airflow/providers/edge3/worker_api/datamodels.py +190 -0
- airflow/providers/edge3/worker_api/routes/__init__.py +16 -0
- airflow/providers/edge3/worker_api/routes/_v2_compat.py +135 -0
- airflow/providers/edge3/worker_api/routes/_v2_routes.py +237 -0
- airflow/providers/edge3/worker_api/routes/health.py +28 -0
- airflow/providers/edge3/worker_api/routes/jobs.py +162 -0
- airflow/providers/edge3/worker_api/routes/logs.py +133 -0
- airflow/providers/edge3/worker_api/routes/worker.py +224 -0
- apache_airflow_providers_edge3-1.0.0rc1.dist-info/METADATA +117 -0
- apache_airflow_providers_edge3-1.0.0rc1.dist-info/RECORD +39 -0
- apache_airflow_providers_edge3-1.0.0rc1.dist-info/WHEEL +4 -0
- apache_airflow_providers_edge3-1.0.0rc1.dist-info/entry_points.txt +6 -0
@@ -0,0 +1,16 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
@@ -0,0 +1,229 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
from __future__ import annotations
|
19
|
+
|
20
|
+
import re
|
21
|
+
from datetime import datetime, timedelta
|
22
|
+
from pathlib import Path
|
23
|
+
from typing import TYPE_CHECKING, Any
|
24
|
+
|
25
|
+
from flask import Blueprint, redirect, request, url_for
|
26
|
+
from flask_appbuilder import BaseView, expose
|
27
|
+
from markupsafe import Markup
|
28
|
+
from sqlalchemy import select
|
29
|
+
|
30
|
+
from airflow.configuration import conf
|
31
|
+
from airflow.exceptions import AirflowConfigException
|
32
|
+
from airflow.models.taskinstance import TaskInstanceState
|
33
|
+
from airflow.plugins_manager import AirflowPlugin
|
34
|
+
from airflow.providers.edge3.version_compat import AIRFLOW_V_3_0_PLUS
|
35
|
+
from airflow.utils.state import State
|
36
|
+
|
37
|
+
if AIRFLOW_V_3_0_PLUS:
|
38
|
+
from airflow.api_fastapi.auth.managers.models.resource_details import AccessView
|
39
|
+
from airflow.providers.fab.www.auth import has_access_view
|
40
|
+
|
41
|
+
else:
|
42
|
+
from airflow.auth.managers.models.resource_details import AccessView # type: ignore[no-redef]
|
43
|
+
from airflow.www.auth import has_access_view # type: ignore[no-redef]
|
44
|
+
from airflow.utils.session import NEW_SESSION, provide_session
|
45
|
+
from airflow.utils.yaml import safe_load
|
46
|
+
|
47
|
+
if TYPE_CHECKING:
|
48
|
+
from sqlalchemy.orm import Session
|
49
|
+
|
50
|
+
|
51
|
+
def _get_airflow_2_api_endpoint() -> Blueprint:
|
52
|
+
from airflow.www.constants import SWAGGER_BUNDLE, SWAGGER_ENABLED
|
53
|
+
from airflow.www.extensions.init_views import _CustomErrorRequestBodyValidator, _LazyResolver
|
54
|
+
|
55
|
+
folder = Path(__file__).parents[1].resolve() # this is airflow/providers/edge3/
|
56
|
+
with folder.joinpath("openapi", "edge_worker_api_v1.yaml").open() as f:
|
57
|
+
specification = safe_load(f)
|
58
|
+
from connexion import FlaskApi
|
59
|
+
|
60
|
+
bp = FlaskApi(
|
61
|
+
specification=specification,
|
62
|
+
resolver=_LazyResolver(),
|
63
|
+
base_path="/edge_worker/v1",
|
64
|
+
strict_validation=True,
|
65
|
+
options={"swagger_ui": SWAGGER_ENABLED, "swagger_path": SWAGGER_BUNDLE.__fspath__()},
|
66
|
+
validate_responses=True,
|
67
|
+
validator_map={"body": _CustomErrorRequestBodyValidator},
|
68
|
+
).blueprint
|
69
|
+
# Need to exempt CSRF to make API usable
|
70
|
+
from airflow.www.app import csrf
|
71
|
+
|
72
|
+
csrf.exempt(bp)
|
73
|
+
return bp
|
74
|
+
|
75
|
+
|
76
|
+
def _get_api_endpoint() -> dict[str, Any]:
|
77
|
+
from airflow.providers.edge3.worker_api.app import create_edge_worker_api_app
|
78
|
+
|
79
|
+
return {
|
80
|
+
"app": create_edge_worker_api_app(),
|
81
|
+
"url_prefix": "/edge_worker/v1",
|
82
|
+
"name": "Airflow Edge Worker API",
|
83
|
+
}
|
84
|
+
|
85
|
+
|
86
|
+
def _state_token(state):
|
87
|
+
"""Return a formatted string with HTML for a given State."""
|
88
|
+
color = State.color(state)
|
89
|
+
fg_color = State.color_fg(state)
|
90
|
+
return Markup(
|
91
|
+
"""
|
92
|
+
<span class="label" style="color:{fg_color}; background-color:{color};"
|
93
|
+
title="Current State: {state}">{state}</span>
|
94
|
+
"""
|
95
|
+
).format(color=color, state=state, fg_color=fg_color)
|
96
|
+
|
97
|
+
|
98
|
+
def modify_maintenance_comment_on_update(maintenance_comment: str | None, username: str) -> str:
|
99
|
+
if maintenance_comment:
|
100
|
+
if re.search(
|
101
|
+
r"^\[[-\d:\s]+\] - .+ put node into maintenance mode\r?\nComment:.*", maintenance_comment
|
102
|
+
):
|
103
|
+
return re.sub(
|
104
|
+
r"^\[[-\d:\s]+\] - .+ put node into maintenance mode\r?\nComment:",
|
105
|
+
f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {username} updated maintenance mode\nComment:",
|
106
|
+
maintenance_comment,
|
107
|
+
)
|
108
|
+
if re.search(r"^\[[-\d:\s]+\] - .+ updated maintenance mode\r?\nComment:.*", maintenance_comment):
|
109
|
+
return re.sub(
|
110
|
+
r"^\[[-\d:\s]+\] - .+ updated maintenance mode\r?\nComment:",
|
111
|
+
f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {username} updated maintenance mode\nComment:",
|
112
|
+
maintenance_comment,
|
113
|
+
)
|
114
|
+
return f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {username} updated maintenance mode\nComment: {maintenance_comment}"
|
115
|
+
return f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {username} updated maintenance mode\nComment:"
|
116
|
+
|
117
|
+
|
118
|
+
# registers airflow/providers/edge3/plugins/templates as a Jinja template folder
|
119
|
+
template_bp = Blueprint(
|
120
|
+
"template_blueprint",
|
121
|
+
__name__,
|
122
|
+
template_folder="templates",
|
123
|
+
)
|
124
|
+
|
125
|
+
|
126
|
+
class EdgeWorkerJobs(BaseView):
|
127
|
+
"""Simple view to show Edge Worker jobs."""
|
128
|
+
|
129
|
+
default_view = "jobs"
|
130
|
+
|
131
|
+
@expose("/jobs")
|
132
|
+
@has_access_view(AccessView.JOBS)
|
133
|
+
@provide_session
|
134
|
+
def jobs(self, session: Session = NEW_SESSION):
|
135
|
+
from airflow.providers.edge3.models.edge_job import EdgeJobModel
|
136
|
+
|
137
|
+
jobs = session.scalars(select(EdgeJobModel).order_by(EdgeJobModel.queued_dttm)).all()
|
138
|
+
html_states = {
|
139
|
+
str(state): _state_token(str(state)) for state in TaskInstanceState.__members__.values()
|
140
|
+
}
|
141
|
+
return self.render_template("edge_worker_jobs.html", jobs=jobs, html_states=html_states)
|
142
|
+
|
143
|
+
|
144
|
+
class EdgeWorkerHosts(BaseView):
|
145
|
+
"""Simple view to show Edge Worker status."""
|
146
|
+
|
147
|
+
default_view = "status"
|
148
|
+
|
149
|
+
@expose("/status")
|
150
|
+
@has_access_view(AccessView.JOBS)
|
151
|
+
@provide_session
|
152
|
+
def status(self, session: Session = NEW_SESSION):
|
153
|
+
from airflow.providers.edge3.models.edge_worker import EdgeWorkerModel
|
154
|
+
|
155
|
+
hosts = session.scalars(select(EdgeWorkerModel).order_by(EdgeWorkerModel.worker_name)).all()
|
156
|
+
five_min_ago = datetime.now() - timedelta(minutes=5)
|
157
|
+
return self.render_template("edge_worker_hosts.html", hosts=hosts, five_min_ago=five_min_ago)
|
158
|
+
|
159
|
+
@expose("/status/maintenance/<string:worker_name>/on", methods=["POST"])
|
160
|
+
@has_access_view(AccessView.JOBS)
|
161
|
+
def worker_to_maintenance(self, worker_name: str):
|
162
|
+
from flask_login import current_user
|
163
|
+
|
164
|
+
from airflow.providers.edge3.models.edge_worker import request_maintenance
|
165
|
+
|
166
|
+
maintenance_comment = request.form.get("maintenance_comment")
|
167
|
+
maintenance_comment = f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {current_user.username} put node into maintenance mode\nComment: {maintenance_comment}"
|
168
|
+
request_maintenance(worker_name, maintenance_comment)
|
169
|
+
return redirect(url_for("EdgeWorkerHosts.status"))
|
170
|
+
|
171
|
+
@expose("/status/maintenance/<string:worker_name>/off", methods=["POST"])
|
172
|
+
@has_access_view(AccessView.JOBS)
|
173
|
+
def remove_worker_from_maintenance(self, worker_name: str):
|
174
|
+
from airflow.providers.edge3.models.edge_worker import exit_maintenance
|
175
|
+
|
176
|
+
exit_maintenance(worker_name)
|
177
|
+
return redirect(url_for("EdgeWorkerHosts.status"))
|
178
|
+
|
179
|
+
@expose("/status/maintenance/<string:worker_name>/remove", methods=["POST"])
|
180
|
+
@has_access_view(AccessView.JOBS)
|
181
|
+
def remove_worker(self, worker_name: str):
|
182
|
+
from airflow.providers.edge3.models.edge_worker import remove_worker
|
183
|
+
|
184
|
+
remove_worker(worker_name)
|
185
|
+
return redirect(url_for("EdgeWorkerHosts.status"))
|
186
|
+
|
187
|
+
@expose("/status/maintenance/<string:worker_name>/change_comment", methods=["POST"])
|
188
|
+
@has_access_view(AccessView.JOBS)
|
189
|
+
def change_maintenance_comment(self, worker_name: str):
|
190
|
+
from flask_login import current_user
|
191
|
+
|
192
|
+
from airflow.providers.edge3.models.edge_worker import change_maintenance_comment
|
193
|
+
|
194
|
+
maintenance_comment = request.form.get("maintenance_comment")
|
195
|
+
maintenance_comment = modify_maintenance_comment_on_update(maintenance_comment, current_user.username)
|
196
|
+
change_maintenance_comment(worker_name, maintenance_comment)
|
197
|
+
return redirect(url_for("EdgeWorkerHosts.status"))
|
198
|
+
|
199
|
+
|
200
|
+
# Check if EdgeExecutor is actually loaded
|
201
|
+
try:
|
202
|
+
EDGE_EXECUTOR_ACTIVE = conf.getboolean("edge", "api_enabled", fallback="False")
|
203
|
+
except AirflowConfigException:
|
204
|
+
EDGE_EXECUTOR_ACTIVE = False
|
205
|
+
|
206
|
+
|
207
|
+
class EdgeExecutorPlugin(AirflowPlugin):
|
208
|
+
"""EdgeExecutor Plugin - provides API endpoints for Edge Workers in Webserver."""
|
209
|
+
|
210
|
+
name = "edge_executor"
|
211
|
+
if EDGE_EXECUTOR_ACTIVE:
|
212
|
+
appbuilder_views = [
|
213
|
+
{
|
214
|
+
"name": "Edge Worker Jobs",
|
215
|
+
"category": "Admin",
|
216
|
+
"view": EdgeWorkerJobs(),
|
217
|
+
},
|
218
|
+
{
|
219
|
+
"name": "Edge Worker Hosts",
|
220
|
+
"category": "Admin",
|
221
|
+
"view": EdgeWorkerHosts(),
|
222
|
+
},
|
223
|
+
]
|
224
|
+
|
225
|
+
if AIRFLOW_V_3_0_PLUS:
|
226
|
+
fastapi_apps = [_get_api_endpoint()]
|
227
|
+
flask_blueprints = [template_bp]
|
228
|
+
else:
|
229
|
+
flask_blueprints = [_get_airflow_2_api_endpoint(), template_bp]
|
@@ -0,0 +1,175 @@
|
|
1
|
+
{#
|
2
|
+
Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
or more contributor license agreements. See the NOTICE file
|
4
|
+
distributed with this work for additional information
|
5
|
+
regarding copyright ownership. The ASF licenses this file
|
6
|
+
to you under the Apache License, Version 2.0 (the
|
7
|
+
"License"); you may not use this file except in compliance
|
8
|
+
with the License. You may obtain a copy of the License at
|
9
|
+
|
10
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
|
12
|
+
Unless required by applicable law or agreed to in writing,
|
13
|
+
software distributed under the License is distributed on an
|
14
|
+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
KIND, either express or implied. See the License for the
|
16
|
+
specific language governing permissions and limitations
|
17
|
+
under the License.
|
18
|
+
#}
|
19
|
+
|
20
|
+
{% extends base_template %}
|
21
|
+
|
22
|
+
{% block title %}
|
23
|
+
Edge Worker Hosts
|
24
|
+
{% endblock %}
|
25
|
+
|
26
|
+
{% block content %}
|
27
|
+
<h2>Edge Worker Hosts</h2>
|
28
|
+
{% if hosts|length == 0 %}
|
29
|
+
<p>No Edge Workers connected or known currently.</p>
|
30
|
+
{% else %}
|
31
|
+
|
32
|
+
<script>
|
33
|
+
function showForm(worker_name) {
|
34
|
+
var button = document.getElementById("button_" + worker_name);
|
35
|
+
var form = document.getElementById("form_" + worker_name);
|
36
|
+
form.style.display = "block";
|
37
|
+
button.style.display = "none";
|
38
|
+
}
|
39
|
+
function showEditComment(worker_name) {
|
40
|
+
var display = document.getElementById("display_" + worker_name);
|
41
|
+
var textarea = document.getElementById("textarea_" + worker_name);
|
42
|
+
var button = document.getElementById("update_" + worker_name);
|
43
|
+
display.style.display = "none";
|
44
|
+
textarea.style.display = "block";
|
45
|
+
button.style.display = "inline";
|
46
|
+
}
|
47
|
+
</script>
|
48
|
+
<table class="table table-striped table-bordered">
|
49
|
+
<tr>
|
50
|
+
<th>Hostname</th>
|
51
|
+
<th>State</th>
|
52
|
+
<th>Queues</th>
|
53
|
+
<th>First Online</th>
|
54
|
+
<th>Last Heart Beat</th>
|
55
|
+
<th>Active Jobs</th>
|
56
|
+
<!-- Stats are not collected (yet) leave the columns out until available
|
57
|
+
<th>Jobs Taken</th>
|
58
|
+
<th>Jobs Success</th>
|
59
|
+
<th>Jobs Failed</th>
|
60
|
+
-->
|
61
|
+
<th>System Information</th>
|
62
|
+
<th>Operations</th>
|
63
|
+
</tr>
|
64
|
+
{% for host in hosts %}
|
65
|
+
<tr>
|
66
|
+
<td>{{ host.worker_name }}</a></td>
|
67
|
+
<td>
|
68
|
+
{%- if host.state in["offline", "offline maintenance"] -%}
|
69
|
+
<span class="label" style="color:white; background-color:black;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
70
|
+
{%- elif host.state == "unknown" -%}
|
71
|
+
<span class="label" style="color:white; background-color:red;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
72
|
+
{%- elif host.last_update.timestamp() <= five_min_ago.timestamp() -%}
|
73
|
+
Reported <span class="label" style="color:white; background-color:red;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
74
|
+
but no heartbeat
|
75
|
+
{%- elif host.state in ["starting", "maintenance request", "maintenance exit"] -%}
|
76
|
+
<span class="label" style="color:black; background-color:gold;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
77
|
+
{%- elif host.state == "running" -%}
|
78
|
+
<span class="label" style="color:white; background-color:green;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
79
|
+
{%- elif host.state == "idle" -%}
|
80
|
+
<span class="label" style="color:black; background-color:gray;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
81
|
+
{%- elif host.state == "terminating" -%}
|
82
|
+
<span class="label" style="color:black; background-color:violet;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
83
|
+
{%- elif host.state in ["maintenance pending", "maintenance mode"] -%}
|
84
|
+
<span class="label" style="color:black; background-color:orange;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
85
|
+
{%- else -%}
|
86
|
+
<span class="label" style="color:white; background-color:hotpink;" title="Current State: {{ host.state }}">{{ host.state }}</span>
|
87
|
+
{%- endif -%}
|
88
|
+
</td>
|
89
|
+
<td>
|
90
|
+
{% if host.queues %}
|
91
|
+
<ul>
|
92
|
+
{% for item in host.queues %}
|
93
|
+
<li>
|
94
|
+
<a href="../taskinstance/list/?_flt_7_state=success&_flt_7_state=failed&_flt_3_queue={{ item }}#"">{{ item }}</a>
|
95
|
+
</li>
|
96
|
+
{% endfor %}
|
97
|
+
</ul>
|
98
|
+
{% else %}
|
99
|
+
(all)
|
100
|
+
{% endif %}
|
101
|
+
</td>
|
102
|
+
<td><time datetime="{{ host.first_online }}">{{ host.first_online }}</time></td>
|
103
|
+
<td>{% if host.last_update %}<time datetime="{{ host.last_update }}">{{ host.last_update }}</time>{% endif %}</td>
|
104
|
+
<td>
|
105
|
+
<a href="../taskinstance/list/?_flt_3_hostname={{ host.worker_name }}&_flt_7_state=success&_flt_7_state=failed#">{{ host.jobs_active }}</a>
|
106
|
+
</td>
|
107
|
+
<!-- Stats are not collected (yet) leave the columns out until available
|
108
|
+
<td>{{ host.jobs_taken }}</td>
|
109
|
+
<td>{{ host.jobs_success }}</td>
|
110
|
+
<td>{{ host.jobs_failed }}</td>
|
111
|
+
-->
|
112
|
+
<td>
|
113
|
+
<ul>
|
114
|
+
{% for item in host.sysinfo_json %}
|
115
|
+
<li>{{ item }}: {{ host.sysinfo_json[item] }}</li>
|
116
|
+
{% endfor %}
|
117
|
+
</ul>
|
118
|
+
</td>
|
119
|
+
{%- if host.state in ["idle", "running"] -%}
|
120
|
+
<td>
|
121
|
+
<button id="button_{{ host.worker_name }}" onclick="showForm('{{ host.worker_name }}')" class="btn btn-sm btn-primary">
|
122
|
+
Enter Maintenance
|
123
|
+
</button>
|
124
|
+
<form id="form_{{ host.worker_name }}" style="display: none" action="../edgeworkerhosts/status/maintenance/{{ host.worker_name }}/on" method="POST">
|
125
|
+
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
126
|
+
<div style="white-space: pre-line;">
|
127
|
+
<label for="maintenance_comment">Maintenance Comment:</label>
|
128
|
+
</div>
|
129
|
+
<textarea name="maintenance_comment" rows="3" maxlength="1024" style="width: 100%; margin-bottom: 5px;" required></textarea>
|
130
|
+
<br />
|
131
|
+
<button type="submit" class="btn btn-sm btn-primary">
|
132
|
+
Confirm Maintenance
|
133
|
+
</button>
|
134
|
+
</td>
|
135
|
+
</form>
|
136
|
+
{%- elif host.state in ["maintenance pending", "maintenance mode", "maintenance request"] -%}
|
137
|
+
<form action="../edgeworkerhosts/status/maintenance/{{ host.worker_name }}/off" method="POST">
|
138
|
+
<td>
|
139
|
+
<div id="display_{{ host.worker_name }}" style="white-space: pre-line;">
|
140
|
+
{{ host.maintenance_comment }}
|
141
|
+
<a onclick="showEditComment('{{ host.worker_name }}')" class="btn btn-sm btn-default" data-toggle="tooltip" rel="tooltip" title="Edit maintenance comment">
|
142
|
+
<span class="sr-only">Edit</span>
|
143
|
+
<i class="fa fa-edit"></i>
|
144
|
+
</a>
|
145
|
+
</div>
|
146
|
+
<textarea id="textarea_{{ host.worker_name }}" name="maintenance_comment" rows="3" maxlength="1024" style="display: none; width:100%;" required>{{ host.maintenance_comment }}</textarea>
|
147
|
+
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
148
|
+
<div style="margin-top: 10px;">
|
149
|
+
<button type="submit" class="btn btn-sm btn-primary">
|
150
|
+
Exit Maintenance
|
151
|
+
</button>
|
152
|
+
<button id="update_{{ host.worker_name }}" type="submit" class="btn btn-sm btn-primary" style="display: none;" formaction="../edgeworkerhosts/status/maintenance/{{ host.worker_name }}/change_comment">
|
153
|
+
Update comment
|
154
|
+
</button>
|
155
|
+
</div>
|
156
|
+
</td>
|
157
|
+
</form>
|
158
|
+
{%- elif host.state in ["offline", "unknown", "offline maintenance"] -%}
|
159
|
+
<form action="../edgeworkerhosts/status/maintenance/{{ host.worker_name }}/remove" method="POST">
|
160
|
+
<td>
|
161
|
+
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
162
|
+
<button type="submit" class="btn btn-sm btn-primary">
|
163
|
+
Remove
|
164
|
+
</button>
|
165
|
+
</td>
|
166
|
+
</form>
|
167
|
+
{%- else -%}
|
168
|
+
<td></td>
|
169
|
+
{% endif %}
|
170
|
+
</tr>
|
171
|
+
{% endfor %}
|
172
|
+
</table>
|
173
|
+
{% endif %}
|
174
|
+
{% endblock %}
|
175
|
+
<html>
|
@@ -0,0 +1,69 @@
|
|
1
|
+
{#
|
2
|
+
Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
or more contributor license agreements. See the NOTICE file
|
4
|
+
distributed with this work for additional information
|
5
|
+
regarding copyright ownership. The ASF licenses this file
|
6
|
+
to you under the Apache License, Version 2.0 (the
|
7
|
+
"License"); you may not use this file except in compliance
|
8
|
+
with the License. You may obtain a copy of the License at
|
9
|
+
|
10
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
|
12
|
+
Unless required by applicable law or agreed to in writing,
|
13
|
+
software distributed under the License is distributed on an
|
14
|
+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
KIND, either express or implied. See the License for the
|
16
|
+
specific language governing permissions and limitations
|
17
|
+
under the License.
|
18
|
+
#}
|
19
|
+
|
20
|
+
{% extends base_template %}
|
21
|
+
|
22
|
+
{% block title %}
|
23
|
+
Edge Worker Jobs
|
24
|
+
{% endblock %}
|
25
|
+
|
26
|
+
{% block content %}
|
27
|
+
<h2>Edge Worker Jobs</h2>
|
28
|
+
{% if jobs|length == 0 %}
|
29
|
+
<p>No jobs running currently</p>
|
30
|
+
{% else %}
|
31
|
+
|
32
|
+
<table class="table table-striped table-bordered">
|
33
|
+
<tr>
|
34
|
+
<th>DAG ID</th>
|
35
|
+
<th>Run ID</th>
|
36
|
+
<th>Task ID</th>
|
37
|
+
<th>Map Index</th>
|
38
|
+
<th>Try Number</th>
|
39
|
+
<th>State</th>
|
40
|
+
<th>Queue</th>
|
41
|
+
<th>Queued DTTM</th>
|
42
|
+
<th>Edge Worker</th>
|
43
|
+
<th>Last Update</th>
|
44
|
+
</tr>
|
45
|
+
|
46
|
+
{% for job in jobs %}
|
47
|
+
<tr title="{{ job.command }}">
|
48
|
+
<td><a href="/dags/{{ job.dag_id }}/grid">{{ job.dag_id }}</a></td>
|
49
|
+
<td><a href="/dags/{{ job.dag_id }}/grid?dag_run_id={{ job.run_id | urlencode }}">{{ job.run_id }}</a></td>
|
50
|
+
<td><a href="/dags/{{ job.dag_id }}/grid?dag_run_id={{ job.run_id | urlencode }}&task_id={{ job.task_id }}&tab=logs">{{ job.task_id }}</a></td>
|
51
|
+
<td>{% if job.map_index >= 0 %}{{ job.map_index }}{% else %}-{% endif %}</td>
|
52
|
+
<td>{{ job.try_number }}</td>
|
53
|
+
<td>{{ html_states[job.state] }}</td>
|
54
|
+
<td>
|
55
|
+
<a href="../taskinstance/list/?_flt_7_state=success&_flt_7_state=failed&_flt_3_queue={{ job.queue }}#"">{{ job.queue }}</a>
|
56
|
+
</td>
|
57
|
+
<td><time datetime="{{ job.queued_dttm }}">{{ job.queued_dttm }}</time></td>
|
58
|
+
<td>
|
59
|
+
{% if job.edge_worker %}
|
60
|
+
<a href="../taskinstance/list/?_flt_3_hostname={{ job.edge_worker }}&_flt_7_state=success&_flt_7_state=failed#">{{ job.edge_worker }}</a>
|
61
|
+
{% endif %}
|
62
|
+
</td>
|
63
|
+
<td>{% if job.last_update %}<time datetime="{{ job.last_update }}">{{ job.last_update | string | truncate(20, False, "") }}</time>{% endif %}</td>
|
64
|
+
</tr>
|
65
|
+
{% endfor %}
|
66
|
+
</table>
|
67
|
+
{% endif %}
|
68
|
+
{% endblock %}
|
69
|
+
<html>
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
#
|
18
|
+
# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY
|
19
|
+
# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS
|
20
|
+
# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
|
21
|
+
# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
|
22
|
+
#
|
23
|
+
from __future__ import annotations
|
24
|
+
|
25
|
+
|
26
|
+
def get_base_airflow_version_tuple() -> tuple[int, int, int]:
|
27
|
+
from packaging.version import Version
|
28
|
+
|
29
|
+
from airflow import __version__
|
30
|
+
|
31
|
+
airflow_version = Version(__version__)
|
32
|
+
return airflow_version.major, airflow_version.minor, airflow_version.micro
|
33
|
+
|
34
|
+
|
35
|
+
AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
|
36
|
+
AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
"""FastAPI implementation modules for Airflow 3.x."""
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
from __future__ import annotations
|
18
|
+
|
19
|
+
from fastapi import FastAPI
|
20
|
+
|
21
|
+
from airflow.providers.edge3.worker_api.routes.health import health_router
|
22
|
+
from airflow.providers.edge3.worker_api.routes.jobs import jobs_router
|
23
|
+
from airflow.providers.edge3.worker_api.routes.logs import logs_router
|
24
|
+
from airflow.providers.edge3.worker_api.routes.worker import worker_router
|
25
|
+
|
26
|
+
|
27
|
+
def create_edge_worker_api_app() -> FastAPI:
|
28
|
+
"""Create FastAPI app for edge worker API."""
|
29
|
+
edge_worker_api_app = FastAPI(
|
30
|
+
title="Airflow Edge Worker API",
|
31
|
+
description=(
|
32
|
+
"This is Airflow Edge Worker API - which is a the access endpoint for workers running on remote "
|
33
|
+
"sites serving for Apache Airflow jobs. It also proxies internal API to edge endpoints. It is "
|
34
|
+
"not intended to be used by any external code. You can find more information in AIP-69 "
|
35
|
+
"https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=301795932"
|
36
|
+
),
|
37
|
+
)
|
38
|
+
|
39
|
+
edge_worker_api_app.include_router(health_router)
|
40
|
+
edge_worker_api_app.include_router(jobs_router)
|
41
|
+
edge_worker_api_app.include_router(logs_router)
|
42
|
+
edge_worker_api_app.include_router(worker_router)
|
43
|
+
return edge_worker_api_app
|