apache-airflow-providers-edge3 1.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. airflow/providers/edge3/LICENSE +201 -0
  2. airflow/providers/edge3/__init__.py +39 -0
  3. airflow/providers/edge3/cli/__init__.py +16 -0
  4. airflow/providers/edge3/cli/api_client.py +206 -0
  5. airflow/providers/edge3/cli/dataclasses.py +95 -0
  6. airflow/providers/edge3/cli/edge_command.py +689 -0
  7. airflow/providers/edge3/example_dags/__init__.py +16 -0
  8. airflow/providers/edge3/example_dags/integration_test.py +164 -0
  9. airflow/providers/edge3/example_dags/win_notepad.py +83 -0
  10. airflow/providers/edge3/example_dags/win_test.py +342 -0
  11. airflow/providers/edge3/executors/__init__.py +22 -0
  12. airflow/providers/edge3/executors/edge_executor.py +367 -0
  13. airflow/providers/edge3/get_provider_info.py +99 -0
  14. airflow/providers/edge3/models/__init__.py +16 -0
  15. airflow/providers/edge3/models/edge_job.py +94 -0
  16. airflow/providers/edge3/models/edge_logs.py +73 -0
  17. airflow/providers/edge3/models/edge_worker.py +230 -0
  18. airflow/providers/edge3/openapi/__init__.py +19 -0
  19. airflow/providers/edge3/openapi/edge_worker_api_v1.yaml +808 -0
  20. airflow/providers/edge3/plugins/__init__.py +16 -0
  21. airflow/providers/edge3/plugins/edge_executor_plugin.py +229 -0
  22. airflow/providers/edge3/plugins/templates/edge_worker_hosts.html +175 -0
  23. airflow/providers/edge3/plugins/templates/edge_worker_jobs.html +69 -0
  24. airflow/providers/edge3/version_compat.py +36 -0
  25. airflow/providers/edge3/worker_api/__init__.py +17 -0
  26. airflow/providers/edge3/worker_api/app.py +43 -0
  27. airflow/providers/edge3/worker_api/auth.py +135 -0
  28. airflow/providers/edge3/worker_api/datamodels.py +190 -0
  29. airflow/providers/edge3/worker_api/routes/__init__.py +16 -0
  30. airflow/providers/edge3/worker_api/routes/_v2_compat.py +135 -0
  31. airflow/providers/edge3/worker_api/routes/_v2_routes.py +237 -0
  32. airflow/providers/edge3/worker_api/routes/health.py +28 -0
  33. airflow/providers/edge3/worker_api/routes/jobs.py +162 -0
  34. airflow/providers/edge3/worker_api/routes/logs.py +133 -0
  35. airflow/providers/edge3/worker_api/routes/worker.py +224 -0
  36. apache_airflow_providers_edge3-1.0.0rc1.dist-info/METADATA +117 -0
  37. apache_airflow_providers_edge3-1.0.0rc1.dist-info/RECORD +39 -0
  38. apache_airflow_providers_edge3-1.0.0rc1.dist-info/WHEEL +4 -0
  39. apache_airflow_providers_edge3-1.0.0rc1.dist-info/entry_points.txt +6 -0
@@ -0,0 +1,16 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
@@ -0,0 +1,229 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ from __future__ import annotations
19
+
20
+ import re
21
+ from datetime import datetime, timedelta
22
+ from pathlib import Path
23
+ from typing import TYPE_CHECKING, Any
24
+
25
+ from flask import Blueprint, redirect, request, url_for
26
+ from flask_appbuilder import BaseView, expose
27
+ from markupsafe import Markup
28
+ from sqlalchemy import select
29
+
30
+ from airflow.configuration import conf
31
+ from airflow.exceptions import AirflowConfigException
32
+ from airflow.models.taskinstance import TaskInstanceState
33
+ from airflow.plugins_manager import AirflowPlugin
34
+ from airflow.providers.edge3.version_compat import AIRFLOW_V_3_0_PLUS
35
+ from airflow.utils.state import State
36
+
37
+ if AIRFLOW_V_3_0_PLUS:
38
+ from airflow.api_fastapi.auth.managers.models.resource_details import AccessView
39
+ from airflow.providers.fab.www.auth import has_access_view
40
+
41
+ else:
42
+ from airflow.auth.managers.models.resource_details import AccessView # type: ignore[no-redef]
43
+ from airflow.www.auth import has_access_view # type: ignore[no-redef]
44
+ from airflow.utils.session import NEW_SESSION, provide_session
45
+ from airflow.utils.yaml import safe_load
46
+
47
+ if TYPE_CHECKING:
48
+ from sqlalchemy.orm import Session
49
+
50
+
51
+ def _get_airflow_2_api_endpoint() -> Blueprint:
52
+ from airflow.www.constants import SWAGGER_BUNDLE, SWAGGER_ENABLED
53
+ from airflow.www.extensions.init_views import _CustomErrorRequestBodyValidator, _LazyResolver
54
+
55
+ folder = Path(__file__).parents[1].resolve() # this is airflow/providers/edge3/
56
+ with folder.joinpath("openapi", "edge_worker_api_v1.yaml").open() as f:
57
+ specification = safe_load(f)
58
+ from connexion import FlaskApi
59
+
60
+ bp = FlaskApi(
61
+ specification=specification,
62
+ resolver=_LazyResolver(),
63
+ base_path="/edge_worker/v1",
64
+ strict_validation=True,
65
+ options={"swagger_ui": SWAGGER_ENABLED, "swagger_path": SWAGGER_BUNDLE.__fspath__()},
66
+ validate_responses=True,
67
+ validator_map={"body": _CustomErrorRequestBodyValidator},
68
+ ).blueprint
69
+ # Need to exempt CSRF to make API usable
70
+ from airflow.www.app import csrf
71
+
72
+ csrf.exempt(bp)
73
+ return bp
74
+
75
+
76
+ def _get_api_endpoint() -> dict[str, Any]:
77
+ from airflow.providers.edge3.worker_api.app import create_edge_worker_api_app
78
+
79
+ return {
80
+ "app": create_edge_worker_api_app(),
81
+ "url_prefix": "/edge_worker/v1",
82
+ "name": "Airflow Edge Worker API",
83
+ }
84
+
85
+
86
+ def _state_token(state):
87
+ """Return a formatted string with HTML for a given State."""
88
+ color = State.color(state)
89
+ fg_color = State.color_fg(state)
90
+ return Markup(
91
+ """
92
+ <span class="label" style="color:{fg_color}; background-color:{color};"
93
+ title="Current State: {state}">{state}</span>
94
+ """
95
+ ).format(color=color, state=state, fg_color=fg_color)
96
+
97
+
98
+ def modify_maintenance_comment_on_update(maintenance_comment: str | None, username: str) -> str:
99
+ if maintenance_comment:
100
+ if re.search(
101
+ r"^\[[-\d:\s]+\] - .+ put node into maintenance mode\r?\nComment:.*", maintenance_comment
102
+ ):
103
+ return re.sub(
104
+ r"^\[[-\d:\s]+\] - .+ put node into maintenance mode\r?\nComment:",
105
+ f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {username} updated maintenance mode\nComment:",
106
+ maintenance_comment,
107
+ )
108
+ if re.search(r"^\[[-\d:\s]+\] - .+ updated maintenance mode\r?\nComment:.*", maintenance_comment):
109
+ return re.sub(
110
+ r"^\[[-\d:\s]+\] - .+ updated maintenance mode\r?\nComment:",
111
+ f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {username} updated maintenance mode\nComment:",
112
+ maintenance_comment,
113
+ )
114
+ return f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {username} updated maintenance mode\nComment: {maintenance_comment}"
115
+ return f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {username} updated maintenance mode\nComment:"
116
+
117
+
118
+ # registers airflow/providers/edge3/plugins/templates as a Jinja template folder
119
+ template_bp = Blueprint(
120
+ "template_blueprint",
121
+ __name__,
122
+ template_folder="templates",
123
+ )
124
+
125
+
126
+ class EdgeWorkerJobs(BaseView):
127
+ """Simple view to show Edge Worker jobs."""
128
+
129
+ default_view = "jobs"
130
+
131
+ @expose("/jobs")
132
+ @has_access_view(AccessView.JOBS)
133
+ @provide_session
134
+ def jobs(self, session: Session = NEW_SESSION):
135
+ from airflow.providers.edge3.models.edge_job import EdgeJobModel
136
+
137
+ jobs = session.scalars(select(EdgeJobModel).order_by(EdgeJobModel.queued_dttm)).all()
138
+ html_states = {
139
+ str(state): _state_token(str(state)) for state in TaskInstanceState.__members__.values()
140
+ }
141
+ return self.render_template("edge_worker_jobs.html", jobs=jobs, html_states=html_states)
142
+
143
+
144
+ class EdgeWorkerHosts(BaseView):
145
+ """Simple view to show Edge Worker status."""
146
+
147
+ default_view = "status"
148
+
149
+ @expose("/status")
150
+ @has_access_view(AccessView.JOBS)
151
+ @provide_session
152
+ def status(self, session: Session = NEW_SESSION):
153
+ from airflow.providers.edge3.models.edge_worker import EdgeWorkerModel
154
+
155
+ hosts = session.scalars(select(EdgeWorkerModel).order_by(EdgeWorkerModel.worker_name)).all()
156
+ five_min_ago = datetime.now() - timedelta(minutes=5)
157
+ return self.render_template("edge_worker_hosts.html", hosts=hosts, five_min_ago=five_min_ago)
158
+
159
+ @expose("/status/maintenance/<string:worker_name>/on", methods=["POST"])
160
+ @has_access_view(AccessView.JOBS)
161
+ def worker_to_maintenance(self, worker_name: str):
162
+ from flask_login import current_user
163
+
164
+ from airflow.providers.edge3.models.edge_worker import request_maintenance
165
+
166
+ maintenance_comment = request.form.get("maintenance_comment")
167
+ maintenance_comment = f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}] - {current_user.username} put node into maintenance mode\nComment: {maintenance_comment}"
168
+ request_maintenance(worker_name, maintenance_comment)
169
+ return redirect(url_for("EdgeWorkerHosts.status"))
170
+
171
+ @expose("/status/maintenance/<string:worker_name>/off", methods=["POST"])
172
+ @has_access_view(AccessView.JOBS)
173
+ def remove_worker_from_maintenance(self, worker_name: str):
174
+ from airflow.providers.edge3.models.edge_worker import exit_maintenance
175
+
176
+ exit_maintenance(worker_name)
177
+ return redirect(url_for("EdgeWorkerHosts.status"))
178
+
179
+ @expose("/status/maintenance/<string:worker_name>/remove", methods=["POST"])
180
+ @has_access_view(AccessView.JOBS)
181
+ def remove_worker(self, worker_name: str):
182
+ from airflow.providers.edge3.models.edge_worker import remove_worker
183
+
184
+ remove_worker(worker_name)
185
+ return redirect(url_for("EdgeWorkerHosts.status"))
186
+
187
+ @expose("/status/maintenance/<string:worker_name>/change_comment", methods=["POST"])
188
+ @has_access_view(AccessView.JOBS)
189
+ def change_maintenance_comment(self, worker_name: str):
190
+ from flask_login import current_user
191
+
192
+ from airflow.providers.edge3.models.edge_worker import change_maintenance_comment
193
+
194
+ maintenance_comment = request.form.get("maintenance_comment")
195
+ maintenance_comment = modify_maintenance_comment_on_update(maintenance_comment, current_user.username)
196
+ change_maintenance_comment(worker_name, maintenance_comment)
197
+ return redirect(url_for("EdgeWorkerHosts.status"))
198
+
199
+
200
+ # Check if EdgeExecutor is actually loaded
201
+ try:
202
+ EDGE_EXECUTOR_ACTIVE = conf.getboolean("edge", "api_enabled", fallback="False")
203
+ except AirflowConfigException:
204
+ EDGE_EXECUTOR_ACTIVE = False
205
+
206
+
207
+ class EdgeExecutorPlugin(AirflowPlugin):
208
+ """EdgeExecutor Plugin - provides API endpoints for Edge Workers in Webserver."""
209
+
210
+ name = "edge_executor"
211
+ if EDGE_EXECUTOR_ACTIVE:
212
+ appbuilder_views = [
213
+ {
214
+ "name": "Edge Worker Jobs",
215
+ "category": "Admin",
216
+ "view": EdgeWorkerJobs(),
217
+ },
218
+ {
219
+ "name": "Edge Worker Hosts",
220
+ "category": "Admin",
221
+ "view": EdgeWorkerHosts(),
222
+ },
223
+ ]
224
+
225
+ if AIRFLOW_V_3_0_PLUS:
226
+ fastapi_apps = [_get_api_endpoint()]
227
+ flask_blueprints = [template_bp]
228
+ else:
229
+ flask_blueprints = [_get_airflow_2_api_endpoint(), template_bp]
@@ -0,0 +1,175 @@
1
+ {#
2
+ Licensed to the Apache Software Foundation (ASF) under one
3
+ or more contributor license agreements. See the NOTICE file
4
+ distributed with this work for additional information
5
+ regarding copyright ownership. The ASF licenses this file
6
+ to you under the Apache License, Version 2.0 (the
7
+ "License"); you may not use this file except in compliance
8
+ with the License. You may obtain a copy of the License at
9
+
10
+ http://www.apache.org/licenses/LICENSE-2.0
11
+
12
+ Unless required by applicable law or agreed to in writing,
13
+ software distributed under the License is distributed on an
14
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ KIND, either express or implied. See the License for the
16
+ specific language governing permissions and limitations
17
+ under the License.
18
+ #}
19
+
20
+ {% extends base_template %}
21
+
22
+ {% block title %}
23
+ Edge Worker Hosts
24
+ {% endblock %}
25
+
26
+ {% block content %}
27
+ <h2>Edge Worker Hosts</h2>
28
+ {% if hosts|length == 0 %}
29
+ <p>No Edge Workers connected or known currently.</p>
30
+ {% else %}
31
+
32
+ <script>
33
+ function showForm(worker_name) {
34
+ var button = document.getElementById("button_" + worker_name);
35
+ var form = document.getElementById("form_" + worker_name);
36
+ form.style.display = "block";
37
+ button.style.display = "none";
38
+ }
39
+ function showEditComment(worker_name) {
40
+ var display = document.getElementById("display_" + worker_name);
41
+ var textarea = document.getElementById("textarea_" + worker_name);
42
+ var button = document.getElementById("update_" + worker_name);
43
+ display.style.display = "none";
44
+ textarea.style.display = "block";
45
+ button.style.display = "inline";
46
+ }
47
+ </script>
48
+ <table class="table table-striped table-bordered">
49
+ <tr>
50
+ <th>Hostname</th>
51
+ <th>State</th>
52
+ <th>Queues</th>
53
+ <th>First Online</th>
54
+ <th>Last Heart Beat</th>
55
+ <th>Active Jobs</th>
56
+ <!-- Stats are not collected (yet) leave the columns out until available
57
+ <th>Jobs Taken</th>
58
+ <th>Jobs Success</th>
59
+ <th>Jobs Failed</th>
60
+ -->
61
+ <th>System Information</th>
62
+ <th>Operations</th>
63
+ </tr>
64
+ {% for host in hosts %}
65
+ <tr>
66
+ <td>{{ host.worker_name }}</a></td>
67
+ <td>
68
+ {%- if host.state in["offline", "offline maintenance"] -%}
69
+ <span class="label" style="color:white; background-color:black;" title="Current State: {{ host.state }}">{{ host.state }}</span>
70
+ {%- elif host.state == "unknown" -%}
71
+ <span class="label" style="color:white; background-color:red;" title="Current State: {{ host.state }}">{{ host.state }}</span>
72
+ {%- elif host.last_update.timestamp() <= five_min_ago.timestamp() -%}
73
+ Reported <span class="label" style="color:white; background-color:red;" title="Current State: {{ host.state }}">{{ host.state }}</span>
74
+ but no heartbeat
75
+ {%- elif host.state in ["starting", "maintenance request", "maintenance exit"] -%}
76
+ <span class="label" style="color:black; background-color:gold;" title="Current State: {{ host.state }}">{{ host.state }}</span>
77
+ {%- elif host.state == "running" -%}
78
+ <span class="label" style="color:white; background-color:green;" title="Current State: {{ host.state }}">{{ host.state }}</span>
79
+ {%- elif host.state == "idle" -%}
80
+ <span class="label" style="color:black; background-color:gray;" title="Current State: {{ host.state }}">{{ host.state }}</span>
81
+ {%- elif host.state == "terminating" -%}
82
+ <span class="label" style="color:black; background-color:violet;" title="Current State: {{ host.state }}">{{ host.state }}</span>
83
+ {%- elif host.state in ["maintenance pending", "maintenance mode"] -%}
84
+ <span class="label" style="color:black; background-color:orange;" title="Current State: {{ host.state }}">{{ host.state }}</span>
85
+ {%- else -%}
86
+ <span class="label" style="color:white; background-color:hotpink;" title="Current State: {{ host.state }}">{{ host.state }}</span>
87
+ {%- endif -%}
88
+ </td>
89
+ <td>
90
+ {% if host.queues %}
91
+ <ul>
92
+ {% for item in host.queues %}
93
+ <li>
94
+ <a href="../taskinstance/list/?_flt_7_state=success&_flt_7_state=failed&_flt_3_queue={{ item }}#"">{{ item }}</a>
95
+ </li>
96
+ {% endfor %}
97
+ </ul>
98
+ {% else %}
99
+ (all)
100
+ {% endif %}
101
+ </td>
102
+ <td><time datetime="{{ host.first_online }}">{{ host.first_online }}</time></td>
103
+ <td>{% if host.last_update %}<time datetime="{{ host.last_update }}">{{ host.last_update }}</time>{% endif %}</td>
104
+ <td>
105
+ <a href="../taskinstance/list/?_flt_3_hostname={{ host.worker_name }}&_flt_7_state=success&_flt_7_state=failed#">{{ host.jobs_active }}</a>
106
+ </td>
107
+ <!-- Stats are not collected (yet) leave the columns out until available
108
+ <td>{{ host.jobs_taken }}</td>
109
+ <td>{{ host.jobs_success }}</td>
110
+ <td>{{ host.jobs_failed }}</td>
111
+ -->
112
+ <td>
113
+ <ul>
114
+ {% for item in host.sysinfo_json %}
115
+ <li>{{ item }}: {{ host.sysinfo_json[item] }}</li>
116
+ {% endfor %}
117
+ </ul>
118
+ </td>
119
+ {%- if host.state in ["idle", "running"] -%}
120
+ <td>
121
+ <button id="button_{{ host.worker_name }}" onclick="showForm('{{ host.worker_name }}')" class="btn btn-sm btn-primary">
122
+ Enter Maintenance
123
+ </button>
124
+ <form id="form_{{ host.worker_name }}" style="display: none" action="../edgeworkerhosts/status/maintenance/{{ host.worker_name }}/on" method="POST">
125
+ <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
126
+ <div style="white-space: pre-line;">
127
+ <label for="maintenance_comment">Maintenance Comment:</label>
128
+ </div>
129
+ <textarea name="maintenance_comment" rows="3" maxlength="1024" style="width: 100%; margin-bottom: 5px;" required></textarea>
130
+ <br />
131
+ <button type="submit" class="btn btn-sm btn-primary">
132
+ Confirm Maintenance
133
+ </button>
134
+ </td>
135
+ </form>
136
+ {%- elif host.state in ["maintenance pending", "maintenance mode", "maintenance request"] -%}
137
+ <form action="../edgeworkerhosts/status/maintenance/{{ host.worker_name }}/off" method="POST">
138
+ <td>
139
+ <div id="display_{{ host.worker_name }}" style="white-space: pre-line;">
140
+ {{ host.maintenance_comment }}
141
+ <a onclick="showEditComment('{{ host.worker_name }}')" class="btn btn-sm btn-default" data-toggle="tooltip" rel="tooltip" title="Edit maintenance comment">
142
+ <span class="sr-only">Edit</span>
143
+ <i class="fa fa-edit"></i>
144
+ </a>
145
+ </div>
146
+ <textarea id="textarea_{{ host.worker_name }}" name="maintenance_comment" rows="3" maxlength="1024" style="display: none; width:100%;" required>{{ host.maintenance_comment }}</textarea>
147
+ <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
148
+ <div style="margin-top: 10px;">
149
+ <button type="submit" class="btn btn-sm btn-primary">
150
+ Exit Maintenance
151
+ </button>
152
+ <button id="update_{{ host.worker_name }}" type="submit" class="btn btn-sm btn-primary" style="display: none;" formaction="../edgeworkerhosts/status/maintenance/{{ host.worker_name }}/change_comment">
153
+ Update comment
154
+ </button>
155
+ </div>
156
+ </td>
157
+ </form>
158
+ {%- elif host.state in ["offline", "unknown", "offline maintenance"] -%}
159
+ <form action="../edgeworkerhosts/status/maintenance/{{ host.worker_name }}/remove" method="POST">
160
+ <td>
161
+ <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
162
+ <button type="submit" class="btn btn-sm btn-primary">
163
+ Remove
164
+ </button>
165
+ </td>
166
+ </form>
167
+ {%- else -%}
168
+ <td></td>
169
+ {% endif %}
170
+ </tr>
171
+ {% endfor %}
172
+ </table>
173
+ {% endif %}
174
+ {% endblock %}
175
+ <html>
@@ -0,0 +1,69 @@
1
+ {#
2
+ Licensed to the Apache Software Foundation (ASF) under one
3
+ or more contributor license agreements. See the NOTICE file
4
+ distributed with this work for additional information
5
+ regarding copyright ownership. The ASF licenses this file
6
+ to you under the Apache License, Version 2.0 (the
7
+ "License"); you may not use this file except in compliance
8
+ with the License. You may obtain a copy of the License at
9
+
10
+ http://www.apache.org/licenses/LICENSE-2.0
11
+
12
+ Unless required by applicable law or agreed to in writing,
13
+ software distributed under the License is distributed on an
14
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ KIND, either express or implied. See the License for the
16
+ specific language governing permissions and limitations
17
+ under the License.
18
+ #}
19
+
20
+ {% extends base_template %}
21
+
22
+ {% block title %}
23
+ Edge Worker Jobs
24
+ {% endblock %}
25
+
26
+ {% block content %}
27
+ <h2>Edge Worker Jobs</h2>
28
+ {% if jobs|length == 0 %}
29
+ <p>No jobs running currently</p>
30
+ {% else %}
31
+
32
+ <table class="table table-striped table-bordered">
33
+ <tr>
34
+ <th>DAG ID</th>
35
+ <th>Run ID</th>
36
+ <th>Task ID</th>
37
+ <th>Map Index</th>
38
+ <th>Try Number</th>
39
+ <th>State</th>
40
+ <th>Queue</th>
41
+ <th>Queued DTTM</th>
42
+ <th>Edge Worker</th>
43
+ <th>Last Update</th>
44
+ </tr>
45
+
46
+ {% for job in jobs %}
47
+ <tr title="{{ job.command }}">
48
+ <td><a href="/dags/{{ job.dag_id }}/grid">{{ job.dag_id }}</a></td>
49
+ <td><a href="/dags/{{ job.dag_id }}/grid?dag_run_id={{ job.run_id | urlencode }}">{{ job.run_id }}</a></td>
50
+ <td><a href="/dags/{{ job.dag_id }}/grid?dag_run_id={{ job.run_id | urlencode }}&task_id={{ job.task_id }}&tab=logs">{{ job.task_id }}</a></td>
51
+ <td>{% if job.map_index >= 0 %}{{ job.map_index }}{% else %}-{% endif %}</td>
52
+ <td>{{ job.try_number }}</td>
53
+ <td>{{ html_states[job.state] }}</td>
54
+ <td>
55
+ <a href="../taskinstance/list/?_flt_7_state=success&_flt_7_state=failed&_flt_3_queue={{ job.queue }}#"">{{ job.queue }}</a>
56
+ </td>
57
+ <td><time datetime="{{ job.queued_dttm }}">{{ job.queued_dttm }}</time></td>
58
+ <td>
59
+ {% if job.edge_worker %}
60
+ <a href="../taskinstance/list/?_flt_3_hostname={{ job.edge_worker }}&_flt_7_state=success&_flt_7_state=failed#">{{ job.edge_worker }}</a>
61
+ {% endif %}
62
+ </td>
63
+ <td>{% if job.last_update %}<time datetime="{{ job.last_update }}">{{ job.last_update | string | truncate(20, False, "") }}</time>{% endif %}</td>
64
+ </tr>
65
+ {% endfor %}
66
+ </table>
67
+ {% endif %}
68
+ {% endblock %}
69
+ <html>
@@ -0,0 +1,36 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ #
18
+ # NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY
19
+ # DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS
20
+ # ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
21
+ # THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
22
+ #
23
+ from __future__ import annotations
24
+
25
+
26
+ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
27
+ from packaging.version import Version
28
+
29
+ from airflow import __version__
30
+
31
+ airflow_version = Version(__version__)
32
+ return airflow_version.major, airflow_version.minor, airflow_version.micro
33
+
34
+
35
+ AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
36
+ AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
@@ -0,0 +1,17 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ """FastAPI implementation modules for Airflow 3.x."""
@@ -0,0 +1,43 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from fastapi import FastAPI
20
+
21
+ from airflow.providers.edge3.worker_api.routes.health import health_router
22
+ from airflow.providers.edge3.worker_api.routes.jobs import jobs_router
23
+ from airflow.providers.edge3.worker_api.routes.logs import logs_router
24
+ from airflow.providers.edge3.worker_api.routes.worker import worker_router
25
+
26
+
27
+ def create_edge_worker_api_app() -> FastAPI:
28
+ """Create FastAPI app for edge worker API."""
29
+ edge_worker_api_app = FastAPI(
30
+ title="Airflow Edge Worker API",
31
+ description=(
32
+ "This is Airflow Edge Worker API - which is a the access endpoint for workers running on remote "
33
+ "sites serving for Apache Airflow jobs. It also proxies internal API to edge endpoints. It is "
34
+ "not intended to be used by any external code. You can find more information in AIP-69 "
35
+ "https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=301795932"
36
+ ),
37
+ )
38
+
39
+ edge_worker_api_app.include_router(health_router)
40
+ edge_worker_api_app.include_router(jobs_router)
41
+ edge_worker_api_app.include_router(logs_router)
42
+ edge_worker_api_app.include_router(worker_router)
43
+ return edge_worker_api_app