apache-airflow-providers-edge3 1.1.2rc1__tar.gz → 1.1.2rc2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/PKG-INFO +5 -3
  2. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/README.rst +2 -1
  3. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/architecture.rst +1 -1
  4. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/changelog.rst +18 -0
  5. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/deployment.rst +2 -0
  6. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/index.rst +2 -0
  7. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/pyproject.toml +3 -2
  8. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/__init__.py +1 -1
  9. {apache_airflow_providers_edge3-1.1.2rc1/tests/unit → apache_airflow_providers_edge3-1.1.2rc2/src/airflow/providers}/__init__.py +1 -1
  10. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/cli/edge_command.py +65 -0
  11. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/cli/worker.py +2 -2
  12. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/example_dags/integration_test.py +7 -7
  13. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/example_dags/win_test.py +8 -4
  14. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/executors/edge_executor.py +1 -1
  15. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/models/edge_worker.py +35 -1
  16. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/plugins/edge_executor_plugin.py +14 -1
  17. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/auth.py +1 -1
  18. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/routes/_v2_compat.py +1 -1
  19. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/routes/_v2_routes.py +1 -1
  20. {apache_airflow_providers_edge3-1.1.2rc1/src/airflow/providers → apache_airflow_providers_edge3-1.1.2rc2/tests/unit}/__init__.py +1 -1
  21. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/cli/test_worker.py +1 -0
  22. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/executors/test_edge_executor.py +3 -3
  23. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/plugins/test_edge_executor_plugin.py +27 -2
  24. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/cli-ref.rst +0 -0
  25. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/commits.rst +0 -0
  26. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/conf.py +0 -0
  27. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/configurations-ref.rst +0 -0
  28. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/edge_executor.rst +0 -0
  29. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/img/distributed_architecture.svg +0 -0
  30. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/img/edge_package.svg +0 -0
  31. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/img/worker_hosts.png +0 -0
  32. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/img/worker_maintenance.png +0 -0
  33. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/install_on_windows.rst +0 -0
  34. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/installing-providers-from-sources.rst +0 -0
  35. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/security.rst +0 -0
  36. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/ui_plugin.rst +0 -0
  37. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/docs/why_edge.rst +0 -0
  38. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/provider.yaml +0 -0
  39. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/LICENSE +0 -0
  40. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/__init__.py +0 -0
  41. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/cli/__init__.py +0 -0
  42. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/cli/api_client.py +0 -0
  43. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/cli/dataclasses.py +0 -0
  44. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/cli/signalling.py +0 -0
  45. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/example_dags/__init__.py +0 -0
  46. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/example_dags/win_notepad.py +0 -0
  47. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/executors/__init__.py +0 -0
  48. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/get_provider_info.py +0 -0
  49. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/models/__init__.py +0 -0
  50. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/models/edge_job.py +0 -0
  51. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/models/edge_logs.py +0 -0
  52. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/openapi/__init__.py +0 -0
  53. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/openapi/edge_worker_api_v1.yaml +0 -0
  54. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/plugins/__init__.py +0 -0
  55. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/plugins/templates/edge_worker_hosts.html +0 -0
  56. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/plugins/templates/edge_worker_jobs.html +0 -0
  57. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/version_compat.py +0 -0
  58. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/__init__.py +0 -0
  59. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/app.py +0 -0
  60. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/datamodels.py +0 -0
  61. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/routes/__init__.py +0 -0
  62. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/routes/health.py +0 -0
  63. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/routes/jobs.py +0 -0
  64. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/routes/logs.py +0 -0
  65. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/src/airflow/providers/edge3/worker_api/routes/worker.py +0 -0
  66. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/conftest.py +0 -0
  67. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/__init__.py +0 -0
  68. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/cli/__init__.py +0 -0
  69. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/cli/test_api_client.py +0 -0
  70. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/cli/test_dataclasses.py +0 -0
  71. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/cli/test_edge_command.py +0 -0
  72. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/cli/test_signalling.py +0 -0
  73. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/executors/__init__.py +0 -0
  74. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/models/__init__.py +0 -0
  75. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/plugins/__init__.py +0 -0
  76. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/worker_api/__init__.py +0 -0
  77. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/worker_api/routes/__init__.py +0 -0
  78. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/worker_api/routes/test_health.py +0 -0
  79. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/worker_api/routes/test_jobs.py +0 -0
  80. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/worker_api/routes/test_logs.py +0 -0
  81. {apache_airflow_providers_edge3-1.1.2rc1 → apache_airflow_providers_edge3-1.1.2rc2}/tests/unit/edge3/worker_api/routes/test_worker.py +0 -0
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: apache-airflow-providers-edge3
3
- Version: 1.1.2rc1
3
+ Version: 1.1.2rc2
4
4
  Summary: Provider package apache-airflow-providers-edge3 for Apache Airflow
5
5
  Keywords: airflow-provider,edge3,airflow,integration
6
6
  Author-email: Apache Software Foundation <dev@airflow.apache.org>
7
7
  Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
8
- Requires-Python: ~=3.10
8
+ Requires-Python: >=3.10
9
9
  Description-Content-Type: text/x-rst
10
10
  Classifier: Development Status :: 5 - Production/Stable
11
11
  Classifier: Environment :: Console
@@ -18,6 +18,7 @@ Classifier: License :: OSI Approved :: Apache Software License
18
18
  Classifier: Programming Language :: Python :: 3.10
19
19
  Classifier: Programming Language :: Python :: 3.11
20
20
  Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
21
22
  Classifier: Topic :: System :: Monitoring
22
23
  Requires-Dist: apache-airflow>=2.10.0rc1
23
24
  Requires-Dist: pydantic>=2.11.0
@@ -57,6 +58,7 @@ Package ``apache-airflow-providers-edge3``
57
58
 
58
59
  Release: ``1.1.2``
59
60
 
61
+ Release Date: ``|PypiReleaseDate|``
60
62
 
61
63
  Handle edge workers on remote sites via HTTP(s) connection and orchestrates work over distributed sites.
62
64
 
@@ -89,7 +91,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
89
91
  for the minimum Airflow version supported) via
90
92
  ``pip install apache-airflow-providers-edge3``
91
93
 
92
- The package supports the following python versions: 3.10,3.11,3.12
94
+ The package supports the following python versions: 3.10,3.11,3.12,3.13
93
95
 
94
96
  Requirements
95
97
  ------------
@@ -25,6 +25,7 @@ Package ``apache-airflow-providers-edge3``
25
25
 
26
26
  Release: ``1.1.2``
27
27
 
28
+ Release Date: ``|PypiReleaseDate|``
28
29
 
29
30
  Handle edge workers on remote sites via HTTP(s) connection and orchestrates work over distributed sites.
30
31
 
@@ -57,7 +58,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
57
58
  for the minimum Airflow version supported) via
58
59
  ``pip install apache-airflow-providers-edge3``
59
60
 
60
- The package supports the following python versions: 3.10,3.11,3.12
61
+ The package supports the following python versions: 3.10,3.11,3.12,3.13
61
62
 
62
63
  Requirements
63
64
  ------------
@@ -171,7 +171,7 @@ The following features are known missing and will be implemented in increments:
171
171
 
172
172
  - Tests
173
173
 
174
- - System tests in Github, test the deployment of the worker with a Dag execution
174
+ - System tests in GitHub, test the deployment of the worker with a Dag execution
175
175
  - Test/Support on Windows for Edge Worker
176
176
 
177
177
  - Scaling test - Check and define boundaries of workers/jobs. Today it is known to
@@ -34,10 +34,23 @@ Bug Fixes
34
34
  ~~~~~~~~~
35
35
 
36
36
  * ``Fix UnboundLocalError for 'edge_job_command_len' (#52328)``
37
+ * ``Extend run detection to dev-mode to load plugin (#53576)``
38
+ * ``Add queue and remove queue cli commands for EdgeExecutor (#53505)``
39
+ * ``Ensure Edge Plugin for API endpoint is only loaded on API-Server and AF2 Webserver (#52952)``
40
+ * ``Fix unreachable code mypy warnings in edge3 provider (#53430)``
41
+ * ``Make edge3 provider compatible with mypy 1.16.1 (#53104)``
42
+ * ``Fix task configuration defaults for AbstractOperator (#52871)``
37
43
 
38
44
  Misc
39
45
  ~~~~
40
46
 
47
+ * ``Remove upper-binding for "python-requires" (#52980)``
48
+ * ``Temporarily switch to use >=,< pattern instead of '~=' (#52967)``
49
+ * ``Deprecate decorators from Core (#53629)``
50
+ * ``Add Python 3.13 support for Airflow. (#46891)``
51
+ * ``Cleanup type ignores in edge3 provider where possible (#53248)``
52
+ * ``Remove type ignore across codebase after mypy upgrade (#53243)``
53
+ * ``Remove deprecation in Edge for DEFAULT_QUEUE (#52954)``
41
54
  * ``Move 'BaseHook' implementation to task SDK (#51873)``
42
55
  * ``Force the definition of 'execution_api_server_url' based on 'api_url' (#52184)``
43
56
  * ``Drop support for Python 3.9 (#52072)``
@@ -58,6 +71,11 @@ Doc-only
58
71
  appropriate section above if needed. Do not delete the lines(!):
59
72
  * ``Revert "Revert "Remove FAB dependency from Edge3 Provider (#51995)"" (#52000)``
60
73
  * ``Revert "Remove FAB dependency from Edge3 Provider (#51995)" (#51998)``
74
+ * ``Make dag_version_id in TI non-nullable (#50825)``
75
+ * ``Fix spelling of GitHub brand name (#53735)``
76
+ * ``Replace mock.patch("utcnow") with time_machine in Edge Executor (#53670)``
77
+ * ``Prepare release for July 2025 1st provider wave (#52727)``
78
+
61
79
 
62
80
  1.1.1
63
81
  .....
@@ -190,3 +190,5 @@ instance. The commands are:
190
190
  - ``airflow edge remote-edge-worker-exit-maintenance``: Request a remote edge worker to exit maintenance mode
191
191
  - ``airflow edge shutdown-remote-edge-worker``: Shuts down a remote edge worker gracefully
192
192
  - ``airflow edge remove-remote-edge-worker``: Remove a worker instance from the cluster
193
+ - ``airflow edge add-worker-queues``: Add queues to an edge worker
194
+ - ``airflow edge remove-worker-queues``: Remove queues from an edge worker
@@ -92,6 +92,8 @@ are provided by the API server.
92
92
 
93
93
  Release: 1.1.2
94
94
 
95
+ Release Date: ``|PypiReleaseDate|``
96
+
95
97
  Provider package
96
98
  ----------------
97
99
 
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
25
25
 
26
26
  [project]
27
27
  name = "apache-airflow-providers-edge3"
28
- version = "1.1.2rc1"
28
+ version = "1.1.2rc2"
29
29
  description = "Provider package apache-airflow-providers-edge3 for Apache Airflow"
30
30
  readme = "README.rst"
31
31
  authors = [
@@ -47,9 +47,10 @@ classifiers = [
47
47
  "Programming Language :: Python :: 3.10",
48
48
  "Programming Language :: Python :: 3.11",
49
49
  "Programming Language :: Python :: 3.12",
50
+ "Programming Language :: Python :: 3.13",
50
51
  "Topic :: System :: Monitoring",
51
52
  ]
52
- requires-python = "~=3.10"
53
+ requires-python = ">=3.10"
53
54
 
54
55
  # The dependencies should be modified in place in the generated file.
55
56
  # Any change in the dependencies is preserved when the file is regenerated
@@ -14,4 +14,4 @@
14
14
  # KIND, either express or implied. See the License for the
15
15
  # specific language governing permissions and limitations
16
16
  # under the License.
17
- __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore
17
+ __path__ = __import__("pkgutil").extend_path(__path__, __name__)
@@ -14,4 +14,4 @@
14
14
  # KIND, either express or implied. See the License for the
15
15
  # specific language governing permissions and limitations
16
16
  # under the License.
17
- __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore
17
+ __path__ = __import__("pkgutil").extend_path(__path__, __name__)
@@ -351,6 +351,48 @@ def remote_worker_request_shutdown(args) -> None:
351
351
  logger.info("Requested shutdown of Edge Worker host %s by %s.", args.edge_hostname, getuser())
352
352
 
353
353
 
354
+ @cli_utils.action_cli(check_db=False)
355
+ @providers_configuration_loaded
356
+ def add_worker_queues(args) -> None:
357
+ """Add queues to an edge worker."""
358
+ _check_valid_db_connection()
359
+ _check_if_registered_edge_host(hostname=args.edge_hostname)
360
+ from airflow.providers.edge3.models.edge_worker import add_worker_queues
361
+
362
+ queues = args.queues.split(",") if args.queues else []
363
+ if not queues:
364
+ raise SystemExit("Error: No queues specified to add.")
365
+
366
+ try:
367
+ add_worker_queues(args.edge_hostname, queues)
368
+ logger.info("Added queues %s to Edge Worker host %s by %s.", queues, args.edge_hostname, getuser())
369
+ except TypeError as e:
370
+ logger.error(str(e))
371
+ raise SystemExit
372
+
373
+
374
+ @cli_utils.action_cli(check_db=False)
375
+ @providers_configuration_loaded
376
+ def remove_worker_queues(args) -> None:
377
+ """Remove queues from an edge worker."""
378
+ _check_valid_db_connection()
379
+ _check_if_registered_edge_host(hostname=args.edge_hostname)
380
+ from airflow.providers.edge3.models.edge_worker import remove_worker_queues
381
+
382
+ queues = args.queues.split(",") if args.queues else []
383
+ if not queues:
384
+ raise SystemExit("Error: No queues specified to remove.")
385
+
386
+ try:
387
+ remove_worker_queues(args.edge_hostname, queues)
388
+ logger.info(
389
+ "Removed queues %s from Edge Worker host %s by %s.", queues, args.edge_hostname, getuser()
390
+ )
391
+ except TypeError as e:
392
+ logger.error(str(e))
393
+ raise SystemExit
394
+
395
+
354
396
  ARG_CONCURRENCY = Arg(
355
397
  ("-c", "--concurrency"),
356
398
  type=int,
@@ -380,6 +422,11 @@ ARG_REQUIRED_MAINTENANCE_COMMENT = Arg(
380
422
  help="Maintenance comments to report reason. Required if enabling maintenance",
381
423
  required=True,
382
424
  )
425
+ ARG_QUEUES_MANAGE = Arg(
426
+ ("-q", "--queues"),
427
+ help="Comma delimited list of queues to add or remove.",
428
+ required=True,
429
+ )
383
430
  ARG_WAIT_MAINT = Arg(
384
431
  ("-w", "--wait"),
385
432
  default=False,
@@ -516,4 +563,22 @@ EDGE_COMMANDS: list[ActionCommand] = [
516
563
  func=remote_worker_request_shutdown,
517
564
  args=(ARG_REQUIRED_EDGE_HOSTNAME,),
518
565
  ),
566
+ ActionCommand(
567
+ name="add-worker-queues",
568
+ help=add_worker_queues.__doc__,
569
+ func=add_worker_queues,
570
+ args=(
571
+ ARG_REQUIRED_EDGE_HOSTNAME,
572
+ ARG_QUEUES_MANAGE,
573
+ ),
574
+ ),
575
+ ActionCommand(
576
+ name="remove-worker-queues",
577
+ help=remove_worker_queues.__doc__,
578
+ func=remove_worker_queues,
579
+ args=(
580
+ ARG_REQUIRED_EDGE_HOSTNAME,
581
+ ARG_QUEUES_MANAGE,
582
+ ),
583
+ ),
519
584
  ]
@@ -188,8 +188,8 @@ class EdgeWorker:
188
188
 
189
189
  try:
190
190
  api_url = conf.get("edge", "api_url")
191
- execution_api_server_url = conf.get("core", "execution_api_server_url", fallback=...)
192
- if execution_api_server_url is ...:
191
+ execution_api_server_url = conf.get("core", "execution_api_server_url", fallback="")
192
+ if not execution_api_server_url:
193
193
  parsed = urlparse(api_url)
194
194
  execution_api_server_url = f"{parsed.scheme}://{parsed.netloc}/execution/"
195
195
 
@@ -41,13 +41,13 @@ try:
41
41
  from airflow.sdk import DAG, Param, Variable, task, task_group
42
42
  except ImportError:
43
43
  # Airflow 2.10 compat
44
- from airflow.decorators import task, task_group # type: ignore[no-redef,attr-defined]
45
- from airflow.models.dag import DAG # type: ignore[no-redef,attr-defined,assignment]
46
- from airflow.models.param import Param # type: ignore[no-redef,attr-defined]
47
- from airflow.models.variable import Variable # type: ignore[no-redef,attr-defined]
48
- from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined]
49
- from airflow.operators.empty import EmptyOperator # type: ignore[no-redef,attr-defined]
50
- from airflow.operators.python import PythonOperator # type: ignore[no-redef,attr-defined]
44
+ from airflow.decorators import task, task_group # type: ignore[attr-defined,no-redef]
45
+ from airflow.models.dag import DAG # type: ignore[assignment]
46
+ from airflow.models.param import Param # type: ignore[no-redef]
47
+ from airflow.models.variable import Variable
48
+ from airflow.operators.bash import BashOperator # type: ignore[no-redef]
49
+ from airflow.operators.empty import EmptyOperator # type: ignore[no-redef]
50
+ from airflow.operators.python import PythonOperator # type: ignore[no-redef]
51
51
 
52
52
  with DAG(
53
53
  dag_id="integration_test",
@@ -32,7 +32,11 @@ from subprocess import STDOUT, Popen
32
32
  from time import sleep
33
33
  from typing import TYPE_CHECKING, Any
34
34
 
35
- from airflow.decorators import task, task_group
35
+ try:
36
+ from airflow.sdk import task, task_group
37
+ except ImportError:
38
+ # Airflow 2 path
39
+ from airflow.decorators import task, task_group # type: ignore[attr-defined,no-redef]
36
40
  from airflow.exceptions import AirflowException, AirflowNotFoundException, AirflowSkipException
37
41
  from airflow.models import BaseOperator
38
42
  from airflow.models.dag import DAG
@@ -52,11 +56,11 @@ if TYPE_CHECKING:
52
56
  try:
53
57
  from airflow.sdk.types import RuntimeTaskInstanceProtocol as TaskInstance
54
58
  except ImportError:
55
- from airflow.models import TaskInstance # type: ignore[assignment, no-redef]
59
+ from airflow.models import TaskInstance # type: ignore[assignment]
56
60
  from airflow.utils.context import Context
57
61
 
58
62
  try:
59
- from airflow.operators.python import PythonOperator # type: ignore
63
+ from airflow.operators.python import PythonOperator
60
64
  except ImportError:
61
65
  from airflow.providers.common.compat.standard.operators import PythonOperator
62
66
 
@@ -277,7 +281,7 @@ with DAG(
277
281
 
278
282
  @task.virtualenv(requirements="numpy")
279
283
  def virtualenv():
280
- import numpy # type: ignore
284
+ import numpy
281
285
 
282
286
  print(f"Welcome to virtualenv with numpy version {numpy.__version__}.")
283
287
 
@@ -30,7 +30,6 @@ from sqlalchemy.orm import Session
30
30
  from airflow.cli.cli_config import GroupCommand
31
31
  from airflow.configuration import conf
32
32
  from airflow.executors.base_executor import BaseExecutor
33
- from airflow.models.abstractoperator import DEFAULT_QUEUE
34
33
  from airflow.models.taskinstance import TaskInstance, TaskInstanceState
35
34
  from airflow.providers.edge3.cli.edge_command import EDGE_COMMANDS
36
35
  from airflow.providers.edge3.models.edge_job import EdgeJobModel
@@ -55,6 +54,7 @@ if TYPE_CHECKING:
55
54
  TaskTuple = tuple[TaskInstanceKey, CommandType, str | None, Any | None]
56
55
 
57
56
  PARALLELISM: int = conf.getint("core", "PARALLELISM")
57
+ DEFAULT_QUEUE: str = conf.get_mandatory_value("operators", "default_queue")
58
58
 
59
59
 
60
60
  class EdgeExecutor(BaseExecutor):
@@ -109,7 +109,7 @@ class EdgeWorkerModel(Base, LoggingMixin):
109
109
  super().__init__()
110
110
 
111
111
  @property
112
- def sysinfo_json(self) -> dict:
112
+ def sysinfo_json(self) -> dict | None:
113
113
  return json.loads(self.sysinfo) if self.sysinfo else None
114
114
 
115
115
  @property
@@ -283,3 +283,37 @@ def request_shutdown(worker_name: str, session: Session = NEW_SESSION) -> None:
283
283
  EdgeWorkerState.UNKNOWN,
284
284
  ):
285
285
  worker.state = EdgeWorkerState.SHUTDOWN_REQUEST
286
+
287
+
288
+ @provide_session
289
+ def add_worker_queues(worker_name: str, queues: list[str], session: Session = NEW_SESSION) -> None:
290
+ """Add queues to an edge worker."""
291
+ query = select(EdgeWorkerModel).where(EdgeWorkerModel.worker_name == worker_name)
292
+ worker: EdgeWorkerModel = session.scalar(query)
293
+ if worker.state in (
294
+ EdgeWorkerState.OFFLINE,
295
+ EdgeWorkerState.OFFLINE_MAINTENANCE,
296
+ EdgeWorkerState.UNKNOWN,
297
+ ):
298
+ error_message = f"Cannot add queues to edge worker {worker_name} as it is in {worker.state} state!"
299
+ logger.error(error_message)
300
+ raise TypeError(error_message)
301
+ worker.add_queues(queues)
302
+
303
+
304
+ @provide_session
305
+ def remove_worker_queues(worker_name: str, queues: list[str], session: Session = NEW_SESSION) -> None:
306
+ """Remove queues from an edge worker."""
307
+ query = select(EdgeWorkerModel).where(EdgeWorkerModel.worker_name == worker_name)
308
+ worker: EdgeWorkerModel = session.scalar(query)
309
+ if worker.state in (
310
+ EdgeWorkerState.OFFLINE,
311
+ EdgeWorkerState.OFFLINE_MAINTENANCE,
312
+ EdgeWorkerState.UNKNOWN,
313
+ ):
314
+ error_message = (
315
+ f"Cannot remove queues from edge worker {worker_name} as it is in {worker.state} state!"
316
+ )
317
+ logger.error(error_message)
318
+ raise TypeError(error_message)
319
+ worker.remove_queues(queues)
@@ -17,6 +17,7 @@
17
17
 
18
18
  from __future__ import annotations
19
19
 
20
+ import sys
20
21
  from typing import TYPE_CHECKING, Any
21
22
 
22
23
  from airflow.configuration import conf
@@ -213,12 +214,24 @@ try:
213
214
  except AirflowConfigException:
214
215
  EDGE_EXECUTOR_ACTIVE = False
215
216
 
217
+ # Load the API endpoint only on api-server (Airflow 3.x) or webserver (Airflow 2.x)
218
+ # todo(jscheffl): Remove this check when the discussion in
219
+ # https://lists.apache.org/thread/w170czq6r7bslkqp1tk6bjjjo0789wgl
220
+ # resulted in a proper API to selective initialize. Maybe backcompat-shim
221
+ # is also needed to support Airflow-versions prior the rework.
222
+ if AIRFLOW_V_3_0_PLUS:
223
+ RUNNING_ON_APISERVER = (len(sys.argv) > 1 and sys.argv[1] in ["api-server"]) or (
224
+ len(sys.argv) > 2 and sys.argv[2] == "airflow-core/src/airflow/api_fastapi/main.py"
225
+ )
226
+ else:
227
+ RUNNING_ON_APISERVER = "gunicorn" in sys.argv[0] and "airflow-webserver" in sys.argv
228
+
216
229
 
217
230
  class EdgeExecutorPlugin(AirflowPlugin):
218
231
  """EdgeExecutor Plugin - provides API endpoints for Edge Workers in Webserver."""
219
232
 
220
233
  name = "edge_executor"
221
- if EDGE_EXECUTOR_ACTIVE:
234
+ if EDGE_EXECUTOR_ACTIVE and RUNNING_ON_APISERVER:
222
235
  if AIRFLOW_V_3_0_PLUS:
223
236
  fastapi_apps = [_get_api_endpoint()]
224
237
  else:
@@ -58,7 +58,7 @@ if AIRFLOW_V_3_0_PLUS:
58
58
 
59
59
  else:
60
60
  # Airflow 2.10 compatibility
61
- from airflow.utils.jwt_signer import JWTSigner # type: ignore
61
+ from airflow.utils.jwt_signer import JWTSigner
62
62
 
63
63
  @cache
64
64
  def jwt_signer() -> JWTSigner:
@@ -127,7 +127,7 @@ else:
127
127
 
128
128
  # In Airflow 3 with AIP-72 we get workload addressed by ExecuteTask
129
129
  # But in Airflow 2.10 it is a command line array
130
- ExecuteTask = list[str] # type: ignore[no-redef,assignment,misc]
130
+ ExecuteTask = list[str] # type: ignore[assignment,misc]
131
131
 
132
132
  def parse_command(command: str) -> ExecuteTask:
133
133
  from ast import literal_eval
@@ -66,7 +66,7 @@ def rpcapi_v2(body: dict[str, Any]) -> APIResponse:
66
66
  # Note: Except the method map this _was_ a 100% copy of internal API module
67
67
  # airflow.api_internal.endpoints.rpc_api_endpoint.internal_airflow_api()
68
68
  # As of rework for FastAPI in Airflow 3.0, this is updated and to be removed in the future.
69
- from airflow.api_internal.endpoints.rpc_api_endpoint import ( # type: ignore[attr-defined]
69
+ from airflow.api_internal.endpoints.rpc_api_endpoint import (
70
70
  # Note: This is just for compatibility with Airflow 2.10, not working for Airflow 3 / main as removed
71
71
  initialize_method_map,
72
72
  )
@@ -14,4 +14,4 @@
14
14
  # KIND, either express or implied. See the License for the
15
15
  # specific language governing permissions and limitations
16
16
  # under the License.
17
- __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore
17
+ __path__ = __import__("pkgutil").extend_path(__path__, __name__)
@@ -63,6 +63,7 @@ MOCK_COMMAND = (
63
63
  "dag_id": "mock",
64
64
  "run_id": "mock",
65
65
  "try_number": 1,
66
+ "dag_version_id": "01234567-89ab-cdef-0123-456789abcdef",
66
67
  "pool_slots": 1,
67
68
  "queue": "default",
68
69
  "priority_weight": 1,
@@ -21,6 +21,7 @@ from datetime import datetime, timedelta
21
21
  from unittest.mock import MagicMock, patch
22
22
 
23
23
  import pytest
24
+ import time_machine
24
25
 
25
26
  from airflow.configuration import conf
26
27
  from airflow.models.taskinstancekey import TaskInstanceKey
@@ -275,9 +276,7 @@ class TestEdgeExecutor:
275
276
  )
276
277
  session.commit()
277
278
 
278
- with patch(
279
- "airflow.utils.timezone.utcnow", return_value=datetime(2023, 1, 1, 1, 0, 0, tzinfo=timezone.utc)
280
- ):
279
+ with time_machine.travel(datetime(2023, 1, 1, 1, 0, 0, tzinfo=timezone.utc), tick=False):
281
280
  with conf_vars({("edge", "heartbeat_interval"): "10"}):
282
281
  executor.sync()
283
282
 
@@ -328,6 +327,7 @@ class TestEdgeExecutor:
328
327
  queue="default",
329
328
  priority_weight=1,
330
329
  start_date=timezone.utcnow(),
330
+ dag_version_id="4d828a62-a417-4936-a7a6-2b3fabacecab",
331
331
  ),
332
332
  dag_rel_path="mock.py",
333
333
  log_path="mock.log",
@@ -17,6 +17,7 @@
17
17
  from __future__ import annotations
18
18
 
19
19
  import importlib
20
+ from unittest.mock import patch
20
21
 
21
22
  import pytest
22
23
  import time_machine
@@ -44,17 +45,20 @@ def test_plugin_inactive():
44
45
 
45
46
 
46
47
  @pytest.mark.db_test
47
- def test_plugin_active():
48
- with conf_vars({("edge", "api_enabled"): "true"}):
48
+ def test_plugin_active_apiserver():
49
+ mock_cli = ["airflow", "api-server"] if AIRFLOW_V_3_0_PLUS else ["gunicorn", "airflow-webserver"]
50
+ with conf_vars({("edge", "api_enabled"): "true"}), patch("sys.argv", mock_cli):
49
51
  importlib.reload(edge_executor_plugin)
50
52
 
51
53
  from airflow.providers.edge3.plugins.edge_executor_plugin import (
52
54
  EDGE_EXECUTOR_ACTIVE,
55
+ RUNNING_ON_APISERVER,
53
56
  EdgeExecutorPlugin,
54
57
  )
55
58
 
56
59
  rep = EdgeExecutorPlugin()
57
60
  assert EDGE_EXECUTOR_ACTIVE
61
+ assert RUNNING_ON_APISERVER
58
62
  if AIRFLOW_V_3_0_PLUS:
59
63
  assert len(rep.appbuilder_views) == 0
60
64
  assert len(rep.flask_blueprints) == 0
@@ -64,6 +68,27 @@ def test_plugin_active():
64
68
  assert len(rep.flask_blueprints) == 2
65
69
 
66
70
 
71
+ @patch("sys.argv", ["airflow", "some-other-command"])
72
+ def test_plugin_active_non_apiserver():
73
+ with conf_vars({("edge", "api_enabled"): "true"}):
74
+ importlib.reload(edge_executor_plugin)
75
+
76
+ from airflow.providers.edge3.plugins.edge_executor_plugin import (
77
+ EDGE_EXECUTOR_ACTIVE,
78
+ RUNNING_ON_APISERVER,
79
+ EdgeExecutorPlugin,
80
+ )
81
+
82
+ rep = EdgeExecutorPlugin()
83
+ assert EDGE_EXECUTOR_ACTIVE
84
+ assert not RUNNING_ON_APISERVER
85
+ assert len(rep.appbuilder_views) == 0
86
+ assert len(rep.flask_blueprints) == 0
87
+ assert len(rep.appbuilder_views) == 0
88
+ if AIRFLOW_V_3_0_PLUS:
89
+ assert len(rep.fastapi_apps) == 0
90
+
91
+
67
92
  @pytest.fixture
68
93
  def plugin():
69
94
  from airflow.providers.edge3.plugins.edge_executor_plugin import EdgeExecutorPlugin