apache-airflow-providers-edge3 1.3.1__tar.gz → 1.4.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-edge3 might be problematic. Click here for more details.
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/PKG-INFO +35 -14
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/README.rst +30 -10
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/architecture.rst +10 -11
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/changelog.rst +33 -4
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/deployment.rst +20 -6
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/edge_executor.rst +11 -6
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/index.rst +37 -8
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/ui_plugin.rst +3 -3
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/provider.yaml +2 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/pyproject.toml +6 -4
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/__init__.py +1 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/cli/worker.py +1 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/example_dags/win_test.py +1 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/executors/edge_executor.py +31 -2
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/models/edge_job.py +17 -14
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/models/edge_logs.py +11 -8
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/models/edge_worker.py +23 -19
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/edge_executor_plugin.py +26 -10
- apache_airflow_providers_edge3-1.4.0rc1/src/airflow/providers/edge3/plugins/www/dist/main.umd.cjs +142 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/schemas.gen.ts +4 -4
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/types.gen.ts +1 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/layouts/EdgeLayout.tsx +5 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/main.tsx +6 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/version_compat.py +5 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/routes/jobs.py +1 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/routes/worker.py +1 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/cli/test_worker.py +1 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/executors/test_edge_executor.py +85 -2
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/worker_api/routes/test_logs.py +1 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/worker_api/routes/test_worker.py +1 -1
- apache_airflow_providers_edge3-1.4.0rc1/www-hash.txt +1 -0
- apache_airflow_providers_edge3-1.3.1/src/airflow/providers/edge3/plugins/www/dist/main.umd.cjs +0 -142
- apache_airflow_providers_edge3-1.3.1/www-hash.txt +0 -1
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/cli-ref.rst +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/commits.rst +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/conf.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/configurations-ref.rst +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/img/distributed_architecture.svg +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/img/edge_package.svg +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/img/worker_hosts.png +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/install_on_windows.rst +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/installing-providers-from-sources.rst +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/security.rst +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/why_edge.rst +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/LICENSE +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/cli/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/cli/api_client.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/cli/dataclasses.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/cli/edge_command.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/cli/signalling.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/example_dags/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/example_dags/integration_test.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/example_dags/win_notepad.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/executors/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/get_provider_info.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/models/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/openapi/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/openapi/edge_worker_api_v1.yaml +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/openapi/v2-edge-generated.yaml +3 -3
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/templates/edge_worker_hosts.html +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/templates/edge_worker_jobs.html +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/.gitignore +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/.prettierignore +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/.prettierrc +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/README.md +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/dist/main.d.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/eslint.config.js +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/index.html +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/queries/common.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/queries/ensureQueryData.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/queries/index.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/queries/infiniteQueries.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/queries/prefetch.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/queries/queries.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/queries/suspense.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/core/ApiError.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/core/ApiRequestOptions.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/core/ApiResult.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/core/CancelablePromise.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/core/OpenAPI.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/core/request.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/index.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/openapi-gen/requests/services.gen.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/package.json +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/AddQueueButton.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/ErrorAlert.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/MaintenanceEditCommentButton.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/MaintenanceEnterButton.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/MaintenanceExitButton.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/RemoveQueueButton.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/StateBadge.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/StateIcon.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/WorkerDeleteButton.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/WorkerOperations.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/WorkerShutdownButton.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/WorkerStateBadge.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/WorkerStateIcon.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/ui/Alert.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/ui/CloseButton.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/ui/ScrollToAnchor.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/ui/createToaster.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/components/ui/index.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/context/colorMode/ColorModeProvider.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/context/colorMode/index.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/context/colorMode/useColorMode.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/dev.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/layouts/NavTabs.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/pages/JobsPage.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/pages/WorkerPage.tsx +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/res/README.md +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/res/cloud-computer-dark.svg +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/res/cloud-computer.svg +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/theme.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/utils/config.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/utils/index.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/utils/tokenHandler.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/utils/useContainerWidth.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/src/vite-env.d.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/testsSetup.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/tsconfig.app.json +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/tsconfig.json +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/tsconfig.lib.json +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/tsconfig.node.json +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/plugins/www/vite.config.ts +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/app.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/auth.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/datamodels.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/datamodels_ui.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/routes/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/routes/_v2_compat.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/routes/_v2_routes.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/routes/health.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/routes/logs.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/src/airflow/providers/edge3/worker_api/routes/ui.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/conftest.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/cli/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/cli/test_api_client.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/cli/test_dataclasses.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/cli/test_edge_command.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/cli/test_signalling.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/executors/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/models/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/plugins/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/plugins/test_edge_executor_plugin.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/worker_api/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/worker_api/routes/__init__.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/worker_api/routes/test_health.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/worker_api/routes/test_jobs.py +0 -0
- {apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/tests/unit/edge3/worker_api/routes/test_ui.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-edge3
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0rc1
|
|
4
4
|
Summary: Provider package apache-airflow-providers-edge3 for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,edge3,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -20,12 +20,13 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
|
-
Requires-Dist: apache-airflow>=2.10.
|
|
23
|
+
Requires-Dist: apache-airflow>=2.10.0rc1
|
|
24
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.8.0rc1
|
|
24
25
|
Requires-Dist: pydantic>=2.11.0
|
|
25
26
|
Requires-Dist: retryhttp>=1.2.0,!=1.3.0
|
|
26
27
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
27
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.
|
|
28
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.
|
|
28
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-edge3/1.4.0/changelog.html
|
|
29
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-edge3/1.4.0
|
|
29
30
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
30
31
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
31
32
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -56,7 +57,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
|
56
57
|
|
|
57
58
|
Package ``apache-airflow-providers-edge3``
|
|
58
59
|
|
|
59
|
-
Release: ``1.
|
|
60
|
+
Release: ``1.4.0``
|
|
60
61
|
|
|
61
62
|
|
|
62
63
|
Handle edge workers on remote sites via HTTP(s) connection and orchestrates work over distributed sites.
|
|
@@ -81,7 +82,7 @@ This is a provider package for ``edge3`` provider. All classes for this provider
|
|
|
81
82
|
are in ``airflow.providers.edge3`` python package.
|
|
82
83
|
|
|
83
84
|
You can find package information and changelog for the provider
|
|
84
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.
|
|
85
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.4.0/>`_.
|
|
85
86
|
|
|
86
87
|
Installation
|
|
87
88
|
------------
|
|
@@ -95,14 +96,34 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13
|
|
|
95
96
|
Requirements
|
|
96
97
|
------------
|
|
97
98
|
|
|
98
|
-
|
|
99
|
-
PIP package
|
|
100
|
-
|
|
101
|
-
``apache-airflow``
|
|
102
|
-
``
|
|
103
|
-
``
|
|
104
|
-
|
|
99
|
+
========================================== ===================
|
|
100
|
+
PIP package Version required
|
|
101
|
+
========================================== ===================
|
|
102
|
+
``apache-airflow`` ``>=2.10.0``
|
|
103
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
104
|
+
``pydantic`` ``>=2.11.0``
|
|
105
|
+
``retryhttp`` ``>=1.2.0,!=1.3.0``
|
|
106
|
+
========================================== ===================
|
|
107
|
+
|
|
108
|
+
Cross provider package dependencies
|
|
109
|
+
-----------------------------------
|
|
110
|
+
|
|
111
|
+
Those are dependencies that might be needed in order to use all the features of the package.
|
|
112
|
+
You need to install the specified providers in order to use them.
|
|
113
|
+
|
|
114
|
+
You can install such cross-provider dependencies when installing from PyPI. For example:
|
|
115
|
+
|
|
116
|
+
.. code-block:: bash
|
|
117
|
+
|
|
118
|
+
pip install apache-airflow-providers-edge3[common.compat]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
================================================================================================================== =================
|
|
122
|
+
Dependent package Extra
|
|
123
|
+
================================================================================================================== =================
|
|
124
|
+
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
125
|
+
================================================================================================================== =================
|
|
105
126
|
|
|
106
127
|
The changelog for the provider package can be found in the
|
|
107
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.
|
|
128
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.4.0/changelog.html>`_.
|
|
108
129
|
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
Package ``apache-airflow-providers-edge3``
|
|
25
25
|
|
|
26
|
-
Release: ``1.
|
|
26
|
+
Release: ``1.4.0``
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
Handle edge workers on remote sites via HTTP(s) connection and orchestrates work over distributed sites.
|
|
@@ -48,7 +48,7 @@ This is a provider package for ``edge3`` provider. All classes for this provider
|
|
|
48
48
|
are in ``airflow.providers.edge3`` python package.
|
|
49
49
|
|
|
50
50
|
You can find package information and changelog for the provider
|
|
51
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.
|
|
51
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.4.0/>`_.
|
|
52
52
|
|
|
53
53
|
Installation
|
|
54
54
|
------------
|
|
@@ -62,13 +62,33 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13
|
|
|
62
62
|
Requirements
|
|
63
63
|
------------
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
PIP package
|
|
67
|
-
|
|
68
|
-
``apache-airflow``
|
|
69
|
-
``
|
|
70
|
-
``
|
|
71
|
-
|
|
65
|
+
========================================== ===================
|
|
66
|
+
PIP package Version required
|
|
67
|
+
========================================== ===================
|
|
68
|
+
``apache-airflow`` ``>=2.10.0``
|
|
69
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
70
|
+
``pydantic`` ``>=2.11.0``
|
|
71
|
+
``retryhttp`` ``>=1.2.0,!=1.3.0``
|
|
72
|
+
========================================== ===================
|
|
73
|
+
|
|
74
|
+
Cross provider package dependencies
|
|
75
|
+
-----------------------------------
|
|
76
|
+
|
|
77
|
+
Those are dependencies that might be needed in order to use all the features of the package.
|
|
78
|
+
You need to install the specified providers in order to use them.
|
|
79
|
+
|
|
80
|
+
You can install such cross-provider dependencies when installing from PyPI. For example:
|
|
81
|
+
|
|
82
|
+
.. code-block:: bash
|
|
83
|
+
|
|
84
|
+
pip install apache-airflow-providers-edge3[common.compat]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
================================================================================================================== =================
|
|
88
|
+
Dependent package Extra
|
|
89
|
+
================================================================================================================== =================
|
|
90
|
+
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
91
|
+
================================================================================================================== =================
|
|
72
92
|
|
|
73
93
|
The changelog for the provider package can be found in the
|
|
74
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.
|
|
94
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.4.0/changelog.html>`_.
|
|
@@ -60,10 +60,10 @@ deployed outside of the central Airflow cluster is connected via HTTP(s) to the
|
|
|
60
60
|
edge_worker->api[label="HTTP(s)"]
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
* **Workers** - Execute the assigned tasks - most standard setup has local or centralized workers, e.g. via Celery
|
|
63
|
+
* **Workers** (Central) - Execute the assigned tasks - most standard setup has local or centralized workers, e.g. via Celery
|
|
64
64
|
* **Edge Workers** - Special workers which pull tasks via HTTP(s) as provided as feature via this provider package
|
|
65
65
|
* **Scheduler** - Responsible for adding the necessary tasks to the queue. The EdgeExecutor is running as a module inside the scheduler.
|
|
66
|
-
* **API server** - HTTP REST API Server provides access to Dag/task status information. The required end-points are
|
|
66
|
+
* **API server** (webserver in Airflow 2.x) - HTTP REST API Server provides access to Dag/task status information. The required end-points are
|
|
67
67
|
provided by the Edge provider plugin. The Edge Worker uses this API to pull tasks and send back the results.
|
|
68
68
|
* **Database** - Contains information about the status of tasks, Dags, Variables, connections, etc.
|
|
69
69
|
|
|
@@ -73,19 +73,18 @@ In detail the parts of the Edge provider are deployed as follows:
|
|
|
73
73
|
:alt: Overview and communication of Edge Provider modules
|
|
74
74
|
|
|
75
75
|
* **EdgeExecutor** - The EdgeExecutor is running inside the core Airflow scheduler. It is responsible for
|
|
76
|
-
scheduling tasks and sending them to the Edge job queue in the database.
|
|
77
|
-
``
|
|
78
|
-
``executor`` configuration option in the ``airflow.cfg`` file to
|
|
76
|
+
scheduling tasks and sending them to the Edge job queue in the database. To activate the EdgeExecutor, you
|
|
77
|
+
need to set the ``executor`` configuration option in the ``airflow.cfg`` file to
|
|
79
78
|
``airflow.providers.edge3.executors.EdgeExecutor``. For more details see :doc:`edge_executor`. Note that also
|
|
80
79
|
multiple executors can be used in parallel together with the EdgeExecutor.
|
|
81
|
-
* **API server** - The API server is providing REST endpoints to the web UI as well
|
|
82
|
-
Edge provider adds a plugin that provides additional REST API for the Edge Worker
|
|
83
|
-
manage workers (
|
|
80
|
+
* **API server** (webserver in Airflow 2.x) - The API server is providing REST endpoints to the web UI as well
|
|
81
|
+
as serves static files. The Edge provider adds a plugin that provides additional REST API for the Edge Worker
|
|
82
|
+
as well as UI elements to manage workers (not available in Airflow 3.0).
|
|
84
83
|
The API server is responsible for handling requests from the Edge Worker and sending back the results. To
|
|
85
84
|
activate the API server, you need to set the ``api_enabled`` configuration option in ``edge`` section in the
|
|
86
85
|
``airflow.cfg`` file to ``True``. The API endpoints for edge is not started by default.
|
|
87
|
-
|
|
88
|
-
* **Database** - The Airflow meta database is used to store the status of tasks, Dags, Variables,
|
|
86
|
+
For more details see :doc:`ui_plugin`.
|
|
87
|
+
* **Database** - The Airflow meta database is used to store the status of tasks, Dags, Variables, Connections
|
|
89
88
|
etc. The Edge provider uses the database to store the status of the Edge Worker instances and the tasks that
|
|
90
89
|
are assigned to it. The database is also used to store the results of the tasks that are executed by the
|
|
91
90
|
Edge Worker. Setup of needed tables and migration is done automatically when the provider package is deployed.
|
|
@@ -174,7 +173,7 @@ The following features are known missing and will be implemented in increments:
|
|
|
174
173
|
- Test/Support on Windows for Edge Worker
|
|
175
174
|
|
|
176
175
|
- Scaling test - Check and define boundaries of workers/jobs. Today it is known to
|
|
177
|
-
scale into a range of
|
|
176
|
+
scale into a range of ~80 workers. This is not a hard limit but just an experience reported.
|
|
178
177
|
- Load tests - impact of scaled execution and code optimization
|
|
179
178
|
- Incremental logs during task execution can be served w/o shared log disk on api-server
|
|
180
179
|
- Reduce dependencies during execution: Today the worker depends on the airflow core with a lot
|
{apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/changelog.rst
RENAMED
|
@@ -27,6 +27,39 @@
|
|
|
27
27
|
Changelog
|
|
28
28
|
---------
|
|
29
29
|
|
|
30
|
+
1.4.0
|
|
31
|
+
.....
|
|
32
|
+
|
|
33
|
+
Features
|
|
34
|
+
~~~~~~~~
|
|
35
|
+
|
|
36
|
+
* ``Introduce generic Callbacks to support running callbacks on executors (#54796)``
|
|
37
|
+
* ``Add revoke_task implementation to EdgeExecutor for task queued timeout support (#56240)``
|
|
38
|
+
|
|
39
|
+
Bug Fixes
|
|
40
|
+
~~~~~~~~~
|
|
41
|
+
|
|
42
|
+
* ``Bugfix/remove airflow utils deprecations in edge (#56568)``
|
|
43
|
+
* ``Fix Edge3 provider navigation with webserver base_url configuration (#56189)``
|
|
44
|
+
|
|
45
|
+
Misc
|
|
46
|
+
~~~~
|
|
47
|
+
|
|
48
|
+
* ``Migrate edge3 provider to ''common.compat'' (#56998)``
|
|
49
|
+
* ``SQLA2: Partially fix type hints in the edge3 provider (#56873)``
|
|
50
|
+
|
|
51
|
+
Doc-only
|
|
52
|
+
~~~~~~~~
|
|
53
|
+
|
|
54
|
+
* ``Correct 'Dag' to 'DAG' for code snippets in provider docs (#56727)``
|
|
55
|
+
* ``Revise details of edge documentation after Airflow 3.1 release (#56166)``
|
|
56
|
+
* ``Remove placeholder Release Date in changelog and index files (#56056)``
|
|
57
|
+
|
|
58
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
59
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
60
|
+
* ``Common.Compat: Extract reusable compat utilities and rename to sdk (#56884)``
|
|
61
|
+
* ``Enable PT011 rule to prvoider tests (#56277)``
|
|
62
|
+
|
|
30
63
|
1.3.1
|
|
31
64
|
.....
|
|
32
65
|
|
|
@@ -48,8 +81,6 @@ Misc
|
|
|
48
81
|
.....
|
|
49
82
|
|
|
50
83
|
|
|
51
|
-
Release Date: ``|PypiReleaseDate|``
|
|
52
|
-
|
|
53
84
|
Features
|
|
54
85
|
~~~~~~~~
|
|
55
86
|
|
|
@@ -88,8 +119,6 @@ Doc-only
|
|
|
88
119
|
1.2.0
|
|
89
120
|
.....
|
|
90
121
|
|
|
91
|
-
Release Date: ``|PypiReleaseDate|``
|
|
92
|
-
|
|
93
122
|
Features
|
|
94
123
|
~~~~~~~~
|
|
95
124
|
|
{apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/deployment.rst
RENAMED
|
@@ -21,8 +21,7 @@ Edge Worker Deployment
|
|
|
21
21
|
Edge Workers can be deployed outside of the central Airflow infrastructure. They
|
|
22
22
|
are connected to the Airflow API server via HTTP(s). The Edge Worker is a
|
|
23
23
|
lightweight component that can be deployed on any machine that has outbound
|
|
24
|
-
HTTP(s) access to the Airflow API server.
|
|
25
|
-
lightweight and easy to deploy. It allows you to run Airflow tasks on machines
|
|
24
|
+
HTTP(s) access to the Airflow API server. It allows you to run Airflow tasks on machines
|
|
26
25
|
that are not part of your main data center, e.g. edge servers. This also allows to
|
|
27
26
|
deploy only reduced dependencies on the edge worker.
|
|
28
27
|
|
|
@@ -46,7 +45,8 @@ Minimum Airflow configuration settings for the Edge Worker to make it running is
|
|
|
46
45
|
|
|
47
46
|
- Section ``[api_auth]``
|
|
48
47
|
|
|
49
|
-
- ``jwt_secret``: A
|
|
48
|
+
- ``jwt_secret``: A shared secret between workers and the api-server to authenticate
|
|
49
|
+
(starting from version 3.0.0).
|
|
50
50
|
|
|
51
51
|
- Section ``[core]``
|
|
52
52
|
|
|
@@ -56,7 +56,7 @@ Minimum Airflow configuration settings for the Edge Worker to make it running is
|
|
|
56
56
|
- ``executor``: Executor must be set or added to be ``airflow.providers.edge3.executors.EdgeExecutor``
|
|
57
57
|
- ``internal_api_secret_key``: An encryption key must be set on api-server and Edge Worker component as
|
|
58
58
|
shared secret to authenticate traffic. It should be a random string like the fernet key
|
|
59
|
-
(for versions earlier than 3.0.0).
|
|
59
|
+
(for versions earlier than 3.0.0 - from Airflow 3.0 and above this is using ``api_auth.jwt_secret``).
|
|
60
60
|
|
|
61
61
|
- Section ``[edge]``
|
|
62
62
|
|
|
@@ -72,6 +72,20 @@ subcommand
|
|
|
72
72
|
.. code-block:: bash
|
|
73
73
|
|
|
74
74
|
airflow edge worker
|
|
75
|
+
2025-09-27T12:28:32.954316Z [info ] Starting worker with API endpoint http://localhost:8080/edge_worker/v1/rpcapi
|
|
76
|
+
____________ _____________
|
|
77
|
+
____ |__( )_________ __/__ /________ __
|
|
78
|
+
____ /| |_ /__ ___/_ /_ __ /_ __ \_ | /| / /
|
|
79
|
+
___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ /
|
|
80
|
+
_/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/
|
|
81
|
+
____ __ _ __ __
|
|
82
|
+
/ __/__/ /__ ____ | | /| / /__ ____/ /_____ ____
|
|
83
|
+
/ _// _ / _ / -_) | |/ |/ / _ \/ __/ _/ -_) __/
|
|
84
|
+
/___/\_,_/\_, /\__/ |__/|__/\___/_/ /_/\_\\__/_/
|
|
85
|
+
/___/
|
|
86
|
+
|
|
87
|
+
2025-09-27T12:28:33.171525Z [info ] No new job to process
|
|
88
|
+
|
|
75
89
|
|
|
76
90
|
You can also start this worker in the background by running
|
|
77
91
|
it as a daemonized process. Additionally, you can redirect stdout
|
|
@@ -97,9 +111,9 @@ process as and wait until all running tasks are completed. Also in a console you
|
|
|
97
111
|
``Ctrl-C`` to stop the worker.
|
|
98
112
|
|
|
99
113
|
If you want to monitor the remote activity and worker, use the UI plugin which
|
|
100
|
-
is included in the provider package and install it on the webserver and use the
|
|
114
|
+
is included in the provider package and install it on the api-server / webserver and use the
|
|
101
115
|
"Admin" - "Edge Worker Hosts" and "Edge Worker Jobs" pages.
|
|
102
|
-
(Note: The plugin is not
|
|
116
|
+
(Note: The plugin is not available on Airflow 3.0 UI, it is only in 2.x and 3.1++)
|
|
103
117
|
|
|
104
118
|
If you want to check status of the worker via CLI you can use the command
|
|
105
119
|
|
|
@@ -27,6 +27,9 @@ The configuration parameters of the Edge Executor can be found in the Edge provi
|
|
|
27
27
|
|
|
28
28
|
To understand the setup of the Edge Executor, please also take a look to :doc:`architecture`.
|
|
29
29
|
|
|
30
|
+
See more details Airflow documentation
|
|
31
|
+
:ref:`apache-airflow:using-multiple-executors-concurrently`.
|
|
32
|
+
|
|
30
33
|
.. _edge_executor:queue:
|
|
31
34
|
|
|
32
35
|
Queues
|
|
@@ -43,6 +46,7 @@ Workers can listen to one or multiple queues of tasks. When a worker is
|
|
|
43
46
|
started (using command ``airflow edge worker``), a set of comma-delimited queue
|
|
44
47
|
names (with no whitespace) can be given (e.g. ``airflow edge worker -q remote,wisconsin_site``).
|
|
45
48
|
This worker will then only pick up tasks wired to the specified queue(s).
|
|
49
|
+
If the ``queue`` attribute is not given then a worker will pick tasks from all queues.
|
|
46
50
|
|
|
47
51
|
This can be useful if you need specialized workers, either from a
|
|
48
52
|
resource perspective (for say very lightweight tasks where one worker
|
|
@@ -80,12 +84,12 @@ Here is an example setting pool_slots for a task:
|
|
|
80
84
|
|
|
81
85
|
import pendulum
|
|
82
86
|
|
|
83
|
-
from airflow import
|
|
87
|
+
from airflow import DAG
|
|
84
88
|
from airflow.decorators import task
|
|
85
89
|
from airflow.example_dags.libs.helper import print_stuff
|
|
86
90
|
from airflow.settings import AIRFLOW_HOME
|
|
87
91
|
|
|
88
|
-
with
|
|
92
|
+
with DAG(
|
|
89
93
|
dag_id="example_edge_pool_slots",
|
|
90
94
|
schedule=None,
|
|
91
95
|
start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
|
|
@@ -104,9 +108,10 @@ Current Limitations Edge Executor
|
|
|
104
108
|
|
|
105
109
|
- Some known limitations
|
|
106
110
|
|
|
107
|
-
- Log upload will only work if you use a single
|
|
108
|
-
Logs are uploaded in chunks and are transferred via API. If you use multiple webservers w/o
|
|
109
|
-
the logs will be scattered across the webserver instances
|
|
111
|
+
- Log upload will only work if you use a single api-server / webserver instance or they need to share one log file
|
|
112
|
+
volume. Logs are uploaded in chunks and are transferred via API. If you use multiple api-servers / webservers w/o
|
|
113
|
+
a shared log volume the logs will be scattered across the api-server / webserver instances and if you view the
|
|
114
|
+
logs on UI you will only see fractions of the logs.
|
|
110
115
|
- Performance: No extensive performance assessment and scaling tests have been made. The edge executor package is
|
|
111
116
|
optimized for stability. This will be incrementally improved in future releases. Setups have reported stable
|
|
112
|
-
operation with ~80 workers until now. Note that executed tasks require more webserver API capacity.
|
|
117
|
+
operation with ~80 workers until now. Note that executed tasks require more api-server / webserver API capacity.
|
{apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/index.rst
RENAMED
|
@@ -90,7 +90,7 @@ Additional REST API endpoints are provided to distribute tasks and manage the ed
|
|
|
90
90
|
are provided by the API server.
|
|
91
91
|
|
|
92
92
|
|
|
93
|
-
Release: 1.
|
|
93
|
+
Release: 1.4.0
|
|
94
94
|
|
|
95
95
|
Provider package
|
|
96
96
|
----------------
|
|
@@ -110,10 +110,39 @@ Requirements
|
|
|
110
110
|
|
|
111
111
|
The minimum Apache Airflow version supported by this provider distribution is ``2.10.0``.
|
|
112
112
|
|
|
113
|
-
|
|
114
|
-
PIP package
|
|
115
|
-
|
|
116
|
-
``apache-airflow``
|
|
117
|
-
``
|
|
118
|
-
``
|
|
119
|
-
|
|
113
|
+
========================================== ===================
|
|
114
|
+
PIP package Version required
|
|
115
|
+
========================================== ===================
|
|
116
|
+
``apache-airflow`` ``>=2.10.0``
|
|
117
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
118
|
+
``pydantic`` ``>=2.11.0``
|
|
119
|
+
``retryhttp`` ``>=1.2.0,!=1.3.0``
|
|
120
|
+
========================================== ===================
|
|
121
|
+
|
|
122
|
+
Cross provider package dependencies
|
|
123
|
+
-----------------------------------
|
|
124
|
+
|
|
125
|
+
Those are dependencies that might be needed in order to use all the features of the package.
|
|
126
|
+
You need to install the specified provider distributions in order to use them.
|
|
127
|
+
|
|
128
|
+
You can install such cross-provider dependencies when installing from PyPI. For example:
|
|
129
|
+
|
|
130
|
+
.. code-block:: bash
|
|
131
|
+
|
|
132
|
+
pip install apache-airflow-providers-edge3[common.compat]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
================================================================================================================== =================
|
|
136
|
+
Dependent package Extra
|
|
137
|
+
================================================================================================================== =================
|
|
138
|
+
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
139
|
+
================================================================================================================== =================
|
|
140
|
+
|
|
141
|
+
Downloading official packages
|
|
142
|
+
-----------------------------
|
|
143
|
+
|
|
144
|
+
You can download officially released packages and verify their checksums and signatures from the
|
|
145
|
+
`Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
|
|
146
|
+
|
|
147
|
+
* `The apache-airflow-providers-edge3 1.4.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_edge3-1.4.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_edge3-1.4.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_edge3-1.4.0.tar.gz.sha512>`__)
|
|
148
|
+
* `The apache-airflow-providers-edge3 1.4.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_edge3-1.4.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_edge3-1.4.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_edge3-1.4.0-py3-none-any.whl.sha512>`__)
|
{apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/docs/ui_plugin.rst
RENAMED
|
@@ -22,7 +22,7 @@ The Edge provider uses a Plugin to
|
|
|
22
22
|
|
|
23
23
|
- Extend the REST API endpoints for connecting workers to the Airflow cluster
|
|
24
24
|
- Provide a web UI for managing the workers and monitoring their status and tasks
|
|
25
|
-
(Note: Airflow 3.0 does not have support for UI plugins. The UI plugin is only available in Airflow 2.
|
|
25
|
+
(Note: Airflow 3.0 does not have support for UI plugins. The UI plugin is only available in Airflow 2.x and in 3.1 and newer.)
|
|
26
26
|
|
|
27
27
|
REST API endpoints
|
|
28
28
|
------------------
|
|
@@ -35,14 +35,14 @@ The Edge provider adds the following REST API endpoints to the Airflow API:
|
|
|
35
35
|
- ``/edge_worker/v1/health``: Check that the API endpoint is deployed and active
|
|
36
36
|
|
|
37
37
|
To see full documentation of the API endpoints open the Airflow web UI and navigate to
|
|
38
|
-
the sub-path ``/edge_worker/docs`` (Airflow 3.0) or ``/edge_worker/v1/ui`` (Airflow 2.
|
|
38
|
+
the sub-path ``/edge_worker/docs`` (Airflow 3.0) or ``/edge_worker/v1/ui`` (Airflow 2.x).
|
|
39
39
|
|
|
40
40
|
Web UI Plugin
|
|
41
41
|
-------------
|
|
42
42
|
|
|
43
43
|
.. note::
|
|
44
44
|
|
|
45
|
-
Airflow 3.0 does not support UI plugins. The UI plugin is only available in Airflow 2.
|
|
45
|
+
Airflow 3.0 does not support UI plugins. The UI plugin is only available in Airflow 2.x and in 3.1 and newer.
|
|
46
46
|
Alternatively you can use the CLI commands as described in :ref:`deployment:maintenance-mgmt-cli`.
|
|
47
47
|
|
|
48
48
|
The Edge provider adds a web UI plugin to the Airflow web UI. The plugin is
|
{apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/provider.yaml
RENAMED
|
@@ -33,13 +33,14 @@ description: |
|
|
|
33
33
|
are provided by the API server.
|
|
34
34
|
|
|
35
35
|
state: ready
|
|
36
|
-
source-date-epoch:
|
|
36
|
+
source-date-epoch: 1761116659
|
|
37
37
|
|
|
38
38
|
# Note that those versions are maintained by release manager - do not update them manually
|
|
39
39
|
# with the exception of case where other provider in sources has >= new provider version.
|
|
40
40
|
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
41
41
|
# to be done in the same PR
|
|
42
42
|
versions:
|
|
43
|
+
- 1.4.0
|
|
43
44
|
- 1.3.1
|
|
44
45
|
- 1.3.0
|
|
45
46
|
- 1.2.0
|
{apache_airflow_providers_edge3-1.3.1 → apache_airflow_providers_edge3-1.4.0rc1}/pyproject.toml
RENAMED
|
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
|
|
|
25
25
|
|
|
26
26
|
[project]
|
|
27
27
|
name = "apache-airflow-providers-edge3"
|
|
28
|
-
version = "1.
|
|
28
|
+
version = "1.4.0rc1"
|
|
29
29
|
description = "Provider package apache-airflow-providers-edge3 for Apache Airflow"
|
|
30
30
|
readme = "README.rst"
|
|
31
31
|
authors = [
|
|
@@ -57,7 +57,8 @@ requires-python = ">=3.10"
|
|
|
57
57
|
# Make sure to run ``prek update-providers-dependencies --all-files``
|
|
58
58
|
# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
|
|
59
59
|
dependencies = [
|
|
60
|
-
"apache-airflow>=2.10.
|
|
60
|
+
"apache-airflow>=2.10.0rc1",
|
|
61
|
+
"apache-airflow-providers-common-compat>=1.8.0rc1",
|
|
61
62
|
"pydantic>=2.11.0",
|
|
62
63
|
"retryhttp>=1.2.0,!=1.3.0",
|
|
63
64
|
]
|
|
@@ -67,6 +68,7 @@ dev = [
|
|
|
67
68
|
"apache-airflow",
|
|
68
69
|
"apache-airflow-task-sdk",
|
|
69
70
|
"apache-airflow-devel-common",
|
|
71
|
+
"apache-airflow-providers-common-compat",
|
|
70
72
|
# Additional devel dependencies (do not remove this line and add extra development dependencies)
|
|
71
73
|
]
|
|
72
74
|
|
|
@@ -96,8 +98,8 @@ apache-airflow-providers-common-sql = {workspace = true}
|
|
|
96
98
|
apache-airflow-providers-standard = {workspace = true}
|
|
97
99
|
|
|
98
100
|
[project.urls]
|
|
99
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.
|
|
100
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-edge3/1.
|
|
101
|
+
"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-edge3/1.4.0"
|
|
102
|
+
"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-edge3/1.4.0/changelog.html"
|
|
101
103
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
102
104
|
"Source Code" = "https://github.com/apache/airflow"
|
|
103
105
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "1.
|
|
32
|
+
__version__ = "1.4.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.10.0"
|
|
@@ -34,6 +34,7 @@ from requests import HTTPError
|
|
|
34
34
|
|
|
35
35
|
from airflow import __version__ as airflow_version
|
|
36
36
|
from airflow.configuration import conf
|
|
37
|
+
from airflow.providers.common.compat.sdk import timezone
|
|
37
38
|
from airflow.providers.edge3 import __version__ as edge_provider_version
|
|
38
39
|
from airflow.providers.edge3.cli.api_client import (
|
|
39
40
|
jobs_fetch,
|
|
@@ -52,7 +53,6 @@ from airflow.providers.edge3.cli.signalling import (
|
|
|
52
53
|
)
|
|
53
54
|
from airflow.providers.edge3.models.edge_worker import EdgeWorkerState, EdgeWorkerVersionException
|
|
54
55
|
from airflow.providers.edge3.version_compat import AIRFLOW_V_3_0_PLUS
|
|
55
|
-
from airflow.utils import timezone
|
|
56
56
|
from airflow.utils.net import getfqdn
|
|
57
57
|
from airflow.utils.state import TaskInstanceState
|
|
58
58
|
|
|
@@ -67,7 +67,7 @@ if TYPE_CHECKING:
|
|
|
67
67
|
try:
|
|
68
68
|
from airflow.operators.python import PythonOperator
|
|
69
69
|
except ImportError:
|
|
70
|
-
from airflow.providers.common.compat.standard.operators import PythonOperator
|
|
70
|
+
from airflow.providers.common.compat.standard.operators import PythonOperator # type: ignore[no-redef]
|
|
71
71
|
|
|
72
72
|
|
|
73
73
|
class CmdOperator(BaseOperator):
|
|
@@ -31,13 +31,13 @@ from airflow.cli.cli_config import GroupCommand
|
|
|
31
31
|
from airflow.configuration import conf
|
|
32
32
|
from airflow.executors.base_executor import BaseExecutor
|
|
33
33
|
from airflow.models.taskinstance import TaskInstance, TaskInstanceState
|
|
34
|
+
from airflow.providers.common.compat.sdk import timezone
|
|
34
35
|
from airflow.providers.edge3.cli.edge_command import EDGE_COMMANDS
|
|
35
36
|
from airflow.providers.edge3.models.edge_job import EdgeJobModel
|
|
36
37
|
from airflow.providers.edge3.models.edge_logs import EdgeLogsModel
|
|
37
38
|
from airflow.providers.edge3.models.edge_worker import EdgeWorkerModel, EdgeWorkerState, reset_metrics
|
|
38
39
|
from airflow.providers.edge3.version_compat import AIRFLOW_V_3_0_PLUS
|
|
39
40
|
from airflow.stats import Stats
|
|
40
|
-
from airflow.utils import timezone
|
|
41
41
|
from airflow.utils.db import DBLocks, create_global_lock
|
|
42
42
|
from airflow.utils.session import NEW_SESSION, provide_session
|
|
43
43
|
|
|
@@ -352,7 +352,7 @@ class EdgeExecutor(BaseExecutor):
|
|
|
352
352
|
del self.last_reported_state[job.key]
|
|
353
353
|
self.fail(job.key)
|
|
354
354
|
else:
|
|
355
|
-
self.last_reported_state[job.key] = job.state
|
|
355
|
+
self.last_reported_state[job.key] = TaskInstanceState(job.state)
|
|
356
356
|
if (
|
|
357
357
|
job.state == TaskInstanceState.SUCCESS
|
|
358
358
|
and job.last_update_t < (datetime.now() - timedelta(minutes=job_success_purge)).timestamp()
|
|
@@ -399,6 +399,35 @@ class EdgeExecutor(BaseExecutor):
|
|
|
399
399
|
def terminate(self):
|
|
400
400
|
"""Terminate the executor is not doing anything."""
|
|
401
401
|
|
|
402
|
+
@provide_session
|
|
403
|
+
def revoke_task(self, *, ti: TaskInstance, session: Session = NEW_SESSION):
|
|
404
|
+
"""
|
|
405
|
+
Revoke a task instance from the executor.
|
|
406
|
+
|
|
407
|
+
This method removes the task from the executor's internal state and deletes
|
|
408
|
+
the corresponding EdgeJobModel record to prevent edge workers from picking it up.
|
|
409
|
+
|
|
410
|
+
:param ti: Task instance to revoke
|
|
411
|
+
:param session: Database session
|
|
412
|
+
"""
|
|
413
|
+
# Remove from executor's internal state
|
|
414
|
+
self.running.discard(ti.key)
|
|
415
|
+
self.queued_tasks.pop(ti.key, None)
|
|
416
|
+
if ti.key in self.last_reported_state:
|
|
417
|
+
del self.last_reported_state[ti.key]
|
|
418
|
+
|
|
419
|
+
# Delete the job from the database to prevent edge workers from picking it up
|
|
420
|
+
session.execute(
|
|
421
|
+
delete(EdgeJobModel).where(
|
|
422
|
+
EdgeJobModel.dag_id == ti.dag_id,
|
|
423
|
+
EdgeJobModel.task_id == ti.task_id,
|
|
424
|
+
EdgeJobModel.run_id == ti.run_id,
|
|
425
|
+
EdgeJobModel.map_index == ti.map_index,
|
|
426
|
+
EdgeJobModel.try_number == ti.try_number,
|
|
427
|
+
)
|
|
428
|
+
)
|
|
429
|
+
self.log.info("Revoked task instance %s from EdgeExecutor", ti.key)
|
|
430
|
+
|
|
402
431
|
def try_adopt_task_instances(self, tis: Sequence[TaskInstance]) -> Sequence[TaskInstance]:
|
|
403
432
|
"""
|
|
404
433
|
Try to adopt running task instances that have been abandoned by a SchedulerJob dying.
|