apache-airflow-providers-standard 1.0.0.dev1__py3-none-any.whl → 1.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of apache-airflow-providers-standard might be problematic. Click here for more details.

Files changed (50) hide show
  1. airflow/providers/standard/LICENSE +0 -52
  2. airflow/providers/standard/__init__.py +23 -1
  3. airflow/providers/standard/decorators/__init__.py +16 -0
  4. airflow/providers/standard/decorators/bash.py +121 -0
  5. airflow/providers/standard/decorators/branch_external_python.py +63 -0
  6. airflow/providers/standard/decorators/branch_python.py +62 -0
  7. airflow/providers/standard/decorators/branch_virtualenv.py +62 -0
  8. airflow/providers/standard/decorators/external_python.py +70 -0
  9. airflow/providers/standard/decorators/python.py +86 -0
  10. airflow/providers/standard/decorators/python_virtualenv.py +67 -0
  11. airflow/providers/standard/decorators/sensor.py +83 -0
  12. airflow/providers/standard/decorators/short_circuit.py +65 -0
  13. airflow/providers/standard/get_provider_info.py +78 -7
  14. airflow/providers/standard/hooks/__init__.py +16 -0
  15. airflow/providers/standard/hooks/filesystem.py +89 -0
  16. airflow/providers/standard/hooks/package_index.py +95 -0
  17. airflow/providers/standard/hooks/subprocess.py +119 -0
  18. airflow/providers/standard/operators/bash.py +73 -56
  19. airflow/providers/standard/operators/branch.py +105 -0
  20. airflow/providers/standard/operators/datetime.py +15 -5
  21. airflow/providers/standard/operators/empty.py +39 -0
  22. airflow/providers/standard/operators/latest_only.py +115 -0
  23. airflow/providers/standard/operators/python.py +1143 -0
  24. airflow/providers/standard/operators/smooth.py +38 -0
  25. airflow/providers/standard/operators/trigger_dagrun.py +370 -0
  26. airflow/providers/standard/operators/weekday.py +19 -9
  27. airflow/providers/standard/sensors/bash.py +15 -11
  28. airflow/providers/standard/sensors/date_time.py +32 -8
  29. airflow/providers/standard/sensors/external_task.py +593 -0
  30. airflow/providers/standard/sensors/filesystem.py +158 -0
  31. airflow/providers/standard/sensors/python.py +84 -0
  32. airflow/providers/standard/sensors/time.py +28 -5
  33. airflow/providers/standard/sensors/time_delta.py +68 -15
  34. airflow/providers/standard/sensors/weekday.py +25 -7
  35. airflow/providers/standard/triggers/__init__.py +16 -0
  36. airflow/providers/standard/triggers/external_task.py +288 -0
  37. airflow/providers/standard/triggers/file.py +131 -0
  38. airflow/providers/standard/triggers/temporal.py +113 -0
  39. airflow/providers/standard/utils/__init__.py +16 -0
  40. airflow/providers/standard/utils/python_virtualenv.py +209 -0
  41. airflow/providers/standard/utils/python_virtualenv_script.jinja2 +82 -0
  42. airflow/providers/standard/utils/sensor_helper.py +137 -0
  43. airflow/providers/standard/utils/skipmixin.py +192 -0
  44. airflow/providers/standard/utils/weekday.py +77 -0
  45. airflow/providers/standard/version_compat.py +36 -0
  46. {apache_airflow_providers_standard-1.0.0.dev1.dist-info → apache_airflow_providers_standard-1.0.0rc1.dist-info}/METADATA +12 -31
  47. apache_airflow_providers_standard-1.0.0rc1.dist-info/RECORD +51 -0
  48. {apache_airflow_providers_standard-1.0.0.dev1.dist-info → apache_airflow_providers_standard-1.0.0rc1.dist-info}/WHEEL +1 -1
  49. apache_airflow_providers_standard-1.0.0.dev1.dist-info/RECORD +0 -17
  50. {apache_airflow_providers_standard-1.0.0.dev1.dist-info → apache_airflow_providers_standard-1.0.0rc1.dist-info}/entry_points.txt +0 -0
@@ -199,55 +199,3 @@ distributed under the License is distributed on an "AS IS" BASIS,
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
-
203
- ============================================================================
204
- APACHE AIRFLOW SUBCOMPONENTS:
205
-
206
- The Apache Airflow project contains subcomponents with separate copyright
207
- notices and license terms. Your use of the source code for the these
208
- subcomponents is subject to the terms and conditions of the following
209
- licenses.
210
-
211
-
212
- ========================================================================
213
- Third party Apache 2.0 licenses
214
- ========================================================================
215
-
216
- The following components are provided under the Apache 2.0 License.
217
- See project link for details. The text of each license is also included
218
- at 3rd-party-licenses/LICENSE-[project].txt.
219
-
220
- (ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
221
- (ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
222
- (ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
223
- (ALv2 License) connexion v2.7.0 (https://github.com/zalando/connexion)
224
-
225
- ========================================================================
226
- MIT licenses
227
- ========================================================================
228
-
229
- The following components are provided under the MIT License. See project link for details.
230
- The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
231
-
232
- (MIT License) jquery v3.5.1 (https://jquery.org/license/)
233
- (MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
234
- (MIT License) bootstrap v3.4.1 (https://github.com/twbs/bootstrap/)
235
- (MIT License) d3-tip v0.9.1 (https://github.com/Caged/d3-tip)
236
- (MIT License) dataTables v1.10.25 (https://datatables.net)
237
- (MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
238
- (MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
239
- (MIT License) MomentJS v2.24.0 (http://momentjs.com/)
240
- (MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/)
241
-
242
- ========================================================================
243
- BSD 3-Clause licenses
244
- ========================================================================
245
- The following components are provided under the BSD 3-Clause license. See project links for details.
246
- The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
247
-
248
- (BSD 3 License) d3 v5.16.0 (https://d3js.org)
249
- (BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
250
- (BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
251
-
252
- ========================================================================
253
- See 3rd-party-licenses/LICENSES-ui.txt for packages used in `/airflow/www`
@@ -1,4 +1,3 @@
1
- #
2
1
  # Licensed to the Apache Software Foundation (ASF) under one
3
2
  # or more contributor license agreements. See the NOTICE file
4
3
  # distributed with this work for additional information
@@ -15,3 +14,26 @@
15
14
  # KIND, either express or implied. See the License for the
16
15
  # specific language governing permissions and limitations
17
16
  # under the License.
17
+ #
18
+ # NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
19
+ # OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
20
+ #
21
+ # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
22
+ # `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
23
+ #
24
+ from __future__ import annotations
25
+
26
+ import packaging.version
27
+
28
+ from airflow import __version__ as airflow_version
29
+
30
+ __all__ = ["__version__"]
31
+
32
+ __version__ = "1.0.0"
33
+
34
+ if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
+ "2.9.0"
36
+ ):
37
+ raise RuntimeError(
38
+ f"The package `apache-airflow-providers-standard:{__version__}` needs Apache Airflow 2.9.0+"
39
+ )
@@ -0,0 +1,16 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
@@ -0,0 +1,121 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ from __future__ import annotations
19
+
20
+ import warnings
21
+ from collections.abc import Collection, Mapping, Sequence
22
+ from typing import TYPE_CHECKING, Any, Callable, ClassVar
23
+
24
+ from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
25
+
26
+ if AIRFLOW_V_3_0_PLUS:
27
+ from airflow.sdk.bases.decorator import DecoratedOperator, TaskDecorator, task_decorator_factory
28
+ else:
29
+ from airflow.decorators.base import ( # type: ignore[no-redef]
30
+ DecoratedOperator,
31
+ TaskDecorator,
32
+ task_decorator_factory,
33
+ )
34
+
35
+ from airflow.providers.standard.operators.bash import BashOperator
36
+ from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION
37
+ from airflow.utils.context import context_merge
38
+ from airflow.utils.operator_helpers import determine_kwargs
39
+
40
+ if TYPE_CHECKING:
41
+ from airflow.sdk.definitions.context import Context
42
+
43
+
44
+ class _BashDecoratedOperator(DecoratedOperator, BashOperator):
45
+ """
46
+ Wraps a Python callable and uses the callable return value as the Bash command to be executed.
47
+
48
+ :param python_callable: A reference to an object that is callable.
49
+ :param op_kwargs: A dictionary of keyword arguments that will get unpacked
50
+ in your function (templated).
51
+ :param op_args: A list of positional arguments that will get unpacked when
52
+ calling your callable (templated).
53
+ """
54
+
55
+ template_fields: Sequence[str] = (*DecoratedOperator.template_fields, *BashOperator.template_fields)
56
+ template_fields_renderers: ClassVar[dict[str, str]] = {
57
+ **DecoratedOperator.template_fields_renderers,
58
+ **BashOperator.template_fields_renderers,
59
+ }
60
+
61
+ custom_operator_name: str = "@task.bash"
62
+ overwrite_rtif_after_execution: bool = True
63
+
64
+ def __init__(
65
+ self,
66
+ *,
67
+ python_callable: Callable,
68
+ op_args: Collection[Any] | None = None,
69
+ op_kwargs: Mapping[str, Any] | None = None,
70
+ **kwargs,
71
+ ) -> None:
72
+ if kwargs.pop("multiple_outputs", None):
73
+ warnings.warn(
74
+ f"`multiple_outputs=True` is not supported in {self.custom_operator_name} tasks. Ignoring.",
75
+ UserWarning,
76
+ stacklevel=3,
77
+ )
78
+
79
+ super().__init__(
80
+ python_callable=python_callable,
81
+ op_args=op_args,
82
+ op_kwargs=op_kwargs,
83
+ bash_command=SET_DURING_EXECUTION,
84
+ multiple_outputs=False,
85
+ **kwargs,
86
+ )
87
+
88
+ def execute(self, context: Context) -> Any:
89
+ context_merge(context, self.op_kwargs)
90
+ kwargs = determine_kwargs(self.python_callable, self.op_args, context)
91
+
92
+ self.bash_command = self.python_callable(*self.op_args, **kwargs)
93
+
94
+ if not isinstance(self.bash_command, str) or self.bash_command.strip() == "":
95
+ raise TypeError("The returned value from the TaskFlow callable must be a non-empty string.")
96
+
97
+ self._is_inline_cmd = self._is_inline_command(bash_command=self.bash_command)
98
+ context["ti"].render_templates() # type: ignore[attr-defined]
99
+
100
+ return super().execute(context)
101
+
102
+
103
+ def bash_task(
104
+ python_callable: Callable | None = None,
105
+ **kwargs,
106
+ ) -> TaskDecorator:
107
+ """
108
+ Wrap a function into a BashOperator.
109
+
110
+ Accepts kwargs for operator kwargs. Can be reused in a single DAG. This function is only used only used
111
+ during type checking or auto-completion.
112
+
113
+ :param python_callable: Function to decorate.
114
+
115
+ :meta private:
116
+ """
117
+ return task_decorator_factory(
118
+ python_callable=python_callable,
119
+ decorated_operator_class=_BashDecoratedOperator,
120
+ **kwargs,
121
+ )
@@ -0,0 +1,63 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from typing import TYPE_CHECKING, Callable
20
+
21
+ from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
22
+
23
+ if AIRFLOW_V_3_0_PLUS:
24
+ from airflow.sdk.bases.decorator import task_decorator_factory
25
+ else:
26
+ from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
27
+
28
+ from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
29
+ from airflow.providers.standard.operators.python import BranchExternalPythonOperator
30
+
31
+ if TYPE_CHECKING:
32
+ from airflow.sdk.bases.decorator import TaskDecorator
33
+
34
+
35
+ class _BranchExternalPythonDecoratedOperator(_PythonDecoratedOperator, BranchExternalPythonOperator):
36
+ """Wraps a Python callable and captures args/kwargs when called for execution."""
37
+
38
+ template_fields = BranchExternalPythonOperator.template_fields
39
+ custom_operator_name: str = "@task.branch_external_python"
40
+
41
+
42
+ def branch_external_python_task(
43
+ python_callable: Callable | None = None, multiple_outputs: bool | None = None, **kwargs
44
+ ) -> TaskDecorator:
45
+ """
46
+ Wrap a python function into a BranchExternalPythonOperator.
47
+
48
+ For more information on how to use this operator, take a look at the guide:
49
+ :ref:`concepts:branching`
50
+
51
+ Accepts kwargs for operator kwarg. Can be reused in a single DAG.
52
+
53
+ :param python_callable: Function to decorate
54
+ :param multiple_outputs: if set, function return value will be
55
+ unrolled to multiple XCom values. Dict will unroll to xcom values with keys as XCom keys.
56
+ Defaults to False.
57
+ """
58
+ return task_decorator_factory(
59
+ python_callable=python_callable,
60
+ multiple_outputs=multiple_outputs,
61
+ decorated_operator_class=_BranchExternalPythonDecoratedOperator,
62
+ **kwargs,
63
+ )
@@ -0,0 +1,62 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from typing import TYPE_CHECKING, Callable
20
+
21
+ from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
22
+
23
+ if AIRFLOW_V_3_0_PLUS:
24
+ from airflow.sdk.bases.decorator import task_decorator_factory
25
+ else:
26
+ from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
27
+ from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
28
+ from airflow.providers.standard.operators.python import BranchPythonOperator
29
+
30
+ if TYPE_CHECKING:
31
+ from airflow.sdk.bases.decorator import TaskDecorator
32
+
33
+
34
+ class _BranchPythonDecoratedOperator(_PythonDecoratedOperator, BranchPythonOperator):
35
+ """Wraps a Python callable and captures args/kwargs when called for execution."""
36
+
37
+ template_fields = BranchPythonOperator.template_fields
38
+ custom_operator_name: str = "@task.branch"
39
+
40
+
41
+ def branch_task(
42
+ python_callable: Callable | None = None, multiple_outputs: bool | None = None, **kwargs
43
+ ) -> TaskDecorator:
44
+ """
45
+ Wrap a python function into a BranchPythonOperator.
46
+
47
+ For more information on how to use this operator, take a look at the guide:
48
+ :ref:`concepts:branching`
49
+
50
+ Accepts kwargs for operator kwarg. Can be reused in a single DAG.
51
+
52
+ :param python_callable: Function to decorate
53
+ :param multiple_outputs: if set, function return value will be
54
+ unrolled to multiple XCom values. Dict will unroll to xcom values with keys as XCom keys.
55
+ Defaults to False.
56
+ """
57
+ return task_decorator_factory(
58
+ python_callable=python_callable,
59
+ multiple_outputs=multiple_outputs,
60
+ decorated_operator_class=_BranchPythonDecoratedOperator,
61
+ **kwargs,
62
+ )
@@ -0,0 +1,62 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from typing import TYPE_CHECKING, Callable
20
+
21
+ from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
22
+
23
+ if AIRFLOW_V_3_0_PLUS:
24
+ from airflow.sdk.bases.decorator import task_decorator_factory
25
+ else:
26
+ from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
27
+ from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
28
+ from airflow.providers.standard.operators.python import BranchPythonVirtualenvOperator
29
+
30
+ if TYPE_CHECKING:
31
+ from airflow.sdk.bases.decorator import TaskDecorator
32
+
33
+
34
+ class _BranchPythonVirtualenvDecoratedOperator(_PythonDecoratedOperator, BranchPythonVirtualenvOperator):
35
+ """Wraps a Python callable and captures args/kwargs when called for execution."""
36
+
37
+ template_fields = BranchPythonVirtualenvOperator.template_fields
38
+ custom_operator_name: str = "@task.branch_virtualenv"
39
+
40
+
41
+ def branch_virtualenv_task(
42
+ python_callable: Callable | None = None, multiple_outputs: bool | None = None, **kwargs
43
+ ) -> TaskDecorator:
44
+ """
45
+ Wrap a python function into a BranchPythonVirtualenvOperator.
46
+
47
+ For more information on how to use this operator, take a look at the guide:
48
+ :ref:`concepts:branching`
49
+
50
+ Accepts kwargs for operator kwarg. Can be reused in a single DAG.
51
+
52
+ :param python_callable: Function to decorate
53
+ :param multiple_outputs: if set, function return value will be
54
+ unrolled to multiple XCom values. Dict will unroll to xcom values with keys as XCom keys.
55
+ Defaults to False.
56
+ """
57
+ return task_decorator_factory(
58
+ python_callable=python_callable,
59
+ multiple_outputs=multiple_outputs,
60
+ decorated_operator_class=_BranchPythonVirtualenvDecoratedOperator,
61
+ **kwargs,
62
+ )
@@ -0,0 +1,70 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from typing import TYPE_CHECKING, Callable
20
+
21
+ from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
22
+
23
+ if AIRFLOW_V_3_0_PLUS:
24
+ from airflow.sdk.bases.decorator import task_decorator_factory
25
+ else:
26
+ from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
27
+ from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
28
+ from airflow.providers.standard.operators.python import ExternalPythonOperator
29
+
30
+ if TYPE_CHECKING:
31
+ from airflow.sdk.bases.decorator import TaskDecorator
32
+
33
+
34
+ class _PythonExternalDecoratedOperator(_PythonDecoratedOperator, ExternalPythonOperator):
35
+ """Wraps a Python callable and captures args/kwargs when called for execution."""
36
+
37
+ template_fields = ExternalPythonOperator.template_fields
38
+ custom_operator_name: str = "@task.external_python"
39
+
40
+
41
+ def external_python_task(
42
+ python: str | None = None,
43
+ python_callable: Callable | None = None,
44
+ multiple_outputs: bool | None = None,
45
+ **kwargs,
46
+ ) -> TaskDecorator:
47
+ """
48
+ Wrap a callable into an Airflow operator to run via a Python virtual environment.
49
+
50
+ Accepts kwargs for operator kwarg. Can be reused in a single DAG.
51
+
52
+ This function is only used during type checking or auto-completion.
53
+
54
+ :meta private:
55
+
56
+ :param python: Full path string (file-system specific) that points to a Python binary inside
57
+ a virtualenv that should be used (in ``VENV/bin`` folder). Should be absolute path
58
+ (so usually start with "/" or "X:/" depending on the filesystem/os used).
59
+ :param python_callable: Function to decorate
60
+ :param multiple_outputs: If set to True, the decorated function's return value will be unrolled to
61
+ multiple XCom values. Dict will unroll to XCom values with its keys as XCom keys.
62
+ Defaults to False.
63
+ """
64
+ return task_decorator_factory(
65
+ python=python,
66
+ python_callable=python_callable,
67
+ multiple_outputs=multiple_outputs,
68
+ decorated_operator_class=_PythonExternalDecoratedOperator,
69
+ **kwargs,
70
+ )
@@ -0,0 +1,86 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from collections.abc import Sequence
20
+ from typing import TYPE_CHECKING, Callable
21
+
22
+ from airflow.providers.standard.operators.python import PythonOperator
23
+ from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
24
+
25
+ if AIRFLOW_V_3_0_PLUS:
26
+ from airflow.sdk.bases.decorator import DecoratedOperator, task_decorator_factory
27
+ else:
28
+ from airflow.decorators.base import DecoratedOperator, task_decorator_factory # type: ignore[no-redef]
29
+
30
+ if TYPE_CHECKING:
31
+ from airflow.sdk.bases.decorator import TaskDecorator
32
+
33
+
34
+ class _PythonDecoratedOperator(DecoratedOperator, PythonOperator):
35
+ """
36
+ Wraps a Python callable and captures args/kwargs when called for execution.
37
+
38
+ :param python_callable: A reference to an object that is callable
39
+ :param op_kwargs: a dictionary of keyword arguments that will get unpacked
40
+ in your function (templated)
41
+ :param op_args: a list of positional arguments that will get unpacked when
42
+ calling your callable (templated)
43
+ :param multiple_outputs: If set to True, the decorated function's return value will be unrolled to
44
+ multiple XCom values. Dict will unroll to XCom values with its keys as XCom keys. Defaults to False.
45
+ """
46
+
47
+ template_fields: Sequence[str] = ("templates_dict", "op_args", "op_kwargs")
48
+ template_fields_renderers = {"templates_dict": "json", "op_args": "py", "op_kwargs": "py"}
49
+
50
+ custom_operator_name: str = "@task"
51
+
52
+ def __init__(self, *, python_callable, op_args, op_kwargs, **kwargs) -> None:
53
+ kwargs_to_upstream = {
54
+ "python_callable": python_callable,
55
+ "op_args": op_args,
56
+ "op_kwargs": op_kwargs,
57
+ }
58
+ super().__init__(
59
+ kwargs_to_upstream=kwargs_to_upstream,
60
+ python_callable=python_callable,
61
+ op_args=op_args,
62
+ op_kwargs=op_kwargs,
63
+ **kwargs,
64
+ )
65
+
66
+
67
+ def python_task(
68
+ python_callable: Callable | None = None,
69
+ multiple_outputs: bool | None = None,
70
+ **kwargs,
71
+ ) -> TaskDecorator:
72
+ """
73
+ Wrap a function into an Airflow operator.
74
+
75
+ Accepts kwargs for operator kwarg. Can be reused in a single DAG.
76
+
77
+ :param python_callable: Function to decorate
78
+ :param multiple_outputs: If set to True, the decorated function's return value will be unrolled to
79
+ multiple XCom values. Dict will unroll to XCom values with its keys as XCom keys. Defaults to False.
80
+ """
81
+ return task_decorator_factory(
82
+ python_callable=python_callable,
83
+ multiple_outputs=multiple_outputs,
84
+ decorated_operator_class=_PythonDecoratedOperator,
85
+ **kwargs,
86
+ )
@@ -0,0 +1,67 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from typing import TYPE_CHECKING, Callable
20
+
21
+ from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
22
+
23
+ if AIRFLOW_V_3_0_PLUS:
24
+ from airflow.sdk.bases.decorator import task_decorator_factory
25
+ else:
26
+ from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
27
+
28
+
29
+ from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
30
+ from airflow.providers.standard.operators.python import PythonVirtualenvOperator
31
+
32
+ if TYPE_CHECKING:
33
+ from airflow.sdk.bases.decorator import TaskDecorator
34
+
35
+
36
+ class _PythonVirtualenvDecoratedOperator(_PythonDecoratedOperator, PythonVirtualenvOperator):
37
+ """Wraps a Python callable and captures args/kwargs when called for execution."""
38
+
39
+ template_fields = PythonVirtualenvOperator.template_fields
40
+ custom_operator_name: str = "@task.virtualenv"
41
+
42
+
43
+ def virtualenv_task(
44
+ python_callable: Callable | None = None,
45
+ multiple_outputs: bool | None = None,
46
+ **kwargs,
47
+ ) -> TaskDecorator:
48
+ """
49
+ Wrap a callable into an Airflow operator to run via a Python virtual environment.
50
+
51
+ Accepts kwargs for operator kwarg. Can be reused in a single DAG.
52
+
53
+ This function is only used only used during type checking or auto-completion.
54
+
55
+ :meta private:
56
+
57
+ :param python_callable: Function to decorate
58
+ :param multiple_outputs: If set to True, the decorated function's return value will be unrolled to
59
+ multiple XCom values. Dict will unroll to XCom values with its keys as XCom keys.
60
+ Defaults to False.
61
+ """
62
+ return task_decorator_factory(
63
+ python_callable=python_callable,
64
+ multiple_outputs=multiple_outputs,
65
+ decorated_operator_class=_PythonVirtualenvDecoratedOperator,
66
+ **kwargs,
67
+ )