apache-airflow-providers-standard 1.5.0rc1__py3-none-any.whl → 1.6.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/standard/__init__.py +1 -1
- airflow/providers/standard/decorators/bash.py +1 -1
- airflow/providers/standard/example_dags/example_external_task_parent_deferrable.py +5 -1
- airflow/providers/standard/example_dags/example_hitl_operator.py +93 -4
- airflow/providers/standard/operators/hitl.py +38 -12
- airflow/providers/standard/operators/python.py +5 -5
- airflow/providers/standard/sensors/python.py +2 -3
- airflow/providers/standard/sensors/time.py +8 -3
- airflow/providers/standard/triggers/hitl.py +11 -3
- airflow/providers/standard/triggers/temporal.py +5 -1
- airflow/providers/standard/utils/python_virtualenv.py +10 -0
- airflow/providers/standard/version_compat.py +3 -0
- {apache_airflow_providers_standard-1.5.0rc1.dist-info → apache_airflow_providers_standard-1.6.0rc2.dist-info}/METADATA +6 -6
- {apache_airflow_providers_standard-1.5.0rc1.dist-info → apache_airflow_providers_standard-1.6.0rc2.dist-info}/RECORD +16 -16
- {apache_airflow_providers_standard-1.5.0rc1.dist-info → apache_airflow_providers_standard-1.6.0rc2.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_standard-1.5.0rc1.dist-info → apache_airflow_providers_standard-1.6.0rc2.dist-info}/entry_points.txt +0 -0
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "1.
|
|
32
|
+
__version__ = "1.6.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.10.0"
|
|
@@ -33,8 +33,8 @@ else:
|
|
|
33
33
|
)
|
|
34
34
|
|
|
35
35
|
from airflow.providers.standard.operators.bash import BashOperator
|
|
36
|
+
from airflow.providers.standard.version_compat import context_merge
|
|
36
37
|
from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION
|
|
37
|
-
from airflow.utils.context import context_merge
|
|
38
38
|
from airflow.utils.operator_helpers import determine_kwargs
|
|
39
39
|
|
|
40
40
|
if TYPE_CHECKING:
|
|
@@ -20,7 +20,11 @@ from airflow import DAG
|
|
|
20
20
|
from airflow.providers.standard.operators.empty import EmptyOperator
|
|
21
21
|
from airflow.providers.standard.operators.trigger_dagrun import TriggerDagRunOperator
|
|
22
22
|
from airflow.providers.standard.sensors.external_task import ExternalTaskSensor
|
|
23
|
-
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
from airflow.sdk.timezone import datetime
|
|
26
|
+
except ImportError:
|
|
27
|
+
from airflow.utils.timezone import datetime # type: ignore[no-redef]
|
|
24
28
|
|
|
25
29
|
with DAG(
|
|
26
30
|
dag_id="example_external_task",
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import datetime
|
|
21
|
+
from typing import TYPE_CHECKING
|
|
21
22
|
|
|
22
23
|
import pendulum
|
|
23
24
|
|
|
@@ -28,6 +29,43 @@ from airflow.providers.standard.operators.hitl import (
|
|
|
28
29
|
HITLOperator,
|
|
29
30
|
)
|
|
30
31
|
from airflow.sdk import DAG, Param, task
|
|
32
|
+
from airflow.sdk.bases.notifier import BaseNotifier
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from airflow.sdk.definitions.context import Context
|
|
36
|
+
|
|
37
|
+
# [START hitl_tutorial]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class LocalLogNotifier(BaseNotifier):
|
|
41
|
+
"""Simple notifier to demonstrate HITL notification without setup any connection."""
|
|
42
|
+
|
|
43
|
+
template_fields = ("message",)
|
|
44
|
+
|
|
45
|
+
def __init__(self, message: str) -> None:
|
|
46
|
+
self.message = message
|
|
47
|
+
|
|
48
|
+
def notify(self, context: Context) -> None:
|
|
49
|
+
self.log.info(self.message)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# [START htil_notifer]
|
|
53
|
+
hitl_request_callback = LocalLogNotifier(
|
|
54
|
+
message="""
|
|
55
|
+
[HITL]
|
|
56
|
+
Subject: {{ task.subject }}
|
|
57
|
+
Body: {{ task.body }}
|
|
58
|
+
Options: {{ task.options }}
|
|
59
|
+
Is Multiple Option: {{ task.multiple }}
|
|
60
|
+
Default Options: {{ task.defaults }}
|
|
61
|
+
Params: {{ task.params }}
|
|
62
|
+
"""
|
|
63
|
+
)
|
|
64
|
+
hitl_success_callback = LocalLogNotifier(
|
|
65
|
+
message="{% set task_id = task.task_id -%}{{ ti.xcom_pull(task_ids=task_id) }}"
|
|
66
|
+
)
|
|
67
|
+
hitl_failure_callback = LocalLogNotifier(message="Request to response to '{{ task.subject }}' failed")
|
|
68
|
+
# [END htil_notifer]
|
|
31
69
|
|
|
32
70
|
with DAG(
|
|
33
71
|
dag_id="example_hitl_operator",
|
|
@@ -35,35 +73,83 @@ with DAG(
|
|
|
35
73
|
catchup=False,
|
|
36
74
|
tags=["example", "HITL"],
|
|
37
75
|
):
|
|
76
|
+
# [START howto_hitl_entry_operator]
|
|
38
77
|
wait_for_input = HITLEntryOperator(
|
|
39
78
|
task_id="wait_for_input",
|
|
40
79
|
subject="Please provide required information: ",
|
|
41
80
|
params={"information": Param("", type="string")},
|
|
81
|
+
notifiers=[hitl_request_callback],
|
|
82
|
+
on_success_callback=hitl_success_callback,
|
|
83
|
+
on_failure_callback=hitl_failure_callback,
|
|
42
84
|
)
|
|
85
|
+
# [END howto_hitl_entry_operator]
|
|
43
86
|
|
|
87
|
+
# [START howto_hitl_operator]
|
|
44
88
|
wait_for_option = HITLOperator(
|
|
45
89
|
task_id="wait_for_option",
|
|
46
|
-
subject="Please choose one option to
|
|
90
|
+
subject="Please choose one option to proceed: ",
|
|
47
91
|
options=["option 1", "option 2", "option 3"],
|
|
92
|
+
notifiers=[hitl_request_callback],
|
|
93
|
+
on_success_callback=hitl_success_callback,
|
|
94
|
+
on_failure_callback=hitl_failure_callback,
|
|
95
|
+
)
|
|
96
|
+
# [END howto_hitl_operator]
|
|
97
|
+
|
|
98
|
+
# [START howto_hitl_operator_muliple]
|
|
99
|
+
wait_for_multiple_options = HITLOperator(
|
|
100
|
+
task_id="wait_for_multiple_options",
|
|
101
|
+
subject="Please choose option to proceed: ",
|
|
102
|
+
options=["option 4", "option 5", "option 6"],
|
|
103
|
+
multiple=True,
|
|
104
|
+
notifiers=[hitl_request_callback],
|
|
105
|
+
on_success_callback=hitl_success_callback,
|
|
106
|
+
on_failure_callback=hitl_failure_callback,
|
|
107
|
+
)
|
|
108
|
+
# [END howto_hitl_operator_muliple]
|
|
109
|
+
|
|
110
|
+
# [START howto_hitl_operator_timeout]
|
|
111
|
+
wait_for_default_option = HITLOperator(
|
|
112
|
+
task_id="wait_for_default_option",
|
|
113
|
+
subject="Please choose option to proceed: ",
|
|
114
|
+
options=["option 7", "option 8", "option 9"],
|
|
115
|
+
defaults=["option 7"],
|
|
116
|
+
execution_timeout=datetime.timedelta(seconds=1),
|
|
117
|
+
notifiers=[hitl_request_callback],
|
|
118
|
+
on_success_callback=hitl_success_callback,
|
|
119
|
+
on_failure_callback=hitl_failure_callback,
|
|
48
120
|
)
|
|
121
|
+
# [END howto_hitl_operator_timeout]
|
|
49
122
|
|
|
123
|
+
# [START howto_hitl_approval_operator]
|
|
50
124
|
valid_input_and_options = ApprovalOperator(
|
|
51
125
|
task_id="valid_input_and_options",
|
|
52
126
|
subject="Are the following input and options valid?",
|
|
53
127
|
body="""
|
|
54
|
-
Input: {{
|
|
55
|
-
Option: {{
|
|
128
|
+
Input: {{ ti.xcom_pull(task_ids='wait_for_input')["params_input"]["information"] }}
|
|
129
|
+
Option: {{ ti.xcom_pull(task_ids='wait_for_option')["chosen_options"] }}
|
|
130
|
+
Multiple Options: {{ ti.xcom_pull(task_ids='wait_for_option')["chosen_options"] }}
|
|
131
|
+
Timeout Option: {{ ti.xcom_pull(task_ids='wait_for_option')["chosen_options"] }}
|
|
56
132
|
""",
|
|
57
133
|
defaults="Reject",
|
|
58
134
|
execution_timeout=datetime.timedelta(minutes=1),
|
|
135
|
+
notifiers=[hitl_request_callback],
|
|
136
|
+
on_success_callback=hitl_success_callback,
|
|
137
|
+
on_failure_callback=hitl_failure_callback,
|
|
59
138
|
)
|
|
139
|
+
# [END howto_hitl_approval_operator]
|
|
60
140
|
|
|
141
|
+
# [START howto_hitl_branch_operator]
|
|
61
142
|
choose_a_branch_to_run = HITLBranchOperator(
|
|
62
143
|
task_id="choose_a_branch_to_run",
|
|
63
144
|
subject="You're now allowed to proceeded. Please choose one task to run: ",
|
|
64
145
|
options=["task_1", "task_2", "task_3"],
|
|
146
|
+
notifiers=[hitl_request_callback],
|
|
147
|
+
on_success_callback=hitl_success_callback,
|
|
148
|
+
on_failure_callback=hitl_failure_callback,
|
|
65
149
|
)
|
|
150
|
+
# [END howto_hitl_branch_operator]
|
|
66
151
|
|
|
152
|
+
# [START howto_hitl_workflow]
|
|
67
153
|
@task
|
|
68
154
|
def task_1(): ...
|
|
69
155
|
|
|
@@ -74,8 +160,11 @@ with DAG(
|
|
|
74
160
|
def task_3(): ...
|
|
75
161
|
|
|
76
162
|
(
|
|
77
|
-
[wait_for_input, wait_for_option]
|
|
163
|
+
[wait_for_input, wait_for_option, wait_for_default_option, wait_for_multiple_options]
|
|
78
164
|
>> valid_input_and_options
|
|
79
165
|
>> choose_a_branch_to_run
|
|
80
166
|
>> [task_1(), task_2(), task_3()]
|
|
81
167
|
)
|
|
168
|
+
# [END howto_hitl_workflow]
|
|
169
|
+
|
|
170
|
+
# [END hitl_tutorial]
|
|
@@ -20,21 +20,23 @@ import logging
|
|
|
20
20
|
|
|
21
21
|
from airflow.exceptions import AirflowOptionalProviderFeatureException
|
|
22
22
|
from airflow.providers.standard.version_compat import AIRFLOW_V_3_1_PLUS
|
|
23
|
+
from airflow.sdk.bases.notifier import BaseNotifier
|
|
23
24
|
|
|
24
25
|
if not AIRFLOW_V_3_1_PLUS:
|
|
25
26
|
raise AirflowOptionalProviderFeatureException("Human in the loop functionality needs Airflow 3.1+.")
|
|
26
27
|
|
|
27
28
|
|
|
28
|
-
from collections.abc import Collection, Mapping
|
|
29
|
-
from datetime import datetime, timezone
|
|
29
|
+
from collections.abc import Collection, Mapping, Sequence
|
|
30
30
|
from typing import TYPE_CHECKING, Any
|
|
31
31
|
|
|
32
32
|
from airflow.providers.standard.exceptions import HITLTimeoutError, HITLTriggerEventError
|
|
33
|
+
from airflow.providers.standard.operators.branch import BranchMixIn
|
|
33
34
|
from airflow.providers.standard.triggers.hitl import HITLTrigger, HITLTriggerEventSuccessPayload
|
|
34
35
|
from airflow.providers.standard.utils.skipmixin import SkipMixin
|
|
35
36
|
from airflow.providers.standard.version_compat import BaseOperator
|
|
36
37
|
from airflow.sdk.definitions.param import ParamsDict
|
|
37
38
|
from airflow.sdk.execution_time.hitl import upsert_hitl_detail
|
|
39
|
+
from airflow.sdk.timezone import utcnow
|
|
38
40
|
|
|
39
41
|
if TYPE_CHECKING:
|
|
40
42
|
from airflow.sdk.definitions.context import Context
|
|
@@ -65,6 +67,7 @@ class HITLOperator(BaseOperator):
|
|
|
65
67
|
defaults: str | list[str] | None = None,
|
|
66
68
|
multiple: bool = False,
|
|
67
69
|
params: ParamsDict | dict[str, Any] | None = None,
|
|
70
|
+
notifiers: Sequence[BaseNotifier] | BaseNotifier | None = None,
|
|
68
71
|
**kwargs,
|
|
69
72
|
) -> None:
|
|
70
73
|
super().__init__(**kwargs)
|
|
@@ -77,9 +80,17 @@ class HITLOperator(BaseOperator):
|
|
|
77
80
|
self.multiple = multiple
|
|
78
81
|
|
|
79
82
|
self.params: ParamsDict = params if isinstance(params, ParamsDict) else ParamsDict(params or {})
|
|
83
|
+
self.notifiers: Sequence[BaseNotifier] = (
|
|
84
|
+
[notifiers] if isinstance(notifiers, BaseNotifier) else notifiers or []
|
|
85
|
+
)
|
|
80
86
|
|
|
87
|
+
self.validate_options()
|
|
81
88
|
self.validate_defaults()
|
|
82
89
|
|
|
90
|
+
def validate_options(self) -> None:
|
|
91
|
+
if not self.options:
|
|
92
|
+
raise ValueError('"options" cannot be empty.')
|
|
93
|
+
|
|
83
94
|
def validate_defaults(self) -> None:
|
|
84
95
|
"""
|
|
85
96
|
Validate whether the given defaults pass the following criteria.
|
|
@@ -107,11 +118,16 @@ class HITLOperator(BaseOperator):
|
|
|
107
118
|
multiple=self.multiple,
|
|
108
119
|
params=self.serialized_params,
|
|
109
120
|
)
|
|
121
|
+
|
|
110
122
|
if self.execution_timeout:
|
|
111
|
-
timeout_datetime =
|
|
123
|
+
timeout_datetime = utcnow() + self.execution_timeout
|
|
112
124
|
else:
|
|
113
125
|
timeout_datetime = None
|
|
126
|
+
|
|
114
127
|
self.log.info("Waiting for response")
|
|
128
|
+
for notifier in self.notifiers:
|
|
129
|
+
notifier(context)
|
|
130
|
+
|
|
115
131
|
# Defer the Human-in-the-loop response checking process to HITLTrigger
|
|
116
132
|
self.defer(
|
|
117
133
|
trigger=HITLTrigger(
|
|
@@ -170,6 +186,9 @@ class ApprovalOperator(HITLOperator, SkipMixin):
|
|
|
170
186
|
|
|
171
187
|
FIXED_ARGS = ["options", "multiple"]
|
|
172
188
|
|
|
189
|
+
APPROVE = "Approve"
|
|
190
|
+
REJECT = "Reject"
|
|
191
|
+
|
|
173
192
|
def __init__(self, ignore_downstream_trigger_rules: bool = False, **kwargs) -> None:
|
|
174
193
|
for arg in self.FIXED_ARGS:
|
|
175
194
|
if arg in kwargs:
|
|
@@ -177,13 +196,17 @@ class ApprovalOperator(HITLOperator, SkipMixin):
|
|
|
177
196
|
|
|
178
197
|
self.ignore_downstream_trigger_rules = ignore_downstream_trigger_rules
|
|
179
198
|
|
|
180
|
-
super().__init__(
|
|
199
|
+
super().__init__(
|
|
200
|
+
options=[self.APPROVE, self.REJECT],
|
|
201
|
+
multiple=False,
|
|
202
|
+
**kwargs,
|
|
203
|
+
)
|
|
181
204
|
|
|
182
205
|
def execute_complete(self, context: Context, event: dict[str, Any]) -> Any:
|
|
183
206
|
ret = super().execute_complete(context=context, event=event)
|
|
184
207
|
|
|
185
208
|
chosen_option = ret["chosen_options"][0]
|
|
186
|
-
if chosen_option ==
|
|
209
|
+
if chosen_option == self.APPROVE:
|
|
187
210
|
self.log.info("Approved. Proceeding with downstream tasks...")
|
|
188
211
|
return ret
|
|
189
212
|
|
|
@@ -211,24 +234,27 @@ class ApprovalOperator(HITLOperator, SkipMixin):
|
|
|
211
234
|
return ret
|
|
212
235
|
|
|
213
236
|
|
|
214
|
-
class HITLBranchOperator(HITLOperator):
|
|
237
|
+
class HITLBranchOperator(HITLOperator, BranchMixIn):
|
|
215
238
|
"""BranchOperator based on Human-in-the-loop Response."""
|
|
216
239
|
|
|
217
|
-
|
|
218
|
-
super().__init__(**kwargs)
|
|
240
|
+
inherits_from_skipmixin = True
|
|
219
241
|
|
|
220
|
-
def execute_complete(self, context: Context, event: dict[str, Any]) ->
|
|
221
|
-
|
|
242
|
+
def execute_complete(self, context: Context, event: dict[str, Any]) -> Any:
|
|
243
|
+
ret = super().execute_complete(context=context, event=event)
|
|
244
|
+
chosen_options = ret["chosen_options"]
|
|
245
|
+
return self.do_branch(context=context, branches_to_execute=chosen_options)
|
|
222
246
|
|
|
223
247
|
|
|
224
248
|
class HITLEntryOperator(HITLOperator):
|
|
225
249
|
"""Human-in-the-loop Operator that is used to accept user input through TriggerForm."""
|
|
226
250
|
|
|
251
|
+
OK = "OK"
|
|
252
|
+
|
|
227
253
|
def __init__(self, **kwargs) -> None:
|
|
228
254
|
if "options" not in kwargs:
|
|
229
|
-
kwargs["options"] = [
|
|
255
|
+
kwargs["options"] = [self.OK]
|
|
230
256
|
|
|
231
257
|
if "defaults" not in kwargs:
|
|
232
|
-
kwargs["defaults"] = [
|
|
258
|
+
kwargs["defaults"] = [self.OK]
|
|
233
259
|
|
|
234
260
|
super().__init__(**kwargs)
|
|
@@ -51,9 +51,8 @@ from airflow.exceptions import (
|
|
|
51
51
|
from airflow.models.variable import Variable
|
|
52
52
|
from airflow.providers.standard.hooks.package_index import PackageIndexHook
|
|
53
53
|
from airflow.providers.standard.utils.python_virtualenv import prepare_virtualenv, write_python_script
|
|
54
|
-
from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS, BaseOperator
|
|
54
|
+
from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS, BaseOperator, context_merge
|
|
55
55
|
from airflow.utils import hashlib_wrapper
|
|
56
|
-
from airflow.utils.context import context_copy_partial, context_merge
|
|
57
56
|
from airflow.utils.file import get_unique_dag_module_name
|
|
58
57
|
from airflow.utils.operator_helpers import KeywordParameters
|
|
59
58
|
from airflow.utils.process_utils import execute_in_subprocess
|
|
@@ -487,7 +486,8 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
|
|
|
487
486
|
|
|
488
487
|
def execute(self, context: Context) -> Any:
|
|
489
488
|
serializable_keys = set(self._iter_serializable_context_keys())
|
|
490
|
-
|
|
489
|
+
new = {k: v for k, v in context.items() if k in serializable_keys}
|
|
490
|
+
serializable_context = cast("Context", new)
|
|
491
491
|
return super().execute(context=serializable_context)
|
|
492
492
|
|
|
493
493
|
def get_python_source(self):
|
|
@@ -889,9 +889,9 @@ class PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
|
|
|
889
889
|
|
|
890
890
|
with TemporaryDirectory(prefix="venv") as tmp_dir:
|
|
891
891
|
tmp_path = Path(tmp_dir)
|
|
892
|
-
tmp_dir, temp_venv_dir = tmp_path.relative_to(tmp_path.anchor).parts
|
|
893
892
|
custom_pycache_prefix = Path(sys.pycache_prefix or "")
|
|
894
|
-
|
|
893
|
+
r_path = tmp_path.relative_to(tmp_path.anchor)
|
|
894
|
+
venv_python_cache_dir = Path.cwd() / custom_pycache_prefix / r_path
|
|
895
895
|
self._prepare_venv(tmp_path)
|
|
896
896
|
python_path = tmp_path / "bin" / "python"
|
|
897
897
|
result = self._execute_python_callable_in_subprocess(python_path)
|
|
@@ -20,8 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
from collections.abc import Callable, Mapping, Sequence
|
|
21
21
|
from typing import TYPE_CHECKING, Any
|
|
22
22
|
|
|
23
|
-
from airflow.providers.standard.version_compat import BaseSensorOperator, PokeReturnValue
|
|
24
|
-
from airflow.utils.context import context_merge
|
|
23
|
+
from airflow.providers.standard.version_compat import BaseSensorOperator, PokeReturnValue, context_merge
|
|
25
24
|
from airflow.utils.operator_helpers import determine_kwargs
|
|
26
25
|
|
|
27
26
|
if TYPE_CHECKING:
|
|
@@ -29,7 +28,7 @@ if TYPE_CHECKING:
|
|
|
29
28
|
from airflow.sdk.definitions.context import Context
|
|
30
29
|
except ImportError:
|
|
31
30
|
# TODO: Remove once provider drops support for Airflow 2
|
|
32
|
-
from airflow.utils.context import Context
|
|
31
|
+
from airflow.utils.context import Context # type: ignore[no-redef, attr-defined]
|
|
33
32
|
|
|
34
33
|
|
|
35
34
|
class PythonSensor(BaseSensorOperator):
|
|
@@ -42,7 +42,10 @@ except ImportError:
|
|
|
42
42
|
timeout: datetime.timedelta | None = None
|
|
43
43
|
|
|
44
44
|
|
|
45
|
-
|
|
45
|
+
try:
|
|
46
|
+
from airflow.sdk import timezone
|
|
47
|
+
except ImportError:
|
|
48
|
+
from airflow.utils import timezone # type: ignore[attr-defined,no-redef]
|
|
46
49
|
|
|
47
50
|
if TYPE_CHECKING:
|
|
48
51
|
try:
|
|
@@ -116,9 +119,11 @@ class TimeSensor(BaseSensorOperator):
|
|
|
116
119
|
),
|
|
117
120
|
method_name="execute_complete",
|
|
118
121
|
)
|
|
122
|
+
else:
|
|
123
|
+
super().execute(context)
|
|
119
124
|
|
|
120
|
-
def execute_complete(self, context: Context) -> None:
|
|
121
|
-
return
|
|
125
|
+
def execute_complete(self, context: Context, event: Any = None) -> None:
|
|
126
|
+
return None
|
|
122
127
|
|
|
123
128
|
def poke(self, context: Context) -> bool:
|
|
124
129
|
self.log.info("Checking if the time (%s) has come", self.target_datetime)
|
|
@@ -34,8 +34,8 @@ from airflow.sdk.execution_time.hitl import (
|
|
|
34
34
|
get_hitl_detail_content_detail,
|
|
35
35
|
update_htil_detail_response,
|
|
36
36
|
)
|
|
37
|
+
from airflow.sdk.timezone import utcnow
|
|
37
38
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
|
38
|
-
from airflow.utils import timezone
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class HITLTriggerEventSuccessPayload(TypedDict, total=False):
|
|
@@ -43,6 +43,7 @@ class HITLTriggerEventSuccessPayload(TypedDict, total=False):
|
|
|
43
43
|
|
|
44
44
|
chosen_options: list[str]
|
|
45
45
|
params_input: dict[str, Any]
|
|
46
|
+
timedout: bool
|
|
46
47
|
|
|
47
48
|
|
|
48
49
|
class HITLTriggerEventFailurePayload(TypedDict):
|
|
@@ -96,7 +97,7 @@ class HITLTrigger(BaseTrigger):
|
|
|
96
97
|
async def run(self) -> AsyncIterator[TriggerEvent]:
|
|
97
98
|
"""Loop until the Human-in-the-loop response received or timeout reached."""
|
|
98
99
|
while True:
|
|
99
|
-
if self.timeout_datetime and self.timeout_datetime <
|
|
100
|
+
if self.timeout_datetime and self.timeout_datetime < utcnow():
|
|
100
101
|
if self.defaults is None:
|
|
101
102
|
yield TriggerEvent(
|
|
102
103
|
HITLTriggerEventFailurePayload(
|
|
@@ -111,21 +112,28 @@ class HITLTrigger(BaseTrigger):
|
|
|
111
112
|
chosen_options=self.defaults,
|
|
112
113
|
params_input=self.params,
|
|
113
114
|
)
|
|
115
|
+
self.log.info(
|
|
116
|
+
"[HITL] timeout reached before receiving response, fallback to default %s", self.defaults
|
|
117
|
+
)
|
|
114
118
|
yield TriggerEvent(
|
|
115
119
|
HITLTriggerEventSuccessPayload(
|
|
116
120
|
chosen_options=self.defaults,
|
|
117
121
|
params_input=self.params,
|
|
122
|
+
timedout=True,
|
|
118
123
|
)
|
|
119
124
|
)
|
|
120
125
|
return
|
|
121
126
|
|
|
122
127
|
resp = await sync_to_async(get_hitl_detail_content_detail)(ti_id=self.ti_id)
|
|
123
128
|
if resp.response_received and resp.chosen_options:
|
|
124
|
-
self.log.info(
|
|
129
|
+
self.log.info(
|
|
130
|
+
"[HITL] user=%s options=%s at %s", resp.user_id, resp.chosen_options, resp.response_at
|
|
131
|
+
)
|
|
125
132
|
yield TriggerEvent(
|
|
126
133
|
HITLTriggerEventSuccessPayload(
|
|
127
134
|
chosen_options=resp.chosen_options,
|
|
128
135
|
params_input=resp.params_input,
|
|
136
|
+
timedout=False,
|
|
129
137
|
)
|
|
130
138
|
)
|
|
131
139
|
return
|
|
@@ -24,7 +24,11 @@ from typing import Any
|
|
|
24
24
|
import pendulum
|
|
25
25
|
|
|
26
26
|
from airflow.triggers.base import BaseTrigger, TaskSuccessEvent, TriggerEvent
|
|
27
|
-
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
from airflow.sdk import timezone
|
|
30
|
+
except ImportError:
|
|
31
|
+
from airflow.utils import timezone # type: ignore[attr-defined,no-redef]
|
|
28
32
|
|
|
29
33
|
|
|
30
34
|
class DateTimeTrigger(BaseTrigger):
|
|
@@ -21,6 +21,7 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
import os
|
|
23
23
|
import shutil
|
|
24
|
+
import warnings
|
|
24
25
|
from pathlib import Path
|
|
25
26
|
|
|
26
27
|
import jinja2
|
|
@@ -55,6 +56,15 @@ def _use_uv() -> bool:
|
|
|
55
56
|
|
|
56
57
|
def _generate_uv_cmd(tmp_dir: str, python_bin: str, system_site_packages: bool) -> list[str]:
|
|
57
58
|
"""Build the command to install the venv via UV."""
|
|
59
|
+
if python_bin == "python" or python_bin == "python3":
|
|
60
|
+
python_interpreter_exists = bool(shutil.which(python_bin))
|
|
61
|
+
if not python_interpreter_exists:
|
|
62
|
+
warnings.warn(
|
|
63
|
+
f"uv trying to use `{python_bin}` as the python interpreter. it could lead to errors if the python interpreter not found in PATH. "
|
|
64
|
+
f"please specify python_version in operator.",
|
|
65
|
+
UserWarning,
|
|
66
|
+
stacklevel=3,
|
|
67
|
+
)
|
|
58
68
|
cmd = ["uv", "venv", "--allow-existing", "--seed", "--python", python_bin]
|
|
59
69
|
if system_site_packages:
|
|
60
70
|
cmd.append("--system-site-packages")
|
|
@@ -40,9 +40,11 @@ AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0)
|
|
|
40
40
|
# even though it wasn't used.
|
|
41
41
|
if AIRFLOW_V_3_1_PLUS:
|
|
42
42
|
from airflow.sdk import BaseHook, BaseOperator
|
|
43
|
+
from airflow.sdk.definitions.context import context_merge
|
|
43
44
|
else:
|
|
44
45
|
from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
|
|
45
46
|
from airflow.models.baseoperator import BaseOperator # type: ignore[no-redef]
|
|
47
|
+
from airflow.utils.context import context_merge # type: ignore[no-redef, attr-defined]
|
|
46
48
|
|
|
47
49
|
if AIRFLOW_V_3_0_PLUS:
|
|
48
50
|
from airflow.sdk import BaseOperatorLink
|
|
@@ -59,4 +61,5 @@ __all__ = [
|
|
|
59
61
|
"BaseHook",
|
|
60
62
|
"BaseSensorOperator",
|
|
61
63
|
"PokeReturnValue",
|
|
64
|
+
"context_merge",
|
|
62
65
|
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-standard
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.0rc2
|
|
4
4
|
Summary: Provider package apache-airflow-providers-standard for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,standard,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
23
|
Requires-Dist: apache-airflow>=2.10.0rc1
|
|
24
24
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
25
|
-
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-standard/1.
|
|
26
|
-
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-standard/1.
|
|
25
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-standard/1.6.0/changelog.html
|
|
26
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-standard/1.6.0
|
|
27
27
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
28
28
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
29
29
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -54,7 +54,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
|
54
54
|
|
|
55
55
|
Package ``apache-airflow-providers-standard``
|
|
56
56
|
|
|
57
|
-
Release: ``1.
|
|
57
|
+
Release: ``1.6.0``
|
|
58
58
|
|
|
59
59
|
Release Date: ``|PypiReleaseDate|``
|
|
60
60
|
|
|
@@ -68,7 +68,7 @@ This is a provider package for ``standard`` provider. All classes for this provi
|
|
|
68
68
|
are in ``airflow.providers.standard`` python package.
|
|
69
69
|
|
|
70
70
|
You can find package information and changelog for the provider
|
|
71
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/1.
|
|
71
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-standard/1.6.0/>`_.
|
|
72
72
|
|
|
73
73
|
Installation
|
|
74
74
|
------------
|
|
@@ -89,5 +89,5 @@ PIP package Version required
|
|
|
89
89
|
================== ==================
|
|
90
90
|
|
|
91
91
|
The changelog for the provider package can be found in the
|
|
92
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/1.
|
|
92
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-standard/1.6.0/changelog.html>`_.
|
|
93
93
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
airflow/providers/standard/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
|
|
2
|
-
airflow/providers/standard/__init__.py,sha256=
|
|
2
|
+
airflow/providers/standard/__init__.py,sha256=HRU9nnD6x9Bv6Wqk4LgRJeoJvuUNVQQMsRz1FiSBuSc,1497
|
|
3
3
|
airflow/providers/standard/exceptions.py,sha256=8CTMCs1xVk_06piBoyP3pKX6j29riukL8V2V7miPgEU,2269
|
|
4
4
|
airflow/providers/standard/get_provider_info.py,sha256=jhENLvqCXj0mzBPmJeAvPj7XWaaNuxPLhHVVA-9rqzs,7132
|
|
5
|
-
airflow/providers/standard/version_compat.py,sha256=
|
|
5
|
+
airflow/providers/standard/version_compat.py,sha256=zXebxsadQSmoPBb2ypNHwJO65GY_E_mnNDmPNu2PPKI,2787
|
|
6
6
|
airflow/providers/standard/decorators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
7
|
-
airflow/providers/standard/decorators/bash.py,sha256=
|
|
7
|
+
airflow/providers/standard/decorators/bash.py,sha256=NuWB6q8AeTzWPeIIX-y3M-qN819BEcjNQwS32XTF6cs,4416
|
|
8
8
|
airflow/providers/standard/decorators/branch_external_python.py,sha256=G1l3_sYM38wEWMRnzRuVGmAbd4uMN5D07zm3EM3-ZWo,2584
|
|
9
9
|
airflow/providers/standard/decorators/branch_python.py,sha256=ZngmIIXTZDBfPgOI71ag6q_87eFdoxW5xTqtC60-_xQ,2503
|
|
10
10
|
airflow/providers/standard/decorators/branch_virtualenv.py,sha256=9mwTQsOKFKg0xJ4KmD0qsBPXvQUX0cBJRVtqKqaNocQ,2585
|
|
@@ -22,8 +22,8 @@ airflow/providers/standard/example_dags/example_branch_operator.py,sha256=c4dr2d
|
|
|
22
22
|
airflow/providers/standard/example_dags/example_branch_operator_decorator.py,sha256=mF_87Kqxhui6sCsWvBXJ6m_w9bUoeqrA_rUVJuyIeq4,4801
|
|
23
23
|
airflow/providers/standard/example_dags/example_external_task_child_deferrable.py,sha256=o-ji3leJTBjiChEWoqVu4ykz1YVYUd8-ApmZwHFcNc8,1233
|
|
24
24
|
airflow/providers/standard/example_dags/example_external_task_marker_dag.py,sha256=gssBjlfrGMDLZxTYOxo8ihXLbJ-3Uu31QodINGFWYNU,3650
|
|
25
|
-
airflow/providers/standard/example_dags/example_external_task_parent_deferrable.py,sha256=
|
|
26
|
-
airflow/providers/standard/example_dags/example_hitl_operator.py,sha256=
|
|
25
|
+
airflow/providers/standard/example_dags/example_external_task_parent_deferrable.py,sha256=8R1EqL0x1SgxcJtOb2EO4_NXmFgxiM-a0wPilDf7fis,2172
|
|
26
|
+
airflow/providers/standard/example_dags/example_hitl_operator.py,sha256=i6UC0IX8s5anBx4AStd3-lPYGoguOxscIXl94Q4BdRw,5745
|
|
27
27
|
airflow/providers/standard/example_dags/example_latest_only.py,sha256=ac9WpLMWLzyuxZks74t3HojS7vRG2gynmQfGm13gwOI,1456
|
|
28
28
|
airflow/providers/standard/example_dags/example_python_decorator.py,sha256=jveqPOw1GZzD3Z37_rYc8Q8hcyx8vCNjgetpO_P6qmg,4281
|
|
29
29
|
airflow/providers/standard/example_dags/example_python_operator.py,sha256=3L6CZHK2Fb7zmA9tDhZ5QaEe38WJYlS4l35Gc7xJAoE,4761
|
|
@@ -44,9 +44,9 @@ airflow/providers/standard/operators/bash.py,sha256=yzNbi20dp6PWJzXZqJ05gyMsdb6-
|
|
|
44
44
|
airflow/providers/standard/operators/branch.py,sha256=No3JoaqPjEAu7QlVf-20nzli0yYKbYorjcx885i5eeg,3970
|
|
45
45
|
airflow/providers/standard/operators/datetime.py,sha256=bYDdbfAyAlEXRRHjOgB06UhgDum6SPdd5I3u-ylPSaw,5005
|
|
46
46
|
airflow/providers/standard/operators/empty.py,sha256=BTeZ4KRykaEHLZigSBkevcStCrbPdQpWDMnO3ZdtZqw,1338
|
|
47
|
-
airflow/providers/standard/operators/hitl.py,sha256=
|
|
47
|
+
airflow/providers/standard/operators/hitl.py,sha256=zvMU5FPoaTmSPARhmKsr8qaQ0ajctdkE4V0GNaPHS3M,10054
|
|
48
48
|
airflow/providers/standard/operators/latest_only.py,sha256=VkU-nAI8QbIrmeiv4wYXBcZF0yKMkcFapormg0J5-As,5110
|
|
49
|
-
airflow/providers/standard/operators/python.py,sha256=
|
|
49
|
+
airflow/providers/standard/operators/python.py,sha256=nPRQV6OweIkf976i7AJpZn9vQziqugReJ7xtDu27s8Q,53741
|
|
50
50
|
airflow/providers/standard/operators/smooth.py,sha256=IMs5GjM42XEiroksIZ5flGQgxfRUbXZXCWxpshVinYQ,1396
|
|
51
51
|
airflow/providers/standard/operators/trigger_dagrun.py,sha256=AgJaiB4u-X-HDvdYLdseFQO_zBYb6_UijV-qmDqwqo0,16576
|
|
52
52
|
airflow/providers/standard/operators/weekday.py,sha256=Qg7LhXYtybVSGZn8uQqF-r7RB7zOXfe3R6vSGVa_rJk,5083
|
|
@@ -55,22 +55,22 @@ airflow/providers/standard/sensors/bash.py,sha256=jiysK84IwnVpQj1_lE65E_pSPE0FO8
|
|
|
55
55
|
airflow/providers/standard/sensors/date_time.py,sha256=W4lN-EXCIiJkPf6FKvJ4yx7X9vSCfKT7YjVbnjtmkrM,6481
|
|
56
56
|
airflow/providers/standard/sensors/external_task.py,sha256=lDEg2Zbwp79f6VV6uH3PXI-NiHbL4IMAO4z-1VDl4gA,28695
|
|
57
57
|
airflow/providers/standard/sensors/filesystem.py,sha256=jDgxZQ4WXRv1PSjc2o4K0Iq_AxnaPw7yIUnafK_VpaM,6050
|
|
58
|
-
airflow/providers/standard/sensors/python.py,sha256=
|
|
59
|
-
airflow/providers/standard/sensors/time.py,sha256=
|
|
58
|
+
airflow/providers/standard/sensors/python.py,sha256=gFlZf3h60UtgJCarG9FN30cnPxKyLNVn46nNxu6O9aQ,3415
|
|
59
|
+
airflow/providers/standard/sensors/time.py,sha256=9kfamlG4mNJ1T39rOrhuAW41NiXFPP6RxU1ytiTIsLs,5271
|
|
60
60
|
airflow/providers/standard/sensors/time_delta.py,sha256=ggDSna-m_scLFks9zx1LoC64jQBjw7ZQqH7n96UU2BQ,7579
|
|
61
61
|
airflow/providers/standard/sensors/weekday.py,sha256=sKDQ7xC9c32DZxaGNIjqmW6HXE4hIvKC71Kt-_d9SG8,4470
|
|
62
62
|
airflow/providers/standard/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
63
63
|
airflow/providers/standard/triggers/external_task.py,sha256=R2Wsd21pw9_gGTs9XuHafylt65hMVPisz2g6vnpLJ4o,11521
|
|
64
64
|
airflow/providers/standard/triggers/file.py,sha256=2i8-RwSjEgdOwQNcHCqLmSdpE3Ehqg4GQJ8nE3-fHxo,4886
|
|
65
|
-
airflow/providers/standard/triggers/hitl.py,sha256=
|
|
66
|
-
airflow/providers/standard/triggers/temporal.py,sha256=
|
|
65
|
+
airflow/providers/standard/triggers/hitl.py,sha256=39xqF8om6hLJPhcQ6t4q_ZYhrQTAfC9bwitu1cjbb7k,5127
|
|
66
|
+
airflow/providers/standard/triggers/temporal.py,sha256=rqUMRsKTO46JvUbIXgReei5p9cnEd8PV4zJkLZWWOUI,4552
|
|
67
67
|
airflow/providers/standard/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
68
|
-
airflow/providers/standard/utils/python_virtualenv.py,sha256=
|
|
68
|
+
airflow/providers/standard/utils/python_virtualenv.py,sha256=logUzODR5qnUZYyew-ZEbL7rudrkTEtmnU9qTQhg0-0,8532
|
|
69
69
|
airflow/providers/standard/utils/python_virtualenv_script.jinja2,sha256=3Z334hVq6hQ9EHkOoGnAHc2_XNkZQkOJGxZArDKLc-c,2770
|
|
70
70
|
airflow/providers/standard/utils/sensor_helper.py,sha256=PNIETsl_a4BkmOypFfHdpP0VuTkC6eWKUDuwnNVaWsA,5000
|
|
71
71
|
airflow/providers/standard/utils/skipmixin.py,sha256=PMrP2vtr5Sn6eCVslAqmEpY6Rgo6ZyfR73LPXS5NGVA,8015
|
|
72
72
|
airflow/providers/standard/utils/weekday.py,sha256=ySDrIkWv-lqqxURo9E98IGInDqERec2O4y9o2hQTGiQ,2685
|
|
73
|
-
apache_airflow_providers_standard-1.
|
|
74
|
-
apache_airflow_providers_standard-1.
|
|
75
|
-
apache_airflow_providers_standard-1.
|
|
76
|
-
apache_airflow_providers_standard-1.
|
|
73
|
+
apache_airflow_providers_standard-1.6.0rc2.dist-info/entry_points.txt,sha256=mW2YRh3mVdZdaP5-iGSNgmcCh3YYdALIn28BCLBZZ40,104
|
|
74
|
+
apache_airflow_providers_standard-1.6.0rc2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
|
75
|
+
apache_airflow_providers_standard-1.6.0rc2.dist-info/METADATA,sha256=J3y9RNYBTZdUCuGKTKHv5lG5E8_EFDDNl_PYVEokMfY,3847
|
|
76
|
+
apache_airflow_providers_standard-1.6.0rc2.dist-info/RECORD,,
|
|
File without changes
|