apache-airflow-providers-standard 1.9.0__py3-none-any.whl → 1.9.2rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/standard/__init__.py +1 -1
- airflow/providers/standard/decorators/bash.py +7 -13
- airflow/providers/standard/decorators/branch_external_python.py +2 -8
- airflow/providers/standard/decorators/branch_python.py +2 -7
- airflow/providers/standard/decorators/branch_virtualenv.py +2 -7
- airflow/providers/standard/decorators/external_python.py +2 -7
- airflow/providers/standard/decorators/python.py +2 -7
- airflow/providers/standard/decorators/python_virtualenv.py +2 -9
- airflow/providers/standard/decorators/sensor.py +2 -9
- airflow/providers/standard/decorators/short_circuit.py +2 -8
- airflow/providers/standard/decorators/stub.py +6 -12
- airflow/providers/standard/example_dags/example_bash_decorator.py +1 -6
- airflow/providers/standard/example_dags/example_branch_operator.py +1 -6
- airflow/providers/standard/example_dags/example_branch_operator_decorator.py +1 -6
- airflow/providers/standard/example_dags/example_external_task_parent_deferrable.py +2 -6
- airflow/providers/standard/example_dags/example_hitl_operator.py +1 -1
- airflow/providers/standard/example_dags/example_sensors.py +1 -6
- airflow/providers/standard/example_dags/example_short_circuit_decorator.py +1 -6
- airflow/providers/standard/example_dags/example_short_circuit_operator.py +1 -6
- airflow/providers/standard/hooks/filesystem.py +1 -1
- airflow/providers/standard/hooks/package_index.py +1 -1
- airflow/providers/standard/hooks/subprocess.py +3 -10
- airflow/providers/standard/operators/bash.py +3 -12
- airflow/providers/standard/operators/branch.py +1 -1
- airflow/providers/standard/operators/datetime.py +2 -6
- airflow/providers/standard/operators/empty.py +1 -1
- airflow/providers/standard/operators/hitl.py +12 -9
- airflow/providers/standard/operators/latest_only.py +3 -8
- airflow/providers/standard/operators/python.py +9 -9
- airflow/providers/standard/operators/smooth.py +1 -1
- airflow/providers/standard/operators/trigger_dagrun.py +16 -21
- airflow/providers/standard/operators/weekday.py +2 -6
- airflow/providers/standard/sensors/bash.py +3 -8
- airflow/providers/standard/sensors/date_time.py +2 -6
- airflow/providers/standard/sensors/external_task.py +77 -55
- airflow/providers/standard/sensors/filesystem.py +1 -1
- airflow/providers/standard/sensors/python.py +2 -6
- airflow/providers/standard/sensors/time.py +1 -6
- airflow/providers/standard/sensors/time_delta.py +3 -7
- airflow/providers/standard/sensors/weekday.py +2 -7
- airflow/providers/standard/triggers/external_task.py +36 -36
- airflow/providers/standard/triggers/file.py +1 -1
- airflow/providers/standard/triggers/hitl.py +135 -86
- airflow/providers/standard/triggers/temporal.py +1 -5
- airflow/providers/standard/utils/python_virtualenv.py +36 -3
- airflow/providers/standard/utils/sensor_helper.py +19 -8
- airflow/providers/standard/utils/skipmixin.py +1 -7
- airflow/providers/standard/version_compat.py +4 -21
- {apache_airflow_providers_standard-1.9.0.dist-info → apache_airflow_providers_standard-1.9.2rc1.dist-info}/METADATA +36 -13
- apache_airflow_providers_standard-1.9.2rc1.dist-info/RECORD +78 -0
- apache_airflow_providers_standard-1.9.2rc1.dist-info/licenses/NOTICE +5 -0
- apache_airflow_providers_standard-1.9.0.dist-info/RECORD +0 -77
- {apache_airflow_providers_standard-1.9.0.dist-info → apache_airflow_providers_standard-1.9.2rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_standard-1.9.0.dist-info → apache_airflow_providers_standard-1.9.2rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/standard → apache_airflow_providers_standard-1.9.2rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "1.9.
|
|
32
|
+
__version__ = "1.9.2"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.10.0"
|
|
@@ -21,24 +21,18 @@ import warnings
|
|
|
21
21
|
from collections.abc import Callable, Collection, Mapping, Sequence
|
|
22
22
|
from typing import TYPE_CHECKING, Any, ClassVar
|
|
23
23
|
|
|
24
|
-
from airflow.providers.
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
DecoratedOperator,
|
|
31
|
-
TaskDecorator,
|
|
32
|
-
task_decorator_factory,
|
|
33
|
-
)
|
|
34
|
-
|
|
24
|
+
from airflow.providers.common.compat.sdk import (
|
|
25
|
+
DecoratedOperator,
|
|
26
|
+
TaskDecorator,
|
|
27
|
+
context_merge,
|
|
28
|
+
task_decorator_factory,
|
|
29
|
+
)
|
|
35
30
|
from airflow.providers.standard.operators.bash import BashOperator
|
|
36
|
-
from airflow.providers.standard.version_compat import context_merge
|
|
37
31
|
from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION
|
|
38
32
|
from airflow.utils.operator_helpers import determine_kwargs
|
|
39
33
|
|
|
40
34
|
if TYPE_CHECKING:
|
|
41
|
-
from airflow.
|
|
35
|
+
from airflow.providers.common.compat.sdk import Context
|
|
42
36
|
|
|
43
37
|
|
|
44
38
|
class _BashDecoratedOperator(DecoratedOperator, BashOperator):
|
|
@@ -19,18 +19,12 @@ from __future__ import annotations
|
|
|
19
19
|
from collections.abc import Callable
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
|
-
from airflow.providers.
|
|
23
|
-
|
|
24
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
25
|
-
from airflow.sdk.bases.decorator import task_decorator_factory
|
|
26
|
-
else:
|
|
27
|
-
from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
|
|
28
|
-
|
|
22
|
+
from airflow.providers.common.compat.sdk import task_decorator_factory
|
|
29
23
|
from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
|
|
30
24
|
from airflow.providers.standard.operators.python import BranchExternalPythonOperator
|
|
31
25
|
|
|
32
26
|
if TYPE_CHECKING:
|
|
33
|
-
from airflow.
|
|
27
|
+
from airflow.providers.common.compat.sdk import TaskDecorator
|
|
34
28
|
|
|
35
29
|
|
|
36
30
|
class _BranchExternalPythonDecoratedOperator(_PythonDecoratedOperator, BranchExternalPythonOperator):
|
|
@@ -19,17 +19,12 @@ from __future__ import annotations
|
|
|
19
19
|
from collections.abc import Callable
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
|
-
from airflow.providers.
|
|
23
|
-
|
|
24
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
25
|
-
from airflow.sdk.bases.decorator import task_decorator_factory
|
|
26
|
-
else:
|
|
27
|
-
from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
|
|
22
|
+
from airflow.providers.common.compat.sdk import task_decorator_factory
|
|
28
23
|
from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
|
|
29
24
|
from airflow.providers.standard.operators.python import BranchPythonOperator
|
|
30
25
|
|
|
31
26
|
if TYPE_CHECKING:
|
|
32
|
-
from airflow.
|
|
27
|
+
from airflow.providers.common.compat.sdk import TaskDecorator
|
|
33
28
|
|
|
34
29
|
|
|
35
30
|
class _BranchPythonDecoratedOperator(_PythonDecoratedOperator, BranchPythonOperator):
|
|
@@ -19,17 +19,12 @@ from __future__ import annotations
|
|
|
19
19
|
from collections.abc import Callable
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
|
-
from airflow.providers.
|
|
23
|
-
|
|
24
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
25
|
-
from airflow.sdk.bases.decorator import task_decorator_factory
|
|
26
|
-
else:
|
|
27
|
-
from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
|
|
22
|
+
from airflow.providers.common.compat.sdk import task_decorator_factory
|
|
28
23
|
from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
|
|
29
24
|
from airflow.providers.standard.operators.python import BranchPythonVirtualenvOperator
|
|
30
25
|
|
|
31
26
|
if TYPE_CHECKING:
|
|
32
|
-
from airflow.
|
|
27
|
+
from airflow.providers.common.compat.sdk import TaskDecorator
|
|
33
28
|
|
|
34
29
|
|
|
35
30
|
class _BranchPythonVirtualenvDecoratedOperator(_PythonDecoratedOperator, BranchPythonVirtualenvOperator):
|
|
@@ -19,17 +19,12 @@ from __future__ import annotations
|
|
|
19
19
|
from collections.abc import Callable
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
|
-
from airflow.providers.
|
|
23
|
-
|
|
24
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
25
|
-
from airflow.sdk.bases.decorator import task_decorator_factory
|
|
26
|
-
else:
|
|
27
|
-
from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
|
|
22
|
+
from airflow.providers.common.compat.sdk import task_decorator_factory
|
|
28
23
|
from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
|
|
29
24
|
from airflow.providers.standard.operators.python import ExternalPythonOperator
|
|
30
25
|
|
|
31
26
|
if TYPE_CHECKING:
|
|
32
|
-
from airflow.
|
|
27
|
+
from airflow.providers.common.compat.sdk import TaskDecorator
|
|
33
28
|
|
|
34
29
|
|
|
35
30
|
class _PythonExternalDecoratedOperator(_PythonDecoratedOperator, ExternalPythonOperator):
|
|
@@ -19,16 +19,11 @@ from __future__ import annotations
|
|
|
19
19
|
from collections.abc import Callable, Sequence
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
|
+
from airflow.providers.common.compat.sdk import DecoratedOperator, task_decorator_factory
|
|
22
23
|
from airflow.providers.standard.operators.python import PythonOperator
|
|
23
|
-
from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS
|
|
24
|
-
|
|
25
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
26
|
-
from airflow.sdk.bases.decorator import DecoratedOperator, task_decorator_factory
|
|
27
|
-
else:
|
|
28
|
-
from airflow.decorators.base import DecoratedOperator, task_decorator_factory # type: ignore[no-redef]
|
|
29
24
|
|
|
30
25
|
if TYPE_CHECKING:
|
|
31
|
-
from airflow.
|
|
26
|
+
from airflow.providers.common.compat.sdk import TaskDecorator
|
|
32
27
|
|
|
33
28
|
|
|
34
29
|
class _PythonDecoratedOperator(DecoratedOperator, PythonOperator):
|
|
@@ -19,19 +19,12 @@ from __future__ import annotations
|
|
|
19
19
|
from collections.abc import Callable
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
|
-
from airflow.providers.
|
|
23
|
-
|
|
24
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
25
|
-
from airflow.sdk.bases.decorator import task_decorator_factory
|
|
26
|
-
else:
|
|
27
|
-
from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
|
|
28
|
-
|
|
29
|
-
|
|
22
|
+
from airflow.providers.common.compat.sdk import task_decorator_factory
|
|
30
23
|
from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
|
|
31
24
|
from airflow.providers.standard.operators.python import PythonVirtualenvOperator
|
|
32
25
|
|
|
33
26
|
if TYPE_CHECKING:
|
|
34
|
-
from airflow.
|
|
27
|
+
from airflow.providers.common.compat.sdk import TaskDecorator
|
|
35
28
|
|
|
36
29
|
|
|
37
30
|
class _PythonVirtualenvDecoratedOperator(_PythonDecoratedOperator, PythonVirtualenvOperator):
|
|
@@ -20,18 +20,11 @@ from __future__ import annotations
|
|
|
20
20
|
from collections.abc import Callable, Sequence
|
|
21
21
|
from typing import TYPE_CHECKING, ClassVar
|
|
22
22
|
|
|
23
|
-
from airflow.providers.
|
|
24
|
-
|
|
25
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
26
|
-
from airflow.sdk.bases.decorator import get_unique_task_id, task_decorator_factory
|
|
27
|
-
else:
|
|
28
|
-
from airflow.decorators.base import get_unique_task_id, task_decorator_factory # type: ignore[no-redef]
|
|
29
|
-
|
|
30
|
-
|
|
23
|
+
from airflow.providers.common.compat.sdk import get_unique_task_id, task_decorator_factory
|
|
31
24
|
from airflow.providers.standard.sensors.python import PythonSensor
|
|
32
25
|
|
|
33
26
|
if TYPE_CHECKING:
|
|
34
|
-
from airflow.
|
|
27
|
+
from airflow.providers.common.compat.sdk import TaskDecorator
|
|
35
28
|
|
|
36
29
|
|
|
37
30
|
class DecoratedSensorOperator(PythonSensor):
|
|
@@ -19,18 +19,12 @@ from __future__ import annotations
|
|
|
19
19
|
from collections.abc import Callable
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
|
-
from airflow.providers.
|
|
23
|
-
|
|
24
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
25
|
-
from airflow.sdk.bases.decorator import task_decorator_factory
|
|
26
|
-
else:
|
|
27
|
-
from airflow.decorators.base import task_decorator_factory # type: ignore[no-redef]
|
|
28
|
-
|
|
22
|
+
from airflow.providers.common.compat.sdk import task_decorator_factory
|
|
29
23
|
from airflow.providers.standard.decorators.python import _PythonDecoratedOperator
|
|
30
24
|
from airflow.providers.standard.operators.python import ShortCircuitOperator
|
|
31
25
|
|
|
32
26
|
if TYPE_CHECKING:
|
|
33
|
-
from airflow.
|
|
27
|
+
from airflow.providers.common.compat.sdk import TaskDecorator
|
|
34
28
|
|
|
35
29
|
|
|
36
30
|
class _ShortCircuitDecoratedOperator(_PythonDecoratedOperator, ShortCircuitOperator):
|
|
@@ -21,20 +21,14 @@ import ast
|
|
|
21
21
|
from collections.abc import Callable
|
|
22
22
|
from typing import TYPE_CHECKING, Any
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
except ModuleNotFoundError:
|
|
30
|
-
from airflow.decorators.base import (
|
|
31
|
-
DecoratedOperator,
|
|
32
|
-
TaskDecorator,
|
|
33
|
-
task_decorator_factory,
|
|
34
|
-
)
|
|
24
|
+
from airflow.providers.common.compat.sdk import (
|
|
25
|
+
DecoratedOperator,
|
|
26
|
+
TaskDecorator,
|
|
27
|
+
task_decorator_factory,
|
|
28
|
+
)
|
|
35
29
|
|
|
36
30
|
if TYPE_CHECKING:
|
|
37
|
-
from airflow.
|
|
31
|
+
from airflow.providers.common.compat.sdk import Context
|
|
38
32
|
|
|
39
33
|
|
|
40
34
|
class _StubOperator(DecoratedOperator):
|
|
@@ -20,16 +20,11 @@ from __future__ import annotations
|
|
|
20
20
|
import pendulum
|
|
21
21
|
|
|
22
22
|
from airflow.exceptions import AirflowSkipException
|
|
23
|
+
from airflow.providers.common.compat.sdk import TriggerRule
|
|
23
24
|
from airflow.providers.standard.operators.empty import EmptyOperator
|
|
24
25
|
from airflow.providers.standard.utils.weekday import WeekDay
|
|
25
26
|
from airflow.sdk import chain, dag, task
|
|
26
27
|
|
|
27
|
-
try:
|
|
28
|
-
from airflow.sdk import TriggerRule
|
|
29
|
-
except ImportError:
|
|
30
|
-
# Compatibility for Airflow < 3.1
|
|
31
|
-
from airflow.utils.trigger_rule import TriggerRule # type: ignore[no-redef,attr-defined]
|
|
32
|
-
|
|
33
28
|
|
|
34
29
|
@dag(schedule=None, start_date=pendulum.datetime(2023, 1, 1, tz="UTC"), catchup=False)
|
|
35
30
|
def example_bash_decorator():
|
|
@@ -29,6 +29,7 @@ from pathlib import Path
|
|
|
29
29
|
|
|
30
30
|
import pendulum
|
|
31
31
|
|
|
32
|
+
from airflow.providers.common.compat.sdk import TriggerRule
|
|
32
33
|
from airflow.providers.standard.operators.empty import EmptyOperator
|
|
33
34
|
from airflow.providers.standard.operators.python import (
|
|
34
35
|
BranchExternalPythonOperator,
|
|
@@ -40,12 +41,6 @@ from airflow.providers.standard.operators.python import (
|
|
|
40
41
|
)
|
|
41
42
|
from airflow.sdk import DAG, Label
|
|
42
43
|
|
|
43
|
-
try:
|
|
44
|
-
from airflow.sdk import TriggerRule
|
|
45
|
-
except ImportError:
|
|
46
|
-
# Compatibility for Airflow < 3.1
|
|
47
|
-
from airflow.utils.trigger_rule import TriggerRule # type: ignore[no-redef,attr-defined]
|
|
48
|
-
|
|
49
44
|
PATH_TO_PYTHON_BINARY = sys.executable
|
|
50
45
|
|
|
51
46
|
with DAG(
|
|
@@ -30,15 +30,10 @@ import tempfile
|
|
|
30
30
|
|
|
31
31
|
import pendulum
|
|
32
32
|
|
|
33
|
+
from airflow.providers.common.compat.sdk import TriggerRule
|
|
33
34
|
from airflow.providers.standard.operators.empty import EmptyOperator
|
|
34
35
|
from airflow.sdk import DAG, Label, task
|
|
35
36
|
|
|
36
|
-
try:
|
|
37
|
-
from airflow.sdk import TriggerRule
|
|
38
|
-
except ImportError:
|
|
39
|
-
# Compatibility for Airflow < 3.1
|
|
40
|
-
from airflow.utils.trigger_rule import TriggerRule # type: ignore[no-redef,attr-defined]
|
|
41
|
-
|
|
42
37
|
PATH_TO_PYTHON_BINARY = sys.executable
|
|
43
38
|
|
|
44
39
|
with DAG(
|
|
@@ -17,18 +17,14 @@
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
19
|
from airflow import DAG
|
|
20
|
+
from airflow.providers.common.compat.sdk import timezone
|
|
20
21
|
from airflow.providers.standard.operators.empty import EmptyOperator
|
|
21
22
|
from airflow.providers.standard.operators.trigger_dagrun import TriggerDagRunOperator
|
|
22
23
|
from airflow.providers.standard.sensors.external_task import ExternalTaskSensor
|
|
23
24
|
|
|
24
|
-
try:
|
|
25
|
-
from airflow.sdk.timezone import datetime
|
|
26
|
-
except ImportError:
|
|
27
|
-
from airflow.utils.timezone import datetime # type: ignore[no-redef]
|
|
28
|
-
|
|
29
25
|
with DAG(
|
|
30
26
|
dag_id="example_external_task",
|
|
31
|
-
start_date=datetime(2022, 1, 1),
|
|
27
|
+
start_date=timezone.datetime(2022, 1, 1),
|
|
32
28
|
schedule="@once",
|
|
33
29
|
catchup=False,
|
|
34
30
|
tags=["example", "async", "core"],
|
|
@@ -32,7 +32,7 @@ from airflow.sdk import DAG, Param, task
|
|
|
32
32
|
from airflow.sdk.bases.notifier import BaseNotifier
|
|
33
33
|
|
|
34
34
|
if TYPE_CHECKING:
|
|
35
|
-
from airflow.
|
|
35
|
+
from airflow.providers.common.compat.sdk import Context
|
|
36
36
|
|
|
37
37
|
# [START hitl_tutorial]
|
|
38
38
|
|
|
@@ -21,6 +21,7 @@ import datetime
|
|
|
21
21
|
|
|
22
22
|
import pendulum
|
|
23
23
|
|
|
24
|
+
from airflow.providers.common.compat.sdk import TriggerRule
|
|
24
25
|
from airflow.providers.standard.operators.bash import BashOperator
|
|
25
26
|
from airflow.providers.standard.sensors.bash import BashSensor
|
|
26
27
|
from airflow.providers.standard.sensors.filesystem import FileSensor
|
|
@@ -31,12 +32,6 @@ from airflow.providers.standard.sensors.weekday import DayOfWeekSensor
|
|
|
31
32
|
from airflow.providers.standard.utils.weekday import WeekDay
|
|
32
33
|
from airflow.sdk import DAG
|
|
33
34
|
|
|
34
|
-
try:
|
|
35
|
-
from airflow.sdk import TriggerRule
|
|
36
|
-
except ImportError:
|
|
37
|
-
# Compatibility for Airflow < 3.1
|
|
38
|
-
from airflow.utils.trigger_rule import TriggerRule # type: ignore[no-redef,attr-defined]
|
|
39
|
-
|
|
40
35
|
|
|
41
36
|
# [START example_callables]
|
|
42
37
|
def success_callable():
|
|
@@ -20,15 +20,10 @@ from __future__ import annotations
|
|
|
20
20
|
|
|
21
21
|
import pendulum
|
|
22
22
|
|
|
23
|
+
from airflow.providers.common.compat.sdk import TriggerRule
|
|
23
24
|
from airflow.providers.standard.operators.empty import EmptyOperator
|
|
24
25
|
from airflow.sdk import chain, dag, task
|
|
25
26
|
|
|
26
|
-
try:
|
|
27
|
-
from airflow.sdk import TriggerRule
|
|
28
|
-
except ImportError:
|
|
29
|
-
# Compatibility for Airflow < 3.1
|
|
30
|
-
from airflow.utils.trigger_rule import TriggerRule # type: ignore[no-redef,attr-defined]
|
|
31
|
-
|
|
32
27
|
|
|
33
28
|
@dag(schedule=None, start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), catchup=False, tags=["example"])
|
|
34
29
|
def example_short_circuit_decorator():
|
|
@@ -21,16 +21,11 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
import pendulum
|
|
23
23
|
|
|
24
|
+
from airflow.providers.common.compat.sdk import TriggerRule
|
|
24
25
|
from airflow.providers.standard.operators.empty import EmptyOperator
|
|
25
26
|
from airflow.providers.standard.operators.python import ShortCircuitOperator
|
|
26
27
|
from airflow.sdk import DAG, chain
|
|
27
28
|
|
|
28
|
-
try:
|
|
29
|
-
from airflow.sdk import TriggerRule
|
|
30
|
-
except ImportError:
|
|
31
|
-
# Compatibility for Airflow < 3.1
|
|
32
|
-
from airflow.utils.trigger_rule import TriggerRule # type: ignore[no-redef,attr-defined]
|
|
33
|
-
|
|
34
29
|
with DAG(
|
|
35
30
|
dag_id="example_short_circuit_operator",
|
|
36
31
|
schedule=None,
|
|
@@ -24,7 +24,7 @@ from collections.abc import Iterator
|
|
|
24
24
|
from subprocess import PIPE, STDOUT, Popen
|
|
25
25
|
from tempfile import TemporaryDirectory, gettempdir
|
|
26
26
|
|
|
27
|
-
from airflow.providers.
|
|
27
|
+
from airflow.providers.common.compat.sdk import BaseHook
|
|
28
28
|
|
|
29
29
|
SubprocessResult = namedtuple("SubprocessResult", ["exit_code", "output"])
|
|
30
30
|
|
|
@@ -77,14 +77,6 @@ class SubprocessHook(BaseHook):
|
|
|
77
77
|
"""
|
|
78
78
|
self.log.info("Tmp dir root location: %s", gettempdir())
|
|
79
79
|
with working_directory(cwd=cwd) as cwd:
|
|
80
|
-
|
|
81
|
-
def pre_exec():
|
|
82
|
-
# Restore default signal disposition and invoke setsid
|
|
83
|
-
for sig in ("SIGPIPE", "SIGXFZ", "SIGXFSZ"):
|
|
84
|
-
if hasattr(signal, sig):
|
|
85
|
-
signal.signal(getattr(signal, sig), signal.SIG_DFL)
|
|
86
|
-
os.setsid()
|
|
87
|
-
|
|
88
80
|
self.log.info("Running command: %s", command)
|
|
89
81
|
|
|
90
82
|
self.sub_process = Popen(
|
|
@@ -93,7 +85,8 @@ class SubprocessHook(BaseHook):
|
|
|
93
85
|
stderr=STDOUT,
|
|
94
86
|
cwd=cwd,
|
|
95
87
|
env=env if env or env == {} else os.environ,
|
|
96
|
-
|
|
88
|
+
start_new_session=True,
|
|
89
|
+
restore_signals=True,
|
|
97
90
|
)
|
|
98
91
|
|
|
99
92
|
self.log.info("Output:")
|
|
@@ -25,23 +25,14 @@ from functools import cached_property
|
|
|
25
25
|
from typing import TYPE_CHECKING, Any, cast
|
|
26
26
|
|
|
27
27
|
from airflow.exceptions import AirflowException, AirflowSkipException
|
|
28
|
+
from airflow.providers.common.compat.sdk import context_to_airflow_vars
|
|
28
29
|
from airflow.providers.standard.hooks.subprocess import SubprocessHook, SubprocessResult, working_directory
|
|
29
|
-
from airflow.providers.standard.version_compat import
|
|
30
|
-
|
|
31
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
32
|
-
from airflow.sdk.execution_time.context import context_to_airflow_vars
|
|
33
|
-
else:
|
|
34
|
-
from airflow.utils.operator_helpers import context_to_airflow_vars # type: ignore[no-redef, attr-defined]
|
|
30
|
+
from airflow.providers.standard.version_compat import BaseOperator
|
|
35
31
|
|
|
36
32
|
if TYPE_CHECKING:
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
37
34
|
from airflow.utils.types import ArgNotSet
|
|
38
35
|
|
|
39
|
-
try:
|
|
40
|
-
from airflow.sdk.definitions.context import Context
|
|
41
|
-
except ImportError:
|
|
42
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
43
|
-
from airflow.utils.context import Context
|
|
44
|
-
|
|
45
36
|
|
|
46
37
|
class BashOperator(BaseOperator):
|
|
47
38
|
r"""
|
|
@@ -30,7 +30,7 @@ else:
|
|
|
30
30
|
from airflow.models.skipmixin import SkipMixin
|
|
31
31
|
|
|
32
32
|
if TYPE_CHECKING:
|
|
33
|
-
from airflow.
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
34
34
|
from airflow.sdk.types import RuntimeTaskInstanceProtocol
|
|
35
35
|
|
|
36
36
|
|
|
@@ -21,15 +21,11 @@ from collections.abc import Iterable
|
|
|
21
21
|
from typing import TYPE_CHECKING
|
|
22
22
|
|
|
23
23
|
from airflow.exceptions import AirflowException
|
|
24
|
+
from airflow.providers.common.compat.sdk import timezone
|
|
24
25
|
from airflow.providers.standard.operators.branch import BaseBranchOperator
|
|
25
|
-
from airflow.utils import timezone
|
|
26
26
|
|
|
27
27
|
if TYPE_CHECKING:
|
|
28
|
-
|
|
29
|
-
from airflow.sdk.definitions.context import Context
|
|
30
|
-
except ImportError:
|
|
31
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
32
|
-
from airflow.utils.context import Context
|
|
28
|
+
from airflow.providers.common.compat.sdk import Context
|
|
33
29
|
|
|
34
30
|
|
|
35
31
|
class BranchDateTimeOperator(BaseBranchOperator):
|
|
@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
|
|
|
21
21
|
from airflow.providers.standard.version_compat import BaseOperator
|
|
22
22
|
|
|
23
23
|
if TYPE_CHECKING:
|
|
24
|
-
from airflow.
|
|
24
|
+
from airflow.providers.common.compat.sdk import Context
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class EmptyOperator(BaseOperator):
|
|
@@ -19,7 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
import logging
|
|
20
20
|
|
|
21
21
|
from airflow.exceptions import AirflowOptionalProviderFeatureException
|
|
22
|
-
from airflow.providers.standard.version_compat import AIRFLOW_V_3_1_PLUS
|
|
22
|
+
from airflow.providers.standard.version_compat import AIRFLOW_V_3_1_3_PLUS, AIRFLOW_V_3_1_PLUS
|
|
23
23
|
|
|
24
24
|
if not AIRFLOW_V_3_1_PLUS:
|
|
25
25
|
raise AirflowOptionalProviderFeatureException("Human in the loop functionality needs Airflow 3.1+.")
|
|
@@ -40,7 +40,7 @@ from airflow.sdk.execution_time.hitl import upsert_hitl_detail
|
|
|
40
40
|
from airflow.sdk.timezone import utcnow
|
|
41
41
|
|
|
42
42
|
if TYPE_CHECKING:
|
|
43
|
-
from airflow.
|
|
43
|
+
from airflow.providers.common.compat.sdk import Context
|
|
44
44
|
from airflow.sdk.execution_time.hitl import HITLUser
|
|
45
45
|
from airflow.sdk.types import RuntimeTaskInstanceProtocol
|
|
46
46
|
|
|
@@ -84,6 +84,7 @@ class HITLOperator(BaseOperator):
|
|
|
84
84
|
self.multiple = multiple
|
|
85
85
|
|
|
86
86
|
self.params: ParamsDict = params if isinstance(params, ParamsDict) else ParamsDict(params or {})
|
|
87
|
+
|
|
87
88
|
self.notifiers: Sequence[BaseNotifier] = (
|
|
88
89
|
[notifiers] if isinstance(notifiers, BaseNotifier) else notifiers or []
|
|
89
90
|
)
|
|
@@ -110,6 +111,7 @@ class HITLOperator(BaseOperator):
|
|
|
110
111
|
Raises:
|
|
111
112
|
ValueError: If `"_options"` key is present in `params`, which is not allowed.
|
|
112
113
|
"""
|
|
114
|
+
self.params.validate()
|
|
113
115
|
if "_options" in self.params:
|
|
114
116
|
raise ValueError('"_options" is not allowed in params')
|
|
115
117
|
|
|
@@ -165,8 +167,10 @@ class HITLOperator(BaseOperator):
|
|
|
165
167
|
)
|
|
166
168
|
|
|
167
169
|
@property
|
|
168
|
-
def serialized_params(self) -> dict[str, Any]:
|
|
169
|
-
|
|
170
|
+
def serialized_params(self) -> dict[str, dict[str, Any]]:
|
|
171
|
+
if not AIRFLOW_V_3_1_3_PLUS:
|
|
172
|
+
return self.params.dump() if isinstance(self.params, ParamsDict) else self.params
|
|
173
|
+
return {k: self.params.get_param(k).serialize() for k in self.params}
|
|
170
174
|
|
|
171
175
|
def execute_complete(self, context: Context, event: dict[str, Any]) -> Any:
|
|
172
176
|
if "error" in event:
|
|
@@ -196,13 +200,12 @@ class HITLOperator(BaseOperator):
|
|
|
196
200
|
|
|
197
201
|
def validate_params_input(self, params_input: Mapping) -> None:
|
|
198
202
|
"""Check whether user provide valid params input."""
|
|
199
|
-
if (
|
|
200
|
-
self.serialized_params is not None
|
|
201
|
-
and params_input is not None
|
|
202
|
-
and set(self.serialized_params.keys()) ^ set(params_input)
|
|
203
|
-
):
|
|
203
|
+
if self.params and params_input and set(self.serialized_params.keys()) ^ set(params_input):
|
|
204
204
|
raise ValueError(f"params_input {params_input} does not match params {self.params}")
|
|
205
205
|
|
|
206
|
+
for key, value in params_input.items():
|
|
207
|
+
self.params[key] = value
|
|
208
|
+
|
|
206
209
|
def generate_link_to_ui(
|
|
207
210
|
self,
|
|
208
211
|
*,
|
|
@@ -33,12 +33,7 @@ if TYPE_CHECKING:
|
|
|
33
33
|
from pendulum.datetime import DateTime
|
|
34
34
|
|
|
35
35
|
from airflow.models import DagRun
|
|
36
|
-
|
|
37
|
-
try:
|
|
38
|
-
from airflow.sdk.definitions.context import Context
|
|
39
|
-
except ImportError:
|
|
40
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
41
|
-
from airflow.utils.context import Context
|
|
36
|
+
from airflow.providers.common.compat.sdk import Context
|
|
42
37
|
|
|
43
38
|
|
|
44
39
|
class LatestOnlyOperator(BaseBranchOperator):
|
|
@@ -93,9 +88,9 @@ class LatestOnlyOperator(BaseBranchOperator):
|
|
|
93
88
|
def _get_compare_dates(self, dag_run: DagRun) -> tuple[DateTime, DateTime] | None:
|
|
94
89
|
dagrun_date: DateTime
|
|
95
90
|
if AIRFLOW_V_3_0_PLUS:
|
|
96
|
-
dagrun_date = dag_run.logical_date or dag_run.run_after
|
|
91
|
+
dagrun_date = dag_run.logical_date or dag_run.run_after # type: ignore[assignment]
|
|
97
92
|
else:
|
|
98
|
-
dagrun_date = dag_run.logical_date
|
|
93
|
+
dagrun_date = dag_run.logical_date # type: ignore[assignment]
|
|
99
94
|
|
|
100
95
|
from airflow.timetables.base import DataInterval, TimeRestriction
|
|
101
96
|
|
|
@@ -49,13 +49,17 @@ from airflow.exceptions import (
|
|
|
49
49
|
DeserializingResultError,
|
|
50
50
|
)
|
|
51
51
|
from airflow.models.variable import Variable
|
|
52
|
+
from airflow.providers.common.compat.sdk import context_merge
|
|
52
53
|
from airflow.providers.standard.hooks.package_index import PackageIndexHook
|
|
53
|
-
from airflow.providers.standard.utils.python_virtualenv import
|
|
54
|
-
|
|
54
|
+
from airflow.providers.standard.utils.python_virtualenv import (
|
|
55
|
+
_execute_in_subprocess,
|
|
56
|
+
prepare_virtualenv,
|
|
57
|
+
write_python_script,
|
|
58
|
+
)
|
|
59
|
+
from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS, BaseOperator
|
|
55
60
|
from airflow.utils import hashlib_wrapper
|
|
56
61
|
from airflow.utils.file import get_unique_dag_module_name
|
|
57
62
|
from airflow.utils.operator_helpers import KeywordParameters
|
|
58
|
-
from airflow.utils.process_utils import execute_in_subprocess
|
|
59
63
|
|
|
60
64
|
if AIRFLOW_V_3_0_PLUS:
|
|
61
65
|
from airflow.providers.standard.operators.branch import BaseBranchOperator
|
|
@@ -72,14 +76,10 @@ if TYPE_CHECKING:
|
|
|
72
76
|
|
|
73
77
|
from pendulum.datetime import DateTime
|
|
74
78
|
|
|
79
|
+
from airflow.providers.common.compat.sdk import Context
|
|
75
80
|
from airflow.sdk.execution_time.callback_runner import ExecutionCallableRunner
|
|
76
81
|
from airflow.sdk.execution_time.context import OutletEventAccessorsProtocol
|
|
77
82
|
|
|
78
|
-
try:
|
|
79
|
-
from airflow.sdk.definitions.context import Context
|
|
80
|
-
except ImportError: # TODO: Remove once provider drops support for Airflow 2
|
|
81
|
-
from airflow.utils.context import Context
|
|
82
|
-
|
|
83
83
|
_SerializerTypeDef = Literal["pickle", "cloudpickle", "dill"]
|
|
84
84
|
|
|
85
85
|
|
|
@@ -575,7 +575,7 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta):
|
|
|
575
575
|
os.fspath(termination_log_path),
|
|
576
576
|
os.fspath(airflow_context_path),
|
|
577
577
|
]
|
|
578
|
-
|
|
578
|
+
_execute_in_subprocess(
|
|
579
579
|
cmd=cmd,
|
|
580
580
|
env=env_vars,
|
|
581
581
|
)
|