apache-airflow-providers-databricks 7.7.3rc1__py3-none-any.whl → 7.7.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/databricks/__init__.py +1 -1
- airflow/providers/databricks/operators/databricks.py +2 -8
- airflow/providers/databricks/operators/databricks_repos.py +2 -6
- airflow/providers/databricks/operators/databricks_sql.py +2 -2
- airflow/providers/databricks/operators/databricks_workflow.py +3 -7
- airflow/providers/databricks/plugins/databricks_workflow.py +94 -103
- airflow/providers/databricks/sensors/databricks.py +2 -7
- airflow/providers/databricks/sensors/databricks_partition.py +2 -11
- airflow/providers/databricks/sensors/databricks_sql.py +2 -11
- airflow/providers/databricks/version_compat.py +0 -12
- {apache_airflow_providers_databricks-7.7.3rc1.dist-info → apache_airflow_providers_databricks-7.7.4.dist-info}/METADATA +12 -13
- {apache_airflow_providers_databricks-7.7.3rc1.dist-info → apache_airflow_providers_databricks-7.7.4.dist-info}/RECORD +14 -14
- {apache_airflow_providers_databricks-7.7.3rc1.dist-info → apache_airflow_providers_databricks-7.7.4.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1.dist-info → apache_airflow_providers_databricks-7.7.4.dist-info}/entry_points.txt +0 -0
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "7.7.
|
|
32
|
+
__version__ = "7.7.4"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.10.0"
|
|
@@ -28,6 +28,7 @@ from typing import TYPE_CHECKING, Any
|
|
|
28
28
|
|
|
29
29
|
from airflow.configuration import conf
|
|
30
30
|
from airflow.exceptions import AirflowException
|
|
31
|
+
from airflow.providers.common.compat.sdk import BaseOperator, BaseOperatorLink, XCom
|
|
31
32
|
from airflow.providers.databricks.hooks.databricks import (
|
|
32
33
|
DatabricksHook,
|
|
33
34
|
RunLifeCycleState,
|
|
@@ -51,7 +52,7 @@ from airflow.providers.databricks.utils.databricks import (
|
|
|
51
52
|
validate_trigger_event,
|
|
52
53
|
)
|
|
53
54
|
from airflow.providers.databricks.utils.mixins import DatabricksSQLStatementsMixin
|
|
54
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
55
|
+
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
55
56
|
|
|
56
57
|
if TYPE_CHECKING:
|
|
57
58
|
from airflow.models.taskinstancekey import TaskInstanceKey
|
|
@@ -62,13 +63,6 @@ if TYPE_CHECKING:
|
|
|
62
63
|
from airflow.sdk import TaskGroup
|
|
63
64
|
from airflow.sdk.types import Context, Logger
|
|
64
65
|
|
|
65
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
66
|
-
from airflow.sdk import BaseOperatorLink
|
|
67
|
-
from airflow.sdk.execution_time.xcom import XCom
|
|
68
|
-
else:
|
|
69
|
-
from airflow.models import XCom
|
|
70
|
-
from airflow.models.baseoperatorlink import BaseOperatorLink # type: ignore[no-redef]
|
|
71
|
-
|
|
72
66
|
DEFER_METHOD_NAME = "execute_complete"
|
|
73
67
|
XCOM_RUN_ID_KEY = "run_id"
|
|
74
68
|
XCOM_JOB_ID_KEY = "job_id"
|
|
@@ -26,15 +26,11 @@ from typing import TYPE_CHECKING
|
|
|
26
26
|
from urllib.parse import urlsplit
|
|
27
27
|
|
|
28
28
|
from airflow.exceptions import AirflowException
|
|
29
|
+
from airflow.providers.common.compat.sdk import BaseOperator
|
|
29
30
|
from airflow.providers.databricks.hooks.databricks import DatabricksHook
|
|
30
|
-
from airflow.providers.databricks.version_compat import BaseOperator
|
|
31
31
|
|
|
32
32
|
if TYPE_CHECKING:
|
|
33
|
-
|
|
34
|
-
from airflow.sdk.definitions.context import Context
|
|
35
|
-
except ImportError:
|
|
36
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
37
|
-
from airflow.utils.context import Context
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
38
34
|
|
|
39
35
|
|
|
40
36
|
class DatabricksReposCreateOperator(BaseOperator):
|
|
@@ -28,12 +28,12 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
|
|
28
28
|
from databricks.sql.utils import ParamEscaper
|
|
29
29
|
|
|
30
30
|
from airflow.exceptions import AirflowException
|
|
31
|
+
from airflow.providers.common.compat.sdk import BaseOperator
|
|
31
32
|
from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
|
|
32
33
|
from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
|
|
33
|
-
from airflow.providers.databricks.version_compat import BaseOperator
|
|
34
34
|
|
|
35
35
|
if TYPE_CHECKING:
|
|
36
|
-
from airflow.
|
|
36
|
+
from airflow.providers.common.compat.sdk import Context
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
class DatabricksSqlOperator(SQLExecuteQueryOperator):
|
|
@@ -26,24 +26,20 @@ from typing import TYPE_CHECKING, Any
|
|
|
26
26
|
from mergedeep import merge
|
|
27
27
|
|
|
28
28
|
from airflow.exceptions import AirflowException
|
|
29
|
+
from airflow.providers.common.compat.sdk import BaseOperator, TaskGroup
|
|
29
30
|
from airflow.providers.databricks.hooks.databricks import DatabricksHook, RunLifeCycleState
|
|
30
31
|
from airflow.providers.databricks.plugins.databricks_workflow import (
|
|
31
32
|
WorkflowJobRepairAllFailedLink,
|
|
32
33
|
WorkflowJobRunLink,
|
|
33
34
|
store_databricks_job_run_link,
|
|
34
35
|
)
|
|
35
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
36
|
-
|
|
37
|
-
try:
|
|
38
|
-
from airflow.sdk import TaskGroup
|
|
39
|
-
except ImportError:
|
|
40
|
-
from airflow.utils.task_group import TaskGroup # type: ignore[no-redef]
|
|
36
|
+
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
41
37
|
|
|
42
38
|
if TYPE_CHECKING:
|
|
43
39
|
from types import TracebackType
|
|
44
40
|
|
|
45
41
|
from airflow.models.taskmixin import DAGNode
|
|
46
|
-
from airflow.
|
|
42
|
+
from airflow.providers.common.compat.sdk import Context
|
|
47
43
|
|
|
48
44
|
|
|
49
45
|
@dataclass
|
|
@@ -21,21 +21,13 @@ import os
|
|
|
21
21
|
from typing import TYPE_CHECKING, Any
|
|
22
22
|
from urllib.parse import unquote
|
|
23
23
|
|
|
24
|
-
from flask import flash, redirect, request, url_for
|
|
25
|
-
from flask_appbuilder import BaseView
|
|
26
|
-
from flask_appbuilder.api import expose
|
|
27
|
-
|
|
28
24
|
from airflow.exceptions import AirflowException, TaskInstanceNotFound
|
|
29
25
|
from airflow.models.dagrun import DagRun
|
|
30
26
|
from airflow.models.taskinstance import TaskInstance, TaskInstanceKey, clear_task_instances
|
|
31
27
|
from airflow.plugins_manager import AirflowPlugin
|
|
28
|
+
from airflow.providers.common.compat.sdk import BaseOperatorLink, TaskGroup, XCom
|
|
32
29
|
from airflow.providers.databricks.hooks.databricks import DatabricksHook
|
|
33
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
34
|
-
|
|
35
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
36
|
-
from airflow.providers.fab.www import auth
|
|
37
|
-
else:
|
|
38
|
-
from airflow.www import auth # type: ignore
|
|
30
|
+
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
39
31
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
40
32
|
from airflow.utils.state import TaskInstanceState
|
|
41
33
|
|
|
@@ -43,24 +35,15 @@ if TYPE_CHECKING:
|
|
|
43
35
|
from sqlalchemy.orm.session import Session
|
|
44
36
|
|
|
45
37
|
from airflow.models import BaseOperator
|
|
38
|
+
from airflow.providers.common.compat.sdk import Context
|
|
46
39
|
from airflow.providers.databricks.operators.databricks import DatabricksTaskBaseOperator
|
|
47
40
|
from airflow.sdk.types import Logger
|
|
48
|
-
from airflow.utils.context import Context
|
|
49
41
|
|
|
50
42
|
|
|
51
43
|
REPAIR_WAIT_ATTEMPTS = os.getenv("DATABRICKS_REPAIR_WAIT_ATTEMPTS", 20)
|
|
52
44
|
REPAIR_WAIT_DELAY = os.getenv("DATABRICKS_REPAIR_WAIT_DELAY", 0.5)
|
|
53
45
|
|
|
54
46
|
|
|
55
|
-
def get_auth_decorator():
|
|
56
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
57
|
-
from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
|
|
58
|
-
else:
|
|
59
|
-
from airflow.auth.managers.models.resource_details import DagAccessEntity
|
|
60
|
-
|
|
61
|
-
return auth.has_access_dag("POST", DagAccessEntity.RUN)
|
|
62
|
-
|
|
63
|
-
|
|
64
47
|
def get_databricks_task_ids(
|
|
65
48
|
group_id: str, task_map: dict[str, DatabricksTaskBaseOperator], log: Logger
|
|
66
49
|
) -> list[str]:
|
|
@@ -86,7 +69,64 @@ def get_databricks_task_ids(
|
|
|
86
69
|
# TODO: Need to re-think on how to support the currently unavailable repair functionality in Airflow 3. Probably a
|
|
87
70
|
# good time to re-evaluate this would be once the plugin functionality is expanded in Airflow 3.1.
|
|
88
71
|
if not AIRFLOW_V_3_0_PLUS:
|
|
72
|
+
from flask import flash, redirect, request, url_for
|
|
73
|
+
from flask_appbuilder import BaseView
|
|
74
|
+
from flask_appbuilder.api import expose
|
|
75
|
+
|
|
89
76
|
from airflow.utils.session import NEW_SESSION, provide_session
|
|
77
|
+
from airflow.www import auth
|
|
78
|
+
|
|
79
|
+
def get_auth_decorator():
|
|
80
|
+
from airflow.auth.managers.models.resource_details import DagAccessEntity
|
|
81
|
+
|
|
82
|
+
return auth.has_access_dag("POST", DagAccessEntity.RUN)
|
|
83
|
+
|
|
84
|
+
class RepairDatabricksTasks(BaseView, LoggingMixin):
|
|
85
|
+
"""Repair databricks tasks from Airflow."""
|
|
86
|
+
|
|
87
|
+
default_view = "repair"
|
|
88
|
+
|
|
89
|
+
@expose("/repair_databricks_job/<string:dag_id>/<string:run_id>", methods=("GET",))
|
|
90
|
+
@get_auth_decorator()
|
|
91
|
+
def repair(self, dag_id: str, run_id: str):
|
|
92
|
+
return_url = self._get_return_url(dag_id, run_id)
|
|
93
|
+
|
|
94
|
+
tasks_to_repair = request.values.get("tasks_to_repair")
|
|
95
|
+
self.log.info("Tasks to repair: %s", tasks_to_repair)
|
|
96
|
+
if not tasks_to_repair:
|
|
97
|
+
flash("No tasks to repair. Not sending repair request.")
|
|
98
|
+
return redirect(return_url)
|
|
99
|
+
|
|
100
|
+
databricks_conn_id = request.values.get("databricks_conn_id")
|
|
101
|
+
databricks_run_id = request.values.get("databricks_run_id")
|
|
102
|
+
|
|
103
|
+
if not databricks_conn_id:
|
|
104
|
+
flash("No Databricks connection ID provided. Cannot repair tasks.")
|
|
105
|
+
return redirect(return_url)
|
|
106
|
+
|
|
107
|
+
if not databricks_run_id:
|
|
108
|
+
flash("No Databricks run ID provided. Cannot repair tasks.")
|
|
109
|
+
return redirect(return_url)
|
|
110
|
+
|
|
111
|
+
self.log.info("Repairing databricks job %s", databricks_run_id)
|
|
112
|
+
res = _repair_task(
|
|
113
|
+
databricks_conn_id=databricks_conn_id,
|
|
114
|
+
databricks_run_id=int(databricks_run_id),
|
|
115
|
+
tasks_to_repair=tasks_to_repair.split(","),
|
|
116
|
+
logger=self.log,
|
|
117
|
+
)
|
|
118
|
+
self.log.info("Repairing databricks job query for run %s sent", databricks_run_id)
|
|
119
|
+
|
|
120
|
+
self.log.info("Clearing tasks to rerun in airflow")
|
|
121
|
+
|
|
122
|
+
run_id = unquote(run_id)
|
|
123
|
+
_clear_task_instances(dag_id, run_id, tasks_to_repair.split(","), self.log)
|
|
124
|
+
flash(f"Databricks repair job is starting!: {res}")
|
|
125
|
+
return redirect(return_url)
|
|
126
|
+
|
|
127
|
+
@staticmethod
|
|
128
|
+
def _get_return_url(dag_id: str, run_id: str) -> str:
|
|
129
|
+
return url_for("Airflow.grid", dag_id=dag_id, dag_run_id=run_id)
|
|
90
130
|
|
|
91
131
|
def _get_dag(dag_id: str, session: Session):
|
|
92
132
|
from airflow.models.serialized_dag import SerializedDagModel
|
|
@@ -140,46 +180,45 @@ if not AIRFLOW_V_3_0_PLUS:
|
|
|
140
180
|
raise TaskInstanceNotFound("Task instance not found")
|
|
141
181
|
return ti
|
|
142
182
|
|
|
183
|
+
def _repair_task(
|
|
184
|
+
databricks_conn_id: str,
|
|
185
|
+
databricks_run_id: int,
|
|
186
|
+
tasks_to_repair: list[str],
|
|
187
|
+
logger: Logger,
|
|
188
|
+
) -> int:
|
|
189
|
+
"""
|
|
190
|
+
Repair a Databricks task using the Databricks API.
|
|
143
191
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
databricks_run_id: int,
|
|
147
|
-
tasks_to_repair: list[str],
|
|
148
|
-
logger: Logger,
|
|
149
|
-
) -> int:
|
|
150
|
-
"""
|
|
151
|
-
Repair a Databricks task using the Databricks API.
|
|
152
|
-
|
|
153
|
-
This function allows the Airflow retry function to create a repair job for Databricks.
|
|
154
|
-
It uses the Databricks API to get the latest repair ID before sending the repair query.
|
|
192
|
+
This function allows the Airflow retry function to create a repair job for Databricks.
|
|
193
|
+
It uses the Databricks API to get the latest repair ID before sending the repair query.
|
|
155
194
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
195
|
+
:param databricks_conn_id: The Databricks connection ID.
|
|
196
|
+
:param databricks_run_id: The Databricks run ID.
|
|
197
|
+
:param tasks_to_repair: A list of Databricks task IDs to repair.
|
|
198
|
+
:param logger: The logger to use for logging.
|
|
199
|
+
:return: None
|
|
200
|
+
"""
|
|
201
|
+
hook = DatabricksHook(databricks_conn_id=databricks_conn_id)
|
|
202
|
+
|
|
203
|
+
repair_history_id = hook.get_latest_repair_id(databricks_run_id)
|
|
204
|
+
logger.debug("Latest repair ID is %s", repair_history_id)
|
|
205
|
+
logger.debug(
|
|
206
|
+
"Sending repair query for tasks %s on run %s",
|
|
207
|
+
tasks_to_repair,
|
|
208
|
+
databricks_run_id,
|
|
209
|
+
)
|
|
171
210
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
211
|
+
run_data = hook.get_run(databricks_run_id)
|
|
212
|
+
repair_json = {
|
|
213
|
+
"run_id": databricks_run_id,
|
|
214
|
+
"latest_repair_id": repair_history_id,
|
|
215
|
+
"rerun_tasks": tasks_to_repair,
|
|
216
|
+
}
|
|
178
217
|
|
|
179
|
-
|
|
180
|
-
|
|
218
|
+
if "overriding_parameters" in run_data:
|
|
219
|
+
repair_json["overriding_parameters"] = run_data["overriding_parameters"]
|
|
181
220
|
|
|
182
|
-
|
|
221
|
+
return hook.repair_run(repair_json)
|
|
183
222
|
|
|
184
223
|
|
|
185
224
|
def get_launch_task_id(task_group: TaskGroup) -> str:
|
|
@@ -461,54 +500,6 @@ class WorkflowJobRepairSingleTaskLink(BaseOperatorLink, LoggingMixin):
|
|
|
461
500
|
return url_for("RepairDatabricksTasks.repair", **query_params)
|
|
462
501
|
|
|
463
502
|
|
|
464
|
-
class RepairDatabricksTasks(BaseView, LoggingMixin):
|
|
465
|
-
"""Repair databricks tasks from Airflow."""
|
|
466
|
-
|
|
467
|
-
default_view = "repair"
|
|
468
|
-
|
|
469
|
-
@expose("/repair_databricks_job/<string:dag_id>/<string:run_id>", methods=("GET",))
|
|
470
|
-
@get_auth_decorator()
|
|
471
|
-
def repair(self, dag_id: str, run_id: str):
|
|
472
|
-
return_url = self._get_return_url(dag_id, run_id)
|
|
473
|
-
|
|
474
|
-
tasks_to_repair = request.values.get("tasks_to_repair")
|
|
475
|
-
self.log.info("Tasks to repair: %s", tasks_to_repair)
|
|
476
|
-
if not tasks_to_repair:
|
|
477
|
-
flash("No tasks to repair. Not sending repair request.")
|
|
478
|
-
return redirect(return_url)
|
|
479
|
-
|
|
480
|
-
databricks_conn_id = request.values.get("databricks_conn_id")
|
|
481
|
-
databricks_run_id = request.values.get("databricks_run_id")
|
|
482
|
-
|
|
483
|
-
if not databricks_conn_id:
|
|
484
|
-
flash("No Databricks connection ID provided. Cannot repair tasks.")
|
|
485
|
-
return redirect(return_url)
|
|
486
|
-
|
|
487
|
-
if not databricks_run_id:
|
|
488
|
-
flash("No Databricks run ID provided. Cannot repair tasks.")
|
|
489
|
-
return redirect(return_url)
|
|
490
|
-
|
|
491
|
-
self.log.info("Repairing databricks job %s", databricks_run_id)
|
|
492
|
-
res = _repair_task(
|
|
493
|
-
databricks_conn_id=databricks_conn_id,
|
|
494
|
-
databricks_run_id=int(databricks_run_id),
|
|
495
|
-
tasks_to_repair=tasks_to_repair.split(","),
|
|
496
|
-
logger=self.log,
|
|
497
|
-
)
|
|
498
|
-
self.log.info("Repairing databricks job query for run %s sent", databricks_run_id)
|
|
499
|
-
|
|
500
|
-
self.log.info("Clearing tasks to rerun in airflow")
|
|
501
|
-
|
|
502
|
-
run_id = unquote(run_id)
|
|
503
|
-
_clear_task_instances(dag_id, run_id, tasks_to_repair.split(","), self.log)
|
|
504
|
-
flash(f"Databricks repair job is starting!: {res}")
|
|
505
|
-
return redirect(return_url)
|
|
506
|
-
|
|
507
|
-
@staticmethod
|
|
508
|
-
def _get_return_url(dag_id: str, run_id: str) -> str:
|
|
509
|
-
return url_for("Airflow.grid", dag_id=dag_id, dag_run_id=run_id)
|
|
510
|
-
|
|
511
|
-
|
|
512
503
|
class DatabricksWorkflowPlugin(AirflowPlugin):
|
|
513
504
|
"""
|
|
514
505
|
Databricks Workflows plugin for Airflow.
|
|
@@ -24,18 +24,13 @@ from typing import TYPE_CHECKING, Any
|
|
|
24
24
|
|
|
25
25
|
from airflow.configuration import conf
|
|
26
26
|
from airflow.exceptions import AirflowException
|
|
27
|
+
from airflow.providers.common.compat.sdk import BaseSensorOperator
|
|
27
28
|
from airflow.providers.databricks.hooks.databricks import DatabricksHook, SQLStatementState
|
|
28
29
|
from airflow.providers.databricks.operators.databricks import DEFER_METHOD_NAME
|
|
29
30
|
from airflow.providers.databricks.utils.mixins import DatabricksSQLStatementsMixin
|
|
30
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
31
|
-
|
|
32
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
33
|
-
from airflow.sdk.bases.sensor import BaseSensorOperator
|
|
34
|
-
else:
|
|
35
|
-
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
|
36
31
|
|
|
37
32
|
if TYPE_CHECKING:
|
|
38
|
-
from airflow.
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
39
34
|
|
|
40
35
|
XCOM_STATEMENT_ID_KEY = "statement_id"
|
|
41
36
|
|
|
@@ -28,21 +28,12 @@ from typing import TYPE_CHECKING, Any
|
|
|
28
28
|
from databricks.sql.utils import ParamEscaper
|
|
29
29
|
|
|
30
30
|
from airflow.exceptions import AirflowException
|
|
31
|
+
from airflow.providers.common.compat.sdk import BaseSensorOperator
|
|
31
32
|
from airflow.providers.common.sql.hooks.handlers import fetch_all_handler
|
|
32
33
|
from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
|
|
33
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
34
|
-
|
|
35
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
36
|
-
from airflow.sdk import BaseSensorOperator
|
|
37
|
-
else:
|
|
38
|
-
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
|
39
34
|
|
|
40
35
|
if TYPE_CHECKING:
|
|
41
|
-
|
|
42
|
-
from airflow.sdk.definitions.context import Context
|
|
43
|
-
except ImportError:
|
|
44
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
45
|
-
from airflow.utils.context import Context
|
|
36
|
+
from airflow.providers.common.compat.sdk import Context
|
|
46
37
|
|
|
47
38
|
|
|
48
39
|
class DatabricksPartitionSensor(BaseSensorOperator):
|
|
@@ -25,21 +25,12 @@ from functools import cached_property
|
|
|
25
25
|
from typing import TYPE_CHECKING, Any
|
|
26
26
|
|
|
27
27
|
from airflow.exceptions import AirflowException
|
|
28
|
+
from airflow.providers.common.compat.sdk import BaseSensorOperator
|
|
28
29
|
from airflow.providers.common.sql.hooks.handlers import fetch_all_handler
|
|
29
30
|
from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
|
|
30
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
31
|
-
|
|
32
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
33
|
-
from airflow.sdk import BaseSensorOperator
|
|
34
|
-
else:
|
|
35
|
-
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
|
36
31
|
|
|
37
32
|
if TYPE_CHECKING:
|
|
38
|
-
|
|
39
|
-
from airflow.sdk.definitions.context import Context
|
|
40
|
-
except ImportError:
|
|
41
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
42
|
-
from airflow.utils.context import Context
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
43
34
|
|
|
44
35
|
|
|
45
36
|
class DatabricksSqlSensor(BaseSensorOperator):
|
|
@@ -34,18 +34,6 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
|
|
|
34
34
|
|
|
35
35
|
AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|
|
36
36
|
|
|
37
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
38
|
-
from airflow.sdk import BaseOperator, BaseOperatorLink, TaskGroup
|
|
39
|
-
from airflow.sdk.execution_time.xcom import XCom
|
|
40
|
-
else:
|
|
41
|
-
from airflow.models import BaseOperator, XCom
|
|
42
|
-
from airflow.models.baseoperatorlink import BaseOperatorLink # type: ignore[no-redef]
|
|
43
|
-
from airflow.utils.task_group import TaskGroup # type: ignore[no-redef]
|
|
44
|
-
|
|
45
37
|
__all__ = [
|
|
46
38
|
"AIRFLOW_V_3_0_PLUS",
|
|
47
|
-
"BaseOperator",
|
|
48
|
-
"BaseOperatorLink",
|
|
49
|
-
"TaskGroup",
|
|
50
|
-
"XCom",
|
|
51
39
|
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-databricks
|
|
3
|
-
Version: 7.7.
|
|
3
|
+
Version: 7.7.4
|
|
4
4
|
Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,databricks,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -20,9 +20,9 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
|
-
Requires-Dist: apache-airflow>=2.10.
|
|
24
|
-
Requires-Dist: apache-airflow-providers-common-compat>=1.
|
|
25
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.27.
|
|
23
|
+
Requires-Dist: apache-airflow>=2.10.0
|
|
24
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.8.0
|
|
25
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
|
|
26
26
|
Requires-Dist: requests>=2.32.0,<3
|
|
27
27
|
Requires-Dist: databricks-sql-connector>=4.0.0
|
|
28
28
|
Requires-Dist: databricks-sqlalchemy>=1.0.2
|
|
@@ -33,13 +33,13 @@ Requires-Dist: pandas>=2.2.3; python_version >="3.13"
|
|
|
33
33
|
Requires-Dist: pyarrow>=16.1.0; python_version < '3.13'
|
|
34
34
|
Requires-Dist: pyarrow>=18.0.0; python_version >= '3.13'
|
|
35
35
|
Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
|
|
36
|
-
Requires-Dist: apache-airflow-providers-fab>=2.2.
|
|
37
|
-
Requires-Dist: apache-airflow-providers-openlineage>=2.3.
|
|
36
|
+
Requires-Dist: apache-airflow-providers-fab>=2.2.0 ; extra == "fab" and ( python_version < '3.13')
|
|
37
|
+
Requires-Dist: apache-airflow-providers-openlineage>=2.3.0 ; extra == "openlineage"
|
|
38
38
|
Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
|
|
39
39
|
Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
|
|
40
40
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
41
|
-
Project-URL: Changelog, https://airflow.
|
|
42
|
-
Project-URL: Documentation, https://airflow.
|
|
41
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/changelog.html
|
|
42
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4
|
|
43
43
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
44
44
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
45
45
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -75,7 +75,7 @@ Provides-Extra: standard
|
|
|
75
75
|
|
|
76
76
|
Package ``apache-airflow-providers-databricks``
|
|
77
77
|
|
|
78
|
-
Release: ``7.7.
|
|
78
|
+
Release: ``7.7.4``
|
|
79
79
|
|
|
80
80
|
|
|
81
81
|
`Databricks <https://databricks.com/>`__
|
|
@@ -88,7 +88,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
|
|
|
88
88
|
are in ``airflow.providers.databricks`` python package.
|
|
89
89
|
|
|
90
90
|
You can find package information and changelog for the provider
|
|
91
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.
|
|
91
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/>`_.
|
|
92
92
|
|
|
93
93
|
Installation
|
|
94
94
|
------------
|
|
@@ -106,7 +106,7 @@ Requirements
|
|
|
106
106
|
PIP package Version required
|
|
107
107
|
========================================== ======================================
|
|
108
108
|
``apache-airflow`` ``>=2.10.0``
|
|
109
|
-
``apache-airflow-providers-common-compat`` ``>=1.
|
|
109
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
110
110
|
``apache-airflow-providers-common-sql`` ``>=1.27.0``
|
|
111
111
|
``requests`` ``>=2.32.0,<3``
|
|
112
112
|
``databricks-sql-connector`` ``>=4.0.0``
|
|
@@ -137,7 +137,6 @@ Dependent package
|
|
|
137
137
|
================================================================================================================== =================
|
|
138
138
|
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
139
139
|
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
|
140
|
-
`apache-airflow-providers-fab <https://airflow.apache.org/docs/apache-airflow-providers-fab>`_ ``fab``
|
|
141
140
|
`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_ ``openlineage``
|
|
142
141
|
================================================================================================================== =================
|
|
143
142
|
|
|
@@ -155,5 +154,5 @@ Extra Dependencies
|
|
|
155
154
|
================== ================================================================
|
|
156
155
|
|
|
157
156
|
The changelog for the provider package can be found in the
|
|
158
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.
|
|
157
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/changelog.html>`_.
|
|
159
158
|
|
|
@@ -1,30 +1,30 @@
|
|
|
1
1
|
airflow/providers/databricks/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
|
|
2
|
-
airflow/providers/databricks/__init__.py,sha256=
|
|
2
|
+
airflow/providers/databricks/__init__.py,sha256=47IUUh1RLJbiG9at92L1--O2ozh0fap2Swxx9hN7ySs,1499
|
|
3
3
|
airflow/providers/databricks/exceptions.py,sha256=85RklmLOI_PnTzfXNIUd5fAu2aMMUhelwumQAX0wANE,1261
|
|
4
4
|
airflow/providers/databricks/get_provider_info.py,sha256=NZ-rY6k6ctDZN7rDngN7mAzq7RMhLag5NwfnuBNcKuw,5644
|
|
5
|
-
airflow/providers/databricks/version_compat.py,sha256=
|
|
5
|
+
airflow/providers/databricks/version_compat.py,sha256=RQbdCueLOaFZWekpQmF0BoAoJInW8EoyvJ3Ah-HbrPo,1577
|
|
6
6
|
airflow/providers/databricks/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
|
7
7
|
airflow/providers/databricks/hooks/databricks.py,sha256=rkd1J73Zc8IRnMBapxRHWUvyNAEVyV5j14iBDtRdzFo,29470
|
|
8
8
|
airflow/providers/databricks/hooks/databricks_base.py,sha256=PpXH3LmHpGiWKaxBuInBCV2q7aKwLWs4xyKSx4E-sJ4,36486
|
|
9
9
|
airflow/providers/databricks/hooks/databricks_sql.py,sha256=-oGJxteTW1L7L0MLpiNeucWs3q_k2n0Ax2rLSNGc0F8,17726
|
|
10
10
|
airflow/providers/databricks/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
|
11
|
-
airflow/providers/databricks/operators/databricks.py,sha256=
|
|
12
|
-
airflow/providers/databricks/operators/databricks_repos.py,sha256=
|
|
13
|
-
airflow/providers/databricks/operators/databricks_sql.py,sha256=
|
|
14
|
-
airflow/providers/databricks/operators/databricks_workflow.py,sha256=
|
|
11
|
+
airflow/providers/databricks/operators/databricks.py,sha256=bVhFNTeGboHkmaJkYDYEyF0V1QUOB_RnsvwaCuEtIew,79316
|
|
12
|
+
airflow/providers/databricks/operators/databricks_repos.py,sha256=VRZye45ZMlDxti6ZJjuouox5umiMoeQ-BKugPpE7jnM,13155
|
|
13
|
+
airflow/providers/databricks/operators/databricks_sql.py,sha256=gwpkr660qpk4dUve98RB-hniaMzuXL6znQZZGilJxi0,21842
|
|
14
|
+
airflow/providers/databricks/operators/databricks_workflow.py,sha256=QLsR0pGLWvvQbutsjj4RWwBE-z6tkWiYLHj6waMv8ZE,15019
|
|
15
15
|
airflow/providers/databricks/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
16
|
-
airflow/providers/databricks/plugins/databricks_workflow.py,sha256=
|
|
16
|
+
airflow/providers/databricks/plugins/databricks_workflow.py,sha256=f0p-PefGpkrbJcmcG2_dGjqGwIhILG5sXBAxclWOJM8,20173
|
|
17
17
|
airflow/providers/databricks/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
18
|
-
airflow/providers/databricks/sensors/databricks.py,sha256=
|
|
19
|
-
airflow/providers/databricks/sensors/databricks_partition.py,sha256=
|
|
20
|
-
airflow/providers/databricks/sensors/databricks_sql.py,sha256=
|
|
18
|
+
airflow/providers/databricks/sensors/databricks.py,sha256=RrjSzncvppdp5U8RYHd975MCIQIb_s1VQoxm9Aqbvac,6262
|
|
19
|
+
airflow/providers/databricks/sensors/databricks_partition.py,sha256=FfhSFgxjJapiOhg50UpjczAxQGEv64gD_q0IlrjeUtY,9890
|
|
20
|
+
airflow/providers/databricks/sensors/databricks_sql.py,sha256=shq7ng4LCiaD4Q7lorm4g1A7aijmq3nVUnCFlYtoI7c,5633
|
|
21
21
|
airflow/providers/databricks/triggers/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
|
22
22
|
airflow/providers/databricks/triggers/databricks.py,sha256=DQbXLw1W_e3Iw-hsDph7vPuHc2caj623V7WmA2_PftM,8672
|
|
23
23
|
airflow/providers/databricks/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
24
24
|
airflow/providers/databricks/utils/databricks.py,sha256=ecvzZbC4KdXds47VeSayot9EO-RQnTRJTEwKITH7waQ,5117
|
|
25
25
|
airflow/providers/databricks/utils/mixins.py,sha256=WUmkt3AmXalmV6zOUIJZWbTldxYunAZOstddDhKCC94,7407
|
|
26
26
|
airflow/providers/databricks/utils/openlineage.py,sha256=1jT5Woh9YifawdP-VFWsabfF-ecuCjPlzD5P_W4DAhI,15078
|
|
27
|
-
apache_airflow_providers_databricks-7.7.
|
|
28
|
-
apache_airflow_providers_databricks-7.7.
|
|
29
|
-
apache_airflow_providers_databricks-7.7.
|
|
30
|
-
apache_airflow_providers_databricks-7.7.
|
|
27
|
+
apache_airflow_providers_databricks-7.7.4.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
|
|
28
|
+
apache_airflow_providers_databricks-7.7.4.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
|
29
|
+
apache_airflow_providers_databricks-7.7.4.dist-info/METADATA,sha256=0phl5Y_IcS09vcUV3CaEJBW1b1V2-gf7Iy_C_47X_Tc,7739
|
|
30
|
+
apache_airflow_providers_databricks-7.7.4.dist-info/RECORD,,
|
|
File without changes
|