apache-airflow-providers-snowflake 6.1.1rc1__py3-none-any.whl → 6.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-snowflake might be problematic. Click here for more details.
- airflow/providers/snowflake/__init__.py +1 -1
- airflow/providers/snowflake/get_provider_info.py +4 -3
- airflow/providers/snowflake/hooks/snowflake.py +48 -5
- airflow/providers/snowflake/operators/snowflake.py +1 -1
- airflow/providers/snowflake/utils/openlineage.py +262 -0
- airflow/providers/snowflake/version_compat.py +36 -0
- {apache_airflow_providers_snowflake-6.1.1rc1.dist-info → apache_airflow_providers_snowflake-6.2.0.dist-info}/METADATA +12 -12
- {apache_airflow_providers_snowflake-6.1.1rc1.dist-info → apache_airflow_providers_snowflake-6.2.0.dist-info}/RECORD +10 -9
- {apache_airflow_providers_snowflake-6.1.1rc1.dist-info → apache_airflow_providers_snowflake-6.2.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_snowflake-6.1.1rc1.dist-info → apache_airflow_providers_snowflake-6.2.0.dist-info}/entry_points.txt +0 -0
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "6.
|
|
32
|
+
__version__ = "6.2.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.9.0"
|
|
@@ -27,8 +27,9 @@ def get_provider_info():
|
|
|
27
27
|
"name": "Snowflake",
|
|
28
28
|
"description": "`Snowflake <https://www.snowflake.com/>`__\n",
|
|
29
29
|
"state": "ready",
|
|
30
|
-
"source-date-epoch":
|
|
30
|
+
"source-date-epoch": 1743836721,
|
|
31
31
|
"versions": [
|
|
32
|
+
"6.2.0",
|
|
32
33
|
"6.1.1",
|
|
33
34
|
"6.1.0",
|
|
34
35
|
"6.0.0",
|
|
@@ -154,7 +155,7 @@ def get_provider_info():
|
|
|
154
155
|
],
|
|
155
156
|
"dependencies": [
|
|
156
157
|
"apache-airflow>=2.9.0",
|
|
157
|
-
"apache-airflow-providers-common-compat>=1.
|
|
158
|
+
"apache-airflow-providers-common-compat>=1.6.0",
|
|
158
159
|
"apache-airflow-providers-common-sql>=1.20.0",
|
|
159
160
|
"pandas>=2.1.2,<2.2",
|
|
160
161
|
"pyarrow>=14.0.1",
|
|
@@ -163,5 +164,5 @@ def get_provider_info():
|
|
|
163
164
|
"snowflake-snowpark-python>=1.17.0;python_version<'3.12'",
|
|
164
165
|
],
|
|
165
166
|
"optional-dependencies": {"openlineage": ["apache-airflow-providers-openlineage"]},
|
|
166
|
-
"devel-dependencies": [],
|
|
167
|
+
"devel-dependencies": ["responses>=0.25.0"],
|
|
167
168
|
}
|
|
@@ -544,15 +544,41 @@ class SnowflakeHook(DbApiHook):
|
|
|
544
544
|
uri = fix_snowflake_sqlalchemy_uri(self.get_uri())
|
|
545
545
|
return urlparse(uri).hostname
|
|
546
546
|
|
|
547
|
-
def get_openlineage_database_specific_lineage(self,
|
|
547
|
+
def get_openlineage_database_specific_lineage(self, task_instance) -> OperatorLineage | None:
|
|
548
|
+
"""
|
|
549
|
+
Generate OpenLineage metadata for a Snowflake task instance based on executed query IDs.
|
|
550
|
+
|
|
551
|
+
If a single query ID is present, attach an `ExternalQueryRunFacet` to the lineage metadata.
|
|
552
|
+
If multiple query IDs are present, emits separate OpenLineage events for each query.
|
|
553
|
+
|
|
554
|
+
Note that `get_openlineage_database_specific_lineage` is usually called after task's execution,
|
|
555
|
+
so if multiple query IDs are present, both START and COMPLETE event for each query will be emitted
|
|
556
|
+
after task's execution. If we are able to query Snowflake for query execution metadata,
|
|
557
|
+
query event times will correspond to actual query's start and finish times.
|
|
558
|
+
|
|
559
|
+
Args:
|
|
560
|
+
task_instance: The Airflow TaskInstance object for which lineage is being collected.
|
|
561
|
+
|
|
562
|
+
Returns:
|
|
563
|
+
An `OperatorLineage` object if a single query ID is found; otherwise `None`.
|
|
564
|
+
"""
|
|
548
565
|
from airflow.providers.common.compat.openlineage.facet import ExternalQueryRunFacet
|
|
549
566
|
from airflow.providers.openlineage.extractors import OperatorLineage
|
|
550
567
|
from airflow.providers.openlineage.sqlparser import SQLParser
|
|
568
|
+
from airflow.providers.snowflake.utils.openlineage import (
|
|
569
|
+
emit_openlineage_events_for_snowflake_queries,
|
|
570
|
+
)
|
|
551
571
|
|
|
552
|
-
if self.query_ids:
|
|
553
|
-
self.log.debug("openlineage:
|
|
554
|
-
|
|
555
|
-
|
|
572
|
+
if not self.query_ids:
|
|
573
|
+
self.log.debug("openlineage: no snowflake query ids found.")
|
|
574
|
+
return None
|
|
575
|
+
|
|
576
|
+
self.log.debug("openlineage: getting connection to get database info")
|
|
577
|
+
connection = self.get_connection(self.get_conn_id())
|
|
578
|
+
namespace = SQLParser.create_namespace(self.get_openlineage_database_info(connection))
|
|
579
|
+
|
|
580
|
+
if len(self.query_ids) == 1:
|
|
581
|
+
self.log.debug("Attaching ExternalQueryRunFacet with single query_id to OpenLineage event.")
|
|
556
582
|
return OperatorLineage(
|
|
557
583
|
run_facets={
|
|
558
584
|
"externalQuery": ExternalQueryRunFacet(
|
|
@@ -560,4 +586,21 @@ class SnowflakeHook(DbApiHook):
|
|
|
560
586
|
)
|
|
561
587
|
}
|
|
562
588
|
)
|
|
589
|
+
|
|
590
|
+
self.log.info("Multiple query_ids found. Separate OpenLineage event will be emitted for each query.")
|
|
591
|
+
try:
|
|
592
|
+
from airflow.providers.openlineage.utils.utils import should_use_external_connection
|
|
593
|
+
|
|
594
|
+
use_external_connection = should_use_external_connection(self)
|
|
595
|
+
except ImportError:
|
|
596
|
+
# OpenLineage provider release < 1.8.0 - we always use connection
|
|
597
|
+
use_external_connection = True
|
|
598
|
+
|
|
599
|
+
emit_openlineage_events_for_snowflake_queries(
|
|
600
|
+
query_ids=self.query_ids,
|
|
601
|
+
query_source_namespace=namespace,
|
|
602
|
+
task_instance=task_instance,
|
|
603
|
+
hook=self if use_external_connection else None,
|
|
604
|
+
)
|
|
605
|
+
|
|
563
606
|
return None
|
|
@@ -505,7 +505,7 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator):
|
|
|
505
505
|
raise AirflowException(msg)
|
|
506
506
|
elif "status" in event and event["status"] == "success":
|
|
507
507
|
hook = SnowflakeSqlApiHook(snowflake_conn_id=self.snowflake_conn_id)
|
|
508
|
-
query_ids = cast(list[str], event["statement_query_ids"])
|
|
508
|
+
query_ids = cast("list[str]", event["statement_query_ids"])
|
|
509
509
|
hook.check_query_output(query_ids)
|
|
510
510
|
self.log.info("%s completed successfully.", self.task_id)
|
|
511
511
|
else:
|
|
@@ -16,8 +16,26 @@
|
|
|
16
16
|
# under the License.
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
+
import datetime
|
|
20
|
+
import logging
|
|
21
|
+
from contextlib import closing
|
|
22
|
+
from typing import TYPE_CHECKING
|
|
19
23
|
from urllib.parse import quote, urlparse, urlunparse
|
|
20
24
|
|
|
25
|
+
from airflow.providers.common.compat.openlineage.check import require_openlineage_version
|
|
26
|
+
from airflow.providers.snowflake.version_compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS
|
|
27
|
+
from airflow.utils import timezone
|
|
28
|
+
from airflow.utils.state import TaskInstanceState
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
from openlineage.client.event_v2 import RunEvent
|
|
32
|
+
from openlineage.client.facet_v2 import JobFacet
|
|
33
|
+
|
|
34
|
+
from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
log = logging.getLogger(__name__)
|
|
38
|
+
|
|
21
39
|
|
|
22
40
|
def fix_account_name(name: str) -> str:
|
|
23
41
|
"""Fix account name to have the following format: <account_id>.<region>.<cloud>."""
|
|
@@ -78,3 +96,247 @@ def fix_snowflake_sqlalchemy_uri(uri: str) -> str:
|
|
|
78
96
|
hostname = fix_account_name(hostname)
|
|
79
97
|
# else - its new hostname, just return it
|
|
80
98
|
return urlunparse((parts.scheme, hostname, parts.path, parts.params, parts.query, parts.fragment))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# todo: move this run_id logic into OpenLineage's listener to avoid differences
|
|
102
|
+
def _get_ol_run_id(task_instance) -> str:
|
|
103
|
+
"""
|
|
104
|
+
Get OpenLineage run_id from TaskInstance.
|
|
105
|
+
|
|
106
|
+
It's crucial that the task_instance's run_id creation logic matches OpenLineage's listener implementation.
|
|
107
|
+
Only then can we ensure that the generated run_id aligns with the Airflow task,
|
|
108
|
+
enabling a proper connection between events.
|
|
109
|
+
"""
|
|
110
|
+
from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
|
|
111
|
+
|
|
112
|
+
def _get_logical_date():
|
|
113
|
+
# todo: remove when min airflow version >= 3.0
|
|
114
|
+
if AIRFLOW_V_3_0_PLUS:
|
|
115
|
+
dagrun = task_instance.get_template_context()["dag_run"]
|
|
116
|
+
return dagrun.logical_date or dagrun.run_after
|
|
117
|
+
|
|
118
|
+
if hasattr(task_instance, "logical_date"):
|
|
119
|
+
date = task_instance.logical_date
|
|
120
|
+
else:
|
|
121
|
+
date = task_instance.execution_date
|
|
122
|
+
|
|
123
|
+
return date
|
|
124
|
+
|
|
125
|
+
def _get_try_number_success():
|
|
126
|
+
"""We are running this in the _on_complete, so need to adjust for try_num changes."""
|
|
127
|
+
# todo: remove when min airflow version >= 2.10.0
|
|
128
|
+
if AIRFLOW_V_2_10_PLUS:
|
|
129
|
+
return task_instance.try_number
|
|
130
|
+
if task_instance.state == TaskInstanceState.SUCCESS:
|
|
131
|
+
return task_instance.try_number - 1
|
|
132
|
+
return task_instance.try_number
|
|
133
|
+
|
|
134
|
+
# Generate same OL run id as is generated for current task instance
|
|
135
|
+
return OpenLineageAdapter.build_task_instance_run_id(
|
|
136
|
+
dag_id=task_instance.dag_id,
|
|
137
|
+
task_id=task_instance.task_id,
|
|
138
|
+
logical_date=_get_logical_date(),
|
|
139
|
+
try_number=_get_try_number_success(),
|
|
140
|
+
map_index=task_instance.map_index,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _get_parent_run_facet(task_instance):
|
|
145
|
+
"""
|
|
146
|
+
Retrieve the ParentRunFacet associated with a specific Airflow task instance.
|
|
147
|
+
|
|
148
|
+
This facet helps link OpenLineage events of child jobs - such as queries executed within
|
|
149
|
+
external systems (e.g., Snowflake) by the Airflow task - to the original Airflow task execution.
|
|
150
|
+
Establishing this connection enables better lineage tracking and observability.
|
|
151
|
+
"""
|
|
152
|
+
from openlineage.client.facet_v2 import parent_run
|
|
153
|
+
|
|
154
|
+
from airflow.providers.openlineage.conf import namespace
|
|
155
|
+
|
|
156
|
+
parent_run_id = _get_ol_run_id(task_instance)
|
|
157
|
+
|
|
158
|
+
return parent_run.ParentRunFacet(
|
|
159
|
+
run=parent_run.Run(runId=parent_run_id),
|
|
160
|
+
job=parent_run.Job(
|
|
161
|
+
namespace=namespace(),
|
|
162
|
+
name=f"{task_instance.dag_id}.{task_instance.task_id}",
|
|
163
|
+
),
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _run_single_query_with_hook(hook: SnowflakeHook, sql: str) -> list[dict]:
|
|
168
|
+
"""Execute a query against Snowflake without adding extra logging or instrumentation."""
|
|
169
|
+
with closing(hook.get_conn()) as conn:
|
|
170
|
+
hook.set_autocommit(conn, False)
|
|
171
|
+
with hook._get_cursor(conn, return_dictionaries=True) as cur:
|
|
172
|
+
cur.execute(sql)
|
|
173
|
+
result = cur.fetchall()
|
|
174
|
+
conn.commit()
|
|
175
|
+
return result
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _get_queries_details_from_snowflake(
|
|
179
|
+
hook: SnowflakeHook, query_ids: list[str]
|
|
180
|
+
) -> dict[str, dict[str, str]]:
|
|
181
|
+
"""Retrieve execution details for specific queries from Snowflake's query history."""
|
|
182
|
+
if not query_ids:
|
|
183
|
+
return {}
|
|
184
|
+
query_condition = f"IN {tuple(query_ids)}" if len(query_ids) > 1 else f"= '{query_ids[0]}'"
|
|
185
|
+
query = (
|
|
186
|
+
"SELECT "
|
|
187
|
+
"QUERY_ID, EXECUTION_STATUS, START_TIME, END_TIME, QUERY_TEXT, ERROR_CODE, ERROR_MESSAGE "
|
|
188
|
+
"FROM "
|
|
189
|
+
"table(information_schema.query_history()) "
|
|
190
|
+
f"WHERE "
|
|
191
|
+
f"QUERY_ID {query_condition}"
|
|
192
|
+
f";"
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
result = _run_single_query_with_hook(hook=hook, sql=query)
|
|
196
|
+
|
|
197
|
+
return {row["QUERY_ID"]: row for row in result} if result else {}
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _create_snowflake_event_pair(
|
|
201
|
+
job_namespace: str,
|
|
202
|
+
job_name: str,
|
|
203
|
+
start_time: datetime.datetime,
|
|
204
|
+
end_time: datetime.datetime,
|
|
205
|
+
is_successful: bool,
|
|
206
|
+
run_facets: dict | None = None,
|
|
207
|
+
job_facets: dict | None = None,
|
|
208
|
+
) -> tuple[RunEvent, RunEvent]:
|
|
209
|
+
"""Create a pair of OpenLineage RunEvents representing the start and end of a Snowflake job execution."""
|
|
210
|
+
from openlineage.client.event_v2 import Job, Run, RunEvent, RunState
|
|
211
|
+
from openlineage.client.uuid import generate_new_uuid
|
|
212
|
+
|
|
213
|
+
run = Run(runId=str(generate_new_uuid()), facets=run_facets or {})
|
|
214
|
+
job = Job(namespace=job_namespace, name=job_name, facets=job_facets or {})
|
|
215
|
+
|
|
216
|
+
start = RunEvent(
|
|
217
|
+
eventType=RunState.START,
|
|
218
|
+
eventTime=start_time.isoformat(),
|
|
219
|
+
run=run,
|
|
220
|
+
job=job,
|
|
221
|
+
)
|
|
222
|
+
end = RunEvent(
|
|
223
|
+
eventType=RunState.COMPLETE if is_successful else RunState.FAIL,
|
|
224
|
+
eventTime=end_time.isoformat(),
|
|
225
|
+
run=run,
|
|
226
|
+
job=job,
|
|
227
|
+
)
|
|
228
|
+
return start, end
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@require_openlineage_version(provider_min_version="2.0.0")
|
|
232
|
+
def emit_openlineage_events_for_snowflake_queries(
|
|
233
|
+
query_ids: list[str],
|
|
234
|
+
query_source_namespace: str,
|
|
235
|
+
task_instance,
|
|
236
|
+
hook: SnowflakeHook | None = None,
|
|
237
|
+
additional_run_facets: dict | None = None,
|
|
238
|
+
additional_job_facets: dict | None = None,
|
|
239
|
+
) -> None:
|
|
240
|
+
"""
|
|
241
|
+
Emit OpenLineage events for executed Snowflake queries.
|
|
242
|
+
|
|
243
|
+
Metadata retrieval from Snowflake is attempted only if a `SnowflakeHook` is provided.
|
|
244
|
+
If metadata is available, execution details such as start time, end time, execution status,
|
|
245
|
+
error messages, and SQL text are included in the events. If no metadata is found, the function
|
|
246
|
+
defaults to using the Airflow task instance's state and the current timestamp.
|
|
247
|
+
|
|
248
|
+
Note that both START and COMPLETE event for each query will be emitted at the same time.
|
|
249
|
+
If we are able to query Snowflake for query execution metadata, event times
|
|
250
|
+
will correspond to actual query execution times.
|
|
251
|
+
|
|
252
|
+
Args:
|
|
253
|
+
query_ids: A list of Snowflake query IDs to emit events for.
|
|
254
|
+
query_source_namespace: The namespace to be included in ExternalQueryRunFacet.
|
|
255
|
+
task_instance: The Airflow task instance that run these queries.
|
|
256
|
+
hook: A SnowflakeHook instance used to retrieve query metadata if available.
|
|
257
|
+
additional_run_facets: Additional run facets to include in OpenLineage events.
|
|
258
|
+
additional_job_facets: Additional job facets to include in OpenLineage events.
|
|
259
|
+
"""
|
|
260
|
+
from openlineage.client.facet_v2 import job_type_job
|
|
261
|
+
|
|
262
|
+
from airflow.providers.common.compat.openlineage.facet import (
|
|
263
|
+
ErrorMessageRunFacet,
|
|
264
|
+
ExternalQueryRunFacet,
|
|
265
|
+
SQLJobFacet,
|
|
266
|
+
)
|
|
267
|
+
from airflow.providers.openlineage.conf import namespace
|
|
268
|
+
from airflow.providers.openlineage.plugins.listener import get_openlineage_listener
|
|
269
|
+
|
|
270
|
+
if not query_ids:
|
|
271
|
+
log.debug("No Snowflake query IDs provided; skipping OpenLineage event emission.")
|
|
272
|
+
return
|
|
273
|
+
|
|
274
|
+
query_ids = [q for q in query_ids] # Make a copy to make sure it does not change
|
|
275
|
+
|
|
276
|
+
if hook:
|
|
277
|
+
log.debug("Retrieving metadata for %s queries from Snowflake.", len(query_ids))
|
|
278
|
+
snowflake_metadata = _get_queries_details_from_snowflake(hook, query_ids)
|
|
279
|
+
else:
|
|
280
|
+
log.debug("SnowflakeHook not provided. No extra metadata fill be fetched from Snowflake.")
|
|
281
|
+
snowflake_metadata = {}
|
|
282
|
+
|
|
283
|
+
# If real metadata is unavailable, we send events with eventTime=now
|
|
284
|
+
default_event_time = timezone.utcnow()
|
|
285
|
+
# If no query metadata is provided, we use task_instance's state when checking for success
|
|
286
|
+
default_state = str(task_instance.state) if hasattr(task_instance, "state") else ""
|
|
287
|
+
|
|
288
|
+
common_run_facets = {"parent": _get_parent_run_facet(task_instance)}
|
|
289
|
+
common_job_facets: dict[str, JobFacet] = {
|
|
290
|
+
"jobType": job_type_job.JobTypeJobFacet(
|
|
291
|
+
jobType="QUERY",
|
|
292
|
+
integration="SNOWFLAKE",
|
|
293
|
+
processingType="BATCH",
|
|
294
|
+
)
|
|
295
|
+
}
|
|
296
|
+
additional_run_facets = additional_run_facets or {}
|
|
297
|
+
additional_job_facets = additional_job_facets or {}
|
|
298
|
+
|
|
299
|
+
events: list[RunEvent] = []
|
|
300
|
+
for counter, query_id in enumerate(query_ids, 1):
|
|
301
|
+
query_metadata = snowflake_metadata.get(query_id, {})
|
|
302
|
+
log.debug(
|
|
303
|
+
"Metadata for query no. %s, (ID `%s`): `%s`",
|
|
304
|
+
counter,
|
|
305
|
+
query_id,
|
|
306
|
+
query_metadata if query_metadata else "not found",
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
# TODO(potiuk): likely typing here needs to be fixed
|
|
310
|
+
query_specific_run_facets = { # type : ignore[assignment]
|
|
311
|
+
"externalQuery": ExternalQueryRunFacet(externalQueryId=query_id, source=query_source_namespace)
|
|
312
|
+
}
|
|
313
|
+
if query_metadata.get("ERROR_MESSAGE"):
|
|
314
|
+
query_specific_run_facets["error"] = ErrorMessageRunFacet( # type: ignore[assignment]
|
|
315
|
+
message=f"{query_metadata.get('ERROR_CODE')} : {query_metadata['ERROR_MESSAGE']}",
|
|
316
|
+
programmingLanguage="SQL",
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
query_specific_job_facets = {}
|
|
320
|
+
if query_metadata.get("QUERY_TEXT"):
|
|
321
|
+
query_specific_job_facets["sql"] = SQLJobFacet(query=query_metadata["QUERY_TEXT"])
|
|
322
|
+
|
|
323
|
+
log.debug("Creating OpenLineage event pair for query ID: %s", query_id)
|
|
324
|
+
event_batch = _create_snowflake_event_pair(
|
|
325
|
+
job_namespace=namespace(),
|
|
326
|
+
job_name=f"{task_instance.dag_id}.{task_instance.task_id}.query.{counter}",
|
|
327
|
+
start_time=query_metadata.get("START_TIME", default_event_time), # type: ignore[arg-type]
|
|
328
|
+
end_time=query_metadata.get("END_TIME", default_event_time), # type: ignore[arg-type]
|
|
329
|
+
# `EXECUTION_STATUS` can be `success`, `fail` or `incident` (Snowflake outage, so still failure)
|
|
330
|
+
is_successful=query_metadata.get("EXECUTION_STATUS", default_state).lower() == "success",
|
|
331
|
+
run_facets={**query_specific_run_facets, **common_run_facets, **additional_run_facets},
|
|
332
|
+
job_facets={**query_specific_job_facets, **common_job_facets, **additional_job_facets},
|
|
333
|
+
)
|
|
334
|
+
events.extend(event_batch)
|
|
335
|
+
|
|
336
|
+
log.debug("Generated %s OpenLineage events; emitting now.", len(events))
|
|
337
|
+
client = get_openlineage_listener().adapter.get_or_create_openlineage_client()
|
|
338
|
+
for event in events:
|
|
339
|
+
client.emit(event)
|
|
340
|
+
|
|
341
|
+
log.info("OpenLineage has successfully finished processing information about Snowflake queries.")
|
|
342
|
+
return
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
#
|
|
18
|
+
# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY
|
|
19
|
+
# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS
|
|
20
|
+
# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
|
|
21
|
+
# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
|
|
22
|
+
#
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def get_base_airflow_version_tuple() -> tuple[int, int, int]:
|
|
27
|
+
from packaging.version import Version
|
|
28
|
+
|
|
29
|
+
from airflow import __version__
|
|
30
|
+
|
|
31
|
+
airflow_version = Version(__version__)
|
|
32
|
+
return airflow_version.major, airflow_version.minor, airflow_version.micro
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
|
|
36
|
+
AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-snowflake
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.2.0
|
|
4
4
|
Summary: Provider package apache-airflow-providers-snowflake for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,snowflake,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -20,9 +20,9 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
|
-
Requires-Dist: apache-airflow>=2.9.
|
|
24
|
-
Requires-Dist: apache-airflow-providers-common-compat>=1.
|
|
25
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.20.
|
|
23
|
+
Requires-Dist: apache-airflow>=2.9.0
|
|
24
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.6.0
|
|
25
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
|
|
26
26
|
Requires-Dist: pandas>=2.1.2,<2.2
|
|
27
27
|
Requires-Dist: pyarrow>=14.0.1
|
|
28
28
|
Requires-Dist: snowflake-connector-python>=3.7.1
|
|
@@ -30,11 +30,11 @@ Requires-Dist: snowflake-sqlalchemy>=1.4.0
|
|
|
30
30
|
Requires-Dist: snowflake-snowpark-python>=1.17.0;python_version<'3.12'
|
|
31
31
|
Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
|
|
32
32
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
33
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
34
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
33
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.2.0/changelog.html
|
|
34
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.2.0
|
|
35
|
+
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
35
36
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
36
37
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
37
|
-
Project-URL: Twitter, https://x.com/ApacheAirflow
|
|
38
38
|
Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
39
39
|
Provides-Extra: openlineage
|
|
40
40
|
|
|
@@ -63,7 +63,7 @@ Provides-Extra: openlineage
|
|
|
63
63
|
|
|
64
64
|
Package ``apache-airflow-providers-snowflake``
|
|
65
65
|
|
|
66
|
-
Release: ``6.
|
|
66
|
+
Release: ``6.2.0``
|
|
67
67
|
|
|
68
68
|
|
|
69
69
|
`Snowflake <https://www.snowflake.com/>`__
|
|
@@ -76,7 +76,7 @@ This is a provider package for ``snowflake`` provider. All classes for this prov
|
|
|
76
76
|
are in ``airflow.providers.snowflake`` python package.
|
|
77
77
|
|
|
78
78
|
You can find package information and changelog for the provider
|
|
79
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
79
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.2.0/>`_.
|
|
80
80
|
|
|
81
81
|
Installation
|
|
82
82
|
------------
|
|
@@ -94,7 +94,7 @@ Requirements
|
|
|
94
94
|
PIP package Version required
|
|
95
95
|
========================================== =====================================
|
|
96
96
|
``apache-airflow`` ``>=2.9.0``
|
|
97
|
-
``apache-airflow-providers-common-compat`` ``>=1.
|
|
97
|
+
``apache-airflow-providers-common-compat`` ``>=1.6.0``
|
|
98
98
|
``apache-airflow-providers-common-sql`` ``>=1.20.0``
|
|
99
99
|
``pandas`` ``>=2.1.2,<2.2``
|
|
100
100
|
``pyarrow`` ``>=14.0.1``
|
|
@@ -107,7 +107,7 @@ Cross provider package dependencies
|
|
|
107
107
|
-----------------------------------
|
|
108
108
|
|
|
109
109
|
Those are dependencies that might be needed in order to use all the features of the package.
|
|
110
|
-
You need to install the specified
|
|
110
|
+
You need to install the specified providers in order to use them.
|
|
111
111
|
|
|
112
112
|
You can install such cross-provider dependencies when installing from PyPI. For example:
|
|
113
113
|
|
|
@@ -125,5 +125,5 @@ Dependent package
|
|
|
125
125
|
================================================================================================================== =================
|
|
126
126
|
|
|
127
127
|
The changelog for the provider package can be found in the
|
|
128
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
128
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.2.0/changelog.html>`_.
|
|
129
129
|
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
airflow/providers/snowflake/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
|
|
2
|
-
airflow/providers/snowflake/__init__.py,sha256=
|
|
3
|
-
airflow/providers/snowflake/get_provider_info.py,sha256=
|
|
2
|
+
airflow/providers/snowflake/__init__.py,sha256=2xtoFfDLujuHV11CJo8wzRlZBW19ULr0BmF65Ge6_PU,1496
|
|
3
|
+
airflow/providers/snowflake/get_provider_info.py,sha256=yFTQrfxucLKZ7nEXr0AI6UuFvwJyF4xdYxPwhnC7OKU,5739
|
|
4
|
+
airflow/providers/snowflake/version_compat.py,sha256=aHg90_DtgoSnQvILFICexMyNlHlALBdaeWqkX3dFDug,1605
|
|
4
5
|
airflow/providers/snowflake/decorators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
5
6
|
airflow/providers/snowflake/decorators/snowpark.py,sha256=IXAzhcf7lkim9wsb_7SZlk5JPMQ38KOsEtymQUr0Q68,5298
|
|
6
7
|
airflow/providers/snowflake/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
7
|
-
airflow/providers/snowflake/hooks/snowflake.py,sha256=
|
|
8
|
+
airflow/providers/snowflake/hooks/snowflake.py,sha256=vrK78w0tGpihCHiZW7fPIrQeKuxO0UWKDc-ciI6h0cg,26168
|
|
8
9
|
airflow/providers/snowflake/hooks/snowflake_sql_api.py,sha256=3W3wGAWRUxu1K62qqw682vBrXrRzyEmoCYID7PlMfaA,15486
|
|
9
10
|
airflow/providers/snowflake/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
10
|
-
airflow/providers/snowflake/operators/snowflake.py,sha256=
|
|
11
|
+
airflow/providers/snowflake/operators/snowflake.py,sha256=LHyJed0AEwg-cU8zOnSahPOGwaHyU9Sp_BSPkiCvWn4,22661
|
|
11
12
|
airflow/providers/snowflake/operators/snowpark.py,sha256=Wt3wzcsja0ed4q2KE9WyL74XH6mUVSPNZvcCHWEHQtc,5815
|
|
12
13
|
airflow/providers/snowflake/transfers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
13
14
|
airflow/providers/snowflake/transfers/copy_into_snowflake.py,sha256=UjbznjbK-QWN071ZFMvBHZXoFddMo0vQFK-7VLv3amo,13191
|
|
@@ -15,10 +16,10 @@ airflow/providers/snowflake/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOF
|
|
|
15
16
|
airflow/providers/snowflake/triggers/snowflake_trigger.py,sha256=38tkByMyjbVbSt-69YL8EzRBQT4rhwuOKHgbwHfULL0,4250
|
|
16
17
|
airflow/providers/snowflake/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
17
18
|
airflow/providers/snowflake/utils/common.py,sha256=DG-KLy2KpZWAqZqm_XIECm8lmdoUlzwkXv9onmkQThc,1644
|
|
18
|
-
airflow/providers/snowflake/utils/openlineage.py,sha256=
|
|
19
|
+
airflow/providers/snowflake/utils/openlineage.py,sha256=SKmePIebxQ23nhm6lshguelIA1TXApjRCSf1fTQqpn0,13995
|
|
19
20
|
airflow/providers/snowflake/utils/snowpark.py,sha256=lw_tleNGFJICtTw2qCJ3TjWFOwZK1t8ZCIfYumS2Q18,1616
|
|
20
21
|
airflow/providers/snowflake/utils/sql_api_generate_jwt.py,sha256=9mR-vHIquv60tfAni87f6FAjKsiRHUDDrsVhzw4M9vM,6762
|
|
21
|
-
apache_airflow_providers_snowflake-6.
|
|
22
|
-
apache_airflow_providers_snowflake-6.
|
|
23
|
-
apache_airflow_providers_snowflake-6.
|
|
24
|
-
apache_airflow_providers_snowflake-6.
|
|
22
|
+
apache_airflow_providers_snowflake-6.2.0.dist-info/entry_points.txt,sha256=bCrl5J1PXUMzbgnrKYho61rkbL2gHRT4I6f_1jlxAX4,105
|
|
23
|
+
apache_airflow_providers_snowflake-6.2.0.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
|
|
24
|
+
apache_airflow_providers_snowflake-6.2.0.dist-info/METADATA,sha256=PjEkgXzTnR4ct9ixy04tEcX7EDYrc7eepjHuUsQ8BlM,6204
|
|
25
|
+
apache_airflow_providers_snowflake-6.2.0.dist-info/RECORD,,
|
|
File without changes
|