apache-airflow-providers-snowflake 6.1.0__py3-none-any.whl → 6.1.1rc1__py3-none-any.whl

airflow/providers/snowflake/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "6.1.0"
+__version__ = "6.1.1"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

airflow/providers/snowflake/get_provider_info.py

@@ -27,8 +27,9 @@ def get_provider_info():
         "name": "Snowflake",
         "description": "`Snowflake <https://www.snowflake.com/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1739964397,
+        "source-date-epoch": 1741509691,
         "versions": [
+            "6.1.1",
             "6.1.0",
             "6.0.0",
             "5.8.1",
@@ -162,4 +163,5 @@ def get_provider_info():
             "snowflake-snowpark-python>=1.17.0;python_version<'3.12'",
         ],
         "optional-dependencies": {"openlineage": ["apache-airflow-providers-openlineage"]},
+        "devel-dependencies": [],
     }

airflow/providers/snowflake/hooks/snowflake.py

@@ -28,15 +28,15 @@ from urllib.parse import urlparse
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
+from snowflake import connector
+from snowflake.connector import DictCursor, SnowflakeConnection, util_text
+from snowflake.sqlalchemy import URL
 from sqlalchemy import create_engine
 
 from airflow.exceptions import AirflowException
 from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results
 from airflow.providers.snowflake.utils.openlineage import fix_snowflake_sqlalchemy_uri
 from airflow.utils.strings import to_boolean
-from snowflake import connector
-from snowflake.connector import DictCursor, SnowflakeConnection, util_text
-from snowflake.sqlalchemy import URL
 
 T = TypeVar("T")
 if TYPE_CHECKING:
@@ -367,9 +367,10 @@ class SnowflakeHook(DbApiHook):
 
         :return: the created session.
         """
+        from snowflake.snowpark import Session
+
         from airflow import __version__ as airflow_version
         from airflow.providers.snowflake import __version__ as provider_version
-        from snowflake.snowpark import Session
 
         conn_config = self._get_conn_params
         session = Session.builder.configs(conn_config).create()

airflow/providers/snowflake/operators/snowflake.py

@@ -455,9 +455,13 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator):
                 method_name="execute_complete",
             )
         else:
-            statement_status = self.poll_on_queries()
-            if statement_status["error"]:
-                raise AirflowException(statement_status["error"])
+            while True:
+                statement_status = self.poll_on_queries()
+                if statement_status["error"]:
+                    raise AirflowException(statement_status["error"])
+                if not statement_status["running"]:
+                    break
+
             self._hook.check_query_output(self.query_ids)
 
     def poll_on_queries(self):
@@ -465,6 +469,7 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator):
         queries_in_progress = set(self.query_ids)
         statement_success_status = {}
         statement_error_status = {}
+        statement_running_status = {}
         for query_id in self.query_ids:
             if not len(queries_in_progress):
                 break
@@ -479,8 +484,14 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator):
             if statement_status.get("status") == "success":
                 statement_success_status[query_id] = statement_status
                 queries_in_progress.remove(query_id)
+            if statement_status.get("status") == "running":
+                statement_running_status[query_id] = statement_status
             time.sleep(self.poll_interval)
-        return {"success": statement_success_status, "error": statement_error_status}
+        return {
+            "success": statement_success_status,
+            "error": statement_error_status,
+            "running": statement_running_status,
+        }
 
     def execute_complete(self, context: Context, event: dict[str, str | list[str]] | None = None) -> None:
         """

airflow/providers/snowflake/transfers/copy_into_snowflake.py

@@ -175,7 +175,7 @@ class CopyFromExternalStageToSnowflakeOperator(BaseOperator):
 
         >>> results = [{"file": "azure://my_account.blob.core.windows.net/azure_container/dir3/file.csv"}]
         >>> method(results)
-        ([('wasbs://azure_container@my_account', 'dir3')], [])
+        ([('wasbs://azure_container@my_account', 'dir3/file.csv')], [])
 
         >>> results = [{"file": "azure://my_account.blob.core.windows.net/azure_container"}]
         >>> method(results)
@@ -183,21 +183,28 @@ class CopyFromExternalStageToSnowflakeOperator(BaseOperator):
 
         >>> results = [{"file": "s3://bucket"}, {"file": "gcs://bucket/"}, {"file": "s3://bucket/a.csv"}]
         >>> method(results)
-        ([('gcs://bucket', '/'), ('s3://bucket', '/')], [])
+        ([('gcs://bucket', '/'), ('s3://bucket', '/'), ('s3://bucket', 'a.csv')], [])
 
         >>> results = [{"file": "s3://bucket/dir/file.csv"}, {"file": "gcs://bucket/dir/dir2/a.txt"}]
         >>> method(results)
-        ([('gcs://bucket', 'dir/dir2'), ('s3://bucket', 'dir')], [])
+        ([('gcs://bucket', 'dir/dir2/a.txt'), ('s3://bucket', 'dir/file.csv')], [])
 
         >>> results = [
         ...     {"file": "s3://bucket/dir/file.csv"},
         ...     {"file": "azure://my_account.something_new.windows.net/azure_container"},
         ... ]
         >>> method(results)
-        ([('s3://bucket', 'dir')], ['azure://my_account.something_new.windows.net/azure_container'])
+        ([('s3://bucket', 'dir/file.csv')], ['azure://my_account.something_new.windows.net/azure_container'])
+
+        >>> results = [
+        ...     {"file": "s3://bucket/dir/file.csv"},
+        ...     {"file": "s3:/invalid-s3-uri"},
+        ...     {"file": "gcs:invalid-gcs-uri"},
+        ... ]
+        >>> method(results)
+        ([('s3://bucket', 'dir/file.csv')], ['gcs:invalid-gcs-uri', 's3:/invalid-s3-uri'])
         """
         import re
-        from pathlib import Path
         from urllib.parse import urlparse
 
         azure_regex = r"azure:\/\/(\w+)?\.blob.core.windows.net\/(\w+)\/?(.*)?"
@@ -205,23 +212,31 @@ class CopyFromExternalStageToSnowflakeOperator(BaseOperator):
         unique_dataset_paths = set()
 
         for row in query_result:
-            uri = urlparse(row["file"])
-            if uri.scheme == "azure":
-                match = re.fullmatch(azure_regex, row["file"])
-                if not match:
+            try:
+                uri = urlparse(row["file"])
+
+                # Check for valid URI structure
+                if not uri.scheme or not uri.netloc:
                     extraction_error_files.append(row["file"])
                     continue
-                account_name, container_name, name = match.groups()
-                namespace = f"wasbs://{container_name}@{account_name}"
-            else:
-                namespace = f"{uri.scheme}://{uri.netloc}"
-                name = uri.path.lstrip("/")
 
-            name = Path(name).parent.as_posix()
-            if name in ("", "."):
-                name = "/"
+                if uri.scheme == "azure":
+                    match = re.fullmatch(azure_regex, row["file"])
+                    if not match:
+                        extraction_error_files.append(row["file"])
+                        continue
+                    account_name, container_name, name = match.groups()
+                    namespace = f"wasbs://{container_name}@{account_name}"
+                else:
+                    namespace = f"{uri.scheme}://{uri.netloc}"
+                    name = uri.path.lstrip("/")
+
+                if name in ("", "."):
+                    name = "/"
 
-            unique_dataset_paths.add((namespace, name))
+                unique_dataset_paths.add((namespace, name))
+            except Exception:
+                extraction_error_files.append(row["file"])
 
         return sorted(unique_dataset_paths), sorted(extraction_error_files)
 
@@ -243,9 +258,11 @@ class CopyFromExternalStageToSnowflakeOperator(BaseOperator):
             return OperatorLineage()
 
         query_results = self._result or []
-        # If no files were uploaded we get [{"status": "0 files were uploaded..."}]
-        if len(query_results) == 1 and query_results[0].get("status"):
+
+        # This typically happens when no files were processed (empty directory)
+        if len(query_results) == 1 and ("file" not in query_results[0] or query_results[0]["file"] is None):
             query_results = []
+
         unique_dataset_paths, extraction_error_files = self._extract_openlineage_unique_dataset_paths(
             query_results
         )

{apache_airflow_providers_snowflake-6.1.0.dist-info → apache_airflow_providers_snowflake-6.1.1rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: apache-airflow-providers-snowflake
-Version: 6.1.0
+Version: 6.1.1rc1
 Summary: Provider package apache-airflow-providers-snowflake for Apache Airflow
 Keywords: airflow-provider,snowflake,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,9 +20,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.9.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.1.0
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
+Requires-Dist: apache-airflow>=2.9.0rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.1.0rc0
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
 Requires-Dist: pandas>=2.1.2,<2.2
 Requires-Dist: pyarrow>=14.0.1
 Requires-Dist: snowflake-connector-python>=3.7.1
@@ -30,8 +30,8 @@ Requires-Dist: snowflake-sqlalchemy>=1.4.0
 Requires-Dist: snowflake-snowpark-python>=1.17.0;python_version<'3.12'
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.1.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.1.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.1.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.1.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://x.com/ApacheAirflow
@@ -39,32 +39,31 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: openlineage
 
 
- .. Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
+.. Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
 
- ..   http://www.apache.org/licenses/LICENSE-2.0
+..   http://www.apache.org/licenses/LICENSE-2.0
 
- .. Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
+.. Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
 
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
-
- .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
-    `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 
+.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+   ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY
 
 Package ``apache-airflow-providers-snowflake``
 
-Release: ``6.1.0``
+Release: ``6.1.1``
 
 
 `Snowflake <https://www.snowflake.com/>`__
@@ -77,7 +76,7 @@ This is a provider package for ``snowflake`` provider. All classes for this prov
 are in ``airflow.providers.snowflake`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.1.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.1.1/>`_.
 
 Installation
 ------------
@@ -126,5 +125,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.1.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.1.1/changelog.html>`_.
 

{apache_airflow_providers_snowflake-6.1.0.dist-info → apache_airflow_providers_snowflake-6.1.1rc1.dist-info}/RECORD

@@ -1,16 +1,16 @@
 airflow/providers/snowflake/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/snowflake/__init__.py,sha256=wbduNpHC6V1I9K4YVLh4ZhvnqMDn2Jl-HIR0Cf1eNOE,1496
-airflow/providers/snowflake/get_provider_info.py,sha256=x9uRV99oSHoTBm21ikl2MqVkthQs3bjiqwIJZZDsQT8,5644
+airflow/providers/snowflake/__init__.py,sha256=a9Xtufn-eAD1dLTNyU--Hpz1_KPtC8TnIjkZCd69B5M,1496
+airflow/providers/snowflake/get_provider_info.py,sha256=u5BI-sKjvF49I88JLTYA0XOr6R_GOSMe2hsaW67kKJg,5699
 airflow/providers/snowflake/decorators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/snowflake/decorators/snowpark.py,sha256=IXAzhcf7lkim9wsb_7SZlk5JPMQ38KOsEtymQUr0Q68,5298
 airflow/providers/snowflake/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/hooks/snowflake.py,sha256=cIJMMG76SUh0XrPw8uub-zqqrEG9oQt9_GSk4s4Jucc,24161
+airflow/providers/snowflake/hooks/snowflake.py,sha256=z5d6enjm78OhMUZQwuNXU2yFsv6ikKRsxmwS1TYVKKs,24162
 airflow/providers/snowflake/hooks/snowflake_sql_api.py,sha256=3W3wGAWRUxu1K62qqw682vBrXrRzyEmoCYID7PlMfaA,15486
 airflow/providers/snowflake/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/operators/snowflake.py,sha256=OgaX1Y-26LYG9MLv7vImcQTXR1tD-PtQVI_Ahdc3hCk,22292
+airflow/providers/snowflake/operators/snowflake.py,sha256=KhC-t3N2N3QXhPFW-0ypBULQN_2z8h1-k7uDXc-Xku0,22659
 airflow/providers/snowflake/operators/snowpark.py,sha256=Wt3wzcsja0ed4q2KE9WyL74XH6mUVSPNZvcCHWEHQtc,5815
 airflow/providers/snowflake/transfers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/transfers/copy_into_snowflake.py,sha256=splXX_zn35NecxhWqbs0rAO6zJL0jzeIEdJCqBw5IOI,12511
+airflow/providers/snowflake/transfers/copy_into_snowflake.py,sha256=UjbznjbK-QWN071ZFMvBHZXoFddMo0vQFK-7VLv3amo,13191
 airflow/providers/snowflake/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/snowflake/triggers/snowflake_trigger.py,sha256=38tkByMyjbVbSt-69YL8EzRBQT4rhwuOKHgbwHfULL0,4250
 airflow/providers/snowflake/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -18,7 +18,7 @@ airflow/providers/snowflake/utils/common.py,sha256=DG-KLy2KpZWAqZqm_XIECm8lmdoUl
 airflow/providers/snowflake/utils/openlineage.py,sha256=XkcYvb_cXG8tZQ6h1IwhfGCkOVf7MSBfOxW0WLGBW0s,3257
 airflow/providers/snowflake/utils/snowpark.py,sha256=lw_tleNGFJICtTw2qCJ3TjWFOwZK1t8ZCIfYumS2Q18,1616
 airflow/providers/snowflake/utils/sql_api_generate_jwt.py,sha256=9mR-vHIquv60tfAni87f6FAjKsiRHUDDrsVhzw4M9vM,6762
-apache_airflow_providers_snowflake-6.1.0.dist-info/entry_points.txt,sha256=bCrl5J1PXUMzbgnrKYho61rkbL2gHRT4I6f_1jlxAX4,105
-apache_airflow_providers_snowflake-6.1.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-apache_airflow_providers_snowflake-6.1.0.dist-info/METADATA,sha256=5whcYkTWZjVZpDmsdFgM1tksqylxyncCThrKXMHdlrc,6222
-apache_airflow_providers_snowflake-6.1.0.dist-info/RECORD,,
+apache_airflow_providers_snowflake-6.1.1rc1.dist-info/entry_points.txt,sha256=bCrl5J1PXUMzbgnrKYho61rkbL2gHRT4I6f_1jlxAX4,105
+apache_airflow_providers_snowflake-6.1.1rc1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
+apache_airflow_providers_snowflake-6.1.1rc1.dist-info/METADATA,sha256=TGfduIBC5_a_3lO0o4ueCtGTjtGV_q_-HNv7Wg_MgQs,6220
+apache_airflow_providers_snowflake-6.1.1rc1.dist-info/RECORD,,

{apache_airflow_providers_snowflake-6.1.0.dist-info → apache_airflow_providers_snowflake-6.1.1rc1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.10.1
+Generator: flit 3.11.0
 Root-Is-Purelib: true
 Tag: py3-none-any