apache-airflow-providers-openlineage 1.7.0rc1__py3-none-any.whl → 1.7.1rc1__py3-none-any.whl

airflow/providers/openlineage/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "1.7.0"
+__version__ = "1.7.1"
 
 try:
     from airflow import __version__ as airflow_version

airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1712666247,
+        "source-date-epoch": 1714477058,
         "versions": [
+            "1.7.1",
             "1.7.0",
             "1.6.0",
             "1.5.0",

airflow/providers/openlineage/sqlparser.py

@@ -29,6 +29,7 @@ from openlineage.client.facet import (
     ExtractionErrorRunFacet,
     SqlJobFacet,
 )
+from openlineage.client.run import Dataset
 from openlineage.common.sql import DbTableMeta, SqlMeta, parse
 
 from airflow.providers.openlineage.extractors.base import OperatorLineage
@@ -40,7 +41,6 @@ from airflow.providers.openlineage.utils.sql import (
 from airflow.typing_compat import TypedDict
 
 if TYPE_CHECKING:
-    from openlineage.client.run import Dataset
     from sqlalchemy.engine import Engine
 
     from airflow.hooks.base import BaseHook
@@ -104,6 +104,18 @@ class DatabaseInfo:
     normalize_name_method: Callable[[str], str] = default_normalize_name_method
 
 
+def from_table_meta(
+    table_meta: DbTableMeta, database: str | None, namespace: str, is_uppercase: bool
+) -> Dataset:
+    if table_meta.database:
+        name = table_meta.qualified_name
+    elif database:
+        name = f"{database}.{table_meta.schema}.{table_meta.name}"
+    else:
+        name = f"{table_meta.schema}.{table_meta.name}"
+    return Dataset(namespace=namespace, name=name if not is_uppercase else name.upper())
+
+
 class SQLParser:
     """Interface for openlineage-sql.
 
@@ -117,7 +129,7 @@ class SQLParser:
 
     def parse(self, sql: list[str] | str) -> SqlMeta | None:
         """Parse a single or a list of SQL statements."""
-        return parse(sql=sql, dialect=self.dialect)
+        return parse(sql=sql, dialect=self.dialect, default_schema=self.default_schema)
 
     def parse_table_schemas(
         self,
@@ -156,6 +168,23 @@ class SQLParser:
             else None,
         )
 
+    def get_metadata_from_parser(
+        self,
+        inputs: list[DbTableMeta],
+        outputs: list[DbTableMeta],
+        database_info: DatabaseInfo,
+        namespace: str = DEFAULT_NAMESPACE,
+        database: str | None = None,
+    ) -> tuple[list[Dataset], ...]:
+        database = database if database else database_info.database
+        return [
+            from_table_meta(dataset, database, namespace, database_info.is_uppercase_names)
+            for dataset in inputs
+        ], [
+            from_table_meta(dataset, database, namespace, database_info.is_uppercase_names)
+            for dataset in outputs
+        ]
+
     def attach_column_lineage(
         self, datasets: list[Dataset], database: str | None, parse_result: SqlMeta
     ) -> None:
@@ -204,6 +233,7 @@ class SQLParser:
         database_info: DatabaseInfo,
         database: str | None = None,
         sqlalchemy_engine: Engine | None = None,
+        use_connection: bool = True,
     ) -> OperatorLineage:
         """Parse SQL statement(s) and generate OpenLineage metadata.
 
@@ -242,15 +272,24 @@ class SQLParser:
             )
 
         namespace = self.create_namespace(database_info=database_info)
-        inputs, outputs = self.parse_table_schemas(
-            hook=hook,
-            inputs=parse_result.in_tables,
-            outputs=parse_result.out_tables,
-            namespace=namespace,
-            database=database,
-            database_info=database_info,
-            sqlalchemy_engine=sqlalchemy_engine,
-        )
+        if use_connection:
+            inputs, outputs = self.parse_table_schemas(
+                hook=hook,
+                inputs=parse_result.in_tables,
+                outputs=parse_result.out_tables,
+                namespace=namespace,
+                database=database,
+                database_info=database_info,
+                sqlalchemy_engine=sqlalchemy_engine,
+            )
+        else:
+            inputs, outputs = self.get_metadata_from_parser(
+                inputs=parse_result.in_tables,
+                outputs=parse_result.out_tables,
+                namespace=namespace,
+                database=database,
+                database_info=database_info,
+            )
 
         self.attach_column_lineage(outputs, database or database_info.database, parse_result)
 

airflow/providers/openlineage/utils/utils.py

@@ -384,3 +384,8 @@ def normalize_sql(sql: str | Iterable[str]):
         sql = [stmt for stmt in sql.split(";") if stmt != ""]
     sql = [obj for stmt in sql for obj in stmt.split(";") if obj != ""]
     return ";\n".join(sql)
+
+
+def should_use_external_connection(hook) -> bool:
+    # TODO: Add checking overrides
+    return hook.__class__.__name__ not in ["SnowflakeHook", "SnowflakeSqlApiHook"]

{apache_airflow_providers_openlineage-1.7.0rc1.dist-info → apache_airflow_providers_openlineage-1.7.1rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.7.0rc1
+Version: 1.7.1rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -28,8 +28,8 @@ Requires-Dist: openlineage-integration-common>=0.28.0
 Requires-Dist: openlineage-python>=0.28.0
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.7.0.rc1``
+Release: ``1.7.1.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -93,7 +93,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.1/>`_.
 
 Installation
 ------------
@@ -137,4 +137,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.1/changelog.html>`_.

{apache_airflow_providers_openlineage-1.7.0rc1.dist-info → apache_airflow_providers_openlineage-1.7.1rc1.dist-info}/RECORD

@@ -1,8 +1,8 @@
 airflow/providers/openlineage/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
-airflow/providers/openlineage/__init__.py,sha256=YHyXPr7nPtmtaYO8h8zmWWgRa8LAG_mMnyL09Ph6iKs,1586
+airflow/providers/openlineage/__init__.py,sha256=Xc1UFto3ZKaoNfnpJpkK8cxkSAtXxGbxhpZ624JXDko,1586
 airflow/providers/openlineage/conf.py,sha256=Y76TUM_YwQtn-_081wQicZPTe_8bXH8jNafx-GACgeo,3398
-airflow/providers/openlineage/get_provider_info.py,sha256=Cu_1tldNUETXL4SLKOeXiLEo-Yo7pIi0GcvcTxgbYCk,6605
-airflow/providers/openlineage/sqlparser.py,sha256=fYHNGaf4kiYJYKVLTqdk-3o7tqckJ-wg3WLhtn0dYLw,13806
+airflow/providers/openlineage/get_provider_info.py,sha256=a_A_2VPU8M2vHZ5CX0_8yUawYEiFieMa3g5tmi3pEnU,6626
+airflow/providers/openlineage/sqlparser.py,sha256=-FGWWK0Xu6XkGSXcfn7PXsWIe0Y0fwe-3hivHg7emLA,15308
 airflow/providers/openlineage/extractors/__init__.py,sha256=I0X4f6zUniclyD9zT0DFHRImpCpJVP4MkPJT3cd7X5I,1081
 airflow/providers/openlineage/extractors/base.py,sha256=sj2KS23ocX7LAbkDiR53otkFg1qqEg41PyBivdc-kyM,5070
 airflow/providers/openlineage/extractors/bash.py,sha256=lE7BH9vipRg9jGloPIE6y6wcHw_BbTvGBasfa4PfDBc,2412
@@ -17,8 +17,8 @@ airflow/providers/openlineage/plugins/openlineage.py,sha256=Owlbpp8puiww-4Wh6B46
 airflow/providers/openlineage/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/openlineage/utils/selective_enable.py,sha256=JVTmXdQknBL-9N0drFDkVMf1HCf8C6nbITVaP4-5ba4,3072
 airflow/providers/openlineage/utils/sql.py,sha256=7tEK0zVfIe7v3NI6oyv62x0KAS3sl8Ajfhqob8MdiX8,9366
-airflow/providers/openlineage/utils/utils.py,sha256=lONfDxrn30aMy47gZQlDDdLJfvBJgOQE6n0sVuBPd0k,13115
-apache_airflow_providers_openlineage-1.7.0rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
-apache_airflow_providers_openlineage-1.7.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_openlineage-1.7.0rc1.dist-info/METADATA,sha256=H5CmsWiQxrbCNFvuBBzYmJspl8avkR2OD8QqiMLtBvk,6381
-apache_airflow_providers_openlineage-1.7.0rc1.dist-info/RECORD,,
+airflow/providers/openlineage/utils/utils.py,sha256=duT_rXHQuVFUIbMCplGGw0OI0RN0DAXU8oo8FaqcREg,13285
+apache_airflow_providers_openlineage-1.7.1rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
+apache_airflow_providers_openlineage-1.7.1rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_openlineage-1.7.1rc1.dist-info/METADATA,sha256=5ofVoQHgZeqImH1WQSQuXgyfxHld7oS782eWWbA5n-Q,6381
+apache_airflow_providers_openlineage-1.7.1rc1.dist-info/RECORD,,