apache-airflow-providers-openlineage 1.1.1rc1__tar.gz → 1.2.0rc1__tar.gz

{apache-airflow-providers-openlineage-1.1.1rc1/apache_airflow_providers_openlineage.egg-info → apache-airflow-providers-openlineage-1.2.0rc1}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.1.1rc1
+Version: 1.2.0rc1
 Summary: Provider for Apache Airflow. Implements apache-airflow-providers-openlineage package
 Home-page: https://airflow.apache.org/
 Download-URL: https://archive.apache.org/dist/airflow/providers
 Author: Apache Software Foundation
 Author-email: dev@airflow.apache.org
 License: Apache License 2.0
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/changelog.html
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
@@ -71,7 +71,7 @@ License-File: NOTICE
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.1.1rc1``
+Release: ``1.2.0rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -84,7 +84,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/>`_.
 
 
 Installation
@@ -129,4 +129,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/changelog.html>`_.

{apache-airflow-providers-openlineage-1.1.1rc1 → apache-airflow-providers-openlineage-1.2.0rc1}/README.rst

@@ -36,7 +36,7 @@
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.1.1rc1``
+Release: ``1.2.0rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -49,7 +49,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/>`_.
 
 
 Installation
@@ -94,4 +94,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/changelog.html>`_.

{apache-airflow-providers-openlineage-1.1.1rc1 → apache-airflow-providers-openlineage-1.2.0rc1}/airflow/providers/openlineage/__init__.py

@@ -28,7 +28,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "1.1.1"
+__version__ = "1.2.0"
 
 try:
     from airflow import __version__ as airflow_version
@@ -39,5 +39,5 @@ if packaging.version.parse(packaging.version.parse(airflow_version).base_version
     "2.7.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-openlineage:{__version__}` requires Apache Airflow 2.7.0+"  # NOQA: E501
+        f"The package `apache-airflow-providers-openlineage:{__version__}` requires Apache Airflow 2.7.0+"
     )

{apache-airflow-providers-openlineage-1.1.1rc1 → apache-airflow-providers-openlineage-1.2.0rc1}/airflow/providers/openlineage/get_provider_info.py

@@ -28,7 +28,7 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "suspended": False,
-        "versions": ["1.1.1", "1.1.0", "1.0.2", "1.0.1", "1.0.0"],
+        "versions": ["1.2.0", "1.1.1", "1.1.0", "1.0.2", "1.0.1", "1.0.0"],
         "dependencies": [
             "apache-airflow>=2.7.0",
             "apache-airflow-providers-common-sql>=1.6.0",

{apache-airflow-providers-openlineage-1.1.1rc1 → apache-airflow-providers-openlineage-1.2.0rc1}/airflow/providers/openlineage/sqlparser.py

@@ -20,7 +20,15 @@ from typing import TYPE_CHECKING, Callable
 
 import sqlparse
 from attrs import define
-from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
+from openlineage.client.facet import (
+    BaseFacet,
+    ColumnLineageDatasetFacet,
+    ColumnLineageDatasetFacetFieldsAdditional,
+    ColumnLineageDatasetFacetFieldsAdditionalInputFields,
+    ExtractionError,
+    ExtractionErrorRunFacet,
+    SqlJobFacet,
+)
 from openlineage.common.sql import DbTableMeta, SqlMeta, parse
 
 from airflow.providers.openlineage.extractors.base import OperatorLineage
@@ -143,6 +151,47 @@ class SQLParser:
             else None,
         )
 
+    def attach_column_lineage(
+        self, datasets: list[Dataset], database: str | None, parse_result: SqlMeta
+    ) -> None:
+        """
+        Attaches column lineage facet to the list of datasets.
+
+        Note that currently each dataset has the same column lineage information set.
+        This would be a matter of change after OpenLineage SQL Parser improvements.
+        """
+        if not len(parse_result.column_lineage):
+            return
+        for dataset in datasets:
+            dataset.facets["columnLineage"] = ColumnLineageDatasetFacet(
+                fields={
+                    column_lineage.descendant.name: ColumnLineageDatasetFacetFieldsAdditional(
+                        inputFields=[
+                            ColumnLineageDatasetFacetFieldsAdditionalInputFields(
+                                namespace=dataset.namespace,
+                                name=".".join(
+                                    filter(
+                                        None,
+                                        (
+                                            column_meta.origin.database or database,
+                                            column_meta.origin.schema or self.default_schema,
+                                            column_meta.origin.name,
+                                        ),
+                                    )
+                                )
+                                if column_meta.origin
+                                else "",
+                                field=column_meta.name,
+                            )
+                            for column_meta in column_lineage.lineage
+                        ],
+                        transformationType="",
+                        transformationDescription="",
+                    )
+                    for column_lineage in parse_result.column_lineage
+                }
+            )
+
     def generate_openlineage_metadata_from_sql(
         self,
         sql: list[str] | str,
@@ -198,6 +247,8 @@ class SQLParser:
             sqlalchemy_engine=sqlalchemy_engine,
         )
 
+        self.attach_column_lineage(outputs, database or database_info.database, parse_result)
+
         return OperatorLineage(
             inputs=inputs,
             outputs=outputs,

{apache-airflow-providers-openlineage-1.1.1rc1 → apache-airflow-providers-openlineage-1.2.0rc1/apache_airflow_providers_openlineage.egg-info}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.1.1rc1
+Version: 1.2.0rc1
 Summary: Provider for Apache Airflow. Implements apache-airflow-providers-openlineage package
 Home-page: https://airflow.apache.org/
 Download-URL: https://archive.apache.org/dist/airflow/providers
 Author: Apache Software Foundation
 Author-email: dev@airflow.apache.org
 License: Apache License 2.0
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/changelog.html
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
@@ -71,7 +71,7 @@ License-File: NOTICE
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.1.1rc1``
+Release: ``1.2.0rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -84,7 +84,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/>`_.
 
 
 Installation
@@ -129,4 +129,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/changelog.html>`_.

{apache-airflow-providers-openlineage-1.1.1rc1 → apache-airflow-providers-openlineage-1.2.0rc1}/pyproject.toml

@@ -63,7 +63,8 @@ extend-select = [
     "D419",
     "TCH001",  # typing-only-first-party-import
     "TCH002",  # typing-only-third-party-import
-    "TID251",
+    "TID251",  # Specific modules or module members that may not be imported or accessed
+    "TID253",  # Ban certain modules from being imported at module level
 ]
 extend-ignore = [
     "D203",
@@ -128,15 +129,16 @@ combine-as-imports = true
 
 # Ignore pydoc style from these
 "*.pyi" = ["D"]
-"tests/*" = ["D"]
 "scripts/*" = ["D"]
-"dev/*" = ["D"]
 "docs/*" = ["D"]
 "provider_packages/*" = ["D"]
-"docker_tests/*" = ["D"]
-"kubernetes_tests/*" = ["D"]
 "*/example_dags/*" = ["D"]
 "chart/*" = ["D"]
+# In addition ignore top level imports, e.g. pandas, numpy in tests and dev
+"dev/*" = ["D", "TID253"]
+"tests/*" = ["D", "TID253"]
+"docker_tests/*" = ["D", "TID253"]
+"kubernetes_tests/*" = ["D", "TID253"]
 
 # All of the modules which have an extra license header (i.e. that we copy from another project) need to
 # ignore E402 -- module level import not at top level
@@ -150,6 +152,14 @@ combine-as-imports = true
 "airflow.AirflowException".msg = "Use airflow.exceptions.AirflowException instead."
 "airflow.Dataset".msg = "Use airflow.datasets.Dataset instead."
 
+[tool.ruff.flake8-tidy-imports]
+# Ban certain modules from being imported at module level, instead requiring
+# that they're imported lazily (e.g., within a function definition).
+banned-module-level-imports = ["numpy", "pandas"]
+
+[tool.ruff.flake8-type-checking]
+exempt-modules = ["typing", "typing_extensions"]
+
 [tool.coverage.run]
 branch = true
 relative_files = true
@@ -175,5 +185,3 @@ exclude_also = [
     "@(typing(_extensions)?\\.)?overload",
     "if TYPE_CHECKING:"
 ]
-[tool.ruff.flake8-type-checking]
-exempt-modules = ["typing", "typing_extensions"]

{apache-airflow-providers-openlineage-1.1.1rc1 → apache-airflow-providers-openlineage-1.2.0rc1}/setup.cfg

@@ -27,8 +27,8 @@ classifiers =
 	Programming Language :: Python :: 3.11
 	Topic :: System :: Monitoring
 project_urls = 
-	Documentation=https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/
-	Changelog=https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.1.1/changelog.html
+	Documentation=https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/
+	Changelog=https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.0/changelog.html
 	Bug Tracker=https://github.com/apache/airflow/issues
 	Source Code=https://github.com/apache/airflow
 	Slack Chat=https://s.apache.org/airflow-slack

{apache-airflow-providers-openlineage-1.1.1rc1 → apache-airflow-providers-openlineage-1.2.0rc1}/setup.py

@@ -26,7 +26,7 @@
 
 from setuptools import find_namespace_packages, setup
 
-version = "1.1.1"
+version = "1.2.0"
 
 
 def do_setup():