apache-airflow-providers-openlineage 1.9.0rc1__py3-none-any.whl → 1.9.1__py3-none-any.whl

airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.9.0"
+__version__ = "1.9.1"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

airflow/providers/openlineage/extractors/base.py

@@ -41,7 +41,8 @@ class OperatorLineage:
 
 
 class BaseExtractor(ABC, LoggingMixin):
-    """Abstract base extractor class.
+    """
+    Abstract base extractor class.
 
     This is used mostly to maintain support for custom extractors.
     """
@@ -55,7 +56,8 @@ class BaseExtractor(ABC, LoggingMixin):
     @classmethod
     @abstractmethod
     def get_operator_classnames(cls) -> list[str]:
-        """Get a list of operators that extractor works for.
+        """
+        Get a list of operators that extractor works for.
 
         This is an abstract method that subclasses should implement. There are
         operators that work very similarly and one extractor can cover.
@@ -77,7 +79,8 @@ class DefaultExtractor(BaseExtractor):
 
     @classmethod
     def get_operator_classnames(cls) -> list[str]:
-        """Assign this extractor to *no* operators.
+        """
+        Assign this extractor to *no* operators.
 
         Default extractor is chosen not on the classname basis, but
         by existence of get_openlineage_facets method on operator.

airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1718605195,
+        "source-date-epoch": 1720423723,
         "versions": [
+            "1.9.1",
             "1.9.0",
             "1.8.0",
             "1.7.1",

airflow/providers/openlineage/plugins/adapter.py

@@ -139,7 +139,8 @@ class OpenLineageAdapter(LoggingMixin):
         )
 
     def emit(self, event: RunEvent):
-        """Emit OpenLineage event.
+        """
+        Emit OpenLineage event.
 
         :param event: Event to be emitted.
         :return: Redacted Event.
@@ -295,11 +296,17 @@ class OpenLineageAdapter(LoggingMixin):
         """
         error_facet = {}
         if error:
-            if isinstance(error, BaseException):
+            stack_trace = None
+            if isinstance(error, BaseException) and error.__traceback__:
                 import traceback
 
-                error = "\\n".join(traceback.format_exception(type(error), error, error.__traceback__))
-            error_facet = {"errorMessage": ErrorMessageRunFacet(message=error, programmingLanguage="python")}
+                stack_trace = "\\n".join(traceback.format_exception(type(error), error, error.__traceback__))
+
+            error_facet = {
+                "errorMessage": ErrorMessageRunFacet(
+                    message=str(error), programmingLanguage="python", stackTrace=stack_trace
+                )
+            }
 
         event = RunEvent(
             eventType=RunState.FAIL,

airflow/providers/openlineage/plugins/facets.py

@@ -102,7 +102,8 @@ class AirflowRunFacet(BaseFacet):
 
 @define(slots=False)
 class UnknownOperatorInstance(RedactMixin):
-    """Describes an unknown operator.
+    """
+    Describes an unknown operator.
 
     This specifies the (class) name of the operator and its properties.
     """

airflow/providers/openlineage/plugins/listener.py

@@ -367,7 +367,7 @@ class OpenLineageListener:
             except BaseException:
                 # Kill the process directly.
                 self._terminate_with_wait(process)
-            self.log.warning("Process with pid %s finished - parent", pid)
+            self.log.debug("Process with pid %s finished - parent", pid)
         else:
             setproctitle(getproctitle() + " - OpenLineage - " + callable_name)
             configure_orm(disable_connection_pool=True)
@@ -381,7 +381,7 @@ class OpenLineageListener:
         if not self._executor:
             self._executor = ProcessPoolExecutor(
                 max_workers=conf.dag_state_change_process_pool_size(),
-                initializer=_executor_initializer(),
+                initializer=_executor_initializer,
             )
         return self._executor
 

airflow/providers/openlineage/sqlparser.py

@@ -118,7 +118,8 @@ def from_table_meta(
 
 
 class SQLParser(LoggingMixin):
-    """Interface for openlineage-sql.
+    """
+    Interface for openlineage-sql.
 
     :param dialect: dialect specific to the database
     :param default_schema: schema applied to each table with no schema parsed
@@ -244,7 +245,8 @@ class SQLParser(LoggingMixin):
         sqlalchemy_engine: Engine | None = None,
         use_connection: bool = True,
     ) -> OperatorLineage:
-        """Parse SQL statement(s) and generate OpenLineage metadata.
+        """
+        Parse SQL statement(s) and generate OpenLineage metadata.
 
         Generated OpenLineage metadata contains:
 

airflow/providers/openlineage/utils/selective_enable.py

@@ -32,7 +32,8 @@ log = logging.getLogger(__name__)
 
 
 def enable_lineage(obj: T) -> T:
-    """Set selective enable OpenLineage parameter to True.
+    """
+    Set selective enable OpenLineage parameter to True.
 
     The method also propagates param to tasks if the object is DAG.
     """
@@ -48,7 +49,8 @@ def enable_lineage(obj: T) -> T:
 
 
 def disable_lineage(obj: T) -> T:
-    """Set selective enable OpenLineage parameter to False.
+    """
+    Set selective enable OpenLineage parameter to False.
 
     The method also propagates param to tasks if the object is DAG.
     """
@@ -73,7 +75,8 @@ def is_task_lineage_enabled(task: Operator) -> bool:
 
 
 def is_dag_lineage_enabled(dag: DAG) -> bool:
-    """Check if DAG is selectively enabled to emit OpenLineage events.
+    """
+    Check if DAG is selectively enabled to emit OpenLineage events.
 
     The method also checks if selective enable parameter is set to True
     or if any of the tasks in DAG is selectively enabled.

airflow/providers/openlineage/utils/sql.py

@@ -85,7 +85,8 @@ def get_table_schemas(
     in_query: str | None,
     out_query: str | None,
 ) -> tuple[list[Dataset], list[Dataset]]:
-    """Query database for table schemas.
+    """
+    Query database for table schemas.
 
     Uses provided hook. Responsibility to provide queries for this function is on particular extractors.
     If query for input or output table isn't provided, the query is skipped.
@@ -111,7 +112,8 @@ def get_table_schemas(
 
 
 def parse_query_result(cursor) -> list[TableSchema]:
-    """Fetch results from DB-API 2.0 cursor and creates list of table schemas.
+    """
+    Fetch results from DB-API 2.0 cursor and creates list of table schemas.
 
     For each row it creates :class:`TableSchema`.
     """

airflow/providers/openlineage/utils/utils.py

@@ -150,7 +150,7 @@ class InfoJsonEncodable(dict):
             return value.isoformat()
         if isinstance(value, datetime.timedelta):
             return f"{value.total_seconds()} seconds"
-        if isinstance(value, (set, list, tuple)):
+        if isinstance(value, (set, tuple)):
             return str(list(value))
         return value
 
@@ -214,6 +214,12 @@ class TaskInstanceInfo(InfoJsonEncodable):
     }
 
 
+class DatasetInfo(InfoJsonEncodable):
+    """Defines encoding Airflow Dataset object to JSON."""
+
+    includes = ["uri", "extra"]
+
+
 class TaskInfo(InfoJsonEncodable):
     """Defines encoding BaseOperator/AbstractOperator object to JSON."""
 
@@ -242,6 +248,9 @@ class TaskInfo(InfoJsonEncodable):
         "run_as_user",
         "sla",
         "task_id",
+        "trigger_dag_id",
+        "external_dag_id",
+        "external_task_id",
         "trigger_rule",
         "upstream_task_ids",
         "wait_for_downstream",
@@ -255,6 +264,8 @@ class TaskInfo(InfoJsonEncodable):
             if hasattr(task, "task_group") and getattr(task.task_group, "_group_id", None)
             else None
         ),
+        "inlets": lambda task: [DatasetInfo(inlet) for inlet in task.inlets],
+        "outlets": lambda task: [DatasetInfo(outlet) for outlet in task.outlets],
     }
 
 
@@ -360,12 +371,13 @@ def _get_parsed_dag_tree(dag: DAG) -> dict:
         # Determine the level by counting the leading spaces, assuming 4 spaces per level
         # as defined in airflow.models.dag.DAG._generate_tree_view()
         level = (len(line) - len(stripped_line)) // 4
-        # airflow.models.baseoperator.BaseOperator.__repr__ is used in DAG tree
-        # <Task({op_class}): {task_id}>
-        match = re.match(r"^<Task\((.+)\): (.*?)>$", stripped_line)
+        # airflow.models.baseoperator.BaseOperator.__repr__ or
+        # airflow.models.mappedoperator.MappedOperator.__repr__ is used in DAG tree
+        # <Task({op_class}): {task_id}> or <Mapped({op_class}): {task_id}>
+        match = re.match(r"^<(?:Task|Mapped)\(.+\): (.+)>$", stripped_line)
         if not match:
             return {}
-        current_task_id = match[2]
+        current_task_id = match[1]
 
         if level == 0:  # It's a root task
             task_dict[current_task_id] = {}

{apache_airflow_providers_openlineage-1.9.0rc1.dist-info → apache_airflow_providers_openlineage-1.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.9.0rc1
+Version: 1.9.1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,15 +21,15 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow-providers-common-sql>=1.6.0rc0
-Requires-Dist: apache-airflow>=2.7.0rc0
+Requires-Dist: apache-airflow-providers-common-sql>=1.6.0
+Requires-Dist: apache-airflow>=2.7.0
 Requires-Dist: attrs>=22.2
 Requires-Dist: openlineage-integration-common>=1.16.0
 Requires-Dist: openlineage-python>=1.16.0
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.9.0.rc1``
+Release: ``1.9.1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -93,7 +93,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1/>`_.
 
 Installation
 ------------
@@ -137,4 +137,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1/changelog.html>`_.

{apache_airflow_providers_openlineage-1.9.0rc1.dist-info → apache_airflow_providers_openlineage-1.9.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
 airflow/providers/openlineage/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
-airflow/providers/openlineage/__init__.py,sha256=wyNzY61gCpS2r6Wx-q-JiM8TbOKgTbDVngFZmcVmqOc,1498
+airflow/providers/openlineage/__init__.py,sha256=bDqRpjqfCE-UNXjjy8Ph6IFG8z5fdRrKJ9pOcX_nVuU,1498
 airflow/providers/openlineage/conf.py,sha256=ZHUZF3wjFGupfFa40F1DjZSkOQJCUtWFyQM3nvz6Bq4,5099
-airflow/providers/openlineage/get_provider_info.py,sha256=v-KK3oIOhEr-LE1WE8J6MaduGde709qiUmbkOklREn4,7472
-airflow/providers/openlineage/sqlparser.py,sha256=nMUA7Ak2HNfnGZ1PybWbNPENZfQIcJ1eE8AMh9UD-O8,15653
+airflow/providers/openlineage/get_provider_info.py,sha256=diZ2iRGa-47G5qbtylQ6gIXnN8rHXhTFmSK_mnJoaeE,7493
+airflow/providers/openlineage/sqlparser.py,sha256=qX2ID75PH11-youFvRdRHqZbYzLdbm95H3STvbBzGb8,15667
 airflow/providers/openlineage/extractors/__init__.py,sha256=I0X4f6zUniclyD9zT0DFHRImpCpJVP4MkPJT3cd7X5I,1081
-airflow/providers/openlineage/extractors/base.py,sha256=o6z8bXwNor1hwcUzezJ8LIPynR_BqXkP-qtwEgLtD2Q,5476
+airflow/providers/openlineage/extractors/base.py,sha256=D-ll2cz_WWv1XYh2_UfzmpCl8S5iC2aptFTy2sWZeHY,5499
 airflow/providers/openlineage/extractors/bash.py,sha256=m4hLvDV4-zX4gp8apRuhpAR3Uakr8UOUxf-thTWmOxw,2563
 airflow/providers/openlineage/extractors/manager.py,sha256=9TyszMLAsgPS9NETWq7fPJjxcbTFk47x4kd1NRGCvsw,10315
 airflow/providers/openlineage/extractors/python.py,sha256=EQXCj2aHr2XXw0pNxeX-ii8UQFCoqkdf40ozqmA5d58,3151
@@ -13,16 +13,16 @@ airflow/providers/openlineage/facets/AirflowRunFacet.json,sha256=Mk85_KKwBOEcn_p
 airflow/providers/openlineage/facets/AirflowStateRunFacet.json,sha256=xhHQEKD9Jopw-oqbkCCrrwFjfXnxvuJAritsmegKjuQ,937
 airflow/providers/openlineage/facets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/openlineage/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/openlineage/plugins/adapter.py,sha256=nm8muyX-CwBaCNy_YDYbz-iFNL4msVZX6gFb4EaI-YE,18552
-airflow/providers/openlineage/plugins/facets.py,sha256=t60uCyp-FZKAeO7PSom5eAbn0cilN_Mfciip4FWG22M,4445
-airflow/providers/openlineage/plugins/listener.py,sha256=dk7omQItGSLcVw6bK56ScxLTnYQyQa1bXz2Fw402cjA,17956
+airflow/providers/openlineage/plugins/adapter.py,sha256=s0-eQw7cqzlUbznHdnLWihmdLKczHjuDj1TYYmJSK04,18720
+airflow/providers/openlineage/plugins/facets.py,sha256=kNxxnibLurcv8SV0qXPkwXxpKWhByo745vcqzWGNMIs,4450
+airflow/providers/openlineage/plugins/listener.py,sha256=q6r63SS3ZKdZKyTlEhw8c12onHh7kQCQ-_G1kA67btw,17952
 airflow/providers/openlineage/plugins/macros.py,sha256=hgFA3ZdQibyn4KXIOsKYBm4WRKDLA5q6Asscx5rvNfM,3076
 airflow/providers/openlineage/plugins/openlineage.py,sha256=rsRUW_zpXVAglzsgQRv5T9VWYY7CMQl0qRWm8-3oqDA,1678
 airflow/providers/openlineage/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/openlineage/utils/selective_enable.py,sha256=JVTmXdQknBL-9N0drFDkVMf1HCf8C6nbITVaP4-5ba4,3072
-airflow/providers/openlineage/utils/sql.py,sha256=yNBGMnWMECHtbpTh5nlaQh01O__LWkeQaSqrOVRWz6Y,9524
-airflow/providers/openlineage/utils/utils.py,sha256=QBg4B-isDrizb53cQFOflFx8LVKiabwE08I70I0AxOY,19485
-apache_airflow_providers_openlineage-1.9.0rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
-apache_airflow_providers_openlineage-1.9.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_openlineage-1.9.0rc1.dist-info/METADATA,sha256=kamPdoPzxHZpb788jJ_RhWSQUgR1U76rKpRnraIm5Qg,6381
-apache_airflow_providers_openlineage-1.9.0rc1.dist-info/RECORD,,
+airflow/providers/openlineage/utils/selective_enable.py,sha256=dFJ7wK7J_-BFwcOKp9tqFOSrASV3lmLv7HtRkEuMk3Q,3087
+airflow/providers/openlineage/utils/sql.py,sha256=5K3qFhevq-R4CDUZbq-XfFKPLWixIOtaypRem3jiHvw,9534
+airflow/providers/openlineage/utils/utils.py,sha256=cmPynb5QqZRV6Wzya-6cEcJUrCZjYm-4r4Z6NxwMzCc,19962
+apache_airflow_providers_openlineage-1.9.1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
+apache_airflow_providers_openlineage-1.9.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_openlineage-1.9.1.dist-info/METADATA,sha256=naW8pSs4fWp5BmV8wo5dbJa3buwt3uWWObAVfB0NMMg,6368
+apache_airflow_providers_openlineage-1.9.1.dist-info/RECORD,,