data-rentgen 0.1.0__py3-none-any.whl

data_rentgen/__init__.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+# _raw_version could contain pre-release version, like 0.0.1dev123
+# value is updated automatically by `poetry version ...` and poetry-bumpversion plugin
+_raw_version = "0.0.1"
+
+# version always contain only release number like 0.0.1
+__version__ = ".".join(_raw_version.split(".")[:3])
+
+# version tuple always contains only integer parts, like (0, 0, 1)
+__version_tuple__ = tuple(map(int, __version__.split(".")))

data_rentgen/consumer/__init__.py

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+
+from fast_depends import dependency_provider
+from faststream import FastStream
+from faststream.kafka import KafkaBroker
+from sqlalchemy.ext.asyncio import AsyncSession
+
+import data_rentgen
+from data_rentgen.consumer.settings import ConsumerApplicationSettings
+from data_rentgen.consumer.settings.security import get_broker_security
+from data_rentgen.consumer.subscribers import runs_events_subscriber
+from data_rentgen.db.factory import create_session_factory
+from data_rentgen.logging.setup_logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+
+def broker_factory(settings: ConsumerApplicationSettings) -> KafkaBroker:
+    broker = KafkaBroker(
+        bootstrap_servers=settings.kafka.bootstrap_servers,
+        security=get_broker_security(settings.kafka.security),
+        compression_type=settings.kafka.compression.value if settings.kafka.compression else None,
+        client_id=f"data-rentgen-{data_rentgen.__version__}",
+        logger=logger,
+    )
+
+    # register subscribers using settings
+    consumer_settings = settings.consumer.model_dump(exclude={"topics_list", "topics_pattern"})
+    broker.subscriber(
+        *settings.consumer.topics_list,
+        pattern=settings.consumer.topics_pattern,
+        **consumer_settings,
+        batch=True,
+    )(runs_events_subscriber)
+
+    dependency_provider.override(AsyncSession, create_session_factory(settings.database))
+    return broker
+
+
+def application_factory(settings: ConsumerApplicationSettings) -> FastStream:
+    return FastStream(
+        broker=broker_factory(settings),
+        title="Data.Rentgen",
+        description="Data.Rentgen is a nextgen DataLineage service",
+        version=data_rentgen.__version__,
+        logger=logger,
+    )
+
+
+def get_application():
+    settings = ConsumerApplicationSettings()
+    setup_logging(settings.logging)
+    return application_factory(settings=settings)

data_rentgen/consumer/__main__.py

@@ -0,0 +1,36 @@
+#!/bin/env python3
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+from faststream.cli.main import cli
+
+here = Path(__file__).resolve()
+
+
+def main(prog_name: str | None = None, args: list[str] | None = None):
+    """Run FastStream and pass the command line arguments to it."""
+    if args is None:
+        args = sys.argv.copy()
+        prog_name = args.pop(0)
+
+    if not prog_name:
+        prog_name = os.fspath(here)
+
+    args = args.copy()
+    # prepend config path before command line arguments
+    args.insert(0, "run")
+    args.insert(1, "data_rentgen.consumer:get_application")
+    args.insert(2, "--factory")
+
+    # call uvicorn
+    cli(args, prog_name=prog_name)
+
+
+if __name__ == "__main__":
+    main()

data_rentgen/consumer/extractors/__init__.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+from data_rentgen.consumer.extractors.batch import BatchExtractionResult, extract_batch
+from data_rentgen.consumer.extractors.dataset import (
+    connect_dataset_with_symlinks,
+    extract_dataset,
+    extract_dataset_and_symlinks,
+)
+from data_rentgen.consumer.extractors.input import extract_input
+from data_rentgen.consumer.extractors.job import extract_job
+from data_rentgen.consumer.extractors.operation import extract_operation
+from data_rentgen.consumer.extractors.output import extract_output
+from data_rentgen.consumer.extractors.run import extract_run, extract_run_minimal
+from data_rentgen.consumer.extractors.schema import extract_schema
+
+__all__ = [
+    "extract_dataset_and_symlinks",
+    "extract_dataset",
+    "connect_dataset_with_symlinks",
+    "extract_job",
+    "extract_run",
+    "extract_run_minimal",
+    "extract_operation",
+    "extract_input",
+    "extract_output",
+    "extract_schema",
+    "extract_batch",
+    "BatchExtractionResult",
+]

data_rentgen/consumer/extractors/batch.py

@@ -0,0 +1,252 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from typing import TypeVar
+
+from data_rentgen.consumer.extractors.input import extract_input
+from data_rentgen.consumer.extractors.operation import extract_operation
+from data_rentgen.consumer.extractors.output import extract_output
+from data_rentgen.consumer.extractors.run import extract_run
+from data_rentgen.consumer.openlineage.job_facets.job_type import OpenLineageJobType
+from data_rentgen.consumer.openlineage.run_event import OpenLineageRunEvent
+from data_rentgen.dto import (
+    DatasetDTO,
+    InputDTO,
+    JobDTO,
+    LocationDTO,
+    OperationDTO,
+    OutputDTO,
+    RunDTO,
+    SchemaDTO,
+    UserDTO,
+)
+from data_rentgen.dto.dataset_symlink import DatasetSymlinkDTO
+
+T = TypeVar(
+    "T",
+    LocationDTO,
+    DatasetDTO,
+    DatasetSymlinkDTO,
+    JobDTO,
+    RunDTO,
+    OperationDTO,
+    InputDTO,
+    OutputDTO,
+    SchemaDTO,
+    UserDTO,
+)
+
+
+class BatchExtractionResult:  # noqa: WPS338, WPS214
+    """Track results of batch extraction.
+
+    Calling any ``add_*`` method will add DTO item to the result, including nested DTOs,
+    like ``OperationDTO`` -> ``RunDTO`` -> ``JobDTO`` -> ``LocationDTO``, and so on.
+
+    Each DTO type is tracked separately. DTOs with same ``unique_key`` are merged into one final DTO,
+    by calling ``existing.merge(new)``. The resulting final DTO contains all non-null attributes of original DTOs.
+    Last DTO in the chain has a higher priority than previuos ones.
+    For example ``RunDTO(status=STARTED, started_at=...).merge(RunDTO(status=SUCCEEDED, ended_at=...))``
+    produces final ``RunDTO(status=SUCCEEDED, started_at=..., ended_at=...)``.
+
+    Calling get methods, like ``jobs()``, will return the list of tracked DTOs with resolved
+    cross-links. For example, iterating over ``jobs()`` with return the same objects
+    as in ``[run.job for run in runs()]``.
+    This makes modification of nested DTOs easier, all changes will be reflected in the parent DTOs as well.
+    """
+
+    def __init__(self):
+        self._locations: dict[tuple, LocationDTO] = {}
+        self._datasets: dict[tuple, DatasetDTO] = {}
+        self._dataset_symlinks: dict[tuple, DatasetSymlinkDTO] = {}
+        self._jobs: dict[tuple, JobDTO] = {}
+        self._runs: dict[tuple, RunDTO] = {}
+        self._operations: dict[tuple, OperationDTO] = {}
+        self._inputs: dict[tuple, InputDTO] = {}
+        self._outputs: dict[tuple, OutputDTO] = {}
+        self._schemas: dict[tuple, SchemaDTO] = {}
+        self._users: dict[tuple, UserDTO] = {}
+
+    def __repr__(self):
+        return (
+            "ExtractionResult("  # noqa: WPS237
+            f"locations={len(self._locations)}, "
+            f"datasets={len(self._datasets)}, "
+            f"dataset_symlinks={len(self._dataset_symlinks)}, "
+            f"jobs={len(self._jobs)}, "
+            f"runs={len(self._runs)}, "
+            f"operations={len(self._operations)}, "
+            f"inputs={len(self._inputs)}, "
+            f"outputs={len(self._outputs)}, "
+            f"schemas={len(self._schemas)}, "
+            f"users={len(self._users)}"
+            ")"
+        )
+
+    @staticmethod
+    def _add(context: dict[tuple, T], new_item: T) -> dict[tuple, T]:  # noqa: WPS602
+        if new_item.unique_key in context:
+            context[new_item.unique_key] = context[new_item.unique_key].merge(new_item)
+        else:
+            context[new_item.unique_key] = new_item
+        return context
+
+    def add_location(self, location: LocationDTO):
+        self._add(self._locations, location)
+
+    def add_dataset(self, dataset: DatasetDTO):
+        self._add(self._datasets, dataset)
+        self.add_location(dataset.location)
+
+    def add_dataset_symlink(self, dataset_symlink: DatasetSymlinkDTO):
+        self._add(self._dataset_symlinks, dataset_symlink)
+        self.add_dataset(dataset_symlink.from_dataset)
+        self.add_dataset(dataset_symlink.to_dataset)
+
+    def add_job(self, job: JobDTO):
+        self._add(self._jobs, job)
+        self.add_location(job.location)
+
+    def add_run(self, run: RunDTO):
+        self._add(self._runs, run)
+        self.add_job(run.job)
+        if run.parent_run:
+            self.add_run(run.parent_run)
+        if run.user:
+            self.add_user(run.user)
+
+    def add_operation(self, operation: OperationDTO):
+        self._add(self._operations, operation)
+        self.add_run(operation.run)
+
+    def add_input(self, input: InputDTO):
+        self._add(self._inputs, input)
+        self.add_operation(input.operation)
+        self.add_dataset(input.dataset)
+        if input.schema:
+            self.add_schema(input.schema)
+
+    def add_output(self, output: OutputDTO):
+        self._add(self._outputs, output)
+        self.add_operation(output.operation)
+        self.add_dataset(output.dataset)
+        if output.schema:
+            self.add_schema(output.schema)
+
+    def add_schema(self, schema: SchemaDTO):
+        self._add(self._schemas, schema)
+
+    def add_user(self, user: UserDTO):
+        self._add(self._users, user)
+
+    def _get_location(self, location_key: tuple) -> LocationDTO:
+        return self._locations[location_key]
+
+    def _get_schema(self, schema_key: tuple) -> SchemaDTO:
+        return self._schemas[schema_key]
+
+    def _get_user(self, user_key: tuple) -> UserDTO:
+        return self._users[user_key]
+
+    def _get_dataset(self, dataset_key: tuple) -> DatasetDTO:
+        dataset = self._datasets[dataset_key]
+        dataset.location = self._get_location(dataset.location.unique_key)
+        return dataset
+
+    def _get_dataset_symlink(self, dataset_symlink_key: tuple) -> DatasetSymlinkDTO:
+        dataset_symlink = self._dataset_symlinks[dataset_symlink_key]
+        dataset_symlink.from_dataset = self._get_dataset(dataset_symlink.from_dataset.unique_key)
+        dataset_symlink.to_dataset = self._get_dataset(dataset_symlink.to_dataset.unique_key)
+        return dataset_symlink
+
+    def _get_job(self, job_key: tuple) -> JobDTO:
+        job = self._jobs[job_key]
+        job.location = self._get_location(job.location.unique_key)
+        return job
+
+    def _get_run(self, run_key: tuple) -> RunDTO:
+        run = self._runs[run_key]
+        run.job = self._get_job(run.job.unique_key)
+        if run.parent_run:
+            run.parent_run = self._get_run(run.parent_run.unique_key)
+        if run.user:
+            run.user = self._get_user(run.user.unique_key)
+        return run
+
+    def _get_operation(self, operation_key: tuple) -> OperationDTO:
+        operation = self._operations[operation_key]
+        operation.run = self._get_run(operation.run.unique_key)
+        return operation
+
+    def _get_input(self, input_key: tuple) -> InputDTO:
+        input = self._inputs[input_key]
+        input.operation = self._get_operation(input.operation.unique_key)
+        input.dataset = self._get_dataset(input.dataset.unique_key)
+        if input.schema:
+            input.schema = self._get_schema(input.schema.unique_key)
+        return input
+
+    def _get_output(self, output_key: tuple) -> OutputDTO:
+        output = self._outputs[output_key]
+        output.operation = self._get_operation(output.operation.unique_key)
+        output.dataset = self._get_dataset(output.dataset.unique_key)
+        if output.schema:
+            output.schema = self._get_schema(output.schema.unique_key)
+        return output
+
+    def locations(self) -> list[LocationDTO]:
+        return list(map(self._get_location, self._locations))
+
+    def datasets(self) -> list[DatasetDTO]:
+        return list(map(self._get_dataset, self._datasets))
+
+    def dataset_symlinks(self) -> list[DatasetSymlinkDTO]:
+        return list(map(self._get_dataset_symlink, self._dataset_symlinks))
+
+    def jobs(self) -> list[JobDTO]:
+        return list(map(self._get_job, self._jobs))
+
+    def runs(self) -> list[RunDTO]:
+        return list(map(self._get_run, self._runs))
+
+    def operations(self) -> list[OperationDTO]:
+        return list(map(self._get_operation, self._operations))
+
+    def inputs(self) -> list[InputDTO]:
+        return list(map(self._get_input, self._inputs))
+
+    def outputs(self) -> list[OutputDTO]:
+        return list(map(self._get_output, self._outputs))
+
+    def schemas(self) -> list[SchemaDTO]:
+        return list(map(self._get_schema, self._schemas))
+
+    def users(self) -> list[UserDTO]:
+        return list(map(self._get_user, self._users))
+
+
+def extract_batch(events: list[OpenLineageRunEvent]) -> BatchExtractionResult:
+    result = BatchExtractionResult()
+
+    for event in events:
+        if event.job.facets.jobType and event.job.facets.jobType.jobType == OpenLineageJobType.JOB:
+            operation = extract_operation(event)
+            result.add_operation(operation)
+            for input_dataset in event.inputs:
+                input, symlinks = extract_input(operation, input_dataset)
+                result.add_input(input)
+                for symlink in symlinks:
+                    result.add_dataset_symlink(symlink)
+
+            for output_dataset in event.outputs:
+                output, symlinks = extract_output(operation, output_dataset)
+                result.add_output(output)
+                for symlink in symlinks:  # noqa: WPS440
+                    result.add_dataset_symlink(symlink)
+
+        else:
+            run = extract_run(event)
+            result.add_run(run)
+
+    return result

data_rentgen/consumer/extractors/dataset.py

@@ -0,0 +1,145 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from urllib.parse import urlparse
+
+from data_rentgen.consumer.openlineage.dataset import OpenLineageDataset
+from data_rentgen.consumer.openlineage.dataset_facets import (
+    OpenLineageStorageDatasetFacet,
+    OpenLineageSymlinkIdentifier,
+    OpenLineageSymlinkType,
+)
+from data_rentgen.dto import (
+    DatasetDTO,
+    DatasetSymlinkDTO,
+    DatasetSymlinkTypeDTO,
+    LocationDTO,
+)
+
+logger = logging.getLogger(__name__)
+
+METASTORE = DatasetSymlinkTypeDTO.METASTORE
+WAREHOUSE = DatasetSymlinkTypeDTO.WAREHOUSE
+
+
+def connect_dataset_with_symlinks(
+    dataset: DatasetDTO,
+    symlink: DatasetDTO,
+    type: OpenLineageSymlinkType,
+) -> list[DatasetSymlinkDTO]:
+    result = []
+    is_metastore_symlink = type == OpenLineageSymlinkType.TABLE
+
+    result.append(
+        DatasetSymlinkDTO(
+            from_dataset=dataset,
+            to_dataset=symlink,
+            type=METASTORE if is_metastore_symlink else WAREHOUSE,
+        ),
+    )
+    result.append(
+        DatasetSymlinkDTO(
+            from_dataset=symlink,
+            to_dataset=dataset,
+            type=WAREHOUSE if is_metastore_symlink else METASTORE,
+        ),
+    )
+
+    return sorted(result, key=lambda x: x.type)
+
+
+def extract_dataset(dataset: OpenLineageDataset | OpenLineageSymlinkIdentifier) -> DatasetDTO:
+    return DatasetDTO(
+        name=dataset.name,
+        location=extract_dataset_location(dataset),
+        format=extract_dataset_format(dataset),
+    )
+
+
+def extract_dataset_and_symlinks(dataset: OpenLineageDataset) -> tuple[DatasetDTO, list[DatasetSymlinkDTO]]:
+    if not dataset.facets.symlinks:
+        return extract_dataset(dataset), []
+
+    table_symlinks = [
+        identifier
+        for identifier in dataset.facets.symlinks.identifiers
+        if identifier.type == OpenLineageSymlinkType.TABLE
+    ]
+    if table_symlinks:
+        # We are swapping the dataset with its TABLE symlink to create a cleaner lineage.
+        # For example, by replacing an HDFS file with its corresponding Hive table.
+        # This ensures that all operations interact with a single table instead of multiple files (which may represent different partitions).
+        # Discussion on this issue: https://github.com/OpenLineage/OpenLineage/issues/2718
+
+        # TODO: add support for multiple TABLE symlinks
+        if len(table_symlinks) > 1:
+            logger.warning(
+                "Dataset has more than one TABLE symlink. Only the first one will be used for replacement. Symlink name: %s",
+                table_symlinks[0].name,
+            )
+        table_dataset = table_symlinks[0]
+        return (
+            DatasetDTO(
+                name=table_dataset.name,
+                location=extract_dataset_location(table_dataset),
+                format=extract_dataset_format(table_dataset),
+            ),
+            connect_dataset_with_symlinks(
+                extract_dataset(dataset),
+                extract_dataset(table_dataset),
+                OpenLineageSymlinkType.TABLE,
+            ),
+        )
+
+    symlinks = []
+    for identifier in dataset.facets.symlinks.identifiers:
+        symlinks.extend(
+            connect_dataset_with_symlinks(
+                extract_dataset(dataset),
+                extract_dataset(identifier),
+                identifier.type,
+            ),
+        )
+
+    return (
+        DatasetDTO(
+            name=dataset.name,
+            location=extract_dataset_location(dataset),
+            format=extract_dataset_format(dataset),
+        ),
+        symlinks,
+    )
+
+
+def extract_dataset_location(dataset: OpenLineageDataset | OpenLineageSymlinkIdentifier) -> LocationDTO:
+    namespace = dataset.namespace
+    if namespace == "file":
+        # TODO: remove after https://github.com/OpenLineage/OpenLineage/issues/2709
+        namespace = "file://"
+
+    url = urlparse(namespace)
+    scheme = url.scheme or "unknown"
+
+    # TODO: handle S3 bucket properly after https://github.com/OpenLineage/OpenLineage/issues/2816
+    netloc = url.netloc or url.path
+    hosts = list(filter(None, netloc.split(","))) or ["unknown"]
+    return LocationDTO(
+        type=scheme,
+        name=hosts[0],
+        addresses={f"{scheme}://{host}" for host in hosts},
+    )
+
+
+def extract_dataset_format(dataset: OpenLineageDataset | OpenLineageSymlinkIdentifier) -> str | None:
+    if isinstance(dataset, OpenLineageSymlinkIdentifier):
+        return None
+
+    match dataset.facets.storage:
+        case OpenLineageStorageDatasetFacet(storageLayer="default", fileFormat=file_format):
+            # See https://github.com/OpenLineage/OpenLineage/issues/2770
+            return file_format
+        case OpenLineageStorageDatasetFacet(storageLayer=storage_layer):
+            return storage_layer
+        case _:
+            return None

data_rentgen/consumer/extractors/input.py

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+from data_rentgen.consumer.extractors.dataset import extract_dataset_and_symlinks
+from data_rentgen.consumer.extractors.schema import extract_schema
+from data_rentgen.consumer.openlineage.dataset import OpenLineageInputDataset
+from data_rentgen.dto import DatasetSymlinkDTO, InputDTO
+from data_rentgen.dto.operation import OperationDTO
+
+
+def extract_input(
+    operation: OperationDTO,
+    dataset: OpenLineageInputDataset,
+) -> tuple[InputDTO, list[DatasetSymlinkDTO]]:
+    dataset_dto, symlinks = extract_dataset_and_symlinks(dataset)
+    result = InputDTO(
+        operation=operation,
+        dataset=dataset_dto,
+        schema=extract_schema(dataset),
+    )
+    if dataset.inputFacets.inputStatistics:
+        result.num_rows = dataset.inputFacets.inputStatistics.rows
+        result.num_bytes = dataset.inputFacets.inputStatistics.bytes
+        result.num_files = dataset.inputFacets.inputStatistics.files
+    return result, symlinks

data_rentgen/consumer/extractors/job.py

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+from urllib.parse import urlparse
+
+from data_rentgen.consumer.openlineage.job import OpenLineageJob
+from data_rentgen.consumer.openlineage.run_facets import OpenLineageParentJob
+from data_rentgen.dto import JobDTO, JobTypeDTO, LocationDTO
+
+
+def extract_job(job: OpenLineageJob | OpenLineageParentJob) -> JobDTO:
+    return JobDTO(
+        name=job.name,
+        location=extract_job_location(job),
+        type=extract_job_type(job),
+    )
+
+
+def extract_job_location(job: OpenLineageJob | OpenLineageParentJob) -> LocationDTO:
+    url = urlparse(job.namespace)
+    scheme = url.scheme or "unknown"
+    netloc = url.netloc or url.path
+    return LocationDTO(
+        type=scheme,
+        name=netloc,
+        addresses={f"{scheme}://{netloc}"},
+    )
+
+
+def extract_job_type(job: OpenLineageJob | OpenLineageParentJob) -> JobTypeDTO | None:
+    if isinstance(job, OpenLineageJob) and job.facets.jobType:
+        job_type = job.facets.jobType.jobType
+        integration_type = job.facets.jobType.integration
+        return JobTypeDTO(f"{integration_type}_{job_type}")
+
+    return None

data_rentgen/consumer/extractors/operation.py

@@ -0,0 +1,62 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+from data_rentgen.consumer.extractors.run import extract_run_minimal
+from data_rentgen.consumer.openlineage.run_event import (
+    OpenLineageRunEvent,
+    OpenLineageRunEventType,
+)
+from data_rentgen.dto import OperationDTO, OperationStatusDTO, OperationTypeDTO
+
+
+def extract_operation(event: OpenLineageRunEvent) -> OperationDTO:
+    # operation always has parent
+    run = extract_run_minimal(event.run.facets.parent)  # type: ignore[arg-type]
+
+    # in some cases, operation name may contain raw SELECT query with newlines
+    operation_name = " ".join(line.strip() for line in event.job.name.splitlines()).strip()
+    # remove parent job name from operation name
+    if operation_name.startswith(run.job.name):
+        prefix = len(run.job.name) + 1
+        operation_name = operation_name[prefix:]
+
+    operation = OperationDTO(
+        id=event.run.runId,  # type: ignore [arg-type]
+        run=run,
+        name=operation_name,
+        type=OperationTypeDTO(event.job.facets.jobType.processingType) if event.job.facets.jobType else None,
+    )
+    enrich_operation_status(operation, event)
+    enrich_operation_description(operation, event)
+    return operation
+
+
+def enrich_operation_status(operation: OperationDTO, event: OpenLineageRunEvent) -> OperationDTO:
+    match event.eventType:
+        case OpenLineageRunEventType.START:
+            operation.started_at = event.eventTime
+            operation.status = OperationStatusDTO.STARTED
+        case OpenLineageRunEventType.RUNNING:
+            operation.status = OperationStatusDTO.STARTED
+        case OpenLineageRunEventType.COMPLETE:
+            operation.ended_at = event.eventTime
+            operation.status = OperationStatusDTO.SUCCEEDED
+        case OpenLineageRunEventType.FAIL:
+            operation.ended_at = event.eventTime
+            operation.status = OperationStatusDTO.FAILED
+        case OpenLineageRunEventType.ABORT:
+            operation.ended_at = event.eventTime
+            operation.status = OperationStatusDTO.KILLED
+        case OpenLineageRunEventType.OTHER:
+            # OTHER is used only to update run statistics
+            pass
+    return operation
+
+
+def enrich_operation_description(operation: OperationDTO, event: OpenLineageRunEvent) -> OperationDTO:
+    spark_job_details = event.run.facets.spark_jobDetails
+    if spark_job_details:
+        operation.position = spark_job_details.jobId
+        operation.group = spark_job_details.jobGroup
+        operation.description = spark_job_details.jobDescription
+    return operation

data_rentgen/consumer/extractors/output.py

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: 2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+from data_rentgen.consumer.extractors.dataset import extract_dataset_and_symlinks
+from data_rentgen.consumer.extractors.schema import extract_schema
+from data_rentgen.consumer.openlineage.dataset import OpenLineageOutputDataset
+from data_rentgen.dto import DatasetSymlinkDTO, OutputDTO, OutputTypeDTO
+from data_rentgen.dto.operation import OperationDTO
+
+
+def extract_output(
+    operation: OperationDTO,
+    dataset: OpenLineageOutputDataset,
+) -> tuple[OutputDTO, list[DatasetSymlinkDTO]]:
+    lifecycle_change = dataset.facets.lifecycleStateChange
+    if lifecycle_change:
+        output_type = OutputTypeDTO(lifecycle_change.lifecycleStateChange)
+    else:
+        output_type = OutputTypeDTO.APPEND
+    dataset_dto, symlinks = extract_dataset_and_symlinks(dataset)
+    result = OutputDTO(
+        type=output_type,
+        operation=operation,
+        dataset=dataset_dto,
+        schema=extract_schema(dataset),
+    )
+    if dataset.outputFacets.outputStatistics:
+        result.num_rows = dataset.outputFacets.outputStatistics.rows
+        result.num_bytes = dataset.outputFacets.outputStatistics.bytes
+        result.num_files = dataset.outputFacets.outputStatistics.files
+
+    return result, symlinks