openmetadata-ingestion 1.4.6.0__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

metadata/data_insight/producer/cost_analysis_producer.py

@@ -1,126 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Producer class for data insight entity reports
-"""
-
-import traceback
-from typing import Dict, Iterable, Optional
-
-from pydantic import BaseModel
-
-from metadata.data_insight.producer.producer_interface import ProducerInterface
-from metadata.generated.schema.entity.data.table import Table
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
-from metadata.generated.schema.type.lifeCycle import LifeCycle
-from metadata.ingestion.api.models import Entity
-from metadata.ingestion.ometa.utils import model_str
-from metadata.utils.logger import data_insight_logger
-
-logger = data_insight_logger()
-
-
-class CostAnalysisReportData(BaseModel):
-    """
-    Query executed get life cycle
-    """
-
-    entity: Entity
-    life_cycle: Optional[LifeCycle]
-    size: Optional[float]
-
-
-class CostAnalysisProducer(ProducerInterface):
-    """entity producer class"""
-
-    def _check_profiler_and_usage_support(
-        self, database_service: DatabaseService
-    ) -> bool:
-        return (
-            hasattr(database_service.connection.config, "supportsUsageExtraction")
-            and database_service.connection.config.supportsUsageExtraction.__root__
-            and hasattr(database_service.connection.config, "supportsProfiler")
-            and database_service.connection.config.supportsProfiler.__root__
-        )
-
-    def _check_life_cycle_and_size_data(
-        self, table: Table
-    ) -> Optional[CostAnalysisReportData]:
-        """
-        Method to check if the valid life cycle and table size data is present for the table
-        """
-        cost_analysis_report_data = CostAnalysisReportData(entity=table)
-        if table.lifeCycle and table.lifeCycle.accessed:
-            cost_analysis_report_data.life_cycle = table.lifeCycle
-
-        table_profile = self.metadata.get_latest_table_profile(
-            fqn=table.fullyQualifiedName
-        )
-        if table_profile.profile:
-            cost_analysis_report_data.size = table_profile.profile.sizeInByte
-
-        if cost_analysis_report_data.life_cycle or cost_analysis_report_data.size:
-            return cost_analysis_report_data
-        return None
-
-    def life_cycle_data_dict(
-        self, entities_cache: Optional[Dict], database_service_fqn: str
-    ) -> Iterable[Dict]:
-        """
-        Cache the required lifecycle data to be used by the processors and return the dict
-        """
-        if entities_cache.get(database_service_fqn):
-            yield entities_cache[database_service_fqn]
-        else:
-            tables = self.metadata.list_all_entities(
-                Table,
-                limit=100,
-                skip_on_failure=True,
-                params={"database": database_service_fqn},
-            )
-            entities_cache[database_service_fqn] = {}
-
-            for table in tables:
-                try:
-                    cost_analysis_data = self._check_life_cycle_and_size_data(
-                        table=table
-                    )
-                    if cost_analysis_data:
-                        entities_cache[database_service_fqn][
-                            model_str(table.fullyQualifiedName)
-                        ] = cost_analysis_data
-                except Exception as err:
-                    logger.error(
-                        f"Error trying to fetch cost analysis data for [{model_str(table.fullyQualifiedName)}] -- {err}"
-                    )
-                    logger.debug(traceback.format_exc())
-
-            yield entities_cache[database_service_fqn]
-
-    # pylint: disable=dangerous-default-value
-    def fetch_data(
-        self, limit=100, fields=["*"], entities_cache=None
-    ) -> Optional[Iterable[Dict]]:
-        database_services = self.metadata.list_all_entities(
-            DatabaseService, limit=limit, fields=fields, skip_on_failure=True
-        )
-        for database_service in database_services or []:
-            try:
-                if self._check_profiler_and_usage_support(database_service):
-                    yield from self.life_cycle_data_dict(
-                        entities_cache=entities_cache,
-                        database_service_fqn=model_str(
-                            database_service.fullyQualifiedName
-                        ),
-                    )
-            except Exception as err:
-                logger.error(f"Error trying to fetch entities -- {err}")
-                logger.debug(traceback.format_exc())

metadata/data_insight/producer/entity_producer.py

@@ -1,63 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Producer class for data insight entity reports
-"""
-
-import traceback
-from typing import Iterable
-
-from metadata.data_insight.producer.producer_interface import ProducerInterface
-from metadata.generated.schema.entity.data import (
-    chart,
-    container,
-    dashboard,
-    database,
-    databaseSchema,
-    mlmodel,
-    pipeline,
-    searchIndex,
-    storedProcedure,
-    table,
-    topic,
-)
-from metadata.utils.logger import data_insight_logger
-
-logger = data_insight_logger()
-
-
-class EntityProducer(ProducerInterface):
-    """entity producer class"""
-
-    entities = [
-        chart.Chart,
-        dashboard.Dashboard,
-        database.Database,
-        databaseSchema.DatabaseSchema,
-        mlmodel.MlModel,
-        pipeline.Pipeline,
-        table.Table,
-        topic.Topic,
-        container.Container,
-        storedProcedure.StoredProcedure,
-        searchIndex.SearchIndex,
-    ]
-
-    # pylint: disable=dangerous-default-value
-    def fetch_data(self, limit=100, fields=["*"], entities_cache=None) -> Iterable:
-        for entity in self.entities:
-            try:
-                yield from self.metadata.list_all_entities(
-                    entity, limit=limit, fields=fields, skip_on_failure=True
-                )
-            except Exception as err:
-                logger.error(f"Error trying to fetch entity -- {err}")
-                logger.debug(traceback.format_exc())

metadata/data_insight/producer/producer_factory.py

@@ -1,53 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-OpenMetadata producer factory
-"""
-
-from typing import Type
-
-from metadata.data_insight.producer.cost_analysis_producer import CostAnalysisProducer
-from metadata.data_insight.producer.entity_producer import EntityProducer
-from metadata.data_insight.producer.producer_interface import ProducerInterface
-from metadata.data_insight.producer.web_analytics_producer import WebAnalyticsProducer
-from metadata.generated.schema.analytics.reportData import ReportDataType
-
-
-class ProducerFactory:
-    def __init__(self):
-        self._producer_type = {}
-
-    def register(self, producer_type: str, producer_class: Type[ProducerInterface]):
-        """register a new producer"""
-        self._producer_type[producer_type] = producer_class
-
-    def create(self, producer_type: str, *args, **kwargs) -> ProducerInterface:
-        """create producer object based on producer type"""
-        producer_class = self._producer_type.get(producer_type)
-        if not producer_class:
-            raise RuntimeError(f"Producer type {producer_type} not found")
-        return producer_class(*args, **kwargs)
-
-
-producer_factory = ProducerFactory()
-producer_factory.register(ReportDataType.entityReportData.value, EntityProducer)
-producer_factory.register(
-    ReportDataType.rawCostAnalysisReportData.value, CostAnalysisProducer
-)
-producer_factory.register(
-    ReportDataType.aggregatedCostAnalysisReportData.value, CostAnalysisProducer
-)
-producer_factory.register(
-    ReportDataType.webAnalyticEntityViewReportData.value, WebAnalyticsProducer
-)
-producer_factory.register(
-    ReportDataType.webAnalyticUserActivityReportData.value, WebAnalyticsProducer
-)

metadata/data_insight/producer/web_analytics_producer.py

@@ -1,84 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Producer class for data insight web analytics reports
-"""
-
-
-from typing import Dict, Optional
-
-from metadata.data_insight.producer.producer_interface import ProducerInterface
-from metadata.generated.schema.analytics.webAnalyticEventData import (
-    WebAnalyticEventData,
-)
-from metadata.ingestion.ometa.models import EntityList
-from metadata.utils.time_utils import (
-    get_beginning_of_day_timestamp_mill,
-    get_end_of_day_timestamp_mill,
-)
-
-# We'll get web analytics data for the previous day
-START_TS = str(get_beginning_of_day_timestamp_mill(days=1))
-END_TS = str(get_end_of_day_timestamp_mill(days=1))
-
-# we'll cache events to avoid fetching them multiple times
-# the key will be the entity pagination after value
-CACHED_EVENTS: Dict[str, EntityList[WebAnalyticEventData]] = {}
-
-
-class WebAnalyticsProducer(ProducerInterface):
-    """web analytics producer class"""
-
-    entity_type = WebAnalyticEventData
-    params = {
-        "startTs": START_TS,
-        "endTs": END_TS,
-        "eventType": "PageView",
-    }
-    clear_cache = False
-
-    @staticmethod
-    def _cache_events(
-        key: Optional[str], value: EntityList[WebAnalyticEventData]
-    ) -> None:
-        """cache events"""
-        CACHED_EVENTS[str(key)] = value
-
-    # pylint: disable=dangerous-default-value
-    def _get_events(
-        self, after: Optional[str], limit=100, fields=["*"]
-    ) -> EntityList[WebAnalyticEventData]:
-        """Try to retrieve events from catch otherwise retrieve them from DB"""
-        events = CACHED_EVENTS.get(str(after))
-        if not events:
-            events: EntityList[WebAnalyticEventData] = self.metadata.list_entities(
-                entity=self.entity_type,
-                params=self.params,
-                after=after,
-                limit=limit,
-                fields=fields,
-                skip_on_failure=True,
-            )  # type: ignore
-            self._cache_events(after, events)
-        return events
-
-    def _clear_cache(self):
-        """clear cache"""
-        CACHED_EVENTS.clear()
-
-    def fetch_data(
-        self, limit=100, fields=["*"], entities_cache=None
-    ):  # pylint: disable=dangerous-default-value
-        """fetch data for web analytics event"""
-        events = self._get_events(None, limit, fields)
-
-        for entity in events.entities:
-            yield entity

metadata/data_insight/source/metadata.py

@@ -1,175 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-OpenMetadata source for the data insight workflow
-"""
-
-import traceback
-from datetime import datetime
-from types import MappingProxyType
-from typing import Dict, Iterable, Optional, Union, cast
-
-from pydantic import BaseModel
-
-from metadata.data_insight.processor.reports.cost_analysis_report_data_processor import (
-    AggregatedCostAnalysisReportDataProcessor,
-    RawCostAnalysisReportDataProcessor,
-)
-from metadata.data_insight.processor.reports.data_processor import DataProcessor
-from metadata.data_insight.processor.reports.entity_report_data_processor import (
-    EntityReportDataProcessor,
-)
-from metadata.data_insight.processor.reports.web_analytic_report_data_processor import (
-    WebAnalyticEntityViewReportDataProcessor,
-    WebAnalyticUserActivityReportDataProcessor,
-)
-from metadata.data_insight.producer.producer_factory import producer_factory
-from metadata.generated.schema.analytics.reportData import ReportData, ReportDataType
-from metadata.generated.schema.entity.services.ingestionPipelines.status import (
-    StackTraceError,
-)
-from metadata.ingestion.api.models import Either
-from metadata.ingestion.api.step import Step
-from metadata.ingestion.api.steps import Source
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils.logger import profiler_logger
-
-logger = profiler_logger()
-
-
-class DataInsightRecord(BaseModel):
-    """Return class for the OpenMetadata Profiler Source"""
-
-    class Config:
-        arbitrary_types_allowed = True
-        extra = "forbid"
-
-    data: ReportData
-
-    def __str__(self):
-        """Return the string representation of the entity produced"""
-        return f"Entities: {list(self.data)}"
-
-
-class DataInsightSource(Source):
-    """
-    This source lists and filters the entities that need
-    to be processed by the profiler workflow.
-
-    Note that in order to manage the following steps we need
-    to test the connection against the Database Service Source.
-    We do this here as well.
-    """
-
-    def __init__(self, metadata: OpenMetadata):
-        """Instantiate source object"""
-        super().__init__()
-        self.metadata = metadata
-        self.date = datetime.utcnow().strftime("%Y-%m-%d")
-        self.entities_cache = {}
-
-        _processors = self._instantiate_processors()
-        self._processors: Dict[
-            str,
-            Union[
-                DataProcessor,
-                EntityReportDataProcessor,
-                WebAnalyticEntityViewReportDataProcessor,
-                WebAnalyticUserActivityReportDataProcessor,
-                AggregatedCostAnalysisReportDataProcessor,
-                RawCostAnalysisReportDataProcessor,
-            ],
-        ] = MappingProxyType(
-            _processors
-        )  # make an immutable copy of the dict
-
-    @property
-    def name(self) -> str:
-        return "OpenMetadata Insights"
-
-    @property
-    def processors(self) -> Dict[str, Optional[DataProcessor]]:
-        """dictionary of processors"""
-        return self._processors
-
-    def _instantiate_processors(self) -> Dict[str, DataProcessor]:
-        """Instantiate the data processors"""
-        return {
-            report_data_type.value: DataProcessor.create(
-                _data_processor_type=report_data_type.value,
-                metadata=self.metadata,
-            )
-            for report_data_type in ReportDataType
-        }
-
-    def prepare(self):
-        """Nothing to prepare"""
-
-    def test_connection(self) -> None:
-        """Nothing to test as we'll be connecting to OM"""
-
-    def _iter(self, *_, **__) -> Iterable[Either[DataInsightRecord]]:
-        """Produces the data that need to be processed"""
-
-        for report_data_type in ReportDataType:
-            logger.info(f"Processing data for report type {report_data_type}")
-            try:
-                self.metadata.delete_report_data_at_date(report_data_type, self.date)
-                producer = producer_factory.create(
-                    report_data_type.value, self.metadata
-                )
-
-                processor = self.processors[report_data_type.value]
-                processor = cast(DataProcessor, processor)
-                processor.pre_hook() if processor.pre_hook else None  # pylint: disable=expression-not-assigned
-
-                for data in (
-                    producer.fetch_data(
-                        fields=["owner", "tags"], entities_cache=self.entities_cache
-                    )
-                    or []
-                ):
-                    processor.refine(data)
-
-                processor.post_hook() if processor.post_hook else None  # pylint: disable=expression-not-assigned
-
-                if processor.clean_up_cache:
-                    self.entities_cache.clear()
-
-                for data in processor.yield_refined_data():
-                    yield Either(left=None, right=DataInsightRecord(data=data))
-            except KeyError as key_error:
-                yield Either(
-                    left=StackTraceError(
-                        name=report_data_type.value,
-                        error=f"Error retrieving processor with exception [{key_error}]."
-                        "Available processors are {self.processors}",
-                        stackTrace=traceback.format_exc(),
-                    ),
-                    right=None,
-                )
-            except Exception as exc:
-                yield Either(
-                    left=StackTraceError(
-                        name=report_data_type.value,
-                        error=f"Error listing data for report with exception [{exc}]",
-                        stackTrace=traceback.format_exc(),
-                    ),
-                    right=None,
-                )
-
-    # pylint: disable=arguments-differ
-    @classmethod
-    def create(cls, metadata: OpenMetadata) -> "Step":
-        return cls(metadata)
-
-    def close(self) -> None:
-        """Nothing to close"""

metadata/workflow/data_insight.py

@@ -1,78 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Workflow definition for the data insight
-"""
-from metadata.data_insight.processor.kpi.kpi_runner import KpiRunner
-from metadata.data_insight.source.metadata import DataInsightSource
-from metadata.generated.schema.analytics.basic import WebAnalyticEventType
-from metadata.generated.schema.tests.testSuite import ServiceType
-from metadata.ingestion.api.steps import Sink
-from metadata.utils.importer import import_sink_class
-from metadata.utils.logger import data_insight_logger
-from metadata.utils.time_utils import get_beginning_of_day_timestamp_mill
-from metadata.workflow.ingestion import IngestionWorkflow
-
-logger = data_insight_logger()
-
-
-class DataInsightWorkflow(IngestionWorkflow):
-    """Data insight ingestion workflow implementation"""
-
-    retention_days = 7
-
-    def _retrieve_service_connection_if_needed(self, service_type: ServiceType) -> None:
-        """No service connection needed for data insight"""
-        return None
-
-    def _run_kpi_processor(self) -> None:
-        """Run kpi processor. It will run as a step but as a post execution.
-        We need the source to be executed first to compute the KPI results"""
-        kpi_runner = KpiRunner(self.metadata)
-
-        for kpi_result in kpi_runner.run():
-            # Pick up the sink
-            self.steps[0].run(kpi_result)
-
-    def _clean_up_web_analytics_events(self) -> None:
-        """
-        We will delete web analytics events older than `cls.retention_days`
-        to prevent storage explosion.
-        TODO: deprecate to implement it with backend workflows
-        """
-        tmsp = get_beginning_of_day_timestamp_mill(days=self.retention_days)
-        for web_analytic_event in WebAnalyticEventType:
-            self.metadata.delete_web_analytic_event_before_ts_exclusive(
-                web_analytic_event,
-                tmsp,
-            )
-
-    def _get_sink(self) -> Sink:
-        """Retrieve sink for data insight workflow"""
-        sink_type = "metadata-rest"
-        sink_class = import_sink_class(sink_type=sink_type)
-        sink_config = {"api_endpoint": self.metadata_config.hostPort}
-        sink: Sink = sink_class.create(sink_config, self.metadata)
-        logger.debug(f"Sink type:{self.config.sink.type}, {sink_class} configured")
-
-        return sink
-
-    def execute_internal(self):
-        """Use parent logic and add step to process KPIs"""
-        super().execute_internal()
-        self._run_kpi_processor()
-        self._clean_up_web_analytics_events()
-
-    def set_steps(self):
-        self.source = DataInsightSource.create(self.metadata)  # type: ignore
-
-        sink = self._get_sink()
-        self.steps = (sink,)