openmetadata-ingestion 1.4.8.0__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

{openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/top_level.txt

@@ -1,2 +1,3 @@
+_openmetadata_testutils
 airflow_provider_openmetadata
 metadata

metadata/cli/insight.py

@@ -1,51 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Data Insigt utility for the metadata CLI
-"""
-import sys
-import traceback
-from pathlib import Path
-
-from metadata.config.common import load_config_file
-from metadata.utils.logger import cli_logger
-from metadata.workflow.data_insight import DataInsightWorkflow
-from metadata.workflow.workflow_output_handler import (
-    WorkflowType,
-    print_init_error,
-    print_status,
-)
-
-logger = cli_logger()
-
-
-def run_insight(config_path: Path) -> None:
-    """
-    Run the Data Insigt workflow from a config path
-    to a JSON or YAML file
-    :param config_path: Path to load JSON config
-    """
-
-    config_dict = None
-    try:
-        config_dict = load_config_file(config_path)
-        workflow = DataInsightWorkflow.create(config_dict)
-        logger.debug(f"Using config: {workflow.config}")
-    except Exception as exc:
-        logger.debug(traceback.format_exc())
-        print_init_error(exc, config_dict, WorkflowType.INSIGHT)
-        sys.exit(1)
-
-    workflow.execute()
-    workflow.stop()
-    print_status(workflow)
-    workflow.raise_from_status()

metadata/data_insight/processor/kpi/kpi_runner.py

@@ -1,156 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Runner class used to check KPI status
-"""
-
-from __future__ import annotations
-
-import time as tme
-from datetime import datetime
-from typing import Iterator, Optional
-
-from metadata.data_insight.processor.kpi.run_result_registry import run_result_registry
-from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
-from metadata.generated.schema.dataInsight.dataInsightChartResult import (
-    DataInsightChartResult,
-)
-from metadata.generated.schema.dataInsight.kpi.basic import KpiResult, KpiTarget
-from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
-from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.status import Status
-from metadata.ingestion.ometa.models import EntityList
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils.logger import data_insight_logger
-from metadata.utils.time_utils import (
-    get_beginning_of_day_timestamp_mill,
-    get_end_of_day_timestamp_mill,
-)
-
-logger = data_insight_logger()
-
-TIMEOUT = 30
-
-
-class KpiRunner:
-    """KPI runner class
-
-    Attrs:
-        kpis: list[Kpi]
-        metadata: OpenMetadata
-        processor_status: SourceStatus
-    """
-
-    def __init__(self, metadata: OpenMetadata) -> None:
-        self.metadata = metadata
-        self.datetime = int(datetime.utcnow().timestamp() * 1000)
-        self.processor_status = Status()
-        self.kpis = self.get_active_kpis()
-
-    def _is_kpi_active(self, entity: Kpi) -> bool:
-        """Check if a KPI is active
-
-        Args:
-            entity (Kpi): KPI entity
-
-        Returns:
-            Kpi:
-        """
-
-        start_date = entity.startDate.__root__
-        end_date = entity.endDate.__root__
-
-        if not start_date or not end_date:
-            logger.warning(
-                f"Start date or End date was not defined.\n\t-startDate: {start_date}\n\t-end_date: {end_date}\n"
-                "We won't be running the KPI validation"
-            )
-            return False
-
-        if start_date <= self.datetime <= end_date:
-            return True
-
-        return False
-
-    # pylint: disable=dangerous-default-value
-    def get_kpis(self, limit=100, fields=["*"]) -> EntityList[Kpi]:
-        """Get the list of all the KPIs
-
-        Args:
-            limit (int, optional): limit of result to return. Defaults to 100.
-            fields (list, optional): Fields to include. Defaults to ["*"].
-
-        Returns:
-            EntityList[Kpi]: List of KPIs
-        """
-        return self.metadata.list_entities(
-            entity=Kpi, limit=limit, fields=fields  # type: ignore
-        )
-
-    def get_active_kpis(self):
-        """Get the list of active KPIs"""
-        kpis = self.get_kpis()
-        return [kpi for kpi in kpis.entities if self._is_kpi_active(kpi)]
-
-    def _get_data_insight_chart_result(
-        self, data_insight_chart: EntityReference
-    ) -> DataInsightChartResult:
-        """get data insight result for a specific chart
-
-        Args:
-            data_insight_chart (EntityReference): _description_
-        """
-        results = None
-        data_insight_chart_entity: Optional[
-            DataInsightChart
-        ] = self.metadata.get_by_name(
-            entity=DataInsightChart,
-            fqn=data_insight_chart.fullyQualifiedName,
-            fields=["*"],
-        )
-
-        if not data_insight_chart_entity:
-            logger.warning(
-                f"No entity returned for dataInsightChart {data_insight_chart.name}"
-            )
-            return None
-
-        timeout = tme.time() + TIMEOUT
-        while True:
-            results = self.metadata.get_aggregated_data_insight_results(
-                start_ts=get_beginning_of_day_timestamp_mill(),
-                end_ts=get_end_of_day_timestamp_mill(),
-                data_insight_chart_nane=data_insight_chart_entity.name.__root__,
-                data_report_index=data_insight_chart_entity.dataIndexType.value,
-            )
-            if results.data or tme.time() > timeout:
-                break
-
-        return results
-
-    def run(self) -> Iterator[KpiResult]:
-        """Method to run the KPI status check"""
-        for kpi in self.kpis:
-            kpi_target: list[KpiTarget] = kpi.targetDefinition
-            data_insight_chart_result: DataInsightChartResult = (
-                self._get_data_insight_chart_result(kpi.dataInsightChart)
-            )
-            kpi_result = run_result_registry.registry[kpi.dataInsightChart.name](
-                kpi_target,
-                data_insight_chart_result.data,
-                kpi.fullyQualifiedName,
-                self.datetime,
-            )
-
-            yield kpi_result
-
-    def get_status(self):
-        return self.processor_status

metadata/data_insight/processor/kpi/run_result_registry.py

@@ -1,162 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-function to compute kpi
-"""
-
-from __future__ import annotations
-
-import ast
-import traceback
-
-from metadata.generated.schema.dataInsight.kpi.basic import KpiResult, KpiTarget
-from metadata.generated.schema.dataInsight.type.percentageOfEntitiesWithDescriptionByType import (
-    PercentageOfEntitiesWithDescriptionByType,
-)
-from metadata.generated.schema.dataInsight.type.percentageOfEntitiesWithOwnerByType import (
-    PercentageOfEntitiesWithOwnerByType,
-)
-from metadata.generated.schema.type.basic import FullyQualifiedEntityName
-from metadata.utils.dispatch import enum_register
-from metadata.utils.logger import profiler_interface_registry_logger
-
-logger = profiler_interface_registry_logger()
-
-
-def percentage_of_entities_with_description_kpi_result(
-    kpi_target: list[KpiTarget],
-    results: list[PercentageOfEntitiesWithDescriptionByType],
-    kpi_fqn: FullyQualifiedEntityName,
-    timestamp: int,
-) -> KpiResult:
-    """Validate percentage of entities with description kpi
-
-    Args:
-        kpi_target (list[KpiTarget]):
-        results (list[PercentageOfEntitiesWithDescriptionByType]):
-
-    Raises:
-        ValueError:
-        RuntimeError:
-
-    Returns:
-        KpiResult:
-    """
-    if not results:
-        raise ValueError("Cannot compute KPI. No results found")
-
-    latest_chart_result = {}
-
-    try:
-        total_entity_with_description = sum(
-            result.completedDescription for result in results
-        )
-        total_entity_count = sum(result.entityCount for result in results)
-        latest_chart_result["completedDescriptionFraction"] = (
-            total_entity_with_description / total_entity_count
-        )
-    except (TypeError, ZeroDivisionError) as exc:
-        logger.debug(traceback.format_exc())
-        logger.error(f"Could not run KPI result - {exc}")
-        raise RuntimeError(exc)
-
-    latest_chart_result["completedDescription"] = total_entity_with_description
-
-    target_results = []
-
-    for target in kpi_target:
-        try:
-            value = latest_chart_result[target.name]
-        except KeyError as exc:
-            logger.warning(f"Could not compute KPI result for {target.name} - {exc}")
-        else:
-            target_results.append(
-                KpiTarget(
-                    name=target.name,
-                    value=value,
-                    targetMet=value > ast.literal_eval(target.value),
-                )
-            )
-
-    return KpiResult(
-        timestamp=timestamp,
-        targetResult=target_results,
-        kpiFqn=kpi_fqn,
-    )
-
-
-def percentage_of_entities_with_owner_kpi_result(
-    kpi_target: list[KpiTarget],
-    results: list[PercentageOfEntitiesWithOwnerByType],
-    kpi_fqn: FullyQualifiedEntityName,
-    timestamp: int,
-) -> KpiResult:
-    """_summary_
-
-    Args:
-        kpi_target (list[KpiTarget]): _description_
-        data_insight_chart_result (list[PercentageOfEntitiesWithOwnerByType]): _description_
-
-    Raises:
-        RuntimeError: _description_
-
-    Returns:
-        KpiResult: _description_
-    """
-
-    if not results:
-        raise ValueError("Cannot compute KPI. No results found")
-
-    latest_chart_result = {}
-
-    try:
-        total_entity_with_owner = sum(result.hasOwner for result in results)
-        total_entity_count = sum(result.entityCount for result in results)
-        latest_chart_result["hasOwnerFraction"] = (
-            total_entity_with_owner / total_entity_count
-        )
-    except (TypeError, ZeroDivisionError) as exc:
-        logger.debug(traceback.format_exc())
-        logger.error(f"Could not run KPI result - {exc}")
-        raise RuntimeError(exc)
-
-    latest_chart_result["hasOwner"] = total_entity_with_owner
-
-    target_results = []
-
-    for target in kpi_target:
-        try:
-            value = latest_chart_result[target.name]
-        except KeyError as exc:
-            logger.warning(f"Could not compute KPI result for {target.name} - {exc}")
-        else:
-            target_results.append(
-                KpiTarget(
-                    name=target.name,
-                    value=value,
-                    targetMet=value > ast.literal_eval(target.value),
-                )
-            )
-
-    return KpiResult(
-        timestamp=timestamp,
-        targetResult=target_results,
-        kpiFqn=kpi_fqn,
-    )
-
-
-run_result_registry = enum_register()
-run_result_registry.add("PercentageOfEntitiesWithDescriptionByType")(
-    percentage_of_entities_with_description_kpi_result
-)
-run_result_registry.add("PercentageOfEntitiesWithOwnerByType")(
-    percentage_of_entities_with_owner_kpi_result
-)

metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py

@@ -1,253 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Processor class used to compute refined report data
-"""
-
-from __future__ import annotations
-
-import traceback
-from collections import defaultdict
-from copy import deepcopy
-from typing import Dict, Iterable, Optional
-
-from metadata.data_insight.processor.reports.data_processor import DataProcessor
-from metadata.generated.schema.analytics.reportData import ReportData, ReportDataType
-from metadata.generated.schema.analytics.reportDataType.aggregatedCostAnalysisReportData import (
-    AggregatedCostAnalysisReportData,
-)
-from metadata.generated.schema.analytics.reportDataType.rawCostAnalysisReportData import (
-    RawCostAnalysisReportData,
-)
-from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.generated.schema.type.lifeCycle import LifeCycle
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.ometa.utils import model_str
-from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP
-from metadata.utils.logger import data_insight_logger
-from metadata.utils.time_utils import get_end_of_day_timestamp_mill
-
-logger = data_insight_logger()
-
-
-UNUSED_DATA_ASSETS = "unusedDataAssets"
-FREQUENTLY_USED_DATA_ASSETS = "frequentlyUsedDataAssets"
-COUNT = "count"
-SIZE = "size"
-TOTAL_SIZE = "totalSize"
-TOTAL_COUNT = "totalCount"
-
-THREE_DAYS = "threeDays"
-SEVEN_DAYS = "sevenDays"
-FOURTEEN_DAYS = "fourteenDays"
-THIRTY_DAYS = "thirtyDays"
-SIXTY_DAYS = "sixtyDays"
-
-DAYS = [
-    (3, THREE_DAYS),
-    (7, SEVEN_DAYS),
-    (14, FOURTEEN_DAYS),
-    (30, THIRTY_DAYS),
-    (60, SIXTY_DAYS),
-]
-
-DAYS_WISE_METRIC_DICT = {
-    THREE_DAYS: 0,
-    SEVEN_DAYS: 0,
-    FOURTEEN_DAYS: 0,
-    THIRTY_DAYS: 0,
-    SIXTY_DAYS: 0,
-}
-
-ASSET_METRIC_DICT = {
-    COUNT: deepcopy(DAYS_WISE_METRIC_DICT),
-    SIZE: deepcopy(DAYS_WISE_METRIC_DICT),
-    TOTAL_SIZE: 0,
-    TOTAL_COUNT: 0,
-}
-
-
-class RawCostAnalysisReportDataProcessor(DataProcessor):
-    """Processor class used as a bridge to refine the data"""
-
-    _data_processor_type = ReportDataType.rawCostAnalysisReportData.value
-
-    def __init__(self, metadata: OpenMetadata):
-        super().__init__(metadata)
-        self.pre_hook = self._pre_hook_fn
-
-    def _pre_hook_fn(self):
-        """
-        Method to delete the previous rows of the RawCostAnalysisReportData type report
-        """
-        self.metadata.delete_report_data(ReportDataType.rawCostAnalysisReportData)
-
-    def yield_refined_data(self) -> Iterable[ReportData]:
-        """yield refined data"""
-        for _, value in self._refined_data.items():
-            yield ReportData(
-                timestamp=self.timestamp,
-                reportDataType=ReportDataType.rawCostAnalysisReportData.value,
-                data=value,
-            )  # type: ignore
-
-    def refine(self, entity: Dict) -> None:
-        """Aggregate data
-        Returns:
-            list:
-        """
-
-        for entity_fqn, cost_analysis_report_data in entity.items():
-            try:
-                cost_analysis_data = RawCostAnalysisReportData(
-                    entity=EntityReference(
-                        id=cost_analysis_report_data.entity.id,
-                        fullyQualifiedName=model_str(
-                            cost_analysis_report_data.entity.fullyQualifiedName
-                        ),
-                        type=ENTITY_REFERENCE_TYPE_MAP[
-                            type(cost_analysis_report_data.entity).__name__
-                        ],
-                    ),
-                    lifeCycle=cost_analysis_report_data.life_cycle,
-                    sizeInByte=cost_analysis_report_data.size,
-                )
-                self._refined_data[entity_fqn] = cost_analysis_data
-                self.processor_status.scanned(entity_fqn)
-            except Exception as err:
-                logger.debug(traceback.format_exc())
-                logger.error(f"Error trying fetch cost analysis data -- {err}")
-
-    def get_status(self):
-        return self.processor_status
-
-
-class AggregatedCostAnalysisReportDataProcessor(DataProcessor):
-    """Processor class used as a bridge to refine the data"""
-
-    _data_processor_type = ReportDataType.aggregatedCostAnalysisReportData.value
-
-    def __init__(self, metadata: OpenMetadata):
-        super().__init__(metadata)
-        self._refined_data = defaultdict(lambda: defaultdict(dict))
-        self.post_hook = self._post_hook_fn
-        self.clean_up_cache = True
-
-    def yield_refined_data(self) -> Iterable[ReportData]:
-        """Yield refined data"""
-        for data in self._refined_data:
-            yield ReportData(
-                timestamp=self.timestamp,
-                reportDataType=ReportDataType.aggregatedCostAnalysisReportData.value,
-                data=data,
-            )  # type: ignore
-
-    def refine(self, entity: Dict) -> None:
-        """Aggregate data
-        Returns:
-            list:
-        """
-        try:
-            for entity_fqn, cost_analysis_report_data in entity.items():
-                entity_type = str(cost_analysis_report_data.entity.__class__.__name__)
-                service_type = str(cost_analysis_report_data.entity.serviceType.name)
-                service_name = str(cost_analysis_report_data.entity.service.name)
-                if not self._refined_data[str(entity_type)][service_type].get(
-                    service_name
-                ):
-                    self._refined_data[entity_type][service_type][service_name] = {
-                        TOTAL_SIZE: 0,
-                        TOTAL_COUNT: 0,
-                        UNUSED_DATA_ASSETS: deepcopy(ASSET_METRIC_DICT),
-                        FREQUENTLY_USED_DATA_ASSETS: deepcopy(ASSET_METRIC_DICT),
-                    }
-                else:
-                    self._refined_data[entity_type][service_type][service_name][
-                        TOTAL_SIZE
-                    ] += (cost_analysis_report_data.size or 0)
-                    self._refined_data[entity_type][service_type][service_name][
-                        TOTAL_COUNT
-                    ] += 1
-
-                self._get_data_assets_dict(
-                    life_cycle=cost_analysis_report_data.life_cycle,
-                    size=cost_analysis_report_data.size,
-                    data=self._refined_data[entity_type][service_type][service_name],
-                )
-
-                self.processor_status.scanned(entity_fqn)
-        except Exception as err:
-            logger.debug(traceback.format_exc())
-            logger.error(f"Error trying fetch cost analysis data -- {err}")
-
-    def _post_hook_fn(self):
-        """
-        Method to format the structure of data
-        """
-        flattened_results = []
-        for entity_type, entity_item in self._refined_data.items():
-            for service_type, service_item in entity_item.items():
-                for service_name, service_data in service_item.items():
-                    try:
-                        aggregated_data = AggregatedCostAnalysisReportData(
-                            entityType=str(entity_type),
-                            serviceType=str(service_type),
-                            serviceName=str(service_name),
-                            **service_data,
-                        )
-                        aggregated_data.unusedDataAssets.totalCount = (
-                            aggregated_data.unusedDataAssets.count.threeDays
-                        )
-                        aggregated_data.unusedDataAssets.totalSize = (
-                            aggregated_data.unusedDataAssets.size.threeDays
-                        )
-                        aggregated_data.frequentlyUsedDataAssets.totalCount = (
-                            aggregated_data.frequentlyUsedDataAssets.count.threeDays
-                        )
-                        aggregated_data.frequentlyUsedDataAssets.totalSize = (
-                            aggregated_data.frequentlyUsedDataAssets.size.threeDays
-                        )
-                        flattened_results.append(aggregated_data)
-                    except Exception as err:
-                        logger.debug(traceback.format_exc())
-                        logger.error(f"Unable to yield report data -- {err}")
-
-        self._refined_data = flattened_results
-
-    @staticmethod
-    def _get_data_assets_dict(life_cycle: LifeCycle, size: Optional[float], data: dict):
-        """
-        Helper method to calculate number of data assets within time period
-        """
-        try:
-            if not life_cycle or not life_cycle.accessed:
-                return
-
-            # Iterate over the different time periods and update the data
-            for days, key in DAYS:
-                days_before_timestamp = get_end_of_day_timestamp_mill(days=days)
-                if (
-                    life_cycle.accessed
-                    and life_cycle.accessed.timestamp.__root__ <= days_before_timestamp
-                ):
-                    data[UNUSED_DATA_ASSETS][COUNT][key] += 1
-                    data[UNUSED_DATA_ASSETS][SIZE][key] += size or 0
-                else:
-                    data[FREQUENTLY_USED_DATA_ASSETS][COUNT][key] += 1
-                    data[FREQUENTLY_USED_DATA_ASSETS][SIZE][key] += size or 0
-
-        except Exception as err:
-            logger.debug(traceback.format_exc())
-            logger.error(f"Error calculating data -- {err}")
-
-    def get_status(self):
-        return self.processor_status

metadata/data_insight/processor/reports/data_processor.py

@@ -1,72 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Processor class used to compute refined report data
-"""
-
-from __future__ import annotations
-
-import abc
-from datetime import datetime, timezone
-from typing import Callable, Iterable, Optional
-
-from metadata.generated.schema.analytics.reportData import ReportData
-from metadata.ingestion.api.status import Status
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
-
-
-class DataProcessor(abc.ABC):
-    """_summary_
-
-    Attributes:
-        _data_processor_type: used to instantiate the correct class object
-        subclasses: dictionary mapping _data_processor_type to object
-    """
-
-    _data_processor_type: Optional[str] = None
-    subclasses = {}
-
-    def __init_subclass__(cls, *args, **kwargs) -> None:
-        """Hook to map subclass objects to data processor type"""
-        super().__init_subclass__(*args, **kwargs)
-        cls.subclasses[cls._data_processor_type] = cls
-
-    def __init__(self, metadata: OpenMetadata):
-        self.metadata = metadata
-        self.timestamp = datetime.now(timezone.utc).timestamp() * 1000
-        self.processor_status = Status()
-        self._refined_data = {}
-        self.post_hook: Optional[Callable] = None
-        self.pre_hook: Optional[Callable] = None
-        self.clean_up_cache: bool = False
-
-    @classmethod
-    def create(cls, _data_processor_type, metadata: OpenMetadata):
-        if _data_processor_type not in cls.subclasses:
-            raise NotImplementedError
-        return cls.subclasses[_data_processor_type](metadata)
-
-    @property
-    def refined_data(self) -> Iterable:
-        return self._refined_data
-
-    @abc.abstractmethod
-    def refine(self, entity) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def yield_refined_data(self) -> Iterable[ReportData]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def get_status(self) -> Status:
-        raise NotImplementedError