openmetadata-ingestion 1.6.0.0rc1__py3-none-any.whl → 1.6.0.0rc2__py3-none-any.whl

metadata/data_quality/builders/sqa_validator_builder.py

@@ -1,25 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Builder defining the structure of builders for validators for SQA sources
-"""
-
-
-from metadata.data_quality.builders.i_validator_builder import IValidatorBuilder
-
-
-class SQAValidatorBuilder(IValidatorBuilder):
-    """Builder for SQA validators"""
-
-    def _get_source_type(self) -> str:
-        """Return the test case"""
-        return "sqlalchemy"

metadata/data_quality/interface/test_suite_interface_factory.py

@@ -1,158 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#  pylint: disable=import-outside-toplevel
-"""
-Interface factory
-"""
-import traceback
-from logging import Logger
-from typing import Callable, Dict, Type
-
-from metadata.data_quality.interface.test_suite_interface import TestSuiteInterface
-from metadata.generated.schema.entity.data.table import Table
-from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
-    DatabricksConnection,
-)
-from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
-    DatalakeConnection,
-)
-from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
-    SnowflakeConnection,
-)
-from metadata.generated.schema.entity.services.connections.database.unityCatalogConnection import (
-    UnityCatalogConnection,
-)
-from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils.logger import test_suite_logger
-
-logger: Logger = test_suite_logger()
-
-
-class TestSuiteInterfaceFactory:
-    """Factory class for the data quality interface"""
-
-    def __init__(self):
-        """Initialize the interface factory"""
-        self._interface_type: Dict[str, Callable[[], Type[TestSuiteInterface]]] = {
-            "base": self.sqa,
-        }
-
-    def register(self, interface_type: str, fn: Callable[[], Type[TestSuiteInterface]]):
-        """Register the interface
-
-        Args:
-            interface_type (str): type of the interface
-            interface (callable): a class that implements the TestSuiteInterface
-        """
-        self._interface_type[interface_type] = fn
-
-    def register_many(self, interface_dict):
-        """
-        Registers multiple profiler interfaces at once.
-
-        Args:
-            interface_dict: A dictionary mapping connection class names (strings) to their
-            corresponding profiler interface classes.
-        """
-        for interface_type, interface_fn in interface_dict.items():
-            self.register(interface_type, interface_fn)
-
-    def create(
-        self,
-        service_connection_config: DatabaseConnection,
-        ometa_client: OpenMetadata,
-        table_entity: Table,
-        *args,
-        **kwargs,
-    ) -> TestSuiteInterface:
-        """Create the interface
-
-        Args:
-            service_connection_config (DatabaseService): a database service object
-
-        Raises:
-            AttributeError: if no connection is found in the database service object
-
-        Returns:
-            TestSuiteInterface:
-        """
-        try:
-            connection_type = service_connection_config.__class__.__name__
-        except AttributeError as err:
-            logger.debug(traceback.format_exc())
-            raise AttributeError(f"Could not instantiate interface class: {err}")
-        interface_fn = self._interface_type.get(connection_type)
-
-        if not interface_fn:
-            interface_fn = self._interface_type["base"]
-
-        interface_class = interface_fn()
-        return interface_class(
-            service_connection_config, ometa_client, table_entity, *args, **kwargs
-        )
-
-    @staticmethod
-    def sqa() -> Type[TestSuiteInterface]:
-        """Lazy load the SQATestSuiteInterface"""
-        from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
-            SQATestSuiteInterface,
-        )
-
-        return SQATestSuiteInterface
-
-    @staticmethod
-    def pandas() -> Type[TestSuiteInterface]:
-        """Lazy load the PandasTestSuiteInterface"""
-        from metadata.data_quality.interface.pandas.pandas_test_suite_interface import (
-            PandasTestSuiteInterface,
-        )
-
-        return PandasTestSuiteInterface
-
-    @staticmethod
-    def snowflake() -> Type[TestSuiteInterface]:
-        """Lazy load the SnowflakeTestSuiteInterface"""
-        from metadata.data_quality.interface.sqlalchemy.snowflake.test_suite_interface import (
-            SnowflakeTestSuiteInterface,
-        )
-
-        return SnowflakeTestSuiteInterface
-
-    @staticmethod
-    def unity_catalog() -> Type[TestSuiteInterface]:
-        """Lazy load the UnityCatalogTestSuiteInterface"""
-        from metadata.data_quality.interface.sqlalchemy.unity_catalog.test_suite_interface import (
-            UnityCatalogTestSuiteInterface,
-        )
-
-        return UnityCatalogTestSuiteInterface
-
-    @staticmethod
-    def databricks() -> Type[TestSuiteInterface]:
-        """Lazy load the DatabricksTestSuiteInterface"""
-        from metadata.data_quality.interface.sqlalchemy.databricks.test_suite_interface import (
-            DatabricksTestSuiteInterface,
-        )
-
-        return DatabricksTestSuiteInterface
-
-
-test_suite_interface = {
-    DatabaseConnection.__name__: TestSuiteInterfaceFactory.sqa,
-    DatalakeConnection.__name__: TestSuiteInterfaceFactory.pandas,
-    SnowflakeConnection.__name__: TestSuiteInterfaceFactory.snowflake,
-    UnityCatalogConnection.__name__: TestSuiteInterfaceFactory.unity_catalog,
-    DatabricksConnection.__name__: TestSuiteInterfaceFactory.databricks,
-}
-
-test_suite_interface_factory = TestSuiteInterfaceFactory()
-test_suite_interface_factory.register_many(test_suite_interface)

metadata/data_quality/runner/test_suite_source_factory.py

@@ -1,38 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Factory class for creating test suite source objects
-"""
-
-from metadata.data_quality.runner.base_test_suite_source import BaseTestSuiteRunner
-
-
-class TestSuiteRunnerFactory:
-    """Creational factory for test suite source objects"""
-
-    def __init__(self):
-        self._source_type = {"base": BaseTestSuiteRunner}
-
-    def register_source(self, source_type: str, source_class):
-        """Register a new source type"""
-        self._source_type[source_type] = source_class
-
-    def create(self, source_type: str, *args, **kwargs) -> BaseTestSuiteRunner:
-        """Create source object based on source type"""
-        source_class = self._source_type.get(source_type)
-        if not source_class:
-            source_class = self._source_type["base"]
-            return source_class(*args, **kwargs)
-        return source_class(*args, **kwargs)
-
-
-test_suite_source_factory = TestSuiteRunnerFactory()

metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py

@@ -1,54 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  entity/services/connections/dashboard/mstrConnection.json
-#   timestamp: 2024-11-20T14:26:20+00:00
-
-from __future__ import annotations
-
-from enum import Enum
-from typing import Optional
-
-from pydantic import AnyUrl, ConfigDict, Field
-from typing_extensions import Annotated
-
-from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
-
-from .. import connectionBasicType
-
-
-class MstrType(Enum):
-    Mstr = 'Mstr'
-
-
-class MstrConnection(BaseModel):
-    model_config = ConfigDict(
-        extra='forbid',
-    )
-    type: Annotated[
-        Optional[MstrType],
-        Field(MstrType.Mstr, description='Service Type', title='Service Type'),
-    ]
-    username: Annotated[
-        str,
-        Field(
-            description='Username to connect to MSTR. This user should have privileges to read all the metadata in MSTR.',
-            title='Username',
-        ),
-    ]
-    password: Annotated[
-        Optional[CustomSecretStr],
-        Field(None, description='Password to connect to MSTR.', title='Password'),
-    ]
-    hostPort: Annotated[
-        AnyUrl,
-        Field(
-            description='Host and Port of the Metabase instance.', title='Host and Port'
-        ),
-    ]
-    projectName: Annotated[
-        Optional[str],
-        Field(None, description='MSTR Project Name', title='Project Name'),
-    ]
-    supportsMetadataExtraction: Annotated[
-        Optional[connectionBasicType.SupportsMetadataExtraction],
-        Field(None, title='Supports Metadata Extraction'),
-    ]

metadata/ingestion/source/dashboard/mstr/client.py

@@ -1,209 +0,0 @@
-#  Copyright 2023 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-REST Auth & Client for Mstr
-"""
-import traceback
-from typing import List, Optional
-
-import requests
-from mstr.requests import MSTRRESTSession
-
-from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
-    MstrConnection,
-)
-from metadata.ingestion.connections.test_connections import SourceConnectionException
-from metadata.ingestion.source.dashboard.mstr.models import (
-    MstrDashboard,
-    MstrDashboardDetails,
-    MstrDashboardList,
-    MstrProject,
-    MstrProjectList,
-    MstrSearchResult,
-    MstrSearchResultList,
-)
-from metadata.utils.helpers import clean_uri
-from metadata.utils.logger import ingestion_logger
-
-logger = ingestion_logger()
-
-API_VERSION = "MicroStrategyLibrary/api"
-
-
-class MSTRClient:
-    """
-    Client Handling API communication with Metabase
-    """
-
-    def _get_base_url(self, path=None):
-        if not path:
-            return f"{clean_uri(self.config.hostPort)}/{API_VERSION}/"
-        return f"{clean_uri(self.config.hostPort)}/{API_VERSION}/{path}"
-
-    def _get_mstr_session(self) -> MSTRRESTSession:
-        try:
-            session = MSTRRESTSession(base_url=self._get_base_url())
-            session.login(
-                username=self.config.username,
-                password=self.config.password.get_secret_value(),
-            )
-            return session
-
-        except KeyError as exe:
-            msg = "Failed to fetch mstr session, please validate credentials"
-            raise SourceConnectionException(msg) from exe
-
-        except Exception as exc:
-            msg = f"Unknown error in connection: {exc}."
-            raise SourceConnectionException(msg) from exc
-
-    def __init__(
-        self,
-        config: MstrConnection,
-    ):
-        self.config = config
-        self.session = self._get_mstr_session()
-
-    def is_project_name(self) -> bool:
-        return bool(self.config.projectName)
-
-    def get_projects_list(self) -> List[MstrProject]:
-        """
-        Get List of all projects
-        """
-        try:
-            resp_projects = self.session.get(
-                url=self._get_base_url("projects"), params={"include_auth": True}
-            )
-
-            if not resp_projects.ok:
-                raise requests.ConnectionError()
-
-            project_list = MstrProjectList(projects=resp_projects.json())
-            return project_list.projects
-
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Failed to fetch the project list due to [{exc}]")
-
-        return []
-
-    def get_project_by_name(self) -> Optional[MstrProject]:
-        """
-        Get Project By Name
-        """
-        try:
-            resp_projects = self.session.get(
-                url=self._get_base_url(f"projects/{self.config.projectName}"),
-                params={"include_auth": True},
-            )
-
-            if not resp_projects.ok:
-                raise requests.ConnectionError()
-
-            project = MstrProject(**resp_projects.json())
-            return project
-
-        except Exception:
-            logger.debug(traceback.format_exc())
-            logger.warning("Failed to fetch the project list")
-
-        return None
-
-    def get_search_results_list(
-        self, project_id, object_type
-    ) -> List[MstrSearchResult]:
-        """
-        Get Search Results
-        """
-        try:
-            resp_results = self.session.get(
-                url=self._get_base_url("searches/results"),
-                params={
-                    "include_auth": True,
-                    "project_id": project_id,
-                    "type": object_type,
-                    "getAncestors": False,
-                    "offset": 0,
-                    "limit": -1,
-                    "certifiedStatus": "ALL",
-                    "isCrossCluster": False,
-                    "result.hidden": False,
-                },
-            )
-
-            if not resp_results.ok:
-                raise requests.ConnectionError()
-
-            results = []
-            for resp_result in resp_results.json()["result"]:
-                results.append(resp_result)
-
-            results_list = MstrSearchResultList(results=results)
-            return results_list.results
-
-        except Exception:
-            logger.debug(traceback.format_exc())
-            logger.warning("Failed to fetch the Search Result list")
-
-        return []
-
-    def get_dashboards_list(self, project_id, project_name) -> List[MstrDashboard]:
-        """
-        Get Dashboard
-        """
-        try:
-            results = self.get_search_results_list(
-                project_id=project_id, object_type=55
-            )
-
-            dashboards = []
-            for result in results:
-                dashboards.append(
-                    MstrDashboard(projectName=project_name, **result.model_dump())
-                )
-
-            dashboards_list = MstrDashboardList(dashboards=dashboards)
-            return dashboards_list.dashboards
-
-        except Exception:
-            logger.debug(traceback.format_exc())
-            logger.warning("Failed to fetch the dashboard list")
-
-        return []
-
-    def get_dashboard_details(
-        self, project_id, project_name, dashboard_id
-    ) -> Optional[MstrDashboardDetails]:
-        """
-        Get Dashboard Details
-        """
-        try:
-            resp_dashboard = self.session.get(
-                url=self._get_base_url(f"v2/dossiers/{dashboard_id}/definition"),
-                params={
-                    "include_auth": True,
-                },
-                headers={"X-MSTR-ProjectID": project_id},
-            )
-
-            if not resp_dashboard.ok:
-                raise requests.ConnectionError()
-
-            return MstrDashboardDetails(
-                projectId=project_id, projectName=project_name, **resp_dashboard.json()
-            )
-
-        except Exception:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")
-
-        return None

metadata/ingestion/source/dashboard/mstr/service_spec.py

@@ -1,4 +0,0 @@
-from metadata.ingestion.source.dashboard.mstr.metadata import MstrSource
-from metadata.utils.service_spec import BaseSpec
-
-ServiceSpec = BaseSpec(metadata_source_class=MstrSource)

metadata/profiler/processor/sampler/pandas/sampler.py

@@ -1,170 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Helper module to handle data sampling
-for the profiler
-"""
-import math
-import random
-from typing import List, Optional, cast
-
-from metadata.data_quality.validations.table.pandas.tableRowInsertedCountToBeBetween import (
-    TableRowInsertedCountToBeBetweenValidator,
-)
-from metadata.generated.schema.entity.data.table import (
-    PartitionIntervalTypes,
-    PartitionProfilerConfig,
-    ProfileSampleType,
-    TableData,
-)
-from metadata.profiler.processor.sampler.sampler_interface import SamplerInterface
-from metadata.utils.sqa_like_column import SQALikeColumn
-
-
-class DatalakeSampler(SamplerInterface):
-    """
-    Generates a sample of the data to not
-    run the query in the whole table.
-    """
-
-    def _partitioned_table(self):
-        """Get partitioned table"""
-        self._partition_details = cast(PartitionProfilerConfig, self._partition_details)
-        partition_field = self._partition_details.partitionColumnName
-        if (
-            self._partition_details.partitionIntervalType
-            == PartitionIntervalTypes.COLUMN_VALUE
-        ):
-            return [
-                df[df[partition_field].isin(self._partition_details.partitionValues)]
-                for df in self.table
-            ]
-        if (
-            self._partition_details.partitionIntervalType
-            == PartitionIntervalTypes.INTEGER_RANGE
-        ):
-            return [
-                df[
-                    df[partition_field].between(
-                        self._partition_details.partitionIntegerRangeStart,
-                        self._partition_details.partitionIntegerRangeEnd,
-                    )
-                ]
-                for df in self.table
-            ]
-        return [
-            df[
-                df[partition_field]
-                >= TableRowInsertedCountToBeBetweenValidator._get_threshold_date(  # pylint: disable=protected-access
-                    self._partition_details.partitionIntervalUnit.value,
-                    self._partition_details.partitionInterval,
-                )
-            ]
-            for df in self.table
-        ]
-
-    def _fetch_sample_data_from_user_query(self) -> TableData:
-        """Fetch sample data from user query"""
-        cols, rows = self.get_col_row(data_frame=self._rdn_sample_from_user_query())
-        return TableData(columns=cols, rows=rows)
-
-    def _rdn_sample_from_user_query(self):
-        """Generate sample from user query"""
-        return [df.query(self._profile_sample_query) for df in self.table]
-
-    def _get_sampled_dataframe(self):
-        """
-        returns sampled ometa dataframes
-        """
-        random.shuffle(self.table)  # we'll shuffle the list of dataframes
-        # sampling data based on profiler config (if any)
-        if self.profile_sample_type == ProfileSampleType.PERCENTAGE:
-            try:
-                profile_sample = self.profile_sample / 100
-            except TypeError:
-                # if the profile sample is not a number or is None
-                # we'll set it to 100
-                profile_sample = self.profile_sample = 100
-            return [
-                df.sample(
-                    frac=profile_sample,
-                    random_state=random.randint(0, 100),
-                    replace=True,
-                )
-                for df in self.table
-            ]
-
-        # we'll distribute the sample size equally among the dataframes
-        sample_rows_per_chunk: int = math.floor(self.profile_sample / len(self.table))
-        num_rows = sum(len(df) for df in self.table)
-
-        # if we have less rows than the sample size
-        # we'll return the whole table
-        if sample_rows_per_chunk > num_rows:
-            return self.table
-        return [
-            df.sample(
-                n=sample_rows_per_chunk,
-                random_state=random.randint(0, 100),
-                replace=True,
-            )
-            for df in self.table
-        ]
-
-    def get_col_row(self, data_frame, columns: Optional[List[SQALikeColumn]] = None):
-        """
-        Fetches columns and rows from the data_frame
-        """
-        if columns:
-            cols = [col.name for col in columns]
-        else:
-            # we'll use the first dataframe to get the columns
-            cols = data_frame[0].columns.tolist()
-        rows = []
-        # Sample Data should not exceed sample limit
-        for chunk in data_frame:
-            rows.extend(self._fetch_rows(chunk[cols])[: self.sample_limit])
-            if len(rows) >= self.sample_limit:
-                break
-        return cols, rows
-
-    def random_sample(self, is_sampled: bool = False):
-        """Generate random sample from the table
-
-        Returns:
-            List[DataFrame]
-        """
-        if self._profile_sample_query:
-            return self._rdn_sample_from_user_query()
-
-        if self._partition_details:
-            self.table = self._partitioned_table()
-
-        if not self.profile_sample or is_sampled:
-            return self.table
-        return self._get_sampled_dataframe()
-
-    def _fetch_rows(self, data_frame):
-        return data_frame.dropna().values.tolist()
-
-    def fetch_sample_data(
-        self, columns: Optional[List[SQALikeColumn]] = None
-    ) -> TableData:
-        """Fetch sample data from the table
-
-        Returns:
-            TableData:
-        """
-        if self._profile_sample_query:
-            return self._fetch_sample_data_from_user_query()
-
-        cols, rows = self.get_col_row(data_frame=self.table, columns=columns)
-        return TableData(columns=cols, rows=rows)