openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

metadata/ingestion/ometa/mixins/glossary_mixin.py

@@ -1,501 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Mixin class containing Glossaries specific methods
-
-To be used be OpenMetadata
-"""
-import json
-import traceback
-from typing import Optional, Type, TypeVar, Union
-
-from pydantic import BaseModel
-
-from metadata.generated.schema.entity.data.glossary import Glossary
-from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
-from metadata.generated.schema.type import basic
-from metadata.ingestion.ometa.mixins.patch_mixin_utils import (
-    OMetaPatchMixinBase,
-    PatchField,
-    PatchOperation,
-    PatchPath,
-    PatchValue,
-)
-from metadata.ingestion.ometa.utils import model_str
-from metadata.utils.deprecation import deprecated
-from metadata.utils.logger import ometa_logger
-
-logger = ometa_logger()
-
-T = TypeVar("T", bound=BaseModel)
-
-
-class GlossaryMixin(OMetaPatchMixinBase):
-    """
-    OpenMetadata API methods related to Glossaries.
-
-    To be inherited by OpenMetadata
-    """
-
-    @deprecated(message="Use metadata.create_or_update instead", release="1.3")
-    def create_glossary(self, glossaries_body):
-        """Method to create new Glossary
-        Args:
-            glossaries_body (Glossary): body of the request
-        """
-        resp = self.client.put(
-            path=self.get_suffix(Glossary), data=glossaries_body.json()
-        )
-        logger.info(f"Created a Glossary: {resp}")
-
-    @deprecated(message="Use metadata.create_or_update instead", release="1.3")
-    def create_glossary_term(self, glossary_term_body):
-        """Method to create new Glossary Term
-        Args:
-            glossary_term_body (Glossary): body of the request
-        """
-        resp = self.client.put(
-            path=self.get_suffix(GlossaryTerm), data=glossary_term_body.json()
-        )
-        logger.info(f"Created a Glossary Term: {resp}")
-
-    @deprecated(
-        message="Use metadata.patch instead as the new standard method that will create the jsonpatch dynamically",
-        release="1.3",
-    )
-    def patch_glossary_term_parent(
-        self,
-        entity_id: Union[str, basic.Uuid],
-        parent_fqn: Optional[str] = None,
-    ) -> Optional[GlossaryTerm]:
-        """
-        Given an GlossaryTerm ID, JSON PATCH the parent. If parent FQN is not provided, the parent is removed.
-        Args:
-            entity_id: the ID of the GlossaryTerm to be patched
-            parent_fqn: the fully qualified name of the new parent GlossaryTerm. If None the parent is removed.
-        Return:
-            the updated GlossaryTerm if all was valid, otherwise None
-        """
-        instance: GlossaryTerm = self._fetch_entity_if_exists(
-            entity=GlossaryTerm, entity_id=entity_id
-        )
-        if not instance:
-            return None
-
-        if parent_fqn is not None:
-            try:
-                res = self.client.patch(
-                    path=f"{self.get_suffix(GlossaryTerm)}/{model_str(entity_id)}",
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.ADD
-                                if instance.parent is None
-                                else PatchOperation.REPLACE,
-                                PatchField.PATH: PatchPath.PARENT,
-                                PatchField.VALUE: {
-                                    PatchValue.TYPE: self.get_suffix(GlossaryTerm),
-                                    PatchValue.FQN: model_str(parent_fqn),
-                                },
-                            }
-                        ]
-                    ),
-                )
-                return GlossaryTerm(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH parent for GlossaryTerm [{model_str(entity_id)}]: {exc}"
-                )
-
-        else:
-            try:
-                res = self.client.patch(
-                    path=f"{self.get_suffix(GlossaryTerm)}/{model_str(entity_id)}",
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.REMOVE,
-                                PatchField.PATH: PatchPath.PARENT,
-                            }
-                        ]
-                    ),
-                )
-                return GlossaryTerm(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH parent for GlossaryTerm [{entity_id}]: {exc}"
-                )
-
-        return None
-
-    @deprecated(
-        message="Use metadata.patch instead as the new standard method that will create the jsonpatch dynamically",
-        release="1.3",
-    )
-    def patch_glossary_term_related_terms(
-        self,
-        entity_id: Union[str, basic.Uuid],
-        related_term_id: Optional[Union[str, basic.Uuid]] = None,
-    ) -> Optional[GlossaryTerm]:
-        """
-        Given an GlossaryTerm ID and a related term, JSON PATCH the related terms. If related term ID is not provided,
-        the last term is removed.
-        Args:
-            entity_id: the ID of the GlossaryTerm to be patched
-            related_term_id: the ID of the related term to be added. If the related term ID is None, the last term is
-                removed.
-        Return:
-            the updated GlossaryTerm
-        """
-        instance: GlossaryTerm = self._fetch_entity_if_exists(
-            entity=GlossaryTerm, entity_id=entity_id, fields=["relatedTerms"]
-        )
-        if not instance:
-            return None
-
-        term_index: int = (
-            len(instance.relatedTerms.__root__) - 1 if instance.relatedTerms else 0
-        )
-
-        if related_term_id is None:
-            try:
-                res = self.client.patch(
-                    path=f"{self.get_suffix(GlossaryTerm)}/{model_str(entity_id)}",
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.REMOVE,
-                                PatchField.PATH: PatchPath.RELATED_TERMS.format(
-                                    index=term_index
-                                ),
-                            }
-                        ]
-                    ),
-                )
-                return GlossaryTerm(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH parent for GlossaryTerm [{entity_id}]: {exc}"
-                )
-
-        else:
-            related_term: GlossaryTerm = self.get_by_id(
-                entity=GlossaryTerm, entity_id=related_term_id, fields=["*"]
-            )
-            if related_term is None:
-                logger.error(
-                    f"Unable to PATCH GlossaryTerm [{entity_id}], unknown related term id [{related_term_id}]."
-                )
-                return None
-
-            try:
-                res = self.client.patch(
-                    path=f"{self.get_suffix(GlossaryTerm)}/{model_str(entity_id)}",
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.ADD,
-                                PatchField.PATH: PatchPath.RELATED_TERMS.format(
-                                    index=term_index + 1
-                                ),
-                                PatchField.VALUE: {
-                                    PatchValue.DISPLAY_NAME: related_term.displayName,
-                                    PatchValue.ID: model_str(related_term_id),
-                                    PatchValue.NAME: model_str(related_term.name),
-                                    PatchValue.TYPE: PatchValue.GLOSSARY_TERM,
-                                },
-                            }
-                        ]
-                    ),
-                )
-                return GlossaryTerm(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH parent for GlossaryTerm [{entity_id}]: {exc}"
-                )
-
-        return None
-
-    @deprecated(
-        message="Use metadata.patch instead as the new standard method that will create the jsonpatch dynamically",
-        release="1.3",
-    )
-    def patch_reviewers(
-        self,
-        entity: Union[Type[Glossary], Type[GlossaryTerm]],
-        entity_id: Union[str, basic.Uuid],
-        reviewer_id: Optional[Union[str, basic.Uuid]] = None,
-    ) -> Optional[T]:
-        """
-        Update the reviewers of a Glossary or GlossaryTerm. If the reviewer_id is included, the reviewer is added; if
-        not, the last reviewer is removed.
-        Args
-            entity (T): Entity Type - Glossary or GlossaryTerm
-            entity_id: ID of the entity to be updated
-            reviewer_id: ID of the user to added as a reviewer or None to remove the last reviewer
-        Return
-            The updated entity
-        """
-        instance: Union[Glossary, GlossaryTerm] = self._fetch_entity_if_exists(
-            entity=entity, entity_id=entity_id, fields=["reviewers"]
-        )
-        if not instance:
-            return None
-
-        index: int = (
-            len(instance.reviewers)
-            if entity == Glossary
-            else len(instance.reviewers.__root__)
-        )
-
-        if reviewer_id is not None:
-            try:
-                res = self.client.patch(
-                    path=f"{self.get_suffix(entity)}/{model_str(entity_id)}",
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.ADD,
-                                PatchField.PATH: PatchPath.REVIEWERS.format(
-                                    index=index
-                                ),
-                                PatchField.VALUE: {
-                                    PatchValue.ID: model_str(reviewer_id),
-                                    PatchValue.TYPE: PatchValue.USER,
-                                },
-                            }
-                        ]
-                    ),
-                )
-                return entity(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH reviewers for {entity.__class__.__name__} [{entity_id}]: {exc}"
-                )
-
-        else:
-            if index == 0:
-                logger.debug(
-                    f"Unable to remove reviewer from {entity.__class__.__name__} [{entity_id}]. No current reviewers."
-                )
-                return None
-
-            try:
-                res = self.client.patch(
-                    path=f"{self.get_suffix(entity)}/{model_str(entity_id)}",
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.REMOVE,
-                                PatchField.PATH: PatchPath.REVIEWERS.format(
-                                    index=index - 1
-                                ),
-                            }
-                        ]
-                    ),
-                )
-                return entity(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH reviewers for {entity.__class__.__name__} [{entity_id}]: {exc}"
-                )
-
-        return None
-
-    @deprecated(
-        message="Use metadata.patch instead as the new standard method that will create the jsonpatch dynamically",
-        release="1.3",
-    )
-    def patch_glossary_term_synonyms(
-        self,
-        entity_id: Union[str, basic.Uuid],
-        synonym: Optional[str] = None,
-    ) -> Optional[GlossaryTerm]:
-        """
-        Update the synonyms of a Glossary Term via PATCH
-        Params
-            entity_id: entity ID of the Glossary Term
-            synonym: if provided, the synonym is added. If None, the last synonym is removed.
-        Return
-            the updated entity
-        """
-        instance: GlossaryTerm = self._fetch_entity_if_exists(
-            entity=GlossaryTerm, entity_id=entity_id
-        )
-        if not instance:
-            return None
-
-        index: int = len(instance.synonyms)
-
-        if synonym is not None:
-            try:
-                res = self.client.patch(
-                    path=PatchPath.GLOSSARY_TERMS.format(
-                        entity_id=model_str(entity_id)
-                    ),
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.ADD,
-                                PatchField.PATH: PatchPath.SYNONYMS.format(index=index),
-                                PatchField.VALUE: synonym,
-                            }
-                        ]
-                    ),
-                )
-                return GlossaryTerm(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH synonyms for GlossaryTerm [{entity_id}]: {exc}"
-                )
-
-        else:
-            if index == 0:
-                logger.debug(
-                    f"Unable to remove synonyms from GlossaryTerm [{entity_id}]. No current synonyms."
-                )
-                return None
-
-            try:
-                res = self.client.patch(
-                    path=PatchPath.GLOSSARY_TERMS.format(
-                        entity_id=model_str(entity_id)
-                    ),
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.REMOVE,
-                                PatchField.PATH: PatchPath.SYNONYMS.format(
-                                    index=index - 1
-                                ),
-                            }
-                        ]
-                    ),
-                )
-                return GlossaryTerm(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH synonyms for GlossaryTerm [{entity_id}]: {exc}"
-                )
-
-        return None
-
-    @deprecated(
-        message="Use metadata.patch instead as the new standard method that will create the jsonpatch dynamically",
-        release="1.3",
-    )
-    def patch_glossary_term_references(
-        self,
-        entity_id: Union[str, basic.Uuid],
-        reference_name: Optional[str] = None,
-        reference_endpoint: Optional[str] = None,
-    ) -> Optional[GlossaryTerm]:
-        """
-        Update the references of a GlossaryTerm. If reference_name and reference_endpoint are None, the last reference
-        is removed.
-        Params
-            entity_id: ID of the GlossaryTerm
-            reference_name: the name of the reference
-            reference_endpoint: the valid URI endpoint of the reference
-        Return
-            updated GlossaryTerm
-        """
-        instance: GlossaryTerm = self._fetch_entity_if_exists(
-            entity=GlossaryTerm, entity_id=entity_id
-        )
-        if not instance:
-            return None
-
-        index: int = len(instance.references)
-
-        if bool(reference_name) ^ bool(reference_endpoint):
-            logger.debug(
-                f"Unable to PATCH references from GlossaryTerm [{entity_id}]: reference_name and references."
-            )
-            return None
-
-        if reference_name is not None:
-            try:
-                res = self.client.patch(
-                    path=PatchPath.GLOSSARY_TERMS.format(
-                        entity_id=model_str(entity_id)
-                    ),
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.ADD,
-                                PatchField.PATH: PatchPath.REFERENCES.format(
-                                    index=index
-                                ),
-                                PatchField.VALUE: {
-                                    PatchValue.ENDPOINT: reference_endpoint,
-                                    PatchValue.NAME: reference_name,
-                                },
-                            }
-                        ]
-                    ),
-                )
-                return GlossaryTerm(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH references for GlossaryTerm [{entity_id}]: {exc}"
-                )
-
-        else:
-            if index == 0:
-                logger.debug(
-                    f"Unable to remove reference from GlossaryTerm [{entity_id}]. No current references."
-                )
-                return None
-
-            try:
-                res = self.client.patch(
-                    path=PatchPath.GLOSSARY_TERMS.format(
-                        entity_id=model_str(entity_id)
-                    ),
-                    data=json.dumps(
-                        [
-                            {
-                                PatchField.OPERATION: PatchOperation.REMOVE,
-                                PatchField.PATH: PatchPath.REFERENCES.format(
-                                    index=index - 1
-                                ),
-                            }
-                        ]
-                    ),
-                )
-                return GlossaryTerm(**res)
-
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.error(
-                    f"Error trying to PATCH references for GlossaryTerm [{entity_id}]: {exc}"
-                )
-
-        return None

metadata/ingestion/ometa/provider_registry.py

@@ -1,144 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Register auth provider init functions here
-"""
-# Complains about same imports
-# pylint: disable=duplicate-code
-from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
-    AuthProvider,
-    OpenMetadataConnection,
-)
-from metadata.generated.schema.security.client.auth0SSOClientConfig import (
-    Auth0SSOClientConfig,
-)
-from metadata.generated.schema.security.client.azureSSOClientConfig import (
-    AzureSSOClientConfig,
-)
-from metadata.generated.schema.security.client.customOidcSSOClientConfig import (
-    CustomOIDCSSOClientConfig,
-)
-from metadata.generated.schema.security.client.googleSSOClientConfig import (
-    GoogleSSOClientConfig,
-)
-from metadata.generated.schema.security.client.oktaSSOClientConfig import (
-    OktaSSOClientConfig,
-)
-from metadata.ingestion.ometa.auth_provider import (
-    Auth0AuthenticationProvider,
-    AuthenticationProvider,
-    AzureAuthenticationProvider,
-    CustomOIDCAuthenticationProvider,
-    GoogleAuthenticationProvider,
-    NoOpAuthenticationProvider,
-    OktaAuthenticationProvider,
-    OpenMetadataAuthenticationProvider,
-    OpenMetadataJWTClientConfig,
-)
-from metadata.utils.dispatch import enum_register
-from metadata.utils.logger import ometa_logger
-
-logger = ometa_logger()
-
-
-def warn_auth_deprecation(auth_provider: AuthProvider) -> None:
-    logger.warning(
-        "Please, configure the ingestion-bot with the 'OpenMetadata JWT' configuration.\n"
-        f"The '{auth_provider.value}' configuration is deprecated and will be removed in future releases.\n"
-        f"Visit https://docs.open-metadata.org/deployment/security/enable-jwt-tokens to learn how to "
-        f"configure the 'OpenMetadata JWT'."
-    )
-
-
-def warn_not_supported() -> None:
-    logger.warning("Note that we don't support this implementation on the client.")
-
-
-class InvalidAuthProviderException(Exception):
-    """
-    Raised when we cannot find a valid auth provider
-    in the registry
-    """
-
-
-auth_provider_registry = enum_register()
-
-
-@auth_provider_registry.add(AuthProvider.no_auth.value)
-def no_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    return NoOpAuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.basic.value)
-def basic_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    warn_auth_deprecation(config.authProvider)
-    warn_not_supported()
-    return NoOpAuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.saml.value)
-def saml_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    warn_auth_deprecation(config.authProvider)
-    warn_not_supported()
-    return NoOpAuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.ldap.value)
-def ldap_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    warn_auth_deprecation(config.authProvider)
-    warn_not_supported()
-    return NoOpAuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.google.value)
-def google_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    warn_auth_deprecation(config.authProvider)
-    return GoogleAuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.okta.value)
-def okta_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    warn_auth_deprecation(config.authProvider)
-    return OktaAuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.auth0.value)
-def auth0_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    warn_auth_deprecation(config.authProvider)
-    return Auth0AuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.azure.value)
-def azure_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    warn_auth_deprecation(config.authProvider)
-    return AzureAuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.custom_oidc.value)
-def custom_oidc_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    warn_auth_deprecation(config.authProvider)
-    return CustomOIDCAuthenticationProvider.create(config)
-
-
-@auth_provider_registry.add(AuthProvider.openmetadata.value)
-def om_auth_init(config: OpenMetadataConnection) -> AuthenticationProvider:
-    return OpenMetadataAuthenticationProvider.create(config)
-
-
-PROVIDER_CLASS_MAP = {
-    AuthProvider.no_auth.value: None,
-    AuthProvider.google.value: GoogleSSOClientConfig,
-    AuthProvider.azure.value: AzureSSOClientConfig,
-    AuthProvider.okta.value: OktaSSOClientConfig,
-    AuthProvider.auth0.value: Auth0SSOClientConfig,
-    AuthProvider.custom_oidc.value: CustomOIDCSSOClientConfig,
-    AuthProvider.openmetadata.value: OpenMetadataJWTClientConfig,
-}

metadata/ingestion/source/database/databricks/legacy/lineage.py

@@ -1,51 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-Databricks lineage module
-"""
-import traceback
-from datetime import datetime
-from typing import Iterator
-
-from metadata.generated.schema.type.tableQuery import TableQuery
-from metadata.ingestion.source.database.databricks.query_parser import (
-    DatabricksQueryParserSource,
-)
-from metadata.ingestion.source.database.lineage_source import LineageSource
-from metadata.utils.logger import ingestion_logger
-
-logger = ingestion_logger()
-
-
-class DatabricksLineageLegacySource(DatabricksQueryParserSource, LineageSource):
-    """
-    Databricks Lineage Legacy Source
-    """
-
-    def yield_table_query(self) -> Iterator[TableQuery]:
-        data = self.client.list_query_history(
-            start_date=self.start,
-            end_date=self.end,
-        )
-        for row in data:
-            try:
-                if self.client.is_query_valid(row):
-                    yield TableQuery(
-                        query=row.get("query_text"),
-                        userName=row.get("user_name"),
-                        startTime=row.get("query_start_time_ms"),
-                        endTime=row.get("execution_end_time_ms"),
-                        analysisDate=datetime.now(),
-                        serviceName=self.config.serviceName,
-                    )
-            except Exception as exc:
-                logger.debug(traceback.format_exc())
-                logger.warning(f"Error processing query_dict {row}: {exc}")