openmetadata-ingestion 1.2.5.2__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/utils/helpers.py

@@ -31,7 +31,9 @@ from sqlparse.sql import Statement
 
 from metadata.generated.schema.entity.data.chart import ChartType
 from metadata.generated.schema.entity.data.table import Column, Table
+from metadata.generated.schema.entity.feed.suggestion import Suggestion, SuggestionType
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.type.basic import EntityLink
 from metadata.generated.schema.type.tagLabel import TagLabel
 from metadata.utils.constants import DEFAULT_DATABASE
 from metadata.utils.logger import utils_logger
@@ -123,12 +125,27 @@ def calculate_execution_time_generator(func):
     """
 
     def calculate_debug_time(*args, **kwargs):
-        start = perf_counter()
-        yield from func(*args, **kwargs)
-        end = perf_counter()
-        logger.debug(
-            f"{func.__name__} executed in { pretty_print_time_duration(end - start)}"
-        )
+        # NOTE: We are basically implementing by hand a simplified version of 'yield from'
+        # in order to be able to calculate the time difference correctly.
+        # The 'while True' loop allows us to guarantee we are iterating over all thje values
+        # from func(*args, **kwargs).
+        generator = func(*args, **kwargs)
+
+        while True:
+            start = perf_counter()
+
+            try:
+                element = next(generator)
+            except StopIteration:
+                return
+
+            end = perf_counter()
+
+            logger.debug(
+                f"{func.__name__} executed in { pretty_print_time_duration(end - start)}"
+            )
+
+            yield element
 
     return calculate_debug_time
 
@@ -227,12 +244,38 @@ def find_in_iter(element: Any, container: Iterable[Any]) -> Optional[Any]:
     return next((elem for elem in container if elem == element), None)
 
 
-def find_column_in_table(column_name: str, table: Table) -> Optional[Column]:
+def find_column_in_table(
+    column_name: str, table: Table, case_sensitive: bool = True
+) -> Optional[Column]:
     """
     If the column exists in the table, return it
     """
+
+    def equals(first: str, second: str) -> bool:
+        if case_sensitive:
+            return first == second
+        return first.lower() == second.lower()
+
+    return next(
+        (col for col in table.columns if equals(col.name.__root__, column_name)), None
+    )
+
+
+def find_suggestion(
+    suggestions: List[Suggestion],
+    suggestion_type: SuggestionType,
+    entity_link: EntityLink,
+) -> Optional[Suggestion]:
+    """Given a list of suggestions, a suggestion type and an entity link, find
+    one suggestion in the list that matches the criteria
+    """
     return next(
-        (col for col in table.columns if col.name.__root__ == column_name), None
+        (
+            sugg
+            for sugg in suggestions
+            if sugg.type == suggestion_type and sugg.entityLink == entity_link
+        ),
+        None,
     )
 
 
@@ -354,7 +397,7 @@ def format_large_string_numbers(number: Union[float, int]) -> str:
     units = ["", "K", "M", "B", "T"]
     constant_k = 1000.0
     magnitude = int(floor(log(abs(number), constant_k)))
-    return f"{number / constant_k**magnitude:.2f}{units[magnitude]}"
+    return f"{number / constant_k**magnitude:.3f}{units[magnitude]}"
 
 
 def clean_uri(uri: str) -> str:
@@ -430,6 +473,9 @@ def is_safe_sql_query(sql_query: str) -> bool:
         "SET TRANSACTION",
     }
 
+    if sql_query is None:
+        return True
+
     parsed_queries: Tuple[Statement] = sqlparse.parse(sql_query)
     for parsed_query in parsed_queries:
         validation = [

metadata/utils/importer.py

@@ -14,7 +14,7 @@ Helpers to import python classes and modules dynamically
 import importlib
 import traceback
 from enum import Enum
-from typing import Callable, Optional, Type, TypeVar
+from typing import Any, Callable, Optional, Type, TypeVar
 
 from pydantic import BaseModel
 
@@ -23,7 +23,6 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
 )
 from metadata.generated.schema.entity.services.serviceType import ServiceType
 from metadata.generated.schema.metadataIngestion.workflow import Sink as WorkflowSink
-from metadata.ingestion.api.step import Step
 from metadata.ingestion.api.steps import BulkSink, Processor, Sink, Source, Stage
 from metadata.utils.class_helper import get_service_type_from_source_type
 from metadata.utils.logger import utils_logger
@@ -86,7 +85,7 @@ def get_class_name_root(type_: str) -> str:
     )
 
 
-def import_from_module(key: str) -> Type[Step]:
+def import_from_module(key: str) -> Type[Any]:
     """
     Dynamically import an object from a module path
     """

metadata/utils/life_cycle_utils.py

@@ -27,12 +27,12 @@ QUERY_TYPES_DICT = {
 }
 
 
-select_pattern = re.compile(r"^\s*(SELECT|SHOW|DESCRIBE)", re.IGNORECASE)
-create_pattern = re.compile(r"^\s*CREATE", re.IGNORECASE)
+select_pattern = re.compile(r".*\s*(SELECT|SHOW|DESCRIBE)", re.IGNORECASE)
+create_pattern = re.compile(r".*\s*CREATE", re.IGNORECASE)
 update_pattern = re.compile(
-    r"^\s*(UPDATE|INSERT|DELETE|MERGE|TRUNCATE_TABLE|ALTER)", re.IGNORECASE
+    r".*\s*(UPDATE|INSERT|DELETE|MERGE|TRUNCATE_TABLE|ALTER)", re.IGNORECASE
 )
-drop_pattern = re.compile(r"^\s*DROP", re.IGNORECASE)
+drop_pattern = re.compile(r".*\s*DROP", re.IGNORECASE)
 
 logger = utils_logger()
 

metadata/utils/logger.py

@@ -54,6 +54,7 @@ class Loggers(Enum):
     TEST_SUITE = "TestSuite"
     DATA_INSIGHT = "DataInsight"
     QUERY_RUNNER = "QueryRunner"
+    APP = "App"
 
     @DynamicClassAttribute
     def value(self):
@@ -153,6 +154,14 @@ def great_expectations_logger():
     return logging.getLogger(Loggers.GREAT_EXPECTATIONS.value)
 
 
+def app_logger():
+    """
+    Method to get the APP logger
+    """
+
+    return logging.getLogger(Loggers.APP.value)
+
+
 def query_runner_logger():
     """
     Method to get the QUERY_RUNNER logger
@@ -245,9 +254,11 @@ def _(record: TableAndTests) -> str:
 
 
 @get_log_name.register
-def _(_: TestCaseResults) -> Optional[str]:
+def _(record: TestCaseResults) -> str:
     """We don't want to log this in the status"""
-    return None
+    return ",".join(
+        set(result.testCase.name.__root__ for result in record.test_results)
+    )
 
 
 @get_log_name.register

metadata/utils/partition.py

@@ -15,6 +15,7 @@ from typing import Optional
 
 from metadata.generated.schema.entity.data.table import (
     IntervalType,
+    PartitionIntervalUnit,
     PartitionProfilerConfig,
     Table,
 )
@@ -47,8 +48,10 @@ def get_partition_details(entity: Table) -> Optional[PartitionProfilerConfig]:
                 return PartitionProfilerConfig(
                     enablePartitioning=True,
                     partitionColumnName=entity.tablePartition.columns[0],
-                    partitionIntervalUnit=entity.tablePartition.interval,
-                    partitionInterval=30,
+                    partitionIntervalUnit=PartitionIntervalUnit.DAY
+                    if entity.tablePartition.interval != "HOUR"
+                    else entity.tablePartition.interval,
+                    partitionInterval=1,
                     partitionIntervalType=entity.tablePartition.intervalType.value,
                     partitionValues=None,
                     partitionIntegerRangeStart=None,
@@ -60,8 +63,10 @@ def get_partition_details(entity: Table) -> Optional[PartitionProfilerConfig]:
                     partitionColumnName="_PARTITIONDATE"
                     if entity.tablePartition.interval == "DAY"
                     else "_PARTITIONTIME",
-                    partitionIntervalUnit=entity.tablePartition.interval,
-                    partitionInterval=30,
+                    partitionIntervalUnit=PartitionIntervalUnit.DAY
+                    if entity.tablePartition.interval != "HOUR"
+                    else entity.tablePartition.interval,
+                    partitionInterval=1,
                     partitionIntervalType=entity.tablePartition.intervalType.value,
                     partitionValues=None,
                     partitionIntegerRangeStart=None,
@@ -72,7 +77,7 @@ def get_partition_details(entity: Table) -> Optional[PartitionProfilerConfig]:
                     enablePartitioning=True,
                     partitionColumnName=entity.tablePartition.columns[0],
                     partitionIntervalUnit=None,
-                    partitionInterval=30,
+                    partitionInterval=None,
                     partitionIntervalType=entity.tablePartition.intervalType.value,
                     partitionValues=None,
                     partitionIntegerRangeStart=1,

metadata/utils/secrets/client/loader.py

@@ -63,7 +63,6 @@ def _(provider: SecretsManagerProvider) -> Optional[AWSCredentials]:
 
 @secrets_manager_client_loader.add(SecretsManagerClientLoader.env.value)
 def _(provider: SecretsManagerProvider) -> Optional[AWSCredentials]:
-
     if provider in {
         SecretsManagerProvider.aws,
         SecretsManagerProvider.managed_aws,

metadata/utils/secrets/noop_secrets_manager.py

@@ -18,12 +18,13 @@ from metadata.generated.schema.security.secrets.secretsManagerProvider import (
 from metadata.utils.secrets.secrets_manager import SecretsManager
 
 
-class NoopSecretsManager(SecretsManager):
+class DBSecretsManager(SecretsManager):
     """
-    LocalSecretsManager is used when there is not a secrets' manager configured.
+    Will return the string as received. The client does not need to pick
+    it up from any external resource.
     """
 
-    provider: str = SecretsManagerProvider.noop.name
+    provider: str = SecretsManagerProvider.db.name
 
     def get_string_value(self, secret_id: str) -> str:
         return secret_id

metadata/utils/secrets/secrets_manager_factory.py

@@ -23,7 +23,7 @@ from metadata.generated.schema.security.secrets.secretsManagerProvider import (
 from metadata.utils.secrets.aws_secrets_manager import AWSSecretsManager
 from metadata.utils.secrets.aws_ssm_secrets_manager import AWSSSMSecretsManager
 from metadata.utils.secrets.client.loader import secrets_manager_client_loader
-from metadata.utils.secrets.noop_secrets_manager import NoopSecretsManager
+from metadata.utils.secrets.noop_secrets_manager import DBSecretsManager
 from metadata.utils.secrets.secrets_manager import SecretsManager
 from metadata.utils.singleton import Singleton
 
@@ -85,9 +85,9 @@ class SecretsManagerFactory(metaclass=Singleton):
         """
         if (
             secrets_manager_provider is None
-            or secrets_manager_provider == SecretsManagerProvider.noop
+            or secrets_manager_provider == SecretsManagerProvider.db
         ):
-            return NoopSecretsManager()
+            return DBSecretsManager()
         if secrets_manager_provider in (
             SecretsManagerProvider.aws,
             SecretsManagerProvider.managed_aws,
@@ -104,7 +104,6 @@ class SecretsManagerFactory(metaclass=Singleton):
         return self.secrets_manager
 
     def _load_secrets_manager_credentials(self) -> Optional["AWSCredentials"]:
-
         if not self.secrets_manager_loader:
             return None
 

metadata/utils/{source_hash_utils.py → source_hash.py}

@@ -23,11 +23,20 @@ SOURCE_HASH_EXCLUDE_FIELDS = {
 }
 
 
-def generate_source_hash(create_request: C, exclude_fields: Optional[Dict]) -> str:
+def generate_source_hash(
+    create_request: C, exclude_fields: Optional[Dict] = None
+) -> str:
     """
     Given a create_request model convert it to json string and generate a hash value
     """
 
+    # We always want to exclude the sourceHash when generating the fingerprint
+    exclude_fields = (
+        SOURCE_HASH_EXCLUDE_FIELDS.update(exclude_fields)
+        if exclude_fields
+        else SOURCE_HASH_EXCLUDE_FIELDS
+    )
+
     create_request_json = create_request.json(exclude=exclude_fields)
 
     json_bytes = create_request_json.encode("utf-8")

metadata/utils/sqlalchemy_utils.py

@@ -39,6 +39,27 @@ def get_table_comment_wrapper(self, connection, query, table_name, schema=None):
     return {"text": self.all_table_comments.get((table_name, schema))}
 
 
+@reflection.cache
+def get_all_table_owners(
+    self, connection, query, schema_name, **kw
+):  # pylint: disable=unused-argument
+    """
+    Method to fetch owners of all available tables
+    """
+    self.all_table_owners: Dict[Tuple[str, str], str] = {}
+    result = connection.execute(query)
+    for table in result:
+        self.all_table_owners[(table[0], table[1])] = table[2]
+
+
+def get_table_owner_wrapper(
+    self, connection, query, table_name, schema=None, **kw
+):  # pylint: disable=unused-argument
+    if not hasattr(self, "all_table_owners"):
+        self.get_all_table_owners(connection, query, schema)
+    return self.all_table_owners.get((schema, table_name), "")
+
+
 @reflection.cache
 def get_all_view_definitions(self, connection, query):
     """

metadata/utils/storage_metadata_config.py

@@ -20,6 +20,9 @@ import requests
 from metadata.generated.schema.entity.services.connections.database.datalake.azureConfig import (
     AzureConfig,
 )
+from metadata.generated.schema.entity.services.connections.database.datalake.gcsConfig import (
+    GCSConfig,
+)
 from metadata.generated.schema.entity.services.connections.database.datalake.s3Config import (
     S3Config,
 )
@@ -29,6 +32,9 @@ from metadata.generated.schema.metadataIngestion.storage.manifestMetadataConfig
 from metadata.generated.schema.metadataIngestion.storage.storageMetadataADLSConfig import (
     StorageMetadataAdlsConfig,
 )
+from metadata.generated.schema.metadataIngestion.storage.storageMetadataGCSConfig import (
+    StorageMetadataGcsConfig,
+)
 from metadata.generated.schema.metadataIngestion.storage.storageMetadataHttpConfig import (
     StorageMetadataHttpConfig,
 )
@@ -39,6 +45,7 @@ from metadata.generated.schema.metadataIngestion.storage.storageMetadataS3Config
     StorageMetadataS3Config,
 )
 from metadata.readers.file.config_source_factory import get_reader
+from metadata.utils.credentials import set_google_credentials
 from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
@@ -172,5 +179,39 @@ def _(config: StorageMetadataAdlsConfig) -> ManifestMetadataConfig:
     except Exception as exc:
         logger.debug(traceback.format_exc())
         raise StorageMetadataConfigException(
-            f"Error fetching manifest file from s3: {exc}"
+            f"Error fetching manifest file from adls: {exc}"
+        )
+
+
+@get_manifest.register
+def _(config: StorageMetadataGcsConfig) -> ManifestMetadataConfig:
+    try:
+        bucket_name, prefix = (
+            config.prefixConfig.containerName,
+            config.prefixConfig.objectPrefix,
+        )
+
+        path = (
+            f"{prefix}/{STORAGE_METADATA_MANIFEST_FILE_NAME}"
+            if prefix
+            else STORAGE_METADATA_MANIFEST_FILE_NAME
+        )
+
+        from google.cloud.storage import (  # pylint: disable=import-outside-toplevel
+            Client,
+        )
+
+        set_google_credentials(gcp_credentials=config.securityConfig)
+        gcs_client = Client()
+        reader = get_reader(
+            config_source=GCSConfig(securityConfig=config.securityConfig),
+            client=gcs_client,
+        )
+
+        manifest = reader.read(path=path, bucket_name=bucket_name)
+        return ManifestMetadataConfig.parse_obj(json.loads(manifest))
+    except Exception as exc:
+        logger.debug(traceback.format_exc())
+        raise StorageMetadataConfigException(
+            f"Error fetching manifest file from gcs: {exc}"
         )

metadata/utils/tag_utils.py

@@ -21,6 +21,9 @@ from metadata.generated.schema.api.classification.createClassification import (
 )
 from metadata.generated.schema.api.classification.createTag import CreateTagRequest
 from metadata.generated.schema.entity.classification.tag import Tag
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.type.basic import FullyQualifiedEntityName
 from metadata.generated.schema.type.tagLabel import (
     LabelType,
@@ -28,7 +31,7 @@ from metadata.generated.schema.type.tagLabel import (
     TagLabel,
     TagSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
@@ -72,7 +75,7 @@ def get_ometa_tag_and_classification(
                     left=StackTraceError(
                         name=tag,
                         error=f"Error yielding tag [{tag}]: [{err}]",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/workflow/application.py

@@ -0,0 +1,154 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Generic Workflow entrypoint to execute Applications
+"""
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from metadata.config.common import WorkflowExecutionError
+from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
+    OpenMetadataConnection,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.generated.schema.entity.services.serviceType import ServiceType
+from metadata.generated.schema.metadataIngestion.application import (
+    OpenMetadataApplicationConfig,
+)
+from metadata.generated.schema.metadataIngestion.workflow import LogLevels
+from metadata.ingestion.api.step import Step, Summary
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.importer import import_from_module
+from metadata.utils.logger import ingestion_logger
+from metadata.workflow.base import BaseWorkflow
+from metadata.workflow.workflow_status_mixin import SUCCESS_THRESHOLD_VALUE
+
+logger = ingestion_logger()
+
+
+class InvalidAppConfiguration(Exception):
+    """
+    To be raised if the config received by the App
+    is not the one expected
+    """
+
+
+class AppRunner(Step, ABC):
+    """Class that knows how to execute the Application logic."""
+
+    def __init__(
+        self,
+        config: OpenMetadataApplicationConfig,
+        metadata: OpenMetadata,
+    ):
+        self.app_config = config.appConfig.__root__ if config.appConfig else None
+        self.private_config = (
+            config.appPrivateConfig.__root__ if config.appPrivateConfig else None
+        )
+        self.metadata = metadata
+
+        super().__init__()
+
+    @property
+    def name(self) -> str:
+        return "AppRunner"
+
+    @abstractmethod
+    def run(self) -> None:
+        """App logic to execute"""
+
+    @classmethod
+    def create(cls, config_dict: dict, metadata: OpenMetadata) -> "Step":
+        config = OpenMetadataApplicationConfig.parse_obj(config_dict)
+        return cls(config=config, metadata=metadata)
+
+
+class ApplicationWorkflow(BaseWorkflow, ABC):
+    """Base Application Workflow implementation"""
+
+    config: OpenMetadataApplicationConfig
+    runner: Optional[AppRunner]
+
+    def __init__(self, config_dict: dict):
+        self.runner = None  # Will be passed in post-init
+        # TODO: Create a parse_gracefully method
+        self.config = OpenMetadataApplicationConfig.parse_obj(config_dict)
+
+        # Applications are associated to the OpenMetadata Service
+        self.service_type: ServiceType = ServiceType.Metadata
+
+        metadata_config: OpenMetadataConnection = (
+            self.config.workflowConfig.openMetadataServerConfig
+        )
+        log_level: LogLevels = self.config.workflowConfig.loggerLevel
+
+        super().__init__(
+            config=self.config,
+            log_level=log_level,
+            metadata_config=metadata_config,
+            service_type=self.service_type,
+        )
+
+    @classmethod
+    def create(cls, config_dict: dict):
+        return cls(config_dict)
+
+    def post_init(self) -> None:
+        """
+        Method to execute after we have initialized all the internals.
+        Here we will load the runner since it needs the `metadata` object
+        """
+        runner_class = import_from_module(self.config.sourcePythonClass)
+        if not issubclass(runner_class, AppRunner):
+            raise ValueError(
+                "We need a valid AppRunner to initialize the ApplicationWorkflow!"
+            )
+
+        try:
+            self.runner = runner_class(
+                config=self.config,
+                metadata=self.metadata,
+            )
+        except Exception as exc:
+            logger.error(
+                f"Error trying to init the AppRunner [{self.config.sourcePythonClass}] due to [{exc}]"
+            )
+            raise exc
+
+    def execute_internal(self) -> None:
+        """Workflow-specific logic to execute safely"""
+        self.runner.run()
+
+    def calculate_success(self) -> float:
+        return self.runner.get_status().calculate_success()
+
+    def get_failures(self) -> List[StackTraceError]:
+        return self.workflow_steps()[0].get_status().failures
+
+    def workflow_steps(self) -> List[Step]:
+        return [self.runner]
+
+    def raise_from_status_internal(self, raise_warnings=False):
+        """Check failed status in the runner"""
+        if (
+            self.runner.get_status().failures
+            and self.calculate_success() < SUCCESS_THRESHOLD_VALUE
+        ):
+            raise WorkflowExecutionError(
+                f"{self.runner.name} reported errors: {Summary.from_step(self.runner)}"
+            )
+
+        if raise_warnings and self.runner.get_status().warnings:
+            raise WorkflowExecutionError(
+                f"{self.runner.name} reported warning: {Summary.from_step(self.runner)}"
+            )

metadata/workflow/application_output_handler.py

@@ -0,0 +1,34 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Module handles the output messages for applications
+"""
+import time
+
+from metadata.utils.helpers import pretty_print_time_duration
+from metadata.utils.logger import ANSI, log_ansi_encoded_string
+from metadata.workflow.output_handler import print_workflow_summary
+
+
+def print_status(workflow: "ApplicationWorkflow") -> None:
+    """
+    Print the workflow results
+    """
+    print_workflow_summary(workflow)
+
+    if workflow.runner.get_status().source_start_time:
+        log_ansi_encoded_string(
+            color=ANSI.BRIGHT_CYAN,
+            bold=True,
+            message="Workflow finished in time: "
+            f"{pretty_print_time_duration(time.time()-workflow.runner.get_status().source_start_time)}",
+        )