openmetadata-ingestion 1.6.0.0rc2__py3-none-any.whl → 1.6.0.0rc3__py3-none-any.whl

metadata/ingestion/source/database/snowflake/queries.py

@@ -294,17 +294,38 @@ SELECT
   PROCEDURE_LANGUAGE AS language,
   PROCEDURE_DEFINITION AS definition,
   ARGUMENT_SIGNATURE AS signature,
-  COMMENT as comment
+  COMMENT as comment,
+  'StoredProcedure' as procedure_type
 FROM INFORMATION_SCHEMA.PROCEDURES
 WHERE PROCEDURE_CATALOG = '{database_name}'
   AND PROCEDURE_SCHEMA = '{schema_name}'
     """
 )
 
+SNOWFLAKE_GET_FUNCTIONS = textwrap.dedent(
+    """
+SELECT
+  FUNCTION_NAME AS name,
+  FUNCTION_OWNER AS owner,
+  FUNCTION_LANGUAGE AS language,
+  FUNCTION_DEFINITION AS definition,
+  ARGUMENT_SIGNATURE AS signature,
+  COMMENT as comment,
+  'UDF' as procedure_type
+FROM INFORMATION_SCHEMA.FUNCTIONS
+WHERE FUNCTION_CATALOG = '{database_name}'
+  AND FUNCTION_SCHEMA = '{schema_name}'
+    """
+)
+
 SNOWFLAKE_DESC_STORED_PROCEDURE = (
     "DESC PROCEDURE {database_name}.{schema_name}.{procedure_name}{procedure_signature}"
 )
 
+SNOWFLAKE_DESC_FUNCTION = (
+    "DESC FUNCTION {database_name}.{schema_name}.{procedure_name}{procedure_signature}"
+)
+
 SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
     """
 WITH SP_HISTORY AS (

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -217,14 +217,26 @@ class StoredProcedureLineageMixin(ABC):
         self, procedure_and_query: ProcedureAndQuery
     ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
 
-        yield from self._yield_procedure_lineage(
-            query_by_procedure=procedure_and_query.query_by_procedure,
-            procedure=procedure_and_query.procedure,
-        )
-        yield from self.yield_procedure_query(
-            query_by_procedure=procedure_and_query.query_by_procedure,
-            procedure=procedure_and_query.procedure,
-        )
+        try:
+            yield from self._yield_procedure_lineage(
+                query_by_procedure=procedure_and_query.query_by_procedure,
+                procedure=procedure_and_query.procedure,
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Could not get lineage for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]."
+            )
+        try:
+            yield from self.yield_procedure_query(
+                query_by_procedure=procedure_and_query.query_by_procedure,
+                procedure=procedure_and_query.procedure,
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Could not get query for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]."
+            )
 
     def procedure_lineage_generator(self) -> Iterable[ProcedureAndQuery]:
         query = {

metadata/ingestion/source/pipeline/fivetran/metadata.py

@@ -275,7 +275,7 @@ class FivetranSource(PipelineServiceSource):
                 )
 
     def get_pipeline_name(self, pipeline_details: FivetranPipelineDetails) -> str:
-        return pipeline_details.pipeline_name
+        return pipeline_details.pipeline_display_name or pipeline_details.pipeline_name
 
     def get_source_url(
         self,

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -372,7 +372,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             dictionnary of results
         """
         logger.debug(f"Computing system metrics for {runner.table_name}")
-        return self.system_metrics_computer.get_system_metrics(table=runner.dataset)
+        return self.system_metrics_computer.get_system_metrics(runner=runner)
 
     def _create_thread_safe_runner(self, session, column=None):
         """Create thread safe runner"""

metadata/profiler/orm/functions/table_metric_computer.py

@@ -91,8 +91,8 @@ class AbstractTableMetricComputer(ABC):
             table (DeclarativeMeta): _description_
         """
         try:
-            self._schema_name = self.table.schema
-            self._table_name = self.table.name
+            self._schema_name = self.runner.schema_name
+            self._table_name = self.runner.table_name
         except AttributeError:
             raise AttributeError(ERROR_MSG)
 
@@ -119,10 +119,10 @@ class AbstractTableMetricComputer(ABC):
         Returns:
             Tuple[str, int]
         """
-        col_names = literal(",".join(inspect(self.table).c.keys()), type_=String).label(
-            COLUMN_NAMES
-        )
-        col_count = literal(len(inspect(self.table).c)).label(COLUMN_COUNT)
+        col_names = literal(
+            ",".join(inspect(self.runner.raw_dataset).c.keys()), type_=String
+        ).label(COLUMN_NAMES)
+        col_count = literal(len(inspect(self.runner.raw_dataset).c)).label(COLUMN_COUNT)
         return col_names, col_count
 
     def _build_query(

metadata/profiler/processor/runner.py

@@ -83,6 +83,11 @@ class QueryRunner:
         """Table name attribute access"""
         return self.raw_dataset.__table__.schema
 
+    @property
+    def session(self):
+        """Table name attribute access"""
+        return self._session
+
     def _build_query(self, *entities, **kwargs) -> Query:
         """Build query object
 

metadata/profiler/source/database/base/profiler_source.py

@@ -154,7 +154,6 @@ class ProfilerSource(ProfilerSourceInterface):
                 profile_sample_type=self.source_config.profileSampleType,
                 sampling_method_type=self.source_config.samplingMethodType,
             ),
-            default_sample_data_count=self.source_config.sampleDataCount,
         )
 
         profiler_interface: ProfilerInterface = profiler_class.create(

metadata/sampler/processor.py

@@ -88,11 +88,7 @@ class SamplerProcessor(Processor):
                 schema_entity=schema_entity,
                 database_entity=database_entity,
                 table_config=get_config_for_table(entity, self.profiler_config),
-                default_sample_config=SampleConfig(
-                    profile_sample=self.source_config.profileSample,
-                    profile_sample_type=self.source_config.profileSampleType,
-                    sampling_method_type=self.source_config.samplingMethodType,
-                ),
+                default_sample_config=SampleConfig(),
                 default_sample_data_count=self.source_config.sampleDataCount,
             )
             sample_data = SampleData(

metadata/sampler/sqlalchemy/sampler.py

@@ -223,8 +223,11 @@ class SQASampler(SamplerInterface, SQAInterfaceMixin):
         if not is_safe_sql_query(self.sample_query):
             raise RuntimeError(f"SQL expression is not safe\n\n{self.sample_query}")
 
-        return self.client.query(self.raw_dataset).from_statement(
-            text(f"{self.sample_query}")
+        stmt = text(f"{self.sample_query}")
+        stmt = stmt.columns(*list(inspect(self.raw_dataset).c))
+
+        return self.client.query(stmt.subquery()).cte(
+            f"{self.raw_dataset.__tablename__}_user_sampled"
         )
 
     def _partitioned_table(self) -> Query:

metadata/utils/class_helper.py

@@ -66,6 +66,9 @@ from metadata.generated.schema.metadataIngestion.testSuitePipeline import (
     TestSuitePipeline,
 )
 from metadata.generated.schema.metadataIngestion.workflow import SourceConfig
+from metadata.generated.schema.tests.testSuite import (
+    ServiceType as TestSuiteServiceType,
+)
 
 SERVICE_TYPE_REF = {
     ServiceType.Api.value: "apiService",
@@ -77,6 +80,8 @@ SERVICE_TYPE_REF = {
     ServiceType.Metadata.value: "metadataService",
     ServiceType.Search.value: "searchService",
     ServiceType.Storage.value: "storageService",
+    # We use test suites as "services" for DQ Ingestion Pipelines
+    TestSuiteServiceType.TestSuite.value: "testSuite",
 }
 
 SOURCE_CONFIG_TYPE_INGESTION = {
@@ -123,6 +128,8 @@ def get_pipeline_type_from_source_config(source_config: SourceConfig) -> Pipelin
 def _get_service_type_from(  # pylint: disable=inconsistent-return-statements
     service_subtype: str,
 ) -> ServiceType:
+    if service_subtype.lower() == "testsuite":
+        return TestSuiteServiceType.TestSuite
     for service_type in ServiceType:
         if service_subtype.lower() in [
             subtype.value.lower()

metadata/utils/importer.py

@@ -131,7 +131,7 @@ def import_from_module(key: str) -> Type[Any]:
     try:
         obj = getattr(importlib.import_module(module_name), obj_name)
         return obj
-    except ModuleNotFoundError as err:
+    except (ModuleNotFoundError, ImportError) as err:
         logger.debug(traceback.format_exc())
         raise DynamicImportException(module=module_name, key=obj_name, cause=err)
 

metadata/workflow/data_quality.py

@@ -22,6 +22,7 @@ from metadata.generated.schema.entity.services.connections.serviceConnection imp
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.tests.testSuite import ServiceType, TestSuite
 from metadata.ingestion.api.steps import Processor, Sink
+from metadata.ingestion.ometa.utils import model_str
 from metadata.utils import fqn
 from metadata.utils.importer import import_sink_class
 from metadata.utils.logger import test_suite_logger
@@ -40,6 +41,7 @@ class TestSuiteWorkflow(IngestionWorkflow):
     """
 
     __test__ = False
+    service_type = ServiceType.TestSuite
 
     def set_steps(self):
         self.source = TestSuiteSource.create(self.config.model_dump(), self.metadata)
@@ -113,6 +115,8 @@ class TestSuiteWorkflow(IngestionWorkflow):
             fqn=fqn.build(
                 metadata=None,
                 entity_type=TestSuite,
-                table_fqn=self.config.source.sourceConfig.config.entityFullyQualifiedName,
+                table_fqn=model_str(
+                    self.config.source.sourceConfig.config.entityFullyQualifiedName
+                ),
             ),
         )