openmetadata-ingestion 1.2.4.1__py3-none-any.whl → 1.2.4.3__py3-none-any.whl

metadata/generated/schema/system/ui/knowledgePanel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  system/ui/knowledgePanel.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/system/ui/page.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  system/ui/page.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00

metadata/generated/schema/tests/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/basic.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/customMetric.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/customMetric.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testCase.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testCase.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testDefinition.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testDefinition.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testSuite.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testSuite.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00

metadata/generated/schema/type/auditLog.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/auditLog.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/basic.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/bulkOperationResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/bulkOperationResult.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEvent.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/collectionDescriptor.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/collectionDescriptor.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvDocumentation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvDocumentation.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvErrorType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvErrorType.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvFile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvFile.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvImportResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvImportResult.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/dailyCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/dailyCount.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-01-09T08:06:11+00:00
+#   timestamp: 2024-01-17T12:40:28+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/database/common_db_source.py

@@ -246,6 +246,23 @@ class CommonDbSourceService(
             for table_name in self.inspector.get_table_names(schema_name) or []
         ]
 
+    def query_view_names_and_types(
+        self, schema_name: str
+    ) -> Iterable[TableNameAndType]:
+        """
+        Connect to the source database to get the view
+        name and type. By default, use the inspector method
+        to get the names and pass the View type.
+
+        This is useful for sources where we need fine-grained
+        logic on how to handle table types, e.g., material views,...
+        """
+
+        return [
+            TableNameAndType(name=table_name, type_=TableType.View)
+            for table_name in self.inspector.get_view_names(schema_name) or []
+        ]
+
     def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
         """
         Handle table and views.
@@ -285,8 +302,10 @@ class CommonDbSourceService(
                     yield table_name, table_and_type.type_
 
             if self.source_config.includeViews:
-                for view_name in self.inspector.get_view_names(schema_name):
-                    view_name = self.standardize_table_name(schema_name, view_name)
+                for view_and_type in self.query_view_names_and_types(schema_name):
+                    view_name = self.standardize_table_name(
+                        schema_name, view_and_type.name
+                    )
                     view_fqn = fqn.build(
                         self.metadata,
                         entity_type=Table,
@@ -307,7 +326,7 @@ class CommonDbSourceService(
                             "Table Filtered Out",
                         )
                         continue
-                    yield view_name, TableType.View
+                    yield view_name, view_and_type.type_
         except Exception as err:
             logger.warning(
                 f"Fetching tables names failed for schema {schema_name} due to - {err}"
@@ -317,7 +336,7 @@ class CommonDbSourceService(
     def get_view_definition(
         self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
     ) -> Optional[str]:
-        if table_type == TableType.View:
+        if table_type in (TableType.View, TableType.MaterializedView):
             try:
                 view_definition = inspector.get_view_definition(table_name, schema_name)
                 view_definition = (

metadata/ingestion/source/database/databricks/client.py

@@ -141,6 +141,15 @@ class DatabricksClient:
             or query_text.startswith(QUERY_WITH_OM_VERSION)
         )
 
+    def list_jobs_test_connection(self) -> None:
+        data = {"limit": 1, "expand_tasks": True, "offset": 0}
+        self.client.get(
+            self.jobs_list_url,
+            data=json.dumps(data),
+            headers=self.headers,
+            timeout=API_TIMEOUT,
+        ).json()
+
     def list_jobs(self) -> List[dict]:
         """
         Method returns List all the created jobs in a Databricks Workspace

metadata/ingestion/source/database/snowflake/metadata.py

@@ -86,6 +86,7 @@ from metadata.ingestion.source.database.snowflake.utils import (
     get_unique_constraints,
     get_view_definition,
     get_view_names,
+    get_view_names_reflection,
     normalize_names,
 )
 from metadata.ingestion.source.database.stored_procedures_mixin import (
@@ -119,6 +120,7 @@ SnowflakeDialect._get_schema_columns = (  # pylint: disable=protected-access
     get_schema_columns
 )
 Inspector.get_table_names = get_table_names_reflection
+Inspector.get_view_names = get_view_names_reflection
 SnowflakeDialect._current_database_schema = (  # pylint: disable=protected-access
     _current_database_schema
 )
@@ -516,6 +518,33 @@ class SnowflakeSource(
                     )
                 )
 
+    def query_view_names_and_types(
+        self, schema_name: str
+    ) -> Iterable[TableNameAndType]:
+        """
+        Connect to the source database to get the view
+        name and type. By default, use the inspector method
+        to get the names and pass the View type.
+
+        This is useful for sources where we need fine-grained
+        logic on how to handle table types, e.g., material views,...
+        """
+
+        regular_views = [
+            TableNameAndType(name=view_name, type_=TableType.View)
+            for view_name in self.inspector.get_view_names(schema_name) or []
+        ]
+
+        materialized_views = [
+            TableNameAndType(name=view_name, type_=TableType.MaterializedView)
+            for view_name in self.inspector.get_view_names(
+                schema_name, materialized_views=True
+            )
+            or []
+        ]
+
+        return regular_views + materialized_views
+
     def get_stored_procedures(self) -> Iterable[SnowflakeStoredProcedure]:
         """List Snowflake stored procedures"""
         if self.source_config.includeStoredProcedures:

metadata/ingestion/source/database/snowflake/queries.py

@@ -67,6 +67,12 @@ SNOWFLAKE_GET_VIEW_NAMES = """
 select TABLE_NAME from information_schema.tables 
 where TABLE_SCHEMA = '{}' and TABLE_TYPE = 'VIEW'
 """
+
+SNOWFLAKE_GET_MVIEW_NAMES = """
+select TABLE_NAME from information_schema.tables 
+where TABLE_SCHEMA = '{}' and TABLE_TYPE = 'MATERIALIZED VIEW'
+"""
+
 SNOWFLAKE_GET_TRANSIENT_NAMES = """
 select TABLE_NAME from information_schema.tables 
 where TABLE_SCHEMA = '{}' 

metadata/ingestion/source/database/snowflake/utils.py

@@ -25,6 +25,7 @@ from sqlalchemy.types import FLOAT
 from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_GET_COMMENTS,
     SNOWFLAKE_GET_EXTERNAL_TABLE_NAMES,
+    SNOWFLAKE_GET_MVIEW_NAMES,
     SNOWFLAKE_GET_SCHEMA_COLUMNS,
     SNOWFLAKE_GET_TRANSIENT_NAMES,
     SNOWFLAKE_GET_VIEW_NAMES,
@@ -74,6 +75,20 @@ def get_table_names_reflection(self, schema=None, **kw):
         )
 
 
+def get_view_names_reflection(self, schema=None, **kw):
+    """Return all view names in `schema`.
+
+    :param schema: Optional, retrieve names from a non-default schema.
+        For special quoting, use :class:`.quoted_name`.
+
+    """
+
+    with self._operation_context() as conn:  # pylint: disable=protected-access
+        return self.dialect.get_view_names(
+            conn, schema, info_cache=self.info_cache, **kw
+        )
+
+
 def get_table_names(self, connection, schema, **kw):
     query = SNOWFLAKE_GET_WITHOUT_TRANSIENT_TABLE_NAMES
     if kw.get("include_transient_tables"):
@@ -86,8 +101,12 @@ def get_table_names(self, connection, schema, **kw):
     return result
 
 
-def get_view_names(self, connection, schema, **kw):  # pylint: disable=unused-argument
-    cursor = connection.execute(SNOWFLAKE_GET_VIEW_NAMES.format(schema))
+def get_view_names(self, connection, schema, **kw):
+    if kw.get("materialized_views"):
+        query = SNOWFLAKE_GET_MVIEW_NAMES
+    else:
+        query = SNOWFLAKE_GET_VIEW_NAMES
+    cursor = connection.execute(query.format(schema))
     result = [self.normalize_name(row[0]) for row in cursor]
     return result
 

metadata/ingestion/source/pipeline/databrickspipeline/connection.py

@@ -44,11 +44,7 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    def custom_executor_for_pipeline():
-        result = client.list_jobs()
-        return list(result)
-
-    test_fn = {"GetPipelines": custom_executor_for_pipeline}
+    test_fn = {"GetPipelines": client.list_jobs_test_connection}
 
     test_connection_steps(
         metadata=metadata,

metadata/profiler/interface/profiler_interface.py

@@ -18,7 +18,6 @@ from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Optional, Union
 
 from sqlalchemy import Column
-from typing_extensions import Self
 
 from metadata.generated.schema.entity.data.database import (
     Database,
@@ -151,7 +150,7 @@ class ProfilerInterface(ABC):
         service_connection_config,
         ometa_client: Optional[OpenMetadata],
         **kwargs,
-    ) -> Self:
+    ) -> "ProfilerInterface":
         """create class method is used to dispatch the profiler protocol to the
         correct object based on the service connection object