openmetadata-ingestion 1.5.7.1.dev2__py3-none-any.whl → 1.5.8.0.dev0__py3-none-any.whl

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-10-30T06:52:40+00:00
+#   timestamp: 2024-11-26T05:42:17+00:00
 
 from __future__ import annotations
 

metadata/ingestion/connections/test_connections.py

@@ -40,6 +40,7 @@ from metadata.generated.schema.type.basic import Timestamp
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import kill_active_connections
 from metadata.profiler.orm.functions.conn_test import ConnTestFn
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import cli_logger
 from metadata.utils.timeout import timeout
 
@@ -92,25 +93,24 @@ def _test_connection_steps(
     metadata: OpenMetadata,
     steps: List[TestConnectionStep],
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+) -> TestConnectionResult:
     """
     Run all the function steps and raise any errors
     """
 
     if automation_workflow:
-        _test_connection_steps_automation_workflow(
+        return _test_connection_steps_automation_workflow(
             metadata=metadata, steps=steps, automation_workflow=automation_workflow
         )
 
-    else:
-        _test_connection_steps_during_ingestion(steps=steps)
+    return _test_connection_steps_during_ingestion(steps=steps)
 
 
 def _test_connection_steps_automation_workflow(
     metadata: OpenMetadata,
     steps: List[TestConnectionStep],
     automation_workflow: AutomationWorkflow,
-) -> None:
+) -> TestConnectionResult:
     """
     Run the test connection as part of the automation workflow
     We need to update the automation workflow in each step
@@ -187,33 +187,40 @@ def _test_connection_steps_automation_workflow(
             )
         )
 
+    return test_connection_result
 
-def _test_connection_steps_during_ingestion(steps: List[TestConnectionStep]) -> None:
-    """
-    Run the test connection as part of the ingestion workflow
-    Raise an exception if something fails
-    """
-    test_connection_result = TestConnectionIngestionResult()
+
+def _test_connection_steps_during_ingestion(
+    steps: List[TestConnectionStep],
+) -> TestConnectionResult:
+    """Run the test connection steps during ingestion"""
+    test_connection_result = TestConnectionResult(
+        status=StatusType.Running,
+        steps=[],
+    )
     for step in steps:
         try:
+            logger.info(f"Running {step.name}...")
             step.function()
-            test_connection_result.success.append(f"'{step.name}': Pass")
-
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"{step.name}-{exc}")
-            if step.mandatory:
-                test_connection_result.failed.append(
-                    f"'{step.name}': This is a mandatory step and we won't be able to extract"
-                    f" necessary metadata. Failed due to: {exc}"
+            test_connection_result.steps.append(
+                TestConnectionStepResult(
+                    name=step.name,
+                    mandatory=step.mandatory,
+                    passed=True,
                 )
-
-            else:
-                test_connection_result.warning.append(
-                    f"'{step.name}': This is a optional and the ingestion will continue to work as expected."
-                    f"Failed due to: {exc}"
+            )
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.error(f"{step.name}-{err}")
+            test_connection_result.steps.append(
+                TestConnectionStepResult(
+                    name=step.name,
+                    mandatory=step.mandatory,
+                    passed=False,
+                    message=step.error_message,
+                    errorLog=str(err),
                 )
-
+            )
             if step.short_circuit:
                 # break the workflow if the step is a short circuit step
                 break
@@ -221,10 +228,20 @@ def _test_connection_steps_during_ingestion(steps: List[TestConnectionStep]) ->
     logger.info("Test connection results:")
     logger.info(test_connection_result)
 
-    if test_connection_result.failed:
-        raise SourceConnectionException(
-            f"Some steps failed when testing the connection: [{test_connection_result}]"
-        )
+    return test_connection_result
+
+
+def raise_test_connection_exception(result: TestConnectionResult) -> None:
+    """Raise if needed an exception for the test connection"""
+    for step in result.steps:
+        if not step.passed and step.mandatory:
+            raise SourceConnectionException(
+                f"Failed to run the test connection step: {step.name}"
+            )
+        if not step.passed:
+            logger.warning(
+                f"You might be missing metadata in: {step.name} due to {step.message}"
+            )
 
 
 def test_connection_steps(
@@ -232,8 +249,8 @@ def test_connection_steps(
     service_type: str,
     test_fn: dict,
     automation_workflow: Optional[AutomationWorkflow] = None,
-    timeout_seconds: int = 3 * 60,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test the connection steps with a given timeout
 
@@ -268,9 +285,12 @@ def test_connection_steps(
         for step in test_connection_definition.steps
     ]
 
-    return timeout(timeout_seconds)(_test_connection_steps)(
-        metadata, steps, automation_workflow
-    )
+    if timeout_seconds:
+        return timeout(timeout_seconds)(_test_connection_steps)(
+            metadata, steps, automation_workflow
+        )
+
+    return _test_connection_steps(metadata, steps, automation_workflow)
 
 
 def test_connection_engine_step(connection: Engine) -> None:
@@ -289,8 +309,8 @@ def test_connection_db_common(
     service_connection,
     automation_workflow: Optional[AutomationWorkflow] = None,
     queries: dict = None,
-    timeout_seconds: int = 3 * 60,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -322,7 +342,7 @@ def test_connection_db_common(
     for key, query in queries.items():
         test_fn[key] = partial(test_query, statement=query, engine=engine)
 
-    test_connection_steps(
+    result = test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
@@ -332,6 +352,8 @@ def test_connection_db_common(
 
     kill_active_connections(engine)
 
+    return result
+
 
 def test_connection_db_schema_sources(
     metadata: OpenMetadata,
@@ -339,7 +361,8 @@ def test_connection_db_schema_sources(
     service_connection,
     automation_workflow: Optional[AutomationWorkflow] = None,
     queries: dict = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -388,15 +411,18 @@ def test_connection_db_schema_sources(
     for key, query in queries.items():
         test_fn[key] = partial(test_query, statement=query, engine=engine)
 
-    test_connection_steps(
+    result = test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
 
     kill_active_connections(engine)
 
+    return result
+
 
 def test_query(engine: Engine, statement: str):
     """

metadata/ingestion/source/dashboard/dashboard_service.py

@@ -55,6 +55,9 @@ from metadata.ingestion.api.delete import delete_entity_from_source
 from metadata.ingestion.api.models import Either, Entity
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import C, TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.lineage.sql_lineage import get_column_fqn
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -554,7 +557,10 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
 
     def prepare(self):
         """By default, nothing to prepare"""

metadata/ingestion/source/dashboard/domodashboard/connection.py

@@ -12,7 +12,6 @@
 """
 Source connection handler
 """
-
 from typing import Optional
 
 from pydomo import Domo
@@ -24,11 +23,15 @@ from metadata.generated.schema.entity.automations.workflow import (
 from metadata.generated.schema.entity.services.connections.dashboard.domoDashboardConnection import (
     DomoDashboardConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DomoDashboardConnection) -> OMPyDomoClient:
@@ -57,7 +60,8 @@ def test_connection(
     client: OMPyDomoClient,
     service_connection: DomoDashboardConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -72,9 +76,10 @@ def test_connection(
         "GetCharts": client.custom.test_list_cards,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )

metadata/ingestion/source/dashboard/lightdash/connection.py

@@ -12,7 +12,6 @@
 """
 Source connection handler
 """
-
 from typing import Optional
 
 from metadata.generated.schema.entity.automations.workflow import (
@@ -21,12 +20,16 @@ from metadata.generated.schema.entity.automations.workflow import (
 from metadata.generated.schema.entity.services.connections.dashboard.lightdashConnection import (
     LightdashConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.lightdash.client import LightdashApiClient
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -49,7 +52,8 @@ def test_connection(
     client: LightdashApiClient,
     service_connection: LightdashConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -60,9 +64,10 @@ def test_connection(
 
     test_fn = {"GetDashboards": custom_executor}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )

metadata/ingestion/source/dashboard/looker/connection.py

@@ -24,8 +24,12 @@ from metadata.generated.schema.entity.automations.workflow import (
 from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import (
     LookerConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: LookerConnection) -> Looker40SDK:
@@ -49,7 +53,8 @@ def test_connection(
     client: Looker40SDK,
     service_connection: LookerConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -76,9 +81,10 @@ def test_connection(
         "ListLookMLModels": list_datamodels_test,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )

metadata/ingestion/source/dashboard/metabase/connection.py

@@ -20,9 +20,13 @@ from metadata.generated.schema.entity.automations.workflow import (
 from metadata.generated.schema.entity.services.connections.dashboard.metabaseConnection import (
     MetabaseConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.metabase.client import MetabaseClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: MetabaseConnection) -> MetabaseClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: MetabaseClient,
     service_connection: MetabaseConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -49,9 +54,10 @@ def test_connection(
 
     test_fn = {"GetDashboards": custom_executor}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )

metadata/ingestion/source/dashboard/mode/client.py

@@ -67,15 +67,27 @@ class ModeApiClient:
         )
         self.client = REST(client_config)
 
-    def fetch_all_reports(self, workspace_name: str) -> Optional[list]:
+    def fetch_all_reports(
+        self, workspace_name: str, filter: Optional[str] = "all"
+    ) -> Optional[list]:
         """Method to fetch all reports for Mode
         Args:
             workspace_name:
+            filter:
         Returns:
             dict
         """
+        if filter not in ["custom", "all"]:
+            logger.warning(
+                "Invalid value for filter. Should be one of ['custom', 'all']"
+            )
+            return
+
         all_reports = []
-        response_collections = self.client.get(f"/{workspace_name}/{COLLECTIONS}")
+        filter_param = f"?filter={filter}"
+        response_collections = self.client.get(
+            f"/{workspace_name}/{COLLECTIONS}{filter_param}"
+        )
         collections = response_collections[EMBEDDED]["spaces"]
         for collection in collections:
             response_reports = self.get_all_reports_for_collection(
@@ -175,3 +187,18 @@ class ModeApiClient:
             logger.warning(f"Error fetching all data sources: {exc}")
 
         return None
+
+    def get_workspace(self, workspace_name: str) -> Optional[dict]:
+        """Method to get info about a workspace
+        Args:
+            workspace_name:
+        Returns:
+            dict
+        """
+        try:
+            response = self.client.get(f"/{workspace_name}")
+            return response
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error testing workspace connection: {exc}")
+            raise exc