openmetadata-ingestion 1.6.0.0rc3__py3-none-any.whl → 1.6.2.0__py3-none-any.whl

metadata/generated/schema/type/apiSchema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/apiSchema.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/assetCertification.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/assetCertification.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/auditLog.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/auditLog.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/basic.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/bulkOperationResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/bulkOperationResult.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEvent.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEventType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEventType.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/collectionDescriptor.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/collectionDescriptor.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvDocumentation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvDocumentation.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvErrorType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvErrorType.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvFile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvFile.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvImportResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvImportResult.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00

metadata/generated/schema/type/customProperties/complexTypes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperties/complexTypes.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/enumConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperties/enumConfig.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/tableConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperties/tableConfig.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperty.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/customProperty.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/dailyCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/dailyCount.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-12-05T13:16:41+00:00
+#   timestamp: 2025-01-10T14:18:34+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -257,7 +257,7 @@ class OpenMetadataValidationAction(ValidationAction):
 
         create_test_suite = CreateTestSuiteRequest(
             name=f"{table_entity.fullyQualifiedName.root}.TestSuite",
-            executableEntityReference=table_entity.fullyQualifiedName.root,
+            basicEntityReference=table_entity.fullyQualifiedName.root,
         )  # type: ignore
         test_suite = self.ometa_conn.create_or_update_executable_test_suite(
             create_test_suite

metadata/ingestion/ometa/mixins/tests_mixin.py

@@ -228,7 +228,7 @@ class OMetaTestsMixin:
 
         create_test_suite = CreateTestSuiteRequest(
             name=f"{table_entity.fullyQualifiedName.root}.TestSuite",
-            executableEntityReference=table_entity.fullyQualifiedName.root,
+            basicEntityReference=table_entity.fullyQualifiedName.root,
         )  # type: ignore
         test_suite = self.create_or_update_executable_test_suite(create_test_suite)
         return test_suite

metadata/ingestion/ometa/mixins/user_mixin.py

@@ -13,6 +13,7 @@ Mixin class containing User specific methods
 
 To be used by OpenMetadata class
 """
+import json
 from functools import lru_cache
 from typing import Optional, Type
 
@@ -46,16 +47,29 @@ class OMetaUserMixin:
         )
 
     @staticmethod
-    def name_search_query_es(entity: Type[T]) -> str:
+    def name_search_query_es(entity: Type[T], name: str, from_: int, size: int) -> str:
         """
         Allow for more flexible lookup following what the UI is doing when searching users.
 
         We don't want to stick to `q=name:{name}` since in case a user is named `random.user`
         but looked as `Random User`, we want to find this match.
+
+        Search should only look in name and displayName fields and should not return bots.
         """
+        query_filter = {
+            "query": {
+                "query_string": {
+                    "query": f"{name} AND isBot:false",
+                    "fields": ["name", "displayName"],
+                    "default_operator": "AND",
+                    "fuzziness": "AUTO",
+                }
+            }
+        }
+
         return (
-            "/search/query?q={name} AND isBot:false&from={from_}&size={size}&index="
-            + ES_INDEX_MAP[entity.__name__]
+            f"""/search/query?query_filter={json.dumps(query_filter)}"""
+            f"&from={from_}&size={size}&index=" + ES_INDEX_MAP[entity.__name__]
         )
 
     def _search_by_email(
@@ -103,8 +117,8 @@ class OMetaUserMixin:
             fields: Optional field list to pass to ES request
         """
         if name:
-            query_string = self.name_search_query_es(entity=entity).format(
-                name=name, from_=from_count, size=size
+            query_string = self.name_search_query_es(
+                entity=entity, name=name, from_=from_count, size=size
             )
             return self.get_entity_from_es(
                 entity=entity, query_string=query_string, fields=fields

metadata/ingestion/ometa/routes.py

@@ -54,6 +54,7 @@ from metadata.generated.schema.api.data.createStoredProcedure import (
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
 from metadata.generated.schema.api.data.createTopic import CreateTopicRequest
+from metadata.generated.schema.api.docStore.createDocument import CreateDocumentRequest
 from metadata.generated.schema.api.domains.createDataProduct import (
     CreateDataProductRequest,
 )
@@ -136,6 +137,7 @@ from metadata.generated.schema.entity.data.searchIndex import SearchIndex
 from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.data.topic import Topic
+from metadata.generated.schema.entity.docStore.document import Document
 from metadata.generated.schema.entity.domains.dataProduct import DataProduct
 from metadata.generated.schema.entity.domains.domain import Domain
 from metadata.generated.schema.entity.feed.suggestion import Suggestion
@@ -198,6 +200,8 @@ ROUTES = {
     CreateAPIEndpointRequest.__name__: "/apiEndpoints",
     APICollection.__name__: "/apiCollections",
     CreateAPICollectionRequest.__name__: "/apiCollections",
+    Document.__name__: "/docStore",
+    CreateDocumentRequest.__name__: "/docStore",
     # Classifications
     Tag.__name__: "/tags",
     CreateTagRequest.__name__: "/tags",

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -127,7 +127,13 @@ class PowerBiApiClient:
             List[PowerBIDashboard]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/dashboards")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(
+                f"/myorg/{admin}groups/{group_id}/dashboards"
+            )
+            if not response_data:
+                logger.debug(f"No dashboards found for workspace_id: {group_id}")
+                return None
             response = DashboardsResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -142,7 +148,11 @@ class PowerBiApiClient:
             List[PowerBIReport]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/reports")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(f"/myorg/{admin}groups/{group_id}/reports")
+            if not response_data:
+                logger.debug(f"No reports found for workspace_id: {group_id}")
+                return None
             response = ReportsResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -157,7 +167,11 @@ class PowerBiApiClient:
             List[Dataset]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/datasets")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(f"/myorg/{admin}groups/{group_id}/datasets")
+            if not response_data:
+                logger.debug(f"No datasets found for workspace_id: {group_id}")
+                return None
             response = DatasetResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -174,9 +188,13 @@ class PowerBiApiClient:
             List[Tile]
         """
         try:
+            admin = "admin/" if self.config.useAdminApis else ""
             response_data = self.client.get(
-                f"/myorg/groups/{group_id}/dashboards/{dashboard_id}/tiles"
+                f"/myorg/{admin}dashboards/{dashboard_id}/tiles"
             )
+            if not response_data:
+                logger.debug(f"No dashboard tiles found for workspace_id: {group_id}")
+                return None
             response = TilesResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -216,6 +234,9 @@ class PowerBiApiClient:
             entities_per_page = self.config.pagination_entity_per_page
             params_data = {"$top": "1"}
             response_data = self.client.get(api_url, data=params_data)
+            if not response_data:
+                logger.debug("No groups/workspaces found")
+                return None
             response = GroupsResponse(**response_data)
             count = response.odata_count
             indexes = math.ceil(count / entities_per_page)
@@ -227,6 +248,9 @@ class PowerBiApiClient:
                     "$skip": str(index * entities_per_page),
                 }
                 response_data = self.client.get(api_url, data=params_data)
+                if not response_data:
+                    logger.debug("No more groups/workspaces found")
+                    continue
                 response = GroupsResponse(**response_data)
                 workspaces.extend(response.value)
             return workspaces
@@ -262,6 +286,7 @@ class PowerBiApiClient:
     def fetch_workspace_scan_status(
         self, scan_id: str
     ) -> Optional[WorkSpaceScanResponse]:
+        # deprecated in favour to avoide bulk data prepare
         """Get Workspace scan status by id method
         Args:
             scan_id:
@@ -280,6 +305,7 @@ class PowerBiApiClient:
         return None
 
     def fetch_workspace_scan_result(self, scan_id: str) -> Optional[Workspaces]:
+        # deprecated in favour to avoide bulk data prepare
         """Get Workspace scan result by id method
         Args:
             scan_id:
@@ -301,6 +327,7 @@ class PowerBiApiClient:
         """
         Method to poll the scan status endpoint until the timeout
         """
+        # deprecated in favour to avoide bulk data prepare
         min_sleep_time = 3
         if min_sleep_time > timeout:
             logger.info(f"Timeout is set to minimum sleep time: {timeout}")