openmetadata-ingestion 1.6.7.0__py3-none-any.whl → 1.6.7.2__py3-none-any.whl

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-03T16:57:42+00:00
 
 from __future__ import annotations
 

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -407,11 +407,37 @@ class OMetaLineageMixin(Generic[T]):
                         f"Error while adding lineage: {lineage_request.left.error}"
                     )
 
+    def _set_permissions(self):
+        try:
+            if not hasattr(self, "is_edit_table_allowed"):
+                permissions = self.client.get(path="/permissions")
+                for permission in permissions.get("data", []):
+                    if permission.get("resource") != "table":
+                        continue
+                    for action in permission.get("permissions", []):
+                        if (
+                            action.get("operation") == "EditAll"
+                            and action.get("access") == "allow"
+                        ):
+                            self.is_edit_table_allowed = True
+                            return
+                logger.warning("Please grant EditAll permission to the user")
+        except Exception as exc:
+            logger.debug(f"Error while setting permissions: {exc}")
+            logger.debug(traceback.format_exc())
+        self.is_edit_table_allowed = False
+
     def patch_lineage_processed_flag(
         self,
         entity: Type[T],
         fqn: str,
     ) -> None:
+        """
+        Patch the processed lineage flag for an entity.
+        """
+        self._set_permissions()
+        if not self.is_edit_table_allowed:
+            return
 
         try:
             original_entity = self.get_by_name(entity=entity, fqn=fqn)

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -14,6 +14,7 @@ REST Auth & Client for PowerBi
 import json
 import math
 import traceback
+from copy import deepcopy
 from time import sleep
 from typing import List, Optional, Tuple
 
@@ -23,6 +24,7 @@ from pydantic import BaseModel, ConfigDict
 from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
     PowerBIConnection,
 )
+from metadata.generated.schema.type.filterPattern import FilterPattern
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.source.dashboard.powerbi.file_client import PowerBiFileClient
@@ -43,6 +45,7 @@ from metadata.ingestion.source.dashboard.powerbi.models import (
     Workspaces,
     WorkSpaceScanResponse,
 )
+from metadata.utils.filters import validate_regex
 from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
@@ -253,8 +256,91 @@ class PowerBiApiClient:
 
         return None
 
+    def regex_to_odata_condition(self, regex: str) -> str:
+        """
+        Convert a regex pattern to an OData filter condition
+        """
+        try:
+            # Handle empty pattern
+            if not regex:
+                return ""
+
+            # Exact match
+            if regex.startswith("^") and regex.endswith("$"):
+                literal = regex[1:-1]
+                return f"trim(name) eq '{literal}'"
+
+            # Starts with
+            if regex.startswith("^"):
+                remaining = regex[1:]
+                parts = remaining.split(".*", 1)
+                literal = parts[0] if parts else ""
+                return f"startswith(name, '{literal}')"
+
+            # Ends with
+            if regex.endswith("$"):
+                remaining = regex[:-1]
+                parts = remaining.split(".*")
+                literal = parts[-1] if parts else ""
+                return f"endswith(name, '{literal}')"
+
+            # Contains
+            if regex.startswith(".*") and regex.endswith(".*"):
+                parts = regex.split(".*")
+                literal = parts[1] if len(parts) > 1 else ""
+            else:
+                literal = regex
+            return f"contains(name, '{literal}')"
+        except Exception as exc:
+            logger.warning(
+                f"Error converting regex '{regex}' to OData condition: {exc}"
+            )
+            return ""
+
+    def create_filter_query(self, filter_pattern) -> Optional[str]:
+        """
+        Create a complete filter query for workspaces from filter_pattern
+        """
+        try:
+
+            validate_regex(filter_pattern.includes)
+            validate_regex(filter_pattern.excludes)
+            project_to_include = filter_pattern.includes
+            project_to_exclude = filter_pattern.excludes
+            filter_conditions = []
+            if project_to_include:
+                include_conditions = []
+                for pattern in project_to_include:
+                    condition = self.regex_to_odata_condition(pattern)
+                    if condition:
+                        include_conditions.append(f"{condition}")
+
+                if include_conditions:
+                    filter_conditions.append(f"{' or '.join(include_conditions)}")
+
+            if project_to_exclude:
+                exclude_conditions = []
+                for pattern in project_to_exclude:
+                    condition = self.regex_to_odata_condition(pattern)
+                    if condition:
+                        exclude_conditions.append(f"not({condition})")
+
+                if exclude_conditions:
+                    filter_conditions.append(f"{' and '.join(exclude_conditions)}")
+
+            filter_query = " and ".join(filter_conditions) if filter_conditions else ""
+            return filter_query if filter_query else None
+        except Exception as exc:
+            logger.warning(
+                f"Creating filter query from the project filter pattern failed: {exc}. "
+                "The projects will be filtered further inside OpenMetadata."
+            )
+            return None
+
     # pylint: disable=too-many-branches,too-many-statements
-    def fetch_all_workspaces(self) -> Optional[List[Group]]:
+    def fetch_all_workspaces(
+        self, filter_pattern: Optional[FilterPattern] = None
+    ) -> Optional[List[Group]]:
         """Method to fetch all powerbi workspace details
         Returns:
             Group
@@ -264,7 +350,13 @@ class PowerBiApiClient:
             api_url = f"/myorg/{admin}groups"
             entities_per_page = self.pagination_entity_per_page
             failed_indexes = []
-            params_data = GETGROUPS_DEFAULT_PARAMS
+            parsed_filter_query = None
+            if filter_pattern:
+                parsed_filter_query = self.create_filter_query(filter_pattern)
+                logger.debug(f"Filter query applied = {parsed_filter_query}")
+            params_data = deepcopy(GETGROUPS_DEFAULT_PARAMS)
+            if parsed_filter_query:
+                params_data["$filter"] = parsed_filter_query
             response = self.client.get(api_url, data=params_data)
             if (
                 not response
@@ -287,12 +379,19 @@ class PowerBiApiClient:
                     logger.warning(f"Error processing GetGroups response: {exc}")
                     count = 0
             indexes = math.ceil(count / entities_per_page)
+            logger.debug(
+                f"Total {count} workspaces found, Will run {indexes} iterations fetching"
+                f" maximum {entities_per_page} workspaces in a single iteration"
+            )
             workspaces = []
             for index in range(indexes):
                 params_data = {
                     "$top": str(entities_per_page),
                     "$skip": str(index * entities_per_page),
                 }
+                if parsed_filter_query:
+                    params_data["$filter"] = parsed_filter_query
+
                 response = self.client.get(api_url, data=params_data)
                 if (
                     not response