openmetadata-ingestion 1.6.6.5__py3-none-any.whl → 1.6.7.0__py3-none-any.whl

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-03-25T13:00:08+00:00
+#   timestamp: 2025-03-28T07:59:21+00:00
 
 from __future__ import annotations
 

metadata/ingestion/lineage/models.py

@@ -105,6 +105,7 @@ class Dialect(Enum):
     SQLITE = "sqlite"
     TERADATA = "teradata"
     TSQL = "tsql"
+    MARIADB = "mariadb"
 
 
 MAP_CONNECTION_TYPE_DIALECT: Dict[str, Dialect] = {
@@ -126,7 +127,7 @@ MAP_CONNECTION_TYPE_DIALECT: Dict[str, Dialect] = {
     str(MssqlType.Mssql.value): Dialect.TSQL,
     str(AzureSQLType.AzureSQL.value): Dialect.TSQL,
     str(TeradataType.Teradata.value): Dialect.TERADATA,
-    str(MariaDBType.MariaDB.value): Dialect.MYSQL,
+    str(MariaDBType.MariaDB.value): Dialect.MARIADB,
     str(SingleStoreType.SingleStore.value): Dialect.MYSQL,
 }
 

metadata/ingestion/lineage/sql_lineage.py

@@ -54,6 +54,7 @@ from metadata.utils.lru_cache import LRU_CACHE_SIZE, LRUCache
 
 logger = utils_logger()
 DEFAULT_SCHEMA_NAME = "<default>"
+CUTOFF_NODES = 20
 
 
 def get_column_fqn(table_entity: Table, column: str) -> Optional[str]:
@@ -880,8 +881,9 @@ def _get_paths_from_subtree(subtree: DiGraph) -> List[List[Any]]:
 
     # Find all simple paths from each root to each leaf
     for root in root_nodes:
+        logger.debug(f"Processing root node {root}")
         for leaf in leaf_nodes:
-            paths.extend(nx.all_simple_paths(subtree, root, leaf))
+            paths.extend(nx.all_simple_paths(subtree, root, leaf, cutoff=CUTOFF_NODES))
     return paths
 
 
@@ -903,6 +905,9 @@ def get_lineage_by_graph(
     if graph is None:
         return
 
+    logger.info(
+        f"Processing graph with {graph.number_of_nodes()} nodes and {graph.number_of_edges()} edges"
+    )
     # Get all weakly connected components
     components = list(nx.weakly_connected_components(graph))
 

metadata/ingestion/models/custom_basemodel_validation.py

@@ -20,22 +20,27 @@ logger = logging.getLogger("metadata")
 RESTRICTED_KEYWORDS = ["::", ">"]
 RESERVED_COLON_KEYWORD = "__reserved__colon__"
 RESERVED_ARROW_KEYWORD = "__reserved__arrow__"
+RESERVED_QUOTE_KEYWORD = "__reserved__quote__"
 
 CREATE_ADJACENT_MODELS = {"ProfilerResponse", "SampleData"}
 NAME_FIELDS = {"EntityName", "str", "ColumnName", "TableData"}
-FETCH_MODELS = {"Table", "CustomColumnName"}
+FETCH_MODELS = {"Table", "CustomColumnName", "DashboardDataModel"}
 FIELD_NAMES = {"name", "columns", "root"}
 
 
 def revert_separators(value):
-    return value.replace(RESERVED_COLON_KEYWORD, "::").replace(
-        RESERVED_ARROW_KEYWORD, ">"
+    return (
+        value.replace(RESERVED_COLON_KEYWORD, "::")
+        .replace(RESERVED_ARROW_KEYWORD, ">")
+        .replace(RESERVED_QUOTE_KEYWORD, '"')
     )
 
 
 def replace_separators(value):
-    return value.replace("::", RESERVED_COLON_KEYWORD).replace(
-        ">", RESERVED_ARROW_KEYWORD
+    return (
+        value.replace("::", RESERVED_COLON_KEYWORD)
+        .replace(">", RESERVED_ARROW_KEYWORD)
+        .replace('"', RESERVED_QUOTE_KEYWORD)
     )
 
 

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -171,9 +171,13 @@ class OMetaLineageMixin(Generic[T]):
 
         except APIError as err:
             logger.debug(traceback.format_exc())
-            error = f"Error {err.status_code} trying to PUT lineage for {data.model_dump_json()}: {str(err)}"
-            logger.error(error)
-            return {"error": error}
+            logger.error(
+                "Error %s trying to PUT lineage for %s: %s",
+                err.status_code,
+                data.model_dump_json(),
+                str(err),
+            )
+            raise err
 
         from_entity_lineage = self.get_lineage_by_id(
             data.edge.fromEntity.type, str(data.edge.fromEntity.id.root)
@@ -393,10 +397,6 @@ class OMetaLineageMixin(Generic[T]):
                     resp = self.add_lineage(
                         lineage_request.right, check_patch=check_patch
                     )
-                    if resp.get("error"):
-                        logger.error(resp["error"])
-                        continue
-
                     entity_name = resp.get("entity", {}).get("name")
                     for node in resp.get("nodes", []):
                         logger.info(
@@ -407,37 +407,11 @@ class OMetaLineageMixin(Generic[T]):
                         f"Error while adding lineage: {lineage_request.left.error}"
                     )
 
-    def _set_permissions(self):
-        try:
-            if not hasattr(self, "is_edit_table_allowed"):
-                permissions = self.client.get(path="/permissions")
-                for permission in permissions.get("data", []):
-                    if permission.get("resource") != "table":
-                        continue
-                    for action in permission.get("permissions", []):
-                        if (
-                            action.get("operation") == "EditAll"
-                            and action.get("access") == "allow"
-                        ):
-                            self.is_edit_table_allowed = True
-                            return
-                logger.warning("Please grant EditAll permission to the user")
-        except Exception as exc:
-            logger.debug(f"Error while setting permissions: {exc}")
-            logger.debug(traceback.format_exc())
-        self.is_edit_table_allowed = False
-
     def patch_lineage_processed_flag(
         self,
         entity: Type[T],
         fqn: str,
     ) -> None:
-        """
-        Patch the processed lineage flag for an entity.
-        """
-        self._set_permissions()
-        if not self.is_edit_table_allowed:
-            return
 
         try:
             original_entity = self.get_by_name(entity=entity, fqn=fqn)

metadata/ingestion/ometa/mixins/table_mixin.py

@@ -13,6 +13,7 @@ Mixin class containing Table specific methods
 
 To be used by OpenMetadata class
 """
+import base64
 import traceback
 from typing import List, Optional, Type, TypeVar
 
@@ -56,6 +57,7 @@ class OMetaTableMixin:
 
     client: REST
 
+    # pylint: disable=too-many-nested-blocks
     def ingest_table_sample_data(
         self, table: Table, sample_data: TableData
     ) -> Optional[TableData]:
@@ -67,9 +69,37 @@ class OMetaTableMixin:
         """
         resp = None
         try:
+            # Pre-process sample data to handle binary/non-UTF-8 data before serialization
+            if sample_data and sample_data.rows:
+
+                for row in sample_data.rows:
+                    if not row:
+                        continue
+                    for col_idx, value in enumerate(row):
+                        # Handle binary data explicitly
+                        if isinstance(value, bytes):
+                            # Convert binary data to Base64-encoded string
+                            try:
+                                row[
+                                    col_idx
+                                ] = f"[base64]{base64.b64encode(value).decode('ascii', errors='ignore')}"
+                            except Exception as _:
+                                row[col_idx] = f"[binary]{value}"
+
+            try:
+                data = sample_data.model_dump_json()
+            except Exception as _:
+                logger.debug(traceback.format_exc())
+                logger.warning(
+                    f"Error serializing sample data for {table.fullyQualifiedName.root}"
+                    " please check if the data is valid"
+                )
+                return None
+
+            # Now safely serialize to JSON
             resp = self.client.put(
                 f"{self.get_suffix(Table)}/{table.id.root}/sampleData",
-                data=sample_data.model_dump_json(),
+                data=data,
             )
         except Exception as exc:
             logger.debug(traceback.format_exc())

metadata/ingestion/sink/metadata_rest.py

@@ -250,13 +250,6 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
     @_run_dispatch.register
     def write_lineage(self, add_lineage: AddLineageRequest) -> Either[Dict[str, Any]]:
         created_lineage = self.metadata.add_lineage(add_lineage, check_patch=True)
-        if created_lineage.get("error"):
-            return Either(
-                left=StackTraceError(
-                    name="AddLineageRequestError", error=created_lineage["error"]
-                )
-            )
-
         return Either(right=created_lineage["entity"]["fullyQualifiedName"])
 
     @_run_dispatch.register

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -23,7 +23,6 @@ from pydantic import BaseModel, ConfigDict
 from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
     PowerBIConnection,
 )
-from metadata.generated.schema.type.filterPattern import FilterPattern
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.source.dashboard.powerbi.file_client import PowerBiFileClient
@@ -44,7 +43,6 @@ from metadata.ingestion.source.dashboard.powerbi.models import (
     Workspaces,
     WorkSpaceScanResponse,
 )
-from metadata.utils.filters import validate_regex
 from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
@@ -255,91 +253,8 @@ class PowerBiApiClient:
 
         return None
 
-    def regex_to_odata_condition(self, regex: str) -> str:
-        """
-        Convert a regex pattern to an OData filter condition
-        """
-        try:
-            # Handle empty pattern
-            if not regex:
-                return ""
-
-            # Exact match
-            if regex.startswith("^") and regex.endswith("$"):
-                literal = regex[1:-1]
-                return f"trim(name) eq '{literal}'"
-
-            # Starts with
-            if regex.startswith("^"):
-                remaining = regex[1:]
-                parts = remaining.split(".*", 1)
-                literal = parts[0] if parts else ""
-                return f"startswith(name, '{literal}')"
-
-            # Ends with
-            if regex.endswith("$"):
-                remaining = regex[:-1]
-                parts = remaining.split(".*")
-                literal = parts[-1] if parts else ""
-                return f"endswith(name, '{literal}')"
-
-            # Contains
-            if regex.startswith(".*") and regex.endswith(".*"):
-                parts = regex.split(".*")
-                literal = parts[1] if len(parts) > 1 else ""
-            else:
-                literal = regex
-            return f"contains(name, '{literal}')"
-        except Exception as exc:
-            logger.warning(
-                f"Error converting regex '{regex}' to OData condition: {exc}"
-            )
-            return ""
-
-    def create_filter_query(self, filter_pattern) -> Optional[str]:
-        """
-        Create a complete filter query for workspaces from filter_pattern
-        """
-        try:
-
-            validate_regex(filter_pattern.includes)
-            validate_regex(filter_pattern.excludes)
-            project_to_include = filter_pattern.includes
-            project_to_exclude = filter_pattern.excludes
-            filter_conditions = []
-            if project_to_include:
-                include_conditions = []
-                for pattern in project_to_include:
-                    condition = self.regex_to_odata_condition(pattern)
-                    if condition:
-                        include_conditions.append(f"{condition}")
-
-                if include_conditions:
-                    filter_conditions.append(f"{' or '.join(include_conditions)}")
-
-            if project_to_exclude:
-                exclude_conditions = []
-                for pattern in project_to_exclude:
-                    condition = self.regex_to_odata_condition(pattern)
-                    if condition:
-                        exclude_conditions.append(f"not({condition})")
-
-                if exclude_conditions:
-                    filter_conditions.append(f"{' and '.join(exclude_conditions)}")
-
-            filter_query = " and ".join(filter_conditions) if filter_conditions else ""
-            return filter_query if filter_query else None
-        except Exception as exc:
-            logger.warning(
-                f"Creating filter query from the project filter pattern failed: {exc}. "
-                "The projects will be filtered further inside OpenMetadata."
-            )
-            return None
-
     # pylint: disable=too-many-branches,too-many-statements
-    def fetch_all_workspaces(
-        self, filter_pattern: Optional[FilterPattern] = None
-    ) -> Optional[List[Group]]:
+    def fetch_all_workspaces(self) -> Optional[List[Group]]:
         """Method to fetch all powerbi workspace details
         Returns:
             Group
@@ -378,9 +293,6 @@ class PowerBiApiClient:
                     "$top": str(entities_per_page),
                     "$skip": str(index * entities_per_page),
                 }
-                if filter_pattern:
-                    params_data["$filter"] = self.create_filter_query(filter_pattern)
-
                 response = self.client.get(api_url, data=params_data)
                 if (
                     not response