openmetadata-ingestion 1.6.4.0.dev0__py3-none-any.whl → 1.6.5.1__py3-none-any.whl

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-03-04T10:09:33+00:00
+#   timestamp: 2025-03-05T14:24:08+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/dashboard/lightdash/client.py

@@ -19,6 +19,7 @@ from metadata.ingestion.source.dashboard.lightdash.models import (
     LightdashChart,
     LightdashDashboard,
 )
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
@@ -34,7 +35,7 @@ class LightdashApiClient:
     def __init__(self, config):
         self.config = config
         client_config = ClientConfig(
-            base_url=str(self.config.hostPort),
+            base_url=clean_uri(self.config.hostPort),
             api_version="",
             access_token=self.config.apiKey.get_secret_value(),
             auth_header="Authorization",

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -780,8 +780,10 @@ class LookerSource(DashboardServiceSource):
         try:
             if dashboard_details.user_id is not None:
                 dashboard_owner = self.client.user(dashboard_details.user_id)
-                return self.metadata.get_reference_by_email(dashboard_owner.email)
-
+                if dashboard_owner.email:
+                    return self.metadata.get_reference_by_email(
+                        dashboard_owner.email.lower()
+                    )
         except Exception as err:
             logger.debug(traceback.format_exc())
             logger.warning(f"Could not fetch owner data due to {err}")

metadata/ingestion/source/dashboard/mode/client.py

@@ -18,6 +18,7 @@ from typing import Optional
 from requests._internal_utils import to_native_string
 
 from metadata.ingestion.ometa.client import REST, ClientConfig
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
@@ -50,7 +51,7 @@ class ModeApiClient:
     def __init__(self, config):
         self.config = config
         client_config = ClientConfig(
-            base_url=str(config.hostPort),
+            base_url=clean_uri(config.hostPort),
             api_version="api",
             auth_header="Authorization",
             auth_token_mode="Basic",

metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -59,7 +59,7 @@ class QlikCloudClient:
         self.socket_connection = None
 
         client_config: ClientConfig = ClientConfig(
-            base_url=str(self.config.hostPort),
+            base_url=clean_uri(self.config.hostPort),
             api_version=API_VERSION,
             auth_header=AUTHORIZATION_HEADER,
             auth_token=lambda: (self.config.token.get_secret_value(), 0),

metadata/ingestion/source/dashboard/sigma/client.py

@@ -94,14 +94,39 @@ class SigmaApiClient:
         )
         return result.access_token, result.expires_in
 
-    def get_dashboards(self) -> Optional[List[Workbook]]:
+    def test_get_dashboards(self) -> Optional[List[Workbook]]:
         """
-        method to fetch dashboards from api
+        method to test fetch dashboards from api
         """
+        result = self.client.get("/workbooks")
         result = WorkBookResponseDetails.model_validate(self.client.get("/workbooks"))
         if result:
             return result.entries
 
+    def get_dashboards(self) -> Optional[List[Workbook]]:
+        """
+        method to fetch dashboards from api
+        """
+        workbooks = []
+        try:
+            result = self.client.get("/workbooks")
+            result = WorkBookResponseDetails.model_validate(
+                self.client.get("/workbooks")
+            )
+            if result:
+                workbooks.extend(result.entries)
+                while result.nextPage:
+                    data = {"page": int(result.nextPage)}
+                    result = WorkBookResponseDetails.model_validate(
+                        self.client.get("/workbooks", data=data)
+                    )
+                    if result:
+                        workbooks.extend(result.entries)
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.error(f"Error fetching Dashboards: {exc}")
+        return workbooks
+
     def get_dashboard_detail(self, workbook_id: str) -> Optional[WorkbookDetails]:
         """
         method to fetch dashboard details from api
@@ -134,6 +159,36 @@ class SigmaApiClient:
             logger.warning(f"Failed to fetch owner details for owner {owner_id}: {exc}")
         return None
 
+    def get_page_elements(
+        self, workbook_id: str, page_id: str
+    ) -> Optional[List[Elements]]:
+        """
+        method to fetch dashboards page elements from api
+        """
+        elements = []
+        try:
+            result = ElementsResponse.model_validate(
+                self.client.get(f"/workbooks/{workbook_id}/pages/{page_id}/elements")
+            )
+            if result:
+                elements.extend(result.entries)
+                while result.nextPage:
+                    data = {"page": int(result.nextPage)}
+                    result = ElementsResponse.model_validate(
+                        self.client.get(
+                            f"/workbooks/{workbook_id}/pages/{page_id}/elements",
+                            data=data,
+                        )
+                    )
+                    if result:
+                        elements.extend(result.entries)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to fetch page elements for workbook {workbook_id}: {exc}"
+            )
+        return elements
+
     def get_chart_details(self, workbook_id: str) -> Optional[List[Elements]]:
         """
         method to fetch dashboards chart details from api
@@ -143,13 +198,21 @@ class SigmaApiClient:
             pages = WorkBookPageResponse.model_validate(
                 self.client.get(f"/workbooks/{workbook_id}/pages")
             )
-            for page in pages.entries:
-                elements = ElementsResponse.model_validate(
+            if not pages.entries:
+                return None
+            while pages.nextPage:
+                pages = WorkBookPageResponse.model_validate(
                     self.client.get(
-                        f"/workbooks/{workbook_id}/pages/{page.pageId}/elements"
+                        f"/workbooks/{workbook_id}/pages",
+                        data={"page": int(pages.nextPage)},
                     )
                 )
-                elements_list.extend(elements.entries or [])
+                if not pages.entries:
+                    break
+                for page in pages.entries:
+                    elements_list.extend(
+                        self.get_page_elements(workbook_id, page.pageId)
+                    )
             return elements_list
         except Exception as exc:  # pylint: disable=broad-except
             logger.debug(traceback.format_exc())

metadata/ingestion/source/dashboard/sigma/connection.py

@@ -56,7 +56,10 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    test_fn = {"GetToken": client.get_auth_token, "GetWorkbooks": client.get_dashboards}
+    test_fn = {
+        "GetToken": client.get_auth_token,
+        "GetWorkbooks": client.test_get_dashboards,
+    }
 
     return test_connection_steps(
         metadata=metadata,

metadata/ingestion/source/dashboard/sigma/models.py

@@ -42,6 +42,8 @@ class WorkbookDetails(BaseModel):
 
 class WorkBookResponseDetails(BaseModel):
     entries: Optional[List[Workbook]] = []
+    total: int
+    nextPage: Optional[str] = None
 
 
 class OwnerDetails(BaseModel):
@@ -55,6 +57,8 @@ class WorkBookPage(BaseModel):
 
 class WorkBookPageResponse(BaseModel):
     entries: Optional[List[WorkBookPage]] = []
+    total: int
+    nextPage: Optional[str] = None
 
 
 class Elements(BaseModel):
@@ -66,6 +70,8 @@ class Elements(BaseModel):
 
 class ElementsResponse(BaseModel):
     entries: Optional[List[Elements]] = []
+    total: int
+    nextPage: Optional[str] = None
 
 
 class EdgeSource(BaseModel):

metadata/ingestion/source/dashboard/superset/client.py

@@ -25,6 +25,7 @@ from metadata.ingestion.source.dashboard.superset.models import (
     SupersetDashboardCount,
     SupersetDatasource,
 )
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ometa_logger
 from metadata.utils.ssl_registry import get_verify_ssl_fn
 
@@ -41,7 +42,7 @@ class SupersetAuthenticationProvider(AuthenticationProvider):
         self.service_connection = self.config
         get_verify_ssl = get_verify_ssl_fn(config.connection.verifySSL)
         client_config = ClientConfig(
-            base_url=str(config.hostPort),
+            base_url=clean_uri(config.hostPort),
             api_version="api/v1",
             auth_token=lambda: ("no_token", 0),
             auth_header="Authorization",
@@ -90,7 +91,7 @@ class SupersetAPIClient:
         self._auth_provider = SupersetAuthenticationProvider.create(config)
         get_verify_ssl = get_verify_ssl_fn(config.connection.verifySSL)
         client_config = ClientConfig(
-            base_url=str(config.hostPort),
+            base_url=clean_uri(config.hostPort),
             api_version="api/v1",
             auth_token=self._auth_provider.get_access_token,
             auth_header="Authorization",

metadata/ingestion/source/database/databricks/metadata.py

@@ -765,8 +765,6 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
                 table_name=table_name,
                 schema=schema_name,
             )
-            found_location = False
-            log_location = None
             for result in list(cursor):
                 data = result.values()
                 if data[0] and data[0].strip() == "Comment":
@@ -779,12 +777,6 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
                         if data and data[1] and not data[1].startswith("dbfs")
                         else None
                     )
-                    found_location = True
-                    log_location = data[1]
-            if found_location and log_location:
-                logger.debug(f"Table {schema_name}.{table_name} has location {log_location}")
-            else:
-                logger.debug(f"Table {schema_name}.{table_name} does not have location")
 
         # Catch any exception without breaking the ingestion
         except Exception as exc:  # pylint: disable=broad-except

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -48,6 +48,7 @@ from metadata.ingestion.source.database.dbt.constants import (
 from metadata.ingestion.source.database.dbt.models import DbtFiles
 from metadata.readers.file.config_source_factory import get_reader
 from metadata.utils.credentials import set_google_credentials
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ometa_logger
 from metadata.utils.s3_utils import list_s3_objects
 
@@ -163,7 +164,7 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
         expiry = 0
         auth_token = config.dbtCloudAuthToken.get_secret_value(), expiry
         client_config = ClientConfig(
-            base_url=str(config.dbtCloudUrl),
+            base_url=clean_uri(config.dbtCloudUrl),
             api_version="api/v2",
             auth_token=lambda: auth_token,
             auth_header="Authorization",

metadata/ingestion/source/database/lineage_source.py

@@ -172,6 +172,7 @@ class LineageSource(QueryParserSource, ABC):
                 for row in rows:
                     query_dict = dict(row)
                     try:
+                        query_dict.update({k.lower(): v for k, v in query_dict.items()})
                         yield TableQuery(
                             dialect=self.dialect.value,
                             query=query_dict["query_text"],

metadata/ingestion/source/database/sas/client.py

@@ -19,6 +19,7 @@ from metadata.generated.schema.entity.services.connections.database.sasConnectio
     SASConnection,
 )
 from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -35,7 +36,7 @@ class SASClient:
             config.serverHost, config.username, config.password.get_secret_value()
         )
         client_config: ClientConfig = ClientConfig(
-            base_url=str(config.serverHost),
+            base_url=clean_uri(config.serverHost),
             auth_header="Authorization",
             auth_token=self.get_auth_token,
             api_version="",

metadata/ingestion/source/database/snowflake/lineage.py

@@ -87,6 +87,7 @@ class SnowflakeLineageSource(
             # further process of `yield_query_lineage`
             for row in rows:
                 query_dict = dict(row)
+                query_dict.update({k.lower(): v for k, v in query_dict.items()})
                 try:
                     yield TableQuery(
                         dialect=self.dialect.value,

metadata/ingestion/source/database/snowflake/metadata.py

@@ -115,6 +115,7 @@ from metadata.utils.tag_utils import get_ometa_tag_and_classification
 ischema_names["VARIANT"] = VARIANT
 ischema_names["GEOGRAPHY"] = create_sqlalchemy_type("GEOGRAPHY")
 ischema_names["GEOMETRY"] = create_sqlalchemy_type("GEOMETRY")
+ischema_names["VECTOR"] = create_sqlalchemy_type("VECTOR")
 
 logger = ingestion_logger()
 

metadata/ingestion/source/database/snowflake/utils.py

@@ -300,7 +300,8 @@ def get_schema_columns(self, connection, schema, **kw):
             sa_util.warn(
                 f"Did not recognize type '{coltype}' of column '{column_name}'"
             )
-            col_type = sqltypes.NULLTYPE
+            col_type = sqltypes.NullType
+            type_instance = col_type()
         else:
             if issubclass(col_type, FLOAT):
                 col_type_kw["precision"] = numeric_precision
@@ -310,8 +311,7 @@ def get_schema_columns(self, connection, schema, **kw):
                 col_type_kw["scale"] = numeric_scale
             elif issubclass(col_type, (sqltypes.String, sqltypes.BINARY)):
                 col_type_kw["length"] = character_maximum_length
-
-        type_instance = col_type(**col_type_kw)
+            type_instance = col_type(**col_type_kw)
 
         current_table_pks = schema_primary_keys.get(table_name)
 

metadata/ingestion/source/database/usage_source.py

@@ -121,6 +121,7 @@ class UsageSource(QueryParserSource, ABC):
                         for row in rows:
                             row = dict(row)
                             try:
+                                row.update({k.lower(): v for k, v in row.items()})
                                 logger.debug(f"Processing row: {query}")
                                 query_type = row.get("query_type")
                                 query = self.format_query(row["query_text"])

metadata/ingestion/source/metadata/atlas/client.py

@@ -18,6 +18,7 @@ from metadata.generated.schema.entity.services.connections.metadata.atlasConnect
     AtlasConnection,
 )
 from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
+from metadata.utils.helpers import clean_uri
 
 
 class AtlasClient:
@@ -31,7 +32,7 @@ class AtlasClient:
             config.username, config.password.get_secret_value()
         )
         client_config: ClientConfig = ClientConfig(
-            base_url=str(config.hostPort),
+            base_url=clean_uri(config.hostPort),
             auth_header="Authorization",
             api_version="api",
             auth_token=self.get_auth_token,

metadata/mixins/sqalchemy/sqa_mixin.py

@@ -24,6 +24,12 @@ from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
     DatabricksConnection,
 )
+from metadata.generated.schema.entity.services.connections.database.mariaDBConnection import (
+    MariaDBConnection,
+)
+from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
+    MysqlConnection,
+)
 from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
     SnowflakeType,
 )
@@ -79,21 +85,26 @@ class SQAInterfaceMixin(Root):
             )
 
     def set_catalog(self, session) -> None:
-        """Set catalog for the session. Right now only databricks and unity catalog requires it
+        """Set the catalog or database for the session.
 
         Args:
             session (Session): sqa session object
         """
-        if not isinstance(
+        if isinstance(
             self.service_connection_config,
             (UnityCatalogConnection, DatabricksConnection),
         ):
-            return
-        bind = session.get_bind()
-        bind.execute(
-            "USE CATALOG %(catalog)s;",
-            {"catalog": self.service_connection_config.catalog},
-        ).first()
+            session.get_bind().execute(
+                "USE CATALOG %(catalog)s;",
+                {"catalog": self.service_connection_config.catalog},
+            ).first()
+
+        if isinstance(
+            self.service_connection_config, (MysqlConnection, MariaDBConnection)
+        ):
+            session.get_bind().execute(
+                f"USE {self.table_entity.databaseSchema.name};",
+            )
 
     def close(self):
         """close session"""

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -297,7 +297,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             column: the column to compute the metrics against
             metrics: list of metrics to compute
         Returns:
-            dictionnary of results
+            dictionary of results
         """
 
         if not metrics:

metadata/profiler/metrics/window/first_quartile.py

@@ -88,7 +88,7 @@ class FirstQuartile(StaticMetric, PercentilMixin):
             except MemoryError:
                 logger.error(
                     f"Unable to compute Median for {self.col.name} due to memory constraints."
-                    f"We recommend using a smaller sample size or partitionning."
+                    f"We recommend using a smaller sample size or partitioning."
                 )
                 return None
             # check if nan

metadata/profiler/metrics/window/median.py

@@ -87,7 +87,7 @@ class Median(StaticMetric, PercentilMixin):
             except MemoryError:
                 logger.error(
                     f"Unable to compute Median for {self.col.name} due to memory constraints."
-                    f"We recommend using a smaller sample size or partitionning."
+                    f"We recommend using a smaller sample size or partitioning."
                 )
                 return None
             try:

metadata/profiler/metrics/window/third_quartile.py

@@ -88,7 +88,7 @@ class ThirdQuartile(StaticMetric, PercentilMixin):
             except MemoryError:
                 logger.error(
                     f"Unable to compute Median for {self.col.name} due to memory constraints."
-                    f"We recommend using a smaller sample size or partitionning."
+                    f"We recommend using a smaller sample size or partitioning."
                 )
                 return None
             # check if nan

metadata/sampler/sqlalchemy/bigquery/sampler.py

@@ -119,7 +119,7 @@ class BigQuerySampler(SQASampler):
             and self.raw_dataset_type != TableType.View
         ):
             return self._base_sample_query(column).cte(
-                f"{self.raw_dataset.__tablename__}_sample"
+                f"{self.get_sampler_table_name()}_sample"
             )
 
         return super().get_sample_query(column=column)

metadata/sampler/sqlalchemy/mssql/sampler.py

@@ -46,7 +46,7 @@ class MssqlSampler(SQASampler):
     def get_sample_query(self, *, column=None) -> CTE:
         """get query for sample data"""
         rnd = self._base_sample_query(column).cte(
-            f"{self.raw_dataset.__tablename__}_rnd"
+            f"{self.get_sampler_table_name()}_rnd"
         )
         query = self.client.query(rnd)
-        return query.cte(f"{self.raw_dataset.__tablename__}_sample")
+        return query.cte(f"{self.get_sampler_table_name()}_sample")

metadata/sampler/sqlalchemy/postgres/sampler.py

@@ -85,7 +85,7 @@ class PostgresSampler(SQASampler):
     def get_sample_query(self, *, column=None) -> Query:
         if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE:
             return self._base_sample_query(column).cte(
-                f"{self.raw_dataset.__tablename__}_rnd"
+                f"{self.get_sampler_table_name()}_rnd"
             )
 
         return super().get_sample_query(column=column)

metadata/sampler/sqlalchemy/snowflake/sampler.py

@@ -89,7 +89,7 @@ class SnowflakeSampler(SQASampler):
     def get_sample_query(self, *, column=None) -> CTE:
         """get query for sample data"""
         rnd = self._base_sample_query(column).cte(
-            f"{self.raw_dataset.__tablename__}_rnd"
+            f"{self.get_sampler_table_name()}_rnd"
         )
         query = self.client.query(rnd)
-        return query.cte(f"{self.raw_dataset.__tablename__}_sample")
+        return query.cte(f"{self.get_sampler_table_name()}_sample")