openmetadata-ingestion 1.6.6.4__py3-none-any.whl → 1.6.7.0__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -95,30 +95,6 @@ class PowerBiColumns(BaseModel):
     name: str
     dataType: Optional[str] = None
     columnType: Optional[str] = None
-    description: Optional[str] = None
-
-
-class PowerBiMeasureModel(BaseModel):
-    """
-    Represents a Power BI measure, used before converting to a Column instance.
-    """
-
-    dataType: str
-    dataTypeDisplay: str
-    name: str
-    description: str
-
-
-class PowerBiMeasures(BaseModel):
-    """
-    PowerBI Column Model
-    Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/push-datasets/datasets-get-tables-in-group#measure
-    """
-
-    name: str
-    expression: str
-    description: Optional[str] = None
-    isHidden: bool
 
 
 class PowerBITableSource(BaseModel):
@@ -137,7 +113,6 @@ class PowerBiTable(BaseModel):
 
     name: str
     columns: Optional[List[PowerBiColumns]] = None
-    measures: Optional[List[PowerBiMeasures]] = None
     description: Optional[str] = None
     source: Optional[List[PowerBITableSource]] = None
 

metadata/ingestion/source/database/db2/service_spec.py

@@ -1,4 +1,3 @@
-from metadata.ingestion.source.database.db2.lineage import Db2LineageSource
 from metadata.ingestion.source.database.db2.metadata import Db2Source
 from metadata.profiler.interface.sqlalchemy.db2.profiler_interface import (
     DB2ProfilerInterface,
@@ -6,7 +5,5 @@ from metadata.profiler.interface.sqlalchemy.db2.profiler_interface import (
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=Db2Source,
-    profiler_class=DB2ProfilerInterface,
-    lineage_source_class=Db2LineageSource,
+    metadata_source_class=Db2Source, profiler_class=DB2ProfilerInterface
 )

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -353,40 +353,103 @@ def _(config: DbtS3Config):
 def _(config: DbtGcsConfig):
     try:
         bucket_name, prefix = get_dbt_prefix_config(config)
-        from google.cloud import storage  # pylint: disable=import-outside-toplevel
+        # pylint: disable=import-outside-toplevel
+        from google.auth.exceptions import DefaultCredentialsError, GoogleAuthError
+        from google.cloud import storage
 
-        set_google_credentials(gcp_credentials=config.dbtSecurityConfig)
+        try:
+            set_google_credentials(
+                gcp_credentials=config.dbtSecurityConfig, single_project=True
+            )
+        except (ValueError, GoogleAuthError) as cred_exc:
+            logger.error(
+                f"Failed to set Google Cloud credentials: {str(cred_exc)}. "
+                "Please ensure your credentials are properly formatted and valid."
+            )
+            raise DBTConfigException(
+                "Invalid Google Cloud credentials. Please check the format and validity of your credentials."
+            ) from cred_exc
+
+        try:
+            client = storage.Client()
+        except DefaultCredentialsError as client_exc:
+            logger.error(
+                f"Failed to create Google Cloud Storage client: {str(client_exc)}. "
+                "Please ensure you have valid credentials configured."
+            )
+            raise DBTConfigException(
+                "Failed to initialize Google Cloud Storage client. Please verify your credentials."
+            ) from client_exc
 
-        client = storage.Client()
         if not bucket_name:
-            buckets = client.list_buckets()
+            try:
+                buckets = client.list_buckets()
+            except Exception as bucket_exc:
+                logger.error(f"Failed to list GCS buckets: {str(bucket_exc)}")
+                raise DBTConfigException(
+                    "Unable to list GCS buckets. Please check your permissions and credentials."
+                ) from bucket_exc
         else:
-            buckets = [client.get_bucket(bucket_name)]
+            try:
+                buckets = [client.get_bucket(bucket_name)]
+            except Exception as bucket_exc:
+                logger.error(
+                    f"Failed to access GCS bucket {bucket_name}: {str(bucket_exc)}"
+                )
+                raise DBTConfigException(
+                    f"Unable to access GCS bucket {bucket_name}."
+                    "Please verify the bucket exists and you have proper permissions."
+                ) from bucket_exc
+
         for bucket in buckets:
-            if prefix:
-                obj_list = client.list_blobs(bucket.name, prefix=prefix)
-            else:
-                obj_list = client.list_blobs(bucket.name)
+            try:
+                obj_list = client.list_blobs(
+                    bucket.name, prefix=prefix if prefix else None
+                )
 
-            yield from download_dbt_files(
-                blob_grouped_by_directory=get_blobs_grouped_by_dir(
-                    blobs=[blob.name for blob in obj_list]
-                ),
-                config=config,
-                client=client,
-                bucket_name=bucket.name,
-            )
+                yield from download_dbt_files(
+                    blob_grouped_by_directory=get_blobs_grouped_by_dir(
+                        blobs=[blob.name for blob in obj_list]
+                    ),
+                    config=config,
+                    client=client,
+                    bucket_name=bucket.name,
+                )
+            except Exception as blob_exc:
+                logger.error(
+                    f"Failed to process blobs in bucket {bucket.name}: {str(blob_exc)}"
+                )
+                logger.debug(traceback.format_exc())
+
+    except DBTConfigException as dbt_exc:
+        raise dbt_exc
     except Exception as exc:
         logger.debug(traceback.format_exc())
-        raise DBTConfigException(f"Error fetching dbt files from gcs: {exc}")
+        raise DBTConfigException(f"Error fetching dbt files from GCS: {exc}")
 
 
 @get_dbt_details.register
 def _(config: DbtAzureConfig):
     try:
         bucket_name, prefix = get_dbt_prefix_config(config)
+        # pylint: disable=import-outside-toplevel
+        from azure.core.exceptions import AzureError, ClientAuthenticationError
 
-        client = AzureClient(config.dbtSecurityConfig).create_blob_client()
+        try:
+            client = AzureClient(config.dbtSecurityConfig).create_blob_client()
+        except ClientAuthenticationError as auth_exc:
+            logger.error(
+                f"Failed to authenticate with Azure: {str(auth_exc)}. "
+                "Please check your Azure credentials and permissions."
+            )
+            raise DBTConfigException(
+                "Authentication failed with Azure. Please verify your credentials and permissions."
+            ) from auth_exc
+        except AzureError as azure_exc:
+            logger.error(f"Failed to create Azure client: {str(azure_exc)}")
+            raise DBTConfigException(
+                "Failed to initialize Azure client. Please check your Azure configuration."
+            ) from azure_exc
 
         if not bucket_name:
             container_dicts = client.list_containers()

metadata/ingestion/source/database/druid/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.druid.lineage import DruidLineageSource
 from metadata.ingestion.source.database.druid.metadata import DruidSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=DruidSource,
-    lineage_source_class=DruidLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=DruidSource)

metadata/ingestion/source/database/greenplum/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.greenplum.lineage import GreenplumLineageSource
 from metadata.ingestion.source.database.greenplum.metadata import GreenplumSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=GreenplumSource,
-    lineage_source_class=GreenplumLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=GreenplumSource)

metadata/ingestion/source/database/hive/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.hive.lineage import HiveLineageSource
 from metadata.ingestion.source.database.hive.metadata import HiveSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=HiveSource,
-    lineage_source_class=HiveLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=HiveSource)

metadata/ingestion/source/database/impala/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.impala.lineage import ImpalaLineageSource
 from metadata.ingestion.source.database.impala.metadata import ImpalaSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=ImpalaSource,
-    lineage_source_class=ImpalaLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=ImpalaSource)

metadata/ingestion/source/database/lineage_source.py

@@ -22,7 +22,6 @@ from typing import Any, Callable, Iterable, Iterator, List, Union
 
 from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
-from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.type.basic import FullyQualifiedEntityName, SqlQuery
 from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.api.models import Either
@@ -44,6 +43,8 @@ logger = ingestion_logger()
 
 CHUNK_SIZE = 200
 
+THREAD_TIMEOUT = 600
+
 
 class LineageSource(QueryParserSource, ABC):
     """
@@ -152,7 +153,11 @@ class LineageSource(QueryParserSource, ABC):
 
                 for i, future in enumerate(futures):
                     if future.done():
-                        future.result()
+                        try:
+                            future.result(timeout=THREAD_TIMEOUT)
+                        except Exception as e:
+                            logger.debug(f"Error in future: {e}")
+                            logger.debug(traceback.format_exc())
                         futures.pop(i)
 
             time.sleep(0.01)
@@ -245,6 +250,7 @@ class LineageSource(QueryParserSource, ABC):
         Based on the query logs, prepare the lineage
         and send it to the sink
         """
+        logger.info("Processing Query Lineage")
         connection_type = str(self.service_connection.type.value)
         self.dialect = ConnectionTypeDialectMapper.dialect_of(connection_type)
         producer_fn = self.get_table_query

metadata/ingestion/source/database/mariadb/service_spec.py

@@ -1,12 +1,4 @@
-from metadata.ingestion.source.database.mariadb.lineage import MariadbLineageSource
 from metadata.ingestion.source.database.mariadb.metadata import MariadbSource
-from metadata.profiler.interface.sqlalchemy.mariadb.profiler_interface import (
-    MariaDBProfilerInterface,
-)
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=MariadbSource,
-    lineage_source_class=MariadbLineageSource,
-    profiler_class=MariaDBProfilerInterface,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=MariadbSource)

metadata/ingestion/source/database/pinotdb/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.pinotdb.lineage import PinotdbLineageSource
 from metadata.ingestion.source.database.pinotdb.metadata import PinotdbSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=PinotdbSource,
-    lineage_source_class=PinotdbLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=PinotdbSource)

metadata/ingestion/source/database/singlestore/service_spec.py

@@ -1,6 +1,3 @@
-from metadata.ingestion.source.database.singlestore.lineage import (
-    SinglestoreLineageSource,
-)
 from metadata.ingestion.source.database.singlestore.metadata import SinglestoreSource
 from metadata.profiler.interface.sqlalchemy.single_store.profiler_interface import (
     SingleStoreProfilerInterface,
@@ -8,7 +5,5 @@ from metadata.profiler.interface.sqlalchemy.single_store.profiler_interface impo
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=SinglestoreSource,
-    profiler_class=SingleStoreProfilerInterface,
-    lineage_source_class=SinglestoreLineageSource,
+    metadata_source_class=SinglestoreSource, profiler_class=SingleStoreProfilerInterface
 )

metadata/ingestion/source/database/snowflake/lineage.py

@@ -49,6 +49,7 @@ class SnowflakeLineageSource(
             OR (QUERY_TYPE = 'INSERT' and query_text ILIKE '%%insert%%into%%select%%')
             OR (QUERY_TYPE = 'ALTER' and query_text ILIKE '%%alter%%table%%swap%%')
             OR (QUERY_TYPE = 'CREATE_TABLE' and query_text ILIKE '%%clone%%')
+            OR (QUERY_TYPE = 'CREATE_VIEW' and query_text ILIKE '%%create%%temporary%%view%%')
         )
     """
 

metadata/ingestion/source/database/sqlite/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.sqlite.lineage import SqliteLineageSource
 from metadata.ingestion.source.database.sqlite.metadata import SqliteSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=SqliteSource,
-    lineage_source_class=SqliteLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=SqliteSource)

metadata/ingestion/source/database/teradata/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.teradata.lineage import TeradataLineageSource
 from metadata.ingestion.source.database.teradata.metadata import TeradataSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=TeradataSource,
-    lineage_source_class=TeradataLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=TeradataSource)

metadata/ingestion/source/storage/s3/metadata.py

@@ -305,7 +305,6 @@ class S3Source(StorageServiceSource):
                     client=self.s3_client,
                 )
             except Exception as err:
-                logger.warning()
                 self.status.failed(
                     error=StackTraceError(
                         name=f"{bucket_name}/{sample_key}",

metadata/profiler/orm/converter/converter_registry.py

@@ -19,6 +19,7 @@ from metadata.generated.schema.entity.services.databaseService import (
 )
 from metadata.profiler.orm.converter.bigquery.converter import BigqueryMapTypes
 from metadata.profiler.orm.converter.common import CommonMapTypes
+from metadata.profiler.orm.converter.mariadb.converter import MariaDBMapTypes
 from metadata.profiler.orm.converter.mssql.converter import MssqlMapTypes
 from metadata.profiler.orm.converter.redshift.converter import RedshiftMapTypes
 from metadata.profiler.orm.converter.snowflake.converter import SnowflakeMapTypes
@@ -28,3 +29,4 @@ converter_registry[DatabaseServiceType.BigQuery] = BigqueryMapTypes
 converter_registry[DatabaseServiceType.Snowflake] = SnowflakeMapTypes
 converter_registry[DatabaseServiceType.Redshift] = RedshiftMapTypes
 converter_registry[DatabaseServiceType.Mssql] = MssqlMapTypes
+converter_registry[DatabaseServiceType.MariaDB] = MariaDBMapTypes

metadata/profiler/orm/converter/mariadb/converter.py

@@ -0,0 +1,28 @@
+#  Copyright 2025 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Converter logic to transform an OpenMetadata Table Entity for MariaDB
+to an SQLAlchemy ORM class.
+"""
+
+from metadata.generated.schema.entity.data.table import DataType
+from metadata.profiler.orm.converter.common import CommonMapTypes
+from metadata.profiler.orm.registry import CustomTypes
+
+
+class MariaDBMapTypes(CommonMapTypes):
+    def __init__(self) -> None:
+        self._TYPE_MAP.update(
+            {
+                DataType.TIME: CustomTypes.TIME.value,
+            }
+        )

metadata/profiler/orm/registry.py

@@ -28,6 +28,7 @@ from metadata.profiler.orm.types.custom_datetimerange import CustomDateTimeRange
 from metadata.profiler.orm.types.custom_hex_byte_string import HexByteString
 from metadata.profiler.orm.types.custom_image import CustomImage
 from metadata.profiler.orm.types.custom_ip import CustomIP
+from metadata.profiler.orm.types.custom_time import CustomTime
 from metadata.profiler.orm.types.custom_timestamp import CustomTimestamp
 from metadata.profiler.orm.types.undetermined_type import UndeterminedType
 from metadata.profiler.orm.types.uuid import UUIDString
@@ -44,6 +45,7 @@ class CustomTypes(TypeRegistry):
     IP = CustomIP
     SQADATETIMERANGE = CustomDateTimeRange
     UNDETERMINED = UndeterminedType
+    TIME = CustomTime
 
 
 class PythonDialects(Enum):

metadata/profiler/orm/types/custom_time.py

@@ -0,0 +1,54 @@
+#  Copyright 2025 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# pylint: disable=abstract-method
+
+"""
+Expand sqlalchemy types to map them to OpenMetadata DataType
+"""
+import datetime
+
+from sqlalchemy.sql.sqltypes import TIME, TypeDecorator
+
+from metadata.utils.logger import profiler_logger
+
+logger = profiler_logger()
+
+
+class CustomTime(TypeDecorator):
+    """
+    Convert int time to timedelta object
+    """
+
+    __visit_name__ = "TIME"
+    impl = TIME
+    cache_ok = True
+
+    def result_processor(self, dialect, coltype):
+        time = datetime.time
+
+        def process(value):
+            # convert from a timedelta value
+            if value is not None:
+                if isinstance(value, int):
+                    value = datetime.timedelta(seconds=value)
+                microseconds = value.microseconds
+                seconds = value.seconds
+                minutes = seconds // 60
+                return time(
+                    minutes // 60,
+                    minutes % 60,
+                    seconds - minutes * 60,
+                    microsecond=microseconds,
+                )
+            return None
+
+        return process

metadata/sampler/partition.py

@@ -26,9 +26,6 @@ from metadata.generated.schema.entity.services.databaseService import (
     DatabaseServiceType,
 )
 from metadata.sampler.models import TableConfig
-from metadata.utils.logger import sampler_logger
-
-logger = sampler_logger()
 
 
 def validate_athena_injected_partitioning(
@@ -167,12 +164,8 @@ def _handle_bigquery_partition(
                 partitionIntegerRangeEnd=10000,
             )
         # TODO: Allow External Hive Partitioning for profiler
-        logger.info(
-            f"Unknown partition type {partition.intervalType}. Cannot infer partition. Partitionning "
-            "will be set to None. If partitionning is required for your table, you can set it "
-            "directly in the table config. More information can be found at "
-            "https://docs.open-metadata.org/latest/how-to-guides/data-quality-observability/profiler/"
-            "workflow#4.-updating-profiler-setting-at-the-table-level"
+        raise TypeError(
+            f"Unsupported partition type {partition.intervalType}. Skipping table"
         )
 
     return None