openmetadata-ingestion 1.6.6.6__py3-none-any.whl → 1.6.7.1__py3-none-any.whl

metadata/ingestion/source/database/singlestore/service_spec.py

@@ -1,6 +1,3 @@
-from metadata.ingestion.source.database.singlestore.lineage import (
-    SinglestoreLineageSource,
-)
 from metadata.ingestion.source.database.singlestore.metadata import SinglestoreSource
 from metadata.profiler.interface.sqlalchemy.single_store.profiler_interface import (
     SingleStoreProfilerInterface,
@@ -8,7 +5,5 @@ from metadata.profiler.interface.sqlalchemy.single_store.profiler_interface impo
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=SinglestoreSource,
-    profiler_class=SingleStoreProfilerInterface,
-    lineage_source_class=SinglestoreLineageSource,
+    metadata_source_class=SinglestoreSource, profiler_class=SingleStoreProfilerInterface
 )

metadata/ingestion/source/database/sqlite/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.sqlite.lineage import SqliteLineageSource
 from metadata.ingestion.source.database.sqlite.metadata import SqliteSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=SqliteSource,
-    lineage_source_class=SqliteLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=SqliteSource)

metadata/ingestion/source/database/teradata/service_spec.py

@@ -1,8 +1,4 @@
-from metadata.ingestion.source.database.teradata.lineage import TeradataLineageSource
 from metadata.ingestion.source.database.teradata.metadata import TeradataSource
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
-ServiceSpec = DefaultDatabaseSpec(
-    metadata_source_class=TeradataSource,
-    lineage_source_class=TeradataLineageSource,
-)
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=TeradataSource)

metadata/ingestion/source/storage/s3/metadata.py

@@ -305,7 +305,6 @@ class S3Source(StorageServiceSource):
                     client=self.s3_client,
                 )
             except Exception as err:
-                logger.warning()
                 self.status.failed(
                     error=StackTraceError(
                         name=f"{bucket_name}/{sample_key}",

metadata/profiler/orm/converter/converter_registry.py

@@ -19,6 +19,7 @@ from metadata.generated.schema.entity.services.databaseService import (
 )
 from metadata.profiler.orm.converter.bigquery.converter import BigqueryMapTypes
 from metadata.profiler.orm.converter.common import CommonMapTypes
+from metadata.profiler.orm.converter.mariadb.converter import MariaDBMapTypes
 from metadata.profiler.orm.converter.mssql.converter import MssqlMapTypes
 from metadata.profiler.orm.converter.redshift.converter import RedshiftMapTypes
 from metadata.profiler.orm.converter.snowflake.converter import SnowflakeMapTypes
@@ -28,3 +29,4 @@ converter_registry[DatabaseServiceType.BigQuery] = BigqueryMapTypes
 converter_registry[DatabaseServiceType.Snowflake] = SnowflakeMapTypes
 converter_registry[DatabaseServiceType.Redshift] = RedshiftMapTypes
 converter_registry[DatabaseServiceType.Mssql] = MssqlMapTypes
+converter_registry[DatabaseServiceType.MariaDB] = MariaDBMapTypes

metadata/profiler/orm/converter/mariadb/converter.py

@@ -0,0 +1,28 @@
+#  Copyright 2025 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Converter logic to transform an OpenMetadata Table Entity for MariaDB
+to an SQLAlchemy ORM class.
+"""
+
+from metadata.generated.schema.entity.data.table import DataType
+from metadata.profiler.orm.converter.common import CommonMapTypes
+from metadata.profiler.orm.registry import CustomTypes
+
+
+class MariaDBMapTypes(CommonMapTypes):
+    def __init__(self) -> None:
+        self._TYPE_MAP.update(
+            {
+                DataType.TIME: CustomTypes.TIME.value,
+            }
+        )

metadata/profiler/orm/registry.py

@@ -28,6 +28,7 @@ from metadata.profiler.orm.types.custom_datetimerange import CustomDateTimeRange
 from metadata.profiler.orm.types.custom_hex_byte_string import HexByteString
 from metadata.profiler.orm.types.custom_image import CustomImage
 from metadata.profiler.orm.types.custom_ip import CustomIP
+from metadata.profiler.orm.types.custom_time import CustomTime
 from metadata.profiler.orm.types.custom_timestamp import CustomTimestamp
 from metadata.profiler.orm.types.undetermined_type import UndeterminedType
 from metadata.profiler.orm.types.uuid import UUIDString
@@ -44,6 +45,7 @@ class CustomTypes(TypeRegistry):
     IP = CustomIP
     SQADATETIMERANGE = CustomDateTimeRange
     UNDETERMINED = UndeterminedType
+    TIME = CustomTime
 
 
 class PythonDialects(Enum):

metadata/profiler/orm/types/custom_time.py

@@ -0,0 +1,54 @@
+#  Copyright 2025 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# pylint: disable=abstract-method
+
+"""
+Expand sqlalchemy types to map them to OpenMetadata DataType
+"""
+import datetime
+
+from sqlalchemy.sql.sqltypes import TIME, TypeDecorator
+
+from metadata.utils.logger import profiler_logger
+
+logger = profiler_logger()
+
+
+class CustomTime(TypeDecorator):
+    """
+    Convert int time to timedelta object
+    """
+
+    __visit_name__ = "TIME"
+    impl = TIME
+    cache_ok = True
+
+    def result_processor(self, dialect, coltype):
+        time = datetime.time
+
+        def process(value):
+            # convert from a timedelta value
+            if value is not None:
+                if isinstance(value, int):
+                    value = datetime.timedelta(seconds=value)
+                microseconds = value.microseconds
+                seconds = value.seconds
+                minutes = seconds // 60
+                return time(
+                    minutes // 60,
+                    minutes % 60,
+                    seconds - minutes * 60,
+                    microsecond=microseconds,
+                )
+            return None
+
+        return process

metadata/sampler/partition.py

@@ -26,9 +26,6 @@ from metadata.generated.schema.entity.services.databaseService import (
     DatabaseServiceType,
 )
 from metadata.sampler.models import TableConfig
-from metadata.utils.logger import sampler_logger
-
-logger = sampler_logger()
 
 
 def validate_athena_injected_partitioning(
@@ -167,12 +164,8 @@ def _handle_bigquery_partition(
                 partitionIntegerRangeEnd=10000,
             )
         # TODO: Allow External Hive Partitioning for profiler
-        logger.info(
-            f"Unknown partition type {partition.intervalType}. Cannot infer partition. Partitionning "
-            "will be set to None. If partitionning is required for your table, you can set it "
-            "directly in the table config. More information can be found at "
-            "https://docs.open-metadata.org/latest/how-to-guides/data-quality-observability/profiler/"
-            "workflow#4.-updating-profiler-setting-at-the-table-level"
+        raise TypeError(
+            f"Unsupported partition type {partition.intervalType}. Skipping table"
         )
 
     return None

metadata/utils/db_utils.py

@@ -18,6 +18,9 @@ from typing import Iterable
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseServiceType,
+)
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
@@ -33,6 +36,8 @@ from metadata.utils.logger import utils_logger
 
 logger = utils_logger()
 
+PUBLIC_SCHEMA = "public"
+
 
 def get_host_from_host_port(uri: str) -> str:
     """
@@ -64,7 +69,7 @@ def get_view_lineage(
         schema_name=schema_name,
         table_name=table_name,
     )
-    table_entity = metadata.get_by_name(
+    table_entity: Table = metadata.get_by_name(
         entity=Table,
         fqn=table_fqn,
     )
@@ -79,6 +84,11 @@ def get_view_lineage(
         lineage_parser = LineageParser(
             view_definition, dialect, timeout_seconds=timeout_seconds
         )
+
+        if table_entity.serviceType == DatabaseServiceType.Postgres:
+            # For Postgres, if schema is not defined, we need to use the public schema
+            schema_name = PUBLIC_SCHEMA
+
         if lineage_parser.source_tables and lineage_parser.target_tables:
             yield from get_lineage_by_query(
                 metadata,