openmetadata-ingestion 1.9.12.0__py3-none-any.whl → 1.10.0.0__py3-none-any.whl

metadata/ingestion/source/database/sample_data.py

@@ -151,6 +151,7 @@ from metadata.generated.schema.type.entityLineage import (
     LineageDetails,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.generated.schema.type.lifeCycle import AccessDetails, LifeCycle
 from metadata.generated.schema.type.schema import Topic as TopicSchema
 from metadata.ingestion.api.common import Entity
@@ -192,7 +193,7 @@ NUM_SERVICES = 1
 DATABASES_PER_SERVICE = 5
 SCHEMAS_PER_DATABASE = 5
 TABLES_PER_SCHEMA = 10
-COLUMNS_PER_TABLE = 50
+COLUMNS_PER_TABLE = 200
 NUM_THREADS = 10
 BATCH_SIZE = 10
 COLUMNS = [
@@ -1064,13 +1065,12 @@ class SampleDataSource(
             if directory_data.get("parent"):
                 parent_name = directory_data["parent"]
                 if parent_name in directory_refs:
-                    directory_request.parent = directory_refs[parent_name]
+                    directory_request.parent = FullyQualifiedEntityName(
+                        root=directory_refs[parent_name]
+                    )
                 else:
-                    # For nested references like "Marketing.Campaigns_2024"
-                    # Build parent FQN manually
-                    parent_path = parent_name.replace(".", "/")
-                    directory_request.parent = (
-                        f"{self.drive_service.fullyQualifiedName.root}.{parent_path}"
+                    directory_request.parent = FullyQualifiedEntityName(
+                        f"{self.drive_service.fullyQualifiedName.root}.{parent_name}"
                     )
 
             # Use direct API call instead of yielding since suffix mapping is missing
@@ -2491,9 +2491,25 @@ class SampleDataSource(
 
             # Create with minimal required fields
             try:
+                owner = self.metadata.get_by_name(User, "admin")
                 table_request = Either(
                     right=CreateTableRequest(
-                        name=table_name, databaseSchema=schema_fqn, columns=COLUMNS
+                        name=table_name,
+                        databaseSchema=schema_fqn,
+                        columns=COLUMNS,
+                        description=random.choice(
+                            [f"This is {table_name} description.", None]
+                        ),
+                        owners=random.choice(
+                            [
+                                EntityReferenceList(
+                                    [EntityReference(id=owner.id, type="user")]
+                                ),
+                                None,
+                            ]
+                        )
+                        if owner
+                        else None,
                     )
                 )
                 yield table_request

metadata/ingestion/source/database/snowflake/metadata.py

@@ -20,6 +20,7 @@ import sqlalchemy.types as sqltypes
 import sqlparse
 from snowflake.sqlalchemy.custom_types import VARIANT, StructuredType
 from snowflake.sqlalchemy.snowdialect import SnowflakeDialect, ischema_names
+from sqlalchemy import exc as sa_exc
 from sqlalchemy.engine.reflection import Inspector
 from sqlparse.sql import Function, Identifier, Token
 
@@ -948,17 +949,26 @@ class SnowflakeSource(
                     table_name = result[6].split(".")[-1]
                     # Can't fetch source of stream is source is dropped or no priviledge
                     if table_name == "No privilege or table dropped":
-                        logger.debug(
-                            f"Couldn't fetch columns of stream [{result and result[1]}],"
-                            f" due to error on source: {table_name}. Result: {result}"
+                        logger.warning(
+                            f"Couldn't fetch columns of stream [{result and result[1]}] "
+                            f"(schema: '{schema_name}', db: '{db_name}') due to error on"
+                            f" source: [{table_name}]. Result: {result}"
                         )
                         return []
             except Exception:
                 pass
 
-        columns = inspector.get_columns(
-            table_name, schema_name, table_type=table_type, db_name=db_name
-        )
+        try:
+            columns = inspector.get_columns(
+                table_name, schema_name, table_type=table_type, db_name=db_name
+            )
+        except sa_exc.NoSuchTableError:
+            logger.warning(
+                f"Table [{table_name}] (schema: '{schema_name}', db: '{db_name}') not found."
+                " Unable to fetch columns. Please check if the configured Snowflake user has"
+                " necessary grants on this table."
+            )
+            return []
 
         if table_type == TableType.Stream:
             columns = [*columns, *DEFAULT_STREAM_COLUMNS]

metadata/ingestion/source/database/sql_column_handler.py

@@ -268,9 +268,17 @@ class SqlColumnHandlerMixin:
 
         table_columns = []
 
-        columns = self._get_columns_internal(
-            schema_name, table_name, db_name, inspector, table_type
-        )
+        try:
+            columns = self._get_columns_internal(
+                schema_name, table_name, db_name, inspector, table_type
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Unexpected exception getting columns for table [{table_name}] "
+                f"(schema: '{schema_name}', db: '{db_name}'): {exc}"
+            )
+            columns = []
 
         def process_column(column: dict):
             (

metadata/ingestion/source/database/timescale/__init__.py

@@ -0,0 +1,3 @@
+from metadata.ingestion.source.database.timescale.metadata import TimescaleSource
+
+__all__ = ["TimescaleSource"]

metadata/ingestion/source/database/timescale/connection.py

@@ -0,0 +1,103 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Source connection handler for TimescaleDB
+"""
+from typing import Optional
+
+from sqlalchemy.engine import Engine
+
+from metadata.generated.schema.entity.automations.workflow import (
+    Workflow as AutomationWorkflow,
+)
+from metadata.generated.schema.entity.services.connections.database.common.azureConfig import (
+    AzureConfigurationSource,
+)
+from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
+    BasicAuth,
+)
+from metadata.generated.schema.entity.services.connections.database.timescaleConnection import (
+    TimescaleConnection as TimescaleConnectionConfig,
+)
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
+from metadata.ingestion.connections.builders import (
+    create_generic_db_connection,
+    get_connection_args_common,
+    get_connection_url_common,
+)
+from metadata.ingestion.connections.connection import BaseConnection
+from metadata.ingestion.connections.test_connections import test_connection_db_common
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.postgres.queries import (
+    POSTGRES_GET_DATABASE,
+    POSTGRES_TEST_GET_QUERIES,
+    POSTGRES_TEST_GET_TAGS,
+)
+from metadata.ingestion.source.database.postgres.utils import (
+    get_postgres_time_column_name,
+)
+from metadata.utils.constants import THREE_MIN
+from metadata.utils.credentials import get_azure_access_token
+
+
+class TimescaleConnection(BaseConnection[TimescaleConnectionConfig, Engine]):
+    def _get_client(self) -> Engine:
+        """
+        Return the SQLAlchemy Engine for TimescaleDB.
+        TimescaleDB is a PostgreSQL extension, so we use the same connection logic.
+        """
+        connection = self.service_connection
+
+        if isinstance(connection.authType, AzureConfigurationSource):
+            access_token = get_azure_access_token(connection.authType)
+            connection.authType = BasicAuth(password=access_token)
+        return create_generic_db_connection(
+            connection=connection,
+            get_connection_url_fn=get_connection_url_common,
+            get_connection_args_fn=get_connection_args_common,
+        )
+
+    def get_connection_dict(self) -> dict:
+        """
+        Return the connection dictionary for this service.
+        """
+        raise NotImplementedError(
+            "get_connection_dict is not implemented for TimescaleDB"
+        )
+
+    def test_connection(
+        self,
+        metadata: OpenMetadata,
+        automation_workflow: Optional[AutomationWorkflow] = None,
+        timeout_seconds: Optional[int] = THREE_MIN,
+    ) -> TestConnectionResult:
+        """
+        Test connection. This can be executed either as part
+        of a metadata workflow or during an Automation Workflow
+        """
+        queries = {
+            "GetQueries": POSTGRES_TEST_GET_QUERIES.format(
+                time_column_name=get_postgres_time_column_name(engine=self.client),
+            ),
+            "GetDatabases": POSTGRES_GET_DATABASE,
+            "GetTags": POSTGRES_TEST_GET_TAGS,
+        }
+        return test_connection_db_common(
+            metadata=metadata,
+            engine=self.client,
+            service_connection=self.service_connection,
+            automation_workflow=automation_workflow,
+            timeout_seconds=timeout_seconds,
+            queries=queries,
+        )

metadata/ingestion/source/database/timescale/lineage.py

@@ -0,0 +1,124 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+TimescaleDB lineage module with continuous aggregate support
+"""
+
+import traceback
+from typing import Iterable, Optional
+
+from metadata.generated.schema.entity.services.connections.database.timescaleConnection import (
+    TimescaleConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.postgres.lineage import PostgresLineageSource
+from metadata.ingestion.source.database.timescale.queries import (
+    TIMESCALE_CHECK_EXTENSION,
+    TIMESCALE_GET_CONTINUOUS_AGGREGATE_DEFINITIONS,
+)
+from metadata.ingestion.source.models import TableView
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class TimescaleLineageSource(PostgresLineageSource):
+    """
+    TimescaleDB lineage source that extends PostgreSQL lineage
+    to include continuous aggregates (auto-refreshing materialized views)
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create TimescaleLineageSource"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection = config.serviceConnection.root.config
+        if not isinstance(connection, (TimescaleConnection)):
+            raise InvalidSourceException(
+                f"Expected TimescaleConnection, but got {connection}"
+            )
+        return cls(config, metadata)
+
+    def __init__(self, config, metadata):
+        super().__init__(config, metadata)
+        self.timescaledb_installed = False
+
+    def prepare(self):
+        """Check if TimescaleDB extension is installed"""
+        super().prepare()
+        try:
+            result = self.engine.execute(TIMESCALE_CHECK_EXTENSION).first()
+            self.timescaledb_installed = (
+                result.timescaledb_installed if result else False
+            )
+            logger.info(
+                f"TimescaleDB extension installed for lineage: {self.timescaledb_installed}"
+            )
+        except Exception as exc:
+            logger.warning(f"Could not check TimescaleDB extension: {exc}")
+            self.timescaledb_installed = False
+
+    def view_lineage_producer(self) -> Iterable[TableView]:
+        """
+        Get view definitions from ES and add continuous aggregates
+        """
+        # First yield regular views from parent
+        yield from super().view_lineage_producer()
+
+        # Then yield continuous aggregates if TimescaleDB is installed
+        if self.timescaledb_installed:
+            yield from self._yield_continuous_aggregate_views()
+
+    def _yield_continuous_aggregate_views(self) -> Iterable[TableView]:
+        """
+        Get TimescaleDB continuous aggregates as TableView objects.
+
+        Continuous aggregates are auto-refreshing materialized views that
+        provide clear column-level lineage from source hypertables.
+
+        The view_definition contains the SELECT statement that defines
+        the continuous aggregate, which OpenMetadata can parse for
+        column-level lineage.
+        """
+        try:
+            results = self.engine.execute(
+                TIMESCALE_GET_CONTINUOUS_AGGREGATE_DEFINITIONS
+            )
+
+            for row in results:
+                try:
+                    yield TableView(
+                        table_name=row.view_name,
+                        schema_name=row.view_schema,
+                        db_name=self.service_connection.database,
+                        view_definition=row.view_definition,
+                    )
+
+                    logger.debug(
+                        f"Extracted continuous aggregate view: {row.view_schema}.{row.view_name}"
+                    )
+
+                except Exception as exc:
+                    logger.debug(traceback.format_exc())
+                    logger.warning(
+                        f"Failed to process continuous aggregate {row.view_schema}.{row.view_name}: {exc}"
+                    )
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to extract continuous aggregate views: {exc}")

metadata/ingestion/source/database/timescale/metadata.py

@@ -0,0 +1,228 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+TimescaleDB source module
+"""
+import traceback
+from typing import Iterable, Optional, Tuple
+
+from sqlalchemy.engine import Inspector
+
+from metadata.generated.schema.entity.data.table import (
+    PartitionColumnDetails,
+    PartitionIntervalTypes,
+    Table,
+    TablePartition,
+)
+from metadata.generated.schema.entity.services.connections.database.timescaleConnection import (
+    TimescaleConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.models import Either
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.postgres.metadata import PostgresSource
+from metadata.ingestion.source.database.timescale.models import (
+    CompressionSettings,
+    HypertableInfo,
+)
+from metadata.ingestion.source.database.timescale.queries import (
+    TIMESCALE_CHECK_EXTENSION,
+    TIMESCALE_GET_COMPRESSION_SETTINGS,
+    TIMESCALE_GET_HYPERTABLE_INFO,
+)
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class TimescaleSource(PostgresSource):
+    """
+    Implements the necessary methods to extract
+    Database metadata from TimescaleDB Source
+    """
+
+    def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
+        super().__init__(config, metadata)
+        self.timescaledb_installed = False
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: TimescaleConnection = config.serviceConnection.root.config
+        if not isinstance(connection, TimescaleConnection):
+            raise InvalidSourceException(
+                f"Expected TimescaleConnection, but got {connection}"
+            )
+        return cls(config, metadata)
+
+    def prepare(self):
+        """
+        Check if TimescaleDB extension is installed
+        """
+        super().prepare()
+        try:
+            result = self.engine.execute(TIMESCALE_CHECK_EXTENSION).first()
+            if result:
+                self.timescaledb_installed = result.timescaledb_installed
+                logger.info(
+                    f"TimescaleDB extension installed: {self.timescaledb_installed}"
+                )
+        except Exception as exc:
+            logger.warning(f"Could not check TimescaleDB extension: {exc}")
+            self.timescaledb_installed = False
+
+    def yield_table(
+        self, table_name_and_type: Tuple[str, str]
+    ) -> Iterable[Either[Table]]:
+        """
+        Override to add TimescaleDB-specific metadata
+        """
+        for either_table in super().yield_table(table_name_and_type):
+            if either_table.right and self.timescaledb_installed:
+                try:
+                    self._add_timescale_metadata(either_table.right)
+                except Exception as exc:
+                    logger.debug(traceback.format_exc())
+                    logger.warning(
+                        f"Error adding TimescaleDB metadata for table {either_table.right.name}: {exc}"
+                    )
+            yield either_table
+
+    def _add_timescale_metadata(self, table: Table) -> None:
+        """
+        Add compression and hypertable metadata to the table
+        """
+        try:
+            schema_name = table.databaseSchema.root
+            table_name = table.name.root
+
+            hypertable = self._get_hypertable_info(table_name, schema_name)
+
+            if hypertable:
+                logger.debug(f"Found hypertable: {schema_name}.{table_name}")
+
+                table.compressionEnabled = hypertable.compression_enabled
+
+                if hypertable.compression_enabled:
+                    compression = self._get_compression_settings(
+                        table_name, schema_name
+                    )
+
+                    if compression:
+                        table.compressionCodec = "TimescaleDB Native"
+                        table.compressionStrategy = {
+                            "compressionType": "POLICY_BASED",
+                            "segmentColumns": compression.segment_by_columns or [],
+                            "orderColumns": compression.order_by_columns or [],
+                        }
+
+                partition_info = self._build_hypertable_partition(hypertable)
+                if partition_info:
+                    table.tablePartition = partition_info
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error processing TimescaleDB metadata for {table.name}: {exc}"
+            )
+
+    def _get_hypertable_info(
+        self, table_name: str, schema_name: str
+    ) -> Optional[HypertableInfo]:
+        """
+        Query timescaledb_information.hypertables for metadata
+        """
+        try:
+            result = self.engine.execute(
+                TIMESCALE_GET_HYPERTABLE_INFO,
+                {"table_name": table_name, "schema_name": schema_name},
+            ).first()
+
+            if result:
+                return HypertableInfo.model_validate(dict(result._mapping))
+            return None
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.debug(
+                f"Could not get hypertable info for {schema_name}.{table_name}: {exc}"
+            )
+            return None
+
+    def _get_compression_settings(
+        self, table_name: str, schema_name: str
+    ) -> Optional[CompressionSettings]:
+        """
+        Query timescaledb_information.compression_settings for compression config
+        """
+        try:
+            result = self.engine.execute(
+                TIMESCALE_GET_COMPRESSION_SETTINGS,
+                {"table_name": table_name, "schema_name": schema_name},
+            ).first()
+
+            if result:
+                return CompressionSettings.model_validate(dict(result._mapping))
+            return None
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.debug(
+                f"Could not get compression settings for {schema_name}.{table_name}: {exc}"
+            )
+            return None
+
+    def _build_hypertable_partition(
+        self, hypertable: HypertableInfo
+    ) -> Optional[TablePartition]:
+        """
+        Build partition details from hypertable information
+        """
+        if not hypertable.column_name:
+            return None
+
+        if hypertable.interval_length:
+            interval_type = PartitionIntervalTypes.TIME_UNIT
+            interval = f"{hypertable.interval_length} microseconds"
+        elif hypertable.integer_interval:
+            interval_type = PartitionIntervalTypes.INTEGER_RANGE
+            interval = str(hypertable.integer_interval)
+        else:
+            interval_type = PartitionIntervalTypes.COLUMN_VALUE
+            interval = None
+
+        return TablePartition(
+            columns=[
+                PartitionColumnDetails(
+                    columnName=hypertable.column_name,
+                    intervalType=interval_type,
+                    interval=interval,
+                )
+            ]
+        )
+
+    def get_table_partition_details(
+        self, table_name: str, schema_name: str, inspector: Inspector
+    ) -> Tuple[bool, Optional[TablePartition]]:
+        """
+        Override to check for hypertables first, then fall back to PostgreSQL partitioning
+        """
+        if self.timescaledb_installed:
+            hypertable = self._get_hypertable_info(table_name, schema_name)
+            if hypertable:
+                partition = self._build_hypertable_partition(hypertable)
+                return (True, partition) if partition else (False, None)
+
+        return super().get_table_partition_details(table_name, schema_name, inspector)

metadata/ingestion/source/database/timescale/models.py

@@ -0,0 +1,60 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+TimescaleDB models for metadata extraction
+"""
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class HypertableInfo(BaseModel):
+    """Information about a TimescaleDB hypertable"""
+
+    hypertable_schema: str
+    hypertable_name: str
+    compression_enabled: bool
+    column_name: Optional[str] = None
+    interval_length: Optional[int] = None
+    integer_interval: Optional[int] = None
+    integer_now_func: Optional[str] = None
+    num_dimensions: int = 1
+
+
+class CompressionSettings(BaseModel):
+    """Compression configuration for a hypertable"""
+
+    segment_by_columns: Optional[List[str]] = Field(default_factory=list)
+    order_by_columns: Optional[List[str]] = Field(default_factory=list)
+
+
+class ContinuousAggregateInfo(BaseModel):
+    """Information about a continuous aggregate"""
+
+    view_schema: str
+    view_name: str
+    view_definition: Optional[str] = None
+    compression_enabled: bool = False
+    materialized_only: bool = False
+    materialization_hypertable_schema: Optional[str] = None
+    materialization_hypertable_name: Optional[str] = None
+
+
+class ChunkInfo(BaseModel):
+    """Information about a hypertable chunk"""
+
+    chunk_schema: str
+    chunk_name: str
+    range_start: Optional[str] = None
+    range_end: Optional[str] = None
+    is_compressed: bool = False
+    chunk_tablespace: Optional[str] = None