openmetadata-ingestion 1.3.0.0.dev0__py3-none-any.whl → 1.3.1.0__py3-none-any.whl

metadata/generated/schema/entity/services/connections/database/duckdbConnection.py

@@ -1,96 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  entity/services/connections/database/duckdbConnection.json
-#   timestamp: 2024-01-29T10:59:57+00:00
-
-from __future__ import annotations
-
-from enum import Enum
-from typing import Optional, Union
-
-from pydantic import BaseModel, Extra, Field
-
-from .....security.ssl import verifySSLConfig
-from .. import connectionBasicType
-from .common import basicAuth, iamAuthConfig
-
-
-class SslMode(Enum):
-    disable = 'disable'
-    allow = 'allow'
-    prefer = 'prefer'
-    require = 'require'
-    verify_ca = 'verify-ca'
-    verify_full = 'verify-full'
-
-
-class GreenplumType(Enum):
-    DuckDb = 'DuckDb'
-
-
-class GreenplumScheme(Enum):
-    postgresql_psycopg2 = 'postgresql+psycopg2'
-
-
-class DuckDbConnection(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[GreenplumType] = Field(
-        GreenplumType.DuckDb, description='Service Type', title='Service Type'
-    )
-    scheme: Optional[GreenplumScheme] = Field(
-        GreenplumScheme.postgresql_psycopg2,
-        description='SQLAlchemy driver scheme options.',
-        title='Connection Scheme',
-    )
-    username: str = Field(
-        ...,
-        description='Username to connect to DuckDb. This user should have privileges to read all the metadata in DuckDb.',
-        title='Username',
-    )
-    authType: Optional[
-        Union[basicAuth.BasicAuth, iamAuthConfig.IamAuthConfigurationSource]
-    ] = Field(
-        None, description='Choose Auth Config Type.', title='Auth Configuration Type'
-    )
-    hostPort: str = Field(
-        ..., description='Host and port of the source service.', title='Host and Port'
-    )
-    database: str = Field(
-        ...,
-        description='Database of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single database. When left blank, OpenMetadata Ingestion attempts to scan all the databases.',
-        title='Database',
-    )
-    sslMode: Optional[SslMode] = Field(
-        SslMode.disable,
-        description='SSL Mode to connect to DuckDb database.',
-        title='SSL Mode',
-    )
-    sslConfig: Optional[verifySSLConfig.SslConfig] = None
-    ingestAllDatabases: Optional[bool] = Field(
-        False,
-        description='Ingest data from all databases in DuckDb. You can use databaseFilterPattern on top of this.',
-        title='Ingest All Databases',
-    )
-    connectionOptions: Optional[connectionBasicType.ConnectionOptions] = Field(
-        None, title='Connection Options'
-    )
-    connectionArguments: Optional[connectionBasicType.ConnectionArguments] = Field(
-        None, title='Connection Arguments'
-    )
-    supportsMetadataExtraction: Optional[
-        connectionBasicType.SupportsMetadataExtraction
-    ] = Field(None, title='Supports Metadata Extraction')
-    supportsDBTExtraction: Optional[connectionBasicType.SupportsDBTExtraction] = None
-    supportsProfiler: Optional[connectionBasicType.SupportsProfiler] = Field(
-        None, title='Supports Profiler'
-    )
-    supportsDatabase: Optional[connectionBasicType.SupportsDatabase] = Field(
-        None, title='Supports Database'
-    )
-    supportsQueryComment: Optional[connectionBasicType.SupportsQueryComment] = Field(
-        None, title='Supports Query Comment'
-    )
-    sampleDataStorageConfig: Optional[
-        connectionBasicType.SampleDataStorageConfig
-    ] = Field(None, title='Storage Config for Sample Data')

metadata/generated/schema/events/userAlertConfig.py

@@ -1,25 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  events/userAlertConfig.json
-#   timestamp: 2024-01-29T10:59:57+00:00
-
-from __future__ import annotations
-
-from typing import Optional
-
-from pydantic import BaseModel, Extra, Field
-
-from ..type import entityReferenceList
-
-
-class UserAlertConfig(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    references: Optional[entityReferenceList.EntityReferenceList] = Field(
-        None, description='List of References for internal System Users or Teams'
-    )
-    sendToAdmins: Optional[bool] = Field(False, description='Send the Mails to Admins')
-    sendToOwners: Optional[bool] = Field(False, description='Send the Mails to Owners')
-    sendToFollowers: Optional[bool] = Field(
-        False, description='Send the Mails to Followers'
-    )

metadata/ingestion/source/database/duckdb/connection.py

@@ -1,76 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Source connection handler
-"""
-
-from typing import Optional
-
-from sqlalchemy.engine import Engine
-
-from metadata.generated.schema.entity.automations.workflow import (
-    Workflow as AutomationWorkflow,
-)
-from metadata.generated.schema.entity.services.connections.database.duckdbConnection import (
-    DuckDbConnection,
-    SslMode,
-)
-from metadata.ingestion.connections.builders import (
-    create_generic_db_connection,
-    get_connection_args_common,
-    get_connection_url_common,
-    init_empty_connection_arguments,
-)
-from metadata.ingestion.connections.test_connections import test_connection_db_common
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.database.duckdb.queries import DUCKDB_GET_DATABASE
-
-
-def get_connection(connection: DuckDbConnection) -> Engine:
-    """
-    Create connection
-    """
-    if connection.sslMode:
-        if not connection.connectionArguments:
-            connection.connectionArguments = init_empty_connection_arguments()
-        connection.connectionArguments.__root__["sslmode"] = connection.sslMode.value
-        if connection.sslMode in (SslMode.verify_ca, SslMode.verify_full):
-            connection.connectionArguments.__root__[
-                "sslrootcert"
-            ] = connection.sslConfig.__root__.certificatePath
-    return create_generic_db_connection(
-        connection=connection,
-        get_connection_url_fn=get_connection_url_common,
-        get_connection_args_fn=get_connection_args_common,
-    )
-
-
-def test_connection(
-    metadata: OpenMetadata,
-    engine: Engine,
-    service_connection: DuckDbConnection,
-    automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
-    """
-    Test connection. This can be executed either as part
-    of a metadata workflow or during an Automation Workflow
-    """
-    queries = {
-        "GetDatabases": DUCKDB_GET_DATABASE,
-    }
-    test_connection_db_common(
-        metadata=metadata,
-        engine=engine,
-        service_connection=service_connection,
-        automation_workflow=automation_workflow,
-        queries=queries,
-    )

metadata/ingestion/source/database/duckdb/metadata.py

@@ -1,204 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-DuckDb source module
-"""
-import traceback
-from collections import namedtuple
-from typing import Iterable, Optional, Tuple
-
-from sqlalchemy import sql
-from sqlalchemy.dialects.postgresql.base import PGDialect, ischema_names
-from sqlalchemy.engine.reflection import Inspector
-
-from metadata.generated.schema.entity.data.database import Database
-from metadata.generated.schema.entity.data.table import (
-    IntervalType,
-    TablePartition,
-    TableType,
-)
-from metadata.generated.schema.entity.services.connections.database.duckdbConnection import (
-    DuckDbConnection,
-)
-from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
-    OpenMetadataConnection,
-)
-from metadata.generated.schema.metadataIngestion.workflow import (
-    Source as WorkflowSource,
-)
-from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type
-from metadata.ingestion.source.database.common_db_source import (
-    CommonDbSourceService,
-    TableNameAndType,
-)
-from metadata.ingestion.source.database.duckdb.queries import (
-    DUCKDB_GET_DB_NAMES,
-    DUCKDB_GET_TABLE_NAMES,
-    DUCKDB_PARTITION_DETAILS,
-)
-from metadata.ingestion.source.database.duckdb.utils import (
-    get_column_info,
-    get_columns,
-    get_table_comment,
-    get_view_definition,
-)
-from metadata.ingestion.source.database.multi_db_source import MultiDBSource
-from metadata.utils import fqn
-from metadata.utils.filters import filter_by_database
-from metadata.utils.logger import ingestion_logger
-from metadata.utils.sqlalchemy_utils import (
-    get_all_table_comments,
-    get_all_view_definitions,
-)
-
-TableKey = namedtuple("TableKey", ["schema", "table_name"])
-
-logger = ingestion_logger()
-
-
-INTERVAL_TYPE_MAP = {
-    "list": IntervalType.COLUMN_VALUE.value,
-    "hash": IntervalType.COLUMN_VALUE.value,
-    "range": IntervalType.TIME_UNIT.value,
-}
-
-RELKIND_MAP = {
-    "r": TableType.Regular,
-    "p": TableType.Partitioned,
-    "f": TableType.Foreign,
-}
-
-GEOMETRY = create_sqlalchemy_type("GEOMETRY")
-POINT = create_sqlalchemy_type("POINT")
-POLYGON = create_sqlalchemy_type("POLYGON")
-
-ischema_names.update(
-    {
-        "geometry": GEOMETRY,
-        "point": POINT,
-        "polygon": POLYGON,
-        "box": create_sqlalchemy_type("BOX"),
-        "circle": create_sqlalchemy_type("CIRCLE"),
-        "line": create_sqlalchemy_type("LINE"),
-        "lseg": create_sqlalchemy_type("LSEG"),
-        "path": create_sqlalchemy_type("PATH"),
-        "pg_lsn": create_sqlalchemy_type("PG_LSN"),
-        "pg_snapshot": create_sqlalchemy_type("PG_SNAPSHOT"),
-        "tsquery": create_sqlalchemy_type("TSQUERY"),
-        "txid_snapshot": create_sqlalchemy_type("TXID_SNAPSHOT"),
-        "xml": create_sqlalchemy_type("XML"),
-    }
-)
-
-
-PGDialect.get_all_table_comments = get_all_table_comments
-PGDialect.get_table_comment = get_table_comment
-PGDialect._get_column_info = get_column_info  # pylint: disable=protected-access
-PGDialect.get_view_definition = get_view_definition
-PGDialect.get_columns = get_columns
-PGDialect.get_all_view_definitions = get_all_view_definitions
-
-PGDialect.ischema_names = ischema_names
-
-
-class DuckDbSource(CommonDbSourceService, MultiDBSource):
-    """
-    Implements the necessary methods to extract
-    Database metadata from DuckDb Source
-    """
-
-    @classmethod
-    def create(cls, config_dict, metadata: OpenMetadataConnection):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: DuckDbConnection = config.serviceConnection.__root__.config
-        if not isinstance(connection, DuckDbConnection):
-            raise InvalidSourceException(
-                f"Expected DuckDbConnection, but got {connection}"
-            )
-        return cls(config, metadata)
-
-    def query_table_names_and_types(
-        self, schema_name: str
-    ) -> Iterable[TableNameAndType]:
-        """
-        Overwrite the inspector implementation to handle partitioned
-        and foreign types
-        """
-        result = self.connection.execute(
-            sql.text(DUCKDB_GET_TABLE_NAMES),
-            {"schema": schema_name},
-        )
-
-        return [
-            TableNameAndType(
-                name=name, type_=RELKIND_MAP.get(relkind, TableType.Regular)
-            )
-            for name, relkind in result
-        ]
-
-    def get_configured_database(self) -> Optional[str]:
-        if not self.service_connection.ingestAllDatabases:
-            return self.service_connection.database
-        return None
-
-    def get_database_names_raw(self) -> Iterable[str]:
-        yield from self._execute_database_query(DUCKDB_GET_DB_NAMES)
-
-    def get_database_names(self) -> Iterable[str]:
-        if not self.config.serviceConnection.__root__.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.__root__.config.database
-            self.set_inspector(database_name=configured_db)
-            yield configured_db
-        else:
-            for new_database in self.get_database_names_raw():
-                database_fqn = fqn.build(
-                    self.metadata,
-                    entity_type=Database,
-                    service_name=self.context.database_service,
-                    database_name=new_database,
-                )
-
-                if filter_by_database(
-                    self.source_config.databaseFilterPattern,
-                    database_fqn
-                    if self.source_config.useFqnForFiltering
-                    else new_database,
-                ):
-                    self.status.filter(database_fqn, "Database Filtered Out")
-                    continue
-
-                try:
-                    self.set_inspector(database_name=new_database)
-                    yield new_database
-                except Exception as exc:
-                    logger.debug(traceback.format_exc())
-                    logger.error(
-                        f"Error trying to connect to database {new_database}: {exc}"
-                    )
-
-    def get_table_partition_details(
-        self, table_name: str, schema_name: str, inspector: Inspector
-    ) -> Tuple[bool, TablePartition]:
-        result = self.engine.execute(
-            DUCKDB_PARTITION_DETAILS.format(
-                table_name=table_name, schema_name=schema_name
-            )
-        ).all()
-        if result:
-            partition_details = TablePartition(
-                intervalType=INTERVAL_TYPE_MAP.get(
-                    result[0].partition_strategy, IntervalType.COLUMN_VALUE.value
-                ),
-                columns=[row.column_name for row in result if row.column_name],
-            )
-            return True, partition_details
-        return False, None

metadata/ingestion/source/database/duckdb/queries.py

@@ -1,141 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-"""
-SQL Queries used during ingestion
-"""
-
-import textwrap
-
-# https://www.postgresql.org/docs/current/catalog-pg-class.html
-# r = ordinary table, v = view, m = materialized view, c = composite type, f = foreign table, p = partitioned table,
-DUCKDB_GET_TABLE_NAMES = """
-    select c.relname, c.relkind
-    from pg_catalog.pg_class c
-        left outer join pg_catalog.pg_partition_rule pr on c.oid = pr.parchildrelid
-        JOIN pg_namespace n ON n.oid = c.relnamespace
-    where c.relkind in ('r', 'p', 'f')
-        and pr.oid is null
-        and n.nspname = :schema
-"""
-
-DUCKDB_PARTITION_DETAILS = textwrap.dedent(
-    """
-    select
-        ns.nspname as schema,
-        par.relname as table_name,
-        partition_strategy,
-        col.column_name
-    from
-        (select
-             parrelid,
-             parnatts,
-             case parkind
-                  when 'l' then 'list'
-                  when 'h' then 'hash'
-                  when 'r' then 'range' end as partition_strategy,
-             unnest(paratts) column_index
-         from
-             pg_catalog.pg_partition) pt
-    join
-        pg_class par
-    on
-        par.oid = pt.parrelid
-    left join
-        pg_catalog.pg_namespace ns on par.relnamespace = ns.oid
-    left join
-        information_schema.columns col
-    on
-        col.table_schema = ns.nspname
-        and col.table_name = par.relname
-        and ordinal_position = pt.column_index
-    where par.relname='{table_name}' and  ns.nspname='{schema_name}'
-    """
-)
-
-DUCKDB_TABLE_COMMENTS = """
-    SELECT n.nspname as schema,
-            c.relname as table_name,
-            pgd.description as table_comment
-    FROM pg_catalog.pg_class c
-        LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-        LEFT JOIN pg_catalog.pg_description pgd ON pgd.objsubid = 0 AND pgd.objoid = c.oid
-    WHERE c.relkind in ('r', 'v', 'm', 'f', 'p')
-      AND pgd.description IS NOT NULL
-      AND n.nspname <> 'pg_catalog'
-    ORDER BY "schema", "table_name"
-"""
-
-# Postgres\DuckDb views definitions only contains the select query
-# hence we are appending "create view <schema>.<table> as " to select query
-# to generate the column level lineage
-DUCKDB_VIEW_DEFINITIONS = """
-SELECT 
-	n.nspname "schema",
-	c.relname view_name,
-	'create view ' || n.nspname || '.' || c.relname || ' as ' || pg_get_viewdef(c.oid,true) view_def
-FROM pg_class c 
-JOIN pg_namespace n ON n.oid = c.relnamespace 
-WHERE c.relkind IN ('v', 'm')
-AND n.nspname not in ('pg_catalog','information_schema')
-"""
-
-DUCKDB_GET_DATABASE = """
-select datname from pg_catalog.pg_database
-"""
-
-DUCKDB_GET_DB_NAMES = """
-select datname from pg_catalog.pg_database
-"""
-
-DUCKDB_COL_IDENTITY = """\
-  (SELECT json_build_object(
-      'always', a.attidentity = 'a',
-      'start', s.seqstart,
-      'increment', s.seqincrement,
-      'minvalue', s.seqmin,
-      'maxvalue', s.seqmax,
-      'cache', s.seqcache,
-      'cycle', s.seqcycle)
-  FROM pg_catalog.pg_sequence s
-  JOIN pg_catalog.pg_class c on s.seqrelid = c."oid"
-  WHERE c.relkind = 'S'
-  AND a.attidentity != ''
-  AND s.seqrelid = pg_catalog.pg_get_serial_sequence(
-      a.attrelid::regclass::text, a.attname
-  )::regclass::oid
-  ) as identity_options\
-"""
-
-DUCKDB_SQL_COLUMNS = """
-        SELECT a.attname,
-            pg_catalog.format_type(a.atttypid, a.atttypmod),
-            (
-            SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid)
-            FROM pg_catalog.pg_attrdef d
-            WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum
-            AND a.atthasdef
-            ) AS DEFAULT,
-            a.attnotnull,
-            a.attrelid as table_oid,
-            pgd.description as comment,
-            {generated},
-            {identity}
-        FROM pg_catalog.pg_attribute a
-        LEFT JOIN pg_catalog.pg_description pgd ON (
-            pgd.objoid = a.attrelid AND pgd.objsubid = a.attnum)
-        WHERE a.attrelid = :table_oid
-        AND a.attnum > 0 AND NOT a.attisdropped
-        ORDER BY a.attnum
-    """
-
-DUCKDB_GET_SERVER_VERSION = """
-show server_version
-"""