dbt-ydb 0.0.1__tar.gz

dbt_ydb-0.0.1/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.3
+Name: dbt-ydb
+Version: 0.0.1
+Summary: DBT adapter for YDB
+Author: Yandex LLC
+Author-email: ydb@yandex-team.ru
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: dbt-adapters (>=1.14.8,<2.0.0)
+Requires-Dist: dbt-common (>=1.23.0,<2.0.0)
+Requires-Dist: dbt-core (>=1.8.0,<2.0.0)
+Requires-Dist: ydb-dbapi (>=0.1.12,<0.2.0)
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/dbt-labs/dbt/ec7dee39f793aa4f7dd3dae37282cc87664813e4/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
+</p>
+
+# dbt-ydb
+
+**dbt-ydb** is a plugin for [dbt](https://www.getdbt.com/) that provides support for working with [YDB](https://ydb.tech).
+
+## Installation
+
+To install plugin, execute the following command:
+
+```bash
+pip install dbt-ydb
+```
+
+## Supported features
+
+- [x] Table materialization
+- [x] View materialization
+- [x] Seeds
+- [x] Docs generate
+- [x] Tests
+- [ ] TBD

dbt_ydb-0.0.1/README.md

@@ -0,0 +1,24 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/dbt-labs/dbt/ec7dee39f793aa4f7dd3dae37282cc87664813e4/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
+</p>
+
+# dbt-ydb
+
+**dbt-ydb** is a plugin for [dbt](https://www.getdbt.com/) that provides support for working with [YDB](https://ydb.tech).
+
+## Installation
+
+To install plugin, execute the following command:
+
+```bash
+pip install dbt-ydb
+```
+
+## Supported features
+
+- [x] Table materialization
+- [x] View materialization
+- [x] Seeds
+- [x] Docs generate
+- [x] Tests
+- [ ] TBD

dbt_ydb-0.0.1/dbt/adapters/ydb/__init__.py

@@ -0,0 +1,11 @@
+from dbt.adapters.ydb.connections import YDBConnectionManager  # noqa
+from dbt.adapters.ydb.connections import YDBCredentials
+from dbt.adapters.ydb.impl import YDBAdapter
+
+from dbt.adapters.base import AdapterPlugin
+from dbt.include import ydb
+
+
+Plugin = AdapterPlugin(
+    adapter=YDBAdapter, credentials=YDBCredentials, include_path=ydb.PACKAGE_PATH
+)

dbt_ydb-0.0.1/dbt/adapters/ydb/__version__.py

@@ -0,0 +1 @@
+version = "0.0.1"

dbt_ydb-0.0.1/dbt/adapters/ydb/column.py

@@ -0,0 +1,136 @@
+import re
+from dataclasses import dataclass, field
+from typing import Any, List, Literal, TypeVar
+
+from dbt.adapters.base.column import Column
+from dbt_common.exceptions import DbtRuntimeError
+
+Self = TypeVar('Self', bound='YDBColumn')
+
+
+@dataclass
+class YDBColumn(Column):
+    TYPE_LABELS = {
+        'STRING': 'String',
+        'TIMESTAMP': 'DateTime',
+        'FLOAT': 'Float32',
+        'INTEGER': 'Int32',
+    }
+    is_nullable: bool = False
+    is_low_cardinality: bool = False
+    _low_card_regex = re.compile(r'^LowCardinality\((.*)\)$')
+    _nullable_regex = re.compile(r'^(.*)\?$')
+    _fix_size_regex = re.compile(r'FixedString\((.*?)\)')
+    _decimal_regex = re.compile(r'Decimal\((\d+), (\d+)\)')
+
+    def __init__(self, column: str, dtype: str) -> None:
+        char_size = None
+        numeric_precision = None
+        numeric_scale = None
+
+        dtype = self._inner_dtype(dtype)
+
+        if dtype.lower().startswith('fixedstring'):
+            match_sized = self._fix_size_regex.search(dtype)
+            if match_sized:
+                char_size = int(match_sized.group(1))
+
+        if dtype.lower().startswith('decimal'):
+            match_dec = self._decimal_regex.search(dtype)
+            numeric_precision = 0
+            numeric_scale = 0
+            if match_dec:
+                numeric_precision = int(match_dec.group(1))
+                numeric_scale = int(match_dec.group(2))
+
+        super().__init__(column, dtype, char_size, numeric_precision, numeric_scale)
+
+    def __repr__(self) -> str:
+        return f'<YDBColumn {self.name} ({self.data_type}, is nullable: {self.is_nullable})>'
+
+    @property
+    def data_type(self) -> str:
+        if self.is_string():
+            data_t = self.string_type(self.string_size())
+        elif self.is_numeric():
+            data_t = self.numeric_type(self.dtype, self.numeric_precision, self.numeric_scale)
+        else:
+            data_t = self.dtype
+
+        if self.is_nullable or self.is_low_cardinality:
+            data_t = self.nested_type(data_t, self.is_low_cardinality, self.is_nullable)
+
+        return data_t
+
+    def is_string(self) -> bool:
+        return self.dtype.lower() in [
+            'string',
+            'fixedstring',
+            'longblob',
+            'longtext',
+            'tinytext',
+            'text',
+            'varchar',
+            'mediumblob',
+            'blob',
+            'tinyblob',
+            'char',
+            'mediumtext',
+        ] or self.dtype.lower().startswith('fixedstring')
+
+    def is_integer(self) -> bool:
+        return self.dtype.lower().startswith('int') or self.dtype.lower().startswith('uint')
+
+    def is_numeric(self) -> bool:
+        return self.dtype.lower().startswith('decimal')
+
+    def is_float(self) -> bool:
+        return self.dtype.lower().startswith('float')
+
+    def string_size(self) -> int:
+        if not self.is_string():
+            raise DbtRuntimeError('Called string_size() on non-string field!')
+
+        if not self.dtype.lower().startswith('fixedstring') or self.char_size is None:
+            return 256
+        else:
+            return int(self.char_size)
+
+    @classmethod
+    def string_type(cls, size: int) -> str:
+        return 'String'
+
+    @classmethod
+    def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str:
+        return f'Decimal({precision}, {scale})'
+
+    @classmethod
+    def nested_type(cls, dtype: str, is_low_cardinality: bool, is_nullable: bool) -> str:
+        template = "{}"
+        if is_low_cardinality:
+            template = template.format("LowCardinality({})")
+        if is_nullable:
+            template = template.format("Nullable({})")
+        return template.format(dtype)
+
+    def literal(self, value):
+        return f'to{self.dtype}({value})'
+
+    def can_expand_to(self, other_column: 'Column') -> bool:
+        if not self.is_string() or not other_column.is_string():
+            return False
+
+        return other_column.string_size() > self.string_size()
+
+    def _inner_dtype(self, dtype) -> str:
+        inner_dtype = dtype.strip()
+
+        if low_card_match := self._low_card_regex.search(inner_dtype):
+            self.is_low_cardinality = True
+            inner_dtype = low_card_match.group(1)
+
+        if null_match := self._nullable_regex.search(inner_dtype):
+            self.is_nullable = True
+            inner_dtype = null_match.group(1)
+
+        return inner_dtype

dbt_ydb-0.0.1/dbt/adapters/ydb/connections.py

@@ -0,0 +1,179 @@
+from contextlib import contextmanager
+from dataclasses import dataclass
+import dbt.exceptions
+from dbt.adapters.contracts.connection import AdapterResponse
+from dbt.adapters.contracts.connection import Credentials
+from dbt.adapters.contracts.connection import Connection
+import time
+from dbt.adapters.sql import SQLConnectionManager
+
+from dbt.adapters.events.logging import AdapterLogger
+
+import ydb_dbapi as dbapi
+
+from typing import Any, Optional, Tuple, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import agate
+
+
+logger = AdapterLogger("YDB")
+
+
+@dataclass
+class YDBCredentials(Credentials):
+    """
+    Defines database specific credentials that get added to
+    profiles.yml to connect to new adapter
+    """
+
+    # Add credentials members here, like:
+    host: str = "localhost"
+    port: int = 2136
+    username: Optional[str] = None
+    password: Optional[str] = None
+
+    schema: str = ""
+
+    _ALIASES = {"dbname": "database", "pass": "password", "user": "username"}
+
+    @property
+    def type(self):
+        """Return name of adapter."""
+        return "ydb"
+
+    @property
+    def unique_field(self):
+        """
+        Hashed and included in anonymous telemetry to track adapter adoption.
+        Pick a field that can uniquely identify one team/organization building with this adapter
+        """
+        return self.host
+
+    def _connection_keys(self):
+        """
+        List of keys to display in the `dbt debug` output.
+        """
+        return ("host", "port", "username", "user", "database", "schema")
+
+    def _get_ydb_credentials(self):
+        import ydb
+
+        if self.username is not None:
+            return ydb.StaticCredentials.from_user_password(
+                self.username, self.password
+            )
+
+        return ydb.AnonymousCredentials()
+
+
+class YDBConnectionManager(SQLConnectionManager):
+    TYPE = "ydb"
+
+    @contextmanager
+    def exception_handler(self, sql: str):
+        """
+        Returns a context manager, that will handle exceptions raised
+        from queries, catch, log, and raise dbt exceptions it knows how to handle.
+        """
+        ## Example ##
+        try:
+            yield
+        except dbapi.DatabaseError as exc:
+            self.release()
+
+            logger.debug("YDB error: {}".format(str(exc)))
+            raise dbt.exceptions.DbtRuntimeError(str(exc))
+        except Exception as exc:
+            logger.debug("Error running SQL: {}".format(sql))
+            logger.debug("Rolling back transaction.")
+            self.release()
+            raise dbt.exceptions.DbtRuntimeError(str(exc))
+
+    @classmethod
+    def open(cls, connection):
+        """
+        Receives a connection object and a Credentials object
+        and moves it to the "open" state.
+        """
+        ## Example ##
+        if connection.state == "open":
+            logger.debug("Connection is already open, skipping open.")
+            return connection
+
+        credentials = connection.credentials
+
+        try:
+            handle = dbapi.connect(
+                host=credentials.host,
+                port=credentials.port,
+                database=credentials.database,
+                credentials=credentials._get_ydb_credentials(),
+            )
+            logger.debug("Connect success")
+            connection.state = "open"
+            connection.handle = handle
+        except dbapi.Error as exc:
+            logger.debug("YDB error: {}".format(str(exc)))
+            raise dbt.exceptions.DatabaseException(str(exc))
+
+        return connection
+
+    @classmethod
+    def get_response(cls, _):
+        return "OK"
+
+    def cancel(self, connection):
+        """
+        Gets a connection object and attempts to cancel any ongoing queries.
+        """
+        connection_name = connection.name
+        logger.debug("Cancelling query '{}'", connection_name)
+        connection.handle.close()
+        logger.debug("Cancel query '{}'", connection_name)
+
+    def begin(self):
+        pass
+
+    def commit(self):
+        pass
+
+    def execute(
+        self,
+        sql: str,
+        auto_begin: bool = False,
+        fetch: bool = False,
+        limit: Optional[int] = None,
+    ) -> Tuple[AdapterResponse, "agate.Table"]:
+        from dbt_common.clients.agate_helper import empty_table
+
+        sql = self._add_query_comment(sql)
+        _, cursor = self.add_query(sql, auto_begin)
+        response = self.get_response(cursor)
+        if fetch:
+            table = self.get_result_from_cursor(cursor, limit)
+        else:
+            table = empty_table()
+        return AdapterResponse(response), table
+
+    def add_query(
+        self,
+        sql: str,
+        auto_begin: bool = True,
+        bindings: Optional[Any] = None,
+        abridge_sql_log: bool = False,
+    ) -> Tuple[Connection, Any]:
+        sql = self._add_query_comment(sql)
+        conn = self.get_thread_connection()
+        dbapi_connection = conn.handle
+        with self.exception_handler(sql):
+            logger.debug(f"On {conn.name}: {sql}...")
+            pre = time.time()
+            cursor = dbapi_connection.cursor()
+            logger.info(f"Try to execute SQL: \n{sql}")
+            cursor.execute_scheme(sql) # TODO: split
+            status = self.get_response(cursor)
+            logger.info(
+                f"SQL status: {status} in {(time.time() - pre):0.2f} seconds"
+            )
+            return conn, cursor

dbt_ydb-0.0.1/dbt/adapters/ydb/impl.py

@@ -0,0 +1,333 @@
+from dbt.adapters.sql import SQLAdapter
+from dbt.adapters.base.relation import BaseRelation
+from dbt.adapters.base.impl import ConstraintSupport, ConstraintType
+from dbt.adapters.cache import _make_ref_key_dict
+from dbt.adapters.events.types import ColTypeChange, SchemaCreation, SchemaDrop
+import dbt_common.clients
+from dbt_common.exceptions import DbtRuntimeError, CompilationError
+from dbt_common.events.functions import fire_event
+from dbt.adapters.base.relation import InformationSchema
+
+from dbt.adapters.base.meta import AdapterMeta, available
+from dbt.adapters.events.logging import AdapterLogger
+from dbt_common.clients.agate_helper import DEFAULT_TYPE_TESTER
+# from dbt.adapters.base.column import Column
+
+from dbt.adapters.ydb import YDBConnectionManager
+from dbt.adapters.ydb.column import YDBColumn
+from dbt.adapters.ydb.relation import YDBRelation
+
+from typing import Any, Dict, FrozenSet, Iterable, List, TYPE_CHECKING, Optional, Set, Tuple
+
+import traceback
+
+if TYPE_CHECKING:
+    import agate
+
+logger = AdapterLogger("YDB")
+
+class YDBAdapter(SQLAdapter):
+    """
+    Controls actual implmentation of adapter, and ability to override certain methods.
+    """
+
+    ConnectionManager = YDBConnectionManager
+    Column = YDBColumn
+    Relation = YDBRelation
+
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,
+        ConstraintType.not_null: ConstraintSupport.NOT_SUPPORTED,
+        ConstraintType.unique: ConstraintSupport.NOT_SUPPORTED,
+        ConstraintType.primary_key: ConstraintSupport.ENFORCED,
+        ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,
+    }
+
+    @classmethod
+    def quote(self, identifier):
+        return '`{}`'.format(identifier)
+
+    @classmethod
+    def date_function(cls):
+        """
+        Returns canonical date func
+        """
+        return "CurrentUtcDate()"
+
+    @classmethod
+    def convert_text_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return 'Utf8'
+
+    @classmethod
+    def convert_number_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        import agate
+
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
+        return 'Double' if decimals else 'Int64'
+
+    @classmethod
+    def convert_datetime_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return 'DateTime'
+
+    @classmethod
+    def convert_date_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return 'Date'
+
+    def list_relations_without_caching(
+        self,
+        schema_relation: BaseRelation,
+    ) -> List[BaseRelation]:
+        if not self.check_schema_exists(schema_relation.database, schema_relation.schema):
+            logger.info(f"Unable to list relations in schema {schema_relation.schema}: schema does not exist")
+            return []
+
+        connection = self.connections.get_thread_connection()
+        dbapi_connection = connection.handle
+
+        current_prefix = dbapi_connection.table_path_prefix
+        dbapi_connection.table_path_prefix = schema_relation.schema
+
+        relations = []
+
+        for relation_type, relation_list in [
+            ("table", dbapi_connection.get_table_names()),
+            ("view", dbapi_connection.get_view_names()),
+        ]:
+            for relation in relation_list:
+                r = self.Relation.create(
+                        database=schema_relation.database,
+                        schema=schema_relation.schema,
+                        identifier=relation,
+                        type=relation_type,
+                    )
+
+                relations.append(r)
+
+        dbapi_connection.table_path_prefix = current_prefix
+
+        return relations
+
+    def get_columns_in_relation(self, relation):
+        connection = self.connections.get_thread_connection()
+        dbapi_connection = connection.handle
+
+        logger.info(f"Try to get columns from  {relation}")
+
+        rel_str = relation.render().replace('`', '')
+
+        if not dbapi_connection.check_exists(rel_str):
+            logger.info(f"Relation {rel_str} does not exist, unable to get columns")
+            return []
+
+        with dbapi_connection.cursor() as cur:
+            cur.execute(f"SELECT * FROM {relation} LIMIT 0")
+            return [YDBColumn(col[0], col[1]) for col in cur.description]
+
+    def get_rows_different_sql(
+        self,
+        relation_a: YDBRelation,
+        relation_b: YDBRelation,
+        column_names: Optional[List[str]] = None,
+        except_operator: Optional[str] = None,
+    ) -> str:
+        names: List[str]
+        if column_names is None:
+            columns = self.get_columns_in_relation(relation_a)
+            names = sorted((self.quote(c.name) for c in columns))
+        else:
+            names = sorted((self.quote(n) for n in column_names))
+
+        alias_a = 'ta'
+        alias_b = 'tb'
+        columns_csv_a = ', '.join([f'{alias_a}.{name}' for name in names])
+        columns_csv_b = ', '.join([f'{alias_b}.{name}' for name in names])
+        join_condition = ' AND '.join([f'{alias_a}.{name} = {alias_b}.{name}' for name in names])
+        first_column = names[0]
+
+        sql = COLUMNS_EQUAL_SQL.format(
+            alias_a=alias_a,
+            alias_b=alias_b,
+            first_column=first_column,
+            columns_a=columns_csv_a,
+            columns_b=columns_csv_b,
+            join_condition=join_condition,
+            relation_a=str(relation_a),
+            relation_b=str(relation_b),
+        )
+
+        return sql
+
+    def check_schema_exists(self, database: str, schema: str) -> bool:
+        connection = self.connections.get_thread_connection()
+        dbapi_connection = connection.handle
+
+        return dbapi_connection.check_exists(schema)
+
+    def create_schema(self, relation: BaseRelation) -> None:
+        relation = relation.without_identifier()
+        fire_event(SchemaCreation(relation=_make_ref_key_dict(relation)))
+
+        connection = self.connections.get_thread_connection()
+        dbapi_connection = connection.handle
+
+        if not relation.schema:
+            return
+
+        directory = f"{relation.database}/{relation.schema}"
+
+        dbapi_connection._driver.scheme_client.make_directory(directory)
+
+    def drop_schema(self, relation):
+        logger.info("DROP SCHEMA")
+        # traceback.print_stack()
+
+        if not self.check_schema_exists(relation.database, relation.schema):
+            logger.info("Unable to drop schema: does not exist")
+            return
+        relation = relation.without_identifier()
+        fire_event(SchemaDrop(relation=_make_ref_key_dict(relation)))
+
+        for inner_relation in self.list_relations_without_caching(relation):
+            logger.info(f"Drop child relation before drop schema: {inner_relation}")
+            self.drop_relation(inner_relation)
+
+        connection = self.connections.get_thread_connection()
+        dbapi_connection = connection.handle
+
+        if not relation.schema:
+            return
+
+        directory = f"{relation.database}/{relation.schema}"
+        dbapi_connection._driver.scheme_client.remove_directory(directory)
+
+        # we can update the cache here
+        self.cache.drop_schema(relation.database, relation.schema)
+
+        logger.debug("DROP SCHEMA SUCCESS")
+
+    @available
+    def prepare_insert_values_from_csv(self, table):
+        import agate
+        csv_funcs = [c.csvify for c in table._column_types]
+
+        out = []
+        def prepare_value(i, val):
+            ctype = table._column_types[i]
+            if isinstance(ctype, agate.data_types.DateTime):
+                return f'DateTime("{val}Z")'
+            if isinstance(ctype, agate.data_types.Date):
+                return f'Date("{val}")'
+            if isinstance(ctype, dbt_common.clients.agate_helper.Number):
+                return f"{val}"
+            return f'"{val}"'
+
+        for row in table.rows:
+            out.append(f"({', '.join(prepare_value(i, csv_funcs[i](d)) for i, d in enumerate(row))})")
+
+        return ",".join(out)
+
+    def rename_relation(self, from_relation: YDBRelation, to_relation: YDBRelation):
+        logger.debug(f"Try to rename relation from {from_relation}[{str(from_relation.type)}] to {to_relation}[{str(to_relation.type)}]")
+        self.drop_relation(to_relation)
+        if from_relation.type == "table":
+            import ydb
+            connection = self.connections.get_thread_connection()
+            dbapi_connection = connection.handle
+
+            from_path = f"{from_relation.database}/{from_relation.render()[1:-1]}".strip()
+            to_path = f"{to_relation.database}/{to_relation.render()[1:-1]}".strip()
+
+            rename_item = ydb.RenameItem(
+                from_path,
+                to_path,
+            )
+
+            dbapi_connection._driver.table_client.rename_tables([rename_item])
+        else:
+            raise DbtRuntimeError("View rename does not supported in YDB")
+        self.cache_renamed(from_relation, to_relation)
+
+    def get_catalog_for_single_relation(self, relation):
+        columns: List[Dict[str, Any]] = []
+        logger.debug("Getting table schema for relation {}", str(relation))
+        columns.extend(self.parse_columns_from_relation(relation))
+
+        import agate
+
+        return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER)
+
+    def _get_one_catalog(
+        self,
+        information_schema: InformationSchema,
+        schemas: Set[str],
+        used_schemas: FrozenSet[Tuple[str, str]],
+    ) -> "agate.Table":
+        if len(schemas) != 1:
+            raise CompilationError(
+                f"Expected only one schema in ydb _get_one_catalog, found " f"{schemas}"
+            )
+
+        database = information_schema.database
+        schema = list(schemas)[0]
+
+        columns: List[Dict[str, Any]] = []
+        relations = self.list_relations(database, schema)
+        for relation in relations:
+            logger.debug("Getting table schema for relation {}", str(relation))
+            columns.extend(self.parse_columns_from_relation(relation))
+
+        import agate
+
+        return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER)
+
+    def parse_columns_from_relation(self, relation: BaseRelation) -> List[Dict[str, Any]]:
+        owner = ""
+        ydb_columns = self.get_columns_in_relation(relation)
+        columns = []
+        for col_num, col in enumerate(ydb_columns):
+            column = {
+                "table_database": relation.database,
+                "table_schema": relation.schema,
+                "table_name": relation.identifier,
+                "table_type": relation.type,
+                "column_index": col_num,
+                "table_owner": owner,
+                "column_name": col.name,
+                "column_type": col.dtype
+            }
+            columns.append(column)
+        return columns
+
+
+
+COLUMNS_EQUAL_SQL = '''
+$rel_a = SELECT COUNT(*) as num_rows FROM {relation_a};
+$rel_b = SELECT COUNT(*) as num_rows FROM {relation_b};
+SELECT
+    row_count_diff.difference as row_count_difference,
+    diff_count.num_missing as num_mismatched
+FROM (
+    SELECT
+        1 as id,
+        $rel_a - $rel_b as difference
+    ) as row_count_diff
+INNER JOIN (
+    SELECT
+        1 as id,
+        COUNT(*) as num_missing FROM (
+            SELECT
+                {columns_a}
+            FROM {relation_a} as {alias_a}
+            LEFT OUTER JOIN {relation_b} as {alias_b}
+                ON {join_condition}
+            WHERE {alias_b}.{first_column} IS NULL
+            UNION ALL
+            SELECT
+                {columns_b}
+            FROM {relation_b} as {alias_b}
+            LEFT OUTER JOIN {relation_a} as {alias_a}
+                ON {join_condition}
+            WHERE {alias_a}.{first_column} IS NULL
+        ) as missing
+    ) as diff_count ON row_count_diff.id = diff_count.id
+'''.strip()

dbt_ydb-0.0.1/dbt/adapters/ydb/relation.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass, field
+
+from dbt.adapters.base.relation import BaseRelation, Policy
+
+
+@dataclass
+class YDBQuotePolicy(Policy):
+    database: bool = True
+    schema: bool = True
+    identifier: bool = True
+
+@dataclass
+class YDBIncludePolicy(Policy):
+    database: bool = False
+    schema: bool = True
+    identifier: bool = True
+
+@dataclass(frozen=True, eq=False, repr=False)
+class YDBRelation(BaseRelation):
+    quote_character: str = '`'
+    quote_policy: Policy = field(default_factory=lambda: YDBQuotePolicy())
+    include_policy: Policy = field(default_factory=lambda: YDBIncludePolicy())
+
+    def render(self) -> str:
+        # if there is nothing set, this will return the empty string.
+        return self.quoted("/".join([part for part in [self.schema, self.identifier] if part]))

dbt_ydb-0.0.1/dbt/include/ydb/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

dbt_ydb-0.0.1/dbt/include/ydb/dbt_project.yml

@@ -0,0 +1,5 @@
+name: dbt_ydb
+version: 0.0.0
+config-version: 2
+
+macro-paths: ["macros"]

dbt_ydb-0.0.1/dbt/include/ydb/macros/adapters.sql

@@ -0,0 +1,191 @@
+/* For examples of how to fill out the macros please refer to the postgres adapter and docs
+postgres adapter macros: https://github.com/dbt-labs/dbt-core/blob/main/plugins/postgres/dbt/include/postgres/macros/adapters.sql
+dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter
+*/
+
+{% macro ydb__alter_column_type(relation,column_name,new_column_type) -%}
+'''Changes column name or data type'''
+/*
+    1. Create a new column (w/ temp name and correct type)
+    2. Copy data over to it
+    3. Drop the existing column (cascade!)
+    4. Rename the new column to existing column
+*/
+{% endmacro %}
+
+{% macro ydb__check_schema_exists(information_schema,schema) -%}
+'''Checks if schema name exists and returns number or times it shows up.'''
+/*
+    1. Check if schemas exist
+    2. return number of rows or columns that match searched parameter
+*/
+{% endmacro %}
+
+--  Example from postgres adapter in dbt-core
+--  Notice how you can build out other methods than the designated ones for the impl.py file,
+--  to make a more robust adapter. ex. (verify_database)
+
+/*
+
+ {% macro postgres__create_schema(relation) -%}
+   {% if relation.database -%}
+    {{ adapter.verify_database(relation.database) }}
+  {%- endif -%}   {%- call statement('create_schema') -%}
+     create schema if not exists {{ relation.without_identifier().include(database=False) }}
+   {%- endcall -%}
+ {% endmacro %}
+
+*/
+
+{% macro ydb__create_schema(relation) -%}
+'''Creates a new schema in the  target database, if schema already exists, method is a no-op. '''
+{% endmacro %}
+
+/*
+
+{% macro postgres__drop_schema(relation) -%}
+  {% if relation.database -%}
+    {{ adapter.verify_database(relation.database) }}
+  {%- endif -%}
+  {%- call statement('drop_schema') -%}
+    drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade
+  {%- endcall -%}
+{% endmacro %}
+
+*/
+
+{% macro ydb__drop_relation(relation) -%}
+'''Deletes relatonship identifer between tables.'''
+/*
+  1. If database exists
+  2. Create a new schema if passed schema does not exist already
+*/
+  {%- set obj_type = relation.type -%}
+  {% call statement('drop_relation', auto_begin=False) -%}
+    drop {{ obj_type }} if exists {{ relation }}
+  {%- endcall %}
+
+{% endmacro %}
+
+{% macro ydb__drop_schema(relation) -%}
+'''drops a schema in a target database.'''
+/*
+  1. If database exists
+  2. search all calls of schema, and change include value to False, cascade it to backtrack
+*/
+{% endmacro %}
+
+/*
+
+ Example of 1 of 3 required macros that does not have a default implementation
+{% macro postgres__get_columns_in_relation(relation) -%}
+  {% call statement('get_columns_in_relation', fetch_result=True) %}
+      select
+          column_name,
+          data_type,
+          character_maximum_length,
+          numeric_precision,
+          numeric_scale
+      from {{ relation.information_schema('columns') }}
+      where table_name = '{{ relation.identifier }}'
+        {% if relation.schema %}
+        and table_schema = '{{ relation.schema }}'
+        {% endif %}
+      order by ordinal_position
+  {% endcall %}
+  {% set table = load_result('get_columns_in_relation').table %}
+  {{ return(sql_convert_columns_in_relation(table)) }}
+{% endmacro %}
+*/
+
+
+{% macro ydb__get_columns_in_relation(relation) -%}
+'''Returns a list of Columns in a table.'''
+/*
+  1. select as many values from column as needed
+  2. search relations to columns
+  3. where table name is equal to the relation identifier
+  4. if a relation schema exists and table schema is equal to the relation schema
+  5. order in whatever way you want to call.
+  6. create a table by loading result from call
+  7. return new table
+*/
+{% endmacro %}
+
+--  Example of 2 of 3 required macros that do not come with a default implementation
+
+/*
+
+{% macro postgres__list_relations_without_caching(schema_relation) %}
+  {% call statement('list_relations_without_caching', fetch_result=True) -%}
+    select
+      '{{ schema_relation.database }}' as database,
+      tablename as name,
+      schemaname as schema,
+      'table' as type
+    from pg_tables
+    where schemaname ilike '{{ schema_relation.schema }}'
+    union all
+    select
+      '{{ schema_relation.database }}' as database,
+      viewname as name,
+      schemaname as schema,
+      'view' as type
+    from pg_views
+    where schemaname ilike '{{ schema_relation.schema }}'
+  {% endcall %}
+  {{ return(load_result('list_relations_without_caching').table) }}
+{% endmacro %}
+
+*/
+
+/*
+{% macro ydb__list_relations_without_caching(schema_relation) -%}
+  {{ return(adapter.list_relations_without_caching(schema_relation)) }}
+{%- endmacro %}
+*/
+
+/*
+{% macro ydb__list_schemas(database) -%}
+'''Returns a table of unique schemas.'''
+/*
+  1. search schemea by specific name
+  2. create a table with names
+*/
+{% endmacro %}
+*/
+
+{% macro ydb__rename_relation(from_relation, to_relation) -%}
+'''Renames a relation in the database.'''
+/*
+  1. Search for a specific relation name
+  2. alter table by targeting specific name and passing in new name
+*/
+  {% if from_relation.type == "table" %}
+    {{ adapter.rename_table(from_relation, to_relation) }}
+
+  {% endif %}
+{% endmacro %}
+
+{% macro ydb__truncate_relation(relation) -%}
+'''Removes all rows from a targeted set of tables.'''
+/*
+  1. grab all tables tied to the relation
+  2. remove rows from relations
+*/
+{% endmacro %}
+
+/*
+
+Example 3 of 3 of required macros that does not have a default implementation.
+ ** Good example of building out small methods ** please refer to impl.py for implementation of now() in postgres plugin
+{% macro postgres__current_timestamp() -%}
+  now()
+{%- endmacro %}
+
+*/
+
+{% macro ydb__current_timestamp() -%}
+'''Returns current UTC time'''
+{# docs show not to be implemented currently. #}
+{% endmacro %}

dbt_ydb-0.0.1/dbt/include/ydb/macros/generic_test_sql/accepted_values.sql

@@ -0,0 +1,24 @@
+{% macro ydb__test_accepted_values(model, column_name, values, quote=True) %}
+
+select *
+from (
+    select
+        {{ column_name }} as value_field,
+        count(*) as n_records
+
+    from {{ model }}
+    group by {{ column_name }}
+) all_values
+
+where value_field not in (
+    {% for value in values -%}
+        {% if quote -%}
+        '{{ value }}'
+        {%- else -%}
+        {{ value }}
+        {%- endif -%}
+        {%- if not loop.last -%},{%- endif %}
+    {%- endfor %}
+)
+
+{% endmacro %}

dbt_ydb-0.0.1/dbt/include/ydb/macros/generic_test_sql/relationships.sql

@@ -0,0 +1,17 @@
+{% macro ydb__test_relationships(model, column_name, to, field) %}
+
+select
+    child.from_field
+from (
+    select {{ column_name }} as from_field
+    from {{ model }}
+    where {{ column_name }} is not null
+) as child
+left join (
+    select {{ field }} as to_field
+    from {{ to }}
+) as parent
+on child.from_field = parent.to_field
+where parent.to_field is null
+
+{% endmacro %}

dbt_ydb-0.0.1/dbt/include/ydb/macros/materializations/models/incremental/incremental.sql

@@ -0,0 +1,107 @@
+
+{% materialization incremental, adapter='ydb' -%}
+
+  -- relations
+  {%- set existing_relation = load_cached_relation(this) -%}
+
+  {%- set target_relation = this.incorporate(type='table') -%}
+  {%- set temp_relation = make_temp_relation(target_relation)-%}
+  {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}
+  {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}
+  {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
+
+  -- configs
+  {%- set unique_key = config.get('unique_key') -%}
+  {%- set full_refresh_mode = (should_full_refresh()  or existing_relation.is_view) -%}
+  {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
+
+  -- the temp_ and backup_ relations should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation. This has to happen before
+  -- BEGIN, in a separate transaction
+  {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}
+  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
+   -- grab current tables grants config for comparision later on
+  {% set grant_config = config.get('grants') %}
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  {% set to_drop = [] %}
+
+  {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}
+  {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}
+
+  {% if existing_relation is none %}
+      {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}
+      {% set relation_for_indexes = target_relation %}
+  {% elif full_refresh_mode %}
+      {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}
+      {% set relation_for_indexes = intermediate_relation %}
+      {% set need_swap = true %}
+  {% else %}
+    {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}
+    {% set relation_for_indexes = temp_relation %}
+    {% set contract_config = config.get('contract') %}
+    {% if not contract_config or not contract_config.enforced %}
+      {% do adapter.expand_target_column_types(
+               from_relation=temp_relation,
+               to_relation=target_relation) %}
+    {% endif %}
+    {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}
+    {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}
+    {% if not dest_columns %}
+      {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
+    {% endif %}
+
+    {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}
+    {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}
+    {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}
+    {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
+
+  {% endif %}
+
+  {% call statement("main") %}
+      {{ build_sql }}
+  {% endcall %}
+
+  {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
+    {% do create_indexes(relation_for_indexes) %}
+  {% endif %}
+
+  {% if need_swap %}
+    {% if existing_relation.is_view %}
+      {% call statement('main') -%}
+        {{ get_create_view_as_sql(backup_relation, sql) }}
+      {%- endcall %}
+      {{ drop_relation(existing_relation) }}
+    {% else %}
+      {% do adapter.rename_relation(target_relation, backup_relation) %}
+    {% endif %}
+    {% do adapter.rename_relation(intermediate_relation, target_relation) %}
+    {% do to_drop.append(backup_relation) %}
+  {% endif %}
+
+  {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {% do persist_docs(target_relation, model) %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {% do adapter.commit() %}
+
+  {% for rel in to_drop %}
+      {% do adapter.drop_relation(rel) %}
+  {% endfor %}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{%- endmaterialization %}

dbt_ydb-0.0.1/dbt/include/ydb/macros/materializations/models/table.sql

@@ -0,0 +1,19 @@
+{% macro ydb__create_table_as(temporary, relation, sql) -%}
+  {%- set sql_header = config.get('sql_header', none) -%}
+  {%- set primary_key_column = model['config'].get('primary_key', 'id') -%}
+
+  {{ sql_header if sql_header is not none }}
+
+  create {% if temporary: -%}temporary{%- endif %} table
+    {{ relation.include(database=(not temporary), schema=(not temporary)) }}
+  (primary key ({{ primary_key_column }}))
+  {% set contract_config = config.get('contract') %}
+  {% if contract_config.enforced and (not temporary) %}
+    {{ get_assert_columns_equivalent(sql) }}
+    {{ get_table_columns_and_constraints() }}
+    {%- set sql = get_select_subquery(sql) %}
+  {% endif %}
+  as (
+  {{ sql }}
+  )
+{%- endmacro %}

dbt_ydb-0.0.1/dbt/include/ydb/macros/materializations/models/view.sql

@@ -0,0 +1,91 @@
+{%- materialization view, adapter='ydb' -%}
+
+  {%- set existing_relation = load_cached_relation(this) -%}
+  {%- set target_relation = this.incorporate(type='view') -%}
+  {%- set intermediate_relation =  make_intermediate_relation(target_relation) -%}
+
+  -- the intermediate_relation should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation
+  {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}
+  /*
+     This relation (probably) doesn't exist yet. If it does exist, it's a leftover from
+     a previous run, and we're going to try to drop it immediately. At the end of this
+     materialization, we're going to rename the "existing_relation" to this identifier,
+     and then we're going to drop it. In order to make sure we run the correct one of:
+       - drop view ...
+       - drop table ...
+
+     We need to set the type of this relation to be the type of the existing_relation, if it exists,
+     or else "view" as a sane default if it does not. Note that if the existing_relation does not
+     exist, then there is nothing to move out of the way and subsequentally drop. In that case,
+     this relation will be effectively unused.
+  */
+  {%- set backup_relation_type = 'view' -%}
+
+  {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
+  -- as above, the backup_relation should not already exist
+  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
+  -- grab current tables grants config for comparision later on
+  {% set grant_config = config.get('grants') %}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- drop the temp relations if they exist already in the database
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% call statement('main') -%}
+    {{ get_create_view_as_sql(intermediate_relation, sql) }}
+  {%- endcall %}
+
+  -- cleanup
+  -- move the existing view out of the way
+  {% if existing_relation is not none %}
+     /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped
+        since the variable was first set. */
+    {% set existing_relation = load_cached_relation(existing_relation) %}
+    {% if existing_relation is not none %}
+        {% call statement('main') -%}
+          {{ get_create_view_as_sql(backup_relation, sql) }}
+        {%- endcall %}
+        {{ drop_relation_if_exists(existing_relation) }}
+    {% endif %}
+  {% endif %}
+
+  {% call statement('main') -%}
+    {{ get_create_view_as_sql(target_relation, sql) }}
+  {%- endcall %}
+
+  {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {% do persist_docs(target_relation, model) %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  {{ adapter.commit() }}
+
+  {{ drop_relation_if_exists(backup_relation) }}
+
+  {{ drop_relation_if_exists(intermediate_relation) }}
+
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{%- endmaterialization -%}
+
+{% macro ydb__create_view_as(relation, sql) -%}
+  create view {{ relation.include(database=False) }}
+  with (security_invoker = TRUE)
+  as (
+    {{ sql }}
+  )
+
+{%- endmacro %}

dbt_ydb-0.0.1/dbt/include/ydb/macros/materializations/seeds/seed.sql

@@ -0,0 +1,38 @@
+{% macro ydb__create_csv_table(model, agate_table) %}
+  {%- set column_override = model['config'].get('column_types', {}) -%}
+  {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
+  {%- set primary_key_column = model['config'].get('primary_key', agate_table.column_names[0]) -%}
+
+  {% set sql %}
+    create table {{ this.render() }} (
+        {%- for col_name in agate_table.column_names -%}
+            {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}
+            {%- set type = column_override.get(col_name, inferred_type) -%}
+            {%- set column_name = (col_name | string) -%}
+            {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}
+        {%- endfor -%}
+        , primary key ({{ primary_key_column }})
+    )
+  {% endset %}
+
+  {% call statement('_') -%}
+    {{ sql }}
+  {%- endcall %}
+
+  {{ return(sql) }}
+{% endmacro %}
+
+
+{% macro ydb__load_csv_rows(model, agate_table) %}
+  {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+  {% set data_sql = adapter.prepare_insert_values_from_csv(agate_table) %}
+
+  {% if data_sql %}
+    {% set sql -%}
+      upsert into {{ this.render() }} ({{ cols_sql }}) VALUES
+      {{ data_sql }}
+    {%- endset %}
+
+    {% do adapter.add_query(sql, bindings=agate_table, abridge_sql_log=True) %}
+  {% endif %}
+{% endmacro %}

dbt_ydb-0.0.1/dbt/include/ydb/profile_template.yml

@@ -0,0 +1,19 @@
+fixed:
+  type: ydb
+prompts:
+  host:
+    hint: "your host name"
+  port:
+    default: 5432
+    type: "int"
+  user:
+    hint: "dev username"
+  password:
+    hint: "dev password"
+    hide_input: true
+  dbname:
+    hint: "default database"
+  threads:
+    hint: "1 or more"
+    type: "int"
+    default: 1

dbt_ydb-0.0.1/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "dbt-ydb"
+version = "0.0.1"  # AUTOVERSION
+description = "DBT adapter for YDB"
+authors = [
+    {name = "Yandex LLC", email = "ydb@yandex-team.ru"}
+]
+readme = "README.md"
+requires-python = ">=3.10,<4.0"
+dependencies = [
+    "dbt-core (>=1.8.0,<2.0.0)",
+    "dbt-common (>=1.23.0,<2.0.0)",
+    "ydb-dbapi (>=0.1.12,<0.2.0)",
+    "dbt-adapters (>=1.14.8,<2.0.0)"
+]
+
+[tool.poetry]
+packages = [
+    { include = "dbt" },
+    { include = "dbt/**/*.py" },
+]
+
+[tool.poetry.group.dev.dependencies]
+ruff = "^0.11.7"
+pytest = "^8.3.5"
+dbt-tests-adapter = "^1.11.0"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"