datus-db-core 0.1.0__tar.gz

datus_db_core-0.1.0/.gitignore

@@ -0,0 +1,140 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# uv
+uv.lock
+
+# PEP 582
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+
+.omc

datus_db_core-0.1.0/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.4
+Name: datus-db-core
+Version: 0.1.0
+Summary: Core database abstractions for Datus adapters
+Project-URL: Homepage, https://github.com/Datus-ai/datus-db-adapters
+Project-URL: Repository, https://github.com/Datus-ai/datus-db-adapters
+Project-URL: Issues, https://github.com/Datus-ai/datus-db-adapters/issues
+Author-email: DatusAI <support@datus.ai>
+License: Apache-2.0
+Keywords: adapter,core,database,datus
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: sqlglot>=20.0.0
+Description-Content-Type: text/markdown
+
+# datus-db-core
+
+Core database abstractions for Datus adapters. Provides `BaseSqlConnector`, `ConnectorRegistry`, SQL parsing utilities, and shared models used by all database adapter packages.
+
+## Installation
+
+```bash
+pip install datus-db-core
+```

datus_db_core-0.1.0/README.md

@@ -0,0 +1,9 @@
+# datus-db-core
+
+Core database abstractions for Datus adapters. Provides `BaseSqlConnector`, `ConnectorRegistry`, SQL parsing utilities, and shared models used by all database adapter packages.
+
+## Installation
+
+```bash
+pip install datus-db-core
+```

datus_db_core-0.1.0/datus_db_core/__init__.py

@@ -0,0 +1,36 @@
+# Copyright 2025-present DatusAI, Inc.
+# Licensed under the Apache License, Version 2.0.
+# See http://www.apache.org/licenses/LICENSE-2.0 for details.
+
+from datus_db_core.base import BaseSqlConnector, _to_sql_literal, list_to_in_str
+from datus_db_core.config import ConnectionConfig
+from datus_db_core.constants import SQLType
+from datus_db_core.exceptions import DatusException, ErrorCode
+from datus_db_core.logging import get_logger
+from datus_db_core.mixins import CatalogSupportMixin, MaterializedViewSupportMixin, SchemaNamespaceMixin
+from datus_db_core.models import TABLE_TYPE, ExecuteSQLInput, ExecuteSQLResult
+from datus_db_core.registry import AdapterMetadata, ConnectorRegistry, connector_registry
+from datus_db_core.sql_utils import metadata_identifier, parse_context_switch, parse_sql_type
+
+__all__ = [
+    "BaseSqlConnector",
+    "list_to_in_str",
+    "_to_sql_literal",
+    "ConnectionConfig",
+    "SQLType",
+    "DatusException",
+    "ErrorCode",
+    "get_logger",
+    "CatalogSupportMixin",
+    "MaterializedViewSupportMixin",
+    "SchemaNamespaceMixin",
+    "TABLE_TYPE",
+    "ExecuteSQLInput",
+    "ExecuteSQLResult",
+    "AdapterMetadata",
+    "ConnectorRegistry",
+    "connector_registry",
+    "metadata_identifier",
+    "parse_context_switch",
+    "parse_sql_type",
+]

datus_db_core-0.1.0/datus_db_core/base.py

@@ -0,0 +1,300 @@
+# Copyright 2025-present DatusAI, Inc.
+# Licensed under the Apache License, Version 2.0.
+# See http://www.apache.org/licenses/LICENSE-2.0 for details.
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Iterator, List, Literal, Optional, Set, Tuple
+
+from datus_db_core.config import ConnectionConfig
+from datus_db_core.constants import SQLType
+from datus_db_core.logging import get_logger
+from datus_db_core.models import TABLE_TYPE, ExecuteSQLInput, ExecuteSQLResult
+from datus_db_core.sql_utils import metadata_identifier, parse_sql_type
+
+logger = get_logger(__name__)
+
+
+class BaseSqlConnector(ABC):
+    def __init__(self, config: ConnectionConfig, dialect: str):
+        self.config = config
+        self.timeout_seconds = config.timeout_seconds
+        self.connection: Any = None
+        self.dialect = dialect
+        self.catalog_name = ""
+        self.database_name = ""
+        self.schema_name = ""
+
+    def close(self):
+        if self.connection:
+            self.connection.close()
+            self.connection = None
+
+    def connect(self):
+        return
+
+    def __enter__(self):
+        self.connect()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type:
+            try:
+                self._safe_rollback()
+            except Exception as e:
+                logger.warning(f"Failed to rollback during cleanup: {e}")
+        try:
+            self.close()
+        except Exception as e:
+            logger.warning(f"Failed to close connection during cleanup: {e}")
+        return False
+
+    def _safe_rollback(self):
+        if hasattr(self, "connection") and self.connection:
+            try:
+                if hasattr(self.connection, "rollback"):
+                    self.connection.rollback()
+            except Exception:
+                pass
+
+    def execute(
+        self,
+        input_params: Any,
+        result_format: Optional[Literal["csv", "arrow", "pandas", "list"]] = None,
+    ) -> ExecuteSQLResult:
+        self.validate_input(input_params)
+        if isinstance(input_params, dict):
+            input_params = ExecuteSQLInput(**input_params)
+        sql_query = input_params.sql_query.strip()
+        if result_format is None:
+            result_format = getattr(input_params, "result_format", "csv") or "csv"
+        try:
+            sql_type = parse_sql_type(sql_query, self.dialect)
+            if sql_type == SQLType.INSERT:
+                result = self.execute_insert(sql_query)
+            elif sql_type in (SQLType.UPDATE, SQLType.MERGE):
+                result = self.execute_update(sql_query)
+            elif sql_type == SQLType.DELETE:
+                result = self.execute_delete(sql_query)
+            elif sql_type == SQLType.CONTENT_SET:
+                result = self.execute_content_set(sql_query)
+            elif sql_type == SQLType.DDL:
+                result = self.execute_ddl(sql_query)
+            elif sql_type == SQLType.SELECT:
+                result = self.execute_query(sql_query, result_format)
+            elif sql_type == SQLType.METADATA_SHOW:
+                result = self.execute_query(sql_query, result_format)
+            elif sql_type == SQLType.EXPLAIN:
+                result = self.execute_explain(sql_query, result_format)
+            else:
+                return ExecuteSQLResult(
+                    success=False,
+                    error="Unknown type of SQL",
+                    sql_query=sql_query,
+                    sql_return="",
+                    row_count=0,
+                    result_format=result_format,
+                )
+
+            return result
+        except Exception as e:
+            logger.error(f"Executing SQL query failed: {e}")
+            return ExecuteSQLResult(
+                success=False,
+                error=str(e),
+                sql_query=sql_query,
+                sql_return="",
+                row_count=0,
+                result_format=result_format,
+            )
+
+    @abstractmethod
+    def execute_insert(self, sql: str) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def execute_update(self, sql: str) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def execute_delete(self, sql: str) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+    def validate_input(self, input_params: Any):
+        if isinstance(input_params, dict):
+            if "sql_query" not in input_params:
+                raise ValueError("'sql_query' parameter is required")
+            if not isinstance(input_params["sql_query"], str):
+                raise ValueError("'sql_query' must be a string")
+        else:
+            if not hasattr(input_params, "sql_query"):
+                raise ValueError("'sql_query' parameter is required")
+            if not isinstance(input_params.sql_query, str):
+                raise ValueError("'sql_query' must be a string")
+
+    def execute_arrow(self, sql: str) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def execute_query(
+        self, sql: str, result_format: Literal["csv", "arrow", "pandas", "list"] = "csv"
+    ) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+    def execute_explain(
+        self, sql: str, result_format: Literal["csv", "arrow", "pandas", "list"] = "csv"
+    ) -> ExecuteSQLResult:
+        return self.execute_query(sql, result_format)
+
+    @abstractmethod
+    def execute_pandas(self, sql: str) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def execute_ddl(self, sql: str) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def execute_csv(self, sql: str) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_databases(self, catalog_name: str = "", include_sys: bool = False) -> List[str]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_tables(self, catalog_name: str = "", database_name: str = "", schema_name: str = "") -> List[str]:
+        raise NotImplementedError()
+
+    def get_views(self, catalog_name: str = "", database_name: str = "", schema_name: str = "") -> List[str]:
+        return []
+
+    def _sys_databases(self) -> Set[str]:
+        return set()
+
+    def _sys_schemas(self) -> Set[str]:
+        return set()
+
+    def execute_csv_iterator(
+        self, query: str, max_rows: int = 100, with_header: bool = True
+    ) -> Iterator[Tuple[str, ...]]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def test_connection(self):
+        raise NotImplementedError()
+
+    def get_type(self) -> str:
+        return self.dialect
+
+    @abstractmethod
+    def execute_queries(self, queries: List[str]) -> List[Any]:
+        raise NotImplementedError()
+
+    def get_tables_with_ddl(
+        self,
+        catalog_name: str = "",
+        database_name: str = "",
+        schema_name: str = "",
+        tables: Optional[List[str]] = None,
+    ) -> List[Dict[str, str]]:
+        raise NotImplementedError()
+
+    def _reset_filter_tables(
+        self,
+        tables: Optional[List[str]] = None,
+        catalog_name: str = "",
+        database_name: str = "",
+        schema_name: str = "",
+    ) -> List[str]:
+        filter_tables = []
+        if tables:
+            catalog_name = catalog_name or self.catalog_name
+            database_name = database_name or self.database_name
+            schema_name = schema_name or self.schema_name
+            for table_name in tables:
+                filter_tables.append(
+                    self.full_name(
+                        table_name=table_name,
+                        catalog_name=catalog_name,
+                        database_name=database_name,
+                        schema_name=schema_name,
+                    )
+                )
+        return filter_tables
+
+    def get_views_with_ddl(
+        self, catalog_name: str = "", database_name: str = "", schema_name: str = ""
+    ) -> List[Dict[str, str]]:
+        raise NotImplementedError()
+
+    def switch_context(self, catalog_name: str = "", database_name: str = "", schema_name: str = ""):
+        self.connect()
+        self.do_switch_context(catalog_name=catalog_name, database_name=database_name, schema_name=schema_name)
+        if catalog_name:
+            self.catalog_name = catalog_name
+        if database_name:
+            self.database_name = database_name
+        if schema_name:
+            self.schema_name = schema_name
+
+    def do_switch_context(self, catalog_name: str = "", database_name: str = "", schema_name: str = ""):
+        return None
+
+    def get_schema(
+        self, catalog_name: str = "", database_name: str = "", schema_name: str = "", table_name: str = ""
+    ) -> List[Dict[str, str]]:
+        raise NotImplementedError()
+
+    def get_sample_rows(
+        self,
+        tables: Optional[List[str]] = None,
+        top_n: int = 5,
+        catalog_name: str = "",
+        database_name: str = "",
+        schema_name: str = "",
+        table_type: TABLE_TYPE = "table",
+    ) -> List[Dict[str, Any]]:
+        raise NotImplementedError()
+
+    def full_name(
+        self, catalog_name: str = "", database_name: str = "", schema_name: str = "", table_name: str = ""
+    ) -> str:
+        raise NotImplementedError()
+
+    def identifier(
+        self, catalog_name: str = "", database_name: str = "", schema_name: str = "", table_name: str = ""
+    ) -> str:
+        return metadata_identifier(
+            dialect=self.dialect,
+            catalog_name=catalog_name,
+            database_name=database_name,
+            schema_name=schema_name,
+            table_name=table_name,
+        )
+
+    @abstractmethod
+    def execute_content_set(self, sql_query: str) -> ExecuteSQLResult:
+        raise NotImplementedError()
+
+
+def list_to_in_str(prefix: str, values: Optional[List[str]] = None) -> str:
+    if not values:
+        return ""
+    value_str = ",".join(_to_sql_literal(v, around_with_quotes=True) for v in values)
+    return f"{prefix} ({value_str})"
+
+
+def _escape_sql_string_standard(value: str) -> str:
+    return value.replace("'", "''")
+
+
+def _to_sql_literal(value: Optional[str], around_with_quotes: bool = False) -> str:
+    if value is None:
+        return "NULL"
+    if not value:
+        return "" if not around_with_quotes else "''"
+    replace_value = _escape_sql_string_standard(value)
+    if not around_with_quotes:
+        return replace_value
+    else:
+        return f"'{replace_value}'"

datus_db_core-0.1.0/datus_db_core/config.py

@@ -0,0 +1,13 @@
+# Copyright 2025-present DatusAI, Inc.
+# Licensed under the Apache License, Version 2.0.
+# See http://www.apache.org/licenses/LICENSE-2.0 for details.
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ConnectionConfig(BaseModel):
+    """Base connection configuration for all database connectors."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    timeout_seconds: int = Field(default=30, description="Connection timeout in seconds")

datus_db_core-0.1.0/datus_db_core/constants.py

@@ -0,0 +1,20 @@
+# Copyright 2025-present DatusAI, Inc.
+# Licensed under the Apache License, Version 2.0.
+# See http://www.apache.org/licenses/LICENSE-2.0 for details.
+
+from enum import Enum
+
+
+class SQLType(str, Enum):
+    """SQL statement types."""
+
+    SELECT = "select"
+    INSERT = "insert"
+    UPDATE = "update"
+    DELETE = "delete"
+    MERGE = "merge"
+    DDL = "ddl"
+    METADATA_SHOW = "metadata"
+    EXPLAIN = "explain"
+    CONTENT_SET = "context_set"
+    UNKNOWN = "unknown"

datus_db_core-0.1.0/datus_db_core/exceptions.py

@@ -0,0 +1,74 @@
+# Copyright 2025-present DatusAI, Inc.
+# Licensed under the Apache License, Version 2.0.
+# See http://www.apache.org/licenses/LICENSE-2.0 for details.
+
+from enum import Enum
+from typing import Any, Optional
+
+
+class ErrorCode(Enum):
+    """Error codes for Datus database exceptions."""
+
+    # Common errors
+    COMMON_UNKNOWN = ("100000", "Unknown error occurred")
+    COMMON_FIELD_INVALID = (
+        "100001",
+        "Unexpected value of {field_name}, expected value: {expected_values}, your value: {your_value}",
+    )
+    COMMON_INVALID_PARAMETER = ("100002", "Invalid parameter: {error_message}")
+    COMMON_FIELD_REQUIRED = ("100003", "Missing required field: {field_name}")
+    COMMON_UNSUPPORTED = ("100004", "Unsupported value `{your_value}` for field `{field_name}`")
+    COMMON_CONFIG_ERROR = ("100006", "Configuration error: {config_error}")
+    COMMON_MISSING_DEPENDENCY = ("100007", "Missing required dependency: {dependency}")
+
+    # Database errors
+    DB_FAILED = ("500000", "Database operation failed. Error details: {error_message}")
+    DB_CONNECTION_FAILED = ("500001", "Failed to establish connection to database. Error details: {error_message}")
+    DB_CONNECTION_TIMEOUT = ("500002", "Connection to database timed out. Error details: {error_message}")
+    DB_AUTHENTICATION_FAILED = (
+        "500003",
+        "Authentication failed for database. Please check your credentials. Error details: {error_message}",
+    )
+    DB_PERMISSION_DENIED = (
+        "500004",
+        "Permission denied when performing '{operation}' on database. Error details: {error_message}",
+    )
+    DB_EXECUTION_SYNTAX_ERROR = ("500005", "Invalid SQL syntax in query. Error details: {error_message}")
+    DB_EXECUTION_ERROR = ("500006", "Failed to execute query on database. Error details: {error_message}")
+    DB_EXECUTION_TIMEOUT = ("500007", "Query execution timed out on database. Error details: {error_message}")
+    DB_QUERY_METADATA_FAILED = ("500008", "Failed to retrieve metadata for query. Error details: {error_message}")
+    DB_CONSTRAINT_VIOLATION = ("500011", "Database constraint violation occurred. Error details: {error_message}")
+    DB_TRANSACTION_FAILED = ("500009", "Database transaction failed. Error details: {error_message}")
+    DB_TABLE_NOT_EXISTS = ("500010", "Table {table_name} does not exist.")
+
+    def __init__(self, code: str, desc: str):
+        self.code = code
+        self.desc = desc
+
+
+class DatusException(Exception):
+    """Datus custom exception with error code."""
+
+    def __init__(
+        self,
+        code: ErrorCode,
+        message: Optional[str] = None,
+        message_args: Optional[dict[str, Any]] = None,
+        *args: object,
+    ):
+        self.code = code
+        self.message_args = message_args or {}
+        self.message = self.build_msg(message, message_args)
+        super().__init__(self.message, *args)
+
+    def __str__(self):
+        return self.message
+
+    def build_msg(self, message: Optional[str] = None, message_args: Optional[dict[str, Any]] = None) -> str:
+        if message:
+            final_message = message
+        elif message_args:
+            final_message = self.code.desc.format(**message_args)
+        else:
+            final_message = self.code.desc
+        return f"error_code={self.code.code}, error_message={final_message}"

datus_db_core-0.1.0/datus_db_core/logging.py

@@ -0,0 +1,9 @@
+# Copyright 2025-present DatusAI, Inc.
+# Licensed under the Apache License, Version 2.0.
+# See http://www.apache.org/licenses/LICENSE-2.0 for details.
+
+import logging
+
+
+def get_logger(name: str) -> logging.Logger:
+    return logging.getLogger(name)

datus_db_core-0.1.0/datus_db_core/mixins.py

@@ -0,0 +1,46 @@
+# Copyright 2025-present DatusAI, Inc.
+# Licensed under the Apache License, Version 2.0.
+# See http://www.apache.org/licenses/LICENSE-2.0 for details.
+
+from abc import ABC, abstractmethod
+from typing import Dict, List
+
+
+class CatalogSupportMixin(ABC):
+    """Mixin for databases that support catalog namespace."""
+
+    @abstractmethod
+    def get_catalogs(self) -> List[str]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def switch_catalog(self, catalog_name: str) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def default_catalog(self) -> str:
+        raise NotImplementedError
+
+
+class MaterializedViewSupportMixin(ABC):
+    """Mixin for databases that support materialized views."""
+
+    @abstractmethod
+    def get_materialized_views(
+        self, catalog_name: str = "", database_name: str = "", schema_name: str = ""
+    ) -> List[str]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_materialized_views_with_ddl(
+        self, catalog_name: str = "", database_name: str = "", schema_name: str = ""
+    ) -> List[Dict[str, str]]:
+        raise NotImplementedError
+
+
+class SchemaNamespaceMixin(ABC):
+    """Mixin for databases that support schema-level namespace."""
+
+    @abstractmethod
+    def get_schemas(self, catalog_name: str = "", database_name: str = "", include_sys: bool = False) -> List[str]:
+        raise NotImplementedError