datus-postgresql 0.1.0__tar.gz → 0.1.2__tar.gz

{datus_postgresql-0.1.0 → datus_postgresql-0.1.2}/.gitignore

@@ -135,3 +135,6 @@ dmypy.json
 # OS
 .DS_Store
 Thumbs.db
+
+
+.omc

datus_postgresql-0.1.2/PKG-INFO

@@ -0,0 +1,232 @@
+Metadata-Version: 2.4
+Name: datus-postgresql
+Version: 0.1.2
+Summary: PostgreSQL database adapter for Datus
+Project-URL: Homepage, https://github.com/Datus-ai/datus-db-adapters
+Project-URL: Repository, https://github.com/Datus-ai/datus-db-adapters
+Project-URL: Issues, https://github.com/Datus-ai/datus-db-adapters/issues
+Author-email: DatusAI <support@datus.ai>
+License: Apache-2.0
+Keywords: adapter,database,datus,postgresql
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
+Requires-Dist: datus-db-core>=0.1.0
+Requires-Dist: datus-sqlalchemy>=0.1.2
+Requires-Dist: psycopg2-binary>=2.9.11
+Requires-Dist: pydantic>=2.0.0
+Description-Content-Type: text/markdown
+
+# datus-postgresql
+
+PostgreSQL database adapter for [Datus](https://github.com/Datus-ai/datus-agent).
+
+## Overview
+
+This adapter provides PostgreSQL connectivity for Datus, supporting full SQL operations, metadata discovery, schema management, and materialized views via the standard PostgreSQL protocol.
+
+## Installation
+
+```bash
+pip install datus-postgresql
+```
+
+This will automatically install the required dependencies:
+- `datus-agent`
+- `datus-sqlalchemy` (which includes SQLAlchemy and psycopg2)
+
+## Usage
+
+The adapter is automatically registered with Datus when installed. Configure your database connection:
+
+```yaml
+namespace:
+  postgresql_prod:
+    type: postgresql
+    host: localhost
+    port: 5432
+    username: postgres
+    password: your_password
+    database: mydb
+    schema: public
+```
+
+Or use programmatically:
+
+```python
+from datus_postgresql import PostgreSQLConnector, PostgreSQLConfig
+
+# Using config object
+config = PostgreSQLConfig(
+    host="localhost",
+    port=5432,
+    username="postgres",
+    password="password",
+    database="mydb",
+    schema_name="public",
+)
+
+connector = PostgreSQLConnector(config)
+
+# Or using dict
+connector = PostgreSQLConnector({
+    "host": "localhost",
+    "port": 5432,
+    "username": "postgres",
+    "password": "password",
+    "database": "mydb",
+})
+
+# Use context manager for automatic cleanup
+with connector:
+    # Test connection
+    connector.test_connection()
+
+    # Execute queries
+    result = connector.execute({"sql_query": "SELECT * FROM users LIMIT 10"})
+    print(result.sql_return)
+
+    # Get tables
+    tables = connector.get_tables(schema_name="public")
+    print(f"Tables: {tables}")
+
+    # Get table schema
+    columns = connector.get_schema(schema_name="public", table_name="users")
+    for col in columns:
+        print(f"  {col['name']}: {col['type']}")
+
+    # Get materialized views
+    mvs = connector.get_materialized_views(schema_name="public")
+    print(f"Materialized Views: {mvs}")
+```
+
+## Configuration Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `host` | str | `"127.0.0.1"` | PostgreSQL server host |
+| `port` | int | `5432` | PostgreSQL server port |
+| `username` | str | required | PostgreSQL username |
+| `password` | str | `""` | PostgreSQL password |
+| `database` | str | `None` | Default database name |
+| `schema_name` | str | `"public"` | Default schema name |
+| `sslmode` | str | `"prefer"` | SSL mode (`disable`, `allow`, `prefer`, `require`, `verify-ca`, `verify-full`) |
+| `timeout_seconds` | int | `30` | Connection timeout in seconds |
+
+## Features
+
+- **Full SQL Support**: Execute queries, DDL, DML operations
+- **Metadata Discovery**: Automatic discovery of databases, schemas, tables, views, and materialized views
+- **DDL Generation**: Reconstruct CREATE TABLE / VIEW / MATERIALIZED VIEW statements
+- **Schema Management**: Switch between schemas seamlessly
+- **Sample Data**: Extract sample rows for data profiling
+- **Connection Management**: SQLAlchemy-based connection pooling with SSL support
+- **Multiple Result Formats**: pandas, arrow, csv, list
+
+## Code Structure
+
+```text
+datus-postgresql/
+├── datus_postgresql/
+│   ├── __init__.py          # Package exports
+│   ├── config.py            # PostgreSQLConfig (Pydantic model)
+│   └── connector.py         # PostgreSQLConnector implementation
+├── tests/
+│   ├── unit/                # Unit tests with mocks (no database needed)
+│   └── integration/         # Integration tests (requires PostgreSQL)
+│       ├── conftest.py      # Test fixtures and TPC-H data setup
+│       ├── test_integration.py  # General integration tests
+│       └── test_tpch.py     # TPC-H benchmark data tests
+├── scripts/
+│   └── init_tpch_data.py    # TPC-H data initialization script
+├── docker-compose.yml       # PostgreSQL test container
+├── pyproject.toml
+└── README.md
+```
+
+## Testing
+
+### Quick Start
+
+```bash
+# 1. Start PostgreSQL test container
+docker-compose up -d
+
+# 2. Run tests
+pytest tests/unit/ -v              # Unit tests (no database needed)
+pytest tests/integration/ -v       # Integration tests (requires PostgreSQL)
+pytest tests/ -v                   # All tests
+```
+
+### Test Types
+
+- **Unit tests**: Configuration and connector logic with mocks (no database needed)
+- **Integration tests**: Real database operations (SQL, metadata, schema)
+- **TPC-H tests**: Analytical query tests using TPC-H benchmark data
+
+### TPC-H Integration Tests
+
+The adapter includes TPC-H benchmark data tests for validating analytical query capabilities.
+
+#### TPC-H Tables
+
+| Table | Rows | Description |
+|-------|------|-------------|
+| `tpch_region` | 5 | World regions |
+| `tpch_nation` | 25 | Countries with region references |
+| `tpch_supplier` | 5 | Suppliers with nation references |
+| `tpch_customer` | 10 | Customers with nation references |
+| `tpch_orders` | 15 | Orders with customer references |
+
+#### Running TPC-H Tests
+
+```bash
+# Start PostgreSQL container
+docker-compose up -d
+
+# Run TPC-H integration tests
+pytest tests/integration/test_tpch.py -v
+
+# Initialize TPC-H data manually (for ad-hoc testing)
+python scripts/init_tpch_data.py \
+    --host localhost --port 5432 \
+    --username test_user --password test_password \
+    --database test --schema public
+
+# Drop and recreate TPC-H tables
+python scripts/init_tpch_data.py --drop
+```
+
+#### Environment Variables
+
+Tests use these default values (matching docker-compose.yml):
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `POSTGRESQL_HOST` | `localhost` | PostgreSQL server host |
+| `POSTGRESQL_PORT` | `5432` | PostgreSQL server port |
+| `POSTGRESQL_USER` | `test_user` | PostgreSQL username |
+| `POSTGRESQL_PASSWORD` | `test_password` | PostgreSQL password |
+| `POSTGRESQL_DATABASE` | `test` | Database name |
+| `POSTGRESQL_SCHEMA` | `public` | Schema name |
+
+## Requirements
+
+- Python >= 3.12
+- PostgreSQL >= 12
+- datus-agent >= 0.2.1
+- datus-sqlalchemy >= 0.1.0
+
+## License
+
+Apache License 2.0
+
+## Related Packages
+
+- `datus-sqlalchemy` - SQLAlchemy base connector
+- `datus-mysql` - MySQL adapter
+- `datus-redshift` - Amazon Redshift adapter
+- `datus-snowflake` - Snowflake adapter

datus_postgresql-0.1.2/README.md

@@ -0,0 +1,210 @@
+# datus-postgresql
+
+PostgreSQL database adapter for [Datus](https://github.com/Datus-ai/datus-agent).
+
+## Overview
+
+This adapter provides PostgreSQL connectivity for Datus, supporting full SQL operations, metadata discovery, schema management, and materialized views via the standard PostgreSQL protocol.
+
+## Installation
+
+```bash
+pip install datus-postgresql
+```
+
+This will automatically install the required dependencies:
+- `datus-agent`
+- `datus-sqlalchemy` (which includes SQLAlchemy and psycopg2)
+
+## Usage
+
+The adapter is automatically registered with Datus when installed. Configure your database connection:
+
+```yaml
+namespace:
+  postgresql_prod:
+    type: postgresql
+    host: localhost
+    port: 5432
+    username: postgres
+    password: your_password
+    database: mydb
+    schema: public
+```
+
+Or use programmatically:
+
+```python
+from datus_postgresql import PostgreSQLConnector, PostgreSQLConfig
+
+# Using config object
+config = PostgreSQLConfig(
+    host="localhost",
+    port=5432,
+    username="postgres",
+    password="password",
+    database="mydb",
+    schema_name="public",
+)
+
+connector = PostgreSQLConnector(config)
+
+# Or using dict
+connector = PostgreSQLConnector({
+    "host": "localhost",
+    "port": 5432,
+    "username": "postgres",
+    "password": "password",
+    "database": "mydb",
+})
+
+# Use context manager for automatic cleanup
+with connector:
+    # Test connection
+    connector.test_connection()
+
+    # Execute queries
+    result = connector.execute({"sql_query": "SELECT * FROM users LIMIT 10"})
+    print(result.sql_return)
+
+    # Get tables
+    tables = connector.get_tables(schema_name="public")
+    print(f"Tables: {tables}")
+
+    # Get table schema
+    columns = connector.get_schema(schema_name="public", table_name="users")
+    for col in columns:
+        print(f"  {col['name']}: {col['type']}")
+
+    # Get materialized views
+    mvs = connector.get_materialized_views(schema_name="public")
+    print(f"Materialized Views: {mvs}")
+```
+
+## Configuration Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `host` | str | `"127.0.0.1"` | PostgreSQL server host |
+| `port` | int | `5432` | PostgreSQL server port |
+| `username` | str | required | PostgreSQL username |
+| `password` | str | `""` | PostgreSQL password |
+| `database` | str | `None` | Default database name |
+| `schema_name` | str | `"public"` | Default schema name |
+| `sslmode` | str | `"prefer"` | SSL mode (`disable`, `allow`, `prefer`, `require`, `verify-ca`, `verify-full`) |
+| `timeout_seconds` | int | `30` | Connection timeout in seconds |
+
+## Features
+
+- **Full SQL Support**: Execute queries, DDL, DML operations
+- **Metadata Discovery**: Automatic discovery of databases, schemas, tables, views, and materialized views
+- **DDL Generation**: Reconstruct CREATE TABLE / VIEW / MATERIALIZED VIEW statements
+- **Schema Management**: Switch between schemas seamlessly
+- **Sample Data**: Extract sample rows for data profiling
+- **Connection Management**: SQLAlchemy-based connection pooling with SSL support
+- **Multiple Result Formats**: pandas, arrow, csv, list
+
+## Code Structure
+
+```text
+datus-postgresql/
+├── datus_postgresql/
+│   ├── __init__.py          # Package exports
+│   ├── config.py            # PostgreSQLConfig (Pydantic model)
+│   └── connector.py         # PostgreSQLConnector implementation
+├── tests/
+│   ├── unit/                # Unit tests with mocks (no database needed)
+│   └── integration/         # Integration tests (requires PostgreSQL)
+│       ├── conftest.py      # Test fixtures and TPC-H data setup
+│       ├── test_integration.py  # General integration tests
+│       └── test_tpch.py     # TPC-H benchmark data tests
+├── scripts/
+│   └── init_tpch_data.py    # TPC-H data initialization script
+├── docker-compose.yml       # PostgreSQL test container
+├── pyproject.toml
+└── README.md
+```
+
+## Testing
+
+### Quick Start
+
+```bash
+# 1. Start PostgreSQL test container
+docker-compose up -d
+
+# 2. Run tests
+pytest tests/unit/ -v              # Unit tests (no database needed)
+pytest tests/integration/ -v       # Integration tests (requires PostgreSQL)
+pytest tests/ -v                   # All tests
+```
+
+### Test Types
+
+- **Unit tests**: Configuration and connector logic with mocks (no database needed)
+- **Integration tests**: Real database operations (SQL, metadata, schema)
+- **TPC-H tests**: Analytical query tests using TPC-H benchmark data
+
+### TPC-H Integration Tests
+
+The adapter includes TPC-H benchmark data tests for validating analytical query capabilities.
+
+#### TPC-H Tables
+
+| Table | Rows | Description |
+|-------|------|-------------|
+| `tpch_region` | 5 | World regions |
+| `tpch_nation` | 25 | Countries with region references |
+| `tpch_supplier` | 5 | Suppliers with nation references |
+| `tpch_customer` | 10 | Customers with nation references |
+| `tpch_orders` | 15 | Orders with customer references |
+
+#### Running TPC-H Tests
+
+```bash
+# Start PostgreSQL container
+docker-compose up -d
+
+# Run TPC-H integration tests
+pytest tests/integration/test_tpch.py -v
+
+# Initialize TPC-H data manually (for ad-hoc testing)
+python scripts/init_tpch_data.py \
+    --host localhost --port 5432 \
+    --username test_user --password test_password \
+    --database test --schema public
+
+# Drop and recreate TPC-H tables
+python scripts/init_tpch_data.py --drop
+```
+
+#### Environment Variables
+
+Tests use these default values (matching docker-compose.yml):
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `POSTGRESQL_HOST` | `localhost` | PostgreSQL server host |
+| `POSTGRESQL_PORT` | `5432` | PostgreSQL server port |
+| `POSTGRESQL_USER` | `test_user` | PostgreSQL username |
+| `POSTGRESQL_PASSWORD` | `test_password` | PostgreSQL password |
+| `POSTGRESQL_DATABASE` | `test` | Database name |
+| `POSTGRESQL_SCHEMA` | `public` | Schema name |
+
+## Requirements
+
+- Python >= 3.12
+- PostgreSQL >= 12
+- datus-agent >= 0.2.1
+- datus-sqlalchemy >= 0.1.0
+
+## License
+
+Apache License 2.0
+
+## Related Packages
+
+- `datus-sqlalchemy` - SQLAlchemy base connector
+- `datus-mysql` - MySQL adapter
+- `datus-redshift` - Amazon Redshift adapter
+- `datus-snowflake` - Snowflake adapter

{datus_postgresql-0.1.0 → datus_postgresql-0.1.2}/datus_postgresql/__init__.py

@@ -11,6 +11,15 @@ __all__ = ["PostgreSQLConnector", "PostgreSQLConfig", "register"]
 
 def register():
     """Register PostgreSQL connector with Datus registry."""
-    from datus.tools.db_tools import connector_registry
+    from datus_db_core import connector_registry
 
-    connector_registry.register("postgresql", PostgreSQLConnector, config_class=PostgreSQLConfig)
+    from .handlers import build_postgresql_uri, resolve_postgresql_context
+
+    connector_registry.register(
+        "postgresql",
+        PostgreSQLConnector,
+        config_class=PostgreSQLConfig,
+        capabilities={"database", "schema"},
+        uri_builder=build_postgresql_uri,
+        context_resolver=resolve_postgresql_context,
+    )

{datus_postgresql-0.1.0 → datus_postgresql-0.1.2}/datus_postgresql/connector.py

@@ -5,11 +5,7 @@
 from typing import Any, Dict, List, Optional, Set, Union, override
 from urllib.parse import quote_plus
 
-from datus.schemas.base import TABLE_TYPE
-from datus.tools.db_tools.base import list_to_in_str
-from datus.utils.constants import DBType
-from datus.utils.exceptions import DatusException, ErrorCode
-from datus.utils.loggings import get_logger
+from datus_db_core import TABLE_TYPE, DatusDbException, ErrorCode, get_logger, list_to_in_str
 from datus_sqlalchemy import SQLAlchemyConnector
 from pydantic import BaseModel, Field
 
@@ -43,7 +39,7 @@ METADATA_DICT: Dict[TABLE_TYPE, TableMetadataNames] = {
 def _get_metadata_config(table_type: TABLE_TYPE) -> TableMetadataNames:
     """Get metadata configuration for given table type."""
     if table_type not in METADATA_DICT:
-        raise DatusException(ErrorCode.COMMON_FIELD_INVALID, f"Invalid table type '{table_type}'")
+        raise DatusDbException(ErrorCode.COMMON_FIELD_INVALID, f"Invalid table type '{table_type}'")
     return METADATA_DICT[table_type]
 
 
@@ -80,7 +76,7 @@ class PostgreSQLConnector(SQLAlchemyConnector):
             f"{database}?sslmode={config.sslmode}"
         )
 
-        super().__init__(connection_string, dialect=DBType.POSTGRESQL, timeout_seconds=config.timeout_seconds)
+        super().__init__(connection_string, dialect="postgresql", timeout_seconds=config.timeout_seconds)
         self.database_name = database
         self.schema_name = config.schema_name or "public"
 

datus_postgresql-0.1.2/datus_postgresql/handlers.py

@@ -0,0 +1,80 @@
+# Copyright 2025-present DatusAI, Inc.
+# Licensed under the Apache License, Version 2.0.
+# See http://www.apache.org/licenses/LICENSE-2.0 for details.
+
+"""URI builder and context resolver for PostgreSQL."""
+
+import re
+from typing import Dict, Optional, Tuple, Union
+from urllib.parse import unquote
+
+from sqlalchemy.engine.url import URL, make_url
+
+
+def _clean_str(value: Optional[Union[str, int]]) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, (list, tuple, set)):
+        for item in value:
+            if item:
+                return str(item).strip()
+        return ""
+    return str(value).strip()
+
+
+def _value_or_none(value: Optional[Union[str, int]]) -> Optional[str]:
+    cleaned = _clean_str(value)
+    return cleaned or None
+
+
+def _port_or_none(port_value: Optional[Union[str, int]]) -> Optional[int]:
+    cleaned = _clean_str(port_value)
+    if not cleaned:
+        return None
+    try:
+        return int(cleaned)
+    except ValueError:
+        return None
+
+
+def _extract_schema_from_pg_options(options: str) -> str:
+    if not options:
+        return ""
+    decoded = unquote(options)
+    match = re.search(r"search_path\s*=\s*([^ ,]+)", decoded, flags=re.IGNORECASE)
+    if not match:
+        return ""
+    value = match.group(1)
+    if "," in value:
+        value = value.split(",", 1)[0]
+    return value.strip()
+
+
+def build_postgresql_uri(db_config) -> str:
+    sslmode = _value_or_none(getattr(db_config, "sslmode", None))
+    return str(
+        URL.create(
+            drivername="postgresql+psycopg2",
+            username=_value_or_none(db_config.username),
+            password=_value_or_none(db_config.password),
+            host=_value_or_none(db_config.host),
+            port=_port_or_none(db_config.port),
+            database=_value_or_none(db_config.database),
+            query={"sslmode": sslmode} if sslmode else {},
+        )
+    )
+
+
+def resolve_postgresql_context(db_config, uri: str) -> Tuple[str, str, str, str]:
+    url = make_url(uri)
+    query_params: Dict[str, str] = {k: _clean_str(v) for k, v in url.query.items()}
+    database = _clean_str(url.database) or _clean_str(db_config.database) or "postgres"
+    config_schema = _clean_str(getattr(db_config, "schema_name", None) or getattr(db_config, "schema", None))
+    schema = (
+        query_params.get("currentSchema")
+        or query_params.get("schema")
+        or _extract_schema_from_pg_options(query_params.get("options", ""))
+        or config_schema
+        or "public"
+    )
+    return "postgresql", "", database, schema

{datus_postgresql-0.1.0 → datus_postgresql-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datus-postgresql"
-version = "0.1.0"
+version = "0.1.2"
 description = "PostgreSQL database adapter for Datus"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -18,7 +18,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "datus-agent>0.2.1",
+    "datus-db-core>=0.1.0",
     "datus-sqlalchemy>=0.1.2",
     "psycopg2-binary>=2.9.11",
     "pydantic>=2.0.0",
@@ -33,6 +33,7 @@ Issues = "https://github.com/Datus-ai/datus-db-adapters/issues"
 postgresql = "datus_postgresql:register"
 
 [tool.uv.sources]
+datus-db-core = { workspace = true }
 datus-sqlalchemy = { workspace = true }
 
 [build-system]