hexdag 0.5.0.dev1__py3-none-any.whl

hexdag_plugins/storage/sql/base.py

@@ -0,0 +1,187 @@
+"""Base SQL adapter using SQLAlchemy for connection pooling and async operations."""
+
+from typing import Any
+
+from pydantic import SecretStr
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, create_async_engine
+
+from hexdag.core import AdapterConfig, ConfigurableAdapter, SecretField
+from hexdag.core.ports.healthcheck import HealthStatus
+
+
+class DatabaseConfig(AdapterConfig):
+    """Base configuration for SQL database adapters.
+
+    Attributes
+    ----------
+    connection_string : SecretStr
+        Database connection string (e.g., "postgresql+asyncpg://user:pass@host/db")
+    pool_size : int
+        Number of permanent connections in the pool (default: 5)
+    max_overflow : int
+        Maximum number of connections beyond pool_size (default: 10)
+    pool_timeout : float
+        Timeout in seconds when getting connection from pool (default: 30.0)
+    pool_recycle : int
+        Recycle connections after this many seconds (default: 3600)
+    pool_pre_ping : bool
+        Test connections before using them (default: True)
+    """
+
+    connection_string: SecretStr = SecretField(
+        env_var="DATABASE_URL",
+        description="Database connection string",
+    )
+    pool_size: int = 5
+    max_overflow: int = 10
+    pool_timeout: float = 30.0
+    pool_recycle: int = 3600
+    pool_pre_ping: bool = True
+
+
+class SQLAdapter(ConfigurableAdapter):
+    """Base SQL adapter with SQLAlchemy connection pooling.
+
+    This base class provides common functionality for all SQL databases:
+    - Async connection pooling
+    - Health checks
+    - Basic query execution
+    - Proper resource cleanup
+    """
+
+    Config = DatabaseConfig
+
+    def __init__(self, **kwargs):
+        """Initialize SQL adapter with configuration."""
+        super().__init__(**kwargs)
+        self._engine: AsyncEngine | None = None
+
+    async def asetup(self):
+        """Initialize SQLAlchemy async engine with connection pool."""
+        self._engine = create_async_engine(
+            self.config.connection_string.get_secret_value(),
+            pool_size=self.config.pool_size,
+            max_overflow=self.config.max_overflow,
+            pool_timeout=self.config.pool_timeout,
+            pool_recycle=self.config.pool_recycle,
+            pool_pre_ping=self.config.pool_pre_ping,
+            echo=False,
+        )
+
+    async def aclose(self):
+        """Close database connection pool."""
+        if self._engine:
+            await self._engine.dispose()
+            self._engine = None
+
+    async def aexecute(self, query: str, params: dict[str, Any] | None = None) -> Any:
+        """Execute a SQL query (INSERT, UPDATE, DELETE, DDL).
+
+        Args:
+            query: SQL query string
+            params: Query parameters for parameterized queries
+
+        Returns:
+            Result of the execution
+
+        Raises:
+            RuntimeError: If adapter not set up
+        """
+        if not self._engine:
+            msg = "Adapter not set up. Call asetup() first."
+            raise RuntimeError(msg)
+
+        async with AsyncSession(self._engine) as session:
+            result = await session.execute(text(query), params or {})
+            await session.commit()
+            return result
+
+    async def afetch_one(
+        self, query: str, params: dict[str, Any] | None = None
+    ) -> dict[str, Any] | None:
+        """Fetch a single row from the database.
+
+        Args:
+            query: SQL query string
+            params: Query parameters for parameterized queries
+
+        Returns:
+            Single row as dictionary or None if no results
+
+        Raises:
+            RuntimeError: If adapter not set up
+        """
+        if not self._engine:
+            msg = "Adapter not set up. Call asetup() first."
+            raise RuntimeError(msg)
+
+        async with AsyncSession(self._engine) as session:
+            result = await session.execute(text(query), params or {})
+            row = result.fetchone()
+            return dict(row._mapping) if row else None
+
+    async def afetch_all(
+        self, query: str, params: dict[str, Any] | None = None
+    ) -> list[dict[str, Any]]:
+        """Fetch all rows from the database.
+
+        Args:
+            query: SQL query string
+            params: Query parameters for parameterized queries
+
+        Returns:
+            List of rows as dictionaries
+
+        Raises:
+            RuntimeError: If adapter not set up
+        """
+        if not self._engine:
+            msg = "Adapter not set up. Call asetup() first."
+            raise RuntimeError(msg)
+
+        async with AsyncSession(self._engine) as session:
+            result = await session.execute(text(query), params or {})
+            rows = result.fetchall()
+            return [dict(row._mapping) for row in rows]
+
+    async def ahealth_check(self) -> HealthStatus:
+        """Check database connection health.
+
+        Returns:
+            HealthStatus with database availability and connection pool info
+        """
+        if not self._engine:
+            return HealthStatus(
+                status="unhealthy",
+                adapter_name=self.__class__.__name__,
+                port_name="database",
+                details={"message": "Database not initialized"},
+            )
+
+        try:
+            # Test query
+            async with AsyncSession(self._engine) as session:
+                await session.execute(text("SELECT 1"))
+
+            # Get pool statistics
+            pool = self._engine.pool
+            return HealthStatus(
+                status="healthy",
+                adapter_name=self.__class__.__name__,
+                port_name="database",
+                details={
+                    "pool_size": pool.size(),
+                    "checked_in": pool.checkedin(),
+                    "checked_out": pool.checkedout(),
+                    "overflow": pool.overflow(),
+                },
+            )
+        except Exception as e:
+            return HealthStatus(
+                status="unhealthy",
+                adapter_name=self.__class__.__name__,
+                port_name="database",
+                error=e,
+                details={"error": str(e)},
+            )

hexdag_plugins/storage/sql/mysql.py

@@ -0,0 +1,27 @@
+"""MySQL adapter using SQLAlchemy with aiomysql driver."""
+
+from hexdag.core.registry.decorators import adapter
+
+from .base import DatabaseConfig, SQLAdapter
+
+
+@adapter("database", name="mysql", namespace="storage")
+class MySQLAdapter(SQLAdapter):
+    """MySQL database adapter with async connection pooling.
+
+    Uses SQLAlchemy with aiomysql driver for async MySQL operations.
+    Provides connection pooling, health checks, and standard SQL operations.
+
+    Example connection string:
+        mysql+aiomysql://user:password@host:port/database
+
+    Configuration:
+        - connection_string: MySQL connection URL (env: DATABASE_URL)
+        - pool_size: Number of permanent connections (default: 5)
+        - max_overflow: Extra connections beyond pool_size (default: 10)
+        - pool_timeout: Timeout for getting connection (default: 30.0)
+        - pool_recycle: Recycle connections after seconds (default: 3600)
+        - pool_pre_ping: Test connections before use (default: True)
+    """
+
+    Config = DatabaseConfig

hexdag_plugins/storage/sql/postgresql.py

@@ -0,0 +1,27 @@
+"""PostgreSQL adapter using SQLAlchemy with asyncpg driver."""
+
+from hexdag.core.registry.decorators import adapter
+
+from .base import DatabaseConfig, SQLAdapter
+
+
+@adapter("database", name="postgresql", namespace="storage")
+class PostgreSQLAdapter(SQLAdapter):
+    """PostgreSQL database adapter with async connection pooling.
+
+    Uses SQLAlchemy with asyncpg driver for async PostgreSQL operations.
+    Provides connection pooling, health checks, and standard SQL operations.
+
+    Example connection string:
+        postgresql+asyncpg://user:password@host:port/database
+
+    Configuration:
+        - connection_string: PostgreSQL connection URL (env: DATABASE_URL)
+        - pool_size: Number of permanent connections (default: 5)
+        - max_overflow: Extra connections beyond pool_size (default: 10)
+        - pool_timeout: Timeout for getting connection (default: 30.0)
+        - pool_recycle: Recycle connections after seconds (default: 3600)
+        - pool_pre_ping: Test connections before use (default: True)
+    """
+
+    Config = DatabaseConfig

hexdag_plugins/storage/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for hexdag-storage."""

hexdag_plugins/storage/tests/test_local_file_storage.py

@@ -0,0 +1,161 @@
+"""Tests for local file storage adapter."""
+
+import pytest
+
+from hexdag_plugins.storage.file import LocalFileStorage
+
+
+class TestLocalFileStorage:
+    """Test suite for LocalFileStorage adapter."""
+
+    @pytest.fixture
+    async def storage(self, tmp_path):
+        """Create a test storage instance."""
+        storage = LocalFileStorage(base_path=str(tmp_path))
+        yield storage
+
+    @pytest.fixture
+    def test_file(self, tmp_path):
+        """Create a test file."""
+        test_file = tmp_path / "test.txt"
+        test_file.write_text("Hello, World!")
+        return test_file
+
+    @pytest.mark.asyncio
+    async def test_initialization(self, tmp_path):
+        """Test storage initialization."""
+        storage = LocalFileStorage(base_path=str(tmp_path))
+        assert storage._base_path == tmp_path
+        assert tmp_path.exists()
+
+    @pytest.mark.asyncio
+    async def test_upload(self, storage, test_file, tmp_path):
+        """Test file upload."""
+        result = await storage.aupload(str(test_file), "docs/test.txt")
+
+        assert result["uploaded"] is True
+        assert result["remote_path"] == "docs/test.txt"
+        assert (tmp_path / "docs/test.txt").exists()
+
+    @pytest.mark.asyncio
+    async def test_download(self, storage, test_file, tmp_path):
+        """Test file download."""
+        # First upload a file
+        await storage.aupload(str(test_file), "docs/test.txt")
+
+        # Download to a different location
+        download_path = tmp_path / "downloaded.txt"
+        result = await storage.adownload("docs/test.txt", str(download_path))
+
+        assert result["downloaded"] is True
+        assert download_path.exists()
+        assert download_path.read_text() == "Hello, World!"
+
+    @pytest.mark.asyncio
+    async def test_exists(self, storage, test_file):
+        """Test file existence check."""
+        # File doesn't exist yet
+        assert await storage.aexists("docs/test.txt") is False
+
+        # Upload file
+        await storage.aupload(str(test_file), "docs/test.txt")
+
+        # Now it exists
+        assert await storage.aexists("docs/test.txt") is True
+
+    @pytest.mark.asyncio
+    async def test_delete(self, storage, test_file):
+        """Test file deletion."""
+        # Upload file
+        await storage.aupload(str(test_file), "docs/test.txt")
+        assert await storage.aexists("docs/test.txt") is True
+
+        # Delete file
+        result = await storage.adelete("docs/test.txt")
+
+        assert result["deleted"] is True
+        assert await storage.aexists("docs/test.txt") is False
+
+    @pytest.mark.asyncio
+    async def test_list(self, storage, test_file):
+        """Test file listing."""
+        # Upload multiple files
+        await storage.aupload(str(test_file), "docs/file1.txt")
+        await storage.aupload(str(test_file), "docs/file2.txt")
+        await storage.aupload(str(test_file), "images/pic.jpg")
+
+        # List all files
+        all_files = await storage.alist()
+        assert len(all_files) == 3
+        assert "docs/file1.txt" in all_files
+        assert "docs/file2.txt" in all_files
+        assert "images/pic.jpg" in all_files
+
+        # List with prefix
+        docs_files = await storage.alist(prefix="docs/")
+        assert len(docs_files) == 2
+        assert all(f.startswith("docs/") for f in docs_files)
+
+    @pytest.mark.asyncio
+    async def test_get_metadata(self, storage, test_file):
+        """Test getting file metadata."""
+        await storage.aupload(str(test_file), "test.txt")
+
+        metadata = await storage.aget_metadata("test.txt")
+
+        assert metadata["path"] == "test.txt"
+        assert metadata["size_bytes"] > 0
+        assert metadata["is_file"] is True
+        assert "modified_time" in metadata
+        assert "created_time" in metadata
+
+    @pytest.mark.asyncio
+    async def test_health_check(self, storage):
+        """Test health check."""
+        health = await storage.ahealth_check()
+
+        assert health.status == "healthy"
+        assert health.adapter_name == "local_file_storage"
+        assert health.port_name == "file_storage"
+        assert health.details["writable"] is True
+
+    @pytest.mark.asyncio
+    async def test_upload_creates_directories(self, storage, test_file):
+        """Test that upload creates nested directories."""
+        result = await storage.aupload(str(test_file), "a/b/c/test.txt")
+
+        assert result["uploaded"] is True
+        assert await storage.aexists("a/b/c/test.txt") is True
+
+    @pytest.mark.asyncio
+    async def test_upload_nonexistent_file(self, storage):
+        """Test uploading a nonexistent file."""
+        with pytest.raises(FileNotFoundError):
+            await storage.aupload("/nonexistent/file.txt", "test.txt")
+
+    @pytest.mark.asyncio
+    async def test_download_nonexistent_file(self, storage, tmp_path):
+        """Test downloading a nonexistent file."""
+        with pytest.raises(FileNotFoundError):
+            await storage.adownload("nonexistent.txt", str(tmp_path / "out.txt"))
+
+    @pytest.mark.asyncio
+    async def test_delete_nonexistent_file(self, storage):
+        """Test deleting a nonexistent file."""
+        with pytest.raises(FileNotFoundError):
+            await storage.adelete("nonexistent.txt")
+
+    @pytest.mark.asyncio
+    async def test_get_metadata_nonexistent_file(self, storage):
+        """Test getting metadata for nonexistent file."""
+        with pytest.raises(FileNotFoundError):
+            await storage.aget_metadata("nonexistent.txt")
+
+    @pytest.mark.asyncio
+    async def test_repr(self, tmp_path):
+        """Test string representation."""
+        storage = LocalFileStorage(base_path=str(tmp_path))
+        repr_str = repr(storage)
+
+        assert "LocalFileStorage" in repr_str
+        assert str(tmp_path) in repr_str

hexdag_plugins/storage/tests/test_sql_adapters.py

@@ -0,0 +1,212 @@
+"""Tests for SQL adapters (MySQL, PostgreSQL)."""
+
+import pytest
+
+from hexdag_plugins.storage.sql import MySQLAdapter, PostgreSQLAdapter
+
+
+class TestSQLAdapterInterface:
+    """Test SQL adapter interface and basic functionality."""
+
+    @pytest.mark.asyncio
+    async def test_mysql_adapter_creation(self):
+        """Test MySQL adapter can be created with config."""
+        adapter = MySQLAdapter(connection_string="mysql+aiomysql://test:test@localhost/test")
+        assert adapter is not None
+        assert (
+            adapter.config.connection_string.get_secret_value()
+            == "mysql+aiomysql://test:test@localhost/test"
+        )
+        assert adapter.config.pool_size == 5
+        assert adapter.config.max_overflow == 10
+
+    @pytest.mark.asyncio
+    async def test_postgresql_adapter_creation(self):
+        """Test PostgreSQL adapter can be created with config."""
+        adapter = PostgreSQLAdapter(
+            connection_string="postgresql+asyncpg://test:test@localhost/test"
+        )
+        assert adapter is not None
+        assert (
+            adapter.config.connection_string.get_secret_value()
+            == "postgresql+asyncpg://test:test@localhost/test"
+        )
+        assert adapter.config.pool_size == 5
+        assert adapter.config.max_overflow == 10
+
+    @pytest.mark.asyncio
+    async def test_mysql_adapter_custom_pool_config(self):
+        """Test MySQL adapter with custom pool configuration."""
+        adapter = MySQLAdapter(
+            connection_string="mysql+aiomysql://test:test@localhost/test",
+            pool_size=10,
+            max_overflow=20,
+            pool_timeout=60.0,
+            pool_recycle=7200,
+            pool_pre_ping=False,
+        )
+        assert adapter.config.pool_size == 10
+        assert adapter.config.max_overflow == 20
+        assert adapter.config.pool_timeout == 60.0
+        assert adapter.config.pool_recycle == 7200
+        assert adapter.config.pool_pre_ping is False
+
+    @pytest.mark.asyncio
+    async def test_postgresql_adapter_custom_pool_config(self):
+        """Test PostgreSQL adapter with custom pool configuration."""
+        adapter = PostgreSQLAdapter(
+            connection_string="postgresql+asyncpg://test:test@localhost/test",
+            pool_size=15,
+            max_overflow=25,
+            pool_timeout=45.0,
+            pool_recycle=1800,
+            pool_pre_ping=False,
+        )
+        assert adapter.config.pool_size == 15
+        assert adapter.config.max_overflow == 25
+        assert adapter.config.pool_timeout == 45.0
+        assert adapter.config.pool_recycle == 1800
+        assert adapter.config.pool_pre_ping is False
+
+    @pytest.mark.asyncio
+    async def test_adapter_not_setup_error(self):
+        """Test that operations fail before setup."""
+        adapter = MySQLAdapter(connection_string="mysql+aiomysql://test:test@localhost/test")
+
+        with pytest.raises(RuntimeError, match="Adapter not set up"):
+            await adapter.aexecute("SELECT 1")
+
+        with pytest.raises(RuntimeError, match="Adapter not set up"):
+            await adapter.afetch_one("SELECT 1")
+
+        with pytest.raises(RuntimeError, match="Adapter not set up"):
+            await adapter.afetch_all("SELECT 1")
+
+    @pytest.mark.asyncio
+    async def test_health_check_before_setup(self):
+        """Test health check before setup returns unhealthy status."""
+        adapter = MySQLAdapter(connection_string="mysql+aiomysql://test:test@localhost/test")
+
+        health = await adapter.ahealth_check()
+        assert health.status == "unhealthy"
+        assert "not initialized" in health.details.get("message", "").lower()
+
+
+# Integration tests require real database connections
+# These are marked with @pytest.mark.integration and skipped by default
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_mysql_adapter_integration():
+    """Integration test for MySQL adapter with real database.
+
+    Requires MySQL running at localhost with test database.
+    Set MYSQL_TEST_URL environment variable or skip this test.
+    """
+    import os
+
+    connection_string = os.getenv("MYSQL_TEST_URL", "mysql+aiomysql://test:test@localhost/test")
+
+    adapter = MySQLAdapter(connection_string=connection_string)
+
+    try:
+        await adapter.asetup()
+
+        # Test health check
+        health = await adapter.ahealth_check()
+        assert health.is_healthy() is True
+
+        # Test table creation
+        await adapter.aexecute(
+            """
+            CREATE TABLE IF NOT EXISTS test_table (
+                id INT PRIMARY KEY AUTO_INCREMENT,
+                name VARCHAR(255),
+                value INT
+            )
+        """
+        )
+
+        # Test insert
+        await adapter.aexecute(
+            "INSERT INTO test_table (name, value) VALUES (:name, :value)",
+            {"name": "test", "value": 42},
+        )
+
+        # Test fetch_one
+        result = await adapter.afetch_one(
+            "SELECT * FROM test_table WHERE name = :name", {"name": "test"}
+        )
+        assert result is not None
+        assert result["name"] == "test"
+        assert result["value"] == 42
+
+        # Test fetch_all
+        results = await adapter.afetch_all("SELECT * FROM test_table")
+        assert len(results) >= 1
+
+        # Cleanup
+        await adapter.aexecute("DROP TABLE test_table")
+
+    finally:
+        await adapter.aclose()
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_postgresql_adapter_integration():
+    """Integration test for PostgreSQL adapter with real database.
+
+    Requires PostgreSQL running at localhost with test database.
+    Set POSTGRES_TEST_URL environment variable or skip this test.
+    """
+    import os
+
+    connection_string = os.getenv(
+        "POSTGRES_TEST_URL", "postgresql+asyncpg://test:test@localhost/test"
+    )
+
+    adapter = PostgreSQLAdapter(connection_string=connection_string)
+
+    try:
+        await adapter.asetup()
+
+        # Test health check
+        health = await adapter.ahealth_check()
+        assert health.is_healthy() is True
+
+        # Test table creation
+        await adapter.aexecute(
+            """
+            CREATE TABLE IF NOT EXISTS test_table (
+                id SERIAL PRIMARY KEY,
+                name VARCHAR(255),
+                value INT
+            )
+        """
+        )
+
+        # Test insert
+        await adapter.aexecute(
+            "INSERT INTO test_table (name, value) VALUES (:name, :value)",
+            {"name": "test", "value": 42},
+        )
+
+        # Test fetch_one
+        result = await adapter.afetch_one(
+            "SELECT * FROM test_table WHERE name = :name", {"name": "test"}
+        )
+        assert result is not None
+        assert result["name"] == "test"
+        assert result["value"] == 42
+
+        # Test fetch_all
+        results = await adapter.afetch_all("SELECT * FROM test_table")
+        assert len(results) >= 1
+
+        # Cleanup
+        await adapter.aexecute("DROP TABLE test_table")
+
+    finally:
+        await adapter.aclose()

hexdag_plugins/storage/vector/__init__.py

@@ -0,0 +1,7 @@
+"""Vector store adapters."""
+
+from .chromadb import ChromaDBAdapter
+from .in_memory import InMemoryVectorStore
+from .pgvector import PgVectorAdapter
+
+__all__ = ["ChromaDBAdapter", "InMemoryVectorStore", "PgVectorAdapter"]