hexdag 0.5.0.dev1__py3-none-any.whl

hexdag_plugins/hexdag_etl/pyproject.toml

@@ -0,0 +1,82 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hexdag-etl"
+version = "0.1.0"
+description = "ETL infrastructure for hexDAG pipelines"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "hexDAG Team" }]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.12",
+]
+keywords = ["etl", "data-pipeline", "hexdag", "plugin"]
+requires-python = ">=3.12"
+dependencies = [
+    "pandas>=2.0.0",
+    "aiohttp>=3.8.0",
+    "hexdag",         # <-- plain dependency name
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
+    "pytest-cov>=4.0.0",
+    "ruff>=0.1.0",
+    "mypy>=1.0.0",
+]
+
+[tool.hatch.metadata]
+allow-direct-references = true # now harmless but not strictly needed
+
+[tool.hatch.build.targets.wheel]
+packages = ["hexdag_etl"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/hexdag_etl", "/tests", "/examples", "/README.md"]
+
+[tool.uv.sources]
+hexdag = { path = "../.." } # <-- tell uv to take hexdag from repo root
+
+[tool.ruff]
+target-version = "py312"
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "E",  # pycodestyle errors
+    "W",  # pycodestyle warnings
+    "F",  # pyflakes
+    "I",  # isort
+    "UP", # pyupgrade
+]
+ignore = [
+    "E501",   # line too long
+    "SIM105", # contextlib.suppress doesn't work with async
+    "E402",   # module import not at top of file (test files)
+]
+
+[tool.mypy]
+python_version = "3.12"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+disallow_untyped_defs = false

hexdag_plugins/hexdag_etl/test_transform.py

@@ -0,0 +1,54 @@
+"""Test the pandas transform step by step."""
+
+import sys
+
+sys.path.insert(0, "/Users/jankwapisz/Documents/Praca/Omniviser/hexdag")
+sys.path.insert(0, "/Users/jankwapisz/Documents/Praca/Omniviser/hexdag/hexdag_plugins")
+
+import pandas as pd
+
+from hexdag_etl.nodes.pandas_transform import PandasTransformNode
+
+# Create the node
+node_factory = PandasTransformNode()
+
+# Build operations
+operations = [
+    {"type": "transform", "method": "pandas.DataFrame.sort_values", "kwargs": {"by": "amount", "ascending": False}},
+    {
+        "type": "transform",
+        "method": "pandas.DataFrame.assign",
+        "kwargs": {"amount_doubled": "{{ lambda df: df['amount'] * 2 }}"},
+    },
+    {"type": "transform", "method": "pandas.DataFrame.head", "args": [3]},
+]
+
+# Create node spec
+node_spec = node_factory(name="test_transform", operations=operations)
+
+# Test input
+df = pd.DataFrame(
+    {
+        "customer_id": ["C001", "C002", "C003", "C004", "C005"],
+        "name": ["Alice", "Bob", "Carol", "David", "Emma"],
+        "amount": [150.0, 299.99, 150.0, 29.99, 299.99],
+        "category": ["A", "B", "A", "C", "B"],
+    }
+)
+
+print("Input DataFrame:")
+print(df)
+print()
+
+# Try to run the function
+import asyncio
+
+
+async def test():
+    result = await node_spec.fn({"data": df})
+    print("Output:")
+    print(result)
+    return result
+
+
+asyncio.run(test())

hexdag_plugins/hexdag_etl/tests/test_plugin_integration.py

@@ -0,0 +1,62 @@
+"""Test hexDAG ETL plugin integration."""
+
+from hexdag.core.domain.dag import NodeSpec
+from hexdag.core.registry import registry
+
+
+def test_plugin_components_registered():
+    """Test that all ETL plugin components are properly registered."""
+
+    # Test artifact storage adapter
+    artifact_adapter = registry.get("local", namespace="etl")
+    assert artifact_adapter is not None, "Local artifact adapter should be registered"
+
+    # Test nodes
+
+    # Note: Nodes are factory classes, not directly in registry like adapters
+    # They're registered as node factories that create NodeSpecs
+    pandas_transform_factory = registry.get("pandas_transform", namespace="etl")
+    assert pandas_transform_factory is not None, "Pandas transform node should be registered"
+
+    api_extract_factory = registry.get("api_extract", namespace="etl")
+    assert api_extract_factory is not None, "API extract node should be registered"
+
+    sql_extract_factory = registry.get("sql_extract", namespace="etl")
+    assert sql_extract_factory is not None, "SQL extract node should be registered"
+
+    sql_load_factory = registry.get("sql_load", namespace="etl")
+    assert sql_load_factory is not None, "SQL load node should be registered"
+
+
+def test_artifact_adapter_instantiation():
+    """Test that artifact adapter can be instantiated."""
+    from hexdag.core.registry import registry
+
+    adapter_cls = registry.get("local", namespace="etl")
+    assert adapter_cls is not None
+
+    adapter = adapter_cls(base_path="/tmp/test_artifacts")
+    assert adapter is not None
+    assert hasattr(adapter, "write")
+    assert hasattr(adapter, "read")
+    assert hasattr(adapter, "list")
+
+
+def test_pandas_transform_node_creation():
+    """Test that pandas transform node can be created."""
+    from hexdag.core.registry import registry
+
+    factory_cls = registry.get("pandas_transform", namespace="etl")
+    assert factory_cls is not None
+
+    factory = factory_cls()
+
+    # Create a simple node spec
+    node_spec = factory(
+        name="test_transform",
+        operations=[{"type": "transform", "method": "pandas.DataFrame.head", "kwargs": {"n": 5}}],
+    )
+
+    assert isinstance(node_spec, NodeSpec)
+    assert node_spec.name == "test_transform"
+    assert "operations" in node_spec.params

hexdag_plugins/mysql_adapter/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 HexDAG Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

hexdag_plugins/mysql_adapter/README.md

@@ -0,0 +1,224 @@
+# HexDAG MySQL Adapter
+
+A production-ready MySQL database adapter plugin for the hexDAG framework.
+
+## Features
+
+- 🚀 **Production-Ready**: Built for high-performance production deployments
+- 📦 **JSON Document Storage**: Leverage MySQL's native JSON support
+- 🔄 **Async Operations**: Full async/await support for all operations
+- 🔒 **Transaction Support**: ACID compliance with proper transaction handling
+- 🎯 **Type-Safe**: Full type hints and Pydantic validation
+- 🔌 **Plugin Architecture**: Seamlessly integrates with hexDAG's plugin system
+
+## Installation
+
+### Local Development
+
+For local development within the hexDAG project:
+
+```bash
+# Using uv (recommended for hexDAG development)
+uv pip install -e hexdag_plugins/mysql_adapter/
+
+# Or using pip
+uv pip install -e hexdag_plugins/mysql_adapter/
+```
+
+### With development dependencies
+
+```bash
+pip install -e "hexdag_plugins/mysql_adapter/[dev]"
+```
+
+## Configuration
+
+### Environment Variables
+
+Set these environment variables for default connection:
+
+```bash
+export MYSQL_HOST=localhost
+export MYSQL_PORT=3306
+export MYSQL_USER=your_user
+export MYSQL_PASSWORD=your_password
+export MYSQL_DATABASE=hexdag
+```
+
+### Programmatic Configuration
+
+```python
+from hexdag_plugins.mysql_adapter import MySQLAdapter
+
+adapter = MySQLAdapter(
+    host="your-mysql-host",
+    port=3306,
+    user="your-user",
+    password="your-password",
+    database="hexdag",
+    charset="utf8mb4"
+)
+```
+
+## Usage
+
+### Basic Operations
+
+```python
+import asyncio
+from hexdag_plugins.mysql_adapter import MySQLAdapter
+
+async def main():
+    # Initialize adapter
+    adapter = MySQLAdapter(
+        host="localhost",
+        user="root",
+        password="password",
+        database="hexdag"
+    )
+
+    # Insert document
+    doc_id = await adapter.ainsert("users", {
+        "name": "John Doe",
+        "email": "john@example.com",
+        "metadata": {
+            "role": "admin",
+            "permissions": ["read", "write", "delete"]
+        }
+    })
+
+    # Query documents
+    users = await adapter.aquery(
+        "users",
+        filter={"metadata.role": "admin"}
+    )
+
+    # Update document
+    await adapter.aupdate("users", doc_id, {
+        "last_login": "2024-01-01T12:00:00Z"
+    })
+
+    # Delete document
+    await adapter.adelete("users", doc_id)
+
+asyncio.run(main())
+```
+
+### Integration with hexDAG
+
+The adapter automatically registers with hexDAG's registry system:
+
+```python
+from hexdag.core.bootstrap import bootstrap_registry
+from hexdag.core.registry import registry
+
+# Bootstrap with MySQL plugin
+bootstrap_registry()
+
+# Get MySQL adapter from registry
+mysql_info = registry.get_info("mysql", namespace="plugin")
+adapter = mysql_info.get_instance(
+    host="localhost",
+    user="root",
+    password="password"
+)
+```
+
+## Database Schema
+
+The adapter creates a flexible document store schema:
+
+```sql
+CREATE TABLE hexdag_documents (
+    id VARCHAR(255) PRIMARY KEY,
+    collection VARCHAR(255) NOT NULL,
+    document JSON NOT NULL,
+    metadata JSON,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    INDEX idx_collection (collection),
+    INDEX idx_created (created_at),
+    INDEX idx_updated (updated_at)
+);
+```
+
+## Advanced Features
+
+### JSON Path Queries
+
+Leverage MySQL's JSON functions for complex queries:
+
+```python
+# Query nested JSON fields
+results = await adapter.aquery(
+    "products",
+    filter={
+        "specs.memory": "16GB",
+        "price": {"$lt": 1000}
+    }
+)
+```
+
+### Concurrent Operations
+
+The adapter supports concurrent operations safely:
+
+```python
+import asyncio
+
+async def bulk_insert(adapter, collection, documents):
+    tasks = [
+        adapter.ainsert(collection, doc)
+        for doc in documents
+    ]
+    return await asyncio.gather(*tasks)
+```
+
+### Connection Pooling
+
+The adapter uses PyMySQL with connection management for optimal performance.
+
+## Testing
+
+### Run Tests
+
+```bash
+pytest tests/
+```
+
+### Test with Docker MySQL
+
+```bash
+# Start MySQL container
+docker run -d \
+    --name mysql-test \
+    -e MYSQL_ROOT_PASSWORD=test \
+    -e MYSQL_DATABASE=hexdag_test \
+    -p 3306:3306 \
+    mysql:8.0
+
+# Run tests
+MYSQL_TEST_HOST=localhost \
+MYSQL_TEST_USER=root \
+MYSQL_TEST_PASSWORD=test \
+MYSQL_TEST_DATABASE=hexdag_test \
+pytest tests/
+```
+
+## Requirements
+
+- Python 3.12+
+- MySQL 5.7+ (8.0+ recommended for full JSON support)
+- PyMySQL 1.1.0+
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## Support
+
+For issues and questions, please use the GitHub issue tracker.

hexdag_plugins/mysql_adapter/__init__.py

@@ -0,0 +1,6 @@
+"""MySQL adapter plugin for hexDAG framework."""
+
+from hexdag_plugins.mysql_adapter.mysql_adapter import MySQLAdapter
+
+__version__ = "1.0.0"
+__all__ = ["MySQLAdapter"]