aptdata 0.0.2__tar.gz → 0.0.3__tar.gz

{aptdata-0.0.2 → aptdata-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aptdata
-Version: 0.0.2
+Version: 0.0.3
 Summary: A declarative, extensible framework for building smart data pipelines in Python
 License: MIT
 License-File: LICENSE
@@ -17,12 +17,13 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Provides-Extra: ai
 Provides-Extra: all
 Provides-Extra: pandas
 Provides-Extra: plugins
 Provides-Extra: spark
 Requires-Dist: httpx (>=0.27,<0.28) ; extra == "plugins" or extra == "all"
-Requires-Dist: mcp (>=1.26.0,<2.0.0)
+Requires-Dist: mcp (>=1.26.0,<2.0.0) ; extra == "ai" or extra == "all"
 Requires-Dist: opentelemetry-api (>=1.40.0,<2.0.0)
 Requires-Dist: opentelemetry-sdk (>=1.40.0,<2.0.0)
 Requires-Dist: pandas (>=2.2,<3.0) ; extra == "pandas" or extra == "all"
@@ -95,6 +96,7 @@ pip install aptdata
 pip install aptdata[pandas]   # pandas support
 pip install aptdata[spark]    # PySpark support
 pip install aptdata[plugins]  # REST, PostgreSQL, Parquet I/O
+pip install aptdata[ai]       # MCP server for AI agents
 pip install aptdata[all]      # everything
 ```
 
@@ -273,6 +275,23 @@ See [Governance docs](docs/governance.md) for the full API.
 
 ---
 
+## AI Agents & MCP Server
+
+aptdata ships with a built-in [Model Context Protocol](https://modelcontextprotocol.io/) server (`mcp-start`). This transforms AI assistants (like Claude, Copilot, or Devin) into autonomous data engineers with direct access to:
+
+- **Pipeline Execution:** Trigger and monitor data flows (`run_flow`).
+- **Data Quality:** Audit the latest quality test results (`quality://reports/...`).
+- **Data Governance:** Read business rules to prevent violations (`governance://rules`).
+- **Lineage:** Trace upstream dependencies and column-level provenance (`get_pipeline_lineage`).
+
+```bash
+aptdata mcp-start --transport stdio
+```
+
+See the [MCP Documentation](docs/mcp.md) for setup instructions.
+
+---
+
 ## Release process
 
 Releases are automated via the [Release workflow](.github/workflows/release.yml).

{aptdata-0.0.2 → aptdata-0.0.3}/README.md

@@ -51,6 +51,7 @@ pip install aptdata
 pip install aptdata[pandas]   # pandas support
 pip install aptdata[spark]    # PySpark support
 pip install aptdata[plugins]  # REST, PostgreSQL, Parquet I/O
+pip install aptdata[ai]       # MCP server for AI agents
 pip install aptdata[all]      # everything
 ```
 
@@ -229,6 +230,23 @@ See [Governance docs](docs/governance.md) for the full API.
 
 ---
 
+## AI Agents & MCP Server
+
+aptdata ships with a built-in [Model Context Protocol](https://modelcontextprotocol.io/) server (`mcp-start`). This transforms AI assistants (like Claude, Copilot, or Devin) into autonomous data engineers with direct access to:
+
+- **Pipeline Execution:** Trigger and monitor data flows (`run_flow`).
+- **Data Quality:** Audit the latest quality test results (`quality://reports/...`).
+- **Data Governance:** Read business rules to prevent violations (`governance://rules`).
+- **Lineage:** Trace upstream dependencies and column-level provenance (`get_pipeline_lineage`).
+
+```bash
+aptdata mcp-start --transport stdio
+```
+
+See the [MCP Documentation](docs/mcp.md) for setup instructions.
+
+---
+
 ## Release process
 
 Releases are automated via the [Release workflow](.github/workflows/release.yml).

{aptdata-0.0.2 → aptdata-0.0.3}/aptdata/__init__.py

@@ -1,3 +1,3 @@
 """aptdata: A framework for smart data pipelines."""
 
-__version__ = "0.0.2"
+__version__ = "0.0.3"

{aptdata-0.0.2 → aptdata-0.0.3}/aptdata/cli/commands/mesh_cmd.py

@@ -47,6 +47,25 @@ def _find_mesh_yaml(directory: Path) -> Path | None:  # noqa: UP007
     return candidate if candidate.exists() else None
 
 
+def _resolve_mesh_file(root: Path, component: str) -> Path | None:  # noqa: UP007
+    """Find mesh.yaml for the given component (by name or direct path)."""
+    component_path = root / component
+    if component_path.is_dir():
+        mesh_file = _find_mesh_yaml(component_path)
+        if mesh_file:
+            return mesh_file
+
+    for candidate in root.rglob(_MESH_FILE):
+        try:
+            data = _load_mesh(candidate)
+            if data.get("component") == component:
+                return candidate
+        except Exception:  # noqa: BLE001
+            continue
+
+    return None
+
+
 @mesh_app.command("list")
 def mesh_list(
     directory: Path = typer.Option(
@@ -152,20 +171,7 @@ def mesh_run(
     console = SmartConsole(json_mode=json_mode)
     root = directory.resolve()
 
-    # Find mesh.yaml for the given component (by name or direct path)
-    mesh_file: Path | None = None  # noqa: UP007
-    component_path = root / component
-    if component_path.is_dir():
-        mesh_file = _find_mesh_yaml(component_path)
-    if mesh_file is None:
-        for candidate in root.rglob(_MESH_FILE):
-            try:
-                data = _load_mesh(candidate)
-                if data.get("component") == component:
-                    mesh_file = candidate
-                    break
-            except Exception:  # noqa: BLE001
-                continue
+    mesh_file = _resolve_mesh_file(root, component)
 
     if mesh_file is None:
         msg = f"Component '{component}' not found. No mesh.yaml located under '{root}'."
@@ -271,19 +277,7 @@ def mesh_build(
     console = SmartConsole(json_mode=json_mode)
     root = directory.resolve()
 
-    mesh_file: Path | None = None  # noqa: UP007
-    component_path = root / component
-    if component_path.is_dir():
-        mesh_file = _find_mesh_yaml(component_path)
-    if mesh_file is None:
-        for candidate in root.rglob(_MESH_FILE):
-            try:
-                data = _load_mesh(candidate)
-                if data.get("component") == component:
-                    mesh_file = candidate
-                    break
-            except Exception:  # noqa: BLE001
-                continue
+    mesh_file = _resolve_mesh_file(root, component)
 
     if mesh_file is None:
         msg = f"Component '{component}' not found. No mesh.yaml located under '{root}'."

{aptdata-0.0.2 → aptdata-0.0.3}/aptdata/core/__init__.py

@@ -1,7 +1,12 @@
 """Core interfaces and base classes for aptdata."""
 
-from aptdata.core.context import ExecutionContext
-from aptdata.core.dataset import BaseDataset, IDataset
+from aptdata.core.context import ExecutionContext, IContext
+from aptdata.core.dataset import (
+    BaseDataset,
+    DataContractError,
+    IDataset,
+    PydanticDataset,
+)
 from aptdata.core.state import StateBackend
 from aptdata.core.system import (
     BaseComponent,
@@ -26,7 +31,10 @@ from aptdata.core.workflow import (
 __all__ = [
     "IDataset",
     "BaseDataset",
+    "PydanticDataset",
+    "DataContractError",
     "ExecutionContext",
+    "IContext",
     "ComponentKind",
     "ComponentMeta",
     "IComponent",

aptdata-0.0.3/aptdata/core/context.py

@@ -0,0 +1,95 @@
+"""Injectable execution context for state sharing across runs."""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import field
+from typing import Any
+
+from pydantic.dataclasses import dataclass as pydantic_dataclass
+
+from aptdata.core.events import EventBus, IEventBus
+from aptdata.telemetry import TelemetryProvider
+
+
+class IContext(ABC):
+    """Interface for an execution context with logging and telemetry."""
+
+    @property
+    @abstractmethod
+    def logger(self) -> logging.Logger:
+        """Structured logger for this context."""
+
+    @property
+    @abstractmethod
+    def telemetry(self) -> TelemetryProvider:
+        """Telemetry provider for this context."""
+
+    @property
+    @abstractmethod
+    def event_bus(self) -> IEventBus:
+        """Event bus for decoupled communication and observability."""
+
+    @abstractmethod
+    def get(self, key: str, default: Any = None) -> Any:
+        """Return value for *key* or *default* if not present."""
+
+    @abstractmethod
+    def set(self, key: str, value: Any) -> None:
+        """Store *value* under *key*."""
+
+    @abstractmethod
+    def update(self, values: dict[str, Any]) -> None:
+        """Merge mapping into memory state."""
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all state."""
+
+
+from pydantic import ConfigDict  # noqa: E402
+
+
+@pydantic_dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+class ExecutionContext(IContext):
+    """Simple in-memory key/value state container."""
+
+    memory: dict[str, Any] = field(default_factory=dict)
+    _logger: logging.Logger | None = field(default=None, init=False, repr=False)
+    _telemetry: TelemetryProvider | None = field(default=None, init=False, repr=False)
+    _event_bus: IEventBus | None = field(default=None, init=False, repr=False)
+
+    @property
+    def logger(self) -> logging.Logger:
+        if self._logger is None:
+            self._logger = logging.getLogger("aptdata.context")
+        return self._logger
+
+    @property
+    def telemetry(self) -> TelemetryProvider:
+        if self._telemetry is None:
+            self._telemetry = TelemetryProvider.get_instance()
+        return self._telemetry
+
+    @property
+    def event_bus(self) -> IEventBus:
+        if self._event_bus is None:
+            self._event_bus = EventBus()
+        return self._event_bus
+
+    @event_bus.setter
+    def event_bus(self, bus: IEventBus) -> None:
+        self._event_bus = bus
+
+    def get(self, key: str, default: Any = None) -> Any:
+        return self.memory.get(key, default)
+
+    def set(self, key: str, value: Any) -> None:
+        self.memory[key] = value
+
+    def update(self, values: dict[str, Any]) -> None:
+        self.memory.update(values)
+
+    def clear(self) -> None:
+        self.memory.clear()

aptdata-0.0.3/aptdata/core/dataset.py

@@ -0,0 +1,121 @@
+"""Dataset interface and base class."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Generic, TypeVar
+
+from pydantic import BaseModel
+from pydantic.dataclasses import dataclass as pydantic_dataclass
+
+
+class DataContractError(Exception):
+    """Exception raised when dataset data does not conform to the expected Pydantic
+    contract."""
+
+    pass
+
+
+@dataclass
+class IDataset(ABC):
+    """Dataclass interface for dataset types.
+
+    All dataset contracts must implement :meth:`read` and :meth:`write`.
+    No concrete fields are defined here – field declarations live in
+    :class:`BaseDataset` and its subclasses.
+    """
+
+    @abstractmethod
+    def read(self) -> Any:
+        """Read and return data from the dataset."""
+
+    @abstractmethod
+    def write(self, data: Any) -> None:
+        """Write data to the dataset."""
+
+
+@pydantic_dataclass
+class BaseDataset(IDataset):
+    """Base dataset with Pydantic-validated fields.
+
+    Provides the canonical ``uri`` and ``schema_metadata`` fields.
+    Concrete dataset implementations must inherit from this class and
+    implement the :meth:`read` and :meth:`write` abstract methods
+    inherited from :class:`IDataset`.
+    """
+
+    uri: str
+    schema_metadata: dict[str, Any] = field(default_factory=dict)
+
+
+T = TypeVar("T", bound=BaseModel)
+
+
+@pydantic_dataclass
+class PydanticDataset(BaseDataset, Generic[T]):
+    """A dataset implementation that enforces a Pydantic model contract.
+
+    Data validation is performed when data is written to or read from the dataset.
+    This implementation optimized for Pandas dataframes by converting the Pydantic
+    schema into pandas dtypes, ensuring fail-fast execution without row-by-row
+    iteration.
+    """
+
+    contract: type[T] | None = field(default=None)
+    _data: Any = field(default=None, init=False, repr=False)
+
+    def read(self) -> Any:
+        return self._data
+
+    def write(self, data: Any) -> None:
+        if self.contract is not None:
+            self._validate(data)
+        self._data = data
+
+    def _validate(self, data: Any) -> None:
+        """Validates the input data against the configured Pydantic contract."""
+        if self.contract is None:
+            return
+
+        try:
+            import pandas as pd
+        except ImportError:
+            # Fallback to pure pydantic if pandas isn't installed.
+            # We assume data is a list of dicts.
+            if isinstance(data, list):
+                for row in data:
+                    try:
+                        self.contract.model_validate(row)
+                    except Exception as e:
+                        raise DataContractError(f"Validation failed for row {row}: {e}")
+            return
+
+        if isinstance(data, pd.DataFrame):
+            # Optimised pandas validation
+            schema_fields = self.contract.model_fields
+            actual_columns = set(data.columns)
+
+            # Check for missing columns
+            required_columns = {k for k, v in schema_fields.items() if v.is_required()}
+            missing_required = required_columns - actual_columns
+            if missing_required:
+                raise DataContractError(
+                    f"DataFrame is missing required columns: {missing_required}"
+                )
+
+            # Optionally check types (fail-fast type checking without row-by-row)
+            # This is a basic conversion check
+            for col, field_info in schema_fields.items():
+                if col in data.columns:
+                    # In a real-world scenario, we would map pydantic types to
+                    # numpy/pandas dtypes and ensure the types match perfectly.
+                    # For now we rely on missing columns and basic null checks.
+                    if field_info.is_required() and data[col].isnull().any():
+                        raise DataContractError(
+                            f"Column '{col}' contains null values but is required."
+                        )
+        elif isinstance(data, list):
+            for row in data:
+                try:
+                    self.contract.model_validate(row)
+                except Exception as e:
+                    raise DataContractError(f"Validation failed for row {row}: {e}")

aptdata-0.0.3/aptdata/core/decorators.py

@@ -0,0 +1,140 @@
+import inspect
+from collections.abc import Callable
+from typing import Any
+
+from aptdata.core.context import IContext
+from aptdata.core.dataset import IDataset
+from aptdata.core.registry import ComponentRegistry
+from aptdata.core.system import BaseComponent
+
+
+class FunctionComponentAdapter(BaseComponent):
+    """Adapter to make a simple python function behave like a BaseComponent."""
+
+    def __init__(self, func: Callable, name: str, **kwargs: Any):
+        # Determine component_id. User might have passed it in kwargs via yaml builder.
+        # Otherwise fallback to the decorator's name
+        comp_id = kwargs.pop("component_id", name)
+        super().__init__(component_id=comp_id, **kwargs)
+        self._func = func
+
+    def validate_inputs(self, inputs: list[IDataset]) -> bool:
+        """Default validation passes everything."""
+        return True
+
+    def execute(self, inputs: list[IDataset]) -> list[IDataset]:
+        return self._execute(inputs)
+
+    def _execute(self, inputs: list[IDataset]) -> list[IDataset]:
+        # For simple functional components, we assume the signature can be:
+        # func(inputs: list[IDataset], context: IContext) -> list[IDataset]
+        # or just func(inputs: list[IDataset]) -> list[IDataset]
+
+        sig = inspect.signature(self._func)
+        kwargs = {}
+
+        # Determine if the function expects context
+        has_context_param = False
+        for param_name, param in sig.parameters.items():
+            if param.annotation == IContext or param_name == "context":
+                kwargs[param_name] = self.context
+                has_context_param = True
+            elif param_name == "inputs":
+                kwargs[param_name] = inputs
+
+        # If the parameter isn't explicitly named "inputs", we'll just pass inputs
+        # as the first arg if it takes positional args
+        if "inputs" not in kwargs and len(sig.parameters) > 0:
+            first_param = list(sig.parameters.keys())[0]
+            if first_param != "context" or not has_context_param:
+                kwargs[first_param] = inputs
+
+        return self._func(**kwargs)
+
+
+def component(name: str | None = None) -> Callable:
+    """Decorator to register a component class or a function in the global
+    ComponentRegistry.
+
+    If used on a function, it wraps it in an adapter that implements BaseComponent.
+    """
+
+    def decorator(
+        target: type[BaseComponent] | Callable,
+    ) -> type[BaseComponent] | Callable:
+        # Determine registry name
+        registry_name = name or target.__name__
+
+        if isinstance(target, type) and issubclass(target, BaseComponent):
+            # Target is a component class
+            ComponentRegistry.register(registry_name, target)
+            return target
+        else:
+            # Target is a function
+            # We must create a dynamically generated subclass to
+            # easily instantiate later.
+            class DynamicFunctionComponent(FunctionComponentAdapter):
+                def __init__(self, **kwargs):
+                    super().__init__(func=target, name=registry_name, **kwargs)
+
+            # Change the __name__ to match
+            DynamicFunctionComponent.__name__ = target.__name__ + "Component"
+            ComponentRegistry.register(registry_name, DynamicFunctionComponent)
+            return target
+
+    return decorator
+
+
+def pandas_component(name: str | None = None) -> Callable:
+    """Decorator to register a pandas-specific function in the global
+    ComponentRegistry.
+
+    The decorated function should take a pd.DataFrame and optionally an IContext,
+    and return a pd.DataFrame. The adapter will handle unwrapping/wrapping IDataset.
+    """
+
+    def decorator(target: Callable) -> Callable:
+        registry_name = name or target.__name__
+
+        class DynamicPandasComponent(FunctionComponentAdapter):
+            def __init__(self, **kwargs):
+                super().__init__(func=target, name=registry_name, **kwargs)
+
+            def _execute(self, inputs: list[IDataset]) -> list[IDataset]:
+                from aptdata.plugins.dataset import InMemoryDataset
+
+                if not inputs:
+                    raise ValueError(
+                        f"Pandas component '{self.component_id}' "
+                        "requires at least one input dataset."
+                    )
+
+                # Unwrap the first input dataset to a pandas DataFrame
+                df = inputs[0].read()
+
+                sig = inspect.signature(self._func)
+                kwargs = {}
+
+                for param_name, param in sig.parameters.items():
+                    if param.annotation == IContext or param_name == "context":
+                        kwargs[param_name] = self.context
+                    else:
+                        kwargs[param_name] = df
+
+                if len(kwargs) == 0 and len(sig.parameters) > 0:
+                    first_param = list(sig.parameters.keys())[0]
+                    kwargs[first_param] = df
+
+                # Execute user function
+                result_df = self._func(**kwargs)
+
+                # Wrap the result back into an IDataset
+                out_ds = InMemoryDataset(uri=f"memory://{self.component_id}_out")
+                out_ds.write(result_df)
+                return [out_ds]
+
+        DynamicPandasComponent.__name__ = target.__name__ + "PandasComponent"
+        ComponentRegistry.register(registry_name, DynamicPandasComponent)
+        return target
+
+    return decorator

aptdata-0.0.3/aptdata/core/events.py

@@ -0,0 +1,104 @@
+"""Event Bus and Lifecycle Hooks for observing system execution."""
+
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from datetime import datetime, timezone
+
+from pydantic import BaseModel, ConfigDict, Field
+
+logger = logging.getLogger(__name__)
+
+
+class EventPayload(BaseModel):
+    """Base class for all events emitted by the framework.
+    All events must be serializable to JSON Lines for TUI/MCP."""
+
+    model_config = ConfigDict(extra="allow")
+
+    event_type: str = Field(..., description="The type/topic of the event.")
+    timestamp: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="When the event occurred.",
+    )
+
+
+class ComponentExecutionEvent(EventPayload):
+    """Event emitted during a component's lifecycle."""
+
+    component_id: str
+    status: str
+    execution_time: float | None = None
+    io_uris: list[str] = Field(default_factory=list)
+    error_message: str | None = None
+
+
+class IEventBus(ABC):
+    """Interface for an Event Bus."""
+
+    @abstractmethod
+    def subscribe(
+        self, event_type: str, listener: Callable[[EventPayload], None]
+    ) -> None:
+        """Register a listener for a specific event type."""
+        pass
+
+    @abstractmethod
+    def dispatch(self, event: EventPayload) -> None:
+        """Dispatch an event asynchronously or non-blocking."""
+        pass
+
+
+class EventBus(IEventBus):
+    """An asynchronous, non-blocking event bus for the core system.
+    Dispatches events using a background thread and a thread-safe queue.
+    Exceptions in listeners are caught and logged as warnings to prevent
+    blocking data processing or the background thread."""
+
+    def __init__(self) -> None:
+        self._listeners: dict[str, list[Callable[[EventPayload], None]]] = {}
+        self._queue: queue.Queue[EventPayload] = queue.Queue()
+        self._stop_event = threading.Event()
+        self._worker_thread = threading.Thread(target=self._worker, daemon=True)
+        self._worker_thread.start()
+
+    def subscribe(
+        self, event_type: str, listener: Callable[[EventPayload], None]
+    ) -> None:
+        if event_type not in self._listeners:
+            self._listeners[event_type] = []
+        self._listeners[event_type].append(listener)
+
+    def dispatch(self, event: EventPayload) -> None:
+        """Enqueue event for background dispatch. Non-blocking."""
+        self._queue.put(event)
+
+    def _worker(self) -> None:
+        """Background worker to process events from the queue."""
+        while not self._stop_event.is_set():
+            try:
+                # Use a timeout to allow checking _stop_event periodically
+                event = self._queue.get(timeout=0.1)
+            except queue.Empty:
+                continue
+
+            listeners = self._listeners.get(event.event_type, [])
+            for listener in listeners:
+                try:
+                    listener(event)
+                except Exception as e:
+                    logger.warning(
+                        f"Listener {getattr(listener, '__name__', str(listener))} "
+                        f"failed on event {event.event_type}: {e}"
+                    )
+            self._queue.task_done()
+
+    def shutdown(self, timeout: float | None = None) -> None:
+        """Wait for all events to be processed and shut down the worker thread."""
+        self._queue.join()
+        self._stop_event.set()
+        self._worker_thread.join(timeout=timeout)

aptdata-0.0.3/aptdata/core/registry.py

@@ -0,0 +1,31 @@
+import logging
+
+from aptdata.core.system import BaseComponent
+
+logger = logging.getLogger(__name__)
+
+
+class ComponentRegistry:
+    """Global registry for dynamically registering and resolving components by name."""
+
+    _components: dict[str, type[BaseComponent]] = {}
+
+    @classmethod
+    def register(cls, name: str, component_class: type[BaseComponent]) -> None:
+        """Register a component class with a specific name."""
+        if name in cls._components:
+            logger.warning(f"Component '{name}' is already registered. Overwriting.")
+        cls._components[name] = component_class
+        logger.debug(f"Registered component '{name}' -> {component_class.__name__}")
+
+    @classmethod
+    def get(cls, name: str) -> type[BaseComponent]:
+        """Retrieve a component class by name."""
+        if name not in cls._components:
+            raise KeyError(f"Component '{name}' is not registered.")
+        return cls._components[name]
+
+    @classmethod
+    def clear(cls) -> None:
+        """Clear all registered components."""
+        cls._components.clear()