aptdata 0.0.2__py3-none-any.whl

aptdata/config/__init__.py

@@ -0,0 +1,13 @@
+"""Declarative configuration helpers for aptdata."""
+
+from aptdata.config.parser import ParsedConfig, YamlConfigParser
+from aptdata.config.schema import export_domain_schema, write_domain_schema
+from aptdata.config.secrets import SecretManager
+
+__all__ = [
+    "ParsedConfig",
+    "YamlConfigParser",
+    "export_domain_schema",
+    "write_domain_schema",
+    "SecretManager",
+]

aptdata/config/parser.py

@@ -0,0 +1,136 @@
+"""YAML parser for declarative aptdata system definitions."""
+
+from __future__ import annotations
+
+from dataclasses import field
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import TypeAdapter
+from pydantic.dataclasses import dataclass as pydantic_dataclass
+
+from aptdata.config.secrets import SecretManager
+from aptdata.core.dataset import IDataset
+from aptdata.core.system import BaseComponent, BaseFlow, BaseSystem, IFlow
+
+
+@pydantic_dataclass
+class ConfigEdge:
+    """Serializable flow edge for declarative YAML files."""
+
+    source_id: str
+    target_id: str
+    condition: str = ""
+
+
+@pydantic_dataclass
+class ConfigComponent(BaseComponent):
+    """Concrete component used by configuration hydration."""
+
+    def validate_inputs(self, inputs: list[IDataset]) -> bool:  # noqa: ARG002
+        return True
+
+    def execute(self, inputs: list[IDataset]) -> list[IDataset]:
+        return inputs
+
+
+@pydantic_dataclass
+class ConfigFlow(BaseFlow):
+    """Concrete flow used by configuration hydration."""
+
+    components: list[ConfigComponent] = field(default_factory=list)
+    edges: list[ConfigEdge] = field(default_factory=list)
+
+    def add_component(self, component: BaseComponent) -> None:
+        if not isinstance(component, ConfigComponent):
+            raise TypeError("ConfigFlow only accepts ConfigComponent instances.")
+        self.components.append(component)
+
+    def connect(
+        self,
+        source_id: str,
+        target_id: str,
+        condition: str | None = None,
+    ) -> None:
+        self.edges.append(
+            ConfigEdge(
+                source_id=source_id,
+                target_id=target_id,
+                condition=condition or "",
+            )
+        )
+
+    def compile(self) -> None:
+        pass
+
+    def run(self, initial_inputs: list[IDataset]) -> list[IDataset]:
+        return initial_inputs
+
+
+@pydantic_dataclass
+class ConfigSystem(BaseSystem):
+    """Concrete system used by configuration hydration."""
+
+    flows: list[ConfigFlow] = field(default_factory=list)
+
+    def register_flow(self, flow: IFlow) -> None:
+        if not isinstance(flow, ConfigFlow):
+            raise TypeError("ConfigSystem only accepts ConfigFlow instances.")
+        self.flows.append(flow)
+
+    def run(self) -> None:
+        pass
+
+
+@pydantic_dataclass
+class ParsedConfig:
+    """Hydrated config document with metadata and validated domain objects."""
+
+    metadata: dict[str, Any] = field(default_factory=dict)
+    system: ConfigSystem = field(
+        default_factory=lambda: ConfigSystem(system_id="default")
+    )
+
+
+class YamlConfigParser:
+    """Parse YAML files into validated aptdata domain objects."""
+
+    _component_adapter = TypeAdapter(ConfigComponent)
+    _flow_adapter = TypeAdapter(ConfigFlow)
+    _edge_adapter = TypeAdapter(ConfigEdge)
+    _system_adapter = TypeAdapter(ConfigSystem)
+
+    def __init__(self, secret_manager: SecretManager | None = None) -> None:
+        self._secret_manager = secret_manager or SecretManager()
+
+    def parse_file(self, path: str | Path) -> ParsedConfig:
+        """Read and parse a YAML config file."""
+        config_path = Path(path)
+        raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
+        if not isinstance(raw, dict):
+            raise ValueError("YAML root must be a mapping/object.")
+        return self.parse_data(raw)
+
+    def parse_data(self, payload: dict[str, Any]) -> ParsedConfig:
+        """Parse a loaded YAML dictionary."""
+        payload = self._secret_manager.resolve(payload)
+        metadata = payload.get("metadata", {})
+        system_payload = dict(payload.get("system", {}))
+        flow_payloads = system_payload.pop("flows", payload.get("flows", []))
+
+        system = self._system_adapter.validate_python(system_payload)
+        for flow_payload in flow_payloads:
+            flow_data = dict(flow_payload)
+            component_payloads = flow_data.pop("components", [])
+            edge_payloads = flow_data.pop("edges", [])
+
+            flow = self._flow_adapter.validate_python(flow_data)
+            for component_payload in component_payloads:
+                component = self._component_adapter.validate_python(component_payload)
+                flow.add_component(component)
+            for edge_payload in edge_payloads:
+                flow.edges.append(self._edge_adapter.validate_python(edge_payload))
+            system.register_flow(flow)
+
+        return ParsedConfig(metadata=metadata, system=system)

aptdata/config/schema.py

@@ -0,0 +1,27 @@
+"""JSON Schema utilities for declarative aptdata configs."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import TypeAdapter
+
+from aptdata.config.parser import ParsedConfig
+
+
+def export_domain_schema() -> dict[str, Any]:
+    """Export JSON Schema for the full declarative config domain."""
+    return TypeAdapter(ParsedConfig).json_schema()
+
+
+def write_domain_schema(output: str | Path) -> Path:
+    """Write the domain JSON Schema to *output*."""
+    output_path = Path(output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(
+        json.dumps(export_domain_schema(), indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    return output_path

aptdata/config/secrets.py

@@ -0,0 +1,60 @@
+"""Secret resolution helpers for aptdata configuration and plugins."""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any
+
+from aptdata.telemetry.instrumentation import register_secret
+
+_ENV_PATTERN = re.compile(r"\$\{([A-Za-z0-9_]+)\}")
+
+
+class SecretManager:
+    """Resolve `${ENV_VAR}` placeholders using environment variables."""
+
+    def __init__(self, *, load_dotenv_file: bool = True) -> None:
+        if load_dotenv_file:
+            try:
+                from dotenv import load_dotenv  # noqa: WPS433
+            except ImportError:
+                load_dotenv = None
+            if load_dotenv is not None:
+                load_dotenv(dotenv_path=".env")
+        self._injected_keys: set[str] = set()
+
+    def get(self, key: str, default: str | None = None) -> str:
+        """Return environment value for *key* and register it as a secret."""
+        value = os.getenv(key, default)
+        if value is None:
+            raise KeyError(f"Missing required secret: {key}")
+        self._injected_keys.add(key)
+        register_secret(key, value)
+        return value
+
+    def resolve(self, value: Any) -> Any:
+        """Recursively resolve `${ENV_VAR}` placeholders in nested structures."""
+        if isinstance(value, str):
+            return self._resolve_string(value)
+        if isinstance(value, dict):
+            return {k: self.resolve(v) for k, v in value.items()}
+        if isinstance(value, list):
+            return [self.resolve(item) for item in value]
+        if isinstance(value, tuple):
+            return tuple(self.resolve(item) for item in value)
+        return value
+
+    def injected_keys(self) -> list[str]:
+        """Return sorted secret names injected in this manager session."""
+        return sorted(self._injected_keys)
+
+    def _resolve_string(self, value: str) -> str:
+        matches = _ENV_PATTERN.findall(value)
+        if not matches:
+            return value
+        resolved = value
+        for key in matches:
+            resolved_secret = self.get(key)
+            resolved = resolved.replace(f"${{{key}}}", resolved_secret)
+        return resolved

aptdata/core/__init__.py

@@ -0,0 +1,46 @@
+"""Core interfaces and base classes for aptdata."""
+
+from aptdata.core.context import ExecutionContext
+from aptdata.core.dataset import BaseDataset, IDataset
+from aptdata.core.state import StateBackend
+from aptdata.core.system import (
+    BaseComponent,
+    BaseFlow,
+    BaseSystem,
+    ComponentKind,
+    ComponentMeta,
+    FlowEdge,
+    FlowNode,
+    IComponent,
+    IFlow,
+    ISystem,
+)
+from aptdata.core.workflow import (
+    BaseWorkflow,
+    IWorkflow,
+    Workflow,
+    WorkflowEdge,
+    WorkflowNode,
+)
+
+__all__ = [
+    "IDataset",
+    "BaseDataset",
+    "ExecutionContext",
+    "ComponentKind",
+    "ComponentMeta",
+    "IComponent",
+    "BaseComponent",
+    "FlowEdge",
+    "FlowNode",
+    "IFlow",
+    "BaseFlow",
+    "ISystem",
+    "BaseSystem",
+    "WorkflowEdge",
+    "WorkflowNode",
+    "IWorkflow",
+    "BaseWorkflow",
+    "Workflow",
+    "StateBackend",
+]

aptdata/core/context.py

@@ -0,0 +1,31 @@
+"""Injectable execution context for state sharing across runs."""
+
+from __future__ import annotations
+
+from dataclasses import field
+from typing import Any
+
+from pydantic.dataclasses import dataclass as pydantic_dataclass
+
+
+@pydantic_dataclass
+class ExecutionContext:
+    """Simple in-memory key/value state container."""
+
+    memory: dict[str, Any] = field(default_factory=dict)
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Return value for *key* or *default* if not present."""
+        return self.memory.get(key, default)
+
+    def set(self, key: str, value: Any) -> None:
+        """Store *value* under *key*."""
+        self.memory[key] = value
+
+    def update(self, values: dict[str, Any]) -> None:
+        """Merge mapping into memory state."""
+        self.memory.update(values)
+
+    def clear(self) -> None:
+        """Remove all state."""
+        self.memory.clear()

aptdata/core/dataset.py

@@ -0,0 +1,39 @@
+"""Dataset interface and base class."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from pydantic.dataclasses import dataclass as pydantic_dataclass
+
+
+@dataclass
+class IDataset(ABC):
+    """Dataclass interface for dataset types.
+
+    All dataset contracts must implement :meth:`read` and :meth:`write`.
+    No concrete fields are defined here – field declarations live in
+    :class:`BaseDataset` and its subclasses.
+    """
+
+    @abstractmethod
+    def read(self) -> Any:
+        """Read and return data from the dataset."""
+
+    @abstractmethod
+    def write(self, data: Any) -> None:
+        """Write data to the dataset."""
+
+
+@pydantic_dataclass
+class BaseDataset(IDataset):
+    """Base dataset with Pydantic-validated fields.
+
+    Provides the canonical ``uri`` and ``schema_metadata`` fields.
+    Concrete dataset implementations must inherit from this class and
+    implement the :meth:`read` and :meth:`write` abstract methods
+    inherited from :class:`IDataset`.
+    """
+
+    uri: str
+    schema_metadata: dict[str, Any] = field(default_factory=dict)

aptdata/core/lineage.py

@@ -0,0 +1,213 @@
+"""Data lineage graph for tracking dataset transformations and provenance.
+
+Provides :class:`LineageGraph` which is built up during a workflow run and
+stores :class:`LineageNode` instances that capture every read, transform,
+quality-check, and write event.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any
+from uuid import uuid4
+
+
+class LineageEventType(str, Enum):
+    """Enumeration of lineage event types."""
+
+    READ = "READ"
+    TRANSFORM = "TRANSFORM"
+    QUALITY_CHECK = "QUALITY_CHECK"
+    BUSINESS_RULE = "BUSINESS_RULE"
+    WRITE = "WRITE"
+    SCHEMA_CHANGE = "SCHEMA_CHANGE"
+
+
+@dataclass
+class ColumnLineage:
+    """Column-level lineage mapping a target column to its source columns.
+
+    Parameters
+    ----------
+    target_column:
+        Name of the output column.
+    source_columns:
+        Names of the input columns that contributed to *target_column*.
+    transformation:
+        Human-readable description of the transformation applied.
+    source_dataset_uri:
+        URI of the dataset that contains the source columns.
+    """
+
+    target_column: str
+    source_columns: list[str] = field(default_factory=list)
+    transformation: str = ""
+    source_dataset_uri: str = ""
+
+
+@dataclass
+class LineageNode:
+    """A single event node in the lineage graph.
+
+    Parameters
+    ----------
+    dataset_uri:
+        Logical URI of the dataset involved in this event.
+    event_type:
+        The kind of lineage event (see :class:`LineageEventType`).
+    workflow_name:
+        Name of the workflow that produced this node.
+    step_name:
+        Name of the individual step within the workflow.
+    transformer_name:
+        Name of the transformer (for TRANSFORM events).
+    engine:
+        Processing engine (e.g. ``"pandas"`` or ``"pyspark"``).
+    rows_in:
+        Number of input rows.
+    rows_out:
+        Number of output rows.
+    columns_in:
+        List of input column names.
+    columns_out:
+        List of output column names.
+    column_lineage:
+        Column-level lineage mappings.
+    business_rules_applied:
+        IDs of business rules applied during this event.
+    quality_checks:
+        Names of quality checks executed.
+    quality_status:
+        Overall quality check outcome (e.g. ``"PASSED"`` or ``"FAILED"``).
+    trace_id:
+        OpenTelemetry trace identifier.
+    span_id:
+        OpenTelemetry span identifier.
+    parent_node_ids:
+        IDs of upstream :class:`LineageNode` objects.
+    metadata:
+        Arbitrary extra metadata.
+    node_id:
+        Auto-generated unique identifier (UUID hex).
+    timestamp:
+        UTC ISO-8601 timestamp of the event.
+    """
+
+    dataset_uri: str
+    event_type: LineageEventType
+    workflow_name: str = ""
+    step_name: str = ""
+    transformer_name: str = ""
+    engine: str = ""
+    rows_in: int = 0
+    rows_out: int = 0
+    columns_in: list[str] = field(default_factory=list)
+    columns_out: list[str] = field(default_factory=list)
+    column_lineage: list[ColumnLineage] = field(default_factory=list)
+    business_rules_applied: list[str] = field(default_factory=list)
+    quality_checks: list[str] = field(default_factory=list)
+    quality_status: str = ""
+    trace_id: str = ""
+    span_id: str = ""
+    parent_node_ids: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+    node_id: str = field(default_factory=lambda: uuid4().hex)
+    timestamp: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize this node to a plain dictionary."""
+        return {
+            "node_id": self.node_id,
+            "dataset_uri": self.dataset_uri,
+            "event_type": self.event_type.value,
+            "workflow_name": self.workflow_name,
+            "step_name": self.step_name,
+            "transformer_name": self.transformer_name,
+            "engine": self.engine,
+            "rows_in": self.rows_in,
+            "rows_out": self.rows_out,
+            "columns_in": self.columns_in,
+            "columns_out": self.columns_out,
+            "column_lineage": [
+                {
+                    "target_column": cl.target_column,
+                    "source_columns": cl.source_columns,
+                    "transformation": cl.transformation,
+                    "source_dataset_uri": cl.source_dataset_uri,
+                }
+                for cl in self.column_lineage
+            ],
+            "business_rules_applied": self.business_rules_applied,
+            "quality_checks": self.quality_checks,
+            "quality_status": self.quality_status,
+            "trace_id": self.trace_id,
+            "span_id": self.span_id,
+            "parent_node_ids": self.parent_node_ids,
+            "metadata": self.metadata,
+            "timestamp": self.timestamp,
+        }
+
+
+@dataclass
+class LineageGraph:
+    """A complete lineage graph for a single workflow run.
+
+    Parameters
+    ----------
+    run_id:
+        Unique identifier for the workflow run.
+    workflow_name:
+        Human-readable name of the workflow.
+    nodes:
+        Ordered list of :class:`LineageNode` objects.
+    started_at:
+        UTC ISO-8601 timestamp when the run started.
+    """
+
+    run_id: str
+    workflow_name: str
+    nodes: list[LineageNode] = field(default_factory=list)
+    started_at: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+
+    def add_node(self, node: LineageNode) -> None:
+        """Append *node* to the graph."""
+        self.nodes.append(node)
+
+    def get_upstream(self, node_id: str) -> list[LineageNode]:
+        """Return all nodes that are direct parents of *node_id*.
+
+        Nodes are considered parents when their ``node_id`` appears in
+        the target node's :attr:`~LineageNode.parent_node_ids` list.
+        """
+        target = next((n for n in self.nodes if n.node_id == node_id), None)
+        if target is None:
+            return []
+        parent_ids = set(target.parent_node_ids)
+        return [n for n in self.nodes if n.node_id in parent_ids]
+
+    def get_downstream(self, node_id: str) -> list[LineageNode]:
+        """Return all nodes that list *node_id* as a parent."""
+        return [n for n in self.nodes if node_id in n.parent_node_ids]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the full graph to a plain dictionary."""
+        return {
+            "run_id": self.run_id,
+            "workflow_name": self.workflow_name,
+            "started_at": self.started_at,
+            "nodes": [n.to_dict() for n in self.nodes],
+        }
+
+
+__all__ = [
+    "LineageEventType",
+    "ColumnLineage",
+    "LineageNode",
+    "LineageGraph",
+]

aptdata/core/state.py

@@ -0,0 +1,27 @@
+"""Persistent workflow execution state backend."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+class StateBackend:
+    """Simple JSON-on-disk backend used for checkpointing workflow state."""
+
+    def __init__(self, base_dir: str | Path = ".aptdata_state") -> None:
+        self.base_dir = Path(base_dir)
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+
+    def save(self, run_id: str, state: dict[str, Any]) -> None:
+        """Persist *state* for *run_id*."""
+        path = self.base_dir / f"{run_id}.json"
+        path.write_text(
+            json.dumps(state, ensure_ascii=False, default=str), encoding="utf-8"
+        )
+
+    def load(self, run_id: str) -> dict[str, Any]:
+        """Load state for *run_id*."""
+        path = self.base_dir / f"{run_id}.json"
+        return json.loads(path.read_text(encoding="utf-8"))