loopengt 0.1.0__py3-none-any.whl

loopengt/core/runtime/scheduler.py

@@ -0,0 +1,168 @@
+"""Step scheduling and concurrency management."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any
+
+import anyio
+import structlog
+
+from loopengt.core.models.loop_spec import LoopSpec, StepSpec
+from loopengt.core.models.state import LoopState, StepStatus
+
+logger = structlog.get_logger(__name__)
+
+
+class SchedulerError(Exception):
+    """Raised when the scheduler cannot resolve a valid execution order."""
+
+
+class StepScheduler:
+    """Manages step ordering, dependency resolution, and concurrency.
+
+    Given a ``LoopSpec``, the scheduler computes the execution order
+    respecting step dependencies and concurrency limits.
+
+    Usage::
+
+        scheduler = StepScheduler(spec, state)
+        for batch in scheduler.batches():
+            # execute each batch (possibly in parallel)
+            ...
+    """
+
+    def __init__(self, spec: LoopSpec, state: LoopState) -> None:
+        self._spec = spec
+        self._state = state
+        self._log = logger.bind(loop=spec.name)
+        self._dep_graph = self._build_dependency_graph()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def batches(self) -> list[list[StepSpec]]:
+        """Compute execution batches using topological sort.
+
+        Returns a list of batches, where each batch contains steps that
+        can execute concurrently (all their dependencies are satisfied).
+        """
+        resolved: set[str] = set()
+        remaining = {s.name for s in self._spec.steps}
+        batches: list[list[StepSpec]] = []
+
+        while remaining:
+            # Find steps whose dependencies are all resolved
+            ready = [
+                name
+                for name in remaining
+                if all(dep in resolved for dep in self._dep_graph[name])
+            ]
+
+            if not ready:
+                raise SchedulerError(
+                    f"Circular dependency detected among steps: {remaining}"
+                )
+
+            batch = [
+                s for s in self._spec.steps if s.name in ready
+            ]
+            batches.append(batch)
+            resolved.update(ready)
+            remaining -= set(ready)
+
+        return batches
+
+    def next_ready_steps(self) -> list[StepSpec]:
+        """Return steps that are ready to execute now.
+
+        A step is ready when:
+        1. Its status is PENDING.
+        2. All its dependencies have status COMPLETED.
+        """
+        ready = []
+        for step in self._spec.steps:
+            step_state = self._state.step_states.get(step.name)
+            if step_state is None or step_state.status != StepStatus.PENDING:
+                continue
+
+            deps_met = all(
+                self._state.step_states.get(dep, None) is not None
+                and self._state.step_states[dep].status == StepStatus.COMPLETED
+                for dep in self._dep_graph[step.name]
+            )
+            if deps_met:
+                ready.append(step)
+
+        return ready
+
+    def is_complete(self) -> bool:
+        """Return True when all steps have a terminal status."""
+        terminal = {StepStatus.COMPLETED, StepStatus.FAILED, StepStatus.SKIPPED}
+        return all(
+            self._state.step_states.get(s.name, None) is not None
+            and self._state.step_states[s.name].status in terminal
+            for s in self._spec.steps
+        )
+
+    # ------------------------------------------------------------------
+    # Parallel execution helper
+    # ------------------------------------------------------------------
+
+    async def run_parallel(
+        self,
+        step_fn: Any,
+        *,
+        max_concurrency: int | None = None,
+    ) -> None:
+        """Execute steps respecting dependencies with concurrency control.
+
+        Parameters
+        ----------
+        step_fn:
+            An ``async def step_fn(step: StepSpec) -> None`` coroutine
+            that executes a single step.
+        max_concurrency:
+            Maximum number of concurrent steps.  Defaults to the
+            policy's ``max_concurrency``.
+        """
+        concurrency = max_concurrency or self._spec.policy.max_concurrency
+        sem = anyio.Semaphore(concurrency)
+
+        for batch in self.batches():
+            self._log.info("scheduler.batch", steps=[s.name for s in batch])
+
+            async with anyio.create_task_group() as tg:
+                for step in batch:
+                    async def _run(s: StepSpec = step) -> None:
+                        async with sem:
+                            await step_fn(s)
+
+                    tg.start_soon(_run)
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _build_dependency_graph(self) -> dict[str, set[str]]:
+        """Build a mapping of step name → set of dependency step names."""
+        graph: dict[str, set[str]] = defaultdict(set)
+        valid_names = {s.name for s in self._spec.steps}
+
+        for step in self._spec.steps:
+            for dep in step.dependencies:
+                if dep.step_name not in valid_names:
+                    self._log.warning(
+                        "scheduler.unknown_dependency",
+                        step=step.name,
+                        dependency=dep.step_name,
+                    )
+                    continue
+                graph[step.name].add(dep.step_name)
+
+            # Ensure every step has an entry
+            if step.name not in graph:
+                graph[step.name] = set()
+
+        return dict(graph)

loopengt/core/tracing/__init__.py

@@ -0,0 +1 @@
+"""Tracing subsystem for loop execution observability."""

loopengt/core/tracing/events.py

@@ -0,0 +1,95 @@
+"""Event definitions for the tracing subsystem."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+def _utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _event_id() -> str:
+    return uuid.uuid4().hex[:16]
+
+
+class EventType(StrEnum):
+    """Classification of trace events."""
+
+    LOOP_START = "loop.start"
+    LOOP_COMPLETE = "loop.complete"
+    LOOP_ERROR = "loop.error"
+    STEP_START = "step.start"
+    STEP_COMPLETE = "step.complete"
+    STEP_ERROR = "step.error"
+    STEP_RETRY = "step.retry"
+    AGENT_INVOKE = "agent.invoke"
+    AGENT_RESPONSE = "agent.response"
+    AGENT_ERROR = "agent.error"
+    TOOL_CALL = "tool.call"
+    TOOL_RESULT = "tool.result"
+    TOOL_ERROR = "tool.error"
+    CHECKPOINT_SAVE = "checkpoint.save"
+    CHECKPOINT_LOAD = "checkpoint.load"
+    HANDOFF = "handoff"
+    CUSTOM = "custom"
+
+
+class TraceEvent(BaseModel):
+    """Base trace event emitted during loop execution.
+
+    Every event captures a timestamp, the run/step/agent context,
+    and an arbitrary data payload.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    event_id: str = Field(default_factory=_event_id, description="Unique event ID")
+    event_type: EventType = Field(..., description="Event classification")
+    timestamp: datetime = Field(default_factory=_utc_now, description="When the event occurred")
+    run_id: str = Field(default="", description="Parent run ID")
+    step_name: str | None = Field(default=None, description="Step that emitted the event")
+    agent_name: str | None = Field(default=None, description="Agent involved")
+    tool_name: str | None = Field(default=None, description="Tool involved")
+    turn: int | None = Field(default=None, description="Turn number")
+    data: dict[str, Any] = Field(default_factory=dict, description="Event payload")
+    duration_seconds: float | None = Field(default=None, description="Duration if timed")
+    parent_event_id: str | None = Field(default=None, description="Parent event for nesting")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Arbitrary metadata")
+
+
+class LoopEvent(TraceEvent):
+    """Event specific to loop lifecycle (start, complete, error)."""
+
+    loop_name: str = Field(default="", description="Name of the loop")
+    pattern: str = Field(default="", description="Orchestration pattern")
+
+
+class StepEvent(TraceEvent):
+    """Event specific to step execution."""
+
+    attempt: int = Field(default=1, ge=1, description="Attempt number")
+    output: Any = Field(default=None, description="Step output (on completion)")
+    error: str | None = Field(default=None, description="Error message (on failure)")
+
+
+class AgentEvent(TraceEvent):
+    """Event specific to agent invocations."""
+
+    prompt_tokens: int = Field(default=0, ge=0, description="Prompt token count")
+    completion_tokens: int = Field(default=0, ge=0, description="Completion token count")
+    model: str = Field(default="", description="Model used")
+
+
+class ToolEvent(TraceEvent):
+    """Event specific to tool calls and results."""
+
+    tool_type: str = Field(default="", description="Tool type (mcp/function/builtin)")
+    input_data: dict[str, Any] = Field(default_factory=dict, description="Tool input")
+    output_data: Any = Field(default=None, description="Tool output")
+    error: str | None = Field(default=None, description="Error if tool call failed")

loopengt/core/tracing/exporters.py

@@ -0,0 +1,158 @@
+"""Trace exporters — OTel spans, JSON, markdown reports."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Protocol, runtime_checkable
+
+import structlog
+
+from loopengt.core.tracing.events import TraceEvent
+
+logger = structlog.get_logger(__name__)
+
+
+@runtime_checkable
+class TraceExporter(Protocol):
+    """Protocol for trace exporters."""
+
+    async def export(self, events: list[TraceEvent], dest: Path | None = None) -> str: ...
+
+
+class JSONExporter:
+    """Export trace events as a single JSON file."""
+
+    async def export(
+        self, events: list[TraceEvent], dest: Path | None = None
+    ) -> str:
+        """Export events to JSON.  Returns the JSON string."""
+        data = [e.model_dump(mode="json") for e in events]
+        output = json.dumps(data, indent=2, default=str)
+
+        if dest:
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            dest.write_text(output, encoding="utf-8")
+            logger.info("export.json", path=str(dest), events=len(events))
+
+        return output
+
+
+class MarkdownExporter:
+    """Export trace events as a human-readable markdown report."""
+
+    async def export(
+        self, events: list[TraceEvent], dest: Path | None = None
+    ) -> str:
+        """Export events to markdown.  Returns the markdown string."""
+        if not events:
+            return "# Trace Report\n\nNo events recorded.\n"
+
+        run_id = events[0].run_id if events else "unknown"
+        lines = [
+            f"# Trace Report — Run `{run_id}`\n",
+            f"**Events**: {len(events)}\n",
+            "",
+        ]
+
+        # Group by step
+        by_step: dict[str | None, list[TraceEvent]] = {}
+        for e in events:
+            by_step.setdefault(e.step_name, []).append(e)
+
+        for step_name, step_events in by_step.items():
+            heading = step_name or "Loop-level"
+            lines.append(f"## {heading}\n")
+
+            for e in step_events:
+                ts = e.timestamp.strftime("%H:%M:%S.%f")[:-3]
+                dur = f" ({e.duration_seconds:.2f}s)" if e.duration_seconds else ""
+                lines.append(f"- `{ts}` **{e.event_type}**{dur}")
+                if e.data:
+                    for k, v in e.data.items():
+                        lines.append(f"  - {k}: `{v}`")
+            lines.append("")
+
+        output = "\n".join(lines)
+
+        if dest:
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            dest.write_text(output, encoding="utf-8")
+            logger.info("export.markdown", path=str(dest))
+
+        return output
+
+
+class OTelExporter:
+    """Export trace events as OpenTelemetry spans.
+
+    Requires the ``opentelemetry-sdk`` package.  If not installed,
+    the exporter logs a warning and returns empty output.
+    """
+
+    async def export(
+        self, events: list[TraceEvent], dest: Path | None = None
+    ) -> str:
+        """Export events as OTel spans.
+
+        When *dest* is provided, exports spans to a JSON file in
+        OTLP-compatible format.  Otherwise, sends to the configured
+        OTel collector.
+        """
+        try:
+            return await self._export_otel(events, dest)
+        except ImportError:
+            logger.warning(
+                "export.otel.not_installed",
+                hint="pip install opentelemetry-sdk opentelemetry-exporter-otlp",
+            )
+            return ""
+
+    async def _export_otel(
+        self, events: list[TraceEvent], dest: Path | None
+    ) -> str:
+        """Internal OTel export — separated for import isolation."""
+        from opentelemetry import trace as otel_trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+        from opentelemetry.sdk.trace.export.in_memory import InMemorySpanExporter
+
+        provider = TracerProvider()
+        exporter = InMemorySpanExporter()
+        provider.add_span_processor(SimpleSpanProcessor(exporter))
+        tracer = provider.get_tracer("loopengt")
+
+        for event in events:
+            with tracer.start_as_current_span(
+                name=event.event_type,
+                attributes={
+                    "run_id": event.run_id,
+                    "step_name": event.step_name or "",
+                    "agent_name": event.agent_name or "",
+                    "tool_name": event.tool_name or "",
+                },
+            ):
+                pass  # Span auto-completes
+
+        spans = exporter.get_finished_spans()
+        span_data = [
+            {
+                "name": s.name,
+                "trace_id": format(s.context.trace_id, "032x"),
+                "span_id": format(s.context.span_id, "016x"),
+                "start_time": s.start_time,
+                "end_time": s.end_time,
+                "attributes": dict(s.attributes) if s.attributes else {},
+            }
+            for s in spans
+        ]
+
+        output = json.dumps(span_data, indent=2, default=str)
+
+        if dest:
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            dest.write_text(output, encoding="utf-8")
+            logger.info("export.otel", path=str(dest), spans=len(spans))
+
+        provider.shutdown()
+        return output

loopengt/core/tracing/store.py

@@ -0,0 +1,202 @@
+"""Trace storage backends — SQLite and JSONL."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Protocol, runtime_checkable
+
+import structlog
+
+from loopengt.core.tracing.events import TraceEvent
+
+logger = structlog.get_logger(__name__)
+
+
+@runtime_checkable
+class TraceBackend(Protocol):
+    """Protocol for trace storage backends."""
+
+    async def write(self, event: TraceEvent) -> None: ...
+    async def read_run(self, run_id: str) -> list[TraceEvent]: ...
+    async def list_runs(self) -> list[str]: ...
+    async def close(self) -> None: ...
+
+
+class JSONLTraceStore:
+    """JSONL-file-based trace storage.
+
+    Each run is stored in a separate ``.jsonl`` file where every line is
+    a JSON-serialised ``TraceEvent``.
+    """
+
+    def __init__(self, directory: Path | None = None) -> None:
+        self._dir = directory or Path(".loopengt/runs")
+        self._dir.mkdir(parents=True, exist_ok=True)
+        self._handles: dict[str, Any] = {}
+        self._log = logger.bind(backend="jsonl")
+
+    async def write(self, event: TraceEvent) -> None:
+        """Append an event to the run's JSONL file."""
+        path = self._dir / f"{event.run_id}.trace.jsonl"
+        line = event.model_dump_json() + "\n"
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(line)
+
+    async def read_run(self, run_id: str) -> list[TraceEvent]:
+        """Read all events for a run."""
+        path = self._dir / f"{run_id}.trace.jsonl"
+        if not path.exists():
+            return []
+
+        events: list[TraceEvent] = []
+        with open(path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    data = json.loads(line)
+                    events.append(TraceEvent.model_validate(data))
+        return events
+
+    async def list_runs(self) -> list[str]:
+        """List all run IDs with traces."""
+        return sorted(
+            p.stem.replace(".trace", "")
+            for p in self._dir.glob("*.trace.jsonl")
+        )
+
+    async def close(self) -> None:
+        """Close all open file handles."""
+        self._handles.clear()
+
+
+class SQLiteTraceStore:
+    """SQLite-backed trace storage.
+
+    Uses aiosqlite for async access.  Events are stored in a single
+    ``events`` table with JSON-serialised data columns.
+    """
+
+    def __init__(self, db_path: Path | None = None) -> None:
+        self._db_path = db_path or Path(".loopengt/traces.db")
+        self._db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._db: Any = None
+        self._log = logger.bind(backend="sqlite")
+
+    async def _ensure_db(self) -> Any:
+        """Lazily open and initialise the database."""
+        if self._db is None:
+            import aiosqlite
+
+            self._db = await aiosqlite.connect(str(self._db_path))
+            await self._db.execute(
+                """
+                CREATE TABLE IF NOT EXISTS events (
+                    event_id TEXT PRIMARY KEY,
+                    run_id TEXT NOT NULL,
+                    event_type TEXT NOT NULL,
+                    timestamp TEXT NOT NULL,
+                    step_name TEXT,
+                    agent_name TEXT,
+                    tool_name TEXT,
+                    turn INTEGER,
+                    data TEXT NOT NULL,
+                    duration_seconds REAL,
+                    parent_event_id TEXT,
+                    metadata TEXT
+                )
+                """
+            )
+            await self._db.execute(
+                "CREATE INDEX IF NOT EXISTS idx_events_run_id ON events(run_id)"
+            )
+            await self._db.commit()
+        return self._db
+
+    async def write(self, event: TraceEvent) -> None:
+        """Insert an event into the database."""
+        db = await self._ensure_db()
+        await db.execute(
+            """
+            INSERT OR REPLACE INTO events
+            (event_id, run_id, event_type, timestamp, step_name, agent_name,
+             tool_name, turn, data, duration_seconds, parent_event_id, metadata)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                event.event_id,
+                event.run_id,
+                event.event_type,
+                event.timestamp.isoformat(),
+                event.step_name,
+                event.agent_name,
+                event.tool_name,
+                event.turn,
+                json.dumps(event.data, default=str),
+                event.duration_seconds,
+                event.parent_event_id,
+                json.dumps(event.metadata, default=str),
+            ),
+        )
+        await db.commit()
+
+    async def read_run(self, run_id: str) -> list[TraceEvent]:
+        """Read all events for a run, ordered by timestamp."""
+        db = await self._ensure_db()
+        cursor = await db.execute(
+            "SELECT * FROM events WHERE run_id = ? ORDER BY timestamp",
+            (run_id,),
+        )
+        rows = await cursor.fetchall()
+        columns = [desc[0] for desc in cursor.description]
+
+        events = []
+        for row in rows:
+            row_dict = dict(zip(columns, row))
+            row_dict["data"] = json.loads(row_dict.get("data", "{}"))
+            row_dict["metadata"] = json.loads(row_dict.get("metadata", "{}"))
+            events.append(TraceEvent.model_validate(row_dict))
+        return events
+
+    async def list_runs(self) -> list[str]:
+        """List all distinct run IDs."""
+        db = await self._ensure_db()
+        cursor = await db.execute("SELECT DISTINCT run_id FROM events ORDER BY run_id")
+        rows = await cursor.fetchall()
+        return [row[0] for row in rows]
+
+    async def close(self) -> None:
+        """Close the database connection."""
+        if self._db:
+            await self._db.close()
+            self._db = None
+
+
+class TraceStore:
+    """Unified trace store that delegates to a configured backend.
+
+    Defaults to JSONL.  Switch to SQLite by passing ``backend="sqlite"``.
+    """
+
+    def __init__(
+        self,
+        backend: str = "jsonl",
+        directory: Path | None = None,
+        db_path: Path | None = None,
+    ) -> None:
+        if backend == "sqlite":
+            self._backend: TraceBackend = SQLiteTraceStore(db_path=db_path)
+        else:
+            self._backend = JSONLTraceStore(directory=directory)
+
+    async def write(self, event: TraceEvent) -> None:
+        await self._backend.write(event)
+
+    async def read_run(self, run_id: str) -> list[TraceEvent]:
+        return await self._backend.read_run(run_id)
+
+    async def list_runs(self) -> list[str]:
+        return await self._backend.list_runs()
+
+    async def close(self) -> None:
+        await self._backend.close()

loopengt/mcp/__init__.py

@@ -0,0 +1 @@
+"""MCP (Model Context Protocol) integration layer."""

loopengt/mcp/client/__init__.py

@@ -0,0 +1 @@
+"""MCP client subpackage."""