loopengt 0.1.0__py3-none-any.whl

loopengt/core/evals/regression.py

@@ -0,0 +1,157 @@
+"""Regression test harness for loop specifications."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from loopengt.core.evals.metrics import MetricResult, evaluate_all
+
+logger = structlog.get_logger(__name__)
+
+
+class RegressionResult:
+    """Result of a regression test run."""
+
+    __slots__ = ("spec_name", "baseline_metrics", "current_metrics", "regressions")
+
+    def __init__(
+        self,
+        spec_name: str,
+        baseline_metrics: list[MetricResult],
+        current_metrics: list[MetricResult],
+        regressions: list[str],
+    ) -> None:
+        self.spec_name = spec_name
+        self.baseline_metrics = baseline_metrics
+        self.current_metrics = current_metrics
+        self.regressions = regressions
+
+    @property
+    def passed(self) -> bool:
+        return len(self.regressions) == 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "spec_name": self.spec_name,
+            "passed": self.passed,
+            "regressions": self.regressions,
+            "baseline": [m.to_dict() for m in self.baseline_metrics],
+            "current": [m.to_dict() for m in self.current_metrics],
+        }
+
+
+class RegressionHarness:
+    """Runs loop specs and compares against baseline metrics.
+
+    Usage::
+
+        harness = RegressionHarness(baselines_dir=Path(".loopengt/baselines"))
+        result = await harness.run(spec_path, baseline_id="v1.0")
+    """
+
+    def __init__(self, baselines_dir: Path | None = None) -> None:
+        self._baselines_dir = baselines_dir or Path(".loopengt/baselines")
+        self._baselines_dir.mkdir(parents=True, exist_ok=True)
+        self._log = logger.bind(component="regression")
+
+    async def run(
+        self,
+        spec_path: Path,
+        baseline_id: str | None = None,
+    ) -> RegressionResult:
+        """Execute a spec and compare with baseline metrics."""
+        import yaml
+
+        from loopengt.core.models.loop_spec import LoopSpec
+        from loopengt.core.runtime.executor import LoopExecutor
+
+        # Load and run the spec
+        raw = yaml.safe_load(spec_path.read_text(encoding="utf-8"))
+        spec = LoopSpec.model_validate(raw)
+        executor = LoopExecutor(spec)
+        state = await executor.execute()
+
+        # Evaluate current run
+        trace = state.model_dump(mode="json")
+        current_metrics = evaluate_all(trace)
+
+        # Load baseline
+        baseline_metrics = self._load_baseline(spec.name, baseline_id)
+
+        # Compare
+        regressions = self._compare(baseline_metrics, current_metrics)
+
+        result = RegressionResult(
+            spec_name=spec.name,
+            baseline_metrics=baseline_metrics,
+            current_metrics=current_metrics,
+            regressions=regressions,
+        )
+
+        self._log.info(
+            "regression.complete",
+            spec=spec.name,
+            passed=result.passed,
+            regressions=len(regressions),
+        )
+
+        return result
+
+    async def save_baseline(
+        self, spec_name: str, metrics: list[MetricResult], baseline_id: str = "latest"
+    ) -> None:
+        """Save metrics as a baseline for future comparison."""
+        path = self._baselines_dir / f"{spec_name}_{baseline_id}.json"
+        data = [m.to_dict() for m in metrics]
+        path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+        self._log.info("regression.baseline_saved", spec=spec_name, id=baseline_id)
+
+    def _load_baseline(
+        self, spec_name: str, baseline_id: str | None
+    ) -> list[MetricResult]:
+        """Load baseline metrics.  Returns empty list if none found."""
+        bid = baseline_id or "latest"
+        path = self._baselines_dir / f"{spec_name}_{bid}.json"
+
+        if not path.exists():
+            return []
+
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return [
+            MetricResult(
+                name=m["name"],
+                value=m["value"],
+                passed=m["passed"],
+                detail=m.get("detail", ""),
+            )
+            for m in data
+        ]
+
+    def _compare(
+        self, baseline: list[MetricResult], current: list[MetricResult]
+    ) -> list[str]:
+        """Compare current metrics against baseline.  Returns regression descriptions."""
+        if not baseline:
+            return []  # No baseline to compare against
+
+        baseline_map = {m.name: m for m in baseline}
+        regressions = []
+
+        for metric in current:
+            base = baseline_map.get(metric.name)
+            if base is None:
+                continue
+
+            # Check for regressions
+            if base.passed and not metric.passed:
+                regressions.append(
+                    f"{metric.name}: was passing, now failing "
+                    f"(baseline={base.value}, current={metric.value})"
+                )
+
+        return regressions

loopengt/core/memory/__init__.py

@@ -0,0 +1 @@
+"""Memory subsystem for agent context management."""

loopengt/core/memory/retrieval.py

@@ -0,0 +1,124 @@
+"""Retrieval logic for agent memory — keyword + semantic."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+
+from loopengt.core.memory.store import MemoryEntry, MemoryStore
+
+logger = structlog.get_logger(__name__)
+
+
+class MemoryRetriever:
+    """Retrieves relevant memories using keyword and/or vector search.
+
+    Combines results from both search strategies with configurable
+    weighting.
+
+    Usage::
+
+        retriever = MemoryRetriever(store)
+        results = retriever.retrieve("API design patterns", top_k=5)
+    """
+
+    def __init__(
+        self,
+        store: MemoryStore,
+        *,
+        keyword_weight: float = 0.5,
+        vector_weight: float = 0.5,
+    ) -> None:
+        self._store = store
+        self._keyword_weight = keyword_weight
+        self._vector_weight = vector_weight
+        self._log = logger.bind(component="retriever")
+
+    def retrieve(
+        self,
+        query: str,
+        *,
+        top_k: int = 5,
+        metadata_filter: dict[str, Any] | None = None,
+        query_embedding: list[float] | None = None,
+    ) -> list[MemoryEntry]:
+        """Retrieve top-k relevant memories.
+
+        If *query_embedding* is provided, combines keyword and vector
+        search scores.  Otherwise, uses keyword search only.
+        """
+        keyword_results = self._store.search(
+            query, top_k=top_k * 2, metadata_filter=metadata_filter
+        )
+
+        if query_embedding is None:
+            return keyword_results[:top_k]
+
+        vector_results = self._store.vector_search(
+            query_embedding, top_k=top_k * 2, metadata_filter=metadata_filter
+        )
+
+        # Merge and re-rank
+        merged = self._merge_results(keyword_results, vector_results)
+        return merged[:top_k]
+
+    def retrieve_by_metadata(
+        self,
+        metadata_filter: dict[str, Any],
+        *,
+        top_k: int = 10,
+    ) -> list[MemoryEntry]:
+        """Retrieve entries matching specific metadata criteria."""
+        results = [
+            entry
+            for entry in self._store.all_entries()
+            if all(entry.metadata.get(k) == v for k, v in metadata_filter.items())
+        ]
+        return results[:top_k]
+
+    def retrieve_recent(self, *, top_k: int = 5) -> list[MemoryEntry]:
+        """Retrieve the most recently added entries."""
+        entries = self._store.all_entries()
+        # Entries are stored in insertion order (dict in Python 3.7+)
+        return entries[-top_k:][::-1]
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _merge_results(
+        self,
+        keyword: list[MemoryEntry],
+        vector: list[MemoryEntry],
+    ) -> list[MemoryEntry]:
+        """Merge keyword and vector results with weighted scoring."""
+        combined: dict[str, MemoryEntry] = {}
+
+        for entry in keyword:
+            eid = entry.entry_id
+            if eid not in combined:
+                combined[eid] = MemoryEntry(
+                    entry_id=eid,
+                    content=entry.content,
+                    metadata=entry.metadata,
+                    embedding=entry.embedding,
+                    score=0.0,
+                )
+            combined[eid].score += entry.score * self._keyword_weight
+
+        for entry in vector:
+            eid = entry.entry_id
+            if eid not in combined:
+                combined[eid] = MemoryEntry(
+                    entry_id=eid,
+                    content=entry.content,
+                    metadata=entry.metadata,
+                    embedding=entry.embedding,
+                    score=0.0,
+                )
+            combined[eid].score += entry.score * self._vector_weight
+
+        results = list(combined.values())
+        results.sort(key=lambda e: e.score, reverse=True)
+        return results

loopengt/core/memory/store.py

@@ -0,0 +1,184 @@
+"""In-memory vector + keyword store for agent memory."""
+
+from __future__ import annotations
+
+import hashlib
+import math
+from dataclasses import dataclass, field
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class MemoryEntry:
+    """A single memory entry with text content and optional embedding."""
+
+    entry_id: str
+    content: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    embedding: list[float] | None = None
+    score: float = 0.0
+
+
+class MemoryStore:
+    """Lightweight in-memory store for agent memories.
+
+    Supports keyword search and (when embeddings are provided)
+    cosine-similarity based vector search.
+
+    Usage::
+
+        store = MemoryStore()
+        store.add("plan-1", "Build the API endpoints first", metadata={"step": "plan"})
+        results = store.search("API endpoints", top_k=3)
+    """
+
+    def __init__(self) -> None:
+        self._entries: dict[str, MemoryEntry] = {}
+        self._log = logger.bind(component="memory_store")
+
+    # ------------------------------------------------------------------
+    # Write operations
+    # ------------------------------------------------------------------
+
+    def add(
+        self,
+        entry_id: str | None = None,
+        content: str = "",
+        metadata: dict[str, Any] | None = None,
+        embedding: list[float] | None = None,
+    ) -> str:
+        """Add a memory entry.  Returns the entry ID."""
+        if entry_id is None:
+            entry_id = hashlib.sha256(content.encode()).hexdigest()[:12]
+
+        self._entries[entry_id] = MemoryEntry(
+            entry_id=entry_id,
+            content=content,
+            metadata=metadata or {},
+            embedding=embedding,
+        )
+        self._log.debug("memory.add", entry_id=entry_id, length=len(content))
+        return entry_id
+
+    def remove(self, entry_id: str) -> bool:
+        """Remove an entry by ID.  Returns True if found."""
+        if entry_id in self._entries:
+            del self._entries[entry_id]
+            return True
+        return False
+
+    def clear(self) -> None:
+        """Remove all entries."""
+        self._entries.clear()
+
+    # ------------------------------------------------------------------
+    # Read / search operations
+    # ------------------------------------------------------------------
+
+    def get(self, entry_id: str) -> MemoryEntry | None:
+        """Get an entry by ID."""
+        return self._entries.get(entry_id)
+
+    def search(
+        self,
+        query: str,
+        *,
+        top_k: int = 5,
+        metadata_filter: dict[str, Any] | None = None,
+    ) -> list[MemoryEntry]:
+        """Search entries by keyword matching.
+
+        Scores entries by the fraction of query words found in the
+        content.  Optionally filters by metadata key-value matches.
+        """
+        query_words = set(query.lower().split())
+        if not query_words:
+            return []
+
+        scored: list[MemoryEntry] = []
+        for entry in self._entries.values():
+            if metadata_filter and not self._matches_filter(entry, metadata_filter):
+                continue
+
+            content_words = set(entry.content.lower().split())
+            overlap = query_words & content_words
+            score = len(overlap) / len(query_words) if query_words else 0.0
+
+            if score > 0:
+                # Return a copy with the score
+                scored.append(
+                    MemoryEntry(
+                        entry_id=entry.entry_id,
+                        content=entry.content,
+                        metadata=entry.metadata,
+                        embedding=entry.embedding,
+                        score=score,
+                    )
+                )
+
+        scored.sort(key=lambda e: e.score, reverse=True)
+        return scored[:top_k]
+
+    def vector_search(
+        self,
+        query_embedding: list[float],
+        *,
+        top_k: int = 5,
+        metadata_filter: dict[str, Any] | None = None,
+    ) -> list[MemoryEntry]:
+        """Search entries by cosine similarity of embeddings."""
+        scored: list[MemoryEntry] = []
+
+        for entry in self._entries.values():
+            if entry.embedding is None:
+                continue
+            if metadata_filter and not self._matches_filter(entry, metadata_filter):
+                continue
+
+            sim = self._cosine_similarity(query_embedding, entry.embedding)
+            scored.append(
+                MemoryEntry(
+                    entry_id=entry.entry_id,
+                    content=entry.content,
+                    metadata=entry.metadata,
+                    embedding=entry.embedding,
+                    score=sim,
+                )
+            )
+
+        scored.sort(key=lambda e: e.score, reverse=True)
+        return scored[:top_k]
+
+    @property
+    def size(self) -> int:
+        """Number of entries in the store."""
+        return len(self._entries)
+
+    def all_entries(self) -> list[MemoryEntry]:
+        """Return all entries."""
+        return list(self._entries.values())
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _cosine_similarity(a: list[float], b: list[float]) -> float:
+        """Compute cosine similarity between two vectors."""
+        if len(a) != len(b):
+            return 0.0
+        dot = sum(x * y for x, y in zip(a, b))
+        norm_a = math.sqrt(sum(x * x for x in a))
+        norm_b = math.sqrt(sum(x * x for x in b))
+        if norm_a == 0 or norm_b == 0:
+            return 0.0
+        return dot / (norm_a * norm_b)
+
+    @staticmethod
+    def _matches_filter(entry: MemoryEntry, filt: dict[str, Any]) -> bool:
+        """Check if an entry's metadata matches all filter conditions."""
+        return all(entry.metadata.get(k) == v for k, v in filt.items())

loopengt/core/memory/summarizer.py

@@ -0,0 +1,97 @@
+"""Context window summarization for long-running loops."""
+
+from __future__ import annotations
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+class Summarizer:
+    """Summarises conversation / context history to fit within token budgets.
+
+    Uses a simple extractive strategy by default.  When an LLM provider
+    is available, switches to abstractive summarization.
+
+    Usage::
+
+        summarizer = Summarizer(max_tokens=4096)
+        short = summarizer.summarize(long_history)
+    """
+
+    def __init__(
+        self,
+        *,
+        max_tokens: int = 4096,
+        strategy: str = "extractive",
+    ) -> None:
+        self._max_tokens = max_tokens
+        self._strategy = strategy
+        self._log = logger.bind(component="summarizer")
+
+    def summarize(self, entries: list[str]) -> str:
+        """Produce a condensed summary of the given text entries.
+
+        Parameters
+        ----------
+        entries:
+            List of text chunks (e.g., step outputs, conversation turns)
+            to summarize.
+
+        Returns
+        -------
+        str
+            The summarized text, estimated to be within ``max_tokens``.
+        """
+        if not entries:
+            return ""
+
+        full_text = "\n\n".join(entries)
+        estimated_tokens = len(full_text.split())  # rough word-based estimate
+
+        if estimated_tokens <= self._max_tokens:
+            return full_text
+
+        self._log.info(
+            "summarizer.truncating",
+            original_tokens=estimated_tokens,
+            target_tokens=self._max_tokens,
+        )
+
+        if self._strategy == "extractive":
+            return self._extractive_summarize(entries)
+        else:
+            return self._truncate(full_text)
+
+    def _extractive_summarize(self, entries: list[str]) -> str:
+        """Keep the first and last entries, plus sampled middle entries."""
+        if len(entries) <= 3:
+            return "\n\n".join(entries)
+
+        # Always keep first and last
+        selected = [entries[0]]
+
+        # Sample from the middle, keeping budget
+        middle = entries[1:-1]
+        budget = self._max_tokens - len(entries[0].split()) - len(entries[-1].split())
+
+        used = 0
+        step = max(1, len(middle) // 5)  # keep ~5 middle entries
+        for i in range(0, len(middle), step):
+            entry_tokens = len(middle[i].split())
+            if used + entry_tokens > budget:
+                selected.append("... [context truncated] ...")
+                break
+            selected.append(middle[i])
+            used += entry_tokens
+
+        selected.append(entries[-1])
+        return "\n\n".join(selected)
+
+    def _truncate(self, text: str) -> str:
+        """Simple truncation to fit token budget."""
+        words = text.split()
+        if len(words) <= self._max_tokens:
+            return text
+        truncated = " ".join(words[: self._max_tokens])
+        return truncated + "\n\n... [truncated]"

loopengt/core/models/__init__.py

@@ -0,0 +1,43 @@
+"""Core data models for loop engineering."""
+
+from loopengt.core.models.agent import AgentCapability, AgentConfig, AgentRole
+from loopengt.core.models.loop_spec import (
+    LoopSpec,
+    OutputSchema,
+    StepSpec,
+    StopCondition,
+)
+from loopengt.core.models.policy import (
+    BackoffStrategy,
+    Policy,
+    RetryPolicy,
+    VerificationGate,
+)
+from loopengt.core.models.state import (
+    AgentState,
+    CheckpointData,
+    LoopState,
+    StepState,
+)
+from loopengt.core.models.tool import ToolAuth, ToolSpec, ToolType
+
+__all__ = [
+    "AgentCapability",
+    "AgentConfig",
+    "AgentRole",
+    "AgentState",
+    "BackoffStrategy",
+    "CheckpointData",
+    "LoopSpec",
+    "LoopState",
+    "OutputSchema",
+    "Policy",
+    "RetryPolicy",
+    "StepSpec",
+    "StepState",
+    "StopCondition",
+    "ToolAuth",
+    "ToolSpec",
+    "ToolType",
+    "VerificationGate",
+]