agentkernel-cli 0.1.0__py3-none-any.whl

agentkernel/curation.py

@@ -0,0 +1,183 @@
+"""Self-curating memory: extract durable facts from a transcript, and
+consolidate the notebook with an LLM (design §13, Phase 3).
+
+The storage/recall machinery (``NoteStore``, semantic search, dedup) is mature;
+what was missing is *populating and cleaning* it without the model having to call
+``remember`` in the moment. This module adds two harness operations on top of the
+kernel (an Agent is not required — just a ``NoteStore`` and a ``Provider``):
+
+* ``extract(messages)`` — distil a finished conversation into candidate facts,
+  skipping ones that duplicate (by token overlap) what is already stored.
+* ``consolidate()`` — ask the model to merge related notes and supersede
+  outdated ones, then rebuild the notebook from the cleaned set.
+
+Both are best-effort: an unparseable model reply leaves memory unchanged rather
+than raising or destroying notes.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from agentkernel.memory import MemoryNote, _tokens
+from agentkernel.types import Message
+
+if TYPE_CHECKING:
+    from agentkernel.memory import NoteStore
+    from agentkernel.providers import Provider
+
+_EXTRACT_SYSTEM = (
+    "You curate an AI agent's long-term memory. From a conversation, extract only "
+    "DURABLE facts worth remembering across future sessions: stable user "
+    "preferences, project facts and conventions, decisions, and constraints. "
+    "Exclude transient task details, pleasantries, and anything ephemeral. "
+    'Respond with ONLY a JSON array of objects {"text": "...", "tags": ["..."]}. '
+    "Return [] if nothing is worth remembering."
+)
+
+_CONSOLIDATE_SYSTEM = (
+    "You consolidate an AI agent's long-term memory notes. Produce a cleaner set: "
+    "merge notes that say the same or closely related things, remove redundancy, "
+    "and when two notes conflict keep only the most recent/true statement. Do not "
+    "drop any distinct information. "
+    'Respond with ONLY a JSON array of objects {"text": "...", "tags": ["..."]}.'
+)
+
+
+@dataclass
+class ExtractionResult:
+    added: list[MemoryNote] = field(default_factory=list)
+    skipped_duplicates: int = 0
+
+
+@dataclass
+class ConsolidationResult:
+    before: int
+    after: int
+    notes: list[MemoryNote] = field(default_factory=list)
+
+    @property
+    def removed(self) -> int:
+        return max(self.before - self.after, 0)
+
+
+def _parse_json_array(text: str) -> list[dict]:
+    """Extract a JSON array of objects from a model reply, tolerating prose."""
+    match = re.search(r"\[.*\]", text, re.DOTALL)
+    if not match:
+        return []
+    try:
+        data = json.loads(match.group(0))
+    except json.JSONDecodeError:
+        return []
+    return [item for item in data if isinstance(item, dict)]
+
+
+def _render_transcript(messages: list[Message], max_chars: int) -> str:
+    lines: list[str] = []
+    for m in messages:
+        if m.content and m.role in ("user", "assistant"):
+            lines.append(f"{m.role}: {m.content}")
+        for tc in m.tool_calls:
+            lines.append(f"assistant called {tc.name}")
+        for r in m.tool_results:
+            snippet = r.content if len(r.content) < 200 else r.content[:200] + "…"
+            lines.append(f"tool[{'error' if r.is_error else 'ok'}]: {snippet}")
+    text = "\n".join(lines)
+    if len(text) > max_chars:
+        text = text[:max_chars] + "\n… [transcript truncated]"
+    return text
+
+
+class MemoryCurator:
+    """Populates and cleans a ``NoteStore`` with the help of a model."""
+
+    def __init__(
+        self,
+        notes: NoteStore,
+        provider: Provider,
+        *,
+        max_tokens: int = 1024,
+        dedup_threshold: float = 0.8,
+        transcript_chars: int = 16000,
+    ) -> None:
+        self._notes = notes
+        self._provider = provider
+        self._max_tokens = max_tokens
+        self._dedup_threshold = dedup_threshold
+        self._transcript_chars = transcript_chars
+
+    def extract(self, messages: list[Message]) -> ExtractionResult:
+        transcript = _render_transcript(messages, self._transcript_chars)
+        if not transcript.strip():
+            return ExtractionResult()
+        candidates = self._ask(
+            _EXTRACT_SYSTEM,
+            f"Conversation:\n{transcript}\n\nExtract the durable facts.",
+        )
+        existing = self._notes.all()
+        result = ExtractionResult()
+        for cand in candidates:
+            text = str(cand.get("text", "")).strip()
+            if not text:
+                continue
+            if self._is_duplicate(text, existing):
+                result.skipped_duplicates += 1
+                continue
+            note = self._notes.add(text, tags=cand.get("tags") or [])
+            existing.append(note)  # dedup later candidates against it too
+            result.added.append(note)
+        return result
+
+    def consolidate(self) -> ConsolidationResult:
+        existing = self._notes.all()
+        if len(existing) < 2:
+            return ConsolidationResult(len(existing), len(existing), existing)
+        listing = "\n".join(
+            f"{n.note_id}. {n.text}"
+            + (f"  [tags: {', '.join(n.tags)}]" if n.tags else "")
+            for n in existing
+        )
+        cleaned = [
+            c for c in self._ask(_CONSOLIDATE_SYSTEM, f"Current memory notes:\n{listing}")
+            if str(c.get("text", "")).strip()
+        ]
+        if not cleaned:
+            return ConsolidationResult(len(existing), len(existing), existing)  # no-op
+        # Rebuild from the consolidated set using the store's public API so all
+        # backends (JSONL / SQLite / semantic) stay consistent.
+        for note in existing:
+            self._notes.forget(note_id=note.note_id)
+        new_notes = [
+            self._notes.add(str(c["text"]).strip(), tags=c.get("tags") or [])
+            for c in cleaned
+        ]
+        return ConsolidationResult(len(existing), len(new_notes), new_notes)
+
+    # --- internals ---------------------------------------------------------
+
+    def _ask(self, system: str, user: str) -> list[dict]:
+        resp = self._provider.complete(
+            [Message(role="user", content=user)],
+            [],
+            max_tokens=self._max_tokens,
+            temperature=0.0,
+            system=system,
+        )
+        return _parse_json_array(resp.message.content)
+
+    def _is_duplicate(self, text: str, existing: list[MemoryNote]) -> bool:
+        terms = _tokens(text)
+        if not terms:
+            return False
+        for note in existing:
+            other = _tokens(note.text)
+            if not other:
+                continue
+            jaccard = len(terms & other) / len(terms | other)
+            if jaccard >= self._dedup_threshold:
+                return True
+        return False

agentkernel/doctor.py

@@ -0,0 +1,141 @@
+"""Environment health check (design §18.7).
+
+`agentkernel doctor` runs a set of fast, network-free checks — Python version,
+required and optional dependencies, provider credentials, sandbox availability,
+and writable paths — and prints a checklist. It exits non-zero if any check
+fails, so it doubles as a setup smoke test. Checks are pure functions of the
+config + environment, so they're deterministic and offline-testable.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import shutil
+import sys
+from dataclasses import dataclass
+
+from agentkernel.config import Config
+
+# ASCII marks: plain print() to a Windows cp1252 console can't encode unicode glyphs.
+_MARK = {"ok": "[ OK ]", "warn": "[WARN]", "fail": "[FAIL]"}
+
+# provider -> the env var that holds its key (local needs no key).
+_PROVIDER_KEY_ENV = {"anthropic": "ANTHROPIC_API_KEY", "openai": "OPENAI_API_KEY"}
+
+
+@dataclass
+class Check:
+    name: str
+    status: str  # "ok" | "warn" | "fail"
+    detail: str = ""
+
+
+def _have_module(name: str) -> bool:
+    return importlib.util.find_spec(name) is not None
+
+
+def run_checks(config: Config, *, env: dict[str, str] | None = None) -> list[Check]:
+    """Return the list of health checks for ``config`` and ``env``."""
+    env = os.environ if env is None else env
+    checks: list[Check] = []
+
+    # Python version.
+    v = sys.version_info
+    checks.append(
+        Check(
+            "python",
+            "ok" if v >= (3, 11) else "fail",
+            f"{v.major}.{v.minor}.{v.micro}" + ("" if v >= (3, 11) else " (need >= 3.11)"),
+        )
+    )
+
+    # Required dependencies.
+    for dep in ("jsonschema", "httpx"):
+        checks.append(
+            Check(f"dependency: {dep}", "ok" if _have_module(dep) else "fail")
+        )
+
+    # Provider credentials.
+    provider = config.provider
+    if provider == "local":
+        checks.append(
+            Check(
+                "provider: local",
+                "ok" if config.base_url else "warn",
+                config.base_url or "no base_url set - local endpoint won't be reachable",
+            )
+        )
+    else:
+        key_env = _PROVIDER_KEY_ENV.get(provider)
+        if key_env is None:
+            checks.append(Check(f"provider: {provider}", "warn", "unknown provider"))
+        else:
+            present = bool(env.get(key_env))
+            checks.append(
+                Check(
+                    f"provider: {provider}",
+                    "ok" if present else "fail",
+                    f"{key_env} is set" if present else f"{key_env} is not set",
+                )
+            )
+
+    # Sandbox.
+    if config.sandbox == "docker":
+        have = shutil.which("docker") is not None
+        checks.append(
+            Check("sandbox: docker", "ok" if have else "fail",
+                  "docker CLI found" if have else "docker CLI not on PATH")
+        )
+    else:
+        checks.append(Check("sandbox: local", "ok"))
+
+    # Semantic search needs an embedding key.
+    if config.semantic_search:
+        present = bool(env.get(config.embedding_api_key_env))
+        checks.append(
+            Check(
+                "semantic search",
+                "ok" if present else "warn",
+                f"{config.embedding_api_key_env} "
+                + ("is set" if present else "is not set - recall will error"),
+            )
+        )
+
+    # TUI backend on Windows.
+    if sys.platform == "win32":
+        checks.append(
+            Check(
+                "tui: curses",
+                "ok" if _have_module("curses") else "warn",
+                "available" if _have_module("curses")
+                else "windows-curses not installed - `agentkernel tui` won't run",
+            )
+        )
+
+    # Log dir writable.
+    try:
+        from pathlib import Path
+
+        Path(config.log_dir).mkdir(parents=True, exist_ok=True)
+        checks.append(Check("log dir writable", "ok", config.log_dir))
+    except OSError as exc:
+        checks.append(Check("log dir writable", "fail", f"{config.log_dir}: {exc}"))
+
+    return checks
+
+
+def format_checks(checks: list[Check]) -> str:
+    lines = ["agentkernel doctor", ""]
+    for c in checks:
+        detail = f" - {c.detail}" if c.detail else ""
+        lines.append(f"  {_MARK[c.status]} {c.name}{detail}")
+    fails = sum(1 for c in checks if c.status == "fail")
+    warns = sum(1 for c in checks if c.status == "warn")
+    lines.append("")
+    lines.append(f"{len(checks)} checks: {fails} failed, {warns} warnings.")
+    return "\n".join(lines)
+
+
+def has_failures(checks: list[Check]) -> bool:
+    return any(c.status == "fail" for c in checks)

agentkernel/embeddings.py

@@ -0,0 +1,132 @@
+"""Embedding providers for semantic note search.
+
+This module intentionally stays outside of ``agentkernel.memory`` so the default
+SQLite/JSONL notebook does not depend on an embedding endpoint. The provider is
+only instantiated when ``Config.semantic_search`` is enabled.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import Protocol
+
+from agentkernel.config import Config
+
+
+class EmbeddingProvider(Protocol):
+    """A tiny embedding seam. Anything matching this can back semantic search."""
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Return one dense vector per non-empty text input."""
+        ...
+
+
+class EmbeddingError(RuntimeError):
+    """Raised when an embedding request cannot be completed."""
+
+
+@dataclass
+class OpenAIEmbeddingProvider:
+    """OpenAI-compatible embedding endpoint using only stdlib ``urllib``.
+
+    Works with OpenAI, any OpenAI-compatible local server, or cloud providers
+    that expose the ``/embeddings`` route. The API key is read from the
+    environment (``OPENAI_API_KEY`` by default) and never persisted.
+    """
+
+    model: str = "text-embedding-3-small"
+    base_url: str = "https://api.openai.com/v1"
+    dimensions: int | None = None
+    api_key_env: str = "OPENAI_API_KEY"
+    timeout: float = 60.0
+
+    @classmethod
+    def from_config(
+        cls, config: Config, *, api_key_env: str = "OPENAI_API_KEY"
+    ) -> OpenAIEmbeddingProvider:
+        """Convenience constructor fromConfig.
+
+        Infers a sensible endpoint from Config ``provider``/``base_url`` when
+        ``embedding_base_url`` is not set:
+        - provider "openai" → https://api.openai.com/v1
+        - provider "local"  → config.base_url (fallback to ollama-style default)
+        - provider "anthropic" or other → requires explicit ``embedding_base_url``.
+        """
+        base_url = config.embedding_base_url
+        if base_url is None:
+            if config.provider == "openai":
+                base_url = "https://api.openai.com/v1"
+            elif config.provider == "local":
+                base_url = config.base_url or "http://localhost:11434/v1"
+            else:
+                raise EmbeddingError(
+                    f"Cannot infer embedding endpoint for provider={config.provider!r}. "
+                    "Set `embedding_base_url` in agentkernel.toml."
+                )
+        return cls(
+            model=config.embedding_model,
+            base_url=base_url,
+            dimensions=config.embedding_dimensions,
+            api_key_env=api_key_env,
+        )
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Call the embeddings endpoint and return vectors in input order."""
+        texts = [t.strip() for t in texts]
+        if not texts:
+            return []
+        if all(not t for t in texts):
+            return [[] for _ in texts]
+        key = os.environ.get(self.api_key_env)
+        if not key:
+            raise EmbeddingError(f"Environment variable {self.api_key_env} is not set")
+        payload: dict[str, Any] = {"model": self.model, "input": texts}
+        if self.dimensions:
+            payload["dimensions"] = self.dimensions
+        url = self.base_url.rstrip("/") + "/embeddings"
+        req = urllib.request.Request(
+            url,
+            data=json.dumps(payload).encode("utf-8"),
+            headers={
+                "Authorization": f"Bearer {key}",
+                "Content-Type": "application/json",
+            },
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                result = json.loads(resp.read().decode("utf-8"))
+        except urllib.error.HTTPError as exc:
+            body = exc.read().decode(errors="ignore")
+            raise EmbeddingError(f"Embedding request failed ({exc.code}): {body}") from exc
+        except Exception as exc:
+            raise EmbeddingError(f"Embedding request failed: {exc}") from exc
+
+        embeddings: list[list[float]] = []
+        for item in sorted(result.get("data", []), key=lambda x: x.get("index", 0)):
+            embeddings.append(item.get("embedding", []))
+        if len(embeddings) != len(texts):
+            raise EmbeddingError(
+                f"Embedding response length mismatch: expected {len(texts)}, got {len(embeddings)}"
+            )
+        return embeddings
+
+
+def cosine_similarity(a: list[float], b: list[float]) -> float:
+    """Cosine similarity between two equal-length vectors (0.0 if invalid)."""
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b, strict=True))
+    norm_a = sum(x * x for x in a) ** 0.5
+    norm_b = sum(x * x for x in b) ** 0.5
+    if norm_a == 0.0 or norm_b == 0.0:
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
+# Forward import for type annotations that need a runtime reference to Any.
+from typing import Any  # noqa: E402

agentkernel/evaluation.py

@@ -0,0 +1,186 @@
+"""Evaluator / eval harness (design §13, Phase 5).
+
+"An evaluator is a profile whose final output is a structured score." This runs
+the agent on each case, then asks a judge model to score the answer against a
+rubric, producing a structured ``EvalResult``. A suite aggregates into pass-rate
+and mean score — useful for regression tests and model comparison, and as signal
+for the self-improvement loop.
+
+Built entirely on the kernel (an Agent + a provider); the loop is untouched.
+Judging is best-effort: an unparseable judge reply scores 0 rather than raising.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import tomllib
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from agentkernel.types import Message
+
+if TYPE_CHECKING:
+    from agentkernel.agent import Agent
+    from agentkernel.providers import Provider
+
+# A factory that returns a FRESH agent per case (independent context).
+AgentFactory = Callable[[], "Agent"]
+
+_JUDGE_SYSTEM = (
+    "You are a strict evaluator. Score how well an agent's answer satisfies the "
+    "rubric for the given task. Respond with ONLY a JSON object: "
+    '{"score": <0-100 integer>, "pass": <true|false>, "reasoning": "<one sentence>"}.'
+)
+_DEFAULT_RUBRIC = "The answer is correct, complete, and directly addresses the task."
+
+
+@dataclass
+class EvalCase:
+    name: str
+    prompt: str
+    rubric: str | None = None  # overrides the suite/default rubric
+
+
+@dataclass
+class EvalResult:
+    name: str
+    answer: str
+    score: float  # normalized 0.0–1.0
+    passed: bool
+    reasoning: str
+    raw_judge: str = ""
+
+
+@dataclass
+class EvalSummary:
+    results: list[EvalResult] = field(default_factory=list)
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for r in self.results if r.passed)
+
+    @property
+    def pass_rate(self) -> float:
+        return self.passed / self.total if self.total else 0.0
+
+    @property
+    def mean_score(self) -> float:
+        return sum(r.score for r in self.results) / self.total if self.total else 0.0
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "total": self.total,
+            "passed": self.passed,
+            "pass_rate": self.pass_rate,
+            "mean_score": self.mean_score,
+            "results": [
+                {
+                    "name": r.name,
+                    "answer": r.answer,
+                    "score": r.score,
+                    "passed": r.passed,
+                    "reasoning": r.reasoning,
+                    "raw_judge": r.raw_judge,
+                }
+                for r in self.results
+            ],
+        }
+
+
+def _parse_score(text: str, pass_threshold: float) -> tuple[float, bool, str]:
+    """Extract ``(score 0-1, passed, reasoning)`` from a judge reply.
+
+    Tolerant of prose around the JSON; if nothing parseable is found, the case
+    scores 0 and fails (a non-answer should never silently pass)."""
+    match = re.search(r"\{.*\}", text, re.DOTALL)
+    if not match:
+        return 0.0, False, "could not parse judge output"
+    try:
+        data = json.loads(match.group(0))
+    except json.JSONDecodeError:
+        return 0.0, False, "could not parse judge output"
+
+    raw = data.get("score", 0)
+    try:
+        score = float(raw)
+    except (TypeError, ValueError):
+        score = 0.0
+    if score > 1.0:  # judges return 0-100; normalize to 0-1
+        score = score / 100.0
+    score = max(0.0, min(1.0, score))
+
+    passed = data.get("pass")
+    if not isinstance(passed, bool):
+        passed = score >= pass_threshold
+    return score, passed, str(data.get("reasoning", ""))
+
+
+class Evaluator:
+    """Runs cases through an agent and scores answers with a judge provider."""
+
+    def __init__(
+        self,
+        agent_factory: AgentFactory,
+        judge: Provider,
+        *,
+        default_rubric: str = _DEFAULT_RUBRIC,
+        pass_threshold: float = 0.6,
+        judge_max_tokens: int = 512,
+    ) -> None:
+        self._agent_factory = agent_factory
+        self._judge = judge
+        self._default_rubric = default_rubric
+        self._pass_threshold = pass_threshold
+        self._judge_max_tokens = judge_max_tokens
+
+    def evaluate_case(self, case: EvalCase) -> EvalResult:
+        answer = self._agent_factory().run(case.prompt)
+        rubric = case.rubric or self._default_rubric
+        raw = self._score(case.prompt, rubric, answer)
+        score, passed, reasoning = _parse_score(raw, self._pass_threshold)
+        return EvalResult(case.name, answer, score, passed, reasoning, raw)
+
+    def run_suite(self, cases: list[EvalCase]) -> EvalSummary:
+        return EvalSummary([self.evaluate_case(c) for c in cases])
+
+    def _score(self, prompt: str, rubric: str, answer: str) -> str:
+        judge_prompt = (
+            f"Task:\n{prompt}\n\nRubric:\n{rubric}\n\nAgent answer:\n{answer}\n\n"
+            "Score the answer against the rubric."
+        )
+        resp = self._judge.complete(
+            [Message(role="user", content=judge_prompt)],
+            [],
+            max_tokens=self._judge_max_tokens,
+            temperature=0.0,
+            system=_JUDGE_SYSTEM,
+        )
+        return resp.message.content.strip()
+
+
+def load_eval_suite(path: str | Path) -> tuple[str, list[EvalCase]]:
+    """Load ``(default_rubric, cases)`` from a TOML suite file.
+
+    Format::
+
+        rubric = "default rubric for all cases"   # optional
+        [[cases]]
+        name = "..."
+        prompt = "..."
+        rubric = "..."                            # optional per-case override
+    """
+    with Path(path).open("rb") as fh:
+        data = tomllib.load(fh)
+    default_rubric = data.get("rubric", _DEFAULT_RUBRIC)
+    cases = [
+        EvalCase(name=c["name"], prompt=c["prompt"], rubric=c.get("rubric"))
+        for c in data.get("cases", [])
+    ]
+    return default_rubric, cases