ags-cli 0.1.0__py3-none-any.whl

harness/mcp/tools.py

@@ -0,0 +1,81 @@
+"""Pure helpers for turning MCP server tools into the harness's tool protocol.
+
+No I/O and no MCP SDK import here — this is the testable core shared by both
+integration paths:
+
+* the OpenAI-compatible adapter, which needs OpenAI ``tools`` specs and a way to
+  route a tool call back to the owning server;
+* the CLI adapters, which reuse the namespacing convention in their generated
+  native MCP config and ``--allowedTools`` lists.
+
+Tools are namespaced ``mcp__<server>__<tool>`` — the same scheme the Claude CLI
+uses — so a tool call reads identically across every adapter and in the ledger.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+PREFIX = "mcp"
+SEP = "__"
+
+
+def namespaced_name(server: str, tool: str) -> str:
+    """``("github", "create_issue") -> "mcp__github__create_issue"``."""
+    return f"{PREFIX}{SEP}{server}{SEP}{tool}"
+
+
+def parse_namespaced(name: str) -> tuple[str, str] | None:
+    """Inverse of :func:`namespaced_name`. Returns ``(server, tool)`` or ``None`` if
+    ``name`` is not an MCP-namespaced tool. The tool segment may itself contain the
+    separator, so we split off only the prefix and server."""
+    if not name.startswith(f"{PREFIX}{SEP}"):
+        return None
+    rest = name[len(PREFIX) + len(SEP):]
+    server, sep, tool = rest.partition(SEP)
+    if not sep or not server or not tool:
+        return None
+    return server, tool
+
+
+def is_mcp_tool(name: str) -> bool:
+    return parse_namespaced(name) is not None
+
+
+@dataclass(frozen=True)
+class McpTool:
+    """A tool advertised by an MCP server (transport-agnostic projection)."""
+
+    server: str
+    name: str
+    description: str = ""
+    input_schema: dict | None = None
+
+    def openai_spec(self) -> dict:
+        """OpenAI ``tools`` function spec with a namespaced name."""
+        schema = self.input_schema or {"type": "object", "properties": {}}
+        return {
+            "type": "function",
+            "function": {
+                "name": namespaced_name(self.server, self.name),
+                "description": self.description or f"{self.name} (via MCP server {self.server})",
+                "parameters": schema,
+            },
+        }
+
+
+def allowed(tool_name: str, allowlist: list[str] | None) -> bool:
+    """A tool is allowed if the allowlist is empty/absent or names it (bare name)."""
+    return not allowlist or tool_name in allowlist
+
+
+def to_openai_specs(
+    tools: list[McpTool], allowlist_by_server: dict[str, list[str]] | None = None
+) -> list[dict]:
+    """Convert tools to OpenAI specs, applying each server's allowlist."""
+    allowlist_by_server = allowlist_by_server or {}
+    specs: list[dict] = []
+    for t in tools:
+        if allowed(t.name, allowlist_by_server.get(t.server)):
+            specs.append(t.openai_spec())
+    return specs

harness/memory/__init__.py

@@ -0,0 +1,13 @@
+from harness.memory.extract import extract_session_memory
+from harness.memory.ingest import ingest_memory
+from harness.memory.service import MemoryService, RetrievalResult, RetrievedItem
+from harness.memory.summarize import summarize_session
+
+__all__ = [
+    "MemoryService",
+    "RetrievalResult",
+    "RetrievedItem",
+    "extract_session_memory",
+    "ingest_memory",
+    "summarize_session",
+]

harness/memory/context_budget.py

@@ -0,0 +1,116 @@
+"""Per-model context-window budgeting for the injected conversation context.
+
+Adapters are stateless, so the orchestrator re-injects the session history into every
+turn's prompt. This module trims that history to the **active model's** context window —
+keeping recent turns verbatim and falling back to the session summary for older ones —
+so small-window models don't overflow. Large (1M) models effectively never trim.
+
+Token counts are estimated by a char heuristic (no tokenizer dependency); a reserve is
+kept for the model's response, and the estimate intentionally over-counts so we trim a
+little early rather than overflow.
+"""
+
+from __future__ import annotations
+
+import math
+
+from harness.models.enums import AdapterKind
+
+# Known model context windows (tokens), keyed by a normalized substring of the model
+# name — first match wins, so list more specific keys first. Override per provider with
+# ``params.max_context``.
+MODEL_CONTEXT_WINDOWS: dict[str, int] = {
+    "claude-opus-4-8": 1_000_000,
+    "claude-sonnet-4-6": 1_000_000,
+    "claude-sonnet-4": 1_000_000,
+    "claude-fable-5": 1_000_000,
+    "claude-haiku": 200_000,
+    "gemini": 1_000_000,        # 3-pro / 3.1-pro / 3.5-flash all 1M
+    "gemma": 131_072,
+    "gpt-4o": 128_000,
+    "gpt-4.1": 1_000_000,
+    "llama": 131_072,
+    "qwen": 131_072,
+    "mock": 8_192,
+}
+
+_KIND_DEFAULT: dict[AdapterKind, int] = {
+    AdapterKind.claude_code: 200_000,
+    AdapterKind.antigravity: 1_000_000,
+    AdapterKind.openai_local: 8_192,
+    AdapterKind.mock: 8_192,
+}
+
+_CHARS_PER_TOKEN = 3.5  # conservative: over-counts tokens so we trim early, never overflow
+
+_HISTORY_HEADER = "## Session History"
+_SUMMARY_HEADER = "## Earlier in this session (summary)"
+_SEP = "\n\n"
+
+
+def estimate_tokens(text: str) -> int:
+    return math.ceil(len(text) / _CHARS_PER_TOKEN) if text else 0
+
+
+def context_window(kind: AdapterKind, model: str | None, params: dict | None = None) -> int:
+    """Resolve a provider's active-model context window (tokens):
+    explicit ``params.max_context`` → per-model registry → per-kind default."""
+    params = params or {}
+    mc = params.get("max_context")
+    if mc is not None:
+        try:
+            return int(mc)
+        except (TypeError, ValueError):
+            pass
+    name = (model or "").lower()
+    for key, win in MODEL_CONTEXT_WINDOWS.items():
+        if key in name:
+            return win
+    return _KIND_DEFAULT.get(kind, 8_192)
+
+
+def _join_history(blocks: list[str], summary: str | None) -> str:
+    body = _SEP.join(blocks)
+    if summary:
+        if body:
+            return f"{_SUMMARY_HEADER}\n{summary}{_SEP}{_HISTORY_HEADER}\n{body}"
+        return f"{_SUMMARY_HEADER}\n{summary}"
+    return f"{_HISTORY_HEADER}\n{body}" if body else ""
+
+
+def build_memory_context(
+    semantic_block: str,
+    transcript_blocks: list[str],
+    summary: str | None,
+    *,
+    window: int,
+    message_tokens: int = 0,
+    reserve_tokens: int | None = None,
+) -> str:
+    """Assemble the injected context (semantic memory + session history), trimmed to
+    ``window``. Keeps the newest transcript blocks verbatim; if older blocks must be
+    dropped, prepends the session ``summary``. When everything fits, the full transcript
+    is returned UNCHANGED — so 1M-window models never trim."""
+    reserve = reserve_tokens if reserve_tokens is not None else min(window // 5, 32_000)
+    budget = window - reserve - message_tokens - estimate_tokens(semantic_block)
+
+    full = _join_history(transcript_blocks, None)
+    if not transcript_blocks or budget <= 0 or estimate_tokens(full) <= budget:
+        out = full  # fits (or can't meaningfully budget) → no trim
+    else:
+        overhead = estimate_tokens(f"{_SUMMARY_HEADER}\n{summary}") if summary else 0
+        overhead += estimate_tokens(f"\n{_HISTORY_HEADER}\n")
+        kept: list[str] = []
+        used = overhead
+        for block in reversed(transcript_blocks):
+            t = estimate_tokens(block + _SEP)
+            if used + t > budget and kept:
+                break
+            kept.insert(0, block)
+            used += t
+        dropped = len(kept) < len(transcript_blocks)
+        out = _join_history(kept, summary if (dropped and summary) else None)
+
+    if not out:
+        return semantic_block
+    return f"{semantic_block}{_SEP}{out}".strip() if semantic_block else out

harness/memory/embed.py

@@ -0,0 +1,217 @@
+"""Pluggable embedding providers.
+
+- ``HashEmbedder``: deterministic, offline, dependency-free. Default for dev/CI so
+  the whole memory pipeline works without a model server. Not semantically strong,
+  but stable and fast — good enough to validate retrieval plumbing.
+- ``OpenAICompatibleEmbedder``: any ``/v1/embeddings`` endpoint (Ollama, LM Studio,
+  vLLM, LocalAI). Used in real deployments.
+- ``FastEmbedEmbedder``: offline, local-model embedder via ``fastembed``.
+  Install the optional extra to enable: ``pip install 'ags-cli[embed]'``.
+
+Resolution for ``embedding_provider = "auto"`` (the default):
+  1. ``fastembed`` importable → ``FastEmbedEmbedder`` (BAAI/bge-small-en-v1.5, dim 384)
+  2. A provider config named ``local`` with an enabled entry + ``base_url``
+     → ``OpenAICompatibleEmbedder`` with ``base_url`` stripped of any trailing
+     slash; ``.embed()`` appends ``/embeddings`` at call time, so the final
+     request URL is ``{base_url}/embeddings``
+  3. ``HashEmbedder`` + one log warning about the degraded path.
+
+Explicit values (``hash`` / ``openai_compatible`` / ``fastembed``) bypass resolution.
+``fastembed`` without the package → ``RuntimeError``.
+
+The effective dimension comes from the chosen embedder's ``.dim`` attribute;
+callers should not rely on ``settings.embedding_dim`` when using auto-resolution.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import importlib.util
+import logging
+import math
+from typing import Protocol
+
+import httpx
+
+from harness.settings import Settings, get_settings
+
+log = logging.getLogger(__name__)
+
+
+class Embedder(Protocol):
+    model: str
+    async def embed(self, texts: list[str]) -> list[list[float]]: ...
+
+
+class HashEmbedder:
+    """Deterministic hashing embedder: feature-hash tokens into ``dim`` buckets,
+    then L2-normalize. Cosine similarity is meaningful for lexical overlap."""
+
+    def __init__(self, dim: int) -> None:
+        self.dim = dim
+        self.model = f"hash-{dim}"
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        return [self._embed_one(t) for t in texts]
+
+    def _embed_one(self, text: str) -> list[float]:
+        vec = [0.0] * self.dim
+        for token in text.lower().split():
+            digest = hashlib.sha1(token.encode()).digest()
+            idx = int.from_bytes(digest[:4], "big") % self.dim
+            sign = 1.0 if digest[4] & 1 else -1.0
+            vec[idx] += sign
+        norm = math.sqrt(sum(v * v for v in vec)) or 1.0
+        return [v / norm for v in vec]
+
+
+class OpenAICompatibleEmbedder:
+    def __init__(self, base_url: str, api_key: str, model: str, dim: int) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.dim = dim
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        # Fast-fail timeout: protects callers from slow-network hangs on unreachable endpoints.
+        async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=5.0)) as client:
+            resp = await client.post(
+                f"{self.base_url}/embeddings",
+                headers=headers,
+                json={"model": self.model, "input": texts},
+            )
+            resp.raise_for_status()
+            data = resp.json()["data"]
+        return [item["embedding"] for item in data]
+
+
+class FastEmbedEmbedder:
+    """Local offline embedder backed by ``fastembed``.
+
+    The ``fastembed`` package is an optional dependency — install with
+    ``pip install 'ags-cli[embed]'`` or ``uv tool install 'ags-cli[embed]'``.
+    The import is deferred to ``__init__`` so the class can be defined and
+    imported without the package installed (used for branch selection in
+    ``build_embedder``).
+    """
+
+    MODEL = "BAAI/bge-small-en-v1.5"
+    DIM = 384
+
+    def __init__(self, model: str = MODEL) -> None:
+        from fastembed import TextEmbedding  # lazy import — package optional
+        self._te = TextEmbedding(model_name=model)
+        self.model = model
+        self.dim = self.DIM
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        """Embed ``texts`` using the fastembed model (runs sync model in a thread)."""
+        def _sync_embed() -> list[list[float]]:
+            return [v.tolist() for v in self._te.embed(texts)]
+
+        return await asyncio.to_thread(_sync_embed)
+
+
+# ---------------------------------------------------------------------------
+# Factory
+# ---------------------------------------------------------------------------
+
+def build_embedder(
+    settings: Settings,
+    provider_configs: dict | None = None,
+) -> Embedder:
+    """Build the embedder dictated by ``settings.embedding_provider``.
+
+    Parameters
+    ----------
+    settings:
+        Application settings. ``embedding_provider`` controls selection.
+    provider_configs:
+        Optional dict of provider config dicts keyed by provider name.
+        Only consulted when ``embedding_provider == "auto"`` for the
+        ``local`` provider fallback. Expected shape::
+
+            {
+                "local": {
+                    "name": "local",
+                    "enabled": True,
+                    "base_url": "http://localhost:11434/v1",
+                },
+            }
+    """
+    provider = settings.embedding_provider
+
+    # --- Explicit "hash" --------------------------------------------------
+    if provider == "hash":
+        return HashEmbedder(settings.embedding_dim)
+
+    # --- Explicit "openai_compatible" -------------------------------------
+    if provider == "openai_compatible":
+        return OpenAICompatibleEmbedder(
+            base_url=settings.embedding_base_url,
+            api_key=settings.embedding_api_key,
+            model=settings.embedding_model,
+            dim=settings.embedding_dim,
+        )
+
+    # --- Explicit "fastembed" ---------------------------------------------
+    if provider == "fastembed":
+        if importlib.util.find_spec("fastembed") is None:
+            raise RuntimeError(
+                "embedding_provider='fastembed' but the fastembed package is not "
+                "installed. Install it with: pip install 'ags-cli[embed]'"
+            )
+        # Explicit user choice — they own any migration; honour it.
+        return FastEmbedEmbedder()
+
+    # --- "auto" resolution ------------------------------------------------
+    # (a) fastembed importable → prefer it.
+    if importlib.util.find_spec("fastembed") is not None:
+        return FastEmbedEmbedder()
+
+    # (b) enabled local provider with a base_url
+    if provider_configs:
+        local_cfg = provider_configs.get("local")
+        if (
+            local_cfg
+            and local_cfg.get("enabled", False)
+            and local_cfg.get("base_url")
+        ):
+            base_url = local_cfg["base_url"]
+            return OpenAICompatibleEmbedder(
+                base_url=base_url.rstrip("/"),
+                api_key=local_cfg.get("api_key", ""),
+                model=settings.embedding_model,
+                dim=settings.embedding_dim,
+            )
+
+    # (c) last resort: hash embedder with a degradation warning
+    log.warning(
+        "semantic memory degraded: hash embedder in use — pip install 'ags-cli[embed]'"
+    )
+    return HashEmbedder(settings.embedding_dim)
+
+
+def get_embedder(settings: Settings | None = None) -> Embedder:
+    """Backwards-compatible accessor.
+
+    Resolves ``provider_configs`` from the settings config() so that the
+    ``local`` provider's ``base_url`` is available for auto-resolution.
+    Callers that need explicit provider_configs should call ``build_embedder``
+    directly.
+    """
+    settings = settings or get_settings()
+    # Extract a provider_configs dict from the loaded config (if any).
+    provider_configs: dict | None = None
+    try:
+        cfg = settings.config()
+        providers = cfg.get("providers") or []
+        if providers:
+            provider_configs = {p["name"]: p for p in providers if p.get("name")}
+    except Exception:
+        pass  # config file absent in tests / minimal environments
+    return build_embedder(settings, provider_configs=provider_configs)

harness/memory/extract.py

@@ -0,0 +1,74 @@
+"""Memory extraction pipeline.
+
+After a session completes, distill its runs into durable, provider-independent
+memory: each succeeded run's output becomes an **episodic** memory item (what was
+produced, by whom, with outcome), and the canonical session summary becomes a
+**summary** item. Secrets are redacted by the ingest layer before storage/embed;
+dedup keeps re-runs idempotent. This is what lets a later session — on any
+provider — retrieve "what we already concluded" via the shared memory seam.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from harness.memory.ingest import ingest_memory
+from harness.memory.namespaces import provider_ns, workspace_ns
+from harness.models.enums import MemoryScope, MemoryType, RunStatus
+from harness.models.run import Run
+from harness.models.session import Session
+from harness.models.workspace import Workspace
+from harness.observability.logging import get_logger
+
+log = get_logger("memory-extract")
+
+
+async def extract_session_memory(db: AsyncSession, session_id: uuid.UUID) -> int:
+    """Ingest episodic + summary memory for a completed session. Returns the number
+    of memory items written (existing duplicates are not double-counted)."""
+    sess = (
+        await db.execute(select(Session).where(Session.id == session_id))
+    ).scalar_one()
+    workspace = (
+        await db.execute(select(Workspace).where(Workspace.id == sess.workspace_id))
+    ).scalar_one()
+    runs = (
+        await db.execute(select(Run).where(Run.session_id == session_id))
+    ).scalars().all()
+
+    written = 0
+    objective_tag = sess.canonical_objective[:40]
+
+    for run in runs:
+        if run.status != RunStatus.succeeded or not run.final_output:
+            continue
+        item = await ingest_memory(
+            db,
+            namespace=provider_ns(workspace.slug, run.adapter_kind.value),
+            scope=MemoryScope.workspace,
+            mem_type=MemoryType.episodic,
+            content=(
+                f"[{run.adapter_kind.value}/{run.role.value}] for objective "
+                f"'{objective_tag}':\n{run.final_output}"
+            ),
+            tags=["episodic", run.adapter_kind.value, objective_tag],
+            source_run_id=run.id,
+        )
+        written += 1 if item is not None else 0
+
+    if sess.summary:
+        await ingest_memory(
+            db,
+            namespace=workspace_ns(workspace.slug),
+            scope=MemoryScope.workspace,
+            mem_type=MemoryType.summary,
+            content=sess.summary,
+            tags=["session_summary", objective_tag],
+        )
+        written += 1
+
+    log.info("memory_extracted", session_id=str(session_id), items=written)
+    return written

harness/memory/ingest.py

@@ -0,0 +1,106 @@
+"""Memory ingestion pipeline: redact → dedup → store item → chunk → embed.
+
+Idempotent on ``(namespace, dedup_hash)`` — re-ingesting identical content is a
+no-op. Secrets are redacted before anything is stored or embedded."""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from harness.memory.embed import Embedder, get_embedder
+from harness.memory.namespaces import ttl_for
+from harness.memory.redact import redact
+from harness.memory.sqlite_vec_backend import SqliteVecIndex
+from harness.models.enums import MemoryScope, MemoryType
+from harness.models.memory import MemoryEmbedding, MemoryItem
+
+
+def _dedup_hash(namespace: str, content: str) -> str:
+    return hashlib.sha256(f"{namespace}\x00{content}".encode()).hexdigest()
+
+
+def _chunk(text: str, *, size: int = 800, overlap: int = 100) -> list[str]:
+    if len(text) <= size:
+        return [text]
+    chunks, start = [], 0
+    while start < len(text):
+        chunks.append(text[start : start + size])
+        start += size - overlap
+    return chunks
+
+
+async def ingest_memory(
+    session: AsyncSession,
+    *,
+    namespace: str,
+    scope: MemoryScope,
+    mem_type: MemoryType,
+    content: str,
+    tags: list[str] | None = None,
+    source_run_id: uuid.UUID | None = None,
+    embedder: Embedder | None = None,
+    ttl_hours_session: int | None = 72,
+    ttl_hours_workspace: int | None = None,
+) -> MemoryItem:
+    """Ingest one memory item. Returns the existing item if a duplicate."""
+    redacted_content, was_redacted = redact(content)
+    dedup = _dedup_hash(namespace, redacted_content)
+
+    existing = (
+        await session.execute(
+            select(MemoryItem).where(
+                MemoryItem.namespace == namespace, MemoryItem.dedup_hash == dedup
+            )
+        )
+    ).scalar_one_or_none()
+    if existing is not None:
+        return existing
+
+    # Embed BEFORE touching the DB — network I/O must never run inside the
+    # SQLite write transaction (holds the write lock and starves concurrent writers).
+    embedder = embedder or get_embedder()
+    chunks = _chunk(redacted_content)
+    vectors = await embedder.embed(chunks)
+
+    item = MemoryItem(
+        namespace=namespace,
+        scope=scope,
+        type=mem_type,
+        content=redacted_content,
+        tags=tags or [],
+        source_run_id=source_run_id,
+        dedup_hash=dedup,
+        redacted=was_redacted,
+        ttl_expires_at=ttl_for(
+            scope,
+            session_volatile_hours=ttl_hours_session,
+            workspace_durable_hours=ttl_hours_workspace,
+        ),
+    )
+    session.add(item)
+    await session.flush()  # assign item.id
+
+    vec_loaded = await SqliteVecIndex.loaded_for_session(session)
+    if vec_loaded:
+        await SqliteVecIndex.ensure_table(session, embedder.dim)
+
+    for idx, (chunk, vec) in enumerate(zip(chunks, vectors, strict=True)):
+        session.add(
+            MemoryEmbedding(
+                memory_item_id=item.id,
+                chunk_idx=idx,
+                model=embedder.model,
+                embedding=vec,
+                meta={"chars": len(chunk)},
+            )
+        )
+        if vec_loaded:
+            # Mirror into the vec0 ANN table; item.id is a UUID, stored as str.
+            await SqliteVecIndex.upsert(session, str(item.id), idx, vec)
+
+    await session.flush()
+    return item

harness/memory/namespaces.py

@@ -0,0 +1,57 @@
+"""Memory namespaces and TTL policy.
+
+A namespace is a hierarchical key that enforces isolation and enables targeted
+retrieval. Format:
+
+    ws/<workspace_slug>[/project/<name>][/provider/<name>][/session/<id>]
+
+Retrieval is namespace-bounded by default — no implicit cross-project leakage.
+Workspace-scoped durable memory uses the ``ws/<slug>`` prefix; session-scoped
+volatile memory appends ``/session/<id>`` and carries a TTL.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from harness.models.enums import MemoryScope
+
+
+def workspace_ns(slug: str, *, project: str | None = None) -> str:
+    ns = f"ws/{slug}"
+    if project:
+        ns += f"/project/{project}"
+    return ns
+
+
+def session_ns(slug: str, session_id: str, *, project: str | None = None) -> str:
+    return f"{workspace_ns(slug, project=project)}/session/{session_id}"
+
+
+def provider_ns(slug: str, provider: str, *, project: str | None = None) -> str:
+    return f"{workspace_ns(slug, project=project)}/provider/{provider}"
+
+
+def namespace_prefixes_for_retrieval(
+    slug: str, *, session_id: str | None = None, project: str | None = None
+) -> list[str]:
+    """Namespaces a retrieval should search: the session (if any) plus the
+    workspace, most specific first. Never crosses into another workspace."""
+    prefixes = [workspace_ns(slug, project=project)]
+    if session_id:
+        prefixes.insert(0, session_ns(slug, session_id, project=project))
+    return prefixes
+
+
+def ttl_for(
+    scope: MemoryScope,
+    *,
+    session_volatile_hours: int | None = 72,
+    workspace_durable_hours: int | None = None,
+) -> datetime | None:
+    """Compute an absolute expiry from policy. Workspace memory is durable
+    (no expiry) by default; session memory is volatile."""
+    hours = session_volatile_hours if scope == MemoryScope.session else workspace_durable_hours
+    if hours is None:
+        return None
+    return datetime.now(UTC) + timedelta(hours=hours)