forgeoptimizer 0.1.0__py3-none-any.whl

forgecli/graph/repository.py

@@ -0,0 +1,164 @@
+"""Abstract interface for a repository knowledge graph.
+
+This module is the boundary between ForgeCLI's graph-aware features and any
+concrete back-end (Graphify today; custom AST-based backends tomorrow). The
+interface is intentionally small: build, query, explain, path, affected.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class GraphNode:
+    """A typed view of a single node from the knowledge graph."""
+
+    id: str
+    label: str
+    file_type: str | None = None
+    source_file: str | None = None
+    source_location: str | None = None
+    community: int | None = None
+    norm_label: str | None = None
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class GraphEdge:
+    """A typed view of a directed/undirected edge."""
+
+    source: str
+    target: str
+    relation: str
+    confidence: str | None = None
+    confidence_score: float | None = None
+    source_file: str | None = None
+    source_location: str | None = None
+    weight: float = 1.0
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class GraphCommunity:
+    """A Leiden-detected community of nodes."""
+
+    id: int
+    size: int
+    label: str | None = None
+    members: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class GraphSnapshot:
+    """An immutable view of a fully-built knowledge graph."""
+
+    root: str
+    nodes: tuple[GraphNode, ...]
+    edges: tuple[GraphEdge, ...]
+    communities: tuple[GraphCommunity, ...] = ()
+    directed: bool = False
+    multigraph: bool = False
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def node(self, node_id: str) -> GraphNode | None:
+        """Return the node with ``node_id`` or ``None``."""
+        for node in self.nodes:
+            if node.id == node_id:
+                return node
+        return None
+
+    def neighbors(self, node_id: str) -> list[GraphEdge]:
+        """Return all edges incident to ``node_id`` (in or out)."""
+        return [e for e in self.edges if e.source == node_id or e.target == node_id]
+
+    def search(self, query: str, *, limit: int = 20) -> list[GraphNode]:
+        """Return nodes whose ``label`` or ``norm_label`` contain ``query``."""
+        needle = query.lower()
+        hits: list[GraphNode] = []
+        for node in self.nodes:
+            if needle in (node.label or "").lower() or needle in (node.norm_label or "").lower():
+                hits.append(node)
+                if len(hits) >= limit:
+                    break
+        return hits
+
+
+@dataclass(frozen=True)
+class BuildResult:
+    """The outcome of :meth:`RepositoryGraph.build`."""
+
+    snapshot: GraphSnapshot
+    artifacts: dict[str, str] = field(default_factory=dict)
+    raw_output: str = ""
+
+
+@dataclass(frozen=True)
+class QueryResult:
+    """A natural-language answer produced by :meth:`RepositoryGraph.query`."""
+
+    question: str
+    answer: str
+    cited_nodes: tuple[str, ...] = ()
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class ExplainResult:
+    """A plain-language explanation of a node and its neighbors."""
+
+    target: str
+    explanation: str
+    related: tuple[GraphNode, ...] = ()
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+class RepositoryGraph(ABC):
+    """Strategy interface for any repository knowledge graph backend."""
+
+    name: str = "abstract"
+
+    @abstractmethod
+    async def is_available(self) -> bool:
+        """Return True if the back-end is installed and runnable."""
+
+    @abstractmethod
+    async def build(self, *, force: bool = False) -> BuildResult:
+        """Build (or rebuild) the graph rooted at the configured path."""
+
+    @abstractmethod
+    async def load(self) -> GraphSnapshot:
+        """Load a previously-built graph from disk without rebuilding."""
+
+    @abstractmethod
+    async def query(self, question: str, *, budget: int = 2000) -> QueryResult:
+        """Ask a free-form question; the back-end traverses the graph."""
+
+    @abstractmethod
+    async def explain(self, target: str) -> ExplainResult:
+        """Return a plain-language explanation of ``target`` and its neighbors."""
+
+    @abstractmethod
+    async def shortest_path(self, a: str, b: str) -> list[GraphEdge]:
+        """Return the shortest edge path between two node labels/ids."""
+
+    @abstractmethod
+    async def affected(
+        self, target: str, *, relation: Iterable[str] | None = None, depth: int = 2
+    ) -> list[GraphEdge]:
+        """Return the reverse-traversal edges from ``target``."""
+
+
+__all__ = [
+    "BuildResult",
+    "ExplainResult",
+    "GraphCommunity",
+    "GraphEdge",
+    "GraphNode",
+    "GraphSnapshot",
+    "QueryResult",
+    "RepositoryGraph",
+]

forgecli/memory/__init__.py

@@ -0,0 +1,12 @@
+"""Local memory store and history (SQLite-backed)."""
+
+from forgecli.memory.cache import Cache
+from forgecli.memory.history import HistoryEntry, HistoryRepository
+from forgecli.memory.store import MemoryStore
+
+__all__ = [
+    "Cache",
+    "HistoryEntry",
+    "HistoryRepository",
+    "MemoryStore",
+]

forgecli/memory/cache.py

@@ -0,0 +1,61 @@
+"""Lightweight in-memory cache with optional TTL.
+
+A future iteration can swap this for SQLite-backed or Redis-backed caching
+without affecting call sites.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from threading import RLock
+from typing import Generic, TypeVar
+
+K = TypeVar("K")
+V = TypeVar("V")
+
+
+@dataclass
+class _Entry(Generic[V]):
+    value: V
+    expires_at: float | None  # epoch seconds, or None for "no expiry"
+
+
+class Cache(Generic[K, V]):
+    """Thread-safe key/value cache supporting optional per-entry TTL."""
+
+    def __init__(self, *, default_ttl: float | None = None) -> None:
+        self._store: dict[K, _Entry[V]] = {}
+        self._default_ttl = default_ttl
+        self._lock = RLock()
+
+    def get(self, key: K) -> V | None:
+        with self._lock:
+            entry = self._store.get(key)
+            if entry is None:
+                return None
+            if entry.expires_at is not None and entry.expires_at < time.time():
+                self._store.pop(key, None)
+                return None
+            return entry.value
+
+    def set(self, key: K, value: V, *, ttl: float | None = None) -> None:
+        effective_ttl = self._default_ttl if ttl is None else ttl
+        expires = time.time() + effective_ttl if effective_ttl else None
+        with self._lock:
+            self._store[key] = _Entry(value=value, expires_at=expires)
+
+    def delete(self, key: K) -> None:
+        with self._lock:
+            self._store.pop(key, None)
+
+    def clear(self) -> None:
+        with self._lock:
+            self._store.clear()
+
+    def __contains__(self, key: K) -> bool:
+        return self.get(key) is not None
+
+    def __len__(self) -> int:
+        with self._lock:
+            return len(self._store)

forgecli/memory/history.py

@@ -0,0 +1,136 @@
+"""Append-only history of CLI invocations and AI interactions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+from forgecli.memory.store import MemoryStore
+
+
+@dataclass(frozen=True)
+class HistoryEntry:
+    """A single persisted history record."""
+
+    id: int
+    timestamp: datetime
+    command: str
+    provider: str | None
+    model: str | None
+    prompt_tokens: int
+    completion_tokens: int
+    duration_ms: int
+    success: bool
+    error: str | None
+
+
+class HistoryRepository:
+    """Persistence of CLI/AI invocations keyed by time."""
+
+    def __init__(self, store: MemoryStore, *, history_limit: int = 1000) -> None:
+        self._store = store
+        self._limit = history_limit
+        self._ensure_schema()
+
+    def _ensure_schema(self) -> None:
+        self._store.execute(
+            """
+            CREATE TABLE IF NOT EXISTS history (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                ts TEXT NOT NULL,
+                command TEXT NOT NULL,
+                provider TEXT,
+                model TEXT,
+                prompt_tokens INTEGER NOT NULL DEFAULT 0,
+                completion_tokens INTEGER NOT NULL DEFAULT 0,
+                duration_ms INTEGER NOT NULL DEFAULT 0,
+                success INTEGER NOT NULL DEFAULT 1,
+                error TEXT
+            )
+            """
+        )
+        self._store.execute("CREATE INDEX IF NOT EXISTS idx_history_ts ON history(ts DESC)")
+
+    def record(
+        self,
+        *,
+        command: str,
+        provider: str | None = None,
+        model: str | None = None,
+        prompt_tokens: int = 0,
+        completion_tokens: int = 0,
+        duration_ms: int = 0,
+        success: bool = True,
+        error: str | None = None,
+    ) -> int:
+        """Insert a new history row and return its id."""
+        cursor = self._store.execute(
+            """
+            INSERT INTO history (
+                ts, command, provider, model,
+                prompt_tokens, completion_tokens,
+                duration_ms, success, error
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                datetime.now(UTC).isoformat(timespec="seconds"),
+                command,
+                provider,
+                model,
+                prompt_tokens,
+                completion_tokens,
+                duration_ms,
+                int(success),
+                error,
+            ),
+        )
+        self._trim()
+        return int(cursor.lastrowid or 0)
+
+    def list_recent(self, limit: int = 50) -> list[HistoryEntry]:
+        """Return the most recent history entries (newest first)."""
+        rows = self._store.execute(
+            """
+            SELECT id, ts, command, provider, model,
+                   prompt_tokens, completion_tokens,
+                   duration_ms, success, error
+            FROM history
+            ORDER BY id DESC
+            LIMIT ?
+            """,
+            (limit,),
+        ).fetchall()
+        return [self._to_entry(r) for r in rows]
+
+    def _trim(self) -> None:
+        """Drop oldest rows past the configured retention limit."""
+        row = self._store.execute("SELECT COUNT(*) AS n FROM history").fetchone()
+        total = int(row["n"]) if row else 0
+        if total <= self._limit:
+            return
+        excess = total - self._limit
+        self._store.execute(
+            """
+            DELETE FROM history
+            WHERE id IN (
+                SELECT id FROM history ORDER BY id ASC LIMIT ?
+            )
+            """,
+            (excess,),
+        )
+
+    @staticmethod
+    def _to_entry(row: Any) -> HistoryEntry:
+        return HistoryEntry(
+            id=int(row["id"]),
+            timestamp=datetime.fromisoformat(row["ts"]),
+            command=str(row["command"]),
+            provider=row["provider"],
+            model=row["model"],
+            prompt_tokens=int(row["prompt_tokens"] or 0),
+            completion_tokens=int(row["completion_tokens"] or 0),
+            duration_ms=int(row["duration_ms"] or 0),
+            success=bool(row["success"]),
+            error=row["error"],
+        )

forgecli/memory/store.py

@@ -0,0 +1,85 @@
+"""SQLite-backed local memory store (singleton lifecycle owned by AppContext)."""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from forgecli.utils.fs import ensure_dir
+
+
+class MemoryStore:
+    """Thin wrapper around a local SQLite database.
+
+    The store is intentionally minimal: it owns the connection and provides
+    a few named-table helpers. Higher-level repositories (e.g. history)
+    layer domain logic on top.
+    """
+
+    SCHEMA_VERSION = 1
+
+    def __init__(self, db_path: Path) -> None:
+        self._db_path = Path(db_path)
+        ensure_dir(self._db_path.parent)
+        self._conn: sqlite3.Connection | None = None
+
+    @property
+    def db_path(self) -> Path:
+        return self._db_path
+
+    def connect(self) -> None:
+        """Open the database and run schema migrations."""
+        if self._conn is not None:
+            return
+        self._conn = sqlite3.connect(
+            str(self._db_path),
+            check_same_thread=False,
+            isolation_level=None,  # autocommit; we use explicit transactions
+        )
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute("PRAGMA journal_mode = WAL")
+        self._conn.execute("PRAGMA foreign_keys = ON")
+        self._migrate()
+
+    def close(self) -> None:
+        """Close the underlying connection if open."""
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+    def __enter__(self) -> MemoryStore:
+        self.connect()
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+    @property
+    def conn(self) -> sqlite3.Connection:
+        if self._conn is None:
+            raise RuntimeError("MemoryStore.connect() must be called first")
+        return self._conn
+
+    def _migrate(self) -> None:
+        self.conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS schema_meta (
+                key TEXT PRIMARY KEY,
+                value TEXT NOT NULL
+            )
+            """
+        )
+        row = self.conn.execute("SELECT value FROM schema_meta WHERE key = 'version'").fetchone()
+        if row is None:
+            self.conn.execute(
+                "INSERT INTO schema_meta (key, value) VALUES ('version', ?)",
+                (str(self.SCHEMA_VERSION),),
+            )
+        # Future migrations can be chained here by inspecting ``row``.
+
+    def execute(self, sql: str, params: tuple[Any, ...] = ()) -> sqlite3.Cursor:
+        return self.conn.execute(sql, params)
+
+    def executemany(self, sql: str, params: list[tuple[Any, ...]]) -> sqlite3.Cursor:
+        return self.conn.executemany(sql, params)

forgecli/optimizer/__init__.py

@@ -0,0 +1,13 @@
+"""Context optimization: chunking, ranking, summarization, token optimization."""
+
+from forgecli.optimizer.chunker import Chunker
+from forgecli.optimizer.optimizer import ContextOptimizer
+from forgecli.optimizer.ranker import Ranker
+from forgecli.optimizer.summarizer import Summarizer
+
+__all__ = [
+    "Chunker",
+    "ContextOptimizer",
+    "Ranker",
+    "Summarizer",
+]

forgecli/optimizer/caveman/__init__.py

@@ -0,0 +1,169 @@
+"""Caveman token-optimizer integration.
+
+Caveman is a prompt-level optimization technique that reduces LLM output
+tokens by ~65% on average. It works by injecting a system prompt that
+tells the model to communicate in a compressed, token-efficient style —
+dropping filler words, pleasantries, and using fragments — while keeping
+all technical terms exact.
+
+Two implementations are shipped:
+
+* :class:`CavemanRulesetOptimizer` — a self-contained Python
+  implementation of the Caveman ruleset (lite / full / ultra / wenyan).
+  Always available, no external dependencies.
+* :class:`CavemanCLIOptimizer` — an optional adapter that shells out
+  to an external ``caveman`` binary if one is on ``PATH``.
+
+The :class:`CavemanCompositeOptimizer` picks between them at runtime.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+from enum import Enum
+
+from forgecli.optimizer.ponytail import OptimizedRequest
+from forgecli.providers.base import ChatMessage, ChatRequest
+
+
+class CavemanIntensity(str, Enum):
+    """How aggressively to compress LLM output.
+
+    * ``off``    - no rewriting; pass prompts through unchanged.
+    * ``lite``   - be concise; drop filler words and pleasantries.
+    * ``full``   - full caveman: fragments, [thing][action][reason] pattern.
+    * ``ultra``  - maximum compression; grunt-level communication.
+    * ``wenyan`` - Classical Chinese literary style (文言) for max density.
+    """
+
+    OFF = "off"
+    LITE = "lite"
+    FULL = "full"
+    ULTRA = "ultra"
+    WENYAN = "wenyan"
+
+    @classmethod
+    def parse(cls, value: str | CavemanIntensity | None) -> CavemanIntensity:
+        """Parse a string into an :class:`CavemanIntensity`, falling back to LITE."""
+        if value is None or value == "":
+            return cls.LITE
+        if isinstance(value, cls):
+            return value
+        try:
+            return cls(value.lower())
+        except ValueError as exc:
+            raise ValueError(
+                f"Unknown caveman intensity {value!r}; expected one of "
+                f"{', '.join(i.value for i in cls)}"
+            ) from exc
+
+
+class CavemanPromptOptimizer(ABC):
+    """Strategy interface for Caveman prompt optimization.
+
+    Implementations must be deterministic and side-effect free
+    (other than logging), so they can be invoked transparently before
+    every model call.
+    """
+
+    name: str = "abstract-caveman"
+
+    @abstractmethod
+    async def optimize_chat(self, request: ChatRequest) -> OptimizedRequest:
+        """Return an optimized copy of ``request``."""
+
+    async def is_available(self) -> bool:
+        """Return whether this optimizer is available to use."""
+        return True
+
+
+class CavemanCompositeOptimizer(CavemanPromptOptimizer):
+    """Pick the right optimizer for the configured :class:`CavemanIntensity`."""
+
+    name = "caveman-composite"
+
+    def __init__(
+        self,
+        *,
+        intensity: CavemanIntensity = CavemanIntensity.LITE,
+        ruleset: CavemanPromptOptimizer | None = None,
+        external: CavemanPromptOptimizer | None = None,
+    ) -> None:
+        self._intensity = intensity
+        self._ruleset = ruleset
+        self._external = external
+        self._sync_ruleset_intensity()
+
+    def _sync_ruleset_intensity(self) -> None:
+        """Propagate the composite's intensity to the ruleset, if compatible."""
+        ruleset = self._ruleset
+        if isinstance(ruleset, CavemanRulesetOptimizer):
+            ruleset.set_intensity(self._intensity)
+
+    @property
+    def intensity(self) -> CavemanIntensity:
+        return self._intensity
+
+    def set_intensity(self, intensity: CavemanIntensity | str) -> None:
+        self._intensity = CavemanIntensity.parse(intensity)
+        if isinstance(self._ruleset, CavemanRulesetOptimizer):
+            self._ruleset.set_intensity(self._intensity)
+
+    async def optimize_chat(self, request: ChatRequest) -> OptimizedRequest:
+        if self._intensity is CavemanIntensity.OFF:
+            return OptimizedRequest(
+                request=request,
+                notes=("caveman off",),
+                intensity=self._intensity,
+                source="caveman-passthrough",
+            )
+
+        if self._external is not None and await self._external.is_available():
+            result = await self._external.optimize_chat(request)
+            return OptimizedRequest(
+                request=result.request,
+                notes=(*result.notes, "caveman external"),
+                intensity=self._intensity,
+                source="caveman-external",
+            )
+
+        if self._ruleset is None:
+            return OptimizedRequest(
+                request=request,
+                notes=("no caveman ruleset registered",),
+                intensity=self._intensity,
+                source="caveman-passthrough",
+            )
+
+        result = await self._ruleset.optimize_chat(request)
+        return OptimizedRequest(
+            request=result.request,
+            notes=result.notes,
+            intensity=self._intensity,
+            source="caveman-ruleset",
+        )
+
+
+def _ensure_user_message(messages: Sequence[ChatMessage]) -> bool:
+    return any(m.role.value == "user" for m in messages)
+
+
+def _clone_request(request: ChatRequest, messages: Sequence[ChatMessage]) -> ChatRequest:
+    """Return a copy of ``request`` with ``messages`` replaced."""
+    return request.model_copy(update={"messages": list(messages)})
+
+
+__all__ = [
+    "CavemanCLIOptimizer",
+    "CavemanCompositeOptimizer",
+    "CavemanIntensity",
+    "CavemanPromptOptimizer",
+    "CavemanProvider",
+    "CavemanRulesetOptimizer",
+    "OptimizedRequest",
+]
+
+from forgecli.optimizer.caveman.cli import CavemanCLIOptimizer  # noqa: E402
+from forgecli.optimizer.caveman.decorator import CavemanProvider  # noqa: E402
+from forgecli.optimizer.caveman.ruleset import CavemanRulesetOptimizer  # noqa: E402