forgeoptimizer 0.1.0__py3-none-any.whl

forgecli/build/llm.py

@@ -0,0 +1,167 @@
+"""Stage 3 — LLM call.
+
+Graphify retrieval as context) and dispatches it through the active
+provider. The selected provider is the one chosen by the router (see
+:mod:`forgecli.providers.router`).
+
+The stage supports a small retry loop: when ``context.extras`` carries
+a positive ``retries`` count, transient ``ProviderError`` exceptions
+are retried with a short backoff. Non-transient errors (e.g. the
+provider returns a 4xx) are surfaced immediately.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from forgecli.build import BuildContext
+from forgecli.core.errors import ProviderError
+from forgecli.providers.base import (
+    ChatMessage,
+    ChatRequest,
+    Provider,
+    Role,
+)
+
+_SYSTEM_PROMPT = (
+    "  1. Speculative need = skip it ().\n"
+    "  2. Reuse the helper/pattern that already lives in the codebase.\n"
+    "  3. Use the standard library.\n"
+    "  4. Native platform feature > third-party library.\n"
+    "  5. Use an already-installed dependency before adding a new one.\n"
+    "  6. One line beats many.\n"
+    "  7. Only then: the minimum code that works.\n\n"
+    "Return ONLY a unified diff in the response. No prose, no explanation, "
+    "no code fences. The diff must apply with `git apply`.\n"
+)
+
+_TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
+_BACKOFF_SECONDS = 1.5
+_log = logging.getLogger("forgecli.build.llm")
+
+
+async def llm_call(context: BuildContext) -> BuildContext:
+    """Send the assembled prompt to the active provider and store the response."""
+    provider: Provider | None = context.extras.get("provider")
+    if provider is None:
+        raise RuntimeError("no provider wired into the build context")
+
+    intent = context.extras.get("intent")
+
+    retries = int(context.extras.get("retries", 0) or 0)
+    user_content = _format_user_prompt(context)
+    base_request: ChatRequest = context.optimized_request or ChatRequest(
+        messages=[ChatMessage(role=Role.USER, content=context.prompt)]
+    )
+    request = base_request.model_copy(
+        update={
+            "model": context.decision.model if context.decision else None,
+            "messages": _assemble_messages(base_request.messages, user_content, intent),
+        }
+    )
+
+    last_exc: Exception | None = None
+    for attempt in range(retries + 1):
+        try:
+            response = await provider.chat(request)
+            context.response = response
+            return context
+        except ProviderError as exc:
+            last_exc = exc
+            if not _is_transient(exc) or attempt >= retries:
+                raise
+            wait = _BACKOFF_SECONDS * (attempt + 1)
+            _log.warning(
+                "transient provider error (attempt %d/%d): %s; retrying in %.1fs",
+                attempt + 1,
+                retries + 1,
+                exc,
+                wait,
+            )
+            await asyncio.sleep(wait)
+    # Unreachable, but keep mypy happy.
+    raise last_exc if last_exc else RuntimeError("llm_call: no attempts made")
+
+
+def _is_transient(exc: ProviderError) -> bool:
+    """Return True if ``exc`` looks like a transient HTTP failure."""
+    message = str(exc)
+    for code in _TRANSIENT_HTTP_CODES:
+        if f"({code})" in message:
+            return True
+    return False
+
+
+def _format_user_prompt(context: BuildContext) -> str:
+    parts: list[str] = [context.prompt]
+    if context.retrieval:
+        parts.append("\n" + context.retrieval)
+    intent = context.extras.get("intent")
+    from forgecli.plugins import Intent
+
+    if intent in (None, Intent.BUILD, Intent.UNKNOWN, "build"):
+        parts.append(
+            "\nGenerate code that exactly matches the request above: use the "
+            "language, filename(s), framework, approximate size, and behavior "
+            "the user asked for. "
+            "Respond with a unified diff only that applies with `git apply`. "
+            "Do not include prose, explanations, or markdown fences."
+        )
+    return "\n".join(parts)
+
+
+def _assemble_messages(
+    base: list[ChatMessage], user_content: str, intent: Any
+) -> list[ChatMessage]:
+    """Insert the system prompt at the head, replace the user message.
+
+    instructions on top by *replacing* the first user message with our
+    assembled content.
+    """
+    out: list[ChatMessage] = []
+    replaced_user = False
+    has_system = any(m.role is Role.SYSTEM for m in base)
+
+    if not has_system:
+        if intent is None or intent == "build":
+            out.append(ChatMessage(role=Role.SYSTEM, content=_SYSTEM_PROMPT))
+        else:
+            out.append(ChatMessage(role=Role.SYSTEM, content=(
+                "You are a senior software engineer. "
+                "Answer the user's query or perform the request clearly and concisely in natural language / Markdown. "
+                "Use the codebase context provided if relevant. "
+            )))
+
+    for message in base:
+        if not replaced_user and message.role is Role.USER:
+            out.append(ChatMessage(role=Role.USER, content=user_content))
+            replaced_user = True
+            continue
+        if message.role is Role.SYSTEM:
+            if intent is None or intent == "build":
+                extra = "\n\n" + _SYSTEM_PROMPT
+            else:
+                extra = (
+                    "\n\nAnswer the user's query or perform the request clearly and concisely in natural language / Markdown. "
+                    "Use the codebase context provided if relevant. "
+                )
+            out.append(
+                ChatMessage(
+                    role=Role.SYSTEM,
+                    content=message.content + extra,
+                )
+            )
+            continue
+        out.append(message)
+    if not replaced_user:
+        out.append(ChatMessage(role=Role.USER, content=user_content))
+    return out
+
+
+__all__ = ["llm_call"]
+
+
+# Silence unused-import warnings for symbols only used in some branches.
+_ = asyncio

forgecli/build/optimize.py

@@ -0,0 +1,37 @@
+"""Stage 2 — Ponytail optimization.
+
+Wraps the user prompt with the configured :class:`PromptOptimizer`
+(default: :class:`PonytailRulesetOptimizer`). The optimized request is
+re-used by the LLM stage; the ruleset's notes are propagated to the
+final summary.
+"""
+
+from __future__ import annotations
+
+from forgecli.build import BuildContext
+from forgecli.optimizer.ponytail import PromptOptimizer
+from forgecli.providers.base import ChatMessage, ChatRequest, Role
+
+
+async def ponytail_optimization(context: BuildContext) -> BuildContext:
+    """Run the configured :class:`PromptOptimizer` and stash the result."""
+    optimizer: PromptOptimizer | None = context.extras.get("optimizer")
+    if optimizer is None:
+        # No optimizer wired: produce a minimal request so downstream stages
+        # always have something to consume.
+        context.optimized_request = ChatRequest(
+            messages=[ChatMessage(role=Role.USER, content=context.prompt)],
+        )
+        context.optimized_notes = ()
+        return context
+
+    request = ChatRequest(
+        messages=[ChatMessage(role=Role.USER, content=context.prompt)],
+    )
+    optimized = await optimizer.optimize_chat(request)
+    context.optimized_request = optimized.request
+    context.optimized_notes = optimized.notes
+    return context
+
+
+__all__ = ["ponytail_optimization"]

forgecli/build/pipeline.py

@@ -0,0 +1,76 @@
+"""Default pipeline composition."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from forgecli.build import (
+    BuildContext,
+    BuildPipeline,
+    PipelineStage,
+)
+from forgecli.build.apply import apply_diff
+from forgecli.build.diff_extract import diff_extraction
+from forgecli.build.llm import llm_call
+from forgecli.build.optimize import ponytail_optimization
+from forgecli.build.retrieval import graphify_retrieval
+from forgecli.build.summarize import summarize
+from forgecli.build.test_run import run_tests
+from forgecli.optimizer.ponytail import PromptOptimizer
+from forgecli.providers.base import Provider
+from forgecli.providers.router import ModelRouter
+from forgecli.providers.router_state import RouterState, load_state
+
+
+def default_pipeline(
+    *,
+    provider: Provider,
+    optimizer: PromptOptimizer | None,
+    graph: Any | None = None,
+    test_command: str | None = None,
+    test_timeout: float = 120.0,
+    skip_tests: bool = False,
+) -> BuildPipeline:
+    """Construct the canonical pipeline."""
+    stages: list[tuple[str, PipelineStage]] = [
+        ("graphify-retrieval", _stage_with(graphify_retrieval, graph=graph)),
+        ("ponytail-optimize", _stage_with(ponytail_optimization, optimizer=optimizer)),
+        ("llm", _stage_with(llm_call, provider=provider)),
+        ("diff-extract", diff_extraction),
+        ("apply-diff", apply_diff),
+    ]
+    if not skip_tests:
+        stages.append(("run-tests", _stage_with(run_tests, test_command=test_command, test_timeout=test_timeout)))
+    stages.append(("summarize", summarize))
+    return BuildPipeline(stages)
+
+
+def _stage_with(
+    stage: PipelineStage, **extras: Any
+) -> PipelineStage:
+    """Bind extras into a stage by closing over them."""
+    async def _wrapped(context: BuildContext) -> BuildContext:
+        context.extras.update(extras)
+        return await stage(context)
+    return _wrapped
+
+
+def build_context_from(
+    prompt: str,
+    *,
+    root: Path,
+    router: ModelRouter | None = None,
+    state: RouterState | None = None,
+) -> BuildContext:
+    """Build a :class:`BuildContext` with the router's decision pre-resolved."""
+    state = state or RouterState()
+    decision = (router or ModelRouter()).select(state.choice)
+    return BuildContext(prompt=prompt, root=root, decision=decision)
+
+
+__all__ = ["build_context_from", "default_pipeline"]
+
+
+# Re-export helper for callers that want to load the persisted state too.
+_ = load_state

forgecli/build/retrieval.py

@@ -0,0 +1,157 @@
+"""Stage 1 — Graphify retrieval.
+
+Asks the configured :class:`RepositoryGraph` to find nodes relevant to
+the user prompt. The output is a compact text block listing the
+matching node labels, files, and edges, which is fed into the LLM as
+context in :mod:`forgecli.build.llm`.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from forgecli.build import BuildContext
+from forgecli.graph.repository import GraphNode, GraphSnapshot, RepositoryGraph
+
+_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]+")
+_FILE_LIKE_RE = re.compile(r"^[\w./-]+\.[A-Za-z0-9]+$")
+_REPO_HINTS = (
+    "project",
+    "repository",
+    "repo",
+    "code",
+    "file",
+    "files",
+    "architecture",
+    "implementation",
+    "bug",
+    "bugs",
+    "docs",
+    "documentation",
+    "structure",
+    "folder",
+    "directory",
+    "dir",
+    "function",
+    "method",
+    "class",
+    "module",
+    "current",
+    "this",
+    "here",
+)
+_STANDALONE_HINTS = (
+    "10 lines",
+    "hello world",
+    "landing page",
+    "button",
+    "snippet",
+)
+
+_RETRIEVAL_CACHE: dict[tuple[int, str, int], str] = {}
+
+
+def needs_repository_context(prompt: str) -> bool:
+    """Return True when the prompt likely needs repository context."""
+    text = (prompt or "").strip().lower()
+    if not text:
+        return False
+
+    words = set(_TOKEN_RE.findall(text))
+    if len(words) <= 4 and any(word in {"hi", "hello", "hey", "howdy", "greetings"} for word in words):
+        return False
+
+    if any(hint in text for hint in _STANDALONE_HINTS):
+        return False
+
+    if any(ext in text for ext in (".py", ".ts", ".tsx", ".js", ".jsx", ".json", ".md", ".html", ".css", ".yml", ".yaml", ".toml", "package.json")):
+        return True
+
+    if any(hint in words for hint in _REPO_HINTS):
+        return True
+
+    return "what is this" in text or "explain this" in text or "this project" in text
+
+
+async def graphify_retrieval(
+    context: BuildContext, *, top_k: int = 8
+) -> BuildContext:
+    """Return a context with ``context.retrieval`` populated."""
+    graph: RepositoryGraph | None = context.extras.get("graph")
+    if graph is None or not needs_repository_context(context.prompt):
+        context.retrieval = ""
+        return context
+
+    try:
+        snapshot = await graph.load()
+    except Exception as exc:
+        context.retrieval = f"[graph: failed to load ({exc!r})]"
+        return context
+
+    cache_key = (id(snapshot), context.prompt.strip().lower(), top_k)
+    cached = _RETRIEVAL_CACHE.get(cache_key)
+    if cached is not None:
+        context.retrieval = cached
+        return context
+
+    matches = _rank_nodes(snapshot, context.prompt, limit=top_k)
+    if not matches:
+        context.retrieval = "[graph: no matches]"
+        return context
+
+    lines = ["[graph retrieval]"]
+    for node, _score in matches:
+        location = (
+            f" ({node.source_file}:{node.source_location})"
+            if node.source_file
+            else ""
+        )
+        lines.append(f"- {node.label}{location}")
+    context.retrieval = "\n".join(lines)
+    _RETRIEVAL_CACHE[cache_key] = context.retrieval
+    return context
+
+
+def _rank_nodes(
+    snapshot: GraphSnapshot, query: str, *, limit: int
+) -> list[tuple[GraphNode, int]]:
+    """Return the top ``limit`` nodes most relevant to ``query``.
+
+    Scoring is intentionally simple and deterministic: each token in
+    ``query`` contributes one point for every node whose label or
+    source file contains the token as a substring. Tokens that look
+    like filenames (``foo.py``, ``auth/bar.ts``) get a bonus when
+    they match a node's ``source_file`` exactly.
+    """
+    tokens = [tok.lower() for tok in _TOKEN_RE.findall(query or "")]
+    if not tokens:
+        return []
+    scored: list[tuple[GraphNode, int]] = []
+    for node in snapshot.nodes:
+        label = (node.label or "").lower()
+        source = (node.source_file or "").lower()
+        if not label and not source:
+            continue
+        score = 0
+        for token in tokens:
+            if token and token in label:
+                score += 1
+            if token and token in source:
+                score += 1
+            if (
+                _FILE_LIKE_RE.match(token)
+                and node.source_file
+                and token == node.source_file.lower()
+            ):
+                score += 3
+        if score > 0:
+            scored.append((node, score))
+    scored.sort(key=lambda pair: (pair[1], pair[0].label), reverse=True)
+    return scored[:limit]
+
+
+__all__ = ["graphify_retrieval", "needs_repository_context"]
+
+
+_ = Any  # keep typing.Any referenced for the test-only type hints

forgecli/build/summarize.py

@@ -0,0 +1,76 @@
+"""Stage 7 — summarize.
+
+Composes a short human-readable summary of the run. The summary is
+shown in the terminal and stored in ``context.summary``.
+"""
+
+from __future__ import annotations
+
+import time
+
+from forgecli.build import BuildContext, BuildResult, StageStatus
+
+
+def build_summary(context: BuildContext) -> str:
+    """Return a multi-line summary of the run."""
+    lines: list[str] = []
+    lines.append(f"Goal: {context.prompt}")
+    if context.decision is not None:
+        d = context.decision
+        lines.append(
+            f"Route: {d.provider_name}/{d.model} "
+            f"(mode={d.mode.value}, "
+            f"in=${d.cost_in:.5f}/1k, out=${d.cost_out:.5f}/1k)"
+        )
+    if context.applied_files:
+        rel = [str(p.relative_to(context.root)) for p in context.applied_files]
+        lines.append(f"Files touched ({len(rel)}):")
+        for path in rel:
+            lines.append(f"  - {path}")
+    else:
+        lines.append("Files touched: (none)")
+    if context.test_returncode is None:
+        lines.append("Tests: skipped (no test runner available)")
+    elif context.test_returncode == 0:
+        lines.append("Tests: passed")
+    else:
+        lines.append(f"Tests: FAILED (exit {context.test_returncode})")
+        if context.test_stderr.strip():
+            lines.append("--- stderr ---")
+            lines.append(context.test_stderr.strip())
+    total = sum(s.duration_seconds or 0.0 for s in context.stages)
+    lines.append(f"Total time: {total:.2f}s across {len(context.stages)} stages")
+    return "\n".join(lines)
+
+
+async def summarize(context: BuildContext) -> BuildContext:
+    """Populate ``context.summary`` with a human-readable overview."""
+    context.summary = build_summary(context)
+    return context
+
+
+def result_to_dict(result: BuildResult) -> dict[str, object]:
+    """Return a JSON-serializable view of the result (for --json output)."""
+    return {
+        "success": result.success,
+        "failure_stage": result.failure_stage,
+        "summary": result.context.summary,
+        "applied_files": [str(p) for p in result.context.applied_files],
+        "test_returncode": result.context.test_returncode,
+        "stages": [
+            {
+                "name": s.name,
+                "status": s.status.value,
+                "duration_seconds": s.duration_seconds,
+                "notes": list(s.notes),
+                "error": s.error,
+            }
+            for s in result.context.stages
+        ],
+    }
+
+
+__all__ = ["build_summary", "result_to_dict", "summarize"]
+
+
+# Silence unused-import warnings.

forgecli/build/test_run.py

@@ -0,0 +1,75 @@
+"""Stage 6 — run tests.
+
+Runs an optional test command (default: ``pytest -q``) under the
+project root and captures stdout/stderr/return code. The build does
+not fail when tests fail — failures are recorded in the context and
+surfaced in the summary.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from forgecli.build import BuildContext
+
+
+DEFAULT_TEST_COMMAND = "pytest -q"
+
+
+async def run_tests(context: BuildContext) -> BuildContext:
+    """Execute the test command and record the outcome."""
+    command = context.extras.get("test_command") or DEFAULT_TEST_COMMAND
+    if not shutil.which(command.split()[0]):
+        context.test_stdout = ""
+        context.test_stderr = f"`{command.split()[0]}` not found on PATH; skipping tests."
+        context.test_returncode = None
+        return context
+
+    proc = await asyncio.create_subprocess_shell(
+        command,
+        cwd=str(context.root),
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        env=os.environ.copy(),
+    )
+    try:
+        stdout, stderr = await asyncio.wait_for(
+            proc.communicate(), timeout=_timeout_for(context)
+        )
+    except asyncio.TimeoutError:
+        proc.kill()
+        context.test_stdout = ""
+        context.test_stderr = "tests timed out"
+        context.test_returncode = 124
+        return context
+    context.test_stdout = stdout.decode(errors="replace")
+    context.test_stderr = stderr.decode(errors="replace")
+    context.test_returncode = proc.returncode
+    return context
+
+
+def _timeout_for(context: BuildContext) -> float:
+    timeout = context.extras.get("test_timeout")
+    if isinstance(timeout, (int, float)):
+        return float(timeout)
+    return 120.0
+
+
+def run_subprocess_sync(command: str, root: Path, *, timeout: float) -> subprocess.CompletedProcess:
+    """Synchronous helper used by tests."""
+    return subprocess.run(
+        command,
+        cwd=str(root),
+        shell=True,
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+        check=False,
+    )
+
+
+__all__ = ["DEFAULT_TEST_COMMAND", "run_subprocess_sync", "run_tests"]

forgecli/builder/__init__.py

@@ -0,0 +1,11 @@
+"""Code building pipeline: format, generate, apply edits."""
+
+from forgecli.builder.builder import Builder
+from forgecli.builder.editor import Editor
+from forgecli.builder.formatter import Formatter
+
+__all__ = [
+    "Builder",
+    "Editor",
+    "Formatter",
+]

forgecli/builder/builder.py

@@ -0,0 +1,53 @@
+"""Top-level builder pipeline orchestrator."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from forgecli.builder.editor import Editor, FileEdit
+from forgecli.builder.formatter import Formatter
+from forgecli.core.service import Service
+
+
+@dataclass
+class BuildResult:
+    """The outcome of a build run."""
+
+    success: bool
+    touched: list[Path] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+
+
+class Builder(Service):
+    """Coordinates editing, formatting, and verification of a change set."""
+
+    name = "builder"
+
+    def __init__(
+        self,
+        *,
+        editor: Editor | None = None,
+        formatter: Formatter | None = None,
+        max_iterations: int = 3,
+    ) -> None:
+        super().__init__()
+        self._editor = editor or Editor()
+        self._formatter = formatter or Formatter()
+        self._max_iterations = max_iterations
+
+    @property
+    def editor(self) -> Editor:
+        return self._editor
+
+    @property
+    def formatter(self) -> Formatter:
+        return self._formatter
+
+    def build(self, edits: Iterable[FileEdit]) -> BuildResult:
+        """Apply ``edits`` and run post-processing; placeholder for now."""
+        touched = self._editor.apply(edits)
+        if touched:
+            self._formatter.format(touched)
+        return BuildResult(success=True, touched=touched)

forgecli/builder/editor.py

@@ -0,0 +1,36 @@
+"""File editing primitives."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+from forgecli.core.service import Service
+from forgecli.utils.fs import atomic_write, read_text
+
+
+@dataclass(frozen=True)
+class FileEdit:
+    """A single replacement operation targeting ``path``."""
+
+    path: Path
+    new_content: str
+
+
+class Editor(Service):
+    """Apply a batch of file edits atomically (best effort)."""
+
+    name = "builder.editor"
+
+    def apply(self, edits: Iterable[FileEdit]) -> list[Path]:
+        """Write each edit to disk; return the list of touched paths."""
+        touched: list[Path] = []
+        for edit in edits:
+            atomic_write(edit.path, edit.new_content)
+            touched.append(Path(edit.path))
+        return touched
+
+    @staticmethod
+    def read(path: Path) -> str:
+        return read_text(path)