agentforge-ml 0.1.0__py3-none-any.whl

agentforge/__init__.py

@@ -0,0 +1,12 @@
+"""AgentForge — ReAct agents on open-weight LLMs with tools and an eval harness."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from agentforge.core.agent import Agent, AgentResult, Step
+
+try:
+    __version__ = version("agentforge-ml")
+except PackageNotFoundError:
+    __version__ = "0.0.0+local"
+
+__all__ = ["Agent", "AgentResult", "Step", "__version__"]

agentforge/cli.py

@@ -0,0 +1,175 @@
+"""Command-line interface — ``agentforge`` / ``af``."""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+app = typer.Typer(
+    name="agentforge",
+    help="ReAct agents on open-weight LLMs with tools and an eval harness.",
+    no_args_is_help=True,
+    rich_markup_mode="rich",
+)
+console = Console()
+
+_TOOL_FACTORY = {
+    "calculator": lambda: _import("agentforge.tools.Calculator")(),
+    "python_repl": lambda: _import("agentforge.tools.PythonREPL")(),
+    "web_search": lambda: _import("agentforge.tools.WebSearch")(),
+}
+
+
+def _import(dotted: str):
+    mod_name, cls_name = dotted.rsplit(".", 1)
+    import importlib
+
+    return getattr(importlib.import_module(mod_name), cls_name)
+
+
+def _build_tools(spec: str):
+    if spec.lower() in ("all", "*"):
+        spec = ",".join(_TOOL_FACTORY)
+    tools = []
+    for name in (x.strip() for x in spec.split(",")):
+        if not name:
+            continue
+        if name not in _TOOL_FACTORY:
+            console.print(f"[red]unknown tool: {name}[/]  (available: {', '.join(_TOOL_FACTORY)})")
+            raise typer.Exit(2)
+        tools.append(_TOOL_FACTORY[name]())
+    return tools
+
+
+@app.command()
+def ask(
+    question: Annotated[str, typer.Argument()],
+    model_id: Annotated[str, typer.Option()] = "Qwen/Qwen2.5-3B-Instruct",
+    tools: Annotated[str, typer.Option(help="Comma-separated or 'all'")] = "calculator,python_repl",
+    max_steps: Annotated[int, typer.Option()] = 6,
+    max_new_tokens: Annotated[int, typer.Option()] = 256,
+    quantize: Annotated[str | None, typer.Option(help="Optional turboquant-ml method")] = None,
+    verbose: Annotated[bool, typer.Option()] = False,
+) -> None:
+    """Run the ReAct agent on one question."""
+    from agentforge import Agent
+
+    console.print(Panel.fit(f"[bold teal]agentforge ask[/]  [dim]{question}[/]"))
+    tool_list = _build_tools(tools)
+    agent = Agent.from_defaults(
+        model_id=model_id, tools=tool_list, max_steps=max_steps, verbose=verbose, quantize=quantize
+    )
+    result = agent.run(question, max_new_tokens=max_new_tokens)
+
+    if verbose or not result.success:
+        table = Table(title="Steps", show_header=True)
+        table.add_column("#", style="dim")
+        table.add_column("tool")
+        table.add_column("input")
+        table.add_column("observation")
+        for i, s in enumerate(result.steps, 1):
+            table.add_row(
+                str(i), s.tool or "-", (s.action_input or "")[:60], (s.observation or "")[:80]
+            )
+        console.print(table)
+
+    console.print(
+        f"\n[bold]Final answer[/]  [dim]({result.latency_ms:.0f} ms, {result.n_steps} steps)[/]"
+    )
+    console.print(result.final_answer)
+
+
+@app.command()
+def eval(
+    dataset: Annotated[
+        Path, typer.Argument(help="JSONL with {question, ground_truth, expected_tools?}")
+    ],
+    model_id: Annotated[str, typer.Option()] = "Qwen/Qwen2.5-3B-Instruct",
+    tools: Annotated[str, typer.Option()] = "all",
+    max_steps: Annotated[int, typer.Option()] = 6,
+    out: Annotated[Path | None, typer.Option()] = None,
+    limit: Annotated[int | None, typer.Option()] = None,
+) -> None:
+    """Run the eval harness over a JSONL dataset."""
+    from agentforge import Agent
+    from agentforge.eval import evaluate
+    from agentforge.eval.report import EvalReport
+
+    samples = _read_jsonl(dataset, limit=limit)
+    console.print(Panel.fit(f"[bold teal]agentforge eval[/]  n={len(samples)}"))
+
+    tool_list = _build_tools(tools)
+    agent = Agent.from_defaults(model_id=model_id, tools=tool_list, max_steps=max_steps)
+
+    results = []
+    latencies: list[float] = []
+    for s in samples:
+        t0 = time.perf_counter()
+        r = agent.run(s["question"])
+        latencies.append((time.perf_counter() - t0) * 1000)
+        results.append(r)
+
+    res = evaluate(samples, results)
+    report = EvalReport(
+        n=res["n"], means=res["means"], per_sample=res["per_sample"], latencies_ms=latencies
+    )
+    console.print(report.as_table())
+    if out:
+        report.save(out)
+        console.print(f"[green]ok[/]  saved {out}")
+
+
+@app.command()
+def serve(
+    model_id: Annotated[str, typer.Option()] = "Qwen/Qwen2.5-3B-Instruct",
+    tools: Annotated[str, typer.Option()] = "calculator,python_repl",
+    max_steps: Annotated[int, typer.Option()] = 6,
+    host: Annotated[str, typer.Option()] = "127.0.0.1",
+    port: Annotated[int, typer.Option()] = 8000,
+) -> None:
+    """Start the FastAPI agent server."""
+    import uvicorn
+
+    from agentforge import Agent
+    from agentforge.serve import build_app
+
+    tool_list = _build_tools(tools)
+    agent = Agent.from_defaults(model_id=model_id, tools=tool_list, max_steps=max_steps)
+    app_ = build_app(agent)
+    uvicorn.run(app_, host=host, port=port, log_level="info")
+
+
+@app.command(name="tools")
+def list_tools_cmd() -> None:
+    """List the built-in tools."""
+    table = Table(title="Built-in tools")
+    table.add_column("name")
+    table.add_column("description")
+    for name, factory in _TOOL_FACTORY.items():
+        t = factory()
+        table.add_row(name, t.description)
+    console.print(table)
+
+
+def _read_jsonl(path: Path, *, limit: int | None) -> list[dict]:
+    rows: list[dict] = []
+    with path.open(encoding="utf-8") as f:
+        for line in f:
+            s = line.strip()
+            if not s:
+                continue
+            rows.append(json.loads(s))
+            if limit and len(rows) >= limit:
+                break
+    return rows
+
+
+if __name__ == "__main__":
+    app()

agentforge/core/__init__.py

@@ -0,0 +1,15 @@
+"""ReAct loop + parser + prompts."""
+
+from agentforge.core.agent import Agent, AgentResult, Step
+from agentforge.core.parser import ParsedStep, parse_step
+from agentforge.core.prompts import REACT_SYSTEM_PROMPT, build_user_prompt
+
+__all__ = [
+    "REACT_SYSTEM_PROMPT",
+    "Agent",
+    "AgentResult",
+    "ParsedStep",
+    "Step",
+    "build_user_prompt",
+    "parse_step",
+]

agentforge/core/agent.py

@@ -0,0 +1,186 @@
+"""ReAct agent — the loop.
+
+    while step < max_steps:
+        thought, action, action_input = LLM(question + scratchpad)
+        if final_answer: return
+        observation = tool(action_input)
+        scratchpad += step
+
+That is the whole idea. Everything else is just plumbing.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from agentforge.core.parser import parse_step
+from agentforge.core.prompts import build_system_prompt, build_user_prompt, format_scratchpad
+from agentforge.llm import LLM
+from agentforge.tools import Tool, ToolRegistry
+
+logger = logging.getLogger("agentforge")
+
+
+@dataclass
+class Step:
+    thought: str
+    tool: str | None
+    action_input: str | None
+    observation: str | None
+    elapsed_ms: float
+
+
+@dataclass
+class AgentResult:
+    question: str
+    final_answer: str
+    steps: list[Step]
+    n_steps: int
+    success: bool
+    latency_ms: float
+    extras: dict[str, Any] = field(default_factory=dict)
+
+
+class Agent:
+    """ReAct agent with a tool registry and a configurable LLM."""
+
+    def __init__(
+        self,
+        llm: LLM,
+        tools: list[Tool] | ToolRegistry,
+        *,
+        max_steps: int = 6,
+        stop: tuple[str, ...] = ("\nObservation:",),
+        verbose: bool = False,
+    ) -> None:
+        self.llm = llm
+        self.tools = tools if isinstance(tools, ToolRegistry) else ToolRegistry(tools)
+        self.max_steps = max_steps
+        self.stop = stop
+        self.verbose = verbose
+
+    @classmethod
+    def from_defaults(
+        cls,
+        model_id: str = "Qwen/Qwen2.5-3B-Instruct",
+        *,
+        tools: list[Tool] | None = None,
+        max_steps: int = 6,
+        verbose: bool = False,
+        quantize: str | None = None,
+        device_map: str | dict | None = "auto",
+    ) -> Agent:
+        if quantize:
+            from agentforge.llm import QuantizedHFLLM
+
+            llm = QuantizedHFLLM(model_id, method=quantize, device_map=device_map)
+        else:
+            from agentforge.llm import HFLLM
+
+            llm = HFLLM(model_id, device_map=device_map)
+
+        return cls(llm=llm, tools=tools or [], max_steps=max_steps, verbose=verbose)
+
+    # ------------------------------------------------------------------ public
+
+    def run(
+        self,
+        question: str,
+        *,
+        max_steps: int | None = None,
+        max_new_tokens: int = 256,
+    ) -> AgentResult:
+        max_steps = max_steps or self.max_steps
+        sys_prompt = build_system_prompt(list(self.tools))
+        steps: list[Step] = []
+        t0 = time.perf_counter()
+
+        for _ in range(max_steps):
+            scratchpad = format_scratchpad(steps)
+            user_prompt = build_user_prompt(question, scratchpad)
+            full_prompt = f"{sys_prompt}\n\n{user_prompt}"
+
+            llm_t0 = time.perf_counter()
+            raw = self.llm.generate(
+                full_prompt,
+                max_new_tokens=max_new_tokens,
+                stop=list(self.stop),
+            )
+            elapsed = (time.perf_counter() - llm_t0) * 1000
+
+            parsed = parse_step(raw)
+            if self.verbose:
+                logger.info("step parse: %s", parsed)
+
+            if parsed.is_final:
+                steps.append(
+                    Step(
+                        thought=parsed.thought,
+                        tool=None,
+                        action_input=None,
+                        observation=None,
+                        elapsed_ms=elapsed,
+                    )
+                )
+                return _success(question, parsed.final_answer or "", steps, t0)
+
+            # Need a tool to continue. If the LLM didn't emit one, treat the
+            # raw response as the final answer (graceful degradation).
+            if not parsed.tool:
+                return _success(question, raw.strip(), steps, t0, success=False)
+
+            tool = self.tools.get(parsed.tool)
+            if tool is None:
+                observation = (
+                    f"Error: unknown tool '{parsed.tool}'. "
+                    f"Available: {', '.join(t.name for t in self.tools)}"
+                )
+            else:
+                try:
+                    observation = tool.run(parsed.action_input or "")
+                except Exception as e:
+                    observation = f"Error running {parsed.tool}: {type(e).__name__}: {e}"
+
+            steps.append(
+                Step(
+                    thought=parsed.thought,
+                    tool=parsed.tool,
+                    action_input=parsed.action_input,
+                    observation=observation,
+                    elapsed_ms=elapsed,
+                )
+            )
+
+        # Out of steps.
+        return _success(
+            question,
+            steps[-1].observation if steps and steps[-1].observation else "",
+            steps,
+            t0,
+            success=False,
+        )
+
+
+def _success(
+    question: str,
+    answer: str,
+    steps: list[Step],
+    t0: float,
+    *,
+    success: bool = True,
+) -> AgentResult:
+    latency_ms = (time.perf_counter() - t0) * 1000
+    return AgentResult(
+        question=question,
+        final_answer=answer,
+        steps=steps,
+        n_steps=len(steps),
+        success=success,
+        latency_ms=round(latency_ms, 2),
+    )
+
+
+__all__ = ["Agent", "AgentResult", "Step"]

agentforge/core/parser.py

@@ -0,0 +1,85 @@
+"""ReAct output parser.
+
+The LLM emits free-form text shaped roughly like:
+
+    Thought: ...
+    Action: tool_name
+    Action Input: ...
+
+or:
+
+    Thought: ...
+    Final Answer: ...
+
+This parser is intentionally **forgiving** — small open models often forget
+whitespace, mix cases, or fail to close the action input. We grab the most
+recent plausible step from the buffer and stop at the first unambiguous
+boundary.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+_FINAL = re.compile(r"final\s*answer\s*:\s*(.*)", re.IGNORECASE | re.DOTALL)
+_THOUGHT = re.compile(
+    r"thought\s*:\s*(.*?)(?=\n\s*(?:action|final\s*answer)\s*:|$)", re.IGNORECASE | re.DOTALL
+)
+_ACTION = re.compile(r"action\s*:\s*([^\n]+)", re.IGNORECASE)
+_INPUT = re.compile(
+    r"action\s*input\s*:\s*(.*?)(?=\n\s*(?:observation|thought|action|final\s*answer)\s*:|$)",
+    re.IGNORECASE | re.DOTALL,
+)
+
+
+@dataclass
+class ParsedStep:
+    thought: str = ""
+    tool: str | None = None
+    action_input: str | None = None
+    final_answer: str | None = None
+
+    @property
+    def is_final(self) -> bool:
+        return self.final_answer is not None
+
+
+def parse_step(text: str) -> ParsedStep:
+    """Pull the next Thought/Action/Action Input *or* Final Answer out of ``text``."""
+    text = text.strip()
+
+    final = _FINAL.search(text)
+    if final:
+        # If the model emitted a Final Answer, anything before it is the thought.
+        thought = _extract_thought(text[: final.start()])
+        ans = final.group(1).strip()
+        # Trim a possible trailing block ("Observation: ...") in case the model went past.
+        ans = _trim_after(ans, ("\nThought:", "\nObservation:", "\nAction:"))
+        return ParsedStep(thought=thought, final_answer=ans)
+
+    thought = _extract_thought(text)
+    action = _ACTION.search(text)
+    inp = _INPUT.search(text)
+    tool = action.group(1).strip() if action else None
+    action_input = inp.group(1).strip() if inp else None
+    if action_input is not None:
+        action_input = _trim_after(action_input, ("\nThought:", "\nObservation:", "\nAction:"))
+    return ParsedStep(thought=thought, tool=tool, action_input=action_input)
+
+
+def _extract_thought(text: str) -> str:
+    m = _THOUGHT.search(text)
+    if not m:
+        return ""
+    return m.group(1).strip()
+
+
+def _trim_after(s: str, stops: tuple[str, ...]) -> str:
+    earliest = len(s)
+    lower = s.lower()
+    for stop in stops:
+        idx = lower.find(stop.lower())
+        if idx != -1:
+            earliest = min(earliest, idx)
+    return s[:earliest].strip()

agentforge/core/prompts.py

@@ -0,0 +1,59 @@
+"""ReAct prompt templates.
+
+The system prompt explains the loop format and lists the available tools with
+their docstrings. We keep it short and concrete — long prompts confuse small
+open models more than they help.
+"""
+
+from __future__ import annotations
+
+REACT_SYSTEM_PROMPT = """You are a helpful assistant that solves problems step by step.
+
+You have access to the following tools:
+
+{tool_block}
+
+Use this exact format, one Thought/Action/Action Input per step:
+
+Thought: <your reasoning about what to do next>
+Action: <one of: {tool_names}>
+Action Input: <the input to the tool>
+
+After each step, you will receive an Observation: with the tool's output.
+Continue until you can answer, then write:
+
+Thought: <reasoning summarizing the result>
+Final Answer: <your final, concise answer>
+
+Important rules:
+- Use only one Action per step. Do not write multiple actions at once.
+- If the answer is straightforward and no tool is needed, you may go directly to "Final Answer:".
+- Do not invent observations — wait for the system to provide them.
+"""
+
+
+def build_system_prompt(tools: list) -> str:
+    """Render the tool registry into the system prompt."""
+    tool_block = "\n".join(f"- {t.name}: {t.description}" for t in tools)
+    tool_names = ", ".join(t.name for t in tools) if tools else "(none)"
+    return REACT_SYSTEM_PROMPT.format(tool_block=tool_block, tool_names=tool_names)
+
+
+def build_user_prompt(question: str, scratchpad: str = "") -> str:
+    """Render the user turn: the question + the running scratchpad of prior steps."""
+    if not scratchpad:
+        return f"Question: {question}\n\nThought:"
+    return f"Question: {question}\n\n{scratchpad}\nThought:"
+
+
+def format_scratchpad(steps: list) -> str:
+    """Concatenate completed steps back into the prompt for the next iteration."""
+    parts = []
+    for s in steps:
+        parts.append(
+            f"Thought: {s.thought}\n"
+            f"Action: {s.tool}\n"
+            f"Action Input: {s.action_input}\n"
+            f"Observation: {s.observation}"
+        )
+    return "\n\n".join(parts)

agentforge/eval/__init__.py

@@ -0,0 +1,19 @@
+"""Evaluation harness for ReAct agents."""
+
+from agentforge.eval.metrics import (
+    evaluate,
+    final_answer_match,
+    step_efficiency,
+    task_completion,
+    tool_accuracy,
+)
+from agentforge.eval.report import EvalReport
+
+__all__ = [
+    "EvalReport",
+    "evaluate",
+    "final_answer_match",
+    "step_efficiency",
+    "task_completion",
+    "tool_accuracy",
+]

agentforge/eval/metrics.py

@@ -0,0 +1,105 @@
+"""Agent-quality metrics.
+
+Pure Python, no judge model. The metrics are intentionally simple so the
+relationship between "the agent did X" and "the metric says Y" is auditable
+from the code alone.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+from agentforge.core.agent import AgentResult
+
+
+def task_completion(result: AgentResult, sample: dict) -> float:
+    """Did the agent produce *any* final answer? (Did the loop terminate cleanly?)"""
+    return 1.0 if result.success and result.final_answer.strip() else 0.0
+
+
+def final_answer_match(result: AgentResult, sample: dict) -> float:
+    """Does the final answer contain the ground-truth string (case-folded)?
+
+    Substring rather than exact match because LLMs tend to wrap answers in
+    natural-language framing ("The answer is 42."). For numeric questions,
+    pass the bare number as ``ground_truth`` and it will be matched.
+    """
+    gt = sample.get("ground_truth")
+    if not gt:
+        return 0.0
+    return 1.0 if str(gt).strip().lower() in result.final_answer.lower() else 0.0
+
+
+def tool_accuracy(result: AgentResult, sample: dict) -> float:
+    """Fraction of tool calls that match the expected tool(s).
+
+    ``expected_tools`` in the sample can be:
+        - a string (one tool expected at any step)
+        - a list of strings (each step's expected tool, in order, with "any" wildcard)
+        - missing (returns 1.0 for trivia-style samples)
+    """
+    expected = sample.get("expected_tools")
+    if expected is None:
+        return 1.0
+
+    actual = [s.tool for s in result.steps if s.tool]
+    if isinstance(expected, str):
+        return 1.0 if expected in actual else 0.0
+
+    if not isinstance(expected, list):
+        return 0.0
+    if not actual:
+        return 0.0
+
+    # Index-wise comparison up to the shorter sequence.
+    n = min(len(actual), len(expected))
+    if n == 0:
+        return 0.0
+    matched = sum(1 for i in range(n) if expected[i] in ("*", "any", actual[i]))
+    return matched / max(len(expected), 1)
+
+
+def step_efficiency(result: AgentResult, sample: dict) -> float:
+    """``ground_truth_steps / actual_steps`` clipped to [0, 1].
+
+    Encourages reaching the answer in as few steps as possible. If the sample
+    omits ``ground_truth_steps``, defaults to 1 step.
+    """
+    if not result.success or result.n_steps == 0:
+        return 0.0
+    gt_steps = max(1, int(sample.get("ground_truth_steps", 1)))
+    return min(1.0, gt_steps / result.n_steps)
+
+
+_REGISTRY = {
+    "task_completion": task_completion,
+    "final_answer_match": final_answer_match,
+    "tool_accuracy": tool_accuracy,
+    "step_efficiency": step_efficiency,
+}
+
+
+def evaluate(
+    samples: list[dict[str, Any]],
+    results: list[AgentResult],
+    *,
+    metrics: Iterable[str] = (
+        "task_completion",
+        "final_answer_match",
+        "tool_accuracy",
+        "step_efficiency",
+    ),
+) -> dict:
+    """Score a parallel list of samples + results across one or more metrics."""
+    if len(samples) != len(results):
+        raise ValueError(f"len(samples)={len(samples)} != len(results)={len(results)}")
+
+    by_metric: dict[str, list[float]] = {m: [] for m in metrics}
+    for sample, result in zip(samples, results, strict=True):
+        for m in metrics:
+            fn = _REGISTRY[m]
+            by_metric[m].append(float(fn(result, sample)))
+
+    means = {m: (sum(v) / len(v) if v else 0.0) for m, v in by_metric.items()}
+    return {"n": len(samples), "means": means, "per_sample": by_metric}