perferox 0.1.0__tar.gz

perferox-0.1.0/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: perferox
+Version: 0.1.0
+Summary: Agentic benchmark explorer for SGLang.
+Author: Naman Chetwani
+Keywords: sglang,benchmark,fuzzing,inference,agents
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: langchain
+Requires-Dist: langgraph
+Requires-Dist: langchain-openai
+Requires-Dist: textual
+Requires-Dist: paramiko
+Requires-Dist: pydantic
+Requires-Dist: numpy
+Requires-Dist: sentence-transformers>=5.6.0
+
+<h1 align="center">perferox</h1>
+
+Perferox is a small agentic benchmark explorer for inference engines and ML
+systems performance. The first target is SGLang.
+
+This is alpha software. It is not ready for common use yet. Expect rough edges,
+missing docs, changing APIs, and places where you still need to understand the
+repo before trusting a run.
+
+The shape is intentionally simple: give an agent a bounded objective, let it
+delegate benchmark workers, run experiments through narrow tools, and save the
+parts worth keeping. The point is not to make a giant benchmark platform. The
+point is to make it easy to poke at weird corners of an inference stack without
+losing track of what already happened.
+
+The first use case is SGLang serving benchmarks. Perferox is meant to try
+different models, backends, cache settings, request patterns, chips, and setup
+conditions, then preserve the useful results in SQLite. It records the normal
+perf signals, like request throughput, token throughput, TTFT, TPOT, cache hit
+rate, accept length, and error rate. It also has a place to store human-readable
+anomalies, because a lot of the interesting bugs are not just "number went down"
+but "this specific setup behaved strangely in a way a person should inspect."
+
+The important constraint is that the host owns state. Agents do not choose run
+IDs, write SQLite directly, or decide whether a benchmark should bypass caps.
+Real experiments go through a benchmark tool. Started runs get logged. Failed
+runs still count once they actually start. The database is the source of truth
+for runs, experiments, anomalies, trace references, and compact exploration
+state.
+
+Current pieces:
+
+- a `perferox` CLI
+- a Textual TUI for starting and stopping exploration
+- a LangGraph main coordinator
+- tmux-wrapped benchmark subagents
+- Paramiko remote sessions
+- structured SGLang `bench_serving` command construction and parsing
+- SQLite-backed runs, experiments, anomalies, notifications, and ExplorerState
+- local SGLang docs search plumbing
+
+This is still early software. The goal is to keep the code small enough that it
+stays easy to inspect and change. The preferred path is boring Python, explicit
+state, small tools, and very few abstractions.
+
+```bash
+uvx perferox --help
+```

perferox-0.1.0/README.md

@@ -0,0 +1,48 @@
+<h1 align="center">perferox</h1>
+
+Perferox is a small agentic benchmark explorer for inference engines and ML
+systems performance. The first target is SGLang.
+
+This is alpha software. It is not ready for common use yet. Expect rough edges,
+missing docs, changing APIs, and places where you still need to understand the
+repo before trusting a run.
+
+The shape is intentionally simple: give an agent a bounded objective, let it
+delegate benchmark workers, run experiments through narrow tools, and save the
+parts worth keeping. The point is not to make a giant benchmark platform. The
+point is to make it easy to poke at weird corners of an inference stack without
+losing track of what already happened.
+
+The first use case is SGLang serving benchmarks. Perferox is meant to try
+different models, backends, cache settings, request patterns, chips, and setup
+conditions, then preserve the useful results in SQLite. It records the normal
+perf signals, like request throughput, token throughput, TTFT, TPOT, cache hit
+rate, accept length, and error rate. It also has a place to store human-readable
+anomalies, because a lot of the interesting bugs are not just "number went down"
+but "this specific setup behaved strangely in a way a person should inspect."
+
+The important constraint is that the host owns state. Agents do not choose run
+IDs, write SQLite directly, or decide whether a benchmark should bypass caps.
+Real experiments go through a benchmark tool. Started runs get logged. Failed
+runs still count once they actually start. The database is the source of truth
+for runs, experiments, anomalies, trace references, and compact exploration
+state.
+
+Current pieces:
+
+- a `perferox` CLI
+- a Textual TUI for starting and stopping exploration
+- a LangGraph main coordinator
+- tmux-wrapped benchmark subagents
+- Paramiko remote sessions
+- structured SGLang `bench_serving` command construction and parsing
+- SQLite-backed runs, experiments, anomalies, notifications, and ExplorerState
+- local SGLang docs search plumbing
+
+This is still early software. The goal is to keep the code small enough that it
+stays easy to inspect and change. The preferred path is boring Python, explicit
+state, small tools, and very few abstractions.
+
+```bash
+uvx perferox --help
+```

perferox-0.1.0/perferox/__init__.py

@@ -0,0 +1 @@
+"""Perferox package."""

perferox-0.1.0/perferox/agent_runner.py

@@ -0,0 +1,158 @@
+"""Tiny CLI for tmux-wrapped Perferox agent processes."""
+
+from __future__ import annotations
+
+import argparse
+import shlex
+import shutil
+import subprocess
+import time
+from contextlib import closing
+from pathlib import Path
+
+from langchain_core.messages import HumanMessage
+
+from perferox import db
+from perferox.auth import build_chat_model
+from perferox.main_agent import build_main_agent_graph
+from perferox.remote import SessionRegistry
+from perferox.subagent import build_subagent_graph, stream_with_trace
+
+MAIN_SESSION = "perferox-main"
+
+
+def main(argv: list[str] | None = None) -> int:
+  """Parse the runner command and run the requested whole-agent process."""
+  parser = argparse.ArgumentParser(prog="python -m perferox.agent_runner")
+  subparsers = parser.add_subparsers(dest="command", required=True)
+  for name in ("launch-main", "main"):
+    subparser = subparsers.add_parser(name)
+    subparser.add_argument("--db-path", required=True)
+    subparser.add_argument("--trace-dir", default="traces")
+    subparser.add_argument("--objective", required=True)
+    subparser.add_argument("--cwd", default=".")
+  subparsers.choices["main"].add_argument("--poll-s", type=float, default=5.0)
+  subagent = subparsers.add_parser("subagent")
+  for name in ("agent-id", "db-path", "trace-path", "goal-file", "attempt-cap"):
+    subagent.add_argument(f"--{name}", required=True)
+  args = parser.parse_args(argv)
+
+  if args.command == "launch-main":
+    tmux = shutil.which("tmux")
+    if tmux is None:
+      print("tmux is not installed or not on PATH")
+      return 1
+    cwd = Path(args.cwd).resolve()
+    db_path = (cwd / args.db_path).resolve()
+    trace_dir = (cwd / args.trace_dir).resolve()
+    trace_dir.mkdir(parents=True, exist_ok=True)
+    command = shlex.join(["uv", "run", "python", "-m", "perferox.agent_runner", "main", "--db-path", str(db_path), "--trace-dir", str(trace_dir), "--objective", args.objective, "--cwd", str(cwd)])
+    result = subprocess.run([tmux, "new-session", "-d", "-s", MAIN_SESSION, "-c", str(cwd), "--", "bash", "-lc", command], text=True, capture_output=True, check=False)
+    print(f"started {MAIN_SESSION}; attach with: tmux attach -t {MAIN_SESSION}" if result.returncode == 0 else (result.stderr or result.stdout).strip())
+    return result.returncode
+
+  if args.command == "main":
+    cwd = Path(args.cwd).resolve()
+    db_path = (cwd / args.db_path).resolve()
+    trace_dir = (cwd / args.trace_dir).resolve()
+    trace_dir.mkdir(parents=True, exist_ok=True)
+    trace_path = trace_dir / f"{MAIN_SESSION}-{int(time.time())}.jsonl"
+    try:
+      with closing(db.connect(db_path)) as conn:
+        db.init_db(conn)
+        db.record_agent_session(conn, session_name=MAIN_SESSION, role="main", trace_ref=str(trace_path))
+      graph = build_main_agent_graph(build_chat_model(), db_path, cwd=cwd, trace_dir=trace_dir)
+      state = {"objective": args.objective, "messages": [HumanMessage(content=args.objective)]}
+      while True:
+        for event in stream_with_trace(graph, state, trace_path):
+          state = _collect_update(state, event)
+        update = _wait_for_main_event(db_path, args.poll_s)
+        if update is None:
+          return 0
+        state["messages"] = [*state.get("messages", []), HumanMessage(content=update)]
+    finally:
+      with closing(db.connect(db_path)) as conn:
+        db.finish_agent_session(conn, session_name=MAIN_SESSION, status="exited")
+
+  agent_id = int(args.agent_id)
+  db_path = Path(args.db_path).resolve()
+  trace_path = Path(args.trace_path).resolve()
+  session_name = f"perferox-agent-{agent_id}"
+  registry = SessionRegistry()
+  try:
+    with closing(db.connect(db_path)) as conn:
+      db.init_db(conn)
+      db.record_agent_session(conn, session_name=session_name, role="subagent", agent_id=agent_id, trace_ref=str(trace_path))
+    attempt_cap = int(args.attempt_cap)
+    graph = build_subagent_graph(build_chat_model(), agent_id, registry, db_path, attempt_cap=attempt_cap, trace_ref=str(trace_path))
+    state = {"agent_id": agent_id, "messages": [HumanMessage(content=Path(args.goal_file).read_text(encoding="utf-8"))]}
+    for _ in stream_with_trace(graph, state, trace_path):
+      pass
+    return 0
+  finally:
+    registry.close(f"agent-{agent_id}")
+    with closing(db.connect(db_path)) as conn:
+      db.init_db(conn)
+      if db.finish_agent_session(conn, session_name=session_name, status="exited"):
+        db.append_explorer_state(conn, agent_id=agent_id, line=f"agent-{agent_id} tmux exited; trace {trace_path.name}")
+
+
+def _collect_update(state: dict, event: object) -> dict:
+  """Merge one streamed LangGraph update into a reusable graph state."""
+  if not isinstance(event, dict):
+    return state
+  for update in event.values():
+    if not isinstance(update, dict):
+      continue
+    for key, value in update.items():
+      if key == "messages":
+        state.setdefault(key, []).extend(value)
+      else:
+        state[key] = value
+  return state
+
+
+def _wait_for_main_event(db_path: Path, poll_s: float) -> str | None:
+  """Wait for a main-agent wakeup or a human End request."""
+  previous_running = None
+  while True:
+    with closing(db.connect(db_path)) as conn:
+      db.init_db(conn)
+      main_row = conn.execute("SELECT status FROM agent_sessions WHERE session_name = ?", (MAIN_SESSION,)).fetchone()
+      notifications = db.take_main_notifications(conn)
+      if notifications:
+        lines = ["Subagent SQLite write notifications:"]
+        for row in notifications:
+          prefix = "ANOMALY " if row["kind"] == "anomaly_logged" else ""
+          lines.append(
+            f"{prefix}notification_id={row['notification_id']} kind={row['kind']} "
+            f"agent_id={row['agent_id']} run_id={row['run_id']} table={row['table_name']}"
+          )
+          lines.append(row["row_json"])
+        return "\n".join(lines)
+      tmux = shutil.which("tmux")
+      active_query = "SELECT session_name, agent_id, trace_ref FROM agent_sessions WHERE status IN ('running', 'ending') AND role = 'subagent'"
+      rows = conn.execute(active_query).fetchall()
+      for row in rows:
+        alive = tmux and subprocess.run([tmux, "has-session", "-t", row["session_name"]], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False).returncode == 0
+        if not alive and db.finish_agent_session(conn, session_name=row["session_name"], status="missing"):
+          line = f"agent-{row['agent_id']} tmux missing; trace {Path(row['trace_ref']).name}"
+          db.append_explorer_state(conn, agent_id=row["agent_id"], line=line)
+          return f"Tmux session update:\n{line}"
+      if main_row is not None and main_row["status"] == "ending":
+        if not rows:
+          return None
+        time.sleep(poll_s)
+        return "End requested. Do not delegate new subagents; wait for active subagents to wrap up, then summarize."
+    running = {row["session_name"] for row in rows}
+    if previous_running is not None and running != previous_running:
+      return "Tmux session update:\nsubagent tmux sessions changed" if running else "Tmux session update:\nall subagent tmux sessions completed"
+    if not running:
+      time.sleep(poll_s)
+      return "No active subagents are running. Continue exploration from the objective and ExplorerState."
+    previous_running = running
+    time.sleep(poll_s)
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())

perferox-0.1.0/perferox/auth.py

@@ -0,0 +1,46 @@
+"""ChatGPT OAuth auth and model construction for Perferox."""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+  from langchain_core.language_models.chat_models import BaseChatModel
+
+DEFAULT_CHAT_MODEL = "gpt-5.5"
+MODEL_ENV_VAR = "PERFEROX_CHAT_MODEL"
+ORIGINATOR = "perferox"
+
+
+def chatgpt_auth_ready() -> bool:
+  """Return whether a refreshable ChatGPT OAuth token is available."""
+  try:
+    from langchain_openai.chatgpt_oauth import _FileChatGPTOAuthTokenProvider
+
+    _FileChatGPTOAuthTokenProvider.from_default_store().get_token()
+  except Exception:  # noqa: BLE001
+    return False
+  return True
+
+
+def login_chatgpt_oauth(timeout_s: float = 300.0) -> None:
+  """Run the browser-based ChatGPT OAuth login flow."""
+  from langchain_openai.chatgpt_oauth import login_chatgpt
+
+  login_chatgpt(timeout=timeout_s, open_browser=True)
+
+
+def build_chat_model(model: str | None = None) -> BaseChatModel:
+  """Build Perferox's OAuth-backed LangChain chat model."""
+  from langchain_openai.chat_models.codex import _ChatOpenAICodex
+  from langchain_openai.chatgpt_oauth import _FileChatGPTOAuthTokenProvider
+
+  provider = _FileChatGPTOAuthTokenProvider.from_default_store()
+  provider.get_token()
+  model_name = model or os.environ.get(MODEL_ENV_VAR, DEFAULT_CHAT_MODEL)
+  return _ChatOpenAICodex(
+    model=model_name,
+    originator=ORIGINATOR,
+    token_provider=provider,
+  )

perferox-0.1.0/perferox/bench.py

@@ -0,0 +1,114 @@
+"""SGLang benchmark command builders and parsers."""
+
+import json
+from typing import Any, Self
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+BENCH_TIMEOUT_S = 6 * 60 * 60.0
+_CONSOLE_METRIC_LABELS = {
+  "Request throughput (req/s)": "request_rps",
+  "Input token throughput (tok/s)": "input_tps",
+  "Output token throughput (tok/s)": "output_tps",
+  "Median TTFT (ms)": "ttft_p50_ms",
+  "P99 TTFT (ms)": "ttft_p99_ms",
+  "Median TPOT (ms)": "tpot_p50_ms",
+  "P99 TPOT (ms)": "tpot_p99_ms",
+  "Cache hit rate": "cache_hit_rate",
+  "Accept length": "accept_length",
+}
+
+class BenchServingArgs(BaseModel):
+  """Typed common inputs plus raw pass-through flags for SGLang's CLI."""
+
+  model_config = ConfigDict(extra="forbid")
+
+  backend: str = "sglang"
+  base_url: str | None = None
+  host: str | None = None
+  port: int | None = Field(None, ge=1, le=65535)
+  ready_check_timeout_sec: int | None = Field(None, ge=0)
+  dataset_name: str = "sharegpt"
+  dataset_path: str | None = None
+  model: str | None = None
+  served_model_name: str | None = None
+  tokenizer: str | None = None
+  num_prompts: int | None = Field(None, ge=1)
+  sharegpt_output_len: int | None = Field(None, ge=4)
+  sharegpt_context_len: int | None = Field(None, ge=1)
+  random_input_len: int | None = Field(None, ge=1)
+  random_output_len: int | None = Field(None, ge=0)
+  request_rate: float | None = Field(None, gt=0)
+  max_concurrency: int | None = Field(None, ge=0)
+  output_file: str | None = None
+  output_details: bool = False
+  print_requests: bool = False
+  disable_tqdm: bool = False
+  disable_stream: bool = False
+  cache_report: bool = False
+  seed: int | None = Field(None, ge=0, le=4294967295)
+  disable_ignore_eos: bool = False
+  temperature: float | None = Field(None, ge=0)
+  top_p: float | None = Field(None, gt=0, le=1)
+  extra_request_body: dict[str, Any] | None = None
+  apply_chat_template: bool = False
+  flush_cache: bool = False
+  warmup_requests: int | None = Field(None, ge=0)
+  tokenize_prompt: bool = False
+  header: dict[str, str] | None = None
+  extra_args: list[str] = Field(default_factory=list)
+  timeout_s: float | None = Field(BENCH_TIMEOUT_S, gt=0)
+
+  @model_validator(mode="after")
+  def check_serving_constraints(self) -> Self:
+    """Mirror bench_serving's parse-time and early runtime argument checks."""
+    if self.backend in ("trt", "truss") and self.model is None:
+      raise ValueError(f"backend='{self.backend}' requires model")
+    if self.print_requests and self.backend != "sglang-oai-chat":
+      raise ValueError("print_requests is only supported with backend='sglang-oai-chat'")
+    if self.tokenize_prompt and self.backend != "sglang":
+      raise ValueError("tokenize_prompt is only supported with backend='sglang'")
+    return self
+
+
+def bench_serving_argv(args: BenchServingArgs) -> list[str]:
+  """Build the SGLang serving benchmark argv from typed fields."""
+  data = args.model_dump(exclude={"timeout_s", "extra_request_body", "header", "extra_args"}, exclude_none=True)
+  data["extra_request_body"] = json.dumps(args.extra_request_body, separators=(",", ":")) if args.extra_request_body else None
+  data["header"] = [f"{key}={value}" for key, value in (args.header or {}).items()]
+  argv = ["python", "-m", "sglang.benchmark.serving"]
+  for name, value in data.items():
+    if value is False or value is None or value == []:
+      continue
+    argv.append(f"--{name.replace('_', '-')}")
+    if value is True:
+      continue
+    argv.extend(map(str, value if isinstance(value, list) else (value,)))
+  argv.extend(args.extra_args)
+  return argv
+
+
+def parse_bench_serving_metrics(output: str, expected_requests: int | None = None) -> dict[str, float]:
+  """Extract Perferox experiment metrics from SGLang benchmark output."""
+  metrics = {}
+  for raw_line in output.splitlines():
+    label, separator, raw_value = raw_line.partition(":")
+    if not separator:
+      continue
+    label = label.strip()
+    metric_name = _CONSOLE_METRIC_LABELS.get(label)
+    if metric_name is None and (label != "Successful requests" or not expected_requests):
+      continue
+    raw_value = raw_value.strip()
+    if not raw_value:
+      continue
+    raw_number = raw_value.split(maxsplit=1)[0].rstrip("%")
+    try:
+      value = float(raw_number)
+    except ValueError:
+      continue
+    if metric_name is None:
+      metrics["error_rate"] = max(expected_requests - value, 0.0) / expected_requests
+      continue
+    metrics[metric_name] = value / 100.0 if metric_name == "cache_hit_rate" else value
+  return metrics

perferox-0.1.0/perferox/cli.py

@@ -0,0 +1,47 @@
+"""Small command-line entry point for Perferox."""
+
+from __future__ import annotations
+
+import argparse
+from contextlib import closing
+from pathlib import Path
+
+from perferox import db
+
+
+def main(argv: list[str] | None = None) -> int:
+  """Open the TUI, launch the main agent, or request a soft stop."""
+  parser = argparse.ArgumentParser(prog="perferox")
+  parser.add_argument("--cwd", type=Path, default=Path("."), help="repository/root directory")
+  parser.add_argument("--db-path", type=Path, default=Path("perferox.sqlite"), help="SQLite state path")
+  parser.add_argument("--trace-dir", type=Path, default=Path("traces"), help="trace directory")
+  subparsers = parser.add_subparsers(dest="command")
+  run_parser = subparsers.add_parser("run", help="start the main graph without opening the TUI")
+  run_parser.add_argument("objective", nargs="+", help="objective for the main agent")
+  subparsers.add_parser("end", help="request a soft stop without opening the TUI")
+  args = parser.parse_args(argv)
+
+  cwd = args.cwd.resolve()
+  db_path = (cwd / args.db_path).resolve()
+  trace_dir = (cwd / args.trace_dir).resolve()
+
+  if args.command is None:
+    from perferox.tui import PerferoxTUI
+
+    PerferoxTUI(cwd=cwd, db_path=db_path, trace_dir=trace_dir).run()
+    return 0
+  if args.command == "run":
+    from perferox.agent_runner import main as run_agent
+
+    objective = " ".join(args.objective)
+    return run_agent(["launch-main", "--db-path", str(db_path), "--trace-dir", str(trace_dir), "--objective", objective, "--cwd", str(cwd)])
+  with closing(db.connect(db_path)) as conn:
+    db.init_db(conn)
+    stopped = db.request_soft_stop(conn)
+    db.append_explorer_state(conn, agent_id=None, line="soft stop requested from CLI")
+  print(f"soft stop requested for {stopped} running session(s)")
+  return 0
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())