clawperf 0.1.0__py3-none-any.whl

clawperf/__init__.py

@@ -0,0 +1,8 @@
+"""ClawPerf - Performance benchmarking tool for LLM Serving backends.
+
+Reuses EvalScope's perf infrastructure for HTTP, streaming, and timing,
+and adds multi-turn long-context workloads with append-mode compaction,
+user arrival scheduling, and system metrics polling.
+"""
+
+__version__ = "0.1.0"

clawperf/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for running ClawPerfBench as a module: python -m clawperf"""
+
+from clawperf.cli import main
+
+if __name__ == "__main__":
+    main()

clawperf/cli.py

@@ -0,0 +1,107 @@
+"""CLI argument parser for ClawPerf."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+
+from clawperf.config import BenchmarkConfig
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="clawperf",
+        description=(
+            "ClawPerf - Performance testing tool for LLM Serving backends. "
+            "Simulates multi-user, multi-turn, long-context workloads against "
+            "vLLM, SGLang, and MindIE backends. "
+            "Built on EvalScope's perf infrastructure."
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    # ── User configuration ──
+    g = parser.add_argument_group("User Configuration")
+    g.add_argument("--num-users", type=int, default=1, help="Total concurrent users.")
+    g.add_argument(
+        "--user-arrival", type=str, default="burst",
+        help="'burst', 'steady:<seconds>', or 'poisson:<lambda>'.",
+    )
+
+    # ── Context configuration ──
+    g = parser.add_argument_group("Context Configuration")
+    g.add_argument("--system-prefix-tokens", type=int, default=15000)
+    g.add_argument("--system-prefix-source", type=str, default="random")
+    g.add_argument("--user-prefix-tokens", type=int, default=5000)
+    g.add_argument("--input-tokens-per-turn", type=int, default=5000)
+    g.add_argument("--output-tokens-per-turn", type=int, default=1000)
+    g.add_argument("--max-context-tokens", type=int, default=128000)
+    g.add_argument("--compaction-prefix-increment", type=int, default=5000)
+
+    # ── Run configuration ──
+    g = parser.add_argument_group("Run Configuration")
+    g.add_argument("--max-turns", type=int, default=100)
+
+    # ── API configuration ──
+    g = parser.add_argument_group("API Configuration")
+    g.add_argument("--endpoint", type=str, required=True)
+    g.add_argument("--model", type=str, required=True)
+    g.add_argument("--api-key", type=str, default="")
+    g.add_argument("--tokenizer", type=str, default="")
+    g.add_argument("--ignore-eos", action="store_true", default=True)
+    g.add_argument("--no-ignore-eos", action="store_false", dest="ignore_eos")
+    g.add_argument("--request-timeout", type=int, default=600)
+
+    # ── System metrics ──
+    g = parser.add_argument_group("System Metrics")
+    g.add_argument("--metrics-endpoint", type=str, default=None)
+    g.add_argument("--metrics-interval", type=int, default=5)
+    g.add_argument("--backend", type=str, default="vllm", choices=["vllm", "sglang", "mindie"])
+
+    # ── Output ──
+    g = parser.add_argument_group("Output")
+    g.add_argument("--output", type=str, default="results.json")
+    g.add_argument("-v", "--verbose", action="store_true", default=False,
+                   help="Print per-turn progress lines (default: tqdm progress bar)")
+
+    return parser
+
+
+def parse_args(argv: list[str] | None = None) -> BenchmarkConfig:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return BenchmarkConfig(
+        num_users=args.num_users,
+        user_arrival=args.user_arrival,
+        system_prefix_tokens=args.system_prefix_tokens,
+        system_prefix_source=args.system_prefix_source,
+        user_prefix_tokens=args.user_prefix_tokens,
+        input_tokens_per_turn=args.input_tokens_per_turn,
+        output_tokens_per_turn=args.output_tokens_per_turn,
+        max_context_tokens=args.max_context_tokens,
+        compaction_prefix_increment=args.compaction_prefix_increment,
+        max_turns=args.max_turns,
+        endpoint=args.endpoint,
+        model=args.model,
+        api_key=args.api_key,
+        tokenizer=args.tokenizer,
+        ignore_eos=args.ignore_eos,
+        request_timeout=args.request_timeout,
+        metrics_endpoint=args.metrics_endpoint,
+        metrics_interval=args.metrics_interval,
+        backend=args.backend,
+        output=args.output,
+        verbose=args.verbose,
+    )
+
+
+def main():
+    config = parse_args()
+    from clawperf.runner import BenchmarkRunner
+
+    runner = BenchmarkRunner(config)
+    try:
+        asyncio.run(runner.run())
+    except KeyboardInterrupt:
+        print("\n[ClawPerf] Interrupted. Saving partial results...")
+        asyncio.run(runner.shutdown_and_save())

clawperf/config.py

@@ -0,0 +1,107 @@
+"""Configuration for ClawPerfBench.
+
+Wraps EvalScope's Arguments where possible, extends with
+ClawPerfBench-specific context/compaction/scheduling parameters.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Optional
+
+
+@dataclasses.dataclass
+class BenchmarkConfig:
+    """All configurable parameters for a benchmark run."""
+
+    # ── User configuration ──
+    num_users: int = 1
+    user_arrival: str = "burst"  # "burst", "steady:<interval>", "poisson:<lambda>"
+
+    # ── Context configuration ──
+    system_prefix_tokens: int = 15000
+    system_prefix_source: str = "random"  # "random" or file path
+    user_prefix_tokens: int = 5000
+    input_tokens_per_turn: int = 5000
+    output_tokens_per_turn: int = 1000
+    max_context_tokens: int = 128000
+    compaction_prefix_increment: int = 5000
+
+    # ── Run configuration ──
+    max_turns: int = 100
+
+    # ── API configuration ──
+    endpoint: str = ""
+    model: str = ""
+    api_key: str = ""
+    tokenizer: str = ""  # defaults to model if empty
+    ignore_eos: bool = True
+    request_timeout: int = 600
+
+    # ── System metrics configuration ──
+    metrics_endpoint: Optional[str] = None
+    metrics_interval: int = 5
+    backend: str = "vllm"  # "vllm", "sglang", "mindie"
+
+    # ── Output configuration ──
+    output: str = "results.json"
+    verbose: bool = False  # per-turn detailed logging
+
+    # ── Derived fields ──
+    arrival_mode: str = ""
+    arrival_param: float = 0.0
+
+    def __post_init__(self):
+        if not self.tokenizer:
+            self.tokenizer = self.model
+        self._parse_arrival_mode()
+
+    def _parse_arrival_mode(self):
+        if self.user_arrival == "burst":
+            self.arrival_mode = "burst"
+            self.arrival_param = 0.0
+        elif self.user_arrival.startswith("steady:"):
+            self.arrival_mode = "steady"
+            self.arrival_param = float(self.user_arrival.split(":")[1])
+        elif self.user_arrival.startswith("poisson:"):
+            self.arrival_mode = "poisson"
+            self.arrival_param = float(self.user_arrival.split(":")[1])
+        else:
+            raise ValueError(
+                f"Invalid user_arrival format: {self.user_arrival!r}. "
+                "Expected 'burst', 'steady:<interval>', or 'poisson:<lambda>'."
+            )
+
+    def to_evalscope_args(self):
+        """Build an EvalScope Arguments object from this config.
+
+        Reuses EvalScope's connection/timeout/stream/model settings.
+        Note: EvalScope's number/parallel are lists after validation.
+        """
+        from evalscope.perf.arguments import Arguments
+
+        args = Arguments(
+            model=self.model,
+            url=self.endpoint,
+            tokenizer_path=self.tokenizer,
+            stream=True,
+            max_tokens=self.output_tokens_per_turn,
+            number=[self.num_users],
+            parallel=[self.num_users],
+            total_timeout=self.request_timeout,
+            api="openai",
+            no_test_connection=True,
+        )
+        if self.api_key:
+            args.headers["Authorization"] = f"Bearer {self.api_key}"
+
+        # Handle ignore_eos via extra_args
+        if self.ignore_eos:
+            extra = dict(args.extra_args) if args.extra_args else {}
+            extra["ignore_eos"] = True
+            args.extra_args = extra
+
+        return args
+
+    def to_dict(self) -> dict:
+        return dataclasses.asdict(self)

clawperf/context.py

@@ -0,0 +1,138 @@
+"""Context assembly and compaction logic.
+
+Implements the per-turn context structure:
+  [System Prefix] [User Prefix] [History] [Current Input]
+
+And the append-mode compaction mechanism.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import List, Tuple
+
+logger = logging.getLogger("clawperf")
+
+
+@dataclass
+class CompactionEvent:
+    user_id: int
+    turn: int
+    time: float
+    old_prefix_len: int
+    new_prefix_len: int
+
+
+@dataclass
+class UserContext:
+    """Maintains the evolving context state for a single user."""
+
+    user_id: int
+    system_prefix: str
+    user_prefix_tokens: int
+    user_prefix_content: str
+    input_tokens_per_turn: int
+    max_context_tokens: int
+    compaction_prefix_increment: int
+    max_turns: int
+
+    history: List[Tuple[str, str]] = field(default_factory=list)
+    compaction_events: List[CompactionEvent] = field(default_factory=list)
+
+    def prepare_turn(
+        self,
+        turn_id: int,
+        current_input_content: str,
+        tokenizer_manager,
+    ) -> dict:
+        """Prepare context for a turn. May trigger compaction.
+
+        Returns dict with: messages, context_tokens, compaction_triggered,
+        compaction_event, context_overflow.
+        """
+        compaction_event = None
+        context_overflow = False
+        messages = self._build_messages(current_input_content)
+        context_tokens = tokenizer_manager.count_chat_tokens(messages)
+
+        if context_tokens >= self.max_context_tokens:
+            # Check if base context (without history) already exceeds limit
+            # — compaction can't help if system+prefix+input alone is too large
+            base_messages = self._build_messages(current_input_content, skip_history=True)
+            base_tokens = tokenizer_manager.count_chat_tokens(base_messages)
+            if base_tokens >= self.max_context_tokens:
+                context_overflow = True
+                logger.warning(
+                    "[User %02d] Base context (%d tokens) already exceeds limit (%d) "
+                    "— compaction cannot help, skipping turn %d",
+                    self.user_id, base_tokens, self.max_context_tokens, turn_id,
+                )
+            else:
+                old_prefix_len = self.user_prefix_tokens
+                self.history.clear()
+                self.user_prefix_tokens += self.compaction_prefix_increment
+                self.user_prefix_content = tokenizer_manager.generate_random_content(
+                    self.user_prefix_tokens
+                )
+                compaction_event = CompactionEvent(
+                    user_id=self.user_id,
+                    turn=turn_id,
+                    time=0.0,
+                    old_prefix_len=old_prefix_len,
+                    new_prefix_len=self.user_prefix_tokens,
+                )
+                self.compaction_events.append(compaction_event)
+                logger.info(
+                    "[User %02d] Compaction at turn %d: prefix %d → %d",
+                    self.user_id, turn_id, old_prefix_len, self.user_prefix_tokens,
+                )
+                messages = self._build_messages(current_input_content)
+                context_tokens = tokenizer_manager.count_chat_tokens(messages)
+
+                if context_tokens >= self.max_context_tokens:
+                    context_overflow = True
+                    logger.warning(
+                        "[User %02d] Context still exceeds limit after compaction: "
+                        "%d >= %d tokens",
+                        self.user_id, context_tokens, self.max_context_tokens,
+                    )
+
+        return {
+            "messages": messages,
+            "context_tokens": context_tokens,
+            "compaction_triggered": compaction_event is not None,
+            "compaction_event": compaction_event,
+            "context_overflow": context_overflow,
+        }
+
+    def _build_messages(self, current_input: str, skip_history: bool = False) -> list[dict]:
+        messages = []
+        if self.system_prefix:
+            messages.append({"role": "system", "content": self.system_prefix})
+
+        user_parts = []
+        if self.user_prefix_content:
+            user_parts.append(self.user_prefix_content)
+
+        for i, (user_msg, assistant_msg) in enumerate(self.history):
+            if skip_history:
+                continue
+            if i == 0 and user_parts:
+                user_parts.append(user_msg)
+                messages.append({"role": "user", "content": "\n".join(user_parts)})
+                user_parts = []
+            else:
+                messages.append({"role": "user", "content": user_msg})
+            messages.append({"role": "assistant", "content": assistant_msg})
+
+        if user_parts:
+            user_parts.append(current_input)
+            messages.append({"role": "user", "content": "\n".join(user_parts)})
+        else:
+            messages.append({"role": "user", "content": current_input})
+
+        return messages
+
+    def append_history(self, user_message: str, assistant_reply: str):
+        self.history.append((user_message, assistant_reply))