agentperf-local 0.1.0__py3-none-any.whl

agentperf_local/__init__.py

@@ -0,0 +1,3 @@
+"""Replay recorded agent tasks against a local model endpoint."""
+
+__version__ = "0.1.0"

agentperf_local/__main__.py

@@ -0,0 +1,6 @@
+"""Run the agentperf-local command line interface."""
+
+from agentperf_local.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentperf_local/cli/__init__.py

@@ -0,0 +1,51 @@
+"""Run one agentperf-local command.
+
+Every subcommand is defined in a sibling module and named here once.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Callable, Sequence
+
+from agentperf_local.cli.inspect import deployment_options_command, doctor_command
+from agentperf_local.cli.options import INTERRUPTED_STATUS
+from agentperf_local.cli.parsers import DEFAULT_COMMAND, build_parser
+from agentperf_local.cli.replay import managed_run_command, run_command, tui_command
+from agentperf_local.cli.submit import (
+    prepare_submission_command,
+    submission_status_command,
+    submit_command,
+)
+from agentperf_local.cli.workload import convert_command
+from agentperf_local.common.argparse_fields import read_string
+
+COMMANDS: dict[str, Callable[[argparse.Namespace], int]] = {
+    "convert": convert_command,
+    "run": run_command,
+    DEFAULT_COMMAND: tui_command,
+    "doctor": doctor_command,
+    "deployment-options": deployment_options_command,
+    "managed-run": managed_run_command,
+    "prepare-submission": prepare_submission_command,
+    "submit": submit_command,
+    "submission-status": submission_status_command,
+}
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run one command and return its process status.
+
+    An empty command line opens the TUI with its default settings.
+    """
+    arguments = list(sys.argv[1:] if argv is None else argv)
+    namespace = build_parser().parse_args(arguments or [DEFAULT_COMMAND])
+    try:
+        return COMMANDS[read_string(namespace, "command")](namespace)
+    except KeyboardInterrupt:
+        print("error: interrupted", file=sys.stderr)
+        return INTERRUPTED_STATUS
+    except Exception as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1

agentperf_local/cli/inspect.py

@@ -0,0 +1,55 @@
+"""Report local facts: hardware and managed deployment choices."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from agentperf_local.cli.options import (
+    print_json,
+    read_bound_device,
+    read_managed_target,
+)
+from agentperf_local.common.argparse_fields import (
+    read_boolean,
+)
+from agentperf_local.deployment.frameworks import framework_offers
+from agentperf_local.provenance.hardware import (
+    collect_hardware_snapshot,
+)
+from agentperf_local.reports.progress import render_hardware
+
+
+def doctor_command(namespace: argparse.Namespace) -> int:
+    snapshot = collect_hardware_snapshot()
+    if read_boolean(namespace, "json"):
+        print_json(snapshot.to_json())
+    else:
+        print(render_hardware(snapshot))
+    # A benchmark run binds exactly one accelerator, so any other count is a blocking finding.
+    accelerator_count = len(snapshot.accelerators)
+    if accelerator_count == 1:
+        return 0
+    print(f"doctor requires exactly one detected accelerator; found {accelerator_count}", file=sys.stderr)
+    return 1
+
+
+def deployment_options_command(namespace: argparse.Namespace) -> int:
+    candidate = read_managed_target(namespace).candidate
+    bound = read_bound_device(namespace)
+    snapshot = bound.snapshot
+    offers = framework_offers(snapshot, candidate)
+    can_deploy_now = any(offer.installed and offer.memory_fit is True for offer in offers)
+    print_json(
+        {
+            "kind": "managed_deployment_options",
+            "profile_id": candidate.profile_id,
+            "model": candidate.hf_repository,
+            "device_index": bound.device_index,
+            "hardware": snapshot.to_json(),
+            "offers": [offer.to_json() for offer in offers],
+            "can_deploy_now": can_deploy_now,
+            "upload_performed": False,
+        }
+    )
+    return 0 if can_deploy_now else 1

agentperf_local/cli/options.py

@@ -0,0 +1,243 @@
+"""Read and validate one parsed command line, one value at a time."""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+import orjson
+
+from agentperf_local.client.backends import CLIENT_BACKENDS, ClientBackend
+from agentperf_local.common.argparse_fields import (
+    read_optional_integer,
+    read_optional_number,
+    read_optional_path,
+    read_optional_string,
+    read_path,
+    read_string,
+)
+from agentperf_local.common.json_fields import one_of
+from agentperf_local.common.json_types import JsonObject
+from agentperf_local.deployment.catalog import (
+    BUNDLED_MODEL_CATALOG_PATH,
+    DEPLOYMENT_FRAMEWORK_ORDER,
+    DeploymentFramework,
+    ModelCandidate,
+    ModelCatalog,
+    ModelDeployment,
+    load_model_catalog,
+)
+from agentperf_local.deployment.managed import (
+    DEPLOYMENT_LOG_FILENAME,
+    DEPLOYMENT_RECORD_FILENAME,
+    BoundDeploymentDevice,
+    bind_snapshot_to_device,
+)
+from agentperf_local.deployment.qualification import (
+    QUALIFICATION_FILENAME,
+)
+from agentperf_local.provenance.benchmark import (
+    MEASUREMENT_BINDING_FILENAME,
+)
+from agentperf_local.provenance.hardware import (
+    collect_hardware_snapshot,
+)
+from agentperf_local.replay.config import (
+    API_KEY_ENV_PATTERN,
+    DEFAULT_OUTPUT_TOKEN_MARGIN,
+    DEFAULT_TOOL_DELAY_SCALE,
+    OUTPUT_TOKEN_POLICIES,
+    OutputTokenPolicy,
+    SamplingPreset,
+    ToolReplayMode,
+    read_api_key_env,
+)
+from agentperf_local.reports.reporting import (
+    FAILURES_FILENAME,
+    SUMMARY_FILENAME,
+    TASKS_FILENAME,
+    TOOLS_FILENAME,
+    TURNS_FILENAME,
+    validate_run_artifact_output,
+)
+from agentperf_local.submission.client import (
+    read_submit_token,
+)
+from agentperf_local.telemetry.power import (
+    POWER_SUMMARY_FILENAME,
+    TELEMETRY_FILENAME,
+)
+from agentperf_local.workload.bundled import find_bundled_replay
+from agentperf_local.workload.schema import MessageSource
+
+MESSAGE_SOURCES: tuple[MessageSource, ...] = ("provider-request", "request-messages")
+
+
+SAMPLING_PRESETS: tuple[SamplingPreset, ...] = ("standard", "custom")
+
+
+TOOL_REPLAY_MODES: tuple[ToolReplayMode, ...] = ("none", "recorded", "live")
+
+
+DEFAULT_MANAGED_PROFILE_ID = "gemma4-12b-it-q4-0"
+
+
+BOUND_RESULT_FILENAMES = (
+    MEASUREMENT_BINDING_FILENAME,
+    SUMMARY_FILENAME,
+    TURNS_FILENAME,
+    TASKS_FILENAME,
+    TOOLS_FILENAME,
+    FAILURES_FILENAME,
+    DEPLOYMENT_RECORD_FILENAME,
+    TELEMETRY_FILENAME,
+    POWER_SUMMARY_FILENAME,
+    QUALIFICATION_FILENAME,
+    DEPLOYMENT_LOG_FILENAME,
+)
+
+
+DEFAULT_MODEL_CATALOG_PATH = BUNDLED_MODEL_CATALOG_PATH
+
+
+# Shells report an interrupted process as 128 plus the signal number, and SIGINT is signal 2.
+INTERRUPTED_STATUS = 130
+
+
+def nonempty_path(value: str) -> Path:
+    """Parse one path argument, refusing the empty value.
+
+    An empty string becomes the current directory, which silently scatters new files
+    into the working tree; argparse prefixes the offending argument's name itself.
+    """
+    if not value:
+        raise argparse.ArgumentTypeError("must not be empty")
+    return Path(value)
+
+
+def read_api_key(namespace: argparse.Namespace) -> str | None:
+    """Read one explicitly named endpoint secret."""
+    name = read_optional_string(namespace, "api_key_env")
+    if name is None:
+        return None
+    if API_KEY_ENV_PATTERN.fullmatch(name) is None:
+        raise ValueError("--api-key-env must name a portable environment variable")
+    value = read_api_key_env(name)
+    if value is None:
+        raise ValueError(f"API key environment variable {name} is not set, is empty, or contains whitespace")
+    return value
+
+
+def read_message_source(namespace: argparse.Namespace) -> MessageSource:
+    return one_of(read_string(namespace, "message_source"), MESSAGE_SOURCES, "--message-source")
+
+
+def read_client_backend(namespace: argparse.Namespace) -> ClientBackend:
+    return one_of(read_string(namespace, "client"), CLIENT_BACKENDS, "--client")
+
+
+def read_requested_output_token_policy(namespace: argparse.Namespace) -> OutputTokenPolicy | None:
+    """Return the policy the user passed, or None when they left the choice to the command."""
+    policy = read_optional_string(namespace, "output_token_policy")
+    return None if policy is None else one_of(policy, OUTPUT_TOKEN_POLICIES, "--output-token-policy")
+
+
+def read_replay_manifest_path(namespace: argparse.Namespace) -> Path:
+    """Resolve the custom manifest, or the bundled replay's manifest for this host."""
+    manifest = read_optional_path(namespace, "manifest")
+    if manifest is not None:
+        return manifest
+    replay_id = read_string(namespace, "replay")
+    replay = find_bundled_replay(replay_id)
+    if replay is None:
+        raise ValueError(f"unknown bundled replay {replay_id}")
+    return replay.manifest_path
+
+
+def read_sampling_preset(namespace: argparse.Namespace) -> SamplingPreset:
+    return one_of(read_string(namespace, "sampling"), SAMPLING_PRESETS, "--sampling")
+
+
+def read_tool_mode(namespace: argparse.Namespace) -> ToolReplayMode:
+    return one_of(read_string(namespace, "tool_mode"), TOOL_REPLAY_MODES, "--tool-mode")
+
+
+def read_output_token_margin(namespace: argparse.Namespace, policy: OutputTokenPolicy) -> int:
+    """Resolve the recorded-target margin and reject it under any other policy."""
+    margin = read_optional_integer(namespace, "output_token_margin")
+    if margin is None:
+        return DEFAULT_OUTPUT_TOKEN_MARGIN
+    if policy != "recorded":
+        raise ValueError("output token margin applies only to the recorded policy")
+    return margin
+
+
+def read_tool_delay_scale(namespace: argparse.Namespace, tool_mode: ToolReplayMode) -> float:
+    """Resolve the recorded delay scale and reject it without tool replay."""
+    scale = read_optional_number(namespace, "tool_delay_scale")
+    if scale is None:
+        return DEFAULT_TOOL_DELAY_SCALE
+    if tool_mode == "none":
+        raise ValueError("tool delay scale requires a tool replay mode")
+    return scale
+
+
+def read_live_workspace_root(namespace: argparse.Namespace, tool_mode: ToolReplayMode) -> Path | None:
+    """Resolve the live workspace root, defaulting live runs to a folder inside the output directory."""
+    root = read_optional_path(namespace, "live_workspace_root")
+    if root is not None or tool_mode != "live":
+        return root
+    return read_path(namespace, "output_dir") / "workspaces"
+
+
+def read_deployment_framework(namespace: argparse.Namespace) -> DeploymentFramework:
+    return one_of(read_string(namespace, "framework"), DEPLOYMENT_FRAMEWORK_ORDER, "--framework")
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ManagedTarget:
+    """Name the catalog, candidate, and recipe one managed command targets."""
+
+    catalog: ModelCatalog
+    candidate: ModelCandidate
+    deployment: ModelDeployment
+
+
+def read_managed_target(namespace: argparse.Namespace) -> ManagedTarget:
+    catalog = load_model_catalog(read_path(namespace, "catalog"))
+    candidate = catalog_candidate(catalog, read_string(namespace, "profile_id"))
+    if candidate.deployment is None:
+        raise ValueError(f"model profile {candidate.profile_id} does not define a managed deployment")
+    return ManagedTarget(catalog=catalog, candidate=candidate, deployment=candidate.deployment)
+
+
+def read_bound_device(namespace: argparse.Namespace) -> BoundDeploymentDevice:
+    return bind_snapshot_to_device(collect_hardware_snapshot(), read_optional_integer(namespace, "device"))
+
+
+def require_fresh_output_dir(output_dir: Path, label: str) -> None:
+    """Refuse an output directory that already holds any bound result artifact."""
+    validate_run_artifact_output(output_dir)
+    existing = tuple(
+        name for name in BOUND_RESULT_FILENAMES if (output_dir / name).exists() or (output_dir / name).is_symlink()
+    )
+    if existing:
+        raise ValueError(f"{label} output must be fresh; existing artifacts: {', '.join(existing)}")
+
+
+def catalog_candidate(catalog: ModelCatalog, profile_id: str) -> ModelCandidate:
+    candidate = next((model for model in catalog.models if model.profile_id == profile_id), None)
+    if candidate is None:
+        available = ", ".join(sorted(model.profile_id for model in catalog.models))
+        raise ValueError(f"model catalog does not contain profile {profile_id}; available: {available}")
+    return candidate
+
+
+def print_json(value: JsonObject) -> None:
+    print(orjson.dumps(value, option=orjson.OPT_INDENT_2).decode("utf-8"))
+
+
+def resolve_submit_token(namespace: argparse.Namespace, attribute: str = "token_env") -> str | None:
+    """Read the submit token from the named variable; an unset variable means anonymous."""
+    return read_submit_token(read_string(namespace, attribute))