agents-remember-mcp 0.1.0__tar.gz

agents_remember_mcp-0.1.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: agents-remember-mcp
+Version: 0.1.0
+Summary: Model Context Protocol server for Agents Remember.
+Requires-Python: >=3.11
+Requires-Dist: mcp<2,>=1.12.4
+Provides-Extra: dev
+Requires-Dist: coverage<8,>=7.14; extra == "dev"
+Requires-Dist: pytest<10,>=9; extra == "dev"
+Requires-Dist: pytest-cov<8,>=7; extra == "dev"
+Requires-Dist: radon<7,>=6; extra == "dev"
+Requires-Dist: ruff<1,>=0.15; extra == "dev"

agents_remember_mcp-0.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "agents-remember-mcp"
+version = "0.1.0"
+description = "Model Context Protocol server for Agents Remember."
+requires-python = ">=3.11"
+dependencies = [
+  "mcp>=1.12.4,<2",
+]
+
+[project.optional-dependencies]
+dev = [
+  "coverage>=7.14,<8",
+  "pytest>=9,<10",
+  "pytest-cov>=7,<8",
+  "radon>=6,<7",
+  "ruff>=0.15,<1",
+]
+
+[project.scripts]
+agents-remember-mcp = "agents_remember.mcp.__main__:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+agents_remember = [
+  "package_data/**/*",
+  "package_data/benchmarks/.gitignore",
+]

agents_remember_mcp-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

agents_remember_mcp-0.1.0/src/agents_remember/__init__.py

@@ -0,0 +1 @@
+"""Agents Remember Python package."""

agents_remember_mcp-0.1.0/src/agents_remember/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark runner helpers for Agents Remember."""

agents_remember_mcp-0.1.0/src/agents_remember/benchmarks/runner.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""Compatibility facade for Agents Remember benchmark prepare/run/analyze tools."""
+
+from __future__ import annotations
+
+import shutil  # noqa: F401 - compatibility surface for existing benchmark tests.
+import subprocess  # noqa: F401 - compatibility surface for existing benchmark tests.
+import sys
+
+from agents_remember.benchmarks.runner_modules import workspace as _workspace
+from agents_remember.benchmarks.runner_modules.analysis import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.cli import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.cli import main
+from agents_remember.benchmarks.runner_modules.commands import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.commands import repo_has_commit, run_command
+from agents_remember.benchmarks.runner_modules.constants import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.execution import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.filesystem import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.filesystem import remove_path
+from agents_remember.benchmarks.runner_modules.manifest import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.mcp_registration import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.models import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.roots import *  # noqa: F403
+from agents_remember.benchmarks.runner_modules.services import *  # noqa: F403
+from agents_remember.providers import provider_setup  # noqa: F401
+
+
+def prepare_repo(repository, repo_root, dry_run, force_clone=False):
+    """Prepare a benchmark repository while preserving facade monkeypatch hooks."""
+    _workspace.run_command = run_command
+    _workspace.remove_path = remove_path
+    _workspace.repo_has_commit = repo_has_commit
+    return _workspace.prepare_repo(repository, repo_root, dry_run, force_clone=force_clone)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

agents_remember_mcp-0.1.0/src/agents_remember/benchmarks/runner_modules/__init__.py

@@ -0,0 +1 @@
+"""Focused implementation modules for the benchmark runner facade."""

agents_remember_mcp-0.1.0/src/agents_remember/benchmarks/runner_modules/analysis.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from agents_remember.benchmarks.runner_modules.constants import TOKEN_KEYS
+from agents_remember.benchmarks.runner_modules.manifest import load_json
+
+
+def walk_values(value: Any) -> list[Any]:
+    values: list[Any] = []
+    pending = [value]
+    while pending:
+        current = pending.pop()
+        values.append(current)
+        if isinstance(current, dict):
+            pending.extend(current.values())
+        elif isinstance(current, list):
+            pending.extend(current)
+    return values
+
+
+def collect_strings(value: Any, keys: set[str]) -> list[str]:
+    found: list[str] = []
+    for current in walk_values(value):
+        if isinstance(current, dict):
+            found.extend(
+                child for key, child in current.items() if key in keys and isinstance(child, str)
+            )
+    return found
+
+
+def update_token_metrics(event: Any, metrics: dict[str, Any]) -> None:
+    for current in walk_values(event):
+        if not isinstance(current, dict):
+            continue
+        for key, value in current.items():
+            update_token_metric(metrics, key, value)
+
+
+def update_token_metric(metrics: dict[str, Any], key: str, value: Any) -> None:
+    metric_key = TOKEN_KEYS.get(key)
+    if metric_key and isinstance(value, int):
+        metrics[metric_key] = max(int(metrics.get(metric_key, 0)), value)
+
+
+def empty_jsonl_metrics(path: Path) -> dict[str, Any]:
+    return {
+        "jsonl_size_bytes": path.stat().st_size,
+        "event_count": 0,
+        "command_event_count": 0,
+        "errors": [],
+        "final_answer": "",
+    }
+
+
+def iter_jsonl_events(path: Path, metrics: dict[str, Any]) -> list[Any]:
+    events: list[Any] = []
+    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+        if not line.strip():
+            continue
+        try:
+            events.append(json.loads(line))
+        except json.JSONDecodeError as error:
+            metrics["errors"].append(f"invalid jsonl line: {error}")
+    return events
+
+
+def event_has_command_marker(event: Any) -> bool:
+    raw = json.dumps(event, sort_keys=True)
+    return any(marker in raw for marker in ("exec_command", "tool_call", '"cmd"', '"command"'))
+
+
+def update_final_answer(metrics: dict[str, Any], event: Any) -> None:
+    for candidate in collect_strings(event, {"content", "text", "message"}):
+        if len(candidate) > len(str(metrics.get("final_answer", ""))):
+            metrics["final_answer"] = candidate
+
+
+def update_event_metrics(metrics: dict[str, Any], event: Any) -> None:
+    metrics["event_count"] += 1
+    if event_has_command_marker(event):
+        metrics["command_event_count"] += 1
+    update_token_metrics(event, metrics)
+    metrics["errors"].extend(
+        error for error in collect_strings(event, {"error", "stderr"}) if error
+    )
+    update_final_answer(metrics, event)
+
+
+def analyze_jsonl(path: Path) -> dict[str, Any]:
+    metrics = empty_jsonl_metrics(path)
+    for event in iter_jsonl_events(path, metrics):
+        update_event_metrics(metrics, event)
+    return metrics
+
+
+def load_metadata(jsonl_path: Path) -> dict[str, Any]:
+    metadata_path = jsonl_path.with_suffix(".metadata.json")
+    if not metadata_path.exists():
+        return {}
+    try:
+        return load_json(metadata_path)
+    except (OSError, ValueError, json.JSONDecodeError):
+        return {}
+
+
+def analyze_run_root(run_root: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for jsonl_path in sorted(run_root.rglob("*.jsonl")):
+        metadata = load_metadata(jsonl_path)
+        metrics = analyze_jsonl(jsonl_path)
+        row = {
+            "path": jsonl_path,
+            "prompt": metadata.get("prompt", jsonl_path.parent.parent.name),
+            "variant": metadata.get("variant", jsonl_path.parent.name),
+            "repetition": metadata.get("repetition", jsonl_path.stem),
+            "duration_seconds": metadata.get("durationSeconds"),
+            "exit_code": metadata.get("exitCode"),
+            **metrics,
+        }
+        rows.append(row)
+    return rows
+
+
+def range_text(values: list[Any]) -> str:
+    clean = [value for value in values if isinstance(value, (int, float))]
+    if not clean:
+        return "n/a"
+    low = min(clean)
+    high = max(clean)
+    if low == high:
+        return str(low)
+    return f"{low} - {high}"
+
+
+def grouped(rows: list[dict[str, Any]]) -> dict[tuple[str, str], list[dict[str, Any]]]:
+    result: dict[tuple[str, str], list[dict[str, Any]]] = {}
+    for row in rows:
+        key = (str(row.get("prompt", "unknown")), str(row.get("variant", "unknown")))
+        result.setdefault(key, []).append(row)
+    return result
+
+
+def summary_numeric_keys() -> list[str]:
+    return [
+        "duration_seconds",
+        "event_count",
+        "command_event_count",
+        "input_tokens",
+        "fresh_input_tokens",
+        "output_tokens",
+        "reasoning_tokens",
+        "jsonl_size_bytes",
+    ]
+
+
+def exit_code_text(group_rows: list[dict[str, Any]]) -> str:
+    exit_codes = sorted(
+        {str(row.get("exit_code")) for row in group_rows if row.get("exit_code") is not None}
+    )
+    return ", ".join(exit_codes) if exit_codes else "n/a"
+
+
+def append_metric_rows(lines: list[str], group_rows: list[dict[str, Any]]) -> None:
+    for key in summary_numeric_keys():
+        lines.append(f"| {key} | {range_text([row.get(key) for row in group_rows])} |")
+    lines.append(f"| exit_code | {exit_code_text(group_rows)} |")
+    lines.append("")
+
+
+def append_run_rows(lines: list[str], group_rows: list[dict[str, Any]]) -> None:
+    lines.extend(["| Run | Duration | JSONL Size | Errors |", "| --- | ---: | ---: | --- |"])
+    for row in group_rows:
+        error_count = len(row.get("errors", []))
+        lines.append(
+            f"| {row.get('repetition')} | {row.get('duration_seconds', 'n/a')} | "
+            f"{row.get('jsonl_size_bytes', 'n/a')} | {error_count} |"
+        )
+    lines.append("")
+
+
+def append_group_summary(
+    lines: list[str],
+    prompt: str,
+    variant: str,
+    group_rows: list[dict[str, Any]],
+) -> None:
+    lines.extend([f"## {prompt} / {variant}", "", "| Metric | Range |", "| --- | --- |"])
+    append_metric_rows(lines, group_rows)
+    append_run_rows(lines, group_rows)
+
+
+def summary_header(run_root: Path) -> list[str]:
+    return [
+        f"# Benchmark Summary: {run_root.name}",
+        "",
+        f"Run root: `{run_root}`",
+        "",
+    ]
+
+
+def summary_markdown(run_root: Path, rows: list[dict[str, Any]]) -> str:
+    lines = summary_header(run_root)
+    if not rows:
+        lines.extend(["No JSONL files found.", ""])
+        return "\n".join(lines)
+    for (prompt, variant), group_rows in grouped(rows).items():
+        append_group_summary(lines, prompt, variant, group_rows)
+    return "\n".join(lines)
+
+
+def write_summary(run_root: Path, rows: list[dict[str, Any]]) -> Path:
+    summary_path = run_root / "summary.md"
+    summary_path.write_text(summary_markdown(run_root, rows), encoding="utf-8")
+    return summary_path

agents_remember_mcp-0.1.0/src/agents_remember/benchmarks/runner_modules/cli.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import argparse
+import subprocess
+from pathlib import Path
+
+from agents_remember.benchmarks.runner_modules.analysis import (
+    analyze_run_root,
+    summary_markdown,
+    write_summary,
+)
+from agents_remember.benchmarks.runner_modules.constants import (
+    CODEX_BENCHMARK_SANDBOX,
+    CODEX_BENCHMARK_SANDBOX_MODES,
+    SKILL_EXPOSURE_MODES,
+)
+from agents_remember.benchmarks.runner_modules.manifest import load_cases
+from agents_remember.benchmarks.runner_modules.models import (
+    BenchmarkPrepareRequest,
+    BenchmarkRunRequest,
+)
+from agents_remember.benchmarks.runner_modules.roots import benchmark_root_context
+from agents_remember.benchmarks.runner_modules.services import (
+    prepare_benchmarks,
+    run_codex_benchmark,
+)
+
+
+def command_list(args: argparse.Namespace) -> int:
+    with benchmark_root_context(args.benchmarks_root) as selected_benchmarks_root:
+        benchmarks_root = selected_benchmarks_root.resolve()
+        for case in load_cases(benchmarks_root):
+            repository = case.repository
+            repo_name = repository.get("name")
+            repo_commit = repository.get("commit")
+            print(
+                f"{case.case_id}\t{case.data.get('status', 'unknown')}\t"
+                f"{case.data.get('sizeBand', 'unknown')}\t{repo_name}@{repo_commit}"
+            )
+    return 0
+
+
+def command_prepare(args: argparse.Namespace) -> int:
+    with benchmark_root_context(args.benchmarks_root) as benchmarks_root:
+        payload = prepare_benchmarks(
+            BenchmarkPrepareRequest(
+                benchmarks_root=benchmarks_root,
+                target=args.target,
+                case_id=args.case_id,
+                dry_run=args.dry_run,
+                skill_exposure_mode=args.skill_exposure_mode,
+                force_clone=args.force_clone,
+                provider_timeout=args.provider_timeout,
+            )
+        )
+    for message in payload["messages"]:
+        print(message)
+    return 0
+
+
+def command_run(args: argparse.Namespace) -> int:
+    with benchmark_root_context(args.benchmarks_root) as benchmarks_root:
+        payload = run_codex_benchmark(
+            BenchmarkRunRequest(
+                benchmarks_root=benchmarks_root,
+                target=args.target,
+                case_id=args.case_id,
+                prompt=args.prompt,
+                variant=args.variant,
+                repetitions=args.repetitions,
+                jobs=args.jobs,
+                dry_run=args.dry_run,
+                skip_prepare=args.skip_prepare,
+                skill_exposure_mode=args.skill_exposure_mode,
+                force_clone=args.force_clone,
+                provider_timeout=args.provider_timeout,
+                codex_sandbox=args.codex_sandbox,
+            )
+        )
+    for message in payload["messages"]:
+        print(message)
+    for output_root in payload["runOutputRoots"]:
+        print(f"Run output: {output_root}")
+    return 0
+
+
+def command_analyze(args: argparse.Namespace) -> int:
+    run_root = args.run_root.resolve()
+    rows = analyze_run_root(run_root)
+    markdown = summary_markdown(run_root, rows)
+    if args.write_summary:
+        path = write_summary(run_root, rows)
+        print(f"Wrote {path}")
+    else:
+        print(markdown)
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--benchmarks-root",
+        type=Path,
+        default=None,
+        help="Benchmark root. Defaults to packaged benchmark cases.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    list_parser = subparsers.add_parser("list", help="List benchmark cases.")
+    list_parser.set_defaults(func=command_list)
+
+    prepare_parser = subparsers.add_parser(
+        "prepare", help="Prepare resettable benchmark workspaces."
+    )
+    prepare_parser.add_argument("target", choices=("all", "case"))
+    prepare_parser.add_argument("case_id", nargs="?")
+    prepare_parser.add_argument(
+        "--skill-exposure-mode", choices=SKILL_EXPOSURE_MODES, default="copy"
+    )
+    prepare_parser.add_argument(
+        "--force-clone",
+        action="store_true",
+        help="Discard existing benchmark repository checkouts before cloning.",
+    )
+    prepare_parser.add_argument(
+        "--provider-timeout",
+        type=int,
+        default=1800,
+        help="Seconds allowed for each benchmark provider install/index command.",
+    )
+    prepare_parser.add_argument("--dry-run", action="store_true")
+    prepare_parser.set_defaults(func=command_prepare)
+
+    run_parser = subparsers.add_parser("run", help="Run benchmark cases with codex exec --json.")
+    run_parser.add_argument("target", choices=("all", "case"))
+    run_parser.add_argument("case_id", nargs="?")
+    run_parser.add_argument("--prompt", help="Run only one prompt id.")
+    run_parser.add_argument("--variant", help="Run only one variant id.")
+    run_parser.add_argument(
+        "--repetitions", type=int, help="Override repetitions per prompt variant."
+    )
+    run_parser.add_argument(
+        "--jobs",
+        type=int,
+        default=None,
+        help="Maximum concurrent Codex runs. Defaults to the number of selected variants.",
+    )
+    run_parser.add_argument("--dry-run", action="store_true")
+    run_parser.add_argument(
+        "--skip-prepare", action="store_true", help="Use the existing workspace fixture state."
+    )
+    run_parser.add_argument("--skill-exposure-mode", choices=SKILL_EXPOSURE_MODES, default="copy")
+    run_parser.add_argument(
+        "--force-clone",
+        action="store_true",
+        help="Discard existing benchmark repository checkouts during preparation.",
+    )
+    run_parser.add_argument(
+        "--provider-timeout",
+        type=int,
+        default=1800,
+        help="Seconds allowed for each benchmark provider install/index command during preparation.",
+    )
+    run_parser.add_argument(
+        "--codex-sandbox",
+        choices=CODEX_BENCHMARK_SANDBOX_MODES,
+        default=CODEX_BENCHMARK_SANDBOX,
+        help=(
+            "Codex benchmark sandbox mode. Use 'default' to omit --sandbox and "
+            "let the Codex CLI use its configured default."
+        ),
+    )
+    run_parser.set_defaults(func=command_run)
+
+    analyze_parser = subparsers.add_parser(
+        "analyze", help="Analyze an existing user-runs directory."
+    )
+    analyze_parser.add_argument("run_root", type=Path)
+    analyze_parser.add_argument("--write-summary", action="store_true")
+    analyze_parser.set_defaults(func=command_analyze)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return int(args.func(args))
+    except (RuntimeError, ValueError, subprocess.CalledProcessError) as error:
+        parser.error(str(error))
+    return 1

agents_remember_mcp-0.1.0/src/agents_remember/benchmarks/runner_modules/commands.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+def run_command(command: list[str], dry_run: bool, cwd: Path | None = None) -> None:
+    printable = " ".join(command)
+    if dry_run:
+        location = f" in {cwd}" if cwd else ""
+        print(f"Would run{location}: {printable}")
+        return
+    subprocess.run(command, cwd=cwd, check=True)
+
+
+def git_command(*args: str) -> list[str]:
+    return ["git", "-c", "core.longpaths=true", "-c", "safe.directory=*", *args]
+
+
+def repo_has_commit(repo_root: Path, commit: str) -> bool:
+    result = subprocess.run(
+        git_command("-C", str(repo_root), "cat-file", "-e", f"{commit}^{{commit}}"),
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        check=False,
+    )
+    return result.returncode == 0

agents_remember_mcp-0.1.0/src/agents_remember/benchmarks/runner_modules/constants.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+AGENTS_MD_TARGETS = {
+    Path("runtime/agents-md-files/coordinator/AGENTS.md"): Path("AGENTS.md"),
+    Path("runtime/agents-md-files/system/AGENTS.md"): Path("system/AGENTS.md"),
+    Path("runtime/agents-md-files/skills/AGENTS.md"): Path("skills/AGENTS.md"),
+    Path("runtime/agents-md-files/tasks/AGENTS.md"): Path("tasks/AGENTS.md"),
+}
+PROVIDER_ASSET_DIRS = (Path("requirements"), Path("patches"))
+BENCHMARK_PROVIDER_IDS = ("grepai-memory", "codegraphcontext-code")
+
+TOKEN_KEYS = {
+    "input_tokens": "input_tokens",
+    "inputTokens": "input_tokens",
+    "total_input_tokens": "input_tokens",
+    "totalInputTokens": "input_tokens",
+    "fresh_input_tokens": "fresh_input_tokens",
+    "freshInputTokens": "fresh_input_tokens",
+    "output_tokens": "output_tokens",
+    "outputTokens": "output_tokens",
+    "total_output_tokens": "output_tokens",
+    "totalOutputTokens": "output_tokens",
+    "reasoning_tokens": "reasoning_tokens",
+    "reasoningTokens": "reasoning_tokens",
+}
+
+WORKSPACE_AGENTS_TEMPLATE = Path("templates/workspace-AGENTS.md")
+SOURCE_ONLY_AGENTS_TEMPLATE = Path("templates/source-only-AGENTS.md")
+BENCHMARK_ROOT_MARKER = ".benchmark-root"
+SKILLS_EXPOSURE_NAMESPACE = "agents-remember-md"
+CODEX_HARNESS_DIR = ".codex"
+COPYTREE_IGNORE = shutil.ignore_patterns("__pycache__", "*.pyc", "*.pyo")
+SKILL_EXPOSURE_MODES = ("copy", "none")
+CODEX_EXECUTABLE_NAME = "codex"
+CODEX_EXECUTABLE_RESOLUTION = "PATH"
+CODEX_SANDBOX_DANGER_FULL_ACCESS = "danger-full-access"
+CODEX_SANDBOX_DEFAULT = "default"
+CODEX_BENCHMARK_SANDBOX = CODEX_SANDBOX_DANGER_FULL_ACCESS
+CODEX_BENCHMARK_SANDBOX_MODES = (CODEX_SANDBOX_DANGER_FULL_ACCESS, CODEX_SANDBOX_DEFAULT)
+CODEX_BENCHMARK_SCOPE = "codex-benchmark-only"
+BENCHMARK_MCP_SERVER_NAME = "agents_remember_benchmark"
+BENCHMARK_MCP_SETTINGS_NAME = "agents-remember-benchmark.settings.json"
+BENCHMARK_MCP_STARTUP_TIMEOUT_SECONDS = 120
+BENCHMARK_MCP_SOURCE_ENV = "AGENTS_REMEMBER_BENCHMARK_MCP_SRC"