pluginbench 0.5.0__py3-none-any.whl

pluginbench/native_treatment_runtime.js

@@ -0,0 +1,221 @@
+// Executed inside the attempt's private environment, before the coding command.
+// This uses the CLI's installation and discovery APIs; it never starts a model turn.
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const crypto = require("node:crypto");
+const { spawn, execFileSync } = require("node:child_process");
+const readline = require("node:readline");
+
+const [root, cwd] = process.argv.slice(2);
+const payload = JSON.parse(fs.readFileSync(path.join(root, "treatment.json"), "utf8"));
+const codexHome = fs.realpathSync(process.env.CODEX_HOME);
+const home = fs.realpathSync(process.env.HOME);
+const hash = bytes => crypto.createHash("sha256").update(bytes).digest("hex");
+const check = (condition, message) => { if (!condition) throw new Error(message); };
+const canonical = value => fs.realpathSync(value);
+const inside = (value, parent) => value === parent || value.startsWith(parent + path.sep);
+
+function cli(args) {
+  return JSON.parse(execFileSync("codex", args, {
+    cwd, env: process.env, encoding: "utf8", timeout: 30000, maxBuffer: 8 * 1024 * 1024,
+  }));
+}
+
+function treeEntries(directory, relative = ".") {
+  const file = path.join(directory, relative);
+  const stat = fs.lstatSync(file);
+  check(!stat.isSymbolicLink(), "Installed treatment contains a symlink: " + file);
+  check(stat.isDirectory() || stat.isFile(), "Installed treatment has a nonregular asset: " + file);
+  const entry = { path: relative, kind: stat.isDirectory() ? "directory" : "file", mode: stat.mode & 0o7777 };
+  if (stat.isFile()) {
+    const bytes = fs.readFileSync(file);
+    Object.assign(entry, { sha256: hash(bytes), size: bytes.length });
+  }
+  const entries = [entry];
+  if (stat.isDirectory()) {
+    for (const name of fs.readdirSync(file).sort()) {
+      entries.push(...treeEntries(directory, relative === "." ? name : relative + "/" + name));
+    }
+  }
+  return entries;
+}
+
+function verifyTree(directory, expected) {
+  const observed = treeEntries(directory);
+  const expectedByPath = new Map(expected.entries.map(entry => [entry.path, entry]));
+  check(observed.length === expectedByPath.size, "Installed treatment inventory differs: " + directory);
+  // Harbor's upload step grants readability. Restore and check the frozen modes.
+  for (const entry of observed) {
+    const original = expectedByPath.get(entry.path);
+    check(original && original.kind === entry.kind && original.sha256 === entry.sha256
+      && original.size === entry.size, "Installed treatment bytes differ: " + directory + "/" + entry.path);
+    fs.chmodSync(path.join(directory, entry.path), original.mode);
+    check((fs.statSync(path.join(directory, entry.path)).mode & 0o7777) === original.mode,
+      "Installed treatment mode differs: " + directory + "/" + entry.path);
+  }
+}
+
+async function discover() {
+  const child = spawn("codex", ["app-server"], { cwd, env: process.env, stdio: ["pipe", "pipe", "pipe"] });
+  const pending = new Map();
+  let nextId = 1;
+  let stderr = "";
+  child.stderr.on("data", bytes => { stderr += bytes.toString(); });
+  const lines = readline.createInterface({ input: child.stdout });
+  const fail = error => {
+    for (const item of pending.values()) item.reject(error);
+    pending.clear();
+  };
+  child.on("error", fail);
+  child.on("exit", code => fail(new Error("Native discovery exited: " + code + " " + stderr.slice(0, 1000))));
+  lines.on("line", line => {
+    let message;
+    try { message = JSON.parse(line); } catch { fail(new Error("Malformed native discovery JSON")); return; }
+    if (!Object.hasOwn(message, "id")) return;
+    const item = pending.get(message.id);
+    if (!item) return;
+    pending.delete(message.id);
+    if (message.error) item.reject(new Error(JSON.stringify(message.error)));
+    else item.resolve(message.result);
+  });
+  const request = (method, params) => new Promise((resolve, reject) => {
+    const id = nextId++;
+    const timer = setTimeout(() => {
+      pending.delete(id);
+      reject(new Error("Native discovery timed out: " + method));
+    }, 20000);
+    pending.set(id, {
+      resolve: value => { clearTimeout(timer); resolve(value); },
+      reject: error => { clearTimeout(timer); reject(error); },
+    });
+    child.stdin.write(JSON.stringify({ id, method, params }) + "\n");
+  });
+  try {
+    await request("initialize", { clientInfo: { name: "pluginbench", version: "1" }, capabilities: { experimentalApi: true } });
+    child.stdin.write(JSON.stringify({ method: "initialized", params: {} }) + "\n");
+    const skills = await request("skills/list", { cwds: [cwd], forceReload: true });
+    const hooks = await request("hooks/list", { cwds: [cwd] });
+    return { skills, hooks };
+  } finally {
+    lines.close();
+    child.stdin.end();
+    if (child.exitCode === null && child.signalCode === null) {
+      await new Promise(resolve => {
+        const timer = setTimeout(() => { child.kill("SIGKILL"); }, 1000);
+        child.once("exit", () => { clearTimeout(timer); resolve(); });
+        child.kill("SIGTERM");
+      });
+    }
+  }
+}
+
+async function main() {
+  const guidancePath = path.join(codexHome, "AGENTS.md");
+  check(!fs.existsSync(guidancePath), "Undeclared native global instructions already exist");
+  // The prepared source must remain unchanged. Native global instructions apply
+  // to this single attempt and cannot become part of its source patch.
+  if (payload.inventory.guidance.length) {
+    const bytes = fs.readFileSync(path.join(root, "AGENTS.md"));
+    check(hash(bytes) === payload.guidance_sha256 && bytes.length === payload.guidance_bytes,
+      "Staged instructions changed");
+    fs.writeFileSync(guidancePath, bytes, { flag: "wx", mode: 0o600 });
+  }
+  const expectedSkills = new Map();
+  for (const skill of payload.inventory.skills) {
+    const directory = path.join(home, ".agents/skills", skill.directory);
+    verifyTree(directory, skill);
+    expectedSkills.set(canonical(path.join(directory, "SKILL.md")), { name: skill.name, pluginId: null });
+  }
+  const installs = [];
+  if (payload.inventory.plugins.length) {
+    const added = cli(["plugin", "marketplace", "add", path.join(root, "marketplace"), "--json"]);
+    check(added.marketplaceName === payload.marketplace, "Native marketplace identity differs");
+    for (const plugin of payload.inventory.plugins) {
+      const id = plugin.name + "@" + payload.marketplace;
+      const installed = cli(["plugin", "add", id, "--json"]);
+      check(installed.pluginId === id && typeof installed.installedPath === "string", "Native plugin identity differs");
+      const installedRoot = canonical(installed.installedPath);
+      check(inside(installedRoot, path.join(codexHome, "plugins/cache")), "Native plugin installed outside private cache");
+      verifyTree(installedRoot, plugin);
+      installs.push({ ...installed, expected: plugin });
+      for (const skill of plugin.skills) {
+        expectedSkills.set(canonical(path.join(installedRoot, skill.path, "SKILL.md")), {
+          name: plugin.name + ":" + skill.name, pluginId: id,
+        });
+      }
+    }
+  }
+  const listed = cli(["plugin", "list", "--json"]);
+  check(Array.isArray(listed.installed) && listed.installed.length === installs.length,
+    "Native plugin inventory contains missing or undeclared installations: " + JSON.stringify({
+      expected: installs.map(item => item.pluginId),
+      installed: listed.installed?.map(item => ({
+        pluginId: item.pluginId, installed: item.installed, enabled: item.enabled,
+        marketplaceName: item.marketplaceName, installPolicy: item.installPolicy,
+        installPolicySource: item.installPolicySource,
+      })),
+    }));
+  for (const installed of installs) {
+    check(listed.installed.some(item => item.pluginId === installed.pluginId && item.installed === true && item.enabled === true),
+      "Native plugin is not installed and enabled: " + installed.pluginId);
+  }
+  const discovery = await discover();
+  const skillsEntry = discovery.skills.data?.find(entry => canonical(entry.cwd) === canonical(cwd));
+  const hooksEntry = discovery.hooks.data?.find(entry => canonical(entry.cwd) === canonical(cwd));
+  check(skillsEntry && skillsEntry.errors.length === 0, "Native skill discovery failed");
+  check(hooksEntry && hooksEntry.errors.length === 0 && hooksEntry.warnings.length === 0,
+    "Native hook discovery failed or warned");
+  const found = new Set();
+  for (const skill of skillsEntry.skills) {
+    const location = canonical(skill.path);
+    const expected = expectedSkills.get(location);
+    if (!expected) {
+      check(skill.scope === "system" && inside(location, path.join(codexHome, "skills/.system")),
+        "Undeclared native skill discovered: " + skill.name);
+      continue;
+    }
+    check(skill.enabled === true && skill.name === expected.name && (skill.pluginId || null) === expected.pluginId,
+      "Native skill identity/availability differs: " + skill.name);
+    found.add(location);
+  }
+  check(found.size === expectedSkills.size, "Native runtime did not discover every declared skill");
+  let expectedHookCount = 0;
+  for (const plugin of installs) {
+    const expected = [];
+    for (const config of plugin.expected.hooks) {
+      for (const [event, groups] of Object.entries(config.hooks || config)) {
+        for (const group of groups) for (const hook of group.hooks) expected.push({ event, group, hook });
+      }
+    }
+    const observed = hooksEntry.hooks.filter(hook => hook.pluginId === plugin.pluginId);
+    check(observed.length === expected.length, "Native plugin hook inventory differs: " + plugin.pluginId);
+    for (const { event, group, hook } of expected) {
+      const eventName = event[0].toLowerCase() + event.slice(1);
+      check(observed.some(item => item.eventName === eventName && item.enabled === true
+        && item.source === "plugin" && item.handlerType === "command" && !item.async
+        && item.command === hook.command && (item.matcher || null) === (group.matcher || null)
+        && inside(canonical(item.sourcePath), canonical(plugin.installedPath))),
+      "Native plugin hook was not registered faithfully: " + plugin.pluginId + "/" + event);
+    }
+    expectedHookCount += expected.length;
+  }
+  check(hooksEntry.hooks.length === expectedHookCount, "Undeclared native hooks discovered");
+  const receipt = {
+    status: "verified", delivery: payload.delivery, inventory: payload.inventory,
+    guidance: { path: payload.inventory.guidance.length ? guidancePath : null,
+      sha256: payload.guidance_sha256, bytes: payload.guidance_bytes, scope: "native global instructions" },
+    plugins: listed.installed, skills: skillsEntry, hooks: hooksEntry,
+    hook_trust: "per-coding-invocation bypass for the verified frozen package sources",
+    hook_execution: "not observed by discovery; target execution evidence is separate",
+    skill_activation: "not observed by discovery; native availability is verified",
+    effective_config: fs.readFileSync(path.join(codexHome, "config.toml"), "utf8"),
+  };
+  process.stdout.write(JSON.stringify(receipt) + "\n");
+}
+
+main().catch(error => {
+  process.stderr.write(String(error.stack || error) + "\n");
+  process.exitCode = 1;
+});

pluginbench/native_treatments.py

@@ -0,0 +1,155 @@
+"""Install declared treatments through the native Codex CLI in an owned runtime.
+
+Harbor owns skill upload, configuration and execution. This adapter runs after
+Harbor's configuration/skill setup and before its first coding command. The
+companion Node program checks the actual installation and native discovery.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shlex
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from pluginbench.treatments import Treatment, resolve_treatment, treatment_inventory
+
+DELIVERY_POLICY = "native-instructions-skills-plugins-v1"
+MARKETPLACE = "pluginbench-frozen"
+REMOTE_ROOT = "/tmp/pluginbench-treatments"
+MAX_GUIDANCE_BYTES = 32768
+
+
+class NativeCapabilities(BaseModel):
+    """One capability policy for vanilla and every treatment in a study."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    subagents: bool = True
+    max_threads: int = Field(default=4, strict=True, ge=1, le=16)
+    max_depth: int = Field(
+        default=1, strict=True, ge=1, le=2,
+        description="Native V1 depth preference; ignored by model-selected V2 agents.",
+    )
+
+    def codex_config(self, *, model: str, effort: str) -> dict[str, object]:
+        return {
+            "features": {
+                "code_mode": True,
+                "code_mode_only": True,
+                "plugins": True,
+                "remote_plugin": False,
+                "apps": False,
+                "hooks": True,
+                "multi_agent": self.subagents,
+            },
+            "project_doc_max_bytes": MAX_GUIDANCE_BYTES,
+            "agents": {
+                # Unlike feature flags, this also disables model-selected V2 agents.
+                "enabled": self.subagents,
+                "max_threads": self.max_threads,
+                "max_depth": self.max_depth,
+                "default_subagent_model": model,
+                "default_subagent_reasoning_effort": effort,
+            },
+        }
+
+
+def admit(treatment: Treatment, harness: str) -> dict[str, Any]:
+    """Reject unsupported inputs before container preparation or any model call."""
+    if harness != "codex":
+        raise ValueError("native treatment execution is currently qualified for Codex only")
+    resolved = resolve_treatment(treatment, Path.cwd(), harness)
+    inventory = treatment_inventory(resolved)
+    guidance = b"\n\n".join(path.read_bytes() for path in resolved.guidance)
+    if len(guidance) > MAX_GUIDANCE_BYTES:
+        raise ValueError(
+            f"native guidance exceeds {MAX_GUIDANCE_BYTES} bytes; refusing silent truncation"
+        )
+    return inventory
+
+
+def stage(treatment: Treatment, directory: Path) -> dict[str, Any]:
+    """Build a local marketplace from complete, already-frozen packages."""
+    inventory = admit(treatment, "codex")
+    directory.mkdir(parents=True, exist_ok=True)
+    guidance = b"\n\n".join(path.read_bytes() for path in treatment.guidance)
+    (directory / "AGENTS.md").write_bytes(guidance)
+    market = directory / "marketplace"
+    manifest_dir = market / ".agents/plugins"
+    manifest_dir.mkdir(parents=True)
+    packages = []
+    for source, item in zip(treatment.plugins, inventory["plugins"], strict=True):
+        name = item["name"]
+        shutil.copytree(source, market / "plugins" / name, ignore=shutil.ignore_patterns(".git"))
+        packages.append({
+            "name": name,
+            "source": {"source": "local", "path": "./plugins/" + name},
+            "policy": {"installation": "AVAILABLE", "authentication": "ON_INSTALL"},
+            "category": "Productivity",
+        })
+    (manifest_dir / "marketplace.json").write_text(
+        json.dumps({"name": MARKETPLACE, "plugins": packages}) + "\n"
+    )
+    payload = {
+        "delivery": DELIVERY_POLICY,
+        "inventory": inventory,
+        "guidance_sha256": hashlib.sha256(guidance).hexdigest(),
+        "guidance_bytes": len(guidance),
+        "marketplace": MARKETPLACE,
+    }
+    (directory / "treatment.json").write_text(json.dumps(payload, sort_keys=True) + "\n")
+    shutil.copy2(Path(__file__).with_name("native_treatment_runtime.js"), directory / "install.cjs")
+    return payload
+
+
+async def install(
+    *, environment: object, treatment: Treatment,
+    env: dict[str, str], cwd: str, artifacts: Path,
+) -> dict[str, Any]:
+    """Use the same environment/configuration as the upcoming native coding CLI."""
+    if env.get("CODEX_HOME") != "/tmp/codex-home":
+        raise ValueError("native treatment installation requires Harbor's private CODEX_HOME")
+    with tempfile.TemporaryDirectory(prefix="pluginbench-treatment-") as temporary:
+        local = Path(temporary).resolve()
+        payload = stage(treatment, local)
+        await getattr(environment, "upload_dir")(source_dir=local, target_dir=REMOTE_ROOT)
+        result = await getattr(environment, "exec")(
+            command=(
+                "if [ -s ~/.nvm/nvm.sh ]; then . ~/.nvm/nvm.sh; fi; "
+                f"node {REMOTE_ROOT}/install.cjs {REMOTE_ROOT} {shlex.quote(cwd)}"
+                " 2>/logs/agent/native-treatment.stderr"
+            ),
+            env=env,
+            cwd=cwd,
+            timeout_sec=120,
+        )
+        receipt_file = artifacts / "native-treatment.json"
+        stdout = str(getattr(result, "stdout", ""))
+        (artifacts / "native-treatment.stdout").write_text(stdout)
+        await getattr(environment, "download_file")(
+            source_path="/logs/agent/native-treatment.stderr",
+            target_path=artifacts / "native-treatment.stderr",
+        )
+        if getattr(result, "return_code") != 0:
+            receipt_file.write_text(json.dumps({
+                "status": "failed", "delivery": DELIVERY_POLICY,
+                "inventory": payload["inventory"],
+                "reason": "native installation or discovery failed; coding was not started",
+            }, indent=2) + "\n")
+            raise ValueError("native treatment installation/discovery failed; model call refused")
+        try:
+            receipt = json.loads(stdout)
+        except ValueError as error:
+            raise ValueError("native treatment discovery returned invalid evidence") from error
+        if not isinstance(receipt, dict) or receipt.get("status") != "verified":
+            raise ValueError("native treatment discovery did not verify installation")
+        if receipt.get("inventory") != payload["inventory"]:
+            raise ValueError("native treatment receipt describes different package contents")
+        receipt_file.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
+        return receipt

pluginbench/observer_data/go-analysis-v1/go.mod

@@ -0,0 +1,10 @@
+module pluginbench/go-analysis
+
+go 1.25.0
+
+require (
+	github.com/fzipp/gocyclo v0.6.0
+	github.com/uudashr/gocognit v1.3.0
+)
+
+require golang.org/x/tools v0.42.0 // indirect

pluginbench/observer_data/go-analysis-v1/go.sum

@@ -0,0 +1,10 @@
+github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
+github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
+github.com/uudashr/gocognit v1.3.0 h1:Xdw5JwAoYmX3WAB+wsXY3nxEwdw7+MWAnM0wCAAv7A4=
+github.com/uudashr/gocognit v1.3.0/go.mod h1:acaubQc6xYlXFEMb9nWX2dYBzJ/bIjEkc1zzvyIZg5Q=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=

pluginbench/observer_data/go-analysis-v1/main.go

@@ -0,0 +1,64 @@
+// Parse spans with Go's standard parser; all scores come from pinned analyzer libraries.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/fzipp/gocyclo"
+	"github.com/uudashr/gocognit"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+)
+
+type function struct {
+	Path       string `json:"path"`
+	Name       string `json:"name"`
+	Line       int    `json:"line"`
+	Col        int    `json:"col"`
+	EndLine    int    `json:"end_line"`
+	LineCount  int    `json:"line_count"`
+	Cognitive  int    `json:"cognitive"`
+	Cyclomatic int    `json:"cyclomatic"`
+}
+
+func main() {
+	if len(os.Args) == 2 && os.Args[1] == "--version" {
+		fmt.Println("go-analysis-v1 gocognit=1.3.0 gocyclo=0.6.0")
+		return
+	}
+	rows := []function{}
+	for _, path := range os.Args[1:] {
+		fset := token.NewFileSet()
+		file, err := parser.ParseFile(fset, path, nil, 0)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(2)
+		}
+		ast.Inspect(file, func(node ast.Node) bool {
+			var fn *ast.FuncDecl
+			name := ""
+			switch n := node.(type) {
+			case *ast.FuncDecl:
+				fn = n
+				name = n.Name.Name
+				if n.Recv != nil {
+					name = fmt.Sprintf("method@%d:%s", fset.Position(n.Pos()).Line, name)
+				}
+			case *ast.FuncLit:
+				name = fmt.Sprintf("closure@%d:%d", fset.Position(n.Pos()).Line, fset.Position(n.Pos()).Column)
+				fn = &ast.FuncDecl{Type: n.Type, Body: n.Body, Name: ast.NewIdent(name)}
+			default:
+				return true
+			}
+			start, end := fset.Position(node.Pos()), fset.Position(node.End())
+			rows = append(rows, function{path, name, start.Line, start.Column, end.Line, end.Line - start.Line + 1, gocognit.Complexity(fn), gocyclo.Complexity(node)})
+			return true
+		})
+	}
+	if err := json.NewEncoder(os.Stdout).Encode(rows); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(2)
+	}
+}