vis-agent 0.3.4.dev7992__tar.gz → 0.3.4.dev8033__tar.gz

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/.gitignore

@@ -117,3 +117,6 @@ own_your_harness.pptx
 # Call journal written by the e2e schema-probe fixture when it runs from this
 # checkout instead of a temporary workspace.
 /schema-calls.jsonl
+
+# uv lockfiles. The SDK and the examples are libraries; uv writes the lock locally.
+uv.lock

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vis-agent
-Version: 0.3.4.dev7992
+Version: 0.3.4.dev8033
 Summary: The Vis Python SDK — author extensions and control local or remote engines.
 Project-URL: Homepage, https://vis.blockether.com
 Project-URL: Documentation, https://vis.blockether.com

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/contracts/schema/agents.json

@@ -15,6 +15,9 @@
           "maxLength": 8192,
           "pattern": "\\S"
         },
+        "role": {
+          "$ref": "#/$defs/role"
+        },
         "model": {
           "type": "string",
           "minLength": 1
@@ -77,6 +80,9 @@
                   "task": {
                     "$ref": "#/$defs/spawn/properties/task"
                   },
+                  "role": {
+                    "$ref": "#/$defs/spawn/properties/role"
+                  },
                   "model": {
                     "$ref": "#/$defs/spawn/properties/model"
                   },
@@ -191,6 +197,27 @@
       ],
       "additionalProperties": false
     },
+    "answer": {
+      "$comment": "A person answers the open question of a subagent for its parent.",
+      "type": "object",
+      "properties": {
+        "session_id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "text": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 8192,
+          "pattern": "\\S"
+        }
+      },
+      "required": [
+        "session_id",
+        "text"
+      ],
+      "additionalProperties": false
+    },
     "grant": {
       "$comment": "Add file rules to the grant of a subagent. A child without write rules already writes where its parent writes, so a write grant changes nothing for it. A write rule under a read_only or deny rule of the child fails.",
       "type": "object",
@@ -274,6 +301,32 @@
       ],
       "additionalProperties": false
     },
+    "role": {
+      "description": "Choose the role by the result that you need: facts, changed files, a pass or fail, or findings.",
+      "$comment": "A role is a label for people and the leader. It sets no tool, file or model limit. The TUI writes the role name. The Companion draws a Lucide icon for each role.",
+      "oneOf": [
+        {
+          "const": "research",
+          "title": "Finds facts",
+          "description": "Finds facts. Reads code, docs, logs and history, and answers one question with evidence. Use it before you decide or plan."
+        },
+        {
+          "const": "implement",
+          "title": "Changes files",
+          "description": "Changes files. Writes code, tests, docs or configuration and runs the checks for its change. Use it for one bounded change."
+        },
+        {
+          "const": "verify",
+          "title": "Proves it works",
+          "description": "Proves that something works. Runs tests, reproduces a bug or checks the app, and reports pass or fail with evidence."
+        },
+        {
+          "const": "audit",
+          "title": "Judges the work",
+          "description": "Judges finished work. Reviews a change, plan or release for risks and mistakes, and reports findings. Give it the work, not your conclusion."
+        }
+      ]
+    },
     "model_pair": {
       "type": "object",
       "properties": {
@@ -342,6 +395,9 @@
           "type": "string",
           "minLength": 1
         },
+        "role": {
+          "$ref": "#/$defs/role"
+        },
         "status": {
           "enum": [
             "queued",
@@ -385,6 +441,38 @@
           "type": "string",
           "minLength": 1
         },
+        "question_entry_id": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "budget_request": {
+          "type": "object",
+          "properties": {
+            "iterations": {
+              "type": "integer",
+              "minimum": 1
+            },
+            "reason": {
+              "type": "string"
+            }
+          }
+        },
+        "files_request": {
+          "type": "object",
+          "properties": {
+            "files": {
+              "$ref": "#/$defs/files"
+            },
+            "reason": {
+              "type": "string"
+            }
+          }
+        },
+        "activity": {
+          "$comment": "The latest Activity line of a queued or running agent, such as Patched ~/src/a.clj.",
+          "type": "string",
+          "minLength": 1
+        },
         "usage": {
           "type": [
             "object",

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/contracts/schema/config.json

@@ -694,6 +694,32 @@
       },
       "additionalProperties": false
     },
+    "toolList": {
+      "type": "array",
+      "maxItems": 256,
+      "uniqueItems": true,
+      "items": {
+        "type": "string",
+        "pattern": "^[A-Za-z_*][A-Za-z0-9_.*]*$",
+        "maxLength": 200
+      },
+      "description": "Tool names as python_execution binds them, such as memo.recall. * matches any characters."
+    },
+    "tools": {
+      "type": "object",
+      "properties": {
+        "allow": {
+          "$ref": "#/$defs/toolList",
+          "description": "Keep only these tools of each extension that the list names. Other tools of that extension, also later ones, are off."
+        },
+        "deny": {
+          "$ref": "#/$defs/toolList",
+          "description": "Turn these tools off."
+        }
+      },
+      "additionalProperties": false,
+      "description": "Tools of optional extensions that a session can call. Every scope applies: a narrower scope can turn tools off, never on."
+    },
     "jail": {
       "type": "object",
       "properties": {
@@ -1328,6 +1354,9 @@
         "jail": {
           "$ref": "#/$defs/jail"
         },
+        "tools": {
+          "$ref": "#/$defs/tools"
+        },
         "environment": {
           "$ref": "#/$defs/environment"
         },

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/contracts/schema/gateway.json

@@ -736,6 +736,17 @@
       },
       "path": "/v1/sessions/:sid/agents"
     },
+    {
+      "audience": "sdk",
+      "operations": {
+        "post": {
+          "request": "json",
+          "response": "json",
+          "extension": "background"
+        }
+      },
+      "path": "/v1/sessions/:sid/agents/answer"
+    },
     {
       "audience": "sdk",
       "operations": {
@@ -3000,14 +3011,27 @@
             "switch",
             "select",
             "text",
+            "secret",
             "multiline",
             "number",
             "paths",
             "filesystem",
             "network",
-            "list"
+            "list",
+            "tools"
           ]
         },
+        "is_set": {
+          "type": "boolean",
+          "description": "Whether a secret row has a saved value. A secret row never carries the value itself."
+        },
+        "suggestions": {
+          "type": "array",
+          "description": "Values that an editor offers as completions. Other values can still be valid.",
+          "items": {
+            "type": "string"
+          }
+        },
         "schema": {
           "type": "string"
         },

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/contracts/schema/toggle.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://contract.blockether.com/vis/schema/toggle.json",
   "title": "Feature-toggle contract",
-  "description": "Feature-toggle contributions and the boolean tokens accepted by settings.",
+  "description": "Feature-toggle contributions, and the boolean tokens and string values that settings accept.",
   "$defs": {
     "boolean_true": {
       "type": "string",
@@ -32,6 +32,12 @@
         }
       ]
     },
+    "string_value": {
+      "type": "string",
+      "maxLength": 4096,
+      "pattern": "^[^\\u0000-\\u001f\\u007f]*$",
+      "description": "The value of a string setting: one line without control characters."
+    },
     "contribution": {
       "type": "object",
       "additionalProperties": true,
@@ -83,10 +89,15 @@
         "type": {
           "enum": [
             "boolean",
-            "enum"
+            "enum",
+            "string"
           ],
           "default": "boolean"
         },
+        "is_secret": {
+          "type": "boolean",
+          "description": "Makes a string setting write-only. Settings show only whether it is set, never its value."
+        },
         "choices": {
           "type": "array",
           "items": {},
@@ -169,6 +180,60 @@
               }
             ]
           }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {
+                "const": "string"
+              }
+            },
+            "required": [
+              "type"
+            ]
+          },
+          "then": {
+            "properties": {
+              "default": {
+                "$ref": "#/$defs/string_value"
+              }
+            },
+            "not": {
+              "required": [
+                "choices"
+              ]
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "is_secret": {
+                "const": true
+              }
+            },
+            "required": [
+              "is_secret"
+            ]
+          },
+          "then": {
+            "properties": {
+              "type": {
+                "const": "string"
+              },
+              "default": {
+                "const": ""
+              },
+              "scopes": {
+                "const": [
+                  "global"
+                ]
+              }
+            },
+            "required": [
+              "type"
+            ]
+          }
         }
       ]
     },

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vis-agent"
-version = "0.3.4.dev7992"
+version = "0.3.4.dev8033"
 description = "The Vis Python SDK — author extensions and control local or remote engines."
 readme = "README.md"
 requires-python = ">=3.12"

vis_agent-0.3.4.dev8033/release-tag

@@ -0,0 +1 @@
+beta-8906d28bc9e78c61bfe556c52e15d3045f046731

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/src/blockether/vis/engine/_agents.py

@@ -18,10 +18,16 @@ class Subagent:
     status: str
     iteration_budget: int
     iterations_used: int
+    role: str | None = None
     provider: str | None = None
     model: str | None = None
     routing_locked: bool = False
     pending_input: bool = False
+    question: str | None = None
+    question_entry_id: int | None = None
+    budget_request: dict[str, Any] | None = None
+    files_request: dict[str, Any] | None = None
+    activity: str | None = None
     usage: dict[str, Any] | None = None
     files: dict[str, list[str]] | None = None
 

{vis_agent-0.3.4.dev7992 → vis_agent-0.3.4.dev8033}/src/blockether/vis/engine/_council.py

@@ -231,6 +231,7 @@ class Council:
         self,
         task: str,
         *,
+        role: str | None = None,
         provider: str | None = None,
         model: str | None = None,
         iteration_budget: int | None = None,
@@ -247,12 +248,15 @@ class Council:
         it with different input is rejected. This is not an ordinary independent fork,
         and it does not grant broader permissions. ``files`` narrows file access with
         ``write``, ``read_only`` and ``deny`` rules: paths or globs, also for future files.
+        ``role`` is ``research``, ``implement``, ``verify`` or ``audit``: a label for people
+        that sets no limit.
         """
         body = {"task": task}
         body.update(
             {
                 name: value
                 for name, value in {
+                    "role": role,
                     "provider": provider,
                     "iteration_budget": iteration_budget,
                     "model": model,
@@ -291,8 +295,8 @@ class Council:
     ) -> tuple[SubagentRecord, ...]:
         """Spawn one subagent for each task, wait for all of them and return their records.
 
-        A task is a string, or a dict with ``task`` and optional ``model``, ``provider``,
-        ``iteration_budget`` and ``files``. The wait ends when every child answers, asks
+        A task is a string, or a dict with ``task`` and optional ``role``, ``model``,
+        ``provider``, ``iteration_budget`` and ``files``. The wait ends when every child answers, asks
         or stops, or after ``timeout_secs`` (at most 1800). A running child stays running.
         """
         body = {"tasks": tasks, "timeout_secs": timeout_secs}
@@ -352,6 +356,11 @@ class Council:
         body = {"session_id": session_id, "reason": reason}
         return self._records("/deny", body, "deny")[0]
 
+    def answer(self, session_id: str, text: str) -> SubagentRecord:
+        """Answer the open question of a subagent for its parent; the child continues."""
+        body = {"session_id": session_id, "text": text}
+        return self._records("/answer", body, "answer")[0]
+
     def grant(self, session_id: str, files: dict[str, list[str]]) -> SubagentRecord:
         """Add ``write``, ``read_only`` or ``deny`` file rules to the grant of a subagent.
 

vis_agent-0.3.4.dev8033/src/blockether/vis/evals/mrcr.py

@@ -0,0 +1,273 @@
+"""Write OpenAI MRCR conversations as long multi-turn Harbor tasks.
+
+[MRCR](https://huggingface.co/datasets/openai/mrcr) (multi-round coreference resolution)
+hides several similar requests in one long conversation, for example two requests for
+a poem about tapirs. The last request asks for one of the earlier answers, for example
+the second poem about tapirs, word for word, with a random string at the start. The
+model must find the correct answer among similar ones and repeat it exactly.
+
+A task pastes the earlier conversation in user turns of about `turn_tokens` tokens
+each, and the agent gives a short reply. The last turn gives the request. Only the
+answer to that request counts. When the conversation is larger than the context budget
+of the agent, the agent must fold old turns. A fold note seldom keeps a whole answer
+word for word, so MRCR shows what a fold loses. Compare it with
+`blockether.vis.evals.longmemeval`, which asks only for short facts.
+
+The grader follows the MRCR grader: the answer must start with the random string, and
+`difflib.SequenceMatcher` compares the rest with the expected answer. The MRCR score is
+that ratio. A Harbor reward is a pass or a failure, so a task passes when the ratio is
+at least `MIN_RATIO`. The protocol differs from MRCR in two ways:
+
+- The conversation arrives as pasted text in user turns, not as one prompt with
+  assistant messages.
+- The score is a pass at `MIN_RATIO`, not the mean ratio.
+
+Importing this module makes no network calls. `load_rows` downloads only the rows that
+it needs, about 1 MB for each long row. Run `python -m blockether.vis.evals.mrcr DATASET
+--cache FILE` to write the tasks.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import urllib.request
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from .harbor import HarborTask, write_task
+
+__all__ = [
+    "MIN_RATIO",
+    "NEEDLES",
+    "fetch_row",
+    "load_rows",
+    "mrcr_task_name",
+    "task_turns",
+    "write_mrcr_tasks",
+]
+
+ROWS_URL = (
+    "https://datasets-server.huggingface.co/rows"
+    "?dataset=openai/mrcr&config=default&split=train&offset={index}&length=1"
+)
+NEEDLES = {2: 0, 4: 800, 8: 1600}
+"""The first row of each needle count. Each needle count has `ROWS_PER_NEEDLES` rows."""
+ROWS_PER_NEEDLES = 800
+PROGRAM_FILE = "check_answer.py"
+EXPECTED_FILE = "expected.json"
+CHARS_PER_TOKEN = 4
+TURN_TOKENS = 10_000
+TURN_TIMEOUT_SEC = 300.0
+MIN_RATIO = 0.9
+MIN_CHARS = 600_000
+MAX_CHARS = 1_300_000
+
+_FIRST = (
+    "Over the next messages I will paste an earlier conversation between a user and "
+    "an AI assistant. Later I will ask you about it. Do not create or edit files: keep "
+    "what you need in this conversation. Until I give my request, a short reply is "
+    "enough."
+)
+_MORE = "More of the earlier conversation."
+_LAST = (
+    "That is the whole earlier conversation. Answer my request about it. Give only the "
+    "requested text, without a code block or other words: {request}"
+)
+
+_PROGRAM = r"""# Grade an MRCR answer like the MRCR grader. The answer is on stdin.
+import json
+import sys
+from difflib import SequenceMatcher
+from pathlib import Path
+
+expected = json.loads(Path(__file__).with_name("expected.json").read_text())
+prefix = expected["prefix"]
+answer = sys.stdin.read().strip()
+if not answer.startswith(prefix):
+    sys.exit(f"the answer does not start with {prefix!r}: {answer[:80]!r}")
+ratio = SequenceMatcher(
+    None, answer.removeprefix(prefix), expected["answer"].removeprefix(prefix)
+).ratio()
+if ratio < expected["min_ratio"]:
+    sys.exit(f"the answer matches {ratio:.3f} of the expected text, less than {expected['min_ratio']}")
+"""
+
+
+def fetch_row(index: int) -> dict[str, Any]:
+    """Download the MRCR row `index` from the Hugging Face dataset server."""
+    request = urllib.request.Request(
+        ROWS_URL.format(index=index), headers={"User-Agent": "vis-evals"}
+    )
+    with urllib.request.urlopen(request, timeout=300) as response:
+        rows = json.load(response)["rows"]
+    if len(rows) != 1:
+        raise ValueError(f"MRCR row {index} does not exist")
+    return {"row_idx": index, **rows[0]["row"]}
+
+
+def load_rows(
+    cache: Path | str,
+    *,
+    needles: int = 2,
+    min_chars: int = MIN_CHARS,
+    max_chars: int = MAX_CHARS,
+    limit: int = 5,
+    fetch: Callable[[int], Mapping[str, Any]] = fetch_row,
+) -> list[dict[str, Any]]:
+    """Return the first `limit` MRCR rows with `needles` needles and a size in range.
+
+    `min_chars` and `max_chars` bound the `n_chars` size of a row. The JSON Lines file
+    `cache` keeps each row that `fetch` downloaded, and only the size of a row out of
+    range. So a second call with the same or a smaller selection downloads nothing. An
+    unknown needle count raises ValueError.
+    """
+    if needles not in NEEDLES:
+        raise ValueError(f"needles must be one of {sorted(NEEDLES)}: {needles!r}")
+    path = Path(cache)
+    known: dict[int, dict[str, Any]] = {}
+    if path.is_file():
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if line.strip():
+                row = json.loads(line)
+                known[int(row["row_idx"])] = row
+    chosen: list[dict[str, Any]] = []
+    first = NEEDLES[needles]
+    for index in range(first, first + ROWS_PER_NEEDLES):
+        if len(chosen) >= limit:
+            break
+        row = known.get(index)
+        if row is None:
+            row = dict(fetch(index))
+            row["row_idx"] = index
+            if not min_chars <= int(row["n_chars"]) <= max_chars:
+                row = {key: row[key] for key in ("row_idx", "n_needles", "n_chars")}
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with path.open("a", encoding="utf-8") as target:
+                target.write(json.dumps(row) + "\n")
+            known[index] = row
+        if "prompt" in row and min_chars <= int(row["n_chars"]) <= max_chars:
+            chosen.append(row)
+    return chosen
+
+
+def mrcr_task_name(row: Mapping[str, Any]) -> str:
+    """Return the Harbor task name of `row`, for example `mrcr-2n-17`."""
+    return f"mrcr-{int(row['n_needles'])}n-{int(row['row_idx'])}"
+
+
+def _tokens(text: str) -> int:
+    return len(text) // CHARS_PER_TOKEN
+
+
+def _message_text(message: Mapping[str, Any]) -> str:
+    role = "USER" if message.get("role") == "user" else "ASSISTANT"
+    return f"{role}: {str(message.get('content', '')).strip()}"
+
+
+def task_turns(row: Mapping[str, Any], *, turn_tokens: int = TURN_TOKENS) -> list[str]:
+    """Return the user messages of the task for `row`.
+
+    The earlier messages of the MRCR conversation come in pasted turns of about
+    `turn_tokens` tokens. A message never splits. The last turn gives the last request
+    of the conversation.
+    """
+    messages = json.loads(row["prompt"])
+    if not messages or messages[-1].get("role") != "user":
+        raise ValueError(f"MRCR row {row.get('row_idx')} does not end with a request")
+    turns: list[list[str]] = [[]]
+    size = 0
+    for message in messages[:-1]:
+        text = _message_text(message)
+        if turns[-1] and size + _tokens(text) > turn_tokens:
+            turns.append([])
+            size = 0
+        turns[-1].append(text)
+        size += _tokens(text)
+    pasted = [
+        f"{_FIRST if number == 0 else _MORE}\n\n" + "\n\n".join(chunk)
+        for number, chunk in enumerate(turns)
+        if chunk
+    ]
+    return [*pasted, _LAST.format(request=str(messages[-1]["content"]).strip())]
+
+
+def write_mrcr_tasks(
+    dataset: Path | str,
+    rows: Iterable[Mapping[str, Any]],
+    *,
+    turn_tokens: int = TURN_TOKENS,
+    min_ratio: float = MIN_RATIO,
+) -> tuple[HarborTask, ...]:
+    """Write one Harbor task for each MRCR row in the folder `dataset`.
+
+    A task passes when the answer starts with the random string of the row and matches
+    at least `min_ratio` of the expected answer.
+    """
+    written: list[HarborTask] = []
+    for row in rows:
+        turns = task_turns(row, turn_tokens=turn_tokens)
+        name = mrcr_task_name(row)
+        written.append(
+            write_task(
+                dataset,
+                name,
+                turns,
+                {"want_answer_program": PROGRAM_FILE},
+                agent_timeout_sec=TURN_TIMEOUT_SEC * len(turns),
+                metadata={
+                    "benchmark": "mrcr",
+                    "row": int(row["row_idx"]),
+                    "needles": int(row["n_needles"]),
+                    "chars": int(row["n_chars"]),
+                    "turns": len(turns),
+                    "history_tokens": sum(_tokens(turn) for turn in turns),
+                },
+            )
+        )
+        tests = Path(dataset) / name / "tests"
+        (tests / PROGRAM_FILE).write_text(_PROGRAM, encoding="utf-8")
+        expected = {
+            "answer": str(row["answer"]),
+            "prefix": str(row["random_string_to_prepend"]),
+            "min_ratio": min_ratio,
+        }
+        (tests / EXPECTED_FILE).write_text(
+            json.dumps(expected, indent=2) + "\n", encoding="utf-8"
+        )
+    return tuple(written)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m blockether.vis.evals.mrcr",
+        description="Write OpenAI MRCR conversations as long multi-turn Harbor tasks.",
+    )
+    parser.add_argument("dataset", type=Path, help="folder for the new tasks")
+    parser.add_argument(
+        "--cache", type=Path, required=True, help="JSON Lines file of downloaded rows"
+    )
+    parser.add_argument("--needles", type=int, choices=sorted(NEEDLES), default=2)
+    parser.add_argument("--min-chars", type=int, default=MIN_CHARS)
+    parser.add_argument("--max-chars", type=int, default=MAX_CHARS)
+    parser.add_argument("--limit", type=int, default=5, help="rows to write")
+    parser.add_argument("--turn-tokens", type=int, default=TURN_TOKENS)
+    parser.add_argument("--min-ratio", type=float, default=MIN_RATIO)
+    args = parser.parse_args(argv)
+    rows = load_rows(
+        args.cache,
+        needles=args.needles,
+        min_chars=args.min_chars,
+        max_chars=args.max_chars,
+        limit=args.limit,
+    )
+    written = write_mrcr_tasks(
+        args.dataset, rows, turn_tokens=args.turn_tokens, min_ratio=args.min_ratio
+    )
+    print(f"wrote {len(written)} tasks to {args.dataset}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())