visru 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- visru/__init__.py +1 -0
- visru/agent/__init__.py +1 -0
- visru/agent/context.py +276 -0
- visru/agent/loop.py +184 -0
- visru/agent/prompts/system.md +89 -0
- visru/agent/subagents.py +116 -0
- visru/agent/trajectory.py +183 -0
- visru/backends/__init__.py +40 -0
- visru/backends/anthropic.py +136 -0
- visru/backends/base.py +29 -0
- visru/backends/openai_compat.py +274 -0
- visru/backends/replay.py +74 -0
- visru/cli.py +148 -0
- visru/config.py +90 -0
- visru/evidence/__init__.py +3 -0
- visru/evidence/ledger.py +364 -0
- visru/evidence/report.py +111 -0
- visru/geometry.py +52 -0
- visru/models.py +145 -0
- visru/servers/__init__.py +0 -0
- visru/servers/vision_mcp.py +140 -0
- visru/tools/__init__.py +122 -0
- visru/tools/annotate.py +138 -0
- visru/tools/navigate.py +182 -0
- visru/tools/registry.py +367 -0
- visru/tools/search.py +474 -0
- visru/tools/verify.py +460 -0
- visru/workspace/__init__.py +3 -0
- visru/workspace/artifacts.py +142 -0
- visru/workspace/index.py +191 -0
- visru/workspace/registry.py +218 -0
- visru/workspace/video.py +99 -0
- visru-0.0.1.dist-info/METADATA +289 -0
- visru-0.0.1.dist-info/RECORD +37 -0
- visru-0.0.1.dist-info/WHEEL +4 -0
- visru-0.0.1.dist-info/entry_points.txt +2 -0
- visru-0.0.1.dist-info/licenses/LICENSE +21 -0
visru/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.1"
|
visru/agent/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The harness: the loop, the context it keeps, and the record it leaves."""
|
visru/agent/context.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
"""What the model is holding: the live-image budget, elision, and compaction.
|
|
2
|
+
|
|
3
|
+
Images dominate context cost. One 2000x1333 render is about 3,400 visual tokens,
|
|
4
|
+
so twenty of them cost more than the rest of a long session put together. A loop
|
|
5
|
+
that simply appends every image it takes dies around step thirty, and it dies by
|
|
6
|
+
silently truncating rather than by raising.
|
|
7
|
+
|
|
8
|
+
So this keeps a fixed number of images live and swaps older ones for a text stub
|
|
9
|
+
naming the artifact that made them (README.md:91). The stub is only honest
|
|
10
|
+
because `view("a_0042")` brings the image back — see `tools.resolve_ref`.
|
|
11
|
+
|
|
12
|
+
Messages are provider-neutral. Blocks are plain dicts with a `type`, and a
|
|
13
|
+
backend maps them to whatever its API wants; nothing here imports a vendor SDK.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from typing import Any, Iterator
|
|
19
|
+
|
|
20
|
+
from PIL import Image
|
|
21
|
+
|
|
22
|
+
from visru.config import Config
|
|
23
|
+
from visru.geometry import visual_tokens
|
|
24
|
+
from visru.tools import ToolResult
|
|
25
|
+
|
|
26
|
+
# Rough, and deliberately so: a real tokeniser call per step would cost a network
|
|
27
|
+
# round trip to learn something the budget only needs to the nearest few percent.
|
|
28
|
+
CHARS_PER_TOKEN = 4
|
|
29
|
+
STUB_CHARS = 140
|
|
30
|
+
REACTION_CHARS = 300 # README.md:415 — what the model said after seeing the image
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _clip(text: str, limit: int) -> str:
|
|
34
|
+
return text if len(text) <= limit else text[:limit].rstrip() + "…"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class ToolCall:
|
|
39
|
+
id: str
|
|
40
|
+
name: str
|
|
41
|
+
arguments: dict[str, Any]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class ToolOutput:
|
|
46
|
+
"""One tool result, and whether its images are still live."""
|
|
47
|
+
|
|
48
|
+
call_id: str
|
|
49
|
+
name: str
|
|
50
|
+
text: str
|
|
51
|
+
images: list[Image.Image] = field(default_factory=list)
|
|
52
|
+
artifacts: list[str] = field(default_factory=list)
|
|
53
|
+
elided: bool = False
|
|
54
|
+
is_error: bool = False
|
|
55
|
+
reaction: str = "" # what the model said after seeing this, set by add_turn
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def live_images(self) -> int:
|
|
59
|
+
return 0 if self.elided else len(self.images)
|
|
60
|
+
|
|
61
|
+
def stub(self) -> str:
|
|
62
|
+
"""The note that replaces the pixels.
|
|
63
|
+
|
|
64
|
+
Two halves, per README.md:415. The tool's own first line names the asset,
|
|
65
|
+
the box and the scale — enough to decide whether the image is worth asking
|
|
66
|
+
back. What the model said next is usually its reading of that image, and
|
|
67
|
+
losing that is how a session forgets a fact it already established and
|
|
68
|
+
pays to look a second time.
|
|
69
|
+
"""
|
|
70
|
+
head = self.text.splitlines()[0] if self.text else ""
|
|
71
|
+
for artifact_id in self.artifacts: # not repeated inside its own stub
|
|
72
|
+
head = head.replace(f"artifact={artifact_id}", "").strip()
|
|
73
|
+
head = _clip(head, STUB_CHARS)
|
|
74
|
+
|
|
75
|
+
named = ", ".join(self.artifacts) or self.name
|
|
76
|
+
said = _clip(" ".join(self.reaction.split()), REACTION_CHARS)
|
|
77
|
+
return f"[{named} elided: {head}" + (f"; model noted: {said}" if said else "") + "]"
|
|
78
|
+
|
|
79
|
+
def elide(self) -> int:
|
|
80
|
+
"""Drop the pixels, keep the record. Returns how many images went."""
|
|
81
|
+
freed = self.live_images
|
|
82
|
+
self.elided = True
|
|
83
|
+
return freed
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass
|
|
87
|
+
class Step:
|
|
88
|
+
"""One model turn and the tool results it asked for.
|
|
89
|
+
|
|
90
|
+
Compaction works in whole steps because the providers require every tool call
|
|
91
|
+
to be answered; folding away half a step would leave an unanswered call and
|
|
92
|
+
the request would be rejected.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
text: str
|
|
96
|
+
calls: list[ToolCall] = field(default_factory=list)
|
|
97
|
+
outputs: list[ToolOutput] = field(default_factory=list)
|
|
98
|
+
followup: str = "" # a word from the harness when the step called no tools
|
|
99
|
+
# Whatever the backend needs handed back verbatim next turn — on Anthropic,
|
|
100
|
+
# the assistant content blocks, because thinking blocks must be replayed
|
|
101
|
+
# unchanged on the same model and cannot be rebuilt from text.
|
|
102
|
+
raw: Any = None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class Context:
|
|
106
|
+
def __init__(self, task: str, cfg: Config, system: str = ""):
|
|
107
|
+
self.task = task
|
|
108
|
+
self.cfg = cfg
|
|
109
|
+
self.system = system
|
|
110
|
+
self.steps: list[Step] = []
|
|
111
|
+
self.summary: str = ""
|
|
112
|
+
self.elisions = 0
|
|
113
|
+
self.compactions = 0
|
|
114
|
+
|
|
115
|
+
# --- building ---------------------------------------------------------
|
|
116
|
+
def add_turn(self, text: str, calls: list[ToolCall] | None = None,
|
|
117
|
+
raw: Any = None) -> Step:
|
|
118
|
+
# What the model says now is its reading of whatever it was just shown, so
|
|
119
|
+
# it belongs to those results: it is the half of an elision note that
|
|
120
|
+
# carries meaning rather than coordinates.
|
|
121
|
+
if text and self.steps:
|
|
122
|
+
for out in self.steps[-1].outputs:
|
|
123
|
+
if not out.reaction:
|
|
124
|
+
out.reaction = text
|
|
125
|
+
step = Step(text=text, calls=list(calls or []), raw=raw)
|
|
126
|
+
self.steps.append(step)
|
|
127
|
+
return step
|
|
128
|
+
|
|
129
|
+
def add_output(self, call: ToolCall, result: ToolResult,
|
|
130
|
+
is_error: bool = False) -> ToolOutput:
|
|
131
|
+
if not self.steps:
|
|
132
|
+
raise ValueError("a tool result needs a model turn to answer")
|
|
133
|
+
out = ToolOutput(call_id=call.id, name=call.name, text=result.text,
|
|
134
|
+
images=list(result.images), artifacts=list(result.artifacts),
|
|
135
|
+
is_error=is_error)
|
|
136
|
+
self.steps[-1].outputs.append(out)
|
|
137
|
+
self.enforce()
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
def add_user(self, text: str) -> None:
|
|
141
|
+
"""A word from the harness rather than from a tool.
|
|
142
|
+
|
|
143
|
+
Only legal after a turn that called nothing: a step with tool results
|
|
144
|
+
already owes the model a user message, and two in a row is a bad request.
|
|
145
|
+
"""
|
|
146
|
+
if not self.steps:
|
|
147
|
+
raise ValueError("nothing to reply to yet")
|
|
148
|
+
if self.steps[-1].outputs:
|
|
149
|
+
raise ValueError("that step already answers with tool results")
|
|
150
|
+
self.steps[-1].followup = text
|
|
151
|
+
|
|
152
|
+
def add_error(self, call: ToolCall, message: str) -> ToolOutput:
|
|
153
|
+
"""A failed call still has to answer its tool_use, or the next request 400s."""
|
|
154
|
+
return self.add_output(call, ToolResult(text=f"error: {message}"), is_error=True)
|
|
155
|
+
|
|
156
|
+
# --- budget -----------------------------------------------------------
|
|
157
|
+
@property
|
|
158
|
+
def live_images(self) -> int:
|
|
159
|
+
return sum(o.live_images for step in self.steps for o in step.outputs)
|
|
160
|
+
|
|
161
|
+
def _outputs(self) -> Iterator[ToolOutput]:
|
|
162
|
+
for step in self.steps:
|
|
163
|
+
yield from step.outputs
|
|
164
|
+
|
|
165
|
+
def estimate_tokens(self) -> int:
|
|
166
|
+
patch = self.cfg.model.patch
|
|
167
|
+
chars = len(self.system) + len(self.task) + len(self.summary)
|
|
168
|
+
tokens = 0
|
|
169
|
+
for step in self.steps:
|
|
170
|
+
chars += len(step.text)
|
|
171
|
+
for call in step.calls:
|
|
172
|
+
chars += len(call.name) + len(json.dumps(call.arguments, default=str))
|
|
173
|
+
for out in step.outputs:
|
|
174
|
+
chars += len(out.stub()) if out.elided else len(out.text)
|
|
175
|
+
if not out.elided:
|
|
176
|
+
tokens += sum(visual_tokens(im.width, im.height, patch)
|
|
177
|
+
for im in out.images)
|
|
178
|
+
return tokens + chars // CHARS_PER_TOKEN
|
|
179
|
+
|
|
180
|
+
# --- keeping it inside the limits -------------------------------------
|
|
181
|
+
def enforce(self) -> None:
|
|
182
|
+
self._elide()
|
|
183
|
+
self._compact()
|
|
184
|
+
|
|
185
|
+
def _elide(self) -> None:
|
|
186
|
+
"""Swap the oldest images for stubs once the budget is exceeded.
|
|
187
|
+
|
|
188
|
+
A batch at a time, not one per step: rewriting history invalidates the
|
|
189
|
+
cached prefix, so doing it every time an image arrives would pay the
|
|
190
|
+
cache miss on every single turn (config.py:34).
|
|
191
|
+
"""
|
|
192
|
+
limit = self.cfg.context.max_live_images
|
|
193
|
+
if self.live_images <= limit:
|
|
194
|
+
return
|
|
195
|
+
freed = 0
|
|
196
|
+
for out in self._outputs(): # oldest first
|
|
197
|
+
if freed >= self.cfg.context.elide_batch:
|
|
198
|
+
break
|
|
199
|
+
if out.live_images:
|
|
200
|
+
freed += out.elide()
|
|
201
|
+
self.elisions += 1
|
|
202
|
+
|
|
203
|
+
def _compact(self) -> None:
|
|
204
|
+
"""Fold the oldest steps into one note when the whole thing grows too big.
|
|
205
|
+
|
|
206
|
+
Artifact IDs survive verbatim. A compacted summary that dropped them would
|
|
207
|
+
strand the evidence: the model would cite an ID the ledger can still see
|
|
208
|
+
but the model can no longer name, and the claim would be refused for a
|
|
209
|
+
reason invisible to it.
|
|
210
|
+
"""
|
|
211
|
+
if self.estimate_tokens() <= self.cfg.context.compact_at_tokens:
|
|
212
|
+
return
|
|
213
|
+
if len(self.steps) < 4: # nothing to fold that is not still in play
|
|
214
|
+
return
|
|
215
|
+
|
|
216
|
+
keep = max(2, len(self.steps) // 2)
|
|
217
|
+
folded, self.steps = self.steps[:-keep], self.steps[-keep:]
|
|
218
|
+
|
|
219
|
+
lines, seen = [], []
|
|
220
|
+
for step in folded:
|
|
221
|
+
head = step.text.strip().splitlines()[0] if step.text.strip() else ""
|
|
222
|
+
if head:
|
|
223
|
+
lines.append(f"- {head[:STUB_CHARS]}")
|
|
224
|
+
for out in step.outputs:
|
|
225
|
+
for artifact_id in out.artifacts:
|
|
226
|
+
if artifact_id not in seen:
|
|
227
|
+
seen.append(artifact_id)
|
|
228
|
+
|
|
229
|
+
self.summary = (
|
|
230
|
+
f"{self.summary}\n" if self.summary else "") + (
|
|
231
|
+
f"[{len(folded)} earlier steps compacted]\n" + "\n".join(lines) +
|
|
232
|
+
(f"\nArtifacts from those steps, still viewable by ID: {', '.join(seen)}"
|
|
233
|
+
if seen else ""))
|
|
234
|
+
self.compactions += 1
|
|
235
|
+
|
|
236
|
+
# --- rendering --------------------------------------------------------
|
|
237
|
+
def render(self) -> list[dict[str, Any]]:
|
|
238
|
+
"""Provider-neutral messages. Blocks carry PIL images; backends encode them."""
|
|
239
|
+
opening = self.task if not self.summary else f"{self.task}\n\n{self.summary}"
|
|
240
|
+
messages: list[dict[str, Any]] = [
|
|
241
|
+
{"role": "user", "content": [{"type": "text", "text": opening}]}]
|
|
242
|
+
|
|
243
|
+
for step in self.steps:
|
|
244
|
+
assistant: list[dict[str, Any]] = []
|
|
245
|
+
if step.text:
|
|
246
|
+
assistant.append({"type": "text", "text": step.text})
|
|
247
|
+
for call in step.calls:
|
|
248
|
+
assistant.append({"type": "tool_use", "id": call.id,
|
|
249
|
+
"name": call.name, "input": call.arguments})
|
|
250
|
+
if assistant:
|
|
251
|
+
# `raw`, when a backend stored one, is that backend's own version of
|
|
252
|
+
# this turn. It may replay it instead of the neutral blocks; every
|
|
253
|
+
# other backend ignores the key.
|
|
254
|
+
messages.append({"role": "assistant", "content": assistant,
|
|
255
|
+
"raw": step.raw})
|
|
256
|
+
|
|
257
|
+
if step.outputs:
|
|
258
|
+
results = [{"type": "tool_result", "tool_use_id": out.call_id,
|
|
259
|
+
"content": self._result_blocks(out),
|
|
260
|
+
"is_error": out.is_error}
|
|
261
|
+
for out in step.outputs]
|
|
262
|
+
messages.append({"role": "user", "content": results})
|
|
263
|
+
elif step.followup:
|
|
264
|
+
messages.append({"role": "user",
|
|
265
|
+
"content": [{"type": "text", "text": step.followup}]})
|
|
266
|
+
return messages
|
|
267
|
+
|
|
268
|
+
@staticmethod
|
|
269
|
+
def _result_blocks(out: ToolOutput) -> list[dict[str, Any]]:
|
|
270
|
+
"""Images first, then text — the order the MCP server already established."""
|
|
271
|
+
if out.elided:
|
|
272
|
+
return [{"type": "text", "text": out.stub()}]
|
|
273
|
+
blocks: list[dict[str, Any]] = [{"type": "image", "image": im}
|
|
274
|
+
for im in out.images]
|
|
275
|
+
blocks.append({"type": "text", "text": out.text})
|
|
276
|
+
return blocks
|
visru/agent/loop.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""The agent loop: ask the model, run what it asks for, write it all down.
|
|
2
|
+
|
|
3
|
+
Deliberately thin. Everything hard lives elsewhere — the image budget in
|
|
4
|
+
`context.py`, the record in `trajectory.py`, what the evidence warrants in
|
|
5
|
+
`evidence/ledger.py` — and this file is what makes them take turns.
|
|
6
|
+
|
|
7
|
+
The loop ends one of three ways: the model submits a report the ledger accepts,
|
|
8
|
+
the model stops asking for tools, or it runs out of steps. All three are results;
|
|
9
|
+
only the first is an answer.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from collections import Counter
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from visru.agent.context import Context, ToolCall
|
|
19
|
+
from visru.agent.trajectory import Trajectory
|
|
20
|
+
from visru.backends import Backend, Response
|
|
21
|
+
from visru.tools import ToolContext, ToolResult, registry
|
|
22
|
+
from visru.tools.registry import USER_ERRORS
|
|
23
|
+
|
|
24
|
+
PROMPT_PATH = Path(__file__).parent / "prompts" / "system.md"
|
|
25
|
+
|
|
26
|
+
# Said when the model stops calling tools without reporting. Without this a loop
|
|
27
|
+
# just spins: the model says "I think it is a stop sign", nobody asks for
|
|
28
|
+
# evidence, and 200 steps later there is still no report.
|
|
29
|
+
NUDGE = ("Finish by calling submit_report with a claim for each thing you are "
|
|
30
|
+
"asserting, each with the artifact IDs that show it.")
|
|
31
|
+
|
|
32
|
+
# Said instead when the turn died at max_tokens. A truncated turn carries no call,
|
|
33
|
+
# so the nudge above would answer a question the model never finished asking, and
|
|
34
|
+
# the next turn spends another full output budget on more of the same.
|
|
35
|
+
TRUNCATED = ("Your last turn was cut off at the output limit before it called "
|
|
36
|
+
"anything. Call one tool now, with no preamble.")
|
|
37
|
+
|
|
38
|
+
# The same call, with the same arguments, this many times in a run: the model is
|
|
39
|
+
# not reading the answers. Counted over the whole run rather than consecutively,
|
|
40
|
+
# because a stuck model cycles rather than repeats — view, ocr, view, ocr, each
|
|
41
|
+
# pair identical to the last — and any "same as the previous turn" test sees two
|
|
42
|
+
# alternating signatures and never fires. Re-viewing an elided artifact is normal,
|
|
43
|
+
# so the limit leaves room for a few honest repeats.
|
|
44
|
+
REPEAT_LIMIT = 4
|
|
45
|
+
|
|
46
|
+
# Turns that died at max_tokens without calling anything, over the whole run — not
|
|
47
|
+
# in a row. An open model that starts repeating itself alternates: one degenerate
|
|
48
|
+
# turn, one small real call, one degenerate turn. Every consecutive counter resets
|
|
49
|
+
# on the call in between, so this one deliberately never resets.
|
|
50
|
+
TRUNCATION_LIMIT = 3
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def system_prompt() -> str:
|
|
54
|
+
return PROMPT_PATH.read_text()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class Session:
|
|
59
|
+
"""How a run ended, and where to read what happened."""
|
|
60
|
+
|
|
61
|
+
session: str
|
|
62
|
+
steps: int
|
|
63
|
+
reason: str # reported | stopped | stuck | exhausted
|
|
64
|
+
report: str = ""
|
|
65
|
+
trajectory: Path | None = None
|
|
66
|
+
elisions: int = 0
|
|
67
|
+
compactions: int = 0
|
|
68
|
+
peak_live_images: int = 0
|
|
69
|
+
usage: dict[str, int] = field(default_factory=dict)
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def reported(self) -> bool:
|
|
73
|
+
return self.reason == "reported"
|
|
74
|
+
|
|
75
|
+
def summary(self) -> str:
|
|
76
|
+
return (f"{self.session}: {self.reason} after {self.steps} steps "
|
|
77
|
+
f"({self.elisions} images elided, {self.compactions} compactions, "
|
|
78
|
+
f"peak {self.peak_live_images} live)")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _accrue(total: dict[str, int], usage: dict[str, int]) -> None:
|
|
82
|
+
for key, value in usage.items():
|
|
83
|
+
total[key] = total.get(key, 0) + value
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _child_runner(backend: Backend):
|
|
87
|
+
"""A subagent session on this model, offered every tool but the harness-only ones."""
|
|
88
|
+
tools = [name for name in registry.TOOLS if name not in registry.HARNESS_ONLY]
|
|
89
|
+
|
|
90
|
+
def run_child(task: str, child: ToolContext, max_steps: int) -> "Session":
|
|
91
|
+
return run(task, child, backend, max_steps=max_steps, tools=tools)
|
|
92
|
+
return run_child
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def run(task: str, ctx: ToolContext, backend: Backend, max_steps: int = 200,
|
|
96
|
+
system: str | None = None, tools: list[str] | None = None) -> Session:
|
|
97
|
+
"""Drive one session to an answer, a stop, or the step limit.
|
|
98
|
+
|
|
99
|
+
`tools` limits which tools the model is offered and may call; None means all.
|
|
100
|
+
"""
|
|
101
|
+
system = system_prompt() if system is None else system
|
|
102
|
+
context = Context(task=task, cfg=ctx.cfg, system=system)
|
|
103
|
+
ctx.run_child, ctx.child_usage = _child_runner(backend), {} # for `delegate`
|
|
104
|
+
log = Trajectory(ctx.root, ctx.session)
|
|
105
|
+
schemas = registry.schemas(tools)
|
|
106
|
+
log.session_header(ctx.cfg, task, system, [s["name"] for s in schemas])
|
|
107
|
+
|
|
108
|
+
usage: dict[str, int] = {}
|
|
109
|
+
peak, step, nudged = 0, -1, False
|
|
110
|
+
seen: Counter[tuple] = Counter()
|
|
111
|
+
truncations = 0
|
|
112
|
+
reason, report = "exhausted", ""
|
|
113
|
+
|
|
114
|
+
for step in range(max_steps):
|
|
115
|
+
response: Response = backend.complete(system, context.render(), schemas)
|
|
116
|
+
_accrue(usage, response.usage)
|
|
117
|
+
log.model_turn(step, response.text, response.usage)
|
|
118
|
+
context.add_turn(response.text, response.calls, raw=response.raw)
|
|
119
|
+
|
|
120
|
+
if not response.calls:
|
|
121
|
+
if response.stop_reason == "length":
|
|
122
|
+
truncations += 1
|
|
123
|
+
if truncations >= TRUNCATION_LIMIT:
|
|
124
|
+
reason = "stuck"
|
|
125
|
+
break
|
|
126
|
+
context.add_user(TRUNCATED)
|
|
127
|
+
continue
|
|
128
|
+
# Ask once for a report. If it still calls nothing, it has finished
|
|
129
|
+
# thinking, and spending the rest of the budget to prove that is waste.
|
|
130
|
+
if nudged:
|
|
131
|
+
reason = "stopped"
|
|
132
|
+
break
|
|
133
|
+
nudged = True
|
|
134
|
+
context.add_user(NUDGE)
|
|
135
|
+
continue
|
|
136
|
+
nudged = False
|
|
137
|
+
|
|
138
|
+
sig = tuple((c.name, json.dumps(c.arguments, sort_keys=True))
|
|
139
|
+
for c in response.calls)
|
|
140
|
+
seen[sig] += 1
|
|
141
|
+
if seen[sig] >= REPEAT_LIMIT:
|
|
142
|
+
reason = "stuck"
|
|
143
|
+
break
|
|
144
|
+
|
|
145
|
+
accepted, peak = _run_calls(ctx, response.calls, tools, context, log, step, peak)
|
|
146
|
+
if accepted is not None:
|
|
147
|
+
reason, report = "reported", accepted
|
|
148
|
+
break
|
|
149
|
+
|
|
150
|
+
_accrue(usage, ctx.child_usage) # subagents ran on this session's behalf
|
|
151
|
+
return Session(session=ctx.session, steps=step + 1, reason=reason, report=report,
|
|
152
|
+
trajectory=log.path, elisions=context.elisions,
|
|
153
|
+
compactions=context.compactions,
|
|
154
|
+
peak_live_images=max(peak, context.live_images), usage=usage)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _run_calls(ctx: ToolContext, calls: list[ToolCall], tools: list[str] | None,
|
|
158
|
+
context: Context, log: Trajectory, step: int,
|
|
159
|
+
peak: int) -> tuple[str | None, int]:
|
|
160
|
+
"""Run one turn's calls in order. Returns (accepted report text or None, peak)."""
|
|
161
|
+
for call in calls:
|
|
162
|
+
result, error = _execute(ctx, call, tools)
|
|
163
|
+
context.add_output(call, result)
|
|
164
|
+
log.tool_call(step, call.name, call.arguments, result.text,
|
|
165
|
+
result.artifacts, shown=bool(result.images), error=error)
|
|
166
|
+
peak = max(peak, context.live_images)
|
|
167
|
+
if call.name == "submit_report" and error is None:
|
|
168
|
+
return result.text, peak
|
|
169
|
+
return None, peak
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _execute(ctx: ToolContext, call: ToolCall,
|
|
173
|
+
allowed: list[str] | None = None) -> tuple[ToolResult, str | None]:
|
|
174
|
+
"""Run one call. Bad input comes back as a message, not an exception.
|
|
175
|
+
|
|
176
|
+
A raised error would end the session; the model can usually fix a bad box or a
|
|
177
|
+
misspelled asset ID on its next turn, and it only can if it is told what was
|
|
178
|
+
wrong. This mirrors what the MCP server does with ToolError.
|
|
179
|
+
"""
|
|
180
|
+
try:
|
|
181
|
+
return registry.execute(ctx, call.name, call.arguments, allowed), None
|
|
182
|
+
except USER_ERRORS as exc:
|
|
183
|
+
message = exc.args[0] if exc.args else str(exc)
|
|
184
|
+
return ToolResult(text=f"error: {message}"), str(message)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
You are a visual investigator. You answer questions about images and video by
|
|
2
|
+
looking at them with tools, not by guessing from a thumbnail.
|
|
3
|
+
|
|
4
|
+
The images you are shown are renders of much larger assets. Every coordinate you
|
|
5
|
+
send or receive is in **source-asset pixels** — the original file's resolution —
|
|
6
|
+
never the pixels of the render in front of you.
|
|
7
|
+
|
|
8
|
+
# How to work
|
|
9
|
+
|
|
10
|
+
1. **Start coarse.** Call `list_assets` to see what exists. For a video, call
|
|
11
|
+
`sample_frames` before anything else. Find the thing first, then look closely.
|
|
12
|
+
|
|
13
|
+
2. **Zoom before claiming.** A render below `inspected_scale` is not evidence.
|
|
14
|
+
Every tool result tells you its scale and says plainly whether it is at or
|
|
15
|
+
below that line; if it says BELOW, call `view` on a tighter box before you
|
|
16
|
+
assert anything about small text or small objects. Reading a whole 4K frame
|
|
17
|
+
at once produces confident nonsense.
|
|
18
|
+
|
|
19
|
+
3. **Refer to things by mark ID.** `detect`, `ocr` and `segment` return numbered
|
|
20
|
+
marks. Say "mark 3", and pass marks to `segment` or `overlay` with
|
|
21
|
+
`from_artifact`. Do not retype pixel coordinates you were just given.
|
|
22
|
+
|
|
23
|
+
4. **Check counts, text and locations a second way, with `cross_check`.** A
|
|
24
|
+
reading from one method is a guess that sounds like a fact. Get a second
|
|
25
|
+
method's output — a `detect` count against your own reading of a zoomed
|
|
26
|
+
`view`, an `ocr` line against what you can see — then call `cross_check` with
|
|
27
|
+
both artifact IDs and your reading. It compares them and returns a check
|
|
28
|
+
artifact. Two methods only count as independent if they do not share a model:
|
|
29
|
+
a `segment` mask prompted from a `detect` box inherits that detection's
|
|
30
|
+
errors, and `cross_check` will refuse the pair.
|
|
31
|
+
|
|
32
|
+
5. **Know what you have not looked at.** Call `coverage` before you report. It
|
|
33
|
+
lists what you viewed at or above `inspected_scale` and what you never
|
|
34
|
+
opened; say so in your report rather than implying you looked everywhere.
|
|
35
|
+
|
|
36
|
+
6. **Look at your overlays.** `overlay` draws a claim back onto the pixels. If the
|
|
37
|
+
shape does not sit on the thing it names, the claim is wrong — say so and fix
|
|
38
|
+
it rather than reporting it.
|
|
39
|
+
|
|
40
|
+
# Video
|
|
41
|
+
|
|
42
|
+
Frames are addressed as `vid_<id>#<frame_index>`; times are in seconds.
|
|
43
|
+
|
|
44
|
+
- **Find when first.** `search` with a short description ranks frames sampled
|
|
45
|
+
about once a second and merges neighbouring hits into time windows. A window
|
|
46
|
+
says where to look, not what happened.
|
|
47
|
+
- **Then look.** `sample_frames` with `t0`/`t1` around a candidate window, then
|
|
48
|
+
`view` a frame ref before claiming any detail. Narrow the window with a
|
|
49
|
+
second, tighter `sample_frames` rather than guessing between two cells.
|
|
50
|
+
- **Follow an object** with `track` from one frame: it reports when the object
|
|
51
|
+
is present and when it drops out. Check the frames either side of a change
|
|
52
|
+
with `view` — a mask that drifts is a lost track, not an event.
|
|
53
|
+
- **Long videos:** `delegate` gives each time window to a subagent and returns
|
|
54
|
+
their claims and artifact IDs. View the key frames yourself before reporting.
|
|
55
|
+
- **Answer times as a window**, `start..end` in seconds, e.g. `12.4..15.0`. A
|
|
56
|
+
`cross_check` of kind `temporal` compares your reading of frames against a
|
|
57
|
+
`search` window or a `track`.
|
|
58
|
+
|
|
59
|
+
# Artifacts
|
|
60
|
+
|
|
61
|
+
Every tool that shows you pixels returns an artifact ID like `a_0042`. Two uses:
|
|
62
|
+
|
|
63
|
+
- **Cite it.** Evidence in your report is a list of artifact IDs.
|
|
64
|
+
- **Ask for it back.** Older images are replaced in your context by a note like
|
|
65
|
+
`[a_0042 elided: source=im_9b9 box=[1100, 280, 1460, 580] scale=2.000; model
|
|
66
|
+
noted: the sign reads TRADER JOE'S]`. The pixels are gone, the artifact is not.
|
|
67
|
+
Call `view("a_0042")` to see it again exactly as it was. You do not need to
|
|
68
|
+
re-derive the box.
|
|
69
|
+
|
|
70
|
+
# Finishing
|
|
71
|
+
|
|
72
|
+
End by calling `submit_report` with one claim per thing you are asserting.
|
|
73
|
+
|
|
74
|
+
A claim carries a status, and the ledger checks that your evidence warrants it.
|
|
75
|
+
It will reject a status stronger than what you have shown, and it lists every
|
|
76
|
+
problem at once rather than one per attempt:
|
|
77
|
+
|
|
78
|
+
- No evidence? The claim must be `hypothesis`.
|
|
79
|
+
- Evidence but no passing independent check? `supported`.
|
|
80
|
+
- Evidence plus a `cross_check` artifact that agreed, cited in the claim's
|
|
81
|
+
`checks` as `"a_0009"`? `verified`. A check you describe yourself does not
|
|
82
|
+
count, however you word it.
|
|
83
|
+
- A `cross_check` that disagreed? `contested`.
|
|
84
|
+
- A DISAGREE is a reportable result, not to retry
|
|
85
|
+
|
|
86
|
+
Claim what you actually established. A `hypothesis` stated honestly is worth more
|
|
87
|
+
than a `verified` that gets refused, and far more than a confident wrong answer.
|
|
88
|
+
If you could not determine something, say so and report it as unresolved rather
|
|
89
|
+
than filling the gap.
|
visru/agent/subagents.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Subagents: one agent per time window of a video, run one after another.
|
|
2
|
+
|
|
3
|
+
A long video does not fit in one context at useful resolution: an hour at one
|
|
4
|
+
contact sheet a minute is sixty sheets, and the image budget holds sixteen. So
|
|
5
|
+
the parent splits the video into windows and hands each to a child session with
|
|
6
|
+
its own context, trajectory and ledger. The parent gets back text — each child's
|
|
7
|
+
claims and the artifact IDs behind them — never the child's images, so its own
|
|
8
|
+
context stays small, and it can `view` any of those artifacts itself.
|
|
9
|
+
|
|
10
|
+
A child is an ordinary `loop.run` on a session named `<parent>_w<k>`. That name
|
|
11
|
+
is how coverage credits the parent with what its children looked at.
|
|
12
|
+
|
|
13
|
+
ponytail: sequential. Children in threads would need a lock around artifact IDs
|
|
14
|
+
and one model instance per thread; add when wall-clock on long videos hurts.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
|
|
19
|
+
from visru.evidence.ledger import LEDGER_DIR
|
|
20
|
+
from visru.tools import ToolContext, ToolResult, resolve_ref
|
|
21
|
+
from visru.workspace.registry import parse_ref
|
|
22
|
+
|
|
23
|
+
MAX_WINDOWS = 8
|
|
24
|
+
|
|
25
|
+
CHILD_TASK = """You are one of several investigators, each given one time window of the \
|
|
26
|
+
same video.
|
|
27
|
+
|
|
28
|
+
Video: {asset} ({name}), {duration:.1f} s long.
|
|
29
|
+
Your window: {t0:.2f}..{t1:.2f} s. Look only there; others cover the rest.
|
|
30
|
+
|
|
31
|
+
Question: {question}
|
|
32
|
+
|
|
33
|
+
Investigate with your tools — sample_frames with t0/t1 inside your window is the \
|
|
34
|
+
place to start — then call submit_report. If what the question asks about happens \
|
|
35
|
+
in your window, give a claim whose text states its time window as "start..end" in \
|
|
36
|
+
seconds. If it does not, give a claim saying so, citing the frames you looked at. \
|
|
37
|
+
Every claim needs the artifact IDs that show it."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _windows(duration: float, windows: list[list[float]] | None, n: int
|
|
41
|
+
) -> list[tuple[float, float]]:
|
|
42
|
+
if windows is None:
|
|
43
|
+
if not 1 <= n <= MAX_WINDOWS:
|
|
44
|
+
raise ValueError(f"n must be 1..{MAX_WINDOWS}, got {n}")
|
|
45
|
+
return [(duration * i / n, duration * (i + 1) / n) for i in range(n)]
|
|
46
|
+
if not windows or len(windows) > MAX_WINDOWS:
|
|
47
|
+
raise ValueError(f"give 1..{MAX_WINDOWS} windows as [[t0, t1], ...] in seconds")
|
|
48
|
+
out = []
|
|
49
|
+
for w in windows:
|
|
50
|
+
if len(w) != 2:
|
|
51
|
+
raise ValueError(f"each window is [t0, t1] in seconds, got {w}")
|
|
52
|
+
t0, t1 = max(0.0, float(w[0])), min(duration, float(w[1]))
|
|
53
|
+
if t1 <= t0:
|
|
54
|
+
raise ValueError(f"window {w} is empty inside the video's 0..{duration:.2f}s")
|
|
55
|
+
out.append((t0, t1))
|
|
56
|
+
return out
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _claims(ctx: ToolContext, session: str) -> list[dict]:
|
|
60
|
+
path = ctx.root / LEDGER_DIR / f"{session}.json"
|
|
61
|
+
return json.loads(path.read_text())["claims"] if path.is_file() else []
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _claim_lines(head: str, claims: list[dict], cited: list[str]) -> str:
|
|
65
|
+
"""One window's report as text; the artifact IDs it cites are added to `cited`."""
|
|
66
|
+
lines = [head]
|
|
67
|
+
for c in claims:
|
|
68
|
+
refs = list(c.get("evidence", [])) + [
|
|
69
|
+
k["check"] for k in c.get("checks", []) if isinstance(k, dict)
|
|
70
|
+
and k.get("check")]
|
|
71
|
+
cited.extend(refs)
|
|
72
|
+
lines.append(f" {c['claim_id']} [{c['status']}] {c['text']}"
|
|
73
|
+
+ (f" evidence: {', '.join(refs)}" if refs else ""))
|
|
74
|
+
if not claims:
|
|
75
|
+
lines.append(" no report: treat this window as not inspected")
|
|
76
|
+
return "\n".join(lines)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def delegate(ctx: ToolContext, ref: str, question: str,
|
|
80
|
+
windows: list[list[float]] | None = None, n: int = 4,
|
|
81
|
+
max_steps: int = 15) -> ToolResult:
|
|
82
|
+
"""Run one subagent per time window and collect what each reported."""
|
|
83
|
+
if ctx.run_child is None:
|
|
84
|
+
raise ValueError("delegate runs subagents, which needs the visru agent loop; "
|
|
85
|
+
"under another harness, use its own subagents")
|
|
86
|
+
if not str(question).strip():
|
|
87
|
+
raise ValueError("delegate needs the question each subagent should answer")
|
|
88
|
+
asset_id, _ = parse_ref(resolve_ref(ctx, ref)[0])
|
|
89
|
+
asset = ctx.registry.get(asset_id)
|
|
90
|
+
if asset.kind != "video":
|
|
91
|
+
raise ValueError(f"{asset_id} is an image; delegate splits a video by time")
|
|
92
|
+
duration = asset.duration or 0.0
|
|
93
|
+
spans = _windows(duration, windows, n)
|
|
94
|
+
|
|
95
|
+
blocks, cited = [], []
|
|
96
|
+
for t0, t1 in spans:
|
|
97
|
+
ctx.children += 1
|
|
98
|
+
child = ToolContext(ctx.root, ctx.cfg, session=f"{ctx.session}_w{ctx.children}")
|
|
99
|
+
task = CHILD_TASK.format(asset=asset_id, name=asset.name, duration=duration,
|
|
100
|
+
t0=t0, t1=t1, question=question)
|
|
101
|
+
result = ctx.run_child(task, child, max_steps) # no delegate inside: no recursion
|
|
102
|
+
for key, value in result.usage.items():
|
|
103
|
+
ctx.child_usage[key] = ctx.child_usage.get(key, 0) + value
|
|
104
|
+
|
|
105
|
+
claims = _claims(ctx, child.session) if result.reported else []
|
|
106
|
+
head = (f"[{t0:.2f}..{t1:.2f}s] {child.session}: {result.reason} after "
|
|
107
|
+
f"{result.steps} steps")
|
|
108
|
+
blocks.append(_claim_lines(head, claims, cited))
|
|
109
|
+
|
|
110
|
+
text = (f"delegated {len(spans)} windows of {asset_id} ({duration:.1f}s) to subagents:\n"
|
|
111
|
+
+ "\n".join(blocks)
|
|
112
|
+
+ "\nTheir artifacts are yours to view or cite by ID, and a check they ran "
|
|
113
|
+
"can go in your own claim's checks. Their conclusions are not yours until you "
|
|
114
|
+
"have looked: view the key frames before reporting.")
|
|
115
|
+
return ToolResult(text=text, artifacts=list(dict.fromkeys(cited)))
|
|
116
|
+
|