ph-utcp 0.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ph_utcp/__init__.py ADDED
@@ -0,0 +1,1054 @@
1
+ """`utcp` — tools published as UTCP manuals, as tools a pH agent can call.
2
+
3
+ One row. Its config lists manuals — each a UTCP *manual call template*, the
4
+ object any UTCP harness registers — and another row can add one at mount through
5
+ `ctx.utcp.add`. For each manual the row runs discovery, and every tool the manual
6
+ names becomes a registered pH tool: in the prompt, through the pipeline, in the
7
+ log, and in Code Mode's listing like any other.
8
+
9
+ ## A UTCP tool sees what its agent sees
10
+
11
+ A `cli` tool's command is started by `ctx.shell` for the agent making the call
12
+ (`_library.PhCliProtocol`): in that agent's workspace, under that agent's
13
+ sandbox, ended by the call's own cancellation. So a UTCP tool is bounded the way
14
+ `bash` is bounded — by the sandbox, and not by `permissions-fs`, which governs
15
+ calls through `ctx.fs` and says so of itself. A manual whose tool keeps state
16
+ outside the workspace names those paths in `writable`, or under confinement the
17
+ tool cannot write them.
18
+
19
+ ## Read-only bars the workspace, not a tool's working state
20
+
21
+ A tool that builds something *about* a tree — an index, a graph — changes no
22
+ file of it. A manual names a directory for that (`state`), and each call is told
23
+ where it is for the agent making it: `$PH_TOOL_STATE`, a directory of that
24
+ agent's scratch. The scratch is writable under every sandbox mode and the
25
+ workspace only under `workspace-write`, so the tool works in a read-only session
26
+ and cannot write the repository there; and it is per session and agent, so an
27
+ agent that restarts finds what it built. `writable` is for what agents share — a
28
+ grammar cache, a model's weights — and a grant that overlaps the workspace is
29
+ refused (`ph.seams.sandbox.grants_apart`), so no manual reopens the tree. A
30
+ confined command's temporary files go in the scratch too (`TMPDIR`, set by the
31
+ shell seam), for the same reason.
32
+
33
+ ## A host, for a manual whose tools are slow to start
34
+
35
+ A manual can name a **host** (`Manual.host`): a command that stays up and answers
36
+ the manual's tools on a socket, so what they load — an embedding model — is
37
+ loaded once, and a long piece of work can go on past the call that asked for it.
38
+ The row keeps one running for each tree an agent works in, started as every
39
+ command is: by `ctx.shell`, for that agent, in its workspace and under its
40
+ sandbox. It is a job of that agent's (`ctx.jobs`), so it ends when the agent
41
+ does and when the manual's row unmounts; it leaves by itself when idle, and the
42
+ next call starts another. Each call is told the socket in `$PH_TOOL_HOST`, and a
43
+ command that finds nobody there runs by itself — a host is never required.
44
+
45
+ While a host works in the background the footer says how far it has got, and
46
+ when it finishes the agent it works for is told, with a note that waits for that
47
+ agent's next step and starts no turn.
48
+
49
+ **One host a tree, not one an agent.** A child builds its own only when it has a
50
+ tree it writes for keeps — a worktree, an overlay. One that works in its
51
+ parent's checkout, or was granted only to read the project, uses its parent's
52
+ host and its parent's state (`UtcpSeam.owner_of`): the index is of the tree, and
53
+ a copy for every child of a read-only fan-out would be the same documents
54
+ embedded again and a model held in memory for each.
55
+
56
+ ## What UTCP cannot say, the row's config says
57
+
58
+ A manual carries a name, a description and two schemas. It has no word for "two
59
+ of these may overlap" or "this changes nothing", and pH's scheduler and policy
60
+ rows read exactly those. Each defaults to the careful answer — not safe to
61
+ overlap, has effects — and a manual's `tools` table states otherwise per tool.
62
+
63
+ ## A missing provider is a missing capability, not a failed start
64
+
65
+ A manual whose discovery fails — the command is not installed, the endpoint is
66
+ down, the protocol is not one this deployment allows — is **skipped**: a warning
67
+ in the log, a line in `phern doctor`, and a note in the system prompt so the
68
+ model is not left guessing why a skill names a tool it cannot find. The mount
69
+ goes ahead with the manuals that answered.
70
+
71
+ ## Not stored
72
+
73
+ A template's `${VAR}` references are resolved by the UTCP client from the
74
+ environment when a call is made. Nothing this row holds is a credential.
75
+
76
+ @module ph_utcp
77
+ """
78
+
79
+ from __future__ import annotations
80
+
81
+ import logging
82
+ import os
83
+ import re
84
+ import secrets
85
+ import time
86
+ from collections.abc import Awaitable, Sequence
87
+ from dataclasses import dataclass, field
88
+ from functools import partial
89
+ from pathlib import Path
90
+ from typing import Any, Literal
91
+
92
+ import anyio
93
+ from pydantic import Field
94
+
95
+ from ph.agent.types import AgentHandle, PreStepDecision, PreStepRequest
96
+ from ph.cancel import Canceled, Cancellation
97
+ from ph.cordis import Context, Disposer, Next, ServiceKey, plugin
98
+ from ph.json import as_obj, as_str, dumps
99
+ from ph.keys import AGENTS, FS, JOBS, SHELL, SYSTEM_PROMPT, TOOLS, TUI_STATUS
100
+ from ph.llm.types import CONTEXT_SUMMARY_MAX_CHARS, PluginSource, create_user_message
101
+ from ph.paths import RuntimeDirError, file_stamp, resolve_roots
102
+ from ph.seams._registry import contribute_item
103
+ from ph.seams.diagnostics import Diagnostic, contribute
104
+ from ph.seams.jobs import UNSETTLED, Job
105
+ from ph.seams.sandbox import allowance_path, grants_apart
106
+ from ph.seams.shell import ShellResult
107
+ from ph.seams.tui_status import StatusField, StatusReading
108
+ from ph.seams.workspace import project_access, scratch_of, workspace_of
109
+ from ph.session import Session
110
+ from ph.system_prompt.assembly import ORDER_TOOL_GUIDANCE, AssembleContext, PromptSection
111
+ from ph.text import count_of
112
+ from ph.toolcli import (
113
+ AGENT_ENV,
114
+ HOST_ENV,
115
+ PROGRESS_FILE,
116
+ STATE_ENV,
117
+ Progress,
118
+ read_envelope,
119
+ read_progress,
120
+ value_schema,
121
+ )
122
+ from ph.tools.definition import (
123
+ ToolDefinition,
124
+ ToolRunContext,
125
+ define_tool,
126
+ text_content,
127
+ )
128
+ from ph.tools.errors import HarnessError
129
+ from ph.tools.json_schema import validate_json_schema_value
130
+ from ph.tools.presentation import CardKind, simple_views
131
+ from ph.wire import WireModel
132
+
133
+ from ._library import (
134
+ Call,
135
+ CallTemplateSerializer,
136
+ Tool,
137
+ UtcpClient,
138
+ calling,
139
+ install,
140
+ require_ran,
141
+ )
142
+
143
+ __all__ = [
144
+ "BUNDLE",
145
+ "UTCP",
146
+ "Config",
147
+ "Host",
148
+ "Manual",
149
+ "ManualState",
150
+ "ToolTraits",
151
+ "UtcpSeam",
152
+ "apply",
153
+ "command_said",
154
+ ]
155
+
156
+ BUNDLE = Path(__file__).parent / "bundle.yaml"
157
+ """This distribution's profile layer, for the `ph.bundles` group."""
158
+
159
+ log = logging.getLogger("ph_utcp")
160
+
161
+ UTCP_FAILED = "UTCP_FAILED"
162
+ UTCP_BAD_ARGUMENTS = "UTCP_BAD_ARGUMENTS"
163
+ UTCP_BAD_ANSWER = "UTCP_BAD_ANSWER"
164
+
165
+ _NOT_A_WORD = re.compile(r"\W")
166
+ _CLIP = 600
167
+ """How much of an answer that was not what the tool declared is quoted back."""
168
+
169
+ _HOST_STARTS_S = 15.0
170
+ """How long a call waits for a host it started to listen, before running alone."""
171
+
172
+ _WATCH_EVERY_S = 0.5
173
+ """How often a running host's progress is looked at while a run works. Kept
174
+ here, in memory, so the footer — drawn far more often — never reads a file."""
175
+
176
+ _WATCH_IDLE_S = 2.0
177
+ """How often it is looked at between runs: a new one is noticed this late."""
178
+
179
+
180
+ # ------------------------------------------------------------------- config ----
181
+
182
+
183
+ class ToolView(WireModel):
184
+ """How a front end draws one tool's card: a title over one salient argument."""
185
+
186
+ card: CardKind = "generic"
187
+ title: str
188
+ argument: str
189
+
190
+
191
+ class ToolTraits(WireModel):
192
+ """What a manual cannot say about one of its tools. Every default is the
193
+ careful answer."""
194
+
195
+ concurrency_safe: bool = False
196
+ """Two calls may overlap in one batch."""
197
+ effect_free: bool = False
198
+ """A call changes nothing."""
199
+ self_limits: bool = False
200
+ """The tool bounds its own output and offers a way to page."""
201
+ arguments_disposable: bool = False
202
+ """The arguments are a payload delivered, not something the model refers back to."""
203
+ timeout_ms: int | None = None
204
+ """This tool's budget, over the row's."""
205
+ view: ToolView | None = None
206
+
207
+
208
+ class Host(WireModel):
209
+ """A command that stays up and answers a manual's tools: `ph.toolcli`'s `serve`.
210
+
211
+ The row adds `--socket`, `--root` and `--idle` to `command`, and `--warm` for
212
+ each of `warm` when it starts the host ahead of any call; the state directory
213
+ it is told as each call is, in `$PH_TOOL_STATE`."""
214
+
215
+ command: list[str]
216
+ """The command, up to and including `serve`."""
217
+ label: str
218
+ """What the footer and the completion note call it: `text index`."""
219
+ warm: list[str] = Field(default_factory=list)
220
+ """Tools the host calls once when it is started `eager`ly: an index tool,
221
+ which in a host answers at once and works in the background."""
222
+ eager: bool = True
223
+ """Start it for a root agent at that agent's first step, and not at its first
224
+ call — so an index is being built before anything asks for it. A child that
225
+ works in a tree of its own gets its host at its first call either way: a
226
+ throwaway worktree may never ask."""
227
+ idle_s: float = 900.0
228
+ """How long the host stays up with nothing to do."""
229
+
230
+
231
+ class Manual(WireModel):
232
+ """One UTCP manual, and how pH runs what it names."""
233
+
234
+ template: dict[str, Any]
235
+ """The manual call template, as any UTCP harness would register it. Its `name`
236
+ is the manual's name."""
237
+ names: Literal["prefixed", "bare"] = "prefixed"
238
+ """How its tools are named in pH. `prefixed` is `<manual>_<tool>`, which cannot
239
+ collide with a tool pH or another manual registers. `bare` is the tool's own
240
+ name, for a manual that is the only source of those tools."""
241
+ envelope: bool = False
242
+ """The tool answers `{ok, value, text, error}` — `ph.toolcli`'s envelope.
243
+
244
+ Then the pH tool's value is the envelope's `value`, its content is the
245
+ envelope's `text`, and a failure is raised with the envelope's code: the tool
246
+ reads to an agent exactly as it would registered in-process. Stated rather
247
+ than detected, because an object with an `ok` key is not a promise."""
248
+ state: str = Field("", pattern=r"^([A-Za-z0-9][A-Za-z0-9._-]*)?$")
249
+ """A directory name for this manual's working state, or empty for none.
250
+
251
+ Each call's command is then told `$PH_TOOL_STATE=<the calling agent's
252
+ scratch>/<state>`: writable under every sandbox mode, apart from the
253
+ workspace, and the same directory for the same agent after a restart. One
254
+ name, not a path — where it goes is the agent's, and not the manual's to say."""
255
+ writable: list[str] = Field(default_factory=list)
256
+ """Paths outside the workspace this manual's commands may write: what agents
257
+ share. Honored under every sandbox mode; one that overlaps the workspace is
258
+ refused."""
259
+ env: dict[str, str] = Field(default_factory=dict)
260
+ """Variables set on this manual's commands, over what its templates say."""
261
+ pass_env: list[str] = Field(default_factory=list)
262
+ """Variables this manual's commands are handed from the harness's own
263
+ environment, by name — a credential.
264
+
265
+ A command's environment starts as the deployment's scrubbed one, which drops
266
+ every variable whose name looks like a credential, and a template cannot
267
+ name its way past that. This is how a deployment says one may pass: named in
268
+ a profile, read each time a command is started, and held nowhere."""
269
+ host: Host | None = None
270
+ """A command that stays up and answers this manual's tools, or none."""
271
+ tools: dict[str, ToolTraits] = Field(default_factory=dict)
272
+ """Per tool, by the tool's own name."""
273
+
274
+
275
+ class Config(WireModel):
276
+ """Row config for `utcp`."""
277
+
278
+ manuals: list[Manual] = Field(default_factory=list)
279
+ protocols: list[str] = Field(default_factory=lambda: ["cli"])
280
+ """The call template types a manual may use.
281
+
282
+ `cli` alone by default, because it is the one this row runs through pH's own
283
+ doors. Another type — `http`, `mcp` — runs inside the harness process, by the
284
+ UTCP library's own client, outside the sandbox and its egress rules; naming
285
+ it here is how a deployment says it means that."""
286
+ timeout_ms: int = 120_000
287
+ """How long one call's command may run, unless the tool's own entry says."""
288
+ discovery_timeout_ms: int = 60_000
289
+ """How long a manual's discovery command may run."""
290
+
291
+
292
+ # --------------------------------------------------------------------- seam ----
293
+
294
+
295
+ UTCP: ServiceKey[UtcpSeam] = ServiceKey("utcp")
296
+ """The UTCP manuals this mount holds, and the door another row adds one through."""
297
+
298
+
299
+ @dataclass(frozen=True, slots=True)
300
+ class ManualState:
301
+ """What became of one manual: the tools it gave, or why it gave none."""
302
+
303
+ tools: tuple[str, ...] = ()
304
+ """The pH names registered."""
305
+ problems: tuple[str, ...] = ()
306
+ """Why the manual, or one of its tools, is not available."""
307
+
308
+ @property
309
+ def available(self) -> bool:
310
+ return bool(self.tools)
311
+
312
+ def said(self) -> str:
313
+ """The tools, as a row's `phern doctor` line says them, and why any are
314
+ missing."""
315
+ if self.available and not self.problems:
316
+ return f"{', '.join(self.tools)} (through UTCP)"
317
+ reasons = "; ".join(self.problems) or "the manual named no tools"
318
+ return f"{', '.join(self.tools) or 'unavailable'} — {reasons}"
319
+
320
+
321
+ def command_said(program: str, verb: str, result: ShellResult) -> str:
322
+ """What a manual's own command said, for a person: its output, or that it
323
+ failed and how."""
324
+ said = result.stdout.strip() or result.stderr.strip()
325
+ if result.exit_code != 0:
326
+ return f"{program} {verb} failed (exit {result.exit_code}): {said}"
327
+ return said
328
+
329
+
330
+ @dataclass(slots=True)
331
+ class _Hosted:
332
+ """One running host: whose tree it serves, where it listens, how far it is."""
333
+
334
+ name: str
335
+ """The manual it answers for."""
336
+ host: Host
337
+ owner: AgentHandle
338
+ socket: Path
339
+ home: Path | None
340
+ """Its state directory, where it reports progress."""
341
+ job: Job | None = None
342
+ progress: Progress | None = None
343
+ """What it last reported, as `_watch` read it: nothing until it reports,
344
+ since a report an earlier host left in the directory is not its own."""
345
+ since: float = field(default_factory=time.time)
346
+ """When it was started: a report of a run that began before is one an
347
+ earlier host left in the directory."""
348
+ told: set[float] = field(default_factory=set)
349
+ """The runs, by when each began, whose end its agent has been told of."""
350
+ listening: bool = False
351
+ """Its socket has been seen. Asked once, and a field after."""
352
+
353
+ @property
354
+ def running(self) -> bool:
355
+ return self.job is not None and self.job.state in UNSETTLED
356
+
357
+
358
+ @dataclass(slots=True)
359
+ class UtcpSeam:
360
+ """The service published as `ctx.utcp`."""
361
+
362
+ ctx: Context
363
+ config: Config
364
+ client: Any
365
+ """The UTCP client. `Any` because the library ships no types."""
366
+ states: dict[str, ManualState] = field(default_factory=dict)
367
+ manuals: dict[str, Manual] = field(default_factory=dict)
368
+ """The manuals that gave tools, by name — what a host is started from."""
369
+ hosts: dict[tuple[str, str], _Hosted] = field(default_factory=dict)
370
+ """Running hosts, by manual and the id of the agent whose tree each serves."""
371
+ socket_dir: Path | None = None
372
+ """Where hosts — and anything else a manual's command asks over a socket —
373
+ listen: `$PH_RUNTIME/utcp`, made once at mount. `None` on a deployment with no
374
+ usable runtime directory, which then runs every call alone."""
375
+ warmed: set[tuple[str, str]] = field(default_factory=set)
376
+ """The trees, by manual and owner, whose host has been started unasked once.
377
+ Once: a host that leaves when idle is started again by the next call, and
378
+ not by the next step, which would index the tree again for nobody."""
379
+
380
+ # ----------------------------------------------------------- registering --
381
+
382
+ async def add(self, manual: Manual, *, scope: Context | None = None) -> ManualState:
383
+ """Discover `manual` and register its tools on `scope`. Never raises for a
384
+ manual that cannot be reached: the state says why, and so does the log.
385
+
386
+ `scope` is who the manual belongs to, so a row that adds its own gets its
387
+ tools unregistered, and the manual forgotten, when it unmounts. Adding a
388
+ name that is already held and gave tools changes nothing and answers with
389
+ the state there is — the tools are registered, and a second discovery
390
+ would only find the registry refusing each of them by name.
391
+ """
392
+ name = _NOT_A_WORD.sub("_", as_str(manual.template.get("name")))
393
+ if not name:
394
+ state = ManualState(problems=("its template has no `name`",))
395
+ log.warning("ph_utcp: a manual was skipped: its template has no `name`")
396
+ return state
397
+ held = self.states.get(name)
398
+ if held is not None and held.available:
399
+ return held
400
+ made, unmade = await anyio.to_thread.run_sync(
401
+ _made, manual.writable, allowance_path(str(self.ctx.require(FS).root_for(None)))
402
+ )
403
+ manual = manual.model_copy(update={"writable": made})
404
+ state = await self._discovered(name, manual, scope)
405
+ if unmade:
406
+ state = ManualState(tools=state.tools, problems=(*state.problems, *unmade))
407
+ self.states[name] = state
408
+ if state.available:
409
+ self.manuals[name] = manual
410
+ for problem in state.problems:
411
+ log.warning("ph_utcp: %s: %s", name, problem)
412
+
413
+ async def forget() -> None:
414
+ """What the owning scope's unwinding takes back: the state, so the name
415
+ can be added again, the hosts it started, and the client's own record
416
+ of the manual."""
417
+ if self.states.get(name) is state:
418
+ del self.states[name]
419
+ self.manuals.pop(name, None)
420
+ for key in [key for key in self.hosts if key[0] == name]:
421
+ job = self.hosts.pop(key).job
422
+ if job is not None:
423
+ job.cancel()
424
+ try:
425
+ await self.client.deregister_manual(name)
426
+ except Exception:
427
+ log.debug("ph_utcp: %s was not deregistered", name, exc_info=True)
428
+
429
+ await (scope or self.ctx).effect(lambda: forget, label=f"utcp-manual({name})")
430
+ return state
431
+
432
+ async def _discovered(self, name: str, manual: Manual, scope: Context | None) -> ManualState:
433
+ template = manual.template
434
+ protocol = as_str(template.get("call_template_type"))
435
+ if protocol not in self.config.protocols:
436
+ return ManualState(
437
+ problems=(
438
+ f'it uses the "{protocol}" protocol, which this deployment does not '
439
+ f"allow (the `utcp` row's `protocols` is {self.config.protocols})",
440
+ )
441
+ )
442
+ try:
443
+ call_template = CallTemplateSerializer().validate_dict({**template, "name": name})
444
+ if await self.client.config.tool_repository.get_manual(name) is not None:
445
+ await self.client.deregister_manual(name)
446
+ call = self._call(manual, timeout_ms=self.config.discovery_timeout_ms)
447
+ with calling(call):
448
+ result = await self.client.register_manual(call_template)
449
+ if protocol == "cli":
450
+ require_ran(call, f"discovery of {name}")
451
+ except Exception as error:
452
+ log.debug("ph_utcp: %s was not discovered", name, exc_info=True)
453
+ return ManualState(problems=(f"discovery failed: {error}",))
454
+ if not result.success:
455
+ reasons = "; ".join(str(one) for one in result.errors) or "no reason given"
456
+ return ManualState(problems=(f"discovery failed: {reasons}",))
457
+
458
+ registered: list[str] = []
459
+ problems: list[str] = []
460
+ for tool in result.manual.tools:
461
+ definition = self._definition(name, manual, tool)
462
+ try:
463
+ self.ctx.require(TOOLS).register(definition, scope=scope)
464
+ except ValueError as error:
465
+ problems.append(f"{definition.name} was not registered: {error}")
466
+ else:
467
+ registered.append(definition.name)
468
+ if not registered and not problems:
469
+ problems.append("its manual names no tools")
470
+ return ManualState(tools=tuple(registered), problems=tuple(problems))
471
+
472
+ def _call(
473
+ self,
474
+ manual: Manual,
475
+ *,
476
+ timeout_ms: int,
477
+ agent: AgentHandle | None = None,
478
+ owner: AgentHandle | None = None,
479
+ signal: Cancellation | None = None,
480
+ host: Path | None = None,
481
+ ) -> Call:
482
+ """How one of `manual`'s commands runs for `agent`, whose tree's owner is
483
+ `owner`: asked once by the caller, because each of these needs it."""
484
+ env, writable = self._surroundings(manual, agent, owner)
485
+ if host is not None:
486
+ env = {**env, HOST_ENV: str(host)}
487
+ return Call(
488
+ shell=self.ctx.require(SHELL),
489
+ root=self._tree(manual, agent, owner),
490
+ timeout_ms=timeout_ms,
491
+ env=env,
492
+ writable=writable,
493
+ agent=agent,
494
+ signal=signal,
495
+ )
496
+
497
+ def _surroundings(
498
+ self, manual: Manual, agent: AgentHandle | None, owner: AgentHandle | None
499
+ ) -> tuple[dict[str, str], tuple[str, ...]]:
500
+ """What a command of this manual run for `agent` is given: its variables,
501
+ and the paths outside the workspace it may write. One derivation, for a
502
+ tool call, the host that answers it and a person's provisioning command."""
503
+ env = dict(manual.env)
504
+ for name in manual.pass_env:
505
+ # From the harness's own environment and not the scrubbed one, which
506
+ # is the point: read now, set on this one command, kept nowhere.
507
+ value = os.environ.get(name)
508
+ if value:
509
+ env[name] = value
510
+ writable = list(manual.writable)
511
+ if owner is not None:
512
+ env[AGENT_ENV] = owner.id
513
+ state = self._state(manual, owner)
514
+ if state is not None:
515
+ env[STATE_ENV] = str(state)
516
+ held = workspace_of(self.ctx, agent)
517
+ if held is not None and not state.is_relative_to(held.scratch):
518
+ # Its tree's owner's state, and not in its own scratch: granted,
519
+ # for the call that finds no host and does the work itself.
520
+ writable.append(str(state))
521
+ if manual.host is not None and self.socket_dir is not None:
522
+ # Where a host listens. A caller connects there and a host binds
523
+ # there, and both are confined.
524
+ writable.append(str(self.socket_dir))
525
+ return env, tuple(writable)
526
+
527
+ def _tree(self, manual: Manual, agent: AgentHandle | None, owner: AgentHandle | None) -> Path:
528
+ """Where `manual`'s command runs for `agent`: the tree its state is about —
529
+ the owner's — for a manual that keeps state, and the agent's own root for
530
+ one that does not.
531
+
532
+ The working directory is how a command is told its tree (`ph.toolcli`'s
533
+ default `--root`), so a child that shares its parent's index works on its
534
+ parent's tree rather than indexing its own empty directory into that
535
+ index and sweeping every document out as gone."""
536
+ fs = self.ctx.require(FS)
537
+ return fs.root_for(owner if manual.state else agent)
538
+
539
+ def run_command(
540
+ self, manual: Manual, argv: Sequence[str], agent: AgentHandle | None, *, timeout_ms: int
541
+ ) -> Awaitable[ShellResult]:
542
+ """Run one of `manual`'s own commands for `agent`, as a person asked —
543
+ `install`, `status` — with what a call of its tools is given: the
544
+ variables, the state directory, the tree.
545
+
546
+ Unconfined, as a provisioning command a person types is: `install` needs
547
+ the network, and nobody's model chose it."""
548
+ owner = None if agent is None else self.owner_of(agent)
549
+ env, _ = self._surroundings(manual, agent, owner)
550
+ return self.ctx.require(SHELL).run_argv(
551
+ tuple(argv),
552
+ cwd=self._tree(manual, agent, owner),
553
+ timeout_ms=timeout_ms,
554
+ extra_env=env,
555
+ )
556
+
557
+ def owner_of(self, agent: AgentHandle) -> AgentHandle:
558
+ """The agent whose tree `agent`'s tools are about: itself, or the nearest
559
+ ancestor whose tree it shares.
560
+
561
+ What a tool builds about a tree belongs to the tree, so the question is
562
+ whether a child has a tree of its own to build about:
563
+
564
+ * **It works in its parent's checkout** (`shared`, the same root). One
565
+ tree, so one index and one host.
566
+ * **It was granted only to read the project** — `readonly-scratch`, whose
567
+ root is an empty directory beside a repository it cannot write, and the
568
+ two ephemeral kinds, whose writes reach nobody. It has nothing of its
569
+ own worth indexing, and a fan-out of such children is the case this
570
+ rule is for: they ask their parent's index, and none of them embeds a
571
+ document or holds a model.
572
+ * **It has a tree it writes for keeps** — a `worktree`, an `overlay`. Its
573
+ files diverge from its parent's, so it is the owner of that tree and
574
+ builds its own.
575
+
576
+ `project_access` is the seam's own reading of a kind, the one a spawn
577
+ records as what the child was granted."""
578
+ own = workspace_of(self.ctx, agent)
579
+ while own is not None:
580
+ parent = self._parent(agent)
581
+ theirs = None if parent is None else workspace_of(self.ctx, parent)
582
+ if parent is None or theirs is None:
583
+ break
584
+ if theirs.root != own.root and project_access(own.kind) == "write":
585
+ break
586
+ agent, own = parent, theirs
587
+ return agent
588
+
589
+ def _parent(self, agent: AgentHandle) -> AgentHandle | None:
590
+ """The agent that delegated to `agent`, if it is a sub-agent and that
591
+ agent is still here. An agent's id is its session's."""
592
+ session = agent.session
593
+ wanted = None if session is None else session.header.delegating_parent
594
+ registry = self.ctx.get(AGENTS)
595
+ return None if wanted is None or registry is None else registry.get(wanted)
596
+
597
+ def _state(self, manual: Manual, owner: AgentHandle | None) -> Path | None:
598
+ """The directory `manual` named, in `owner`'s scratch.
599
+
600
+ `None` when the manual names none, and when there is no agent to have a
601
+ scratch — discovery, or a call made for the deployment. The command then
602
+ uses its own default, and nothing confines it: `ctx.shell` confines a
603
+ command only for an agent that holds a workspace, and that agent has a
604
+ scratch. The directory is the command's to make; the scratch is there."""
605
+ if not manual.state or owner is None:
606
+ return None
607
+ session = owner.session
608
+ if session is not None:
609
+ return scratch_of(self.ctx, session.id, owner) / manual.state
610
+ held = workspace_of(self.ctx, owner)
611
+ return None if held is None else held.scratch / manual.state
612
+
613
+ # ---------------------------------------------------------------- hosts --
614
+
615
+ async def _host_for(
616
+ self, name: str, owner: AgentHandle | None, *, warm: bool = False, wait: bool = True
617
+ ) -> Path | None:
618
+ """The socket of the host answering manual `name` for `owner`'s tree,
619
+ started if there is none. `None` when there is no host to be had — the
620
+ manual names none, there is no agent, no job runner or no runtime
621
+ directory, or the host did not come up — and the call then runs alone.
622
+ `wait=False` starts it and answers at once, for a caller that will not
623
+ use it.
624
+
625
+ Never raises: a host is how a call is made faster, and a call that could
626
+ have been answered without one must not fail for want of it.
627
+ """
628
+ manual = self.manuals.get(name)
629
+ jobs = self.ctx.get(JOBS)
630
+ if manual is None or manual.host is None or owner is None:
631
+ return None
632
+ if jobs is None or self.socket_dir is None:
633
+ return None
634
+ key = (name, owner.id)
635
+ hosted = self.hosts.get(key)
636
+ if hosted is None or not hosted.running:
637
+ hosted = self.hosts[key] = _Hosted(
638
+ name=name,
639
+ host=manual.host,
640
+ owner=owner,
641
+ # Short, because a socket's address is at most 108 bytes, and
642
+ # random, because two roots of one daemon share this directory.
643
+ socket=self.socket_dir / f"{secrets.token_hex(6)}.sock",
644
+ home=self._state(manual, owner),
645
+ )
646
+ try:
647
+ hosted.job = await jobs.start(
648
+ kind="utcp-host",
649
+ label=f"{manual.host.label} for {owner.id}",
650
+ run=partial(self._hosting, hosted, manual, warm),
651
+ # The owner's: the host ends when the agent whose tree it
652
+ # serves does. Stopped as well when the manual's row unmounts.
653
+ scope=owner.ctx,
654
+ )
655
+ except Exception:
656
+ log.warning("ph_utcp: no host for %s was started", name, exc_info=True)
657
+ self.hosts.pop(key, None)
658
+ return None
659
+ if not wait:
660
+ return None
661
+ return hosted.socket if await self._listening(hosted) else None
662
+
663
+ async def _listening(self, hosted: _Hosted) -> bool:
664
+ """Wait, briefly, for `hosted` to listen. A host is mounting its rows for
665
+ its first fraction of a second; a call that did not wait would run alone
666
+ beside it and load what the host is about to."""
667
+ if hosted.listening:
668
+ return hosted.running
669
+ with anyio.move_on_after(_HOST_STARTS_S):
670
+ while hosted.running:
671
+ if await anyio.to_thread.run_sync(hosted.socket.exists):
672
+ hosted.listening = True
673
+ return True
674
+ await anyio.sleep(0.02)
675
+ return False
676
+
677
+ async def _hosting(self, hosted: _Hosted, manual: Manual, warm: bool, job: Job) -> None:
678
+ """One host's life: the command, run for its tree's owner until it leaves
679
+ or its job is canceled, and its progress read while it runs."""
680
+ host, owner = hosted.host, hosted.owner
681
+ root = self._tree(manual, owner, owner)
682
+ env, writable = self._surroundings(manual, owner, owner)
683
+ argv = [*host.command, "--socket", str(hosted.socket), "--root", str(root)]
684
+ argv += ["--idle", f"{host.idle_s:g}"]
685
+ if warm:
686
+ for tool in host.warm:
687
+ argv += ["--warm", tool]
688
+ inherited = manual.template.get("inherit_env_vars")
689
+ try:
690
+ async with anyio.create_task_group() as tasks:
691
+ tasks.start_soon(self._watch, hosted)
692
+ result = await self.ctx.require(SHELL).run_argv(
693
+ tuple(argv),
694
+ cwd=root,
695
+ agent=owner,
696
+ # No limit: it leaves when it is idle, and its job ends it.
697
+ timeout_ms=None,
698
+ signal=job.token,
699
+ keep_env=inherited if isinstance(inherited, list) else None,
700
+ extra_env=env,
701
+ writable_extra=writable,
702
+ )
703
+ tasks.cancel_scope.cancel()
704
+ if result.exit_code != 0 and not result.canceled:
705
+ said = command_said(f"the {hosted.name} host", "serve", result)
706
+ log.warning("ph_utcp: %s", said[:_CLIP])
707
+ finally:
708
+ if self.hosts.get((hosted.name, owner.id)) is hosted:
709
+ del self.hosts[(hosted.name, owner.id)]
710
+ # The host removes its socket when it leaves by itself. One that was
711
+ # ended under a sandbox is killed with it and removes nothing.
712
+ with anyio.CancelScope(shield=True):
713
+ await anyio.to_thread.run_sync(partial(hosted.socket.unlink, missing_ok=True))
714
+
715
+ async def _watch(self, hosted: _Hosted) -> None:
716
+ """Read `hosted`'s progress as it changes, and tell its agent when a piece
717
+ of background work ends.
718
+
719
+ Polled rather than watched: one small file, read when its stat says it
720
+ changed — twice a second while a run works, and less often between runs —
721
+ and the reading is kept here, so the footer never reads it. The note is
722
+ `inject`ed: it waits for the agent's next step, starts no turn, and is in
723
+ that agent's log.
724
+ """
725
+ home = hosted.home
726
+ if home is None:
727
+ return
728
+ seen: object = None
729
+ while True:
730
+ stamp = await anyio.to_thread.run_sync(file_stamp, home / PROGRESS_FILE)
731
+ if stamp == seen:
732
+ # Quickly while a run works, and before the first report: a host
733
+ # just started is about to begin one.
734
+ last = hosted.progress
735
+ working = last is None or last.state == "working"
736
+ await anyio.sleep(_WATCH_EVERY_S if working else _WATCH_IDLE_S)
737
+ continue
738
+ seen = stamp
739
+ found = await anyio.to_thread.run_sync(read_progress, home)
740
+ if found is not None and found.started < hosted.since:
741
+ # An earlier host's last report: this one has said nothing yet.
742
+ found = None
743
+ hosted.progress = found
744
+ if (
745
+ found is not None
746
+ and found.state != "working"
747
+ # The run says whether its end is news: a call was answered
748
+ # "still going", or the host began it unasked. One whose caller
749
+ # got its counts back has been told already.
750
+ and found.owed
751
+ and found.started not in hosted.told
752
+ ):
753
+ hosted.told.add(found.started)
754
+ self._tell(hosted, found)
755
+
756
+ def _tell(self, hosted: _Hosted, progress: Progress) -> None:
757
+ label = hosted.host.label
758
+ if progress.state == "failed":
759
+ summary = f"The {label} stopped: {progress.failure}"
760
+ text = (
761
+ f"{summary}. It got through {progress.done} of {progress.total} "
762
+ f"{progress.what}; what it finished is kept, and running its index "
763
+ "tool again goes on from there."
764
+ )
765
+ else:
766
+ summary = f"The {label} is finished: {progress.said()}"
767
+ text = f"{summary}. Searches now cover all of it."
768
+ if progress.note:
769
+ # What its result would have said to a caller that waited.
770
+ text = f"{text}\n{progress.note}"
771
+ registry = self.ctx.get(AGENTS)
772
+ # Asked of the roster, which holds the driver: a handle is what a reader
773
+ # is given, and has no inbox to put a message in.
774
+ driver = None if registry is None else registry.get(hosted.owner.id)
775
+ if driver is None:
776
+ # An agent that has gone. The reading and the tool both still say it.
777
+ return
778
+ try:
779
+ driver.inject(
780
+ create_user_message(
781
+ content=[{"type": "text", "text": text}],
782
+ source=PluginSource(
783
+ plugin="ph_utcp",
784
+ form="notice",
785
+ summary=summary[:CONTEXT_SUMMARY_MAX_CHARS],
786
+ ),
787
+ )
788
+ )
789
+ except Exception:
790
+ log.debug("ph_utcp: %s was not told its %s ended", driver.id, label, exc_info=True)
791
+
792
+ def reading(self, session: Session) -> StatusReading | None:
793
+ """The footer's phrase for this session: what its hosts are building.
794
+ From memory — `_watch` reads the files."""
795
+ said: list[str] = []
796
+ failed = False
797
+ for hosted in self.hosts.values():
798
+ owned = hosted.owner.session
799
+ progress = hosted.progress
800
+ if owned is None or owned.id != session.id or progress is None:
801
+ continue
802
+ if progress.state == "done" or not hosted.running:
803
+ continue
804
+ failed = failed or progress.state == "failed"
805
+ said.append(f"{hosted.host.label} {progress.said()}")
806
+ if not said:
807
+ return None
808
+ return StatusReading(text=", ".join(said), level="warning" if failed else "normal")
809
+
810
+ async def eager(self, request: PreStepRequest, next_: Next[PreStepDecision]) -> PreStepDecision:
811
+ """`agent/pre-step`: before a root agent's step, see that each manual
812
+ whose host is `eager` has one — so its index is being built before the
813
+ model asks for it. After the workspace row's listener, which is what
814
+ gives the agent the tree the host is started in."""
815
+ if not request.session.header.is_subagent:
816
+ agent = request.agent
817
+ for name, manual in list(self.manuals.items()):
818
+ if manual.host is None or not manual.host.eager:
819
+ continue
820
+ if (name, agent.id) in self.warmed:
821
+ continue
822
+ self.warmed.add((name, agent.id))
823
+ # Started, not waited for: the step is not the host's to hold up.
824
+ await self._host_for(name, agent, warm=True, wait=False)
825
+ return await next_()
826
+
827
+ # ------------------------------------------------------------- the tool --
828
+
829
+ def _definition(self, name: str, manual: Manual, tool: Tool) -> ToolDefinition:
830
+ """One UTCP tool as a pH tool."""
831
+ utcp_name: str = tool.name
832
+ own = utcp_name.removeprefix(f"{name}.")
833
+ ph_name = own if manual.names == "bare" else _NOT_A_WORD.sub("_", utcp_name)
834
+ traits = manual.tools.get(own, ToolTraits())
835
+ budget = traits.timeout_ms or self.config.timeout_ms
836
+ inputs = _object_schema(tool.inputs.model_dump(by_alias=True, exclude_none=True))
837
+ outputs: dict[str, Any] = tool.outputs.model_dump(by_alias=True, exclude_none=True)
838
+ declared = value_schema(outputs) if manual.envelope else outputs
839
+ confined = tool.tool_call_template.call_template_type == "cli"
840
+
841
+ async def execute(args: object, run: ToolRunContext) -> object:
842
+ arguments = dict(as_obj(args))
843
+ # The raw-schema path hands the model's arguments through unchecked;
844
+ # the tool owns validation, and this is the tool.
845
+ violations = validate_json_schema_value(inputs, arguments)
846
+ if violations:
847
+ raise HarnessError("; ".join(violations), UTCP_BAD_ARGUMENTS)
848
+ owner = None if run.agent is None else self.owner_of(run.agent)
849
+ call = self._call(
850
+ manual,
851
+ timeout_ms=budget,
852
+ agent=run.agent,
853
+ owner=owner,
854
+ signal=run.signal,
855
+ host=await self._host_for(name, owner),
856
+ )
857
+ try:
858
+ with calling(call):
859
+ answer = await self.client.call_tool(utcp_name, arguments)
860
+ if confined:
861
+ require_ran(call, utcp_name)
862
+ except (HarnessError, Canceled):
863
+ raise
864
+ except Exception as error:
865
+ raise HarnessError(f"{utcp_name} failed: {error}", UTCP_FAILED) from error
866
+ if manual.envelope:
867
+ return self._unwrapped(utcp_name, answer, run, elsewhere=call.root)
868
+ mismatches = validate_json_schema_value(declared, answer)
869
+ if mismatches:
870
+ # The usual cause is a failure: the `cli` protocol ignores exit
871
+ # codes and returns stderr as the result, so a tool that declares
872
+ # an object answers with a sentence. The sentence is what the
873
+ # model needs, so it is the error rather than a schema complaint.
874
+ raise HarnessError(
875
+ f"{utcp_name} did not answer in its declared shape: {_clip(answer)}",
876
+ UTCP_BAD_ANSWER,
877
+ )
878
+ return answer
879
+
880
+ view = traits.view
881
+ views = simple_views(view.card, view.title, view.argument) if view is not None else None
882
+ return define_tool(
883
+ ph_name,
884
+ tool.description,
885
+ parameters=inputs,
886
+ output=declared,
887
+ execute=execute,
888
+ timeout_ms=budget,
889
+ self_limits=traits.self_limits,
890
+ arguments_disposable=traits.arguments_disposable,
891
+ effect_free=traits.effect_free,
892
+ is_concurrency_safe=traits.concurrency_safe,
893
+ present_call=views["present_call"] if views is not None else None,
894
+ present_result=views["present_result"] if views is not None else None,
895
+ )
896
+
897
+ def _unwrapped(
898
+ self, name: str, answer: object, run: ToolRunContext, *, elsewhere: Path
899
+ ) -> object:
900
+ """An envelope's `value`, presented with its `text` as the content.
901
+
902
+ The text was written by the tool's own `render` in its own process, from
903
+ the arguments and the value — the projection a session would have made had
904
+ the tool been registered here — so it is what the registry shows for this
905
+ value (`ToolRunContext.present`), and what every post-execute row sees and
906
+ edits. A failure is raised with the tool's own code
907
+ (`ph.toolcli.read_envelope`).
908
+
909
+ `elsewhere` is where the command ran. When that is not the asking agent's
910
+ own root — a child asking the index of the tree it shares — the text says
911
+ so, because the paths in it start there."""
912
+ try:
913
+ value, text = read_envelope(answer)
914
+ except ValueError as error:
915
+ raise HarnessError(
916
+ f"{name} did not answer with an envelope: {_clip(answer)}", UTCP_BAD_ANSWER
917
+ ) from error
918
+ if text is not None:
919
+ if elsewhere != self.ctx.require(FS).root_for(run.agent):
920
+ text += f"\n[Paths here are relative to {elsewhere}, the tree this was built from.]"
921
+ run.present(value, text_content(text))
922
+ return value
923
+
924
+ # ------------------------------------------------------------ reporting --
925
+
926
+ def report(self) -> list[tuple[str, str]]:
927
+ """`phern doctor`'s section: each manual, and what it gave."""
928
+ rows: list[tuple[str, str]] = []
929
+ for name in sorted(self.states):
930
+ state = self.states[name]
931
+ said = count_of(len(state.tools), "tool") if state.tools else "unavailable"
932
+ if state.problems:
933
+ said += " — " + "; ".join(state.problems)
934
+ rows.append((name, said))
935
+ return rows
936
+
937
+ def unavailable(self) -> str:
938
+ """The note the model reads: which manuals gave no tools, and why."""
939
+ missing = [
940
+ f"- `{name}`: {'; '.join(state.problems)}"
941
+ for name, state in sorted(self.states.items())
942
+ if not state.available
943
+ ]
944
+ if not missing:
945
+ return ""
946
+ return (
947
+ "# Tools that are not available\n\n"
948
+ "These tool providers were configured for this session and could not be "
949
+ "reached, so their tools are not in your tool list. Do not call them; say "
950
+ "so if a task needs one.\n\n" + "\n".join(missing)
951
+ )
952
+
953
+
954
+ # ------------------------------------------------------------------ schemas ----
955
+
956
+
957
+ def _object_schema(schema: dict[str, Any]) -> dict[str, Any]:
958
+ """A manual's `inputs` as the object schema a provider is sent. A tool that
959
+ declares no inputs takes none."""
960
+ if schema.get("type") == "object":
961
+ return schema
962
+ return {"type": "object", "properties": {}, **schema}
963
+
964
+
965
+ def _sockets() -> Path | None:
966
+ """Where hosts listen: a directory of `$PH_RUNTIME`, made owner-only — or
967
+ `None` on a deployment with no usable runtime directory, which then runs
968
+ every call alone. Under the runtime root because a socket's address is short
969
+ and this path is, and because what is there does not outlive a reboot."""
970
+ try:
971
+ directory = resolve_roots().ensure_runtime().runtime / "utcp"
972
+ directory.mkdir(mode=0o700, exist_ok=True)
973
+ except (RuntimeDirError, OSError):
974
+ return None
975
+ return directory
976
+
977
+
978
+ def _made(paths: Sequence[str], root: str) -> tuple[list[str], list[str]]:
979
+ """Make each writable path a directory that exists: those that do, and why the
980
+ rest do not.
981
+
982
+ A path that overlaps the session's workspace `root` is not made and not
983
+ granted (`grants_apart`), and said here once, where a person reads it —
984
+ `ctx.shell` applies the same rule to every call, against the calling agent's
985
+ own root, and says nothing.
986
+
987
+ A sandbox backend grants a path by binding it, and a path that is not there
988
+ cannot be bound — so one missing directory would fail every call the manual's
989
+ tools make, with the backend's own words for it. Made once, here, and a path
990
+ that cannot be made is left out of the grant and said.
991
+
992
+ Spelled as the deployment's own allowances are (`allowance_path`): `~`
993
+ expanded and every symlink resolved, so a profile's `~/.cache/x` is that
994
+ directory and not one named `~` under the harness's working directory, and
995
+ the path the backend binds is the one the kernel will see.
996
+ """
997
+ made: list[str] = []
998
+ apart, overlapping = grants_apart(root, paths)
999
+ unmade = [
1000
+ f"{path} cannot be granted as writable: it overlaps the workspace {root}"
1001
+ for path in overlapping
1002
+ ]
1003
+ for spelled in apart:
1004
+ try:
1005
+ Path(spelled).mkdir(parents=True, exist_ok=True)
1006
+ except OSError as error:
1007
+ unmade.append(f"{spelled} cannot be granted as writable: {error.strerror}")
1008
+ else:
1009
+ made.append(spelled)
1010
+ return made, unmade
1011
+
1012
+
1013
+ def _clip(answer: object) -> str:
1014
+ text = answer if isinstance(answer, str) else dumps(answer)
1015
+ return text if len(text) <= _CLIP else f"{text[:_CLIP]}…"
1016
+
1017
+
1018
+ # ---------------------------------------------------------------------- row ----
1019
+
1020
+
1021
+ @plugin("utcp", affects="environment", inject=[TOOLS, SHELL, FS], config=Config)
1022
+ async def apply(ctx: Context, config: Config) -> None:
1023
+ """Mount the seam, and register the tools of every manual that answers."""
1024
+ install()
1025
+ held: list[Any] = []
1026
+
1027
+ async def client() -> Disposer:
1028
+ held.append(await UtcpClient.create(root_dir=str(ctx.require(FS).root_for(None))))
1029
+
1030
+ async def close() -> None:
1031
+ await held[0].close()
1032
+
1033
+ return close
1034
+
1035
+ await ctx.effect(client, label="utcp-client")
1036
+ seam = UtcpSeam(ctx=ctx, config=config, client=held[0], socket_dir=_sockets())
1037
+ ctx.provide(UTCP, seam)
1038
+ for manual in config.manuals:
1039
+ await seam.add(manual)
1040
+
1041
+ prompt = ctx.get(SYSTEM_PROMPT)
1042
+ if prompt is not None:
1043
+
1044
+ def note(_request: AssembleContext) -> str:
1045
+ return seam.unavailable()
1046
+
1047
+ prompt.section(
1048
+ PromptSection(name="utcp:unavailable", order=ORDER_TOOL_GUIDANCE + 1, text=note)
1049
+ )
1050
+ contribute(ctx, Diagnostic(id="utcp", title="UTCP tools", order=62, read=seam.report))
1051
+ ctx.on("agent/pre-step", seam.eager)
1052
+ contribute_item(
1053
+ ctx, TUI_STATUS, StatusField(id="utcp-hosts", read=seam.reading, order=70), label="utcp"
1054
+ )