phil-agent 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. phil/__init__.py +16 -0
  2. phil/__main__.py +4 -0
  3. phil/agents/__init__.py +0 -0
  4. phil/agents/activity.py +185 -0
  5. phil/agents/check.py +217 -0
  6. phil/agents/collector.py +167 -0
  7. phil/agents/evidence.py +69 -0
  8. phil/agents/factory.py +351 -0
  9. phil/agents/failures.py +204 -0
  10. phil/agents/fake.py +134 -0
  11. phil/agents/invoke.py +721 -0
  12. phil/agents/model_retry.py +123 -0
  13. phil/agents/openrouter_model.py +32 -0
  14. phil/agents/pricing.py +353 -0
  15. phil/agents/providers.py +207 -0
  16. phil/agents/registry.py +108 -0
  17. phil/agents/reminder.py +102 -0
  18. phil/agents/retry.py +260 -0
  19. phil/agents/salvage.py +211 -0
  20. phil/agents/spec.py +55 -0
  21. phil/agents/tool_args.py +171 -0
  22. phil/agents/tool_choice.py +36 -0
  23. phil/agents/tools.py +96 -0
  24. phil/agents/unpriced.py +128 -0
  25. phil/agents/usage.py +21 -0
  26. phil/agents/vision.py +233 -0
  27. phil/chat/__init__.py +0 -0
  28. phil/chat/answer.py +46 -0
  29. phil/chat/approval.py +187 -0
  30. phil/chat/attachments.py +232 -0
  31. phil/chat/btw.py +30 -0
  32. phil/chat/controller.py +3838 -0
  33. phil/chat/decision.py +283 -0
  34. phil/chat/design.py +33 -0
  35. phil/chat/events.py +12 -0
  36. phil/chat/mockup.py +64 -0
  37. phil/chat/mockup_page.py +302 -0
  38. phil/chat/overview.py +18 -0
  39. phil/chat/planning.py +298 -0
  40. phil/chat/session.py +188 -0
  41. phil/chat/snapshot.py +101 -0
  42. phil/chat/state.py +147 -0
  43. phil/chat/terminal.py +338 -0
  44. phil/chat/watcher.py +330 -0
  45. phil/cli/__init__.py +0 -0
  46. phil/cli/attach.py +189 -0
  47. phil/cli/entry.py +80 -0
  48. phil/cli/main.py +1416 -0
  49. phil/cli/memory.py +209 -0
  50. phil/cli/parked.py +72 -0
  51. phil/cli/permissions.py +268 -0
  52. phil/cli/skills.py +209 -0
  53. phil/config.py +539 -0
  54. phil/contracts/__init__.py +106 -0
  55. phil/contracts/base.py +13 -0
  56. phil/contracts/common.py +44 -0
  57. phil/contracts/converse.py +76 -0
  58. phil/contracts/design.py +97 -0
  59. phil/contracts/inputs.py +88 -0
  60. phil/contracts/interface.py +110 -0
  61. phil/contracts/memory.py +43 -0
  62. phil/contracts/planning.py +86 -0
  63. phil/contracts/results.py +112 -0
  64. phil/contracts/routing.py +39 -0
  65. phil/contracts/schema.py +14 -0
  66. phil/contracts/skills.py +31 -0
  67. phil/diagnostics.py +228 -0
  68. phil/git.py +70 -0
  69. phil/key_store.py +150 -0
  70. phil/memory/__init__.py +0 -0
  71. phil/memory/agents_md.py +173 -0
  72. phil/memory/apply.py +132 -0
  73. phil/memory/commands.py +93 -0
  74. phil/memory/context.py +182 -0
  75. phil/memory/doc.py +169 -0
  76. phil/memory/editor.py +95 -0
  77. phil/memory/learn.py +110 -0
  78. phil/memory/listing.py +36 -0
  79. phil/memory/store.py +117 -0
  80. phil/packets/__init__.py +3 -0
  81. phil/packets/packet.py +209 -0
  82. phil/parking.py +33 -0
  83. phil/permissions/__init__.py +0 -0
  84. phil/permissions/catalog.py +179 -0
  85. phil/permissions/listing.py +131 -0
  86. phil/permissions/load.py +231 -0
  87. phil/permissions/rules.py +303 -0
  88. phil/permissions/store.py +362 -0
  89. phil/platform.py +290 -0
  90. phil/prompts/__init__.py +0 -0
  91. phil/prompts/_shared.md +23 -0
  92. phil/prompts/answer.md +15 -0
  93. phil/prompts/architect.md +47 -0
  94. phil/prompts/btw.md +13 -0
  95. phil/prompts/converse.md +30 -0
  96. phil/prompts/critic.md +17 -0
  97. phil/prompts/design.md +21 -0
  98. phil/prompts/implementer.md +26 -0
  99. phil/prompts/intake.md +34 -0
  100. phil/prompts/memory.md +21 -0
  101. phil/prompts/mockup.md +31 -0
  102. phil/prompts/model_check.md +7 -0
  103. phil/prompts/reviewer.md +15 -0
  104. phil/prompts/route.md +9 -0
  105. phil/prompts/skill_labels.md +10 -0
  106. phil/prompts/tester.md +13 -0
  107. phil/publish/__init__.py +16 -0
  108. phil/publish/learnings.py +80 -0
  109. phil/publish/pr_body.py +179 -0
  110. phil/publish/publisher.py +249 -0
  111. phil/publish/service.py +235 -0
  112. phil/repo.py +54 -0
  113. phil/repo_detect.py +73 -0
  114. phil/routing/__init__.py +24 -0
  115. phil/routing/classes.py +83 -0
  116. phil/routing/classify.py +54 -0
  117. phil/routing/jev.py +156 -0
  118. phil/routing/llm.py +24 -0
  119. phil/routing/policy.py +52 -0
  120. phil/routing/types.py +52 -0
  121. phil/run/__init__.py +0 -0
  122. phil/run/checkpoint.py +14 -0
  123. phil/run/cleanup.py +134 -0
  124. phil/run/engine.py +1395 -0
  125. phil/run/gates.py +216 -0
  126. phil/run/launch.py +135 -0
  127. phil/run/naming.py +49 -0
  128. phil/run/runner.py +66 -0
  129. phil/run/state.py +297 -0
  130. phil/run/worker.py +338 -0
  131. phil/setup/__init__.py +4 -0
  132. phil/setup/catalog.py +64 -0
  133. phil/setup/flow.py +660 -0
  134. phil/setup/guard.py +38 -0
  135. phil/setup/io.py +129 -0
  136. phil/setup/skill_labels.py +60 -0
  137. phil/setup/suggestions.py +31 -0
  138. phil/setup/write.py +86 -0
  139. phil/skills/__init__.py +1 -0
  140. phil/skills/discover.py +132 -0
  141. phil/skills/labels.py +137 -0
  142. phil/skills/listing.py +106 -0
  143. phil/skills/runtime.py +55 -0
  144. phil/skills/select.py +155 -0
  145. phil/skills/sources.py +264 -0
  146. phil/skills/tool.py +159 -0
  147. phil/store/__init__.py +0 -0
  148. phil/store/activity.py +193 -0
  149. phil/store/artifacts.py +88 -0
  150. phil/store/db.py +139 -0
  151. phil/store/events.py +89 -0
  152. phil/store/parked.py +102 -0
  153. phil/store/paths.py +64 -0
  154. phil/store/runs.py +179 -0
  155. phil/store/telemetry.py +259 -0
  156. phil/text.py +16 -0
  157. phil/tomlw.py +59 -0
  158. phil/ui/__init__.py +0 -0
  159. phil/ui/answer_view.py +10 -0
  160. phil/ui/banner.py +191 -0
  161. phil/ui/blocks.py +122 -0
  162. phil/ui/brief_view.py +19 -0
  163. phil/ui/bursts.py +167 -0
  164. phil/ui/callout.py +281 -0
  165. phil/ui/chats_view.py +44 -0
  166. phil/ui/diff_view.py +94 -0
  167. phil/ui/feed_view.py +196 -0
  168. phil/ui/fragments.py +16 -0
  169. phil/ui/links.py +16 -0
  170. phil/ui/mascot.py +12 -0
  171. phil/ui/plan_view.py +97 -0
  172. phil/ui/runs.py +20 -0
  173. phil/ui/runs_view.py +43 -0
  174. phil/ui/screen/__init__.py +1 -0
  175. phil/ui/screen/app.py +355 -0
  176. phil/ui/screen/clipboard.py +91 -0
  177. phil/ui/screen/console.py +44 -0
  178. phil/ui/screen/io.py +312 -0
  179. phil/ui/screen/mode.py +30 -0
  180. phil/ui/screen/themed.py +24 -0
  181. phil/ui/screen/widgets.py +487 -0
  182. phil/ui/show_view.py +204 -0
  183. phil/ui/theme.py +76 -0
  184. phil/ui/toolbar.py +353 -0
  185. phil/ui/when.py +28 -0
  186. phil/workspace/__init__.py +0 -0
  187. phil/workspace/shell.py +795 -0
  188. phil/workspace/worktree.py +104 -0
  189. phil_agent-0.1.0.dist-info/METADATA +129 -0
  190. phil_agent-0.1.0.dist-info/RECORD +193 -0
  191. phil_agent-0.1.0.dist-info/WHEEL +4 -0
  192. phil_agent-0.1.0.dist-info/entry_points.txt +2 -0
  193. phil_agent-0.1.0.dist-info/licenses/LICENSE +201 -0
phil/__init__.py ADDED
@@ -0,0 +1,16 @@
1
+ from importlib.metadata import PackageNotFoundError
2
+ from importlib.metadata import version as _dist_version
3
+
4
+ DIST_NAME = "phil-agent"
5
+
6
+
7
+ def _installed_version() -> str:
8
+ """The installed distribution's version, which comes from pyproject.toml. "0+unknown" when
9
+ Phil runs from a source tree that was never installed."""
10
+ try:
11
+ return _dist_version(DIST_NAME)
12
+ except PackageNotFoundError:
13
+ return "0+unknown"
14
+
15
+
16
+ __version__ = _installed_version()
phil/__main__.py ADDED
@@ -0,0 +1,4 @@
1
+ from phil.cli.entry import main
2
+
3
+ if __name__ == "__main__":
4
+ main()
File without changes
@@ -0,0 +1,185 @@
1
+ """Records each agent tool call into the run's activity log (spec 2026-10-07, plan amendment A1:
2
+ a LangChain callback rather than middleware — callbacks reach every harness and every nested
3
+ sub-agent run without changing how agents are built)."""
4
+
5
+ import difflib
6
+ import logging
7
+ import threading
8
+ import time
9
+ from collections.abc import Iterable
10
+
11
+ from langchain_core.callbacks import BaseCallbackHandler
12
+
13
+ from phil.agents.collector import _tool_args
14
+ from phil.store.activity import ActivityLog
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ IGNORED_TOOLS = frozenset({"write_todos"})
19
+ SUB_TOOL = "task" # deepagents' tool that runs a sub-agent
20
+ _SHORT = 60 # a result is at most this many characters
21
+
22
+
23
+ def _first_line(text: str) -> str:
24
+ for line in str(text).splitlines():
25
+ if line.strip():
26
+ return line.strip()[:_SHORT]
27
+ return ""
28
+
29
+
30
+ def summarize_call(tool: str, args: dict | None) -> str:
31
+ args = args or {}
32
+ path = args.get("file_path") or args.get("path") or ""
33
+ if tool == "read_file":
34
+ return f"read {path}"
35
+ if tool == "ls":
36
+ return f"ls {path}".rstrip()
37
+ if tool == "glob":
38
+ return f"glob {args.get('pattern', '')}".rstrip()
39
+ if tool == "grep":
40
+ return f'grep "{args.get("pattern", "")}" {path}'.rstrip()
41
+ if tool == "edit_file":
42
+ return f"edit {path}"
43
+ if tool == "write_file":
44
+ return f"write {path}"
45
+ if tool == "run_shell":
46
+ return f"run {args.get('command', '')}".rstrip()
47
+ if tool == "load_skill":
48
+ name, file = args.get("name", ""), args.get("file")
49
+ return f"skill {name}/{file}" if file else f"skill {name}"
50
+ if tool == "task":
51
+ return f"sub-agent: {_first_line(args.get('description', ''))}".rstrip()
52
+ return tool
53
+
54
+
55
+ def summarize_result(tool: str, args: dict | None, output: str) -> tuple[str, bool, str | None]:
56
+ """(result, ok, detail). Reads, listings and searches have no detail."""
57
+ args, output = args or {}, str(output)
58
+ if tool == "run_shell":
59
+ lines = output.splitlines()
60
+ status = lines[0] if lines else ""
61
+ ok = status.strip() == "exit_code: 0"
62
+ body = lines[1:]
63
+ if body and body[0].startswith("allowed: "): # why it ran: kept in the detail, not the summary
64
+ body = body[1:]
65
+ body = [line for line in body if not line.startswith("full log: ")]
66
+ last = next((line.strip() for line in reversed(body) if line.strip()), status)
67
+ return f"→ {last[:_SHORT]}", ok, output
68
+ if tool == "edit_file":
69
+ old, new = str(args.get("old_string", "")), str(args.get("new_string", ""))
70
+ diff = list(difflib.unified_diff(old.splitlines(), new.splitlines(), "before", "after", lineterm=""))
71
+ added = sum(1 for line in diff if line.startswith("+") and not line.startswith("+++"))
72
+ removed = sum(1 for line in diff if line.startswith("-") and not line.startswith("---"))
73
+ return f"+{added} −{removed}", True, "\n".join(diff)
74
+ if tool == "write_file":
75
+ content = str(args.get("content", ""))
76
+ return f"+{len(content.splitlines())}", True, content
77
+ return "", True, None
78
+
79
+
80
+ class ActivityCallback(BaseCallbackHandler):
81
+ """Writes a start record on `on_tool_start` and an end record on `on_tool_end`/`on_tool_error`,
82
+ for every tool call in the agent's run tree. Never raises.
83
+
84
+ A call made inside a sub-agent (a `task` call) is tagged with that task call's `sub_id` (its
85
+ seq) and `sub` (its description). To find it, the callback remembers the parent of every run it
86
+ sees (chains, models and tools), and walks up from a tool call to the nearest open task call."""
87
+
88
+ def __init__(self, log: ActivityLog, *, task: str | None, role: str, ignore_tools: Iterable[str] = ()) -> None:
89
+ self._log, self._task, self._role = log, task, role
90
+ self._ignore = IGNORED_TOOLS | set(ignore_tools)
91
+ self._open: dict[object, tuple[int | None, float, str, dict | None, str, dict | None]] = {}
92
+ self._parents: dict[object, object] = {} # LangChain run id -> parent run id
93
+ self._subs: dict[object, tuple[int | None, str]] = {} # an open task call's run id -> (seq, description)
94
+ self._lock = threading.Lock()
95
+
96
+ # --- the run tree ----------------------------------------------------------------------
97
+
98
+ def _note_parent(self, run_id, parent_run_id) -> None:
99
+ try:
100
+ if run_id is not None and parent_run_id is not None:
101
+ with self._lock:
102
+ self._parents[run_id] = parent_run_id
103
+ except Exception:
104
+ logger.debug("activity: parent bookkeeping failed", exc_info=True)
105
+
106
+ def on_chain_start(self, serialized, inputs, *, run_id=None, parent_run_id=None, **kwargs) -> None:
107
+ self._note_parent(run_id, parent_run_id)
108
+
109
+ def on_chat_model_start(self, serialized, messages, *, run_id=None, parent_run_id=None, **kwargs) -> None:
110
+ self._note_parent(run_id, parent_run_id)
111
+
112
+ def on_llm_start(self, serialized, prompts, *, run_id=None, parent_run_id=None, **kwargs) -> None:
113
+ self._note_parent(run_id, parent_run_id)
114
+
115
+ def _sub_for(self, parent_run_id) -> dict:
116
+ """The nearest open task call above `parent_run_id`, as record fields; {} if none."""
117
+ seen, node = set(), parent_run_id
118
+ with self._lock:
119
+ while node is not None and node not in seen:
120
+ if node in self._subs:
121
+ seq, description = self._subs[node]
122
+ return {"sub_id": seq, "sub": description} if seq is not None else {}
123
+ seen.add(node)
124
+ node = self._parents.get(node)
125
+ return {}
126
+
127
+ def reset(self) -> None:
128
+ """Forget the run tree (call when the agent call finishes)."""
129
+ with self._lock:
130
+ self._parents.clear()
131
+ self._subs.clear()
132
+
133
+ # --- tool calls ------------------------------------------------------------------------
134
+
135
+ def on_tool_start(self, serialized: dict, input_str: str, *, run_id, parent_run_id=None, inputs=None,
136
+ **kwargs) -> None:
137
+ try:
138
+ self._note_parent(run_id, parent_run_id)
139
+ name = (serialized or {}).get("name") or kwargs.get("name") or "tool"
140
+ if name in self._ignore:
141
+ return
142
+ args = _tool_args(input_str, inputs)
143
+ summary = summarize_call(name, args)
144
+ extra = self._sub_for(parent_run_id) or None # a task call is never inside itself
145
+ seq = self._log.start(task=self._task, role=self._role, tool=name, summary=summary, extra=extra)
146
+ with self._lock:
147
+ self._open[run_id] = (seq, time.monotonic(), name, args, summary, extra)
148
+ if name == SUB_TOOL:
149
+ self._subs[run_id] = (seq, _first_line(args.get("description", "")) if args else "")
150
+ except Exception:
151
+ logger.debug("activity: on_tool_start failed", exc_info=True)
152
+
153
+ def _finish(self, run_id, result: str, ok: bool, detail: str | None) -> None:
154
+ with self._lock:
155
+ opened = self._open.pop(run_id, None)
156
+ self._subs.pop(run_id, None)
157
+ self._parents.pop(run_id, None)
158
+ if opened is None:
159
+ return
160
+ seq, started, name, _args, summary, extra = opened
161
+ self._log.end(seq, task=self._task, role=self._role, tool=name, summary=summary, result=result, ok=ok,
162
+ detail=detail, duration_ms=int((time.monotonic() - started) * 1000), extra=extra)
163
+
164
+ def on_tool_end(self, output, *, run_id, **kwargs) -> None:
165
+ try:
166
+ with self._lock:
167
+ opened = self._open.get(run_id)
168
+ if opened is None:
169
+ return
170
+ text = getattr(output, "content", output)
171
+ text = text if isinstance(text, str) else str(text)
172
+ if getattr(output, "status", None) == "error":
173
+ # deepagents' file tools report a failure as a ToolMessage(status="error"), not by raising.
174
+ self._finish(run_id, _first_line(text) or "error", False, text)
175
+ return
176
+ result, ok, detail = summarize_result(opened[2], opened[3], text)
177
+ self._finish(run_id, result, ok, detail)
178
+ except Exception:
179
+ logger.debug("activity: on_tool_end failed", exc_info=True)
180
+
181
+ def on_tool_error(self, error: BaseException, *, run_id, **kwargs) -> None:
182
+ try:
183
+ self._finish(run_id, _first_line(str(error)) or type(error).__name__, False, str(error))
184
+ except Exception:
185
+ logger.debug("activity: on_tool_error failed", exc_info=True)
phil/agents/check.py ADDED
@@ -0,0 +1,217 @@
1
+ """`phil models check`: one tiny real call per model a role uses, through the agent path Phil uses."""
2
+
3
+ import json
4
+ import tempfile
5
+ import time
6
+ from collections.abc import Callable
7
+ from dataclasses import dataclass
8
+ from pathlib import Path
9
+
10
+ from pydantic import Field
11
+
12
+ from phil.agents.invoke import AgentContext, AgentFactory, ContractViolation, invoke_agent, shared_price_book
13
+ from phil.agents.pricing import PriceBook
14
+ from phil.agents.unpriced import price_label
15
+ from phil.agents.providers import SYSTEMONE, ProviderSpec, missing_key_message, provider_for_model, split_model
16
+ from phil.agents.spec import AgentSpec
17
+ from phil.config import ROLES, TIERS, VISION_KEY, ConfigError, PhilConfig
18
+ from phil.contracts import Contract
19
+ from phil.key_store import key_lookup
20
+ from phil.packets import build_packet
21
+ from phil.routing.jev import JevError, ping_jev
22
+ from phil.store.artifacts import ArtifactStore
23
+ from phil.store.db import connect
24
+
25
+ CHECK_WORD = "pineapple"
26
+ EXCERPT_CHARS = 120
27
+ NO_STRUCTURED_OUTPUT = "no structured output was returned"
28
+
29
+
30
+ class ModelCheckInput(Contract):
31
+ word: str = Field(description="The word to echo back.")
32
+
33
+
34
+ class ModelCheck(Contract):
35
+ ok: bool = Field(description="Always true.")
36
+ echo: str = Field(description="The input's word, exactly as given.")
37
+
38
+
39
+ # Lean, and without _shared.md: its self_check rules don't apply to this tiny contract. A text
40
+ # answer gets the same reminders a real agent gets, so a model passes only if Phil can get an
41
+ # answer from it.
42
+ MODEL_CHECK = AgentSpec(
43
+ "model_check", "orchestrator", ModelCheckInput, ModelCheck,
44
+ harness="lean", shared_prompt=False,
45
+ )
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class CheckResult:
50
+ label: str # the tiers and `role:<name>` keys that use `model`, e.g. "high, low"
51
+ model: str
52
+ ok: bool
53
+ seconds: float
54
+ detail: str # why it failed; empty when ok
55
+ reminded: bool = False # passed, but only after a reminder to call the answer tool
56
+ price: str | None = None # "$in/$out per M", "no price"…; None when it failed (phil.agents.unpriced)
57
+
58
+
59
+ def _role_labels(config: PhilConfig) -> dict[str, str]:
60
+ """Label -> model for each model some role resolves to: the tier the role uses, or
61
+ `role:<name>` when the role's own [models] key overrides its tier. Roles with no model are skipped.
62
+
63
+ `classifier` is special: it is both a role and a tier key under [models], so a `[models]
64
+ classifier` entry is the classifier *tier*, not a `role:classifier` override. Its label is
65
+ always `classifier` (never `role:classifier`); with no classifier set, the role falls back to
66
+ the low model and shares that label instead."""
67
+ used: dict[str, str] = {}
68
+ for role in ROLES:
69
+ try:
70
+ model = config.model_for(role)
71
+ except ConfigError:
72
+ continue
73
+ if role == "classifier":
74
+ label = "classifier" if role in config.models else config.model_owner(role)
75
+ else:
76
+ label = f"role:{role}" if role in config.models else config.model_owner(role)
77
+ used[label] = model
78
+ return used
79
+
80
+
81
+ def check_targets(config: PhilConfig) -> list[tuple[str, list[str]]]:
82
+ """Each distinct model some role resolves to, with the labels that use it: the tiers in tier
83
+ order, then the role keys under [models] (which override their tier), in first-seen order.
84
+ A tier no role uses isn't a target (see `unused_tiers`)."""
85
+ used = _role_labels(config)
86
+ labels: dict[str, list[str]] = {}
87
+ for label in (*TIERS, *(f"role:{role}" for role in ROLES)):
88
+ if label in used:
89
+ labels.setdefault(used[label], []).append(label)
90
+ vision = config.vision_model()
91
+ if vision:
92
+ labels.setdefault(vision, []).append(VISION_KEY)
93
+ return list(labels.items())
94
+
95
+
96
+ def unused_tiers(config: PhilConfig) -> list[tuple[str, str]]:
97
+ """(tier, model) for each tier set under [models] that no role maps to; it is not called."""
98
+ used = _role_labels(config)
99
+ return [(tier, config.models[tier]) for tier in TIERS if tier in config.models and tier not in used]
100
+
101
+
102
+ def _first_line(text: str) -> str:
103
+ return text.strip().splitlines()[0] if text.strip() else ""
104
+
105
+
106
+ def _error_detail(exc: Exception) -> str:
107
+ if isinstance(exc, ConfigError): # Phil's own message (unknown provider, missing key) says it all
108
+ return _first_line(str(exc))
109
+ return f"{type(exc).__name__}: {_first_line(str(exc))}"
110
+
111
+
112
+ def _violation_detail(exc: ContractViolation) -> str:
113
+ raw = None
114
+ if exc.rejected_path is not None:
115
+ raw = json.loads(Path(exc.rejected_path).read_text(encoding="utf-8")).get("raw")
116
+ if exc.problems == [NO_STRUCTURED_OUTPUT]:
117
+ if isinstance(raw, str) and raw.strip():
118
+ excerpt = " ".join(raw.split())[:EXCERPT_CHARS]
119
+ return f'returned text instead of the required structured output: "{excerpt}"'
120
+ return "returned no structured output"
121
+ return f"returned invalid structured output: {'; '.join(exc.problems)}"
122
+
123
+
124
+ # How a decision-model provider is named in a check's failure message.
125
+ PROVIDER_LABELS = {"typesafe": "TypeSafe", "openrouter_decisions": "OpenRouter"}
126
+
127
+
128
+ def _jev_detail(provider: ProviderSpec, exc: JevError) -> str:
129
+ label = PROVIDER_LABELS.get(provider.name, provider.name)
130
+ if exc.reason == "missing key":
131
+ return missing_key_message(provider.name, provider.api_key_env or "", ["classifier"])
132
+ if exc.reason in ("http 401", "http 403"):
133
+ return f"{label} refused the request ({exc.reason}): check {provider.api_key_env}."
134
+ return f"{label} didn't answer ({exc.reason})."
135
+
136
+
137
+ def _check_one(
138
+ ctx: AgentContext, model: str, labels: list[str], call: int, jev_transport: object | None
139
+ ) -> tuple[bool, str, bool]:
140
+ """(ok, detail, reminded): reminded when the model passed only after a reminder to call its answer tool."""
141
+ try:
142
+ # Checked here so the errors name the tiers and roles using the model, not the check's own role.
143
+ provider = provider_for_model(ctx.config, model, labels[0])
144
+ if provider.api_key_env and not key_lookup().get(provider.api_key_env):
145
+ return False, missing_key_message(provider.name, provider.api_key_env, labels), False
146
+ if provider.kind == SYSTEMONE:
147
+ try:
148
+ ping_jev(provider, split_model(model)[1], timeout_s=ctx.config.routing.jev_timeout_s,
149
+ transport=jev_transport)
150
+ except JevError as exc:
151
+ return False, _jev_detail(provider, exc), False
152
+ return True, "", False
153
+ packet = build_packet("model_check", ModelCheckInput(word=CHECK_WORD), budget_tokens=1_000)
154
+ output = invoke_agent(
155
+ MODEL_CHECK, packet, ctx, node="model_check", call=call,
156
+ model=model, max_attempts=1, transient_retries=False,
157
+ )
158
+ except ContractViolation as exc:
159
+ return False, _violation_detail(exc), False
160
+ except Exception as exc:
161
+ return False, _error_detail(exc), False
162
+ assert isinstance(output, ModelCheck)
163
+ if not output.ok:
164
+ return False, "returned ok=false", False
165
+ if output.echo.strip() != CHECK_WORD:
166
+ return False, f'echoed "{output.echo}" instead of "{CHECK_WORD}"', False
167
+ row = ctx.conn.execute(
168
+ "SELECT reminders FROM telemetry WHERE node = ? AND call = ? ORDER BY id DESC LIMIT 1",
169
+ ("model_check", call),
170
+ ).fetchone()
171
+ return True, "", bool(row and row[0])
172
+
173
+
174
+ def check_models(
175
+ config: PhilConfig,
176
+ *,
177
+ factory: AgentFactory | None = None,
178
+ repo_root: Path,
179
+ clock: Callable[[], float] = time.monotonic,
180
+ jev_transport: object | None = None,
181
+ prices: PriceBook | None = None,
182
+ ) -> list[CheckResult]:
183
+ """One call per distinct model a role resolves to (see `check_targets`): a lean agent asked to return
184
+ `ModelCheck(ok=true, echo=<word>)`, with the configured timeout, one try and no retries. Its
185
+ telemetry and artifacts go to a temporary directory, not the project's.
186
+
187
+ A typesafe (systemone) classifier model is pinged with `ping_jev` instead, through the lean
188
+ agent path; `jev_transport` overrides its HTTP transport (tests only).
189
+
190
+ A passing model's result carries its price; `prices` is the price book for it, the
191
+ process-wide one when not given."""
192
+ targets = check_targets(config)
193
+ if not targets:
194
+ return []
195
+ results: list[CheckResult] = []
196
+ with tempfile.TemporaryDirectory(prefix="phil-models-check-") as scratch:
197
+ conn = connect(Path(scratch) / "check.db")
198
+ try:
199
+ ctx = AgentContext(
200
+ config=config,
201
+ conn=conn,
202
+ layer="chat",
203
+ artifacts=ArtifactStore(Path(scratch) / "artifacts"),
204
+ workdir=repo_root,
205
+ factory=factory,
206
+ )
207
+ book = prices if prices is not None else shared_price_book()
208
+ for call, (model, labels) in enumerate(targets, 1):
209
+ started = clock()
210
+ ok, detail, reminded = _check_one(ctx, model, labels, call, jev_transport)
211
+ seconds = clock() - started
212
+ # Looked up after the clock stops: a first fetch of the list isn't the model's time.
213
+ price = price_label(config, model, book) if ok else None
214
+ results.append(CheckResult(", ".join(labels), model, ok, seconds, detail, reminded, price))
215
+ finally:
216
+ conn.close()
217
+ return results
@@ -0,0 +1,167 @@
1
+ import ast
2
+ import json
3
+ import threading
4
+ from collections.abc import Iterable
5
+ from dataclasses import dataclass
6
+
7
+ from langchain_core.callbacks import BaseCallbackHandler
8
+ from langchain_core.outputs import LLMResult
9
+
10
+
11
+ @dataclass(frozen=True)
12
+ class ModelCall:
13
+ model: str | None # model name from the response/serialized llm, if known
14
+ input_tokens: int
15
+ output_tokens: int
16
+ reported_cost: float | None
17
+ cache_read_tokens: int = 0 # of input_tokens, how many were read from the provider's prompt cache
18
+ cache_write_tokens: int = 0 # of input_tokens, how many were written to it
19
+ # Set for a call another model than the agent's served (the `vision` model): `model` is then
20
+ # its configured `provider:model` and this the provider's name, so it's priced at its own rates.
21
+ provider: str | None = None
22
+
23
+
24
+ # phil.agents.vision.SERVED_BY_KEY (not imported: that module loads langchain's agent stack).
25
+ _SERVED_BY_KEY = "phil_served_by"
26
+
27
+
28
+ def _cache_tokens(message: object) -> tuple[int, int]:
29
+ """(read, written) prompt-cache tokens. With Anthropic's per-TTL breakdown, langchain_anthropic
30
+ reports the writes under `ephemeral_5m/1h_input_tokens` and zeroes `cache_creation`."""
31
+ details = (getattr(message, "usage_metadata", None) or {}).get("input_token_details") or {}
32
+ read = int(details.get("cache_read", 0) or 0)
33
+ by_ttl = int(details.get("ephemeral_5m_input_tokens", 0) or 0) + int(details.get("ephemeral_1h_input_tokens", 0) or 0)
34
+ return read, by_ttl or int(details.get("cache_creation", 0) or 0)
35
+
36
+
37
+ def _usage_from_message(message: object) -> tuple[int, int] | None:
38
+ usage = getattr(message, "usage_metadata", None)
39
+ if not usage:
40
+ return None
41
+ return int(usage.get("input_tokens", 0) or 0), int(usage.get("output_tokens", 0) or 0)
42
+
43
+
44
+ def _token_usage(usage: object) -> tuple[int, int] | None:
45
+ """(input, output) tokens from an ``llm_output["token_usage"]`` dict, in either naming:
46
+ ``input_tokens``/``output_tokens`` or OpenAI-style ``prompt_tokens``/``completion_tokens``."""
47
+ if not isinstance(usage, dict):
48
+ return None
49
+ input_tokens = usage.get("input_tokens") or usage.get("prompt_tokens") or 0
50
+ output_tokens = usage.get("output_tokens") or usage.get("completion_tokens") or 0
51
+ try:
52
+ return int(input_tokens), int(output_tokens)
53
+ except (TypeError, ValueError):
54
+ return None
55
+
56
+
57
+ def _model_name(response_metadata: dict) -> str | None:
58
+ return response_metadata.get("model_name") or response_metadata.get("model")
59
+
60
+
61
+ # deepagents' filesystem tools and the argument that names what they touched.
62
+ PATH_TOOLS = ("read_file", "ls", "glob", "grep")
63
+ _PATH_ARGS = ("file_path", "path")
64
+ MAX_TOOL_PATHS = 50
65
+
66
+
67
+ def _tool_args(input_str: str, inputs: object) -> dict | None:
68
+ """The tool's arguments: the `inputs` kwarg when LangChain passes one, else `input_str` parsed
69
+ as JSON or a Python literal; None when neither is a dict."""
70
+ if isinstance(inputs, dict):
71
+ return inputs
72
+ for parse in (json.loads, ast.literal_eval):
73
+ try:
74
+ parsed = parse(input_str)
75
+ except (ValueError, SyntaxError, TypeError, MemoryError, RecursionError):
76
+ continue
77
+ return parsed if isinstance(parsed, dict) else None
78
+ return None
79
+
80
+
81
+ def _tool_path(input_str: str, inputs: object) -> str | None:
82
+ args = _tool_args(input_str, inputs)
83
+ if args is None:
84
+ return None
85
+ for key in _PATH_ARGS:
86
+ value = args.get(key)
87
+ if isinstance(value, str) and value:
88
+ return value
89
+ return None
90
+
91
+
92
+ class UsageCollector(BaseCallbackHandler):
93
+ """A LangChain callback that counts every chat-model call and tool call, including nested
94
+ (sub-agent) runs. Safe to share across threads: appends are protected by a lock.
95
+
96
+ It also records the path each file tool (`PATH_TOOLS`) was called on, in `tool_paths`: at most
97
+ `MAX_TOOL_PATHS` distinct paths per tool, in call order. Pass `tool_paths` to record into a dict
98
+ that outlives this collector (the engine's `CommandLog.tool_paths`).
99
+ """
100
+
101
+ def __init__(self, *, ignore_tools: Iterable[str] = (), tool_paths: dict[str, list[str]] | None = None) -> None:
102
+ self._lock = threading.Lock()
103
+ self.calls: list[ModelCall] = []
104
+ self.tool_calls: dict[str, int] = {}
105
+ self.tool_paths: dict[str, list[str]] = tool_paths if tool_paths is not None else {}
106
+ self._ignore_tools = set(ignore_tools)
107
+ self._served_by: dict[object, str] = {} # run id → the model a tagged chat model call ran on
108
+
109
+ def on_chat_model_start(self, serialized, messages, *, run_id, metadata=None, **kwargs) -> None:
110
+ served_by = (metadata or {}).get(_SERVED_BY_KEY)
111
+ if isinstance(served_by, str):
112
+ with self._lock:
113
+ self._served_by[run_id] = served_by
114
+
115
+ def on_llm_error(self, error, *, run_id, **kwargs) -> None:
116
+ with self._lock:
117
+ self._served_by.pop(run_id, None)
118
+
119
+ def on_llm_end(self, response: LLMResult, *, run_id, parent_run_id=None, **kwargs) -> None:
120
+ with self._lock:
121
+ served_by = self._served_by.pop(run_id, None)
122
+ llm_output = response.llm_output or {}
123
+ # The response-level aggregate, for generations whose message carries no usage. It is
124
+ # for the whole response, so it is applied to one generation only (n > 1 would
125
+ # otherwise count it n times).
126
+ token_usage_fallback = _token_usage(llm_output.get("token_usage"))
127
+ fallback_model = _model_name(llm_output)
128
+
129
+ new_calls: list[ModelCall] = []
130
+ for generation_list in response.generations:
131
+ for generation in generation_list:
132
+ message = getattr(generation, "message", None)
133
+ if message is None:
134
+ continue
135
+ usage = _usage_from_message(message)
136
+ if usage is None:
137
+ usage, token_usage_fallback = token_usage_fallback or (0, 0), None
138
+ response_metadata = getattr(message, "response_metadata", None) or {}
139
+ cost = response_metadata.get("cost")
140
+ model = served_by or _model_name(response_metadata) or fallback_model
141
+ cache_read, cache_write = _cache_tokens(message)
142
+ new_calls.append(
143
+ ModelCall(
144
+ model=model,
145
+ input_tokens=usage[0],
146
+ output_tokens=usage[1],
147
+ reported_cost=float(cost) if cost is not None else None,
148
+ cache_read_tokens=cache_read,
149
+ cache_write_tokens=cache_write,
150
+ provider=served_by.partition(":")[0] if served_by else None,
151
+ )
152
+ )
153
+ if new_calls:
154
+ with self._lock:
155
+ self.calls.extend(new_calls)
156
+
157
+ def on_tool_start(self, serialized: dict, input_str: str, *, run_id, **kwargs) -> None:
158
+ name = (serialized or {}).get("name") or kwargs.get("name")
159
+ if not name or name in self._ignore_tools:
160
+ return
161
+ path = _tool_path(input_str, kwargs.get("inputs")) if name in PATH_TOOLS else None
162
+ with self._lock:
163
+ self.tool_calls[name] = self.tool_calls.get(name, 0) + 1
164
+ if path is not None:
165
+ paths = self.tool_paths.setdefault(name, [])
166
+ if path not in paths and len(paths) < MAX_TOOL_PATHS:
167
+ paths.append(path)
@@ -0,0 +1,69 @@
1
+ import re
2
+ import shlex
3
+ from collections.abc import Iterable
4
+ from pathlib import Path, PurePosixPath
5
+
6
+ from phil.contracts import Contract
7
+
8
+ # The shell tool's own name always counts as a shell call, so a claim naming it must still match
9
+ # by command tokens rather than being waved through just because "run_shell" was called.
10
+ _SHELL_TOOL_NAME = "run_shell"
11
+
12
+
13
+ # Models write a claim's command loosely: "run_shell npm test", "a; b", "cat x (after the build)".
14
+ _TRAILING_NOTE = re.compile(r"\s+\([^()]*\)\s*$")
15
+ _JOINERS = re.compile(r"\s*(?:;|&&)\s*")
16
+
17
+
18
+ def _tokens(command: str) -> tuple[str, ...]:
19
+ try:
20
+ return tuple(shlex.split(command))
21
+ except ValueError:
22
+ return (command.strip(),)
23
+
24
+
25
+ def _parts(command: str) -> list[tuple[str, ...]]:
26
+ """The commands a loosely written claim names: a trailing `(note)` dropped, split on `;` and
27
+ `&&`, and a leading `run_shell` dropped from each part."""
28
+ parts = []
29
+ for part in _JOINERS.split(_TRAILING_NOTE.sub("", command)):
30
+ tokens = _tokens(part)
31
+ if tokens[:1] == (_SHELL_TOOL_NAME,):
32
+ tokens = tokens[1:]
33
+ parts.append(tokens)
34
+ return parts
35
+
36
+
37
+ def check_evidence(
38
+ output: Contract, *, commands: list[str], workdir: Path | None, tools: Iterable[str] = ()
39
+ ) -> list[str]:
40
+ problems: list[str] = []
41
+ ran = {_tokens(command) for command in commands}
42
+ tool_names = set(tools) - {_SHELL_TOOL_NAME}
43
+ self_check = getattr(output, "self_check", None)
44
+ if self_check is not None:
45
+ for claim in self_check.evidence:
46
+ if not claim.command:
47
+ continue
48
+ tokens = _tokens(claim.command)
49
+ if tokens in ran:
50
+ continue
51
+ if tokens and tokens[0] in tool_names:
52
+ continue
53
+ parts = _parts(claim.command)
54
+ if all(part and (part in ran or part[0] in tool_names) for part in parts):
55
+ continue
56
+ problems.append(f"claimed command was never run: {claim.command}")
57
+ if workdir is not None:
58
+ resolved_workdir = workdir.resolve()
59
+ for path in getattr(output, "tests_added", []):
60
+ # The file tools show the worktree root as "/", so a leading "/" means the worktree.
61
+ relative = path.lstrip("/")
62
+ pure_path = PurePosixPath(relative)
63
+ resolved_path = (workdir / relative).resolve()
64
+ outside = ".." in pure_path.parts or not resolved_path.is_relative_to(resolved_workdir)
65
+ if outside:
66
+ problems.append(f"tests_added path is outside the worktree: {path}")
67
+ elif not resolved_path.exists():
68
+ problems.append(f"tests_added file does not exist: {path}")
69
+ return problems