phil-agent 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phil/__init__.py +16 -0
- phil/__main__.py +4 -0
- phil/agents/__init__.py +0 -0
- phil/agents/activity.py +185 -0
- phil/agents/check.py +217 -0
- phil/agents/collector.py +167 -0
- phil/agents/evidence.py +69 -0
- phil/agents/factory.py +351 -0
- phil/agents/failures.py +204 -0
- phil/agents/fake.py +134 -0
- phil/agents/invoke.py +721 -0
- phil/agents/model_retry.py +123 -0
- phil/agents/openrouter_model.py +32 -0
- phil/agents/pricing.py +353 -0
- phil/agents/providers.py +207 -0
- phil/agents/registry.py +108 -0
- phil/agents/reminder.py +102 -0
- phil/agents/retry.py +260 -0
- phil/agents/salvage.py +211 -0
- phil/agents/spec.py +55 -0
- phil/agents/tool_args.py +171 -0
- phil/agents/tool_choice.py +36 -0
- phil/agents/tools.py +96 -0
- phil/agents/unpriced.py +128 -0
- phil/agents/usage.py +21 -0
- phil/agents/vision.py +233 -0
- phil/chat/__init__.py +0 -0
- phil/chat/answer.py +46 -0
- phil/chat/approval.py +187 -0
- phil/chat/attachments.py +232 -0
- phil/chat/btw.py +30 -0
- phil/chat/controller.py +3838 -0
- phil/chat/decision.py +283 -0
- phil/chat/design.py +33 -0
- phil/chat/events.py +12 -0
- phil/chat/mockup.py +64 -0
- phil/chat/mockup_page.py +302 -0
- phil/chat/overview.py +18 -0
- phil/chat/planning.py +298 -0
- phil/chat/session.py +188 -0
- phil/chat/snapshot.py +101 -0
- phil/chat/state.py +147 -0
- phil/chat/terminal.py +338 -0
- phil/chat/watcher.py +330 -0
- phil/cli/__init__.py +0 -0
- phil/cli/attach.py +189 -0
- phil/cli/entry.py +80 -0
- phil/cli/main.py +1416 -0
- phil/cli/memory.py +209 -0
- phil/cli/parked.py +72 -0
- phil/cli/permissions.py +268 -0
- phil/cli/skills.py +209 -0
- phil/config.py +539 -0
- phil/contracts/__init__.py +106 -0
- phil/contracts/base.py +13 -0
- phil/contracts/common.py +44 -0
- phil/contracts/converse.py +76 -0
- phil/contracts/design.py +97 -0
- phil/contracts/inputs.py +88 -0
- phil/contracts/interface.py +110 -0
- phil/contracts/memory.py +43 -0
- phil/contracts/planning.py +86 -0
- phil/contracts/results.py +112 -0
- phil/contracts/routing.py +39 -0
- phil/contracts/schema.py +14 -0
- phil/contracts/skills.py +31 -0
- phil/diagnostics.py +228 -0
- phil/git.py +70 -0
- phil/key_store.py +150 -0
- phil/memory/__init__.py +0 -0
- phil/memory/agents_md.py +173 -0
- phil/memory/apply.py +132 -0
- phil/memory/commands.py +93 -0
- phil/memory/context.py +182 -0
- phil/memory/doc.py +169 -0
- phil/memory/editor.py +95 -0
- phil/memory/learn.py +110 -0
- phil/memory/listing.py +36 -0
- phil/memory/store.py +117 -0
- phil/packets/__init__.py +3 -0
- phil/packets/packet.py +209 -0
- phil/parking.py +33 -0
- phil/permissions/__init__.py +0 -0
- phil/permissions/catalog.py +179 -0
- phil/permissions/listing.py +131 -0
- phil/permissions/load.py +231 -0
- phil/permissions/rules.py +303 -0
- phil/permissions/store.py +362 -0
- phil/platform.py +290 -0
- phil/prompts/__init__.py +0 -0
- phil/prompts/_shared.md +23 -0
- phil/prompts/answer.md +15 -0
- phil/prompts/architect.md +47 -0
- phil/prompts/btw.md +13 -0
- phil/prompts/converse.md +30 -0
- phil/prompts/critic.md +17 -0
- phil/prompts/design.md +21 -0
- phil/prompts/implementer.md +26 -0
- phil/prompts/intake.md +34 -0
- phil/prompts/memory.md +21 -0
- phil/prompts/mockup.md +31 -0
- phil/prompts/model_check.md +7 -0
- phil/prompts/reviewer.md +15 -0
- phil/prompts/route.md +9 -0
- phil/prompts/skill_labels.md +10 -0
- phil/prompts/tester.md +13 -0
- phil/publish/__init__.py +16 -0
- phil/publish/learnings.py +80 -0
- phil/publish/pr_body.py +179 -0
- phil/publish/publisher.py +249 -0
- phil/publish/service.py +235 -0
- phil/repo.py +54 -0
- phil/repo_detect.py +73 -0
- phil/routing/__init__.py +24 -0
- phil/routing/classes.py +83 -0
- phil/routing/classify.py +54 -0
- phil/routing/jev.py +156 -0
- phil/routing/llm.py +24 -0
- phil/routing/policy.py +52 -0
- phil/routing/types.py +52 -0
- phil/run/__init__.py +0 -0
- phil/run/checkpoint.py +14 -0
- phil/run/cleanup.py +134 -0
- phil/run/engine.py +1395 -0
- phil/run/gates.py +216 -0
- phil/run/launch.py +135 -0
- phil/run/naming.py +49 -0
- phil/run/runner.py +66 -0
- phil/run/state.py +297 -0
- phil/run/worker.py +338 -0
- phil/setup/__init__.py +4 -0
- phil/setup/catalog.py +64 -0
- phil/setup/flow.py +660 -0
- phil/setup/guard.py +38 -0
- phil/setup/io.py +129 -0
- phil/setup/skill_labels.py +60 -0
- phil/setup/suggestions.py +31 -0
- phil/setup/write.py +86 -0
- phil/skills/__init__.py +1 -0
- phil/skills/discover.py +132 -0
- phil/skills/labels.py +137 -0
- phil/skills/listing.py +106 -0
- phil/skills/runtime.py +55 -0
- phil/skills/select.py +155 -0
- phil/skills/sources.py +264 -0
- phil/skills/tool.py +159 -0
- phil/store/__init__.py +0 -0
- phil/store/activity.py +193 -0
- phil/store/artifacts.py +88 -0
- phil/store/db.py +139 -0
- phil/store/events.py +89 -0
- phil/store/parked.py +102 -0
- phil/store/paths.py +64 -0
- phil/store/runs.py +179 -0
- phil/store/telemetry.py +259 -0
- phil/text.py +16 -0
- phil/tomlw.py +59 -0
- phil/ui/__init__.py +0 -0
- phil/ui/answer_view.py +10 -0
- phil/ui/banner.py +191 -0
- phil/ui/blocks.py +122 -0
- phil/ui/brief_view.py +19 -0
- phil/ui/bursts.py +167 -0
- phil/ui/callout.py +281 -0
- phil/ui/chats_view.py +44 -0
- phil/ui/diff_view.py +94 -0
- phil/ui/feed_view.py +196 -0
- phil/ui/fragments.py +16 -0
- phil/ui/links.py +16 -0
- phil/ui/mascot.py +12 -0
- phil/ui/plan_view.py +97 -0
- phil/ui/runs.py +20 -0
- phil/ui/runs_view.py +43 -0
- phil/ui/screen/__init__.py +1 -0
- phil/ui/screen/app.py +355 -0
- phil/ui/screen/clipboard.py +91 -0
- phil/ui/screen/console.py +44 -0
- phil/ui/screen/io.py +312 -0
- phil/ui/screen/mode.py +30 -0
- phil/ui/screen/themed.py +24 -0
- phil/ui/screen/widgets.py +487 -0
- phil/ui/show_view.py +204 -0
- phil/ui/theme.py +76 -0
- phil/ui/toolbar.py +353 -0
- phil/ui/when.py +28 -0
- phil/workspace/__init__.py +0 -0
- phil/workspace/shell.py +795 -0
- phil/workspace/worktree.py +104 -0
- phil_agent-0.1.0.dist-info/METADATA +129 -0
- phil_agent-0.1.0.dist-info/RECORD +193 -0
- phil_agent-0.1.0.dist-info/WHEEL +4 -0
- phil_agent-0.1.0.dist-info/entry_points.txt +2 -0
- phil_agent-0.1.0.dist-info/licenses/LICENSE +201 -0
phil/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError
|
|
2
|
+
from importlib.metadata import version as _dist_version
|
|
3
|
+
|
|
4
|
+
DIST_NAME = "phil-agent"
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _installed_version() -> str:
|
|
8
|
+
"""The installed distribution's version, which comes from pyproject.toml. "0+unknown" when
|
|
9
|
+
Phil runs from a source tree that was never installed."""
|
|
10
|
+
try:
|
|
11
|
+
return _dist_version(DIST_NAME)
|
|
12
|
+
except PackageNotFoundError:
|
|
13
|
+
return "0+unknown"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
__version__ = _installed_version()
|
phil/__main__.py
ADDED
phil/agents/__init__.py
ADDED
|
File without changes
|
phil/agents/activity.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Records each agent tool call into the run's activity log (spec 2026-10-07, plan amendment A1:
|
|
2
|
+
a LangChain callback rather than middleware — callbacks reach every harness and every nested
|
|
3
|
+
sub-agent run without changing how agents are built)."""
|
|
4
|
+
|
|
5
|
+
import difflib
|
|
6
|
+
import logging
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import Iterable
|
|
10
|
+
|
|
11
|
+
from langchain_core.callbacks import BaseCallbackHandler
|
|
12
|
+
|
|
13
|
+
from phil.agents.collector import _tool_args
|
|
14
|
+
from phil.store.activity import ActivityLog
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
IGNORED_TOOLS = frozenset({"write_todos"})
|
|
19
|
+
SUB_TOOL = "task" # deepagents' tool that runs a sub-agent
|
|
20
|
+
_SHORT = 60 # a result is at most this many characters
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _first_line(text: str) -> str:
|
|
24
|
+
for line in str(text).splitlines():
|
|
25
|
+
if line.strip():
|
|
26
|
+
return line.strip()[:_SHORT]
|
|
27
|
+
return ""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def summarize_call(tool: str, args: dict | None) -> str:
|
|
31
|
+
args = args or {}
|
|
32
|
+
path = args.get("file_path") or args.get("path") or ""
|
|
33
|
+
if tool == "read_file":
|
|
34
|
+
return f"read {path}"
|
|
35
|
+
if tool == "ls":
|
|
36
|
+
return f"ls {path}".rstrip()
|
|
37
|
+
if tool == "glob":
|
|
38
|
+
return f"glob {args.get('pattern', '')}".rstrip()
|
|
39
|
+
if tool == "grep":
|
|
40
|
+
return f'grep "{args.get("pattern", "")}" {path}'.rstrip()
|
|
41
|
+
if tool == "edit_file":
|
|
42
|
+
return f"edit {path}"
|
|
43
|
+
if tool == "write_file":
|
|
44
|
+
return f"write {path}"
|
|
45
|
+
if tool == "run_shell":
|
|
46
|
+
return f"run {args.get('command', '')}".rstrip()
|
|
47
|
+
if tool == "load_skill":
|
|
48
|
+
name, file = args.get("name", ""), args.get("file")
|
|
49
|
+
return f"skill {name}/{file}" if file else f"skill {name}"
|
|
50
|
+
if tool == "task":
|
|
51
|
+
return f"sub-agent: {_first_line(args.get('description', ''))}".rstrip()
|
|
52
|
+
return tool
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def summarize_result(tool: str, args: dict | None, output: str) -> tuple[str, bool, str | None]:
|
|
56
|
+
"""(result, ok, detail). Reads, listings and searches have no detail."""
|
|
57
|
+
args, output = args or {}, str(output)
|
|
58
|
+
if tool == "run_shell":
|
|
59
|
+
lines = output.splitlines()
|
|
60
|
+
status = lines[0] if lines else ""
|
|
61
|
+
ok = status.strip() == "exit_code: 0"
|
|
62
|
+
body = lines[1:]
|
|
63
|
+
if body and body[0].startswith("allowed: "): # why it ran: kept in the detail, not the summary
|
|
64
|
+
body = body[1:]
|
|
65
|
+
body = [line for line in body if not line.startswith("full log: ")]
|
|
66
|
+
last = next((line.strip() for line in reversed(body) if line.strip()), status)
|
|
67
|
+
return f"→ {last[:_SHORT]}", ok, output
|
|
68
|
+
if tool == "edit_file":
|
|
69
|
+
old, new = str(args.get("old_string", "")), str(args.get("new_string", ""))
|
|
70
|
+
diff = list(difflib.unified_diff(old.splitlines(), new.splitlines(), "before", "after", lineterm=""))
|
|
71
|
+
added = sum(1 for line in diff if line.startswith("+") and not line.startswith("+++"))
|
|
72
|
+
removed = sum(1 for line in diff if line.startswith("-") and not line.startswith("---"))
|
|
73
|
+
return f"+{added} −{removed}", True, "\n".join(diff)
|
|
74
|
+
if tool == "write_file":
|
|
75
|
+
content = str(args.get("content", ""))
|
|
76
|
+
return f"+{len(content.splitlines())}", True, content
|
|
77
|
+
return "", True, None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class ActivityCallback(BaseCallbackHandler):
|
|
81
|
+
"""Writes a start record on `on_tool_start` and an end record on `on_tool_end`/`on_tool_error`,
|
|
82
|
+
for every tool call in the agent's run tree. Never raises.
|
|
83
|
+
|
|
84
|
+
A call made inside a sub-agent (a `task` call) is tagged with that task call's `sub_id` (its
|
|
85
|
+
seq) and `sub` (its description). To find it, the callback remembers the parent of every run it
|
|
86
|
+
sees (chains, models and tools), and walks up from a tool call to the nearest open task call."""
|
|
87
|
+
|
|
88
|
+
def __init__(self, log: ActivityLog, *, task: str | None, role: str, ignore_tools: Iterable[str] = ()) -> None:
|
|
89
|
+
self._log, self._task, self._role = log, task, role
|
|
90
|
+
self._ignore = IGNORED_TOOLS | set(ignore_tools)
|
|
91
|
+
self._open: dict[object, tuple[int | None, float, str, dict | None, str, dict | None]] = {}
|
|
92
|
+
self._parents: dict[object, object] = {} # LangChain run id -> parent run id
|
|
93
|
+
self._subs: dict[object, tuple[int | None, str]] = {} # an open task call's run id -> (seq, description)
|
|
94
|
+
self._lock = threading.Lock()
|
|
95
|
+
|
|
96
|
+
# --- the run tree ----------------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
def _note_parent(self, run_id, parent_run_id) -> None:
|
|
99
|
+
try:
|
|
100
|
+
if run_id is not None and parent_run_id is not None:
|
|
101
|
+
with self._lock:
|
|
102
|
+
self._parents[run_id] = parent_run_id
|
|
103
|
+
except Exception:
|
|
104
|
+
logger.debug("activity: parent bookkeeping failed", exc_info=True)
|
|
105
|
+
|
|
106
|
+
def on_chain_start(self, serialized, inputs, *, run_id=None, parent_run_id=None, **kwargs) -> None:
|
|
107
|
+
self._note_parent(run_id, parent_run_id)
|
|
108
|
+
|
|
109
|
+
def on_chat_model_start(self, serialized, messages, *, run_id=None, parent_run_id=None, **kwargs) -> None:
|
|
110
|
+
self._note_parent(run_id, parent_run_id)
|
|
111
|
+
|
|
112
|
+
def on_llm_start(self, serialized, prompts, *, run_id=None, parent_run_id=None, **kwargs) -> None:
|
|
113
|
+
self._note_parent(run_id, parent_run_id)
|
|
114
|
+
|
|
115
|
+
def _sub_for(self, parent_run_id) -> dict:
|
|
116
|
+
"""The nearest open task call above `parent_run_id`, as record fields; {} if none."""
|
|
117
|
+
seen, node = set(), parent_run_id
|
|
118
|
+
with self._lock:
|
|
119
|
+
while node is not None and node not in seen:
|
|
120
|
+
if node in self._subs:
|
|
121
|
+
seq, description = self._subs[node]
|
|
122
|
+
return {"sub_id": seq, "sub": description} if seq is not None else {}
|
|
123
|
+
seen.add(node)
|
|
124
|
+
node = self._parents.get(node)
|
|
125
|
+
return {}
|
|
126
|
+
|
|
127
|
+
def reset(self) -> None:
|
|
128
|
+
"""Forget the run tree (call when the agent call finishes)."""
|
|
129
|
+
with self._lock:
|
|
130
|
+
self._parents.clear()
|
|
131
|
+
self._subs.clear()
|
|
132
|
+
|
|
133
|
+
# --- tool calls ------------------------------------------------------------------------
|
|
134
|
+
|
|
135
|
+
def on_tool_start(self, serialized: dict, input_str: str, *, run_id, parent_run_id=None, inputs=None,
|
|
136
|
+
**kwargs) -> None:
|
|
137
|
+
try:
|
|
138
|
+
self._note_parent(run_id, parent_run_id)
|
|
139
|
+
name = (serialized or {}).get("name") or kwargs.get("name") or "tool"
|
|
140
|
+
if name in self._ignore:
|
|
141
|
+
return
|
|
142
|
+
args = _tool_args(input_str, inputs)
|
|
143
|
+
summary = summarize_call(name, args)
|
|
144
|
+
extra = self._sub_for(parent_run_id) or None # a task call is never inside itself
|
|
145
|
+
seq = self._log.start(task=self._task, role=self._role, tool=name, summary=summary, extra=extra)
|
|
146
|
+
with self._lock:
|
|
147
|
+
self._open[run_id] = (seq, time.monotonic(), name, args, summary, extra)
|
|
148
|
+
if name == SUB_TOOL:
|
|
149
|
+
self._subs[run_id] = (seq, _first_line(args.get("description", "")) if args else "")
|
|
150
|
+
except Exception:
|
|
151
|
+
logger.debug("activity: on_tool_start failed", exc_info=True)
|
|
152
|
+
|
|
153
|
+
def _finish(self, run_id, result: str, ok: bool, detail: str | None) -> None:
|
|
154
|
+
with self._lock:
|
|
155
|
+
opened = self._open.pop(run_id, None)
|
|
156
|
+
self._subs.pop(run_id, None)
|
|
157
|
+
self._parents.pop(run_id, None)
|
|
158
|
+
if opened is None:
|
|
159
|
+
return
|
|
160
|
+
seq, started, name, _args, summary, extra = opened
|
|
161
|
+
self._log.end(seq, task=self._task, role=self._role, tool=name, summary=summary, result=result, ok=ok,
|
|
162
|
+
detail=detail, duration_ms=int((time.monotonic() - started) * 1000), extra=extra)
|
|
163
|
+
|
|
164
|
+
def on_tool_end(self, output, *, run_id, **kwargs) -> None:
|
|
165
|
+
try:
|
|
166
|
+
with self._lock:
|
|
167
|
+
opened = self._open.get(run_id)
|
|
168
|
+
if opened is None:
|
|
169
|
+
return
|
|
170
|
+
text = getattr(output, "content", output)
|
|
171
|
+
text = text if isinstance(text, str) else str(text)
|
|
172
|
+
if getattr(output, "status", None) == "error":
|
|
173
|
+
# deepagents' file tools report a failure as a ToolMessage(status="error"), not by raising.
|
|
174
|
+
self._finish(run_id, _first_line(text) or "error", False, text)
|
|
175
|
+
return
|
|
176
|
+
result, ok, detail = summarize_result(opened[2], opened[3], text)
|
|
177
|
+
self._finish(run_id, result, ok, detail)
|
|
178
|
+
except Exception:
|
|
179
|
+
logger.debug("activity: on_tool_end failed", exc_info=True)
|
|
180
|
+
|
|
181
|
+
def on_tool_error(self, error: BaseException, *, run_id, **kwargs) -> None:
|
|
182
|
+
try:
|
|
183
|
+
self._finish(run_id, _first_line(str(error)) or type(error).__name__, False, str(error))
|
|
184
|
+
except Exception:
|
|
185
|
+
logger.debug("activity: on_tool_error failed", exc_info=True)
|
phil/agents/check.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""`phil models check`: one tiny real call per model a role uses, through the agent path Phil uses."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import tempfile
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from pydantic import Field
|
|
11
|
+
|
|
12
|
+
from phil.agents.invoke import AgentContext, AgentFactory, ContractViolation, invoke_agent, shared_price_book
|
|
13
|
+
from phil.agents.pricing import PriceBook
|
|
14
|
+
from phil.agents.unpriced import price_label
|
|
15
|
+
from phil.agents.providers import SYSTEMONE, ProviderSpec, missing_key_message, provider_for_model, split_model
|
|
16
|
+
from phil.agents.spec import AgentSpec
|
|
17
|
+
from phil.config import ROLES, TIERS, VISION_KEY, ConfigError, PhilConfig
|
|
18
|
+
from phil.contracts import Contract
|
|
19
|
+
from phil.key_store import key_lookup
|
|
20
|
+
from phil.packets import build_packet
|
|
21
|
+
from phil.routing.jev import JevError, ping_jev
|
|
22
|
+
from phil.store.artifacts import ArtifactStore
|
|
23
|
+
from phil.store.db import connect
|
|
24
|
+
|
|
25
|
+
CHECK_WORD = "pineapple"
|
|
26
|
+
EXCERPT_CHARS = 120
|
|
27
|
+
NO_STRUCTURED_OUTPUT = "no structured output was returned"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ModelCheckInput(Contract):
|
|
31
|
+
word: str = Field(description="The word to echo back.")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ModelCheck(Contract):
|
|
35
|
+
ok: bool = Field(description="Always true.")
|
|
36
|
+
echo: str = Field(description="The input's word, exactly as given.")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# Lean, and without _shared.md: its self_check rules don't apply to this tiny contract. A text
|
|
40
|
+
# answer gets the same reminders a real agent gets, so a model passes only if Phil can get an
|
|
41
|
+
# answer from it.
|
|
42
|
+
MODEL_CHECK = AgentSpec(
|
|
43
|
+
"model_check", "orchestrator", ModelCheckInput, ModelCheck,
|
|
44
|
+
harness="lean", shared_prompt=False,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class CheckResult:
|
|
50
|
+
label: str # the tiers and `role:<name>` keys that use `model`, e.g. "high, low"
|
|
51
|
+
model: str
|
|
52
|
+
ok: bool
|
|
53
|
+
seconds: float
|
|
54
|
+
detail: str # why it failed; empty when ok
|
|
55
|
+
reminded: bool = False # passed, but only after a reminder to call the answer tool
|
|
56
|
+
price: str | None = None # "$in/$out per M", "no price"…; None when it failed (phil.agents.unpriced)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _role_labels(config: PhilConfig) -> dict[str, str]:
|
|
60
|
+
"""Label -> model for each model some role resolves to: the tier the role uses, or
|
|
61
|
+
`role:<name>` when the role's own [models] key overrides its tier. Roles with no model are skipped.
|
|
62
|
+
|
|
63
|
+
`classifier` is special: it is both a role and a tier key under [models], so a `[models]
|
|
64
|
+
classifier` entry is the classifier *tier*, not a `role:classifier` override. Its label is
|
|
65
|
+
always `classifier` (never `role:classifier`); with no classifier set, the role falls back to
|
|
66
|
+
the low model and shares that label instead."""
|
|
67
|
+
used: dict[str, str] = {}
|
|
68
|
+
for role in ROLES:
|
|
69
|
+
try:
|
|
70
|
+
model = config.model_for(role)
|
|
71
|
+
except ConfigError:
|
|
72
|
+
continue
|
|
73
|
+
if role == "classifier":
|
|
74
|
+
label = "classifier" if role in config.models else config.model_owner(role)
|
|
75
|
+
else:
|
|
76
|
+
label = f"role:{role}" if role in config.models else config.model_owner(role)
|
|
77
|
+
used[label] = model
|
|
78
|
+
return used
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def check_targets(config: PhilConfig) -> list[tuple[str, list[str]]]:
|
|
82
|
+
"""Each distinct model some role resolves to, with the labels that use it: the tiers in tier
|
|
83
|
+
order, then the role keys under [models] (which override their tier), in first-seen order.
|
|
84
|
+
A tier no role uses isn't a target (see `unused_tiers`)."""
|
|
85
|
+
used = _role_labels(config)
|
|
86
|
+
labels: dict[str, list[str]] = {}
|
|
87
|
+
for label in (*TIERS, *(f"role:{role}" for role in ROLES)):
|
|
88
|
+
if label in used:
|
|
89
|
+
labels.setdefault(used[label], []).append(label)
|
|
90
|
+
vision = config.vision_model()
|
|
91
|
+
if vision:
|
|
92
|
+
labels.setdefault(vision, []).append(VISION_KEY)
|
|
93
|
+
return list(labels.items())
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def unused_tiers(config: PhilConfig) -> list[tuple[str, str]]:
|
|
97
|
+
"""(tier, model) for each tier set under [models] that no role maps to; it is not called."""
|
|
98
|
+
used = _role_labels(config)
|
|
99
|
+
return [(tier, config.models[tier]) for tier in TIERS if tier in config.models and tier not in used]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _first_line(text: str) -> str:
|
|
103
|
+
return text.strip().splitlines()[0] if text.strip() else ""
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _error_detail(exc: Exception) -> str:
|
|
107
|
+
if isinstance(exc, ConfigError): # Phil's own message (unknown provider, missing key) says it all
|
|
108
|
+
return _first_line(str(exc))
|
|
109
|
+
return f"{type(exc).__name__}: {_first_line(str(exc))}"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _violation_detail(exc: ContractViolation) -> str:
|
|
113
|
+
raw = None
|
|
114
|
+
if exc.rejected_path is not None:
|
|
115
|
+
raw = json.loads(Path(exc.rejected_path).read_text(encoding="utf-8")).get("raw")
|
|
116
|
+
if exc.problems == [NO_STRUCTURED_OUTPUT]:
|
|
117
|
+
if isinstance(raw, str) and raw.strip():
|
|
118
|
+
excerpt = " ".join(raw.split())[:EXCERPT_CHARS]
|
|
119
|
+
return f'returned text instead of the required structured output: "{excerpt}"'
|
|
120
|
+
return "returned no structured output"
|
|
121
|
+
return f"returned invalid structured output: {'; '.join(exc.problems)}"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# How a decision-model provider is named in a check's failure message.
|
|
125
|
+
PROVIDER_LABELS = {"typesafe": "TypeSafe", "openrouter_decisions": "OpenRouter"}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _jev_detail(provider: ProviderSpec, exc: JevError) -> str:
|
|
129
|
+
label = PROVIDER_LABELS.get(provider.name, provider.name)
|
|
130
|
+
if exc.reason == "missing key":
|
|
131
|
+
return missing_key_message(provider.name, provider.api_key_env or "", ["classifier"])
|
|
132
|
+
if exc.reason in ("http 401", "http 403"):
|
|
133
|
+
return f"{label} refused the request ({exc.reason}): check {provider.api_key_env}."
|
|
134
|
+
return f"{label} didn't answer ({exc.reason})."
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _check_one(
|
|
138
|
+
ctx: AgentContext, model: str, labels: list[str], call: int, jev_transport: object | None
|
|
139
|
+
) -> tuple[bool, str, bool]:
|
|
140
|
+
"""(ok, detail, reminded): reminded when the model passed only after a reminder to call its answer tool."""
|
|
141
|
+
try:
|
|
142
|
+
# Checked here so the errors name the tiers and roles using the model, not the check's own role.
|
|
143
|
+
provider = provider_for_model(ctx.config, model, labels[0])
|
|
144
|
+
if provider.api_key_env and not key_lookup().get(provider.api_key_env):
|
|
145
|
+
return False, missing_key_message(provider.name, provider.api_key_env, labels), False
|
|
146
|
+
if provider.kind == SYSTEMONE:
|
|
147
|
+
try:
|
|
148
|
+
ping_jev(provider, split_model(model)[1], timeout_s=ctx.config.routing.jev_timeout_s,
|
|
149
|
+
transport=jev_transport)
|
|
150
|
+
except JevError as exc:
|
|
151
|
+
return False, _jev_detail(provider, exc), False
|
|
152
|
+
return True, "", False
|
|
153
|
+
packet = build_packet("model_check", ModelCheckInput(word=CHECK_WORD), budget_tokens=1_000)
|
|
154
|
+
output = invoke_agent(
|
|
155
|
+
MODEL_CHECK, packet, ctx, node="model_check", call=call,
|
|
156
|
+
model=model, max_attempts=1, transient_retries=False,
|
|
157
|
+
)
|
|
158
|
+
except ContractViolation as exc:
|
|
159
|
+
return False, _violation_detail(exc), False
|
|
160
|
+
except Exception as exc:
|
|
161
|
+
return False, _error_detail(exc), False
|
|
162
|
+
assert isinstance(output, ModelCheck)
|
|
163
|
+
if not output.ok:
|
|
164
|
+
return False, "returned ok=false", False
|
|
165
|
+
if output.echo.strip() != CHECK_WORD:
|
|
166
|
+
return False, f'echoed "{output.echo}" instead of "{CHECK_WORD}"', False
|
|
167
|
+
row = ctx.conn.execute(
|
|
168
|
+
"SELECT reminders FROM telemetry WHERE node = ? AND call = ? ORDER BY id DESC LIMIT 1",
|
|
169
|
+
("model_check", call),
|
|
170
|
+
).fetchone()
|
|
171
|
+
return True, "", bool(row and row[0])
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def check_models(
|
|
175
|
+
config: PhilConfig,
|
|
176
|
+
*,
|
|
177
|
+
factory: AgentFactory | None = None,
|
|
178
|
+
repo_root: Path,
|
|
179
|
+
clock: Callable[[], float] = time.monotonic,
|
|
180
|
+
jev_transport: object | None = None,
|
|
181
|
+
prices: PriceBook | None = None,
|
|
182
|
+
) -> list[CheckResult]:
|
|
183
|
+
"""One call per distinct model a role resolves to (see `check_targets`): a lean agent asked to return
|
|
184
|
+
`ModelCheck(ok=true, echo=<word>)`, with the configured timeout, one try and no retries. Its
|
|
185
|
+
telemetry and artifacts go to a temporary directory, not the project's.
|
|
186
|
+
|
|
187
|
+
A typesafe (systemone) classifier model is pinged with `ping_jev` instead, through the lean
|
|
188
|
+
agent path; `jev_transport` overrides its HTTP transport (tests only).
|
|
189
|
+
|
|
190
|
+
A passing model's result carries its price; `prices` is the price book for it, the
|
|
191
|
+
process-wide one when not given."""
|
|
192
|
+
targets = check_targets(config)
|
|
193
|
+
if not targets:
|
|
194
|
+
return []
|
|
195
|
+
results: list[CheckResult] = []
|
|
196
|
+
with tempfile.TemporaryDirectory(prefix="phil-models-check-") as scratch:
|
|
197
|
+
conn = connect(Path(scratch) / "check.db")
|
|
198
|
+
try:
|
|
199
|
+
ctx = AgentContext(
|
|
200
|
+
config=config,
|
|
201
|
+
conn=conn,
|
|
202
|
+
layer="chat",
|
|
203
|
+
artifacts=ArtifactStore(Path(scratch) / "artifacts"),
|
|
204
|
+
workdir=repo_root,
|
|
205
|
+
factory=factory,
|
|
206
|
+
)
|
|
207
|
+
book = prices if prices is not None else shared_price_book()
|
|
208
|
+
for call, (model, labels) in enumerate(targets, 1):
|
|
209
|
+
started = clock()
|
|
210
|
+
ok, detail, reminded = _check_one(ctx, model, labels, call, jev_transport)
|
|
211
|
+
seconds = clock() - started
|
|
212
|
+
# Looked up after the clock stops: a first fetch of the list isn't the model's time.
|
|
213
|
+
price = price_label(config, model, book) if ok else None
|
|
214
|
+
results.append(CheckResult(", ".join(labels), model, ok, seconds, detail, reminded, price))
|
|
215
|
+
finally:
|
|
216
|
+
conn.close()
|
|
217
|
+
return results
|
phil/agents/collector.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import json
|
|
3
|
+
import threading
|
|
4
|
+
from collections.abc import Iterable
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from langchain_core.callbacks import BaseCallbackHandler
|
|
8
|
+
from langchain_core.outputs import LLMResult
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True)
|
|
12
|
+
class ModelCall:
|
|
13
|
+
model: str | None # model name from the response/serialized llm, if known
|
|
14
|
+
input_tokens: int
|
|
15
|
+
output_tokens: int
|
|
16
|
+
reported_cost: float | None
|
|
17
|
+
cache_read_tokens: int = 0 # of input_tokens, how many were read from the provider's prompt cache
|
|
18
|
+
cache_write_tokens: int = 0 # of input_tokens, how many were written to it
|
|
19
|
+
# Set for a call another model than the agent's served (the `vision` model): `model` is then
|
|
20
|
+
# its configured `provider:model` and this the provider's name, so it's priced at its own rates.
|
|
21
|
+
provider: str | None = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# phil.agents.vision.SERVED_BY_KEY (not imported: that module loads langchain's agent stack).
|
|
25
|
+
_SERVED_BY_KEY = "phil_served_by"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _cache_tokens(message: object) -> tuple[int, int]:
|
|
29
|
+
"""(read, written) prompt-cache tokens. With Anthropic's per-TTL breakdown, langchain_anthropic
|
|
30
|
+
reports the writes under `ephemeral_5m/1h_input_tokens` and zeroes `cache_creation`."""
|
|
31
|
+
details = (getattr(message, "usage_metadata", None) or {}).get("input_token_details") or {}
|
|
32
|
+
read = int(details.get("cache_read", 0) or 0)
|
|
33
|
+
by_ttl = int(details.get("ephemeral_5m_input_tokens", 0) or 0) + int(details.get("ephemeral_1h_input_tokens", 0) or 0)
|
|
34
|
+
return read, by_ttl or int(details.get("cache_creation", 0) or 0)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _usage_from_message(message: object) -> tuple[int, int] | None:
|
|
38
|
+
usage = getattr(message, "usage_metadata", None)
|
|
39
|
+
if not usage:
|
|
40
|
+
return None
|
|
41
|
+
return int(usage.get("input_tokens", 0) or 0), int(usage.get("output_tokens", 0) or 0)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _token_usage(usage: object) -> tuple[int, int] | None:
|
|
45
|
+
"""(input, output) tokens from an ``llm_output["token_usage"]`` dict, in either naming:
|
|
46
|
+
``input_tokens``/``output_tokens`` or OpenAI-style ``prompt_tokens``/``completion_tokens``."""
|
|
47
|
+
if not isinstance(usage, dict):
|
|
48
|
+
return None
|
|
49
|
+
input_tokens = usage.get("input_tokens") or usage.get("prompt_tokens") or 0
|
|
50
|
+
output_tokens = usage.get("output_tokens") or usage.get("completion_tokens") or 0
|
|
51
|
+
try:
|
|
52
|
+
return int(input_tokens), int(output_tokens)
|
|
53
|
+
except (TypeError, ValueError):
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _model_name(response_metadata: dict) -> str | None:
|
|
58
|
+
return response_metadata.get("model_name") or response_metadata.get("model")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# deepagents' filesystem tools and the argument that names what they touched.
|
|
62
|
+
PATH_TOOLS = ("read_file", "ls", "glob", "grep")
|
|
63
|
+
_PATH_ARGS = ("file_path", "path")
|
|
64
|
+
MAX_TOOL_PATHS = 50
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _tool_args(input_str: str, inputs: object) -> dict | None:
|
|
68
|
+
"""The tool's arguments: the `inputs` kwarg when LangChain passes one, else `input_str` parsed
|
|
69
|
+
as JSON or a Python literal; None when neither is a dict."""
|
|
70
|
+
if isinstance(inputs, dict):
|
|
71
|
+
return inputs
|
|
72
|
+
for parse in (json.loads, ast.literal_eval):
|
|
73
|
+
try:
|
|
74
|
+
parsed = parse(input_str)
|
|
75
|
+
except (ValueError, SyntaxError, TypeError, MemoryError, RecursionError):
|
|
76
|
+
continue
|
|
77
|
+
return parsed if isinstance(parsed, dict) else None
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _tool_path(input_str: str, inputs: object) -> str | None:
|
|
82
|
+
args = _tool_args(input_str, inputs)
|
|
83
|
+
if args is None:
|
|
84
|
+
return None
|
|
85
|
+
for key in _PATH_ARGS:
|
|
86
|
+
value = args.get(key)
|
|
87
|
+
if isinstance(value, str) and value:
|
|
88
|
+
return value
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class UsageCollector(BaseCallbackHandler):
|
|
93
|
+
"""A LangChain callback that counts every chat-model call and tool call, including nested
|
|
94
|
+
(sub-agent) runs. Safe to share across threads: appends are protected by a lock.
|
|
95
|
+
|
|
96
|
+
It also records the path each file tool (`PATH_TOOLS`) was called on, in `tool_paths`: at most
|
|
97
|
+
`MAX_TOOL_PATHS` distinct paths per tool, in call order. Pass `tool_paths` to record into a dict
|
|
98
|
+
that outlives this collector (the engine's `CommandLog.tool_paths`).
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
def __init__(self, *, ignore_tools: Iterable[str] = (), tool_paths: dict[str, list[str]] | None = None) -> None:
|
|
102
|
+
self._lock = threading.Lock()
|
|
103
|
+
self.calls: list[ModelCall] = []
|
|
104
|
+
self.tool_calls: dict[str, int] = {}
|
|
105
|
+
self.tool_paths: dict[str, list[str]] = tool_paths if tool_paths is not None else {}
|
|
106
|
+
self._ignore_tools = set(ignore_tools)
|
|
107
|
+
self._served_by: dict[object, str] = {} # run id → the model a tagged chat model call ran on
|
|
108
|
+
|
|
109
|
+
def on_chat_model_start(self, serialized, messages, *, run_id, metadata=None, **kwargs) -> None:
|
|
110
|
+
served_by = (metadata or {}).get(_SERVED_BY_KEY)
|
|
111
|
+
if isinstance(served_by, str):
|
|
112
|
+
with self._lock:
|
|
113
|
+
self._served_by[run_id] = served_by
|
|
114
|
+
|
|
115
|
+
def on_llm_error(self, error, *, run_id, **kwargs) -> None:
|
|
116
|
+
with self._lock:
|
|
117
|
+
self._served_by.pop(run_id, None)
|
|
118
|
+
|
|
119
|
+
def on_llm_end(self, response: LLMResult, *, run_id, parent_run_id=None, **kwargs) -> None:
|
|
120
|
+
with self._lock:
|
|
121
|
+
served_by = self._served_by.pop(run_id, None)
|
|
122
|
+
llm_output = response.llm_output or {}
|
|
123
|
+
# The response-level aggregate, for generations whose message carries no usage. It is
|
|
124
|
+
# for the whole response, so it is applied to one generation only (n > 1 would
|
|
125
|
+
# otherwise count it n times).
|
|
126
|
+
token_usage_fallback = _token_usage(llm_output.get("token_usage"))
|
|
127
|
+
fallback_model = _model_name(llm_output)
|
|
128
|
+
|
|
129
|
+
new_calls: list[ModelCall] = []
|
|
130
|
+
for generation_list in response.generations:
|
|
131
|
+
for generation in generation_list:
|
|
132
|
+
message = getattr(generation, "message", None)
|
|
133
|
+
if message is None:
|
|
134
|
+
continue
|
|
135
|
+
usage = _usage_from_message(message)
|
|
136
|
+
if usage is None:
|
|
137
|
+
usage, token_usage_fallback = token_usage_fallback or (0, 0), None
|
|
138
|
+
response_metadata = getattr(message, "response_metadata", None) or {}
|
|
139
|
+
cost = response_metadata.get("cost")
|
|
140
|
+
model = served_by or _model_name(response_metadata) or fallback_model
|
|
141
|
+
cache_read, cache_write = _cache_tokens(message)
|
|
142
|
+
new_calls.append(
|
|
143
|
+
ModelCall(
|
|
144
|
+
model=model,
|
|
145
|
+
input_tokens=usage[0],
|
|
146
|
+
output_tokens=usage[1],
|
|
147
|
+
reported_cost=float(cost) if cost is not None else None,
|
|
148
|
+
cache_read_tokens=cache_read,
|
|
149
|
+
cache_write_tokens=cache_write,
|
|
150
|
+
provider=served_by.partition(":")[0] if served_by else None,
|
|
151
|
+
)
|
|
152
|
+
)
|
|
153
|
+
if new_calls:
|
|
154
|
+
with self._lock:
|
|
155
|
+
self.calls.extend(new_calls)
|
|
156
|
+
|
|
157
|
+
def on_tool_start(self, serialized: dict, input_str: str, *, run_id, **kwargs) -> None:
|
|
158
|
+
name = (serialized or {}).get("name") or kwargs.get("name")
|
|
159
|
+
if not name or name in self._ignore_tools:
|
|
160
|
+
return
|
|
161
|
+
path = _tool_path(input_str, kwargs.get("inputs")) if name in PATH_TOOLS else None
|
|
162
|
+
with self._lock:
|
|
163
|
+
self.tool_calls[name] = self.tool_calls.get(name, 0) + 1
|
|
164
|
+
if path is not None:
|
|
165
|
+
paths = self.tool_paths.setdefault(name, [])
|
|
166
|
+
if path not in paths and len(paths) < MAX_TOOL_PATHS:
|
|
167
|
+
paths.append(path)
|
phil/agents/evidence.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import shlex
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
from pathlib import Path, PurePosixPath
|
|
5
|
+
|
|
6
|
+
from phil.contracts import Contract
|
|
7
|
+
|
|
8
|
+
# The shell tool's own name always counts as a shell call, so a claim naming it must still match
|
|
9
|
+
# by command tokens rather than being waved through just because "run_shell" was called.
|
|
10
|
+
_SHELL_TOOL_NAME = "run_shell"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# Models write a claim's command loosely: "run_shell npm test", "a; b", "cat x (after the build)".
|
|
14
|
+
_TRAILING_NOTE = re.compile(r"\s+\([^()]*\)\s*$")
|
|
15
|
+
_JOINERS = re.compile(r"\s*(?:;|&&)\s*")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _tokens(command: str) -> tuple[str, ...]:
|
|
19
|
+
try:
|
|
20
|
+
return tuple(shlex.split(command))
|
|
21
|
+
except ValueError:
|
|
22
|
+
return (command.strip(),)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _parts(command: str) -> list[tuple[str, ...]]:
|
|
26
|
+
"""The commands a loosely written claim names: a trailing `(note)` dropped, split on `;` and
|
|
27
|
+
`&&`, and a leading `run_shell` dropped from each part."""
|
|
28
|
+
parts = []
|
|
29
|
+
for part in _JOINERS.split(_TRAILING_NOTE.sub("", command)):
|
|
30
|
+
tokens = _tokens(part)
|
|
31
|
+
if tokens[:1] == (_SHELL_TOOL_NAME,):
|
|
32
|
+
tokens = tokens[1:]
|
|
33
|
+
parts.append(tokens)
|
|
34
|
+
return parts
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def check_evidence(
|
|
38
|
+
output: Contract, *, commands: list[str], workdir: Path | None, tools: Iterable[str] = ()
|
|
39
|
+
) -> list[str]:
|
|
40
|
+
problems: list[str] = []
|
|
41
|
+
ran = {_tokens(command) for command in commands}
|
|
42
|
+
tool_names = set(tools) - {_SHELL_TOOL_NAME}
|
|
43
|
+
self_check = getattr(output, "self_check", None)
|
|
44
|
+
if self_check is not None:
|
|
45
|
+
for claim in self_check.evidence:
|
|
46
|
+
if not claim.command:
|
|
47
|
+
continue
|
|
48
|
+
tokens = _tokens(claim.command)
|
|
49
|
+
if tokens in ran:
|
|
50
|
+
continue
|
|
51
|
+
if tokens and tokens[0] in tool_names:
|
|
52
|
+
continue
|
|
53
|
+
parts = _parts(claim.command)
|
|
54
|
+
if all(part and (part in ran or part[0] in tool_names) for part in parts):
|
|
55
|
+
continue
|
|
56
|
+
problems.append(f"claimed command was never run: {claim.command}")
|
|
57
|
+
if workdir is not None:
|
|
58
|
+
resolved_workdir = workdir.resolve()
|
|
59
|
+
for path in getattr(output, "tests_added", []):
|
|
60
|
+
# The file tools show the worktree root as "/", so a leading "/" means the worktree.
|
|
61
|
+
relative = path.lstrip("/")
|
|
62
|
+
pure_path = PurePosixPath(relative)
|
|
63
|
+
resolved_path = (workdir / relative).resolve()
|
|
64
|
+
outside = ".." in pure_path.parts or not resolved_path.is_relative_to(resolved_workdir)
|
|
65
|
+
if outside:
|
|
66
|
+
problems.append(f"tests_added path is outside the worktree: {path}")
|
|
67
|
+
elif not resolved_path.exists():
|
|
68
|
+
problems.append(f"tests_added file does not exist: {path}")
|
|
69
|
+
return problems
|