fivefold 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefold/__init__.py +4 -0
- fivefold/_data/db/bootstrap.sql +26 -0
- fivefold/_data/db/migrations/0001_schemas.sql +67 -0
- fivefold/_data/db/migrations/0002_gateway.sql +224 -0
- fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
- fivefold/_data/db/migrations/0004_harness.sql +141 -0
- fivefold/_data/db/migrations/0005_sim.sql +679 -0
- fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
- fivefold/_data/db/migrations/0007_eval.sql +33 -0
- fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
- fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
- fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
- fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
- fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
- fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
- fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
- fivefold/_data/db/registry/0001_registry.sql +31 -0
- fivefold/_data/faults/expected.yaml +406 -0
- fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
- fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
- fivefold/agents/__init__.py +7 -0
- fivefold/agents/branch.py +187 -0
- fivefold/agents/checkpoint.py +77 -0
- fivefold/agents/connection.py +153 -0
- fivefold/agents/live.py +492 -0
- fivefold/agents/policy.py +426 -0
- fivefold/agents/remote.py +333 -0
- fivefold/agents/replay.py +170 -0
- fivefold/agents/scripted.py +122 -0
- fivefold/agents/tools.py +229 -0
- fivefold/agents/worker.py +164 -0
- fivefold/domain/__init__.py +5 -0
- fivefold/domain/api.py +517 -0
- fivefold/domain/authority.py +149 -0
- fivefold/domain/contracts.py +80 -0
- fivefold/domain/identity.py +113 -0
- fivefold/domain/prices.py +178 -0
- fivefold/domain/report.py +92 -0
- fivefold/domain/rules.py +346 -0
- fivefold/domain/scenario.py +524 -0
- fivefold/evaluator/__init__.py +155 -0
- fivefold/evaluator/__main__.py +127 -0
- fivefold/evaluator/authorization.py +146 -0
- fivefold/evaluator/checks.py +331 -0
- fivefold/evaluator/evaluate.py +109 -0
- fivefold/evaluator/evidence.py +392 -0
- fivefold/evaluator/freshness.py +136 -0
- fivefold/evaluator/liveness.py +236 -0
- fivefold/evaluator/load.py +354 -0
- fivefold/evaluator/metrics.py +95 -0
- fivefold/evaluator/records.py +452 -0
- fivefold/evaluator/recovery.py +209 -0
- fivefold/evaluator/report.py +1096 -0
- fivefold/evaluator/result.py +344 -0
- fivefold/evaluator/store.py +83 -0
- fivefold/faults/__init__.py +5 -0
- fivefold/faults/catalog.py +278 -0
- fivefold/faults/plan.py +538 -0
- fivefold/gateway/__init__.py +6 -0
- fivefold/gateway/admin.py +179 -0
- fivefold/gateway/admin_api.py +398 -0
- fivefold/gateway/codec.py +372 -0
- fivefold/gateway/destination.py +64 -0
- fivefold/gateway/dispatcher.py +379 -0
- fivefold/gateway/freshness.py +83 -0
- fivefold/gateway/invariants.py +761 -0
- fivefold/gateway/memory.py +277 -0
- fivefold/gateway/pg_store.py +657 -0
- fivefold/gateway/records.py +592 -0
- fivefold/gateway/server.py +102 -0
- fivefold/gateway/sim_client.py +295 -0
- fivefold/gateway/state_machine.py +376 -0
- fivefold/gateway/steps.py +897 -0
- fivefold/gateway/store.py +635 -0
- fivefold/gateway/tools.py +424 -0
- fivefold/gateway/twin.py +199 -0
- fivefold/gateway/wire.py +138 -0
- fivefold/harness/__init__.py +6 -0
- fivefold/harness/actors.py +230 -0
- fivefold/harness/cells.py +303 -0
- fivefold/harness/clock.py +209 -0
- fivefold/harness/demo.py +397 -0
- fivefold/harness/fault_effects.py +122 -0
- fivefold/harness/live_options.py +159 -0
- fivefold/harness/log.py +540 -0
- fivefold/harness/mcp_link.py +409 -0
- fivefold/harness/migrations.py +520 -0
- fivefold/harness/models.py +368 -0
- fivefold/harness/registry.py +105 -0
- fivefold/harness/reports.py +100 -0
- fivefold/harness/rng.py +57 -0
- fivefold/harness/run.py +881 -0
- fivefold/harness/scheduler.py +441 -0
- fivefold/harness/sim_points.py +215 -0
- fivefold/harness/split/__init__.py +34 -0
- fivefold/harness/split/__main__.py +275 -0
- fivefold/harness/split/credentials.py +74 -0
- fivefold/harness/split/faults.py +75 -0
- fivefold/harness/split/gateway.py +769 -0
- fivefold/harness/split/hops.py +291 -0
- fivefold/harness/split/lifecycle.py +322 -0
- fivefold/harness/split/rundir.py +152 -0
- fivefold/harness/split/service.py +157 -0
- fivefold/harness/split/simulator.py +432 -0
- fivefold/harness/study.py +643 -0
- fivefold/harness/transport.py +533 -0
- fivefold/paths.py +24 -0
- fivefold/py.typed +0 -0
- fivefold/recorder/__init__.py +18 -0
- fivefold/recorder/__main__.py +308 -0
- fivefold/recorder/cassette.py +322 -0
- fivefold/recorder/playback.py +316 -0
- fivefold/recorder/recording.py +133 -0
- fivefold/sim/__init__.py +37 -0
- fivefold/sim/config.py +52 -0
- fivefold/sim/endpoint.py +176 -0
- fivefold/sim/ledger.py +583 -0
- fivefold/sim/service.py +834 -0
- fivefold/viewer/__init__.py +14 -0
- fivefold/viewer/__main__.py +75 -0
- fivefold/viewer/app.py +122 -0
- fivefold/viewer/records.py +108 -0
- fivefold/viewer/source.py +196 -0
- fivefold/viewer/timeline.py +61 -0
- fivefold-0.1.0.dist-info/METADATA +169 -0
- fivefold-0.1.0.dist-info/RECORD +128 -0
- fivefold-0.1.0.dist-info/WHEEL +4 -0
- fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
fivefold/agents/live.py
ADDED
|
@@ -0,0 +1,492 @@
|
|
|
1
|
+
"""The live model adapter (0009, Models): a provider's chat model, run on the scheduler.
|
|
2
|
+
|
|
3
|
+
`LiveModel` wraps the chat model of one provider (`PROVIDERS`). Each call:
|
|
4
|
+
|
|
5
|
+
1. takes `think_ms` of virtual time first, as every adapter's call does (0009);
|
|
6
|
+
2. checks the spend cap (`SpendMeter`). Once the spend so far has reached the cap, the call
|
|
7
|
+
is not sent: the reply is the harness's refusal (`spend_cap_reply`), and the worker
|
|
8
|
+
ends as `budget` (`fivefold.agents.worker`);
|
|
9
|
+
3. sends the request and awaits the answer in wall time, inside the scheduler's real wait
|
|
10
|
+
(`Scheduler.hold`): virtual time stands still and no event fires until the provider
|
|
11
|
+
answers, so the call takes exactly `think_ms` of virtual time;
|
|
12
|
+
4. charges the reply's token usage at the model's price in the price file.
|
|
13
|
+
|
|
14
|
+
The cost of the call in flight is bounded. Calls run one at a time: while one is in
|
|
15
|
+
flight, no event fires, so no other worker's think can end and send another. Each call
|
|
16
|
+
is checked just before it is sent, once every earlier call has been charged. So the
|
|
17
|
+
spend passes the cap by at most one call's cost: its input tokens at the input price plus
|
|
18
|
+
its output tokens at the output price. The provider's output limit bounds the output:
|
|
19
|
+
`num_predict` for Ollama, `max_tokens` for Anthropic and OpenAI, `max_output_tokens` for
|
|
20
|
+
Google. A reply that reports no usage, from a model that is not free, makes the spend
|
|
21
|
+
unknown, and the cap then counts as reached.
|
|
22
|
+
|
|
23
|
+
The harness records every live call into the run's cassette (0009), a refused one too, so
|
|
24
|
+
the replay adapter serves the same replies and the run plays back with no network. A call
|
|
25
|
+
the provider fails, for example because no server answers, raises: the worker ends with an
|
|
26
|
+
error, as for any exception in its graph (0008), and nothing is charged or recorded. Its
|
|
27
|
+
cassette then ends before that call, so a replay of the run stops there with `ReplayError`.
|
|
28
|
+
|
|
29
|
+
The providers (`PROVIDERS`), each through its LangChain package, which reads the key and
|
|
30
|
+
the base URL from its own environment variables:
|
|
31
|
+
|
|
32
|
+
- `ollama`, through langchain-ollama and the `ollama` package: no key, or
|
|
33
|
+
`OLLAMA_API_KEY` for a server that needs one; the base URL from `OLLAMA_HOST`;
|
|
34
|
+
- `anthropic`, through langchain-anthropic: the key from `ANTHROPIC_API_KEY`; the base URL
|
|
35
|
+
from `ANTHROPIC_API_URL` or `ANTHROPIC_BASE_URL`, or else Anthropic's API;
|
|
36
|
+
- `openai`, through langchain-openai: the key from `OPENAI_API_KEY`; the base URL from
|
|
37
|
+
`OPENAI_API_BASE` or `OPENAI_BASE_URL`, or else OpenAI's API;
|
|
38
|
+
- `google_genai`, through langchain-google-genai and the `google-genai` package: the key
|
|
39
|
+
from `GOOGLE_API_KEY` or `GEMINI_API_KEY`; the base URL from `GOOGLE_GEMINI_BASE_URL`, or
|
|
40
|
+
else Google's Gemini API.
|
|
41
|
+
|
|
42
|
+
Nothing here passes, prints or records a key or a base URL. A live run of a provider that
|
|
43
|
+
needs a key is refused unless one of its key variables is set, which is all that the
|
|
44
|
+
check reads (`fivefold.harness.models.check_live`). A live run of `google_genai` is also
|
|
45
|
+
refused while `GOOGLE_GENAI_USE_VERTEXAI` or `GOOGLE_GENAI_USE_ENTERPRISE` is set: either
|
|
46
|
+
one sends its requests to Vertex AI instead, with Google's default credentials
|
|
47
|
+
(`Provider.forbidden`).
|
|
48
|
+
|
|
49
|
+
Each run executes on an event loop of its own (`asyncio.run`), but langchain-anthropic
|
|
50
|
+
1.7.10 and langchain-openai 1.8.0 keep one async HTTP client for the whole process: each
|
|
51
|
+
caches it with `functools.lru_cache` (`_client_utils.py`). A connection that client keeps
|
|
52
|
+
open after a reply belongs to the loop that opened it, and a later run's call on it fails.
|
|
53
|
+
Observed with each package against a local server, one `asyncio.run` per call: the
|
|
54
|
+
second call failed, with Anthropic's `APIConnectionError` after its request had reached
|
|
55
|
+
the server, so the package's retry would send it again, or with OpenAI's `RuntimeError:
|
|
56
|
+
Event loop is closed`. So both models send `Connection: close` with every request
|
|
57
|
+
(`CLOSE_AFTER_REPLY`), and each connection ends with its reply, on the loop that opened
|
|
58
|
+
it. Checked in h11 0.16.0 (`_connection.py`, `_keep_alive`): a request with that header
|
|
59
|
+
turns keep-alive off; and in httpcore2 2.16.0 (`_async/http11.py`, `_response_closed`):
|
|
60
|
+
the client then closes the connection once the reply ends, whatever the server does.
|
|
61
|
+
|
|
62
|
+
langchain-google-genai 4.5.0 builds a google-genai client for each model, so each run's
|
|
63
|
+
client is new. But google-genai 3.3.0 closes a collected async client on whatever event
|
|
64
|
+
loop is running at the time (`AsyncHttpx2Client.__del__`, `_api_client.py`). Observed
|
|
65
|
+
against a local server: an earlier run's client, collected during a later run, failed
|
|
66
|
+
there on the connections it kept, and one model called from a second loop failed with
|
|
67
|
+
`RuntimeError: Event loop is closed`. So the Google model sends the same header,
|
|
68
|
+
through `additional_headers`, and keeps no connection either.
|
|
69
|
+
|
|
70
|
+
To add a provider:
|
|
71
|
+
|
|
72
|
+
1. add its LangChain chat model package as an exact dependency, in a commit of its own
|
|
73
|
+
that changes no other locked version;
|
|
74
|
+
2. add an entry to `PROVIDERS`: a function that builds the chat model from the model's
|
|
75
|
+
name and parameters, the names of the generation parameters it accepts, the
|
|
76
|
+
variables its package reads its key from, and any variable that would send its
|
|
77
|
+
requests elsewhere;
|
|
78
|
+
3. add the provider's prices, with their date, to the price file the runs use.
|
|
79
|
+
|
|
80
|
+
Checked in langchain-ollama 1.1.0 (`chat_models.py`): `ChatOllama(model=...)` takes
|
|
81
|
+
`temperature`, `seed`, `top_k`, `top_p`, `num_ctx`, `num_predict`, `repeat_penalty` and
|
|
82
|
+
`reasoning`; with `base_url` None the `ollama` client picks the host; it asks the server
|
|
83
|
+
for its models only when `validate_model_on_init` is set (default False), so building one
|
|
84
|
+
sends nothing; `bind_tools` binds the tools' schemas (`convert_to_openai_tool`) as
|
|
85
|
+
`tools`; a reply's `usage_metadata` holds `input_tokens` (Ollama's `prompt_eval_count`)
|
|
86
|
+
and `output_tokens` (`eval_count`), and is None unless Ollama reported both. Checked in
|
|
87
|
+
ollama 0.6.3 (`_client.py`, `BaseClient`): it reads `OLLAMA_HOST` when given no host and
|
|
88
|
+
sends `OLLAMA_API_KEY` as a bearer token; its httpx timeout defaults to None.
|
|
89
|
+
|
|
90
|
+
Checked in langchain-anthropic 1.7.10 (`chat_models.py`): `ChatAnthropic(model=...)` takes
|
|
91
|
+
`max_tokens`, `temperature`, `top_k`, `top_p`, `thinking` and `reasoning_effort`, and puts
|
|
92
|
+
any argument it does not know into `model_kwargs`, which every request carries
|
|
93
|
+
(`build_extra`); a validator reads the key and the base URL from the environment through
|
|
94
|
+
langchain-core 1.8.0's `_apply_gateway_config` (`utils/_gateway.py`), which can route the
|
|
95
|
+
requests through LangSmith's gateway when `LANGSMITH_GATEWAY` is set; the clients are
|
|
96
|
+
built at the first call (`cached_property`), so building one sends nothing; `max_tokens`
|
|
97
|
+
defaults to the model's maximum output in the package's own profiles, or 4,096;
|
|
98
|
+
`bind_tools` converts each tool with `convert_to_anthropic_tool`, and drops with a warning
|
|
99
|
+
any tool whose `input_schema` has a top-level `anyOf` or `oneOf`; `_agenerate` sends one
|
|
100
|
+
`messages.create`, and the reply's `usage_metadata` holds `input_tokens` (the input, cache
|
|
101
|
+
reads and cache writes) and `output_tokens` (`_create_usage_metadata`).
|
|
102
|
+
|
|
103
|
+
Checked in langchain-openai 1.8.0 (`chat_models/base.py`): `ChatOpenAI(model_name=...)`
|
|
104
|
+
takes `max_tokens` (sent as `max_completion_tokens`), `temperature`, `top_p`, `seed`,
|
|
105
|
+
`presence_penalty`, `frequency_penalty` and `reasoning_effort`, and puts any argument it
|
|
106
|
+
does not know into `model_kwargs` (`build_extra`); `validate_environment` reads the key
|
|
107
|
+
and the base URL through `_resolve_gateway_config`, and builds the `openai` clients, which
|
|
108
|
+
send nothing; openai 3.51.0 refuses to build a client without a key
|
|
109
|
+
(`lib/_client_auth.py`); `bind_tools` converts each tool with `convert_to_openai_tool`;
|
|
110
|
+
`_agenerate` sends one request, to Chat Completions or, for the models and settings that
|
|
111
|
+
need it, to the Responses API (`_use_responses_api`), and the reply's `usage_metadata`
|
|
112
|
+
holds `input_tokens` and `output_tokens` from either (`_create_usage_metadata`,
|
|
113
|
+
`_create_usage_metadata_responses`).
|
|
114
|
+
|
|
115
|
+
Checked in langchain-google-genai 4.5.0 (`_common.py`, `chat_models.py`):
|
|
116
|
+
`ChatGoogleGenerativeAI(model=...)` takes `max_output_tokens`, `temperature`, `top_k`,
|
|
117
|
+
`top_p`, `seed`, `presence_penalty`, `frequency_penalty`, `thinking_budget` and
|
|
118
|
+
`reasoning_effort`, and puts any argument it does not know into `model_kwargs`
|
|
119
|
+
(`build_extra`); the key comes from `GOOGLE_API_KEY` or `GEMINI_API_KEY`
|
|
120
|
+
(`secret_from_env`, then `_apply_gateway_config`); `_determine_backend` picks Vertex AI
|
|
121
|
+
when `GOOGLE_GENAI_USE_VERTEXAI` is true, and google-genai 3.3.0's client does so again
|
|
122
|
+
when it or `GOOGLE_GENAI_USE_ENTERPRISE` is true (`BaseApiClient.__init__`,
|
|
123
|
+
`_api_client.py`); for the Gemini API, `validate_environment` builds a `Client` with the
|
|
124
|
+
key, which loads no default credentials and sends nothing until the first call;
|
|
125
|
+
`bind_tools` binds the tools in OpenAI's format, and each request converts them with
|
|
126
|
+
`convert_to_genai_function_declarations` (`_format_tools`), which drops
|
|
127
|
+
`additionalProperties` from a schema; `_agenerate` sends one `generate_content`, which
|
|
128
|
+
google-genai tries up to `max_retries` times (default 6, `HttpRetryOptions`, through
|
|
129
|
+
tenacity); the reply's `usage_metadata` holds `input_tokens` (the prompt's and tool-use
|
|
130
|
+
prompt's tokens) and `output_tokens` (the candidates' and thoughts' tokens), and is None
|
|
131
|
+
when the reply reports none (`_response_to_result`).
|
|
132
|
+
|
|
133
|
+
Checked in langchain-core 1.8.0: `UsageMetadata` (`messages/ai.py`); `ainvoke` streams
|
|
134
|
+
only when asked to, or when a streaming callback is attached (`_should_stream`,
|
|
135
|
+
`language_models/chat_models.py`). Checked in langgraph 1.2.22 (`pregel/main.py`): a graph
|
|
136
|
+
attaches its message streaming callback only for the stream mode `messages`, and the
|
|
137
|
+
agent's graph runs with `ainvoke`, whose mode is `values`. So each live call is one
|
|
138
|
+
request and one reply.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
from __future__ import annotations
|
|
142
|
+
|
|
143
|
+
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
144
|
+
from dataclasses import dataclass
|
|
145
|
+
from decimal import Decimal
|
|
146
|
+
from typing import Any, Final
|
|
147
|
+
|
|
148
|
+
from langchain_core.language_models import LanguageModelInput
|
|
149
|
+
from langchain_core.language_models.chat_models import BaseChatModel
|
|
150
|
+
from langchain_core.messages import AIMessage, BaseMessage
|
|
151
|
+
from langchain_core.messages.ai import UsageMetadata
|
|
152
|
+
from langchain_core.outputs import ChatGeneration, ChatResult
|
|
153
|
+
from langchain_core.runnables import Runnable
|
|
154
|
+
from pydantic import JsonValue, PrivateAttr, ValidationError
|
|
155
|
+
|
|
156
|
+
from fivefold.agents.worker import SPEND_CAP_TEXT
|
|
157
|
+
from fivefold.domain.prices import ModelPrice, PriceFile
|
|
158
|
+
from fivefold.harness.clock import within
|
|
159
|
+
from fivefold.recorder.cassette import Tool
|
|
160
|
+
|
|
161
|
+
Think = Callable[[], Awaitable[None]]
|
|
162
|
+
"""Takes `think_ms` of virtual time."""
|
|
163
|
+
|
|
164
|
+
Hold = Callable[[Awaitable[AIMessage]], Awaitable[AIMessage]]
|
|
165
|
+
"""Awaits a real wait with virtual time held still (`Scheduler.hold`)."""
|
|
166
|
+
|
|
167
|
+
REFUSED: Final = "fivefold_refused"
|
|
168
|
+
"""The reply metadata that marks a reply the harness gave instead of the model, and why."""
|
|
169
|
+
|
|
170
|
+
SPEND_CAP: Final = "spend_cap"
|
|
171
|
+
"""Why the harness refused a call: the spend cap was reached."""
|
|
172
|
+
|
|
173
|
+
CALL_LIMIT_S: Final = 600.0
|
|
174
|
+
"""How long, in wall-clock seconds, a live call may wait for its provider by default. A
|
|
175
|
+
call still unanswered then fails, and its worker ends with an error, so a provider that
|
|
176
|
+
accepts a request and never answers cannot hold the run forever."""
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class LiveError(ValueError):
|
|
180
|
+
"""A live model that cannot be built, such as one with an unknown provider or
|
|
181
|
+
parameter, or a live call that failed because its provider did not answer in time."""
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
# Providers ----------------------------------------------------------------------------------
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass(frozen=True)
|
|
188
|
+
class Provider:
|
|
189
|
+
"""A live provider: how to build its chat model, the parameters it accepts, where its
|
|
190
|
+
package reads its API key, and the variables it refuses.
|
|
191
|
+
|
|
192
|
+
Only generation parameters are accepted. A key, a base URL, a header or a client
|
|
193
|
+
setting never is: each comes from the provider's own environment variables, so no
|
|
194
|
+
parameter can put one in the repository, in a cassette, which records every
|
|
195
|
+
parameter, or in `harness.run`.
|
|
196
|
+
"""
|
|
197
|
+
|
|
198
|
+
build: Callable[[str, Mapping[str, JsonValue]], BaseChatModel]
|
|
199
|
+
params: frozenset[str]
|
|
200
|
+
keys: tuple[str, ...] = ()
|
|
201
|
+
"""The environment variables the provider's package reads its API key from, any one of
|
|
202
|
+
which is enough; none for a provider that needs no key."""
|
|
203
|
+
forbidden: tuple[str, ...] = ()
|
|
204
|
+
"""Environment variables that would send the provider's requests to another backend,
|
|
205
|
+
with other credentials; a live run is refused while one is set."""
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _ollama(model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
|
|
209
|
+
from langchain_ollama import ChatOllama
|
|
210
|
+
|
|
211
|
+
arguments: dict[str, Any] = dict(params)
|
|
212
|
+
return ChatOllama(model=model, **arguments)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
CLOSE_AFTER_REPLY: Final = {"Connection": "close"}
|
|
216
|
+
"""The header the API providers' models send with every request, so that no connection
|
|
217
|
+
outlives the run's event loop (see the module)."""
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _anthropic(model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
|
|
221
|
+
from langchain_anthropic import ChatAnthropic
|
|
222
|
+
|
|
223
|
+
arguments: dict[str, Any] = dict(params)
|
|
224
|
+
return ChatAnthropic(model=model, default_headers=dict(CLOSE_AFTER_REPLY), **arguments)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _openai(model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
|
|
228
|
+
from langchain_openai import ChatOpenAI
|
|
229
|
+
|
|
230
|
+
arguments: dict[str, Any] = dict(params)
|
|
231
|
+
return ChatOpenAI(model_name=model, default_headers=dict(CLOSE_AFTER_REPLY), **arguments)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _google_genai(model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
|
|
235
|
+
from langchain_google_genai import ChatGoogleGenerativeAI
|
|
236
|
+
|
|
237
|
+
arguments: dict[str, Any] = dict(params)
|
|
238
|
+
return ChatGoogleGenerativeAI(
|
|
239
|
+
model=model, additional_headers=dict(CLOSE_AFTER_REPLY), **arguments
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
OLLAMA_PARAMS: Final = frozenset(
|
|
244
|
+
{
|
|
245
|
+
"temperature",
|
|
246
|
+
"seed",
|
|
247
|
+
"top_k",
|
|
248
|
+
"top_p",
|
|
249
|
+
"num_ctx",
|
|
250
|
+
"num_predict",
|
|
251
|
+
"repeat_penalty",
|
|
252
|
+
"reasoning",
|
|
253
|
+
}
|
|
254
|
+
)
|
|
255
|
+
"""The `ChatOllama` parameters a live run may set; each changes what the model generates."""
|
|
256
|
+
|
|
257
|
+
ANTHROPIC_PARAMS: Final = frozenset(
|
|
258
|
+
{"max_tokens", "temperature", "top_k", "top_p", "thinking", "reasoning_effort"}
|
|
259
|
+
)
|
|
260
|
+
"""The `ChatAnthropic` parameters a live run may set; each changes what the model
|
|
261
|
+
generates."""
|
|
262
|
+
|
|
263
|
+
OPENAI_PARAMS: Final = frozenset(
|
|
264
|
+
{
|
|
265
|
+
"max_tokens",
|
|
266
|
+
"temperature",
|
|
267
|
+
"top_p",
|
|
268
|
+
"seed",
|
|
269
|
+
"presence_penalty",
|
|
270
|
+
"frequency_penalty",
|
|
271
|
+
"reasoning_effort",
|
|
272
|
+
}
|
|
273
|
+
)
|
|
274
|
+
"""The `ChatOpenAI` parameters a live run may set; each changes what the model generates."""
|
|
275
|
+
|
|
276
|
+
GOOGLE_GENAI_PARAMS: Final = frozenset(
|
|
277
|
+
{
|
|
278
|
+
"max_output_tokens",
|
|
279
|
+
"temperature",
|
|
280
|
+
"top_k",
|
|
281
|
+
"top_p",
|
|
282
|
+
"seed",
|
|
283
|
+
"presence_penalty",
|
|
284
|
+
"frequency_penalty",
|
|
285
|
+
"thinking_budget",
|
|
286
|
+
"reasoning_effort",
|
|
287
|
+
}
|
|
288
|
+
)
|
|
289
|
+
"""The `ChatGoogleGenerativeAI` parameters a live run may set; each changes what the model
|
|
290
|
+
generates."""
|
|
291
|
+
|
|
292
|
+
PROVIDERS: dict[str, Provider] = {
|
|
293
|
+
"ollama": Provider(_ollama, OLLAMA_PARAMS),
|
|
294
|
+
"anthropic": Provider(_anthropic, ANTHROPIC_PARAMS, ("ANTHROPIC_API_KEY",)),
|
|
295
|
+
"openai": Provider(_openai, OPENAI_PARAMS, ("OPENAI_API_KEY",)),
|
|
296
|
+
"google_genai": Provider(
|
|
297
|
+
_google_genai,
|
|
298
|
+
GOOGLE_GENAI_PARAMS,
|
|
299
|
+
("GOOGLE_API_KEY", "GEMINI_API_KEY"),
|
|
300
|
+
("GOOGLE_GENAI_USE_VERTEXAI", "GOOGLE_GENAI_USE_ENTERPRISE"),
|
|
301
|
+
),
|
|
302
|
+
}
|
|
303
|
+
"""The live providers, by name."""
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def live_name(provider: str, model: str) -> str:
|
|
307
|
+
"""The name of a live model, as `harness.run` and its cassette record it: the provider
|
|
308
|
+
and the model, such as `ollama:llama3.1:8b`."""
|
|
309
|
+
return f"{provider}:{model}"
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def parse_live_name(name: str) -> tuple[str, str]:
|
|
313
|
+
"""The provider and the model of a live model's name: the text before the first colon,
|
|
314
|
+
and the rest."""
|
|
315
|
+
provider, _, model = name.partition(":")
|
|
316
|
+
if not provider or not model:
|
|
317
|
+
raise LiveError(f"a live model is named PROVIDER:MODEL, such as ollama:MODEL, not {name!r}")
|
|
318
|
+
return provider, model
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def check_provider(provider: str, params: Mapping[str, JsonValue]) -> Provider:
|
|
322
|
+
"""`provider`'s entry, if it exists and accepts every parameter in `params`."""
|
|
323
|
+
entry = PROVIDERS.get(provider)
|
|
324
|
+
if entry is None:
|
|
325
|
+
raise LiveError(
|
|
326
|
+
f"no live provider {provider!r}; this build has {', '.join(sorted(PROVIDERS))}"
|
|
327
|
+
)
|
|
328
|
+
unknown = sorted(set(params) - entry.params)
|
|
329
|
+
if unknown:
|
|
330
|
+
raise LiveError(
|
|
331
|
+
f"{provider} accepts the parameters {', '.join(sorted(entry.params))},"
|
|
332
|
+
f" not {', '.join(unknown)}"
|
|
333
|
+
)
|
|
334
|
+
return entry
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def provider_model(provider: str, model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
|
|
338
|
+
"""`provider`'s chat model for `model`, with `params`. Building it sends nothing.
|
|
339
|
+
|
|
340
|
+
A value the provider's model refuses raises `LiveError`, with the first problem in
|
|
341
|
+
one line.
|
|
342
|
+
"""
|
|
343
|
+
entry = check_provider(provider, params)
|
|
344
|
+
try:
|
|
345
|
+
return entry.build(model, params)
|
|
346
|
+
except ValidationError as error:
|
|
347
|
+
first = error.errors(include_url=False)[0]
|
|
348
|
+
where = ".".join(str(part) for part in first["loc"]) or "a parameter"
|
|
349
|
+
raise LiveError(f"{provider} refuses {where}: {first['msg']}") from None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
# The spend cap -----------------------------------------------------------------------------
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
class SpendMeter:
|
|
356
|
+
"""The spend cap of one command's live runs, and what their model calls spent (0004).
|
|
357
|
+
|
|
358
|
+
One meter serves every run of a command, so the cap holds across them. Each call is
|
|
359
|
+
charged at its model's price in the price file, from its reply's token usage.
|
|
360
|
+
"""
|
|
361
|
+
|
|
362
|
+
def __init__(self, cap_usd: Decimal, prices: PriceFile) -> None:
|
|
363
|
+
if not cap_usd.is_finite() or cap_usd <= 0:
|
|
364
|
+
raise LiveError(
|
|
365
|
+
f"the spend cap must be an amount above 0 USD, not {cap_usd}:"
|
|
366
|
+
" a cap of 0 would refuse every call"
|
|
367
|
+
)
|
|
368
|
+
self.cap_usd = cap_usd
|
|
369
|
+
self.prices = prices
|
|
370
|
+
self.spent_usd = Decimal(0)
|
|
371
|
+
"""What the calls charged so far cost, in USD, exactly."""
|
|
372
|
+
self.calls = 0
|
|
373
|
+
"""The calls charged so far."""
|
|
374
|
+
self.unknown: str | None = None
|
|
375
|
+
"""Why the spend is no longer known, if it is not."""
|
|
376
|
+
|
|
377
|
+
def reached(self) -> bool:
|
|
378
|
+
"""Whether the spend so far reached the cap, or is no longer known."""
|
|
379
|
+
return self.unknown is not None or self.spent_usd >= self.cap_usd
|
|
380
|
+
|
|
381
|
+
def charge(self, model: str, price: ModelPrice, usage: UsageMetadata | None) -> None:
|
|
382
|
+
"""Charge one call of `model` at `price`, from its reply's usage."""
|
|
383
|
+
self.calls += 1
|
|
384
|
+
if usage is not None:
|
|
385
|
+
self.spent_usd += price.cost_usd(usage["input_tokens"], usage["output_tokens"])
|
|
386
|
+
elif not price.free:
|
|
387
|
+
self.unknown = f"a reply of {model} reported no token usage, so its cost is unknown"
|
|
388
|
+
|
|
389
|
+
def summary(self) -> str:
|
|
390
|
+
"""The spend so far against the cap, and the price file's date, in one line."""
|
|
391
|
+
line = (
|
|
392
|
+
f"{self.spent_usd} USD spent of the {self.cap_usd} USD cap on {self.calls} model"
|
|
393
|
+
f" calls, at prices dated {self.prices.price_date}"
|
|
394
|
+
)
|
|
395
|
+
return line if self.unknown is None else f"{line}; {self.unknown}"
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def spend_cap_reply() -> AIMessage:
|
|
399
|
+
"""The reply to a call the spend cap refused. It ends the worker as `budget`."""
|
|
400
|
+
return AIMessage(content=SPEND_CAP_TEXT, response_metadata={REFUSED: SPEND_CAP})
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def refusal(reply: AIMessage) -> str | None:
|
|
404
|
+
"""Why the harness refused the call that `reply` answers, or None if the model answered."""
|
|
405
|
+
reason = reply.response_metadata.get(REFUSED)
|
|
406
|
+
return reason if isinstance(reason, str) else None
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
# The adapter --------------------------------------------------------------------------------
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
class LiveModel(BaseChatModel):
|
|
413
|
+
"""A provider's chat model behind the spend cap, on the scheduler (0009)."""
|
|
414
|
+
|
|
415
|
+
_chat: BaseChatModel = PrivateAttr()
|
|
416
|
+
_bound: Runnable[LanguageModelInput, AIMessage] = PrivateAttr()
|
|
417
|
+
_name: str = PrivateAttr()
|
|
418
|
+
_price: ModelPrice = PrivateAttr()
|
|
419
|
+
_meter: SpendMeter = PrivateAttr()
|
|
420
|
+
_think: Think = PrivateAttr()
|
|
421
|
+
_hold: Hold = PrivateAttr()
|
|
422
|
+
_limit_s: float = PrivateAttr()
|
|
423
|
+
|
|
424
|
+
def __init__(
|
|
425
|
+
self,
|
|
426
|
+
chat: BaseChatModel,
|
|
427
|
+
*,
|
|
428
|
+
name: str,
|
|
429
|
+
price: ModelPrice,
|
|
430
|
+
meter: SpendMeter,
|
|
431
|
+
think: Think,
|
|
432
|
+
hold: Hold,
|
|
433
|
+
limit_s: float = CALL_LIMIT_S,
|
|
434
|
+
) -> None:
|
|
435
|
+
"""Answer with `chat`, the provider's model named `name`, charged at `price`. A
|
|
436
|
+
call that waits more than `limit_s` wall-clock seconds for the provider fails."""
|
|
437
|
+
super().__init__()
|
|
438
|
+
self._chat = chat
|
|
439
|
+
self._bound = chat
|
|
440
|
+
self._name = name
|
|
441
|
+
self._price = price
|
|
442
|
+
self._meter = meter
|
|
443
|
+
self._think = think
|
|
444
|
+
self._hold = hold
|
|
445
|
+
self._limit_s = limit_s
|
|
446
|
+
|
|
447
|
+
@property
|
|
448
|
+
def _llm_type(self) -> str:
|
|
449
|
+
return "fivefold-live"
|
|
450
|
+
|
|
451
|
+
def _generate(
|
|
452
|
+
self,
|
|
453
|
+
messages: list[BaseMessage],
|
|
454
|
+
stop: list[str] | None = None,
|
|
455
|
+
run_manager: Any = None,
|
|
456
|
+
**kwargs: Any,
|
|
457
|
+
) -> ChatResult:
|
|
458
|
+
raise NotImplementedError("the live model runs on the scheduler's loop only")
|
|
459
|
+
|
|
460
|
+
async def _agenerate(
|
|
461
|
+
self,
|
|
462
|
+
messages: list[BaseMessage],
|
|
463
|
+
stop: list[str] | None = None,
|
|
464
|
+
run_manager: Any = None,
|
|
465
|
+
**kwargs: Any,
|
|
466
|
+
) -> ChatResult:
|
|
467
|
+
await self._think()
|
|
468
|
+
if self._meter.reached():
|
|
469
|
+
return ChatResult(generations=[ChatGeneration(message=spend_cap_reply())])
|
|
470
|
+
call = self._bound.ainvoke(messages, stop=stop, **kwargs)
|
|
471
|
+
try:
|
|
472
|
+
reply = await self._hold(within(call, self._limit_s))
|
|
473
|
+
except TimeoutError:
|
|
474
|
+
raise LiveError(
|
|
475
|
+
f"the live model {self._name} did not answer within {self._limit_s:g} s"
|
|
476
|
+
) from None
|
|
477
|
+
self._meter.charge(self._name, self._price, reply.usage_metadata)
|
|
478
|
+
return ChatResult(generations=[ChatGeneration(message=reply)])
|
|
479
|
+
|
|
480
|
+
def bind_tools(self, tools: Sequence[Tool], **kwargs: Any) -> LiveModel:
|
|
481
|
+
"""Bind `tools` to the provider's model, which sends their schemas with each call."""
|
|
482
|
+
bound = LiveModel(
|
|
483
|
+
self._chat,
|
|
484
|
+
name=self._name,
|
|
485
|
+
price=self._price,
|
|
486
|
+
meter=self._meter,
|
|
487
|
+
think=self._think,
|
|
488
|
+
hold=self._hold,
|
|
489
|
+
limit_s=self._limit_s,
|
|
490
|
+
)
|
|
491
|
+
bound._bound = self._chat.bind_tools(tools, **kwargs)
|
|
492
|
+
return bound
|