fivefold 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. fivefold/__init__.py +4 -0
  2. fivefold/_data/db/bootstrap.sql +26 -0
  3. fivefold/_data/db/migrations/0001_schemas.sql +67 -0
  4. fivefold/_data/db/migrations/0002_gateway.sql +224 -0
  5. fivefold/_data/db/migrations/0003_gateway_record_0010.sql +91 -0
  6. fivefold/_data/db/migrations/0004_harness.sql +141 -0
  7. fivefold/_data/db/migrations/0005_sim.sql +679 -0
  8. fivefold/_data/db/migrations/0006_gateway_reads.sql +66 -0
  9. fivefold/_data/db/migrations/0007_eval.sql +33 -0
  10. fivefold/_data/db/migrations/0008_run_stop.sql +18 -0
  11. fivefold/_data/db/migrations/0009_message_copies.sql +69 -0
  12. fivefold/_data/db/migrations/0010_gateway_g1_check.sql +14 -0
  13. fivefold/_data/db/migrations/0011_model_calls.sql +37 -0
  14. fivefold/_data/db/migrations/0012_recovery_latency.sql +23 -0
  15. fivefold/_data/db/migrations/0013_split_topology.sql +33 -0
  16. fivefold/_data/db/migrations/0014_tool_call_args.sql +20 -0
  17. fivefold/_data/db/registry/0001_registry.sql +31 -0
  18. fivefold/_data/faults/expected.yaml +406 -0
  19. fivefold/_data/scenarios/dev/dev-001-single-po.yaml +45 -0
  20. fivefold/_data/scenarios/dev/dev-002-human-approval.yaml +47 -0
  21. fivefold/agents/__init__.py +7 -0
  22. fivefold/agents/branch.py +187 -0
  23. fivefold/agents/checkpoint.py +77 -0
  24. fivefold/agents/connection.py +153 -0
  25. fivefold/agents/live.py +492 -0
  26. fivefold/agents/policy.py +426 -0
  27. fivefold/agents/remote.py +333 -0
  28. fivefold/agents/replay.py +170 -0
  29. fivefold/agents/scripted.py +122 -0
  30. fivefold/agents/tools.py +229 -0
  31. fivefold/agents/worker.py +164 -0
  32. fivefold/domain/__init__.py +5 -0
  33. fivefold/domain/api.py +517 -0
  34. fivefold/domain/authority.py +149 -0
  35. fivefold/domain/contracts.py +80 -0
  36. fivefold/domain/identity.py +113 -0
  37. fivefold/domain/prices.py +178 -0
  38. fivefold/domain/report.py +92 -0
  39. fivefold/domain/rules.py +346 -0
  40. fivefold/domain/scenario.py +524 -0
  41. fivefold/evaluator/__init__.py +155 -0
  42. fivefold/evaluator/__main__.py +127 -0
  43. fivefold/evaluator/authorization.py +146 -0
  44. fivefold/evaluator/checks.py +331 -0
  45. fivefold/evaluator/evaluate.py +109 -0
  46. fivefold/evaluator/evidence.py +392 -0
  47. fivefold/evaluator/freshness.py +136 -0
  48. fivefold/evaluator/liveness.py +236 -0
  49. fivefold/evaluator/load.py +354 -0
  50. fivefold/evaluator/metrics.py +95 -0
  51. fivefold/evaluator/records.py +452 -0
  52. fivefold/evaluator/recovery.py +209 -0
  53. fivefold/evaluator/report.py +1096 -0
  54. fivefold/evaluator/result.py +344 -0
  55. fivefold/evaluator/store.py +83 -0
  56. fivefold/faults/__init__.py +5 -0
  57. fivefold/faults/catalog.py +278 -0
  58. fivefold/faults/plan.py +538 -0
  59. fivefold/gateway/__init__.py +6 -0
  60. fivefold/gateway/admin.py +179 -0
  61. fivefold/gateway/admin_api.py +398 -0
  62. fivefold/gateway/codec.py +372 -0
  63. fivefold/gateway/destination.py +64 -0
  64. fivefold/gateway/dispatcher.py +379 -0
  65. fivefold/gateway/freshness.py +83 -0
  66. fivefold/gateway/invariants.py +761 -0
  67. fivefold/gateway/memory.py +277 -0
  68. fivefold/gateway/pg_store.py +657 -0
  69. fivefold/gateway/records.py +592 -0
  70. fivefold/gateway/server.py +102 -0
  71. fivefold/gateway/sim_client.py +295 -0
  72. fivefold/gateway/state_machine.py +376 -0
  73. fivefold/gateway/steps.py +897 -0
  74. fivefold/gateway/store.py +635 -0
  75. fivefold/gateway/tools.py +424 -0
  76. fivefold/gateway/twin.py +199 -0
  77. fivefold/gateway/wire.py +138 -0
  78. fivefold/harness/__init__.py +6 -0
  79. fivefold/harness/actors.py +230 -0
  80. fivefold/harness/cells.py +303 -0
  81. fivefold/harness/clock.py +209 -0
  82. fivefold/harness/demo.py +397 -0
  83. fivefold/harness/fault_effects.py +122 -0
  84. fivefold/harness/live_options.py +159 -0
  85. fivefold/harness/log.py +540 -0
  86. fivefold/harness/mcp_link.py +409 -0
  87. fivefold/harness/migrations.py +520 -0
  88. fivefold/harness/models.py +368 -0
  89. fivefold/harness/registry.py +105 -0
  90. fivefold/harness/reports.py +100 -0
  91. fivefold/harness/rng.py +57 -0
  92. fivefold/harness/run.py +881 -0
  93. fivefold/harness/scheduler.py +441 -0
  94. fivefold/harness/sim_points.py +215 -0
  95. fivefold/harness/split/__init__.py +34 -0
  96. fivefold/harness/split/__main__.py +275 -0
  97. fivefold/harness/split/credentials.py +74 -0
  98. fivefold/harness/split/faults.py +75 -0
  99. fivefold/harness/split/gateway.py +769 -0
  100. fivefold/harness/split/hops.py +291 -0
  101. fivefold/harness/split/lifecycle.py +322 -0
  102. fivefold/harness/split/rundir.py +152 -0
  103. fivefold/harness/split/service.py +157 -0
  104. fivefold/harness/split/simulator.py +432 -0
  105. fivefold/harness/study.py +643 -0
  106. fivefold/harness/transport.py +533 -0
  107. fivefold/paths.py +24 -0
  108. fivefold/py.typed +0 -0
  109. fivefold/recorder/__init__.py +18 -0
  110. fivefold/recorder/__main__.py +308 -0
  111. fivefold/recorder/cassette.py +322 -0
  112. fivefold/recorder/playback.py +316 -0
  113. fivefold/recorder/recording.py +133 -0
  114. fivefold/sim/__init__.py +37 -0
  115. fivefold/sim/config.py +52 -0
  116. fivefold/sim/endpoint.py +176 -0
  117. fivefold/sim/ledger.py +583 -0
  118. fivefold/sim/service.py +834 -0
  119. fivefold/viewer/__init__.py +14 -0
  120. fivefold/viewer/__main__.py +75 -0
  121. fivefold/viewer/app.py +122 -0
  122. fivefold/viewer/records.py +108 -0
  123. fivefold/viewer/source.py +196 -0
  124. fivefold/viewer/timeline.py +61 -0
  125. fivefold-0.1.0.dist-info/METADATA +169 -0
  126. fivefold-0.1.0.dist-info/RECORD +128 -0
  127. fivefold-0.1.0.dist-info/WHEEL +4 -0
  128. fivefold-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,492 @@
1
+ """The live model adapter (0009, Models): a provider's chat model, run on the scheduler.
2
+
3
+ `LiveModel` wraps the chat model of one provider (`PROVIDERS`). Each call:
4
+
5
+ 1. takes `think_ms` of virtual time first, as every adapter's call does (0009);
6
+ 2. checks the spend cap (`SpendMeter`). Once the spend so far has reached the cap, the call
7
+ is not sent: the reply is the harness's refusal (`spend_cap_reply`), and the worker
8
+ ends as `budget` (`fivefold.agents.worker`);
9
+ 3. sends the request and awaits the answer in wall time, inside the scheduler's real wait
10
+ (`Scheduler.hold`): virtual time stands still and no event fires until the provider
11
+ answers, so the call takes exactly `think_ms` of virtual time;
12
+ 4. charges the reply's token usage at the model's price in the price file.
13
+
14
+ The cost of the call in flight is bounded. Calls run one at a time: while one is in
15
+ flight, no event fires, so no other worker's think can end and send another. Each call
16
+ is checked just before it is sent, once every earlier call has been charged. So the
17
+ spend passes the cap by at most one call's cost: its input tokens at the input price plus
18
+ its output tokens at the output price. The provider's output limit bounds the output:
19
+ `num_predict` for Ollama, `max_tokens` for Anthropic and OpenAI, `max_output_tokens` for
20
+ Google. A reply that reports no usage, from a model that is not free, makes the spend
21
+ unknown, and the cap then counts as reached.
22
+
23
+ The harness records every live call into the run's cassette (0009), a refused one too, so
24
+ the replay adapter serves the same replies and the run plays back with no network. A call
25
+ the provider fails, for example because no server answers, raises: the worker ends with an
26
+ error, as for any exception in its graph (0008), and nothing is charged or recorded. Its
27
+ cassette then ends before that call, so a replay of the run stops there with `ReplayError`.
28
+
29
+ The providers (`PROVIDERS`), each through its LangChain package, which reads the key and
30
+ the base URL from its own environment variables:
31
+
32
+ - `ollama`, through langchain-ollama and the `ollama` package: no key, or
33
+ `OLLAMA_API_KEY` for a server that needs one; the base URL from `OLLAMA_HOST`;
34
+ - `anthropic`, through langchain-anthropic: the key from `ANTHROPIC_API_KEY`; the base URL
35
+ from `ANTHROPIC_API_URL` or `ANTHROPIC_BASE_URL`, or else Anthropic's API;
36
+ - `openai`, through langchain-openai: the key from `OPENAI_API_KEY`; the base URL from
37
+ `OPENAI_API_BASE` or `OPENAI_BASE_URL`, or else OpenAI's API;
38
+ - `google_genai`, through langchain-google-genai and the `google-genai` package: the key
39
+ from `GOOGLE_API_KEY` or `GEMINI_API_KEY`; the base URL from `GOOGLE_GEMINI_BASE_URL`, or
40
+ else Google's Gemini API.
41
+
42
+ Nothing here passes, prints or records a key or a base URL. A live run of a provider that
43
+ needs a key is refused unless one of its key variables is set, which is all that the
44
+ check reads (`fivefold.harness.models.check_live`). A live run of `google_genai` is also
45
+ refused while `GOOGLE_GENAI_USE_VERTEXAI` or `GOOGLE_GENAI_USE_ENTERPRISE` is set: either
46
+ one sends its requests to Vertex AI instead, with Google's default credentials
47
+ (`Provider.forbidden`).
48
+
49
+ Each run executes on an event loop of its own (`asyncio.run`), but langchain-anthropic
50
+ 1.7.10 and langchain-openai 1.8.0 keep one async HTTP client for the whole process: each
51
+ caches it with `functools.lru_cache` (`_client_utils.py`). A connection that client keeps
52
+ open after a reply belongs to the loop that opened it, and a later run's call on it fails.
53
+ Observed with each package against a local server, one `asyncio.run` per call: the
54
+ second call failed, with Anthropic's `APIConnectionError` after its request had reached
55
+ the server, so the package's retry would send it again, or with OpenAI's `RuntimeError:
56
+ Event loop is closed`. So both models send `Connection: close` with every request
57
+ (`CLOSE_AFTER_REPLY`), and each connection ends with its reply, on the loop that opened
58
+ it. Checked in h11 0.16.0 (`_connection.py`, `_keep_alive`): a request with that header
59
+ turns keep-alive off; and in httpcore2 2.16.0 (`_async/http11.py`, `_response_closed`):
60
+ the client then closes the connection once the reply ends, whatever the server does.
61
+
62
+ langchain-google-genai 4.5.0 builds a google-genai client for each model, so each run's
63
+ client is new. But google-genai 3.3.0 closes a collected async client on whatever event
64
+ loop is running at the time (`AsyncHttpx2Client.__del__`, `_api_client.py`). Observed
65
+ against a local server: an earlier run's client, collected during a later run, failed
66
+ there on the connections it kept, and one model called from a second loop failed with
67
+ `RuntimeError: Event loop is closed`. So the Google model sends the same header,
68
+ through `additional_headers`, and keeps no connection either.
69
+
70
+ To add a provider:
71
+
72
+ 1. add its LangChain chat model package as an exact dependency, in a commit of its own
73
+ that changes no other locked version;
74
+ 2. add an entry to `PROVIDERS`: a function that builds the chat model from the model's
75
+ name and parameters, the names of the generation parameters it accepts, the
76
+ variables its package reads its key from, and any variable that would send its
77
+ requests elsewhere;
78
+ 3. add the provider's prices, with their date, to the price file the runs use.
79
+
80
+ Checked in langchain-ollama 1.1.0 (`chat_models.py`): `ChatOllama(model=...)` takes
81
+ `temperature`, `seed`, `top_k`, `top_p`, `num_ctx`, `num_predict`, `repeat_penalty` and
82
+ `reasoning`; with `base_url` None the `ollama` client picks the host; it asks the server
83
+ for its models only when `validate_model_on_init` is set (default False), so building one
84
+ sends nothing; `bind_tools` binds the tools' schemas (`convert_to_openai_tool`) as
85
+ `tools`; a reply's `usage_metadata` holds `input_tokens` (Ollama's `prompt_eval_count`)
86
+ and `output_tokens` (`eval_count`), and is None unless Ollama reported both. Checked in
87
+ ollama 0.6.3 (`_client.py`, `BaseClient`): it reads `OLLAMA_HOST` when given no host and
88
+ sends `OLLAMA_API_KEY` as a bearer token; its httpx timeout defaults to None.
89
+
90
+ Checked in langchain-anthropic 1.7.10 (`chat_models.py`): `ChatAnthropic(model=...)` takes
91
+ `max_tokens`, `temperature`, `top_k`, `top_p`, `thinking` and `reasoning_effort`, and puts
92
+ any argument it does not know into `model_kwargs`, which every request carries
93
+ (`build_extra`); a validator reads the key and the base URL from the environment through
94
+ langchain-core 1.8.0's `_apply_gateway_config` (`utils/_gateway.py`), which can route the
95
+ requests through LangSmith's gateway when `LANGSMITH_GATEWAY` is set; the clients are
96
+ built at the first call (`cached_property`), so building one sends nothing; `max_tokens`
97
+ defaults to the model's maximum output in the package's own profiles, or 4,096;
98
+ `bind_tools` converts each tool with `convert_to_anthropic_tool`, and drops with a warning
99
+ any tool whose `input_schema` has a top-level `anyOf` or `oneOf`; `_agenerate` sends one
100
+ `messages.create`, and the reply's `usage_metadata` holds `input_tokens` (the input, cache
101
+ reads and cache writes) and `output_tokens` (`_create_usage_metadata`).
102
+
103
+ Checked in langchain-openai 1.8.0 (`chat_models/base.py`): `ChatOpenAI(model_name=...)`
104
+ takes `max_tokens` (sent as `max_completion_tokens`), `temperature`, `top_p`, `seed`,
105
+ `presence_penalty`, `frequency_penalty` and `reasoning_effort`, and puts any argument it
106
+ does not know into `model_kwargs` (`build_extra`); `validate_environment` reads the key
107
+ and the base URL through `_resolve_gateway_config`, and builds the `openai` clients, which
108
+ send nothing; openai 3.51.0 refuses to build a client without a key
109
+ (`lib/_client_auth.py`); `bind_tools` converts each tool with `convert_to_openai_tool`;
110
+ `_agenerate` sends one request, to Chat Completions or, for the models and settings that
111
+ need it, to the Responses API (`_use_responses_api`), and the reply's `usage_metadata`
112
+ holds `input_tokens` and `output_tokens` from either (`_create_usage_metadata`,
113
+ `_create_usage_metadata_responses`).
114
+
115
+ Checked in langchain-google-genai 4.5.0 (`_common.py`, `chat_models.py`):
116
+ `ChatGoogleGenerativeAI(model=...)` takes `max_output_tokens`, `temperature`, `top_k`,
117
+ `top_p`, `seed`, `presence_penalty`, `frequency_penalty`, `thinking_budget` and
118
+ `reasoning_effort`, and puts any argument it does not know into `model_kwargs`
119
+ (`build_extra`); the key comes from `GOOGLE_API_KEY` or `GEMINI_API_KEY`
120
+ (`secret_from_env`, then `_apply_gateway_config`); `_determine_backend` picks Vertex AI
121
+ when `GOOGLE_GENAI_USE_VERTEXAI` is true, and google-genai 3.3.0's client does so again
122
+ when it or `GOOGLE_GENAI_USE_ENTERPRISE` is true (`BaseApiClient.__init__`,
123
+ `_api_client.py`); for the Gemini API, `validate_environment` builds a `Client` with the
124
+ key, which loads no default credentials and sends nothing until the first call;
125
+ `bind_tools` binds the tools in OpenAI's format, and each request converts them with
126
+ `convert_to_genai_function_declarations` (`_format_tools`), which drops
127
+ `additionalProperties` from a schema; `_agenerate` sends one `generate_content`, which
128
+ google-genai tries up to `max_retries` times (default 6, `HttpRetryOptions`, through
129
+ tenacity); the reply's `usage_metadata` holds `input_tokens` (the prompt's and tool-use
130
+ prompt's tokens) and `output_tokens` (the candidates' and thoughts' tokens), and is None
131
+ when the reply reports none (`_response_to_result`).
132
+
133
+ Checked in langchain-core 1.8.0: `UsageMetadata` (`messages/ai.py`); `ainvoke` streams
134
+ only when asked to, or when a streaming callback is attached (`_should_stream`,
135
+ `language_models/chat_models.py`). Checked in langgraph 1.2.22 (`pregel/main.py`): a graph
136
+ attaches its message streaming callback only for the stream mode `messages`, and the
137
+ agent's graph runs with `ainvoke`, whose mode is `values`. So each live call is one
138
+ request and one reply.
139
+ """
140
+
141
+ from __future__ import annotations
142
+
143
+ from collections.abc import Awaitable, Callable, Mapping, Sequence
144
+ from dataclasses import dataclass
145
+ from decimal import Decimal
146
+ from typing import Any, Final
147
+
148
+ from langchain_core.language_models import LanguageModelInput
149
+ from langchain_core.language_models.chat_models import BaseChatModel
150
+ from langchain_core.messages import AIMessage, BaseMessage
151
+ from langchain_core.messages.ai import UsageMetadata
152
+ from langchain_core.outputs import ChatGeneration, ChatResult
153
+ from langchain_core.runnables import Runnable
154
+ from pydantic import JsonValue, PrivateAttr, ValidationError
155
+
156
+ from fivefold.agents.worker import SPEND_CAP_TEXT
157
+ from fivefold.domain.prices import ModelPrice, PriceFile
158
+ from fivefold.harness.clock import within
159
+ from fivefold.recorder.cassette import Tool
160
+
161
+ Think = Callable[[], Awaitable[None]]
162
+ """Takes `think_ms` of virtual time."""
163
+
164
+ Hold = Callable[[Awaitable[AIMessage]], Awaitable[AIMessage]]
165
+ """Awaits a real wait with virtual time held still (`Scheduler.hold`)."""
166
+
167
+ REFUSED: Final = "fivefold_refused"
168
+ """The reply metadata that marks a reply the harness gave instead of the model, and why."""
169
+
170
+ SPEND_CAP: Final = "spend_cap"
171
+ """Why the harness refused a call: the spend cap was reached."""
172
+
173
+ CALL_LIMIT_S: Final = 600.0
174
+ """How long, in wall-clock seconds, a live call may wait for its provider by default. A
175
+ call still unanswered then fails, and its worker ends with an error, so a provider that
176
+ accepts a request and never answers cannot hold the run forever."""
177
+
178
+
179
+ class LiveError(ValueError):
180
+ """A live model that cannot be built, such as one with an unknown provider or
181
+ parameter, or a live call that failed because its provider did not answer in time."""
182
+
183
+
184
+ # Providers ----------------------------------------------------------------------------------
185
+
186
+
187
+ @dataclass(frozen=True)
188
+ class Provider:
189
+ """A live provider: how to build its chat model, the parameters it accepts, where its
190
+ package reads its API key, and the variables it refuses.
191
+
192
+ Only generation parameters are accepted. A key, a base URL, a header or a client
193
+ setting never is: each comes from the provider's own environment variables, so no
194
+ parameter can put one in the repository, in a cassette, which records every
195
+ parameter, or in `harness.run`.
196
+ """
197
+
198
+ build: Callable[[str, Mapping[str, JsonValue]], BaseChatModel]
199
+ params: frozenset[str]
200
+ keys: tuple[str, ...] = ()
201
+ """The environment variables the provider's package reads its API key from, any one of
202
+ which is enough; none for a provider that needs no key."""
203
+ forbidden: tuple[str, ...] = ()
204
+ """Environment variables that would send the provider's requests to another backend,
205
+ with other credentials; a live run is refused while one is set."""
206
+
207
+
208
+ def _ollama(model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
209
+ from langchain_ollama import ChatOllama
210
+
211
+ arguments: dict[str, Any] = dict(params)
212
+ return ChatOllama(model=model, **arguments)
213
+
214
+
215
+ CLOSE_AFTER_REPLY: Final = {"Connection": "close"}
216
+ """The header the API providers' models send with every request, so that no connection
217
+ outlives the run's event loop (see the module)."""
218
+
219
+
220
+ def _anthropic(model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
221
+ from langchain_anthropic import ChatAnthropic
222
+
223
+ arguments: dict[str, Any] = dict(params)
224
+ return ChatAnthropic(model=model, default_headers=dict(CLOSE_AFTER_REPLY), **arguments)
225
+
226
+
227
+ def _openai(model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
228
+ from langchain_openai import ChatOpenAI
229
+
230
+ arguments: dict[str, Any] = dict(params)
231
+ return ChatOpenAI(model_name=model, default_headers=dict(CLOSE_AFTER_REPLY), **arguments)
232
+
233
+
234
+ def _google_genai(model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
235
+ from langchain_google_genai import ChatGoogleGenerativeAI
236
+
237
+ arguments: dict[str, Any] = dict(params)
238
+ return ChatGoogleGenerativeAI(
239
+ model=model, additional_headers=dict(CLOSE_AFTER_REPLY), **arguments
240
+ )
241
+
242
+
243
+ OLLAMA_PARAMS: Final = frozenset(
244
+ {
245
+ "temperature",
246
+ "seed",
247
+ "top_k",
248
+ "top_p",
249
+ "num_ctx",
250
+ "num_predict",
251
+ "repeat_penalty",
252
+ "reasoning",
253
+ }
254
+ )
255
+ """The `ChatOllama` parameters a live run may set; each changes what the model generates."""
256
+
257
+ ANTHROPIC_PARAMS: Final = frozenset(
258
+ {"max_tokens", "temperature", "top_k", "top_p", "thinking", "reasoning_effort"}
259
+ )
260
+ """The `ChatAnthropic` parameters a live run may set; each changes what the model
261
+ generates."""
262
+
263
+ OPENAI_PARAMS: Final = frozenset(
264
+ {
265
+ "max_tokens",
266
+ "temperature",
267
+ "top_p",
268
+ "seed",
269
+ "presence_penalty",
270
+ "frequency_penalty",
271
+ "reasoning_effort",
272
+ }
273
+ )
274
+ """The `ChatOpenAI` parameters a live run may set; each changes what the model generates."""
275
+
276
+ GOOGLE_GENAI_PARAMS: Final = frozenset(
277
+ {
278
+ "max_output_tokens",
279
+ "temperature",
280
+ "top_k",
281
+ "top_p",
282
+ "seed",
283
+ "presence_penalty",
284
+ "frequency_penalty",
285
+ "thinking_budget",
286
+ "reasoning_effort",
287
+ }
288
+ )
289
+ """The `ChatGoogleGenerativeAI` parameters a live run may set; each changes what the model
290
+ generates."""
291
+
292
+ PROVIDERS: dict[str, Provider] = {
293
+ "ollama": Provider(_ollama, OLLAMA_PARAMS),
294
+ "anthropic": Provider(_anthropic, ANTHROPIC_PARAMS, ("ANTHROPIC_API_KEY",)),
295
+ "openai": Provider(_openai, OPENAI_PARAMS, ("OPENAI_API_KEY",)),
296
+ "google_genai": Provider(
297
+ _google_genai,
298
+ GOOGLE_GENAI_PARAMS,
299
+ ("GOOGLE_API_KEY", "GEMINI_API_KEY"),
300
+ ("GOOGLE_GENAI_USE_VERTEXAI", "GOOGLE_GENAI_USE_ENTERPRISE"),
301
+ ),
302
+ }
303
+ """The live providers, by name."""
304
+
305
+
306
+ def live_name(provider: str, model: str) -> str:
307
+ """The name of a live model, as `harness.run` and its cassette record it: the provider
308
+ and the model, such as `ollama:llama3.1:8b`."""
309
+ return f"{provider}:{model}"
310
+
311
+
312
+ def parse_live_name(name: str) -> tuple[str, str]:
313
+ """The provider and the model of a live model's name: the text before the first colon,
314
+ and the rest."""
315
+ provider, _, model = name.partition(":")
316
+ if not provider or not model:
317
+ raise LiveError(f"a live model is named PROVIDER:MODEL, such as ollama:MODEL, not {name!r}")
318
+ return provider, model
319
+
320
+
321
+ def check_provider(provider: str, params: Mapping[str, JsonValue]) -> Provider:
322
+ """`provider`'s entry, if it exists and accepts every parameter in `params`."""
323
+ entry = PROVIDERS.get(provider)
324
+ if entry is None:
325
+ raise LiveError(
326
+ f"no live provider {provider!r}; this build has {', '.join(sorted(PROVIDERS))}"
327
+ )
328
+ unknown = sorted(set(params) - entry.params)
329
+ if unknown:
330
+ raise LiveError(
331
+ f"{provider} accepts the parameters {', '.join(sorted(entry.params))},"
332
+ f" not {', '.join(unknown)}"
333
+ )
334
+ return entry
335
+
336
+
337
+ def provider_model(provider: str, model: str, params: Mapping[str, JsonValue]) -> BaseChatModel:
338
+ """`provider`'s chat model for `model`, with `params`. Building it sends nothing.
339
+
340
+ A value the provider's model refuses raises `LiveError`, with the first problem in
341
+ one line.
342
+ """
343
+ entry = check_provider(provider, params)
344
+ try:
345
+ return entry.build(model, params)
346
+ except ValidationError as error:
347
+ first = error.errors(include_url=False)[0]
348
+ where = ".".join(str(part) for part in first["loc"]) or "a parameter"
349
+ raise LiveError(f"{provider} refuses {where}: {first['msg']}") from None
350
+
351
+
352
+ # The spend cap -----------------------------------------------------------------------------
353
+
354
+
355
+ class SpendMeter:
356
+ """The spend cap of one command's live runs, and what their model calls spent (0004).
357
+
358
+ One meter serves every run of a command, so the cap holds across them. Each call is
359
+ charged at its model's price in the price file, from its reply's token usage.
360
+ """
361
+
362
+ def __init__(self, cap_usd: Decimal, prices: PriceFile) -> None:
363
+ if not cap_usd.is_finite() or cap_usd <= 0:
364
+ raise LiveError(
365
+ f"the spend cap must be an amount above 0 USD, not {cap_usd}:"
366
+ " a cap of 0 would refuse every call"
367
+ )
368
+ self.cap_usd = cap_usd
369
+ self.prices = prices
370
+ self.spent_usd = Decimal(0)
371
+ """What the calls charged so far cost, in USD, exactly."""
372
+ self.calls = 0
373
+ """The calls charged so far."""
374
+ self.unknown: str | None = None
375
+ """Why the spend is no longer known, if it is not."""
376
+
377
+ def reached(self) -> bool:
378
+ """Whether the spend so far reached the cap, or is no longer known."""
379
+ return self.unknown is not None or self.spent_usd >= self.cap_usd
380
+
381
+ def charge(self, model: str, price: ModelPrice, usage: UsageMetadata | None) -> None:
382
+ """Charge one call of `model` at `price`, from its reply's usage."""
383
+ self.calls += 1
384
+ if usage is not None:
385
+ self.spent_usd += price.cost_usd(usage["input_tokens"], usage["output_tokens"])
386
+ elif not price.free:
387
+ self.unknown = f"a reply of {model} reported no token usage, so its cost is unknown"
388
+
389
+ def summary(self) -> str:
390
+ """The spend so far against the cap, and the price file's date, in one line."""
391
+ line = (
392
+ f"{self.spent_usd} USD spent of the {self.cap_usd} USD cap on {self.calls} model"
393
+ f" calls, at prices dated {self.prices.price_date}"
394
+ )
395
+ return line if self.unknown is None else f"{line}; {self.unknown}"
396
+
397
+
398
+ def spend_cap_reply() -> AIMessage:
399
+ """The reply to a call the spend cap refused. It ends the worker as `budget`."""
400
+ return AIMessage(content=SPEND_CAP_TEXT, response_metadata={REFUSED: SPEND_CAP})
401
+
402
+
403
+ def refusal(reply: AIMessage) -> str | None:
404
+ """Why the harness refused the call that `reply` answers, or None if the model answered."""
405
+ reason = reply.response_metadata.get(REFUSED)
406
+ return reason if isinstance(reason, str) else None
407
+
408
+
409
+ # The adapter --------------------------------------------------------------------------------
410
+
411
+
412
+ class LiveModel(BaseChatModel):
413
+ """A provider's chat model behind the spend cap, on the scheduler (0009)."""
414
+
415
+ _chat: BaseChatModel = PrivateAttr()
416
+ _bound: Runnable[LanguageModelInput, AIMessage] = PrivateAttr()
417
+ _name: str = PrivateAttr()
418
+ _price: ModelPrice = PrivateAttr()
419
+ _meter: SpendMeter = PrivateAttr()
420
+ _think: Think = PrivateAttr()
421
+ _hold: Hold = PrivateAttr()
422
+ _limit_s: float = PrivateAttr()
423
+
424
+ def __init__(
425
+ self,
426
+ chat: BaseChatModel,
427
+ *,
428
+ name: str,
429
+ price: ModelPrice,
430
+ meter: SpendMeter,
431
+ think: Think,
432
+ hold: Hold,
433
+ limit_s: float = CALL_LIMIT_S,
434
+ ) -> None:
435
+ """Answer with `chat`, the provider's model named `name`, charged at `price`. A
436
+ call that waits more than `limit_s` wall-clock seconds for the provider fails."""
437
+ super().__init__()
438
+ self._chat = chat
439
+ self._bound = chat
440
+ self._name = name
441
+ self._price = price
442
+ self._meter = meter
443
+ self._think = think
444
+ self._hold = hold
445
+ self._limit_s = limit_s
446
+
447
+ @property
448
+ def _llm_type(self) -> str:
449
+ return "fivefold-live"
450
+
451
+ def _generate(
452
+ self,
453
+ messages: list[BaseMessage],
454
+ stop: list[str] | None = None,
455
+ run_manager: Any = None,
456
+ **kwargs: Any,
457
+ ) -> ChatResult:
458
+ raise NotImplementedError("the live model runs on the scheduler's loop only")
459
+
460
+ async def _agenerate(
461
+ self,
462
+ messages: list[BaseMessage],
463
+ stop: list[str] | None = None,
464
+ run_manager: Any = None,
465
+ **kwargs: Any,
466
+ ) -> ChatResult:
467
+ await self._think()
468
+ if self._meter.reached():
469
+ return ChatResult(generations=[ChatGeneration(message=spend_cap_reply())])
470
+ call = self._bound.ainvoke(messages, stop=stop, **kwargs)
471
+ try:
472
+ reply = await self._hold(within(call, self._limit_s))
473
+ except TimeoutError:
474
+ raise LiveError(
475
+ f"the live model {self._name} did not answer within {self._limit_s:g} s"
476
+ ) from None
477
+ self._meter.charge(self._name, self._price, reply.usage_metadata)
478
+ return ChatResult(generations=[ChatGeneration(message=reply)])
479
+
480
+ def bind_tools(self, tools: Sequence[Tool], **kwargs: Any) -> LiveModel:
481
+ """Bind `tools` to the provider's model, which sends their schemas with each call."""
482
+ bound = LiveModel(
483
+ self._chat,
484
+ name=self._name,
485
+ price=self._price,
486
+ meter=self._meter,
487
+ think=self._think,
488
+ hold=self._hold,
489
+ limit_s=self._limit_s,
490
+ )
491
+ bound._bound = self._chat.bind_tools(tools, **kwargs)
492
+ return bound