groundline-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- groundline_core/__init__.py +78 -0
- groundline_core/_clients/__init__.py +5 -0
- groundline_core/_clients/anthropic.py +226 -0
- groundline_core/_clients/retry.py +74 -0
- groundline_core/_core/__init__.py +125 -0
- groundline_core/_core/cache.py +99 -0
- groundline_core/_core/calls.py +68 -0
- groundline_core/_core/deadline.py +40 -0
- groundline_core/_core/errors.py +130 -0
- groundline_core/_core/file_problems.py +45 -0
- groundline_core/_core/implementation.py +191 -0
- groundline_core/_core/inputs.py +25 -0
- groundline_core/_core/model_client.py +42 -0
- groundline_core/_core/parsing.py +109 -0
- groundline_core/_core/profile.py +134 -0
- groundline_core/_core/prompt_stage.py +56 -0
- groundline_core/_core/registry.py +51 -0
- groundline_core/_core/runner.py +216 -0
- groundline_core/_core/spec.py +289 -0
- groundline_core/_core/templates.py +17 -0
- groundline_core/_disclaimer.py +44 -0
- groundline_core/_evaluate.py +240 -0
- groundline_core/_evaluators/__init__.py +13 -0
- groundline_core/_evaluators/alarm.py +70 -0
- groundline_core/_evaluators/builtin.py +13 -0
- groundline_core/_evaluators/commissions/__init__.py +51 -0
- groundline_core/_evaluators/commissions/models.py +81 -0
- groundline_core/_evaluators/commissions/report.py +66 -0
- groundline_core/_evaluators/commissions/stages/__init__.py +25 -0
- groundline_core/_evaluators/commissions/stages/alarm.py +42 -0
- groundline_core/_evaluators/commissions/stages/check.py +106 -0
- groundline_core/_evaluators/commissions/stages/claims.py +60 -0
- groundline_core/_evaluators/commissions/stages/refute.py +178 -0
- groundline_core/_evaluators/commissions/stages/support.py +115 -0
- groundline_core/_evaluators/models.py +46 -0
- groundline_core/_evaluators/omissions/__init__.py +52 -0
- groundline_core/_evaluators/omissions/models.py +105 -0
- groundline_core/_evaluators/omissions/report.py +66 -0
- groundline_core/_evaluators/omissions/stages/__init__.py +17 -0
- groundline_core/_evaluators/omissions/stages/alarm.py +51 -0
- groundline_core/_evaluators/omissions/stages/check.py +93 -0
- groundline_core/_evaluators/omissions/stages/extract.py +57 -0
- groundline_core/_evaluators/omissions/stages/rank.py +104 -0
- groundline_core/_evaluators/omissions/stages/refute.py +127 -0
- groundline_core/_evaluators/omissions/stages/score.py +86 -0
- groundline_core/_evaluators/report.py +103 -0
- groundline_core/_findings.py +147 -0
- groundline_core/_result.py +159 -0
- groundline_core/_settings.py +257 -0
- groundline_core/api/__init__.py +1 -0
- groundline_core/api/commissions.py +172 -0
- groundline_core/api/common.py +375 -0
- groundline_core/api/errors.py +102 -0
- groundline_core/api/limits.py +83 -0
- groundline_core/api/omissions.py +151 -0
- groundline_core/api_keys.py +345 -0
- groundline_core/app.py +254 -0
- groundline_core/auth.py +56 -0
- groundline_core/cli.py +365 -0
- groundline_core/command.py +32 -0
- groundline_core/db/__init__.py +9 -0
- groundline_core/db/config.py +4 -0
- groundline_core/db/connection.py +180 -0
- groundline_core/db/migrations/env.py +30 -0
- groundline_core/db/migrations/script.py.mako +22 -0
- groundline_core/db/migrations/versions/0001_create_api_keys.py +57 -0
- groundline_core/db/setup.py +198 -0
- groundline_core/db/tables.py +39 -0
- groundline_core/extras.py +31 -0
- groundline_core/keys_cli.py +176 -0
- groundline_core/presets/__init__.py +32 -0
- groundline_core/presets/commission_quick.yaml +42 -0
- groundline_core/presets/commission_reverse_lean.yaml +54 -0
- groundline_core/presets/omission_all_opus.yaml +50 -0
- groundline_core/presets/pipeline.schema.json +824 -0
- groundline_core/presets/schema.py +33 -0
- groundline_core/profiles/__init__.py +31 -0
- groundline_core/profiles/clinical/profile.yaml +30 -0
- groundline_core/profiles/clinical/prompts/check_batch_v1.txt +23 -0
- groundline_core/profiles/clinical/prompts/extract_coarse_v1.txt +30 -0
- groundline_core/profiles/clinical/prompts/note_claims_v1.txt +12 -0
- groundline_core/profiles/clinical/prompts/quick_check_v1.txt +27 -0
- groundline_core/profiles/clinical/prompts/rank_v1.txt +20 -0
- groundline_core/profiles/clinical/prompts/refute_v1.txt +21 -0
- groundline_core/profiles/clinical/prompts/support_check_lean_v1.txt +26 -0
- groundline_core/profiles/clinical/prompts/support_refute_v1.txt +14 -0
- groundline_core/profiles/clinical/rubrics/severity_v1.md +34 -0
- groundline_core/profiles/profile.schema.json +65 -0
- groundline_core/profiles/schema.py +31 -0
- groundline_core/py.typed +0 -0
- groundline_core-0.1.0.dist-info/METADATA +498 -0
- groundline_core-0.1.0.dist-info/RECORD +96 -0
- groundline_core-0.1.0.dist-info/WHEEL +4 -0
- groundline_core-0.1.0.dist-info/entry_points.txt +3 -0
- groundline_core-0.1.0.dist-info/licenses/LICENSE +202 -0
- groundline_core-0.1.0.dist-info/licenses/NOTICE +7 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""groundline: checks AI-written summaries against their source.
|
|
2
|
+
|
|
3
|
+
Start with `evaluate` (or `evaluate_sync`), which runs the validated all-Opus
|
|
4
|
+
omission check by default. Pass `spec=load_preset(DEFAULT_COMMISSIONS_PRESET)` to
|
|
5
|
+
run the commission check instead. The result says whether the summary is flagged,
|
|
6
|
+
its score from 0 to 1, what the check found, and what it cost.
|
|
7
|
+
|
|
8
|
+
When a check can't finish, it raises a `GroundlineError`: a `SetupError` to fix
|
|
9
|
+
before trying again, an `InvalidInputError` for a source or summary that can't
|
|
10
|
+
be checked, or a `ModelFailureError` (a `ModelTimeoutError` when time ran out)
|
|
11
|
+
that a later try may get past. Exceptions from your own code pass through as
|
|
12
|
+
they were raised.
|
|
13
|
+
|
|
14
|
+
The names listed in `__all__`, and the documented fields of the result types,
|
|
15
|
+
are the public interface: they keep working across upgrades. Everything else,
|
|
16
|
+
including the modules whose names start with an underscore and
|
|
17
|
+
`EvaluationResult.run`, is internal and may change without notice.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from groundline_core._core import ModelClient, ModelReply, PipelineSpec
|
|
21
|
+
from groundline_core._core.errors import (
|
|
22
|
+
APIKeyRejectedError,
|
|
23
|
+
GroundlineError,
|
|
24
|
+
InvalidInputError,
|
|
25
|
+
ModelFailureError,
|
|
26
|
+
ModelTimeoutError,
|
|
27
|
+
SetupError,
|
|
28
|
+
)
|
|
29
|
+
from groundline_core._core.profile import ProfileError
|
|
30
|
+
from groundline_core._core.spec import SpecError
|
|
31
|
+
from groundline_core._evaluate import (
|
|
32
|
+
DEFAULT_COMMISSIONS_PRESET,
|
|
33
|
+
DEFAULT_PRESET,
|
|
34
|
+
CommissionPreset,
|
|
35
|
+
evaluate,
|
|
36
|
+
evaluate_sync,
|
|
37
|
+
)
|
|
38
|
+
from groundline_core._evaluators.models import Severity
|
|
39
|
+
from groundline_core._result import (
|
|
40
|
+
Alarm,
|
|
41
|
+
Counts,
|
|
42
|
+
EvaluationResult,
|
|
43
|
+
EvaluationType,
|
|
44
|
+
Finding,
|
|
45
|
+
Usage,
|
|
46
|
+
)
|
|
47
|
+
from groundline_core._settings import MissingAPIKeyError, SettingsError
|
|
48
|
+
from groundline_core.presets import builtin_presets, load_preset
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
"APIKeyRejectedError",
|
|
52
|
+
"GroundlineError",
|
|
53
|
+
"InvalidInputError",
|
|
54
|
+
"MissingAPIKeyError",
|
|
55
|
+
"ModelFailureError",
|
|
56
|
+
"ModelTimeoutError",
|
|
57
|
+
"ProfileError",
|
|
58
|
+
"SettingsError",
|
|
59
|
+
"SetupError",
|
|
60
|
+
"SpecError",
|
|
61
|
+
"DEFAULT_COMMISSIONS_PRESET",
|
|
62
|
+
"DEFAULT_PRESET",
|
|
63
|
+
"Alarm",
|
|
64
|
+
"CommissionPreset",
|
|
65
|
+
"Counts",
|
|
66
|
+
"EvaluationResult",
|
|
67
|
+
"EvaluationType",
|
|
68
|
+
"Finding",
|
|
69
|
+
"ModelClient",
|
|
70
|
+
"ModelReply",
|
|
71
|
+
"PipelineSpec",
|
|
72
|
+
"Severity",
|
|
73
|
+
"Usage",
|
|
74
|
+
"builtin_presets",
|
|
75
|
+
"evaluate",
|
|
76
|
+
"evaluate_sync",
|
|
77
|
+
"load_preset",
|
|
78
|
+
]
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""The Anthropic Messages API client.
|
|
2
|
+
|
|
3
|
+
Notes for Claude Opus 5.5:
|
|
4
|
+
- Thinking is always on, so `thinking` isn't sent. Effort is the only control,
|
|
5
|
+
and Opus 5.5 defaults to `medium`, so it's always sent explicitly.
|
|
6
|
+
- The refusal-fallback beta is deliberately not used: it isn't on Anthropic's
|
|
7
|
+
HIPAA feature-eligibility list, so it isn't covered by the BAA. A refusal is
|
|
8
|
+
raised as an error instead.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import logging
|
|
13
|
+
import math
|
|
14
|
+
import time
|
|
15
|
+
from email.utils import parsedate_to_datetime
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
import anthropic
|
|
19
|
+
import httpx2
|
|
20
|
+
from anthropic import AsyncAnthropic
|
|
21
|
+
|
|
22
|
+
from groundline_core._clients.retry import retry_call
|
|
23
|
+
from groundline_core._core import (
|
|
24
|
+
APIKeyRejectedError,
|
|
25
|
+
Effort,
|
|
26
|
+
ModelFailureError,
|
|
27
|
+
ModelRefusal,
|
|
28
|
+
ModelReply,
|
|
29
|
+
ModelTimeoutError,
|
|
30
|
+
SetupError,
|
|
31
|
+
UnusableReplyError,
|
|
32
|
+
)
|
|
33
|
+
from groundline_core._core.deadline import raise_if_cancelling
|
|
34
|
+
from groundline_core._settings import EnvFile, read_model_settings
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger("groundline.model")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _seconds(value: str | None, scale: float = 1) -> float | None:
|
|
40
|
+
try:
|
|
41
|
+
number = float(value) / scale if value is not None else float("nan")
|
|
42
|
+
except ValueError:
|
|
43
|
+
return None
|
|
44
|
+
return number if math.isfinite(number) and number >= 0 else None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def retry_decision(
|
|
48
|
+
error: Exception, *, now: float | None = None
|
|
49
|
+
) -> tuple[bool, float | None]:
|
|
50
|
+
if isinstance(error, (ModelTimeoutError, anthropic.APIConnectionError)):
|
|
51
|
+
return True, None
|
|
52
|
+
if not isinstance(error, anthropic.APIStatusError):
|
|
53
|
+
return False, None
|
|
54
|
+
headers = error.response.headers
|
|
55
|
+
transient = error.status_code in (408, 409, 429) or 500 <= error.status_code < 600
|
|
56
|
+
if not transient or headers.get("x-should-retry", "").strip().lower() == "false":
|
|
57
|
+
return False, None
|
|
58
|
+
minimum = _seconds(headers.get("retry-after-ms"), 1000)
|
|
59
|
+
if minimum is None:
|
|
60
|
+
value = headers.get("retry-after")
|
|
61
|
+
minimum = _seconds(value)
|
|
62
|
+
if minimum is None and value:
|
|
63
|
+
try:
|
|
64
|
+
minimum = max(
|
|
65
|
+
0.0,
|
|
66
|
+
parsedate_to_datetime(value).timestamp()
|
|
67
|
+
- (time.time() if now is None else now),
|
|
68
|
+
)
|
|
69
|
+
except (ValueError, TypeError, OverflowError):
|
|
70
|
+
pass
|
|
71
|
+
return True, minimum
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _sdk_failure_text(error: anthropic.APIError) -> str:
|
|
75
|
+
"""What the Anthropic API did, named by status and error type. The SDK's own
|
|
76
|
+
message is never quoted: it can repeat parts of the request."""
|
|
77
|
+
if isinstance(error, anthropic.APIStatusError):
|
|
78
|
+
what = f"answered {error.status_code}"
|
|
79
|
+
else:
|
|
80
|
+
what = "failed"
|
|
81
|
+
return f"the Anthropic API {what} ({type(error).__name__})"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class AnthropicClient:
|
|
85
|
+
def __init__(
|
|
86
|
+
self,
|
|
87
|
+
api_key: str,
|
|
88
|
+
*,
|
|
89
|
+
key_path: Path | None = None,
|
|
90
|
+
key_name: str = "ANTHROPIC_API_KEY",
|
|
91
|
+
env_file: EnvFile = None,
|
|
92
|
+
max_retries: int | None = None,
|
|
93
|
+
http_client: httpx2.AsyncClient | None = None,
|
|
94
|
+
) -> None:
|
|
95
|
+
"""Read model policy only; `api_key` is supplied by the caller.
|
|
96
|
+
|
|
97
|
+
`key_path` is the `.env` file the key came from, if any, for a rejected
|
|
98
|
+
key's error to point at. `max_retries=N` allows N+1
|
|
99
|
+
total groundline attempts, bounded by model and enclosing check limits.
|
|
100
|
+
SDK retries are disabled so all attempts and waits share those limits.
|
|
101
|
+
"""
|
|
102
|
+
self._key_path = key_path
|
|
103
|
+
self._key_name = key_name
|
|
104
|
+
self._settings = read_model_settings(env_file)
|
|
105
|
+
self._max_attempts = (
|
|
106
|
+
(max_retries + 1)
|
|
107
|
+
if max_retries is not None
|
|
108
|
+
else self._settings.model_max_attempts
|
|
109
|
+
)
|
|
110
|
+
# One outage retry layer belongs to groundline.
|
|
111
|
+
self._sdk = AsyncAnthropic(
|
|
112
|
+
api_key=api_key, max_retries=0, http_client=http_client
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def _rejected(self, what: str, fix: str) -> APIKeyRejectedError:
|
|
116
|
+
"""The error for a key that was refused. The file the key came from is in
|
|
117
|
+
the error's `path`, never in its message."""
|
|
118
|
+
return APIKeyRejectedError(f"{what}\n Fix: {fix}", path=self._key_path)
|
|
119
|
+
|
|
120
|
+
async def complete(
|
|
121
|
+
self, prompt: str, *, model: str, effort: Effort, max_tokens: int
|
|
122
|
+
) -> ModelReply:
|
|
123
|
+
settings = self._settings
|
|
124
|
+
attempt = 0
|
|
125
|
+
|
|
126
|
+
async def operation():
|
|
127
|
+
nonlocal attempt
|
|
128
|
+
attempt += 1
|
|
129
|
+
started = time.monotonic()
|
|
130
|
+
boundary = asyncio.timeout(settings.model_timeout_seconds)
|
|
131
|
+
try:
|
|
132
|
+
async with boundary:
|
|
133
|
+
return await self._sdk.messages.create(
|
|
134
|
+
model=model,
|
|
135
|
+
max_tokens=max_tokens,
|
|
136
|
+
output_config={"effort": effort},
|
|
137
|
+
messages=[{"role": "user", "content": prompt}],
|
|
138
|
+
timeout=httpx2.Timeout(
|
|
139
|
+
settings.model_timeout_seconds,
|
|
140
|
+
connect=settings.model_connect_timeout_seconds,
|
|
141
|
+
),
|
|
142
|
+
)
|
|
143
|
+
except anthropic.APITimeoutError:
|
|
144
|
+
raise ModelTimeoutError("attempt") from None
|
|
145
|
+
except Exception:
|
|
146
|
+
if boundary.expired():
|
|
147
|
+
raise ModelTimeoutError("attempt") from None
|
|
148
|
+
raise_if_cancelling()
|
|
149
|
+
raise
|
|
150
|
+
finally:
|
|
151
|
+
logger.info(
|
|
152
|
+
"model attempt=%d duration=%.3f",
|
|
153
|
+
attempt,
|
|
154
|
+
time.monotonic() - started,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
def decide(error: Exception) -> tuple[bool, float | None]:
|
|
158
|
+
logger.info(
|
|
159
|
+
"model attempt=%d status=%s",
|
|
160
|
+
attempt,
|
|
161
|
+
error.status_code
|
|
162
|
+
if isinstance(error, anthropic.APIStatusError)
|
|
163
|
+
else type(error).__name__,
|
|
164
|
+
)
|
|
165
|
+
return retry_decision(error)
|
|
166
|
+
|
|
167
|
+
try:
|
|
168
|
+
message = await retry_call(
|
|
169
|
+
operation,
|
|
170
|
+
decide=decide,
|
|
171
|
+
max_attempts=self._max_attempts,
|
|
172
|
+
budget_seconds=settings.model_retry_timeout_seconds,
|
|
173
|
+
base_seconds=settings.model_retry_base_seconds,
|
|
174
|
+
cap_seconds=settings.model_retry_cap_seconds,
|
|
175
|
+
)
|
|
176
|
+
except anthropic.AuthenticationError as error:
|
|
177
|
+
raise self._rejected(
|
|
178
|
+
f"{self._key_name} was rejected by the Anthropic API "
|
|
179
|
+
f"({error.status_code}).",
|
|
180
|
+
"check the key is complete, hasn't been revoked, "
|
|
181
|
+
"and belongs to the workspace you expect.",
|
|
182
|
+
) from None
|
|
183
|
+
except anthropic.PermissionDeniedError as error:
|
|
184
|
+
raise self._rejected(
|
|
185
|
+
f"{self._key_name} doesn't have access to {model} "
|
|
186
|
+
f"({error.status_code}).",
|
|
187
|
+
"use a key from a workspace that can use this model, "
|
|
188
|
+
"or change the stage's model setting.",
|
|
189
|
+
) from None
|
|
190
|
+
except anthropic.NotFoundError as error:
|
|
191
|
+
# The model named in the spec doesn't exist, or has been retired:
|
|
192
|
+
# trying again won't help until the spec changes.
|
|
193
|
+
raise SetupError(
|
|
194
|
+
f"the Anthropic API has no model {model} ({error.status_code}).\n"
|
|
195
|
+
" Fix: change the stage's model setting in the pipeline spec."
|
|
196
|
+
) from None
|
|
197
|
+
except anthropic.APIError as error:
|
|
198
|
+
raise ModelFailureError(_sdk_failure_text(error)) from None
|
|
199
|
+
text = "".join(block.text for block in message.content if block.type == "text")
|
|
200
|
+
reply = ModelReply(
|
|
201
|
+
text=text,
|
|
202
|
+
model=message.model,
|
|
203
|
+
input_tokens=message.usage.input_tokens,
|
|
204
|
+
output_tokens=message.usage.output_tokens,
|
|
205
|
+
)
|
|
206
|
+
# A refusal or a cut-off reply is still billed, so the errors carry the
|
|
207
|
+
# reply's usage for the cost report.
|
|
208
|
+
if message.stop_reason == "refusal":
|
|
209
|
+
details = message.stop_details
|
|
210
|
+
category = (
|
|
211
|
+
details.category if details and details.category else "none given"
|
|
212
|
+
)
|
|
213
|
+
raise ModelRefusal(
|
|
214
|
+
f"{message.model} declined to answer (refusal category: {category})",
|
|
215
|
+
reply=reply,
|
|
216
|
+
)
|
|
217
|
+
if message.stop_reason == "max_tokens":
|
|
218
|
+
raise UnusableReplyError(
|
|
219
|
+
f"reply cut off at max_tokens={max_tokens}; "
|
|
220
|
+
"raise the stage's max_tokens",
|
|
221
|
+
reply=reply,
|
|
222
|
+
)
|
|
223
|
+
return reply
|
|
224
|
+
|
|
225
|
+
async def close(self) -> None:
|
|
226
|
+
await self._sdk.close()
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Provider-neutral outage scheduling with cancellable elapsed budgets."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import random as randomness
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Awaitable, Callable
|
|
8
|
+
|
|
9
|
+
from groundline_core._core.deadline import raise_if_cancelling
|
|
10
|
+
from groundline_core._core.deadline import remaining as check_remaining
|
|
11
|
+
from groundline_core._core.errors import ModelTimeoutError
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger("groundline.retry")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
async def retry_call[T](
|
|
17
|
+
operation: Callable[[], Awaitable[T]],
|
|
18
|
+
*,
|
|
19
|
+
decide: Callable[[Exception], tuple[bool, float | None]],
|
|
20
|
+
max_attempts: int,
|
|
21
|
+
budget_seconds: float,
|
|
22
|
+
base_seconds: float,
|
|
23
|
+
cap_seconds: float,
|
|
24
|
+
remaining: Callable[[], float] = check_remaining,
|
|
25
|
+
clock: Callable[[], float] = time.monotonic,
|
|
26
|
+
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
|
27
|
+
random: Callable[[], float] = randomness.random,
|
|
28
|
+
) -> T:
|
|
29
|
+
"""Retry transient failures within the attempt and elapsed-time limits.
|
|
30
|
+
|
|
31
|
+
If backoff or a provider's minimum wait cannot fit the remaining allowance,
|
|
32
|
+
stop immediately with the original error, without sleeping or another call.
|
|
33
|
+
Actual allowance expiry raises ModelTimeoutError; cancellation propagates.
|
|
34
|
+
"""
|
|
35
|
+
end = clock() + budget_seconds
|
|
36
|
+
|
|
37
|
+
def allowance() -> float:
|
|
38
|
+
return min(end - clock(), remaining())
|
|
39
|
+
|
|
40
|
+
delay_cap = min(base_seconds, cap_seconds)
|
|
41
|
+
for attempt in range(max_attempts):
|
|
42
|
+
if allowance() <= 0:
|
|
43
|
+
raise ModelTimeoutError("retry")
|
|
44
|
+
boundary = asyncio.timeout(allowance())
|
|
45
|
+
try:
|
|
46
|
+
async with boundary:
|
|
47
|
+
return await operation()
|
|
48
|
+
except Exception as error:
|
|
49
|
+
if boundary.expired():
|
|
50
|
+
raise ModelTimeoutError("retry") from None
|
|
51
|
+
raise_if_cancelling()
|
|
52
|
+
retryable, minimum = decide(error)
|
|
53
|
+
if not retryable or attempt + 1 >= max_attempts:
|
|
54
|
+
raise
|
|
55
|
+
if allowance() <= 0:
|
|
56
|
+
raise ModelTimeoutError("retry") from None
|
|
57
|
+
wait = max(random() * delay_cap, minimum or 0)
|
|
58
|
+
if wait >= allowance():
|
|
59
|
+
# Neither backoff nor provider guidance fits another attempt.
|
|
60
|
+
# Preserve the upstream error without waiting out the budget.
|
|
61
|
+
raise
|
|
62
|
+
logger.info(
|
|
63
|
+
"outage retry after attempt=%d wait_seconds=%.3f", attempt + 1, wait
|
|
64
|
+
)
|
|
65
|
+
wait_boundary = asyncio.timeout(allowance())
|
|
66
|
+
try:
|
|
67
|
+
async with wait_boundary:
|
|
68
|
+
await sleep(wait)
|
|
69
|
+
except Exception:
|
|
70
|
+
if wait_boundary.expired():
|
|
71
|
+
raise ModelTimeoutError("retry") from None
|
|
72
|
+
raise
|
|
73
|
+
delay_cap = min(cap_seconds, delay_cap * 2)
|
|
74
|
+
raise AssertionError("positive attempt limit required")
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""The pipeline framework: steps, implementations, specs and the runner."""
|
|
2
|
+
|
|
3
|
+
from groundline_core._core.cache import Cache, FileCache, stable_hash
|
|
4
|
+
from groundline_core._core.calls import (
|
|
5
|
+
PRICES,
|
|
6
|
+
CallRecord,
|
|
7
|
+
Price,
|
|
8
|
+
Usage,
|
|
9
|
+
summarize_usage,
|
|
10
|
+
)
|
|
11
|
+
from groundline_core._core.errors import (
|
|
12
|
+
APIKeyRejectedError,
|
|
13
|
+
GroundlineError,
|
|
14
|
+
InvalidInputError,
|
|
15
|
+
ModelFailureError,
|
|
16
|
+
ModelTimeoutError,
|
|
17
|
+
SetupError,
|
|
18
|
+
file_line,
|
|
19
|
+
redact_keys,
|
|
20
|
+
)
|
|
21
|
+
from groundline_core._core.implementation import Implementation, StageContext
|
|
22
|
+
from groundline_core._core.inputs import InputName, Inputs
|
|
23
|
+
from groundline_core._core.model_client import (
|
|
24
|
+
Effort,
|
|
25
|
+
ModelClient,
|
|
26
|
+
ModelRefusal,
|
|
27
|
+
ModelReply,
|
|
28
|
+
UnusableReplyError,
|
|
29
|
+
)
|
|
30
|
+
from groundline_core._core.parsing import (
|
|
31
|
+
ParseError,
|
|
32
|
+
as_object,
|
|
33
|
+
choice,
|
|
34
|
+
evidence_in,
|
|
35
|
+
get_list,
|
|
36
|
+
parse_json_object,
|
|
37
|
+
percent,
|
|
38
|
+
reply_list,
|
|
39
|
+
stripped,
|
|
40
|
+
words,
|
|
41
|
+
)
|
|
42
|
+
from groundline_core._core.profile import (
|
|
43
|
+
DomainProfile,
|
|
44
|
+
Placeholders,
|
|
45
|
+
ProfileError,
|
|
46
|
+
load_profile,
|
|
47
|
+
)
|
|
48
|
+
from groundline_core._core.prompt_stage import (
|
|
49
|
+
RETRY_NOTE,
|
|
50
|
+
PromptImplementation,
|
|
51
|
+
ReplyError,
|
|
52
|
+
)
|
|
53
|
+
from groundline_core._core.registry import Registry
|
|
54
|
+
from groundline_core._core.runner import RunResult, StageRun, run_pipeline
|
|
55
|
+
from groundline_core._core.spec import (
|
|
56
|
+
Pipeline,
|
|
57
|
+
PipelineSpec,
|
|
58
|
+
PipelineStage,
|
|
59
|
+
SpecError,
|
|
60
|
+
StageSpec,
|
|
61
|
+
build_pipeline,
|
|
62
|
+
load_spec,
|
|
63
|
+
pipeline_fingerprint,
|
|
64
|
+
with_settings,
|
|
65
|
+
)
|
|
66
|
+
from groundline_core._core.templates import fill
|
|
67
|
+
|
|
68
|
+
__all__ = [
|
|
69
|
+
"PRICES",
|
|
70
|
+
"RETRY_NOTE",
|
|
71
|
+
"APIKeyRejectedError",
|
|
72
|
+
"Cache",
|
|
73
|
+
"CallRecord",
|
|
74
|
+
"DomainProfile",
|
|
75
|
+
"Effort",
|
|
76
|
+
"FileCache",
|
|
77
|
+
"Implementation",
|
|
78
|
+
"InputName",
|
|
79
|
+
"Inputs",
|
|
80
|
+
"GroundlineError",
|
|
81
|
+
"InvalidInputError",
|
|
82
|
+
"ModelClient",
|
|
83
|
+
"ModelFailureError",
|
|
84
|
+
"ModelRefusal",
|
|
85
|
+
"ModelReply",
|
|
86
|
+
"ModelTimeoutError",
|
|
87
|
+
"ParseError",
|
|
88
|
+
"Pipeline",
|
|
89
|
+
"PipelineSpec",
|
|
90
|
+
"PipelineStage",
|
|
91
|
+
"Placeholders",
|
|
92
|
+
"Price",
|
|
93
|
+
"ProfileError",
|
|
94
|
+
"PromptImplementation",
|
|
95
|
+
"Registry",
|
|
96
|
+
"ReplyError",
|
|
97
|
+
"RunResult",
|
|
98
|
+
"SetupError",
|
|
99
|
+
"file_line",
|
|
100
|
+
"redact_keys",
|
|
101
|
+
"SpecError",
|
|
102
|
+
"StageContext",
|
|
103
|
+
"StageRun",
|
|
104
|
+
"StageSpec",
|
|
105
|
+
"UnusableReplyError",
|
|
106
|
+
"Usage",
|
|
107
|
+
"as_object",
|
|
108
|
+
"build_pipeline",
|
|
109
|
+
"choice",
|
|
110
|
+
"evidence_in",
|
|
111
|
+
"fill",
|
|
112
|
+
"get_list",
|
|
113
|
+
"load_profile",
|
|
114
|
+
"load_spec",
|
|
115
|
+
"parse_json_object",
|
|
116
|
+
"percent",
|
|
117
|
+
"pipeline_fingerprint",
|
|
118
|
+
"reply_list",
|
|
119
|
+
"run_pipeline",
|
|
120
|
+
"stable_hash",
|
|
121
|
+
"stripped",
|
|
122
|
+
"summarize_usage",
|
|
123
|
+
"with_settings",
|
|
124
|
+
"words",
|
|
125
|
+
]
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""An optional cache of stage outputs, so identical re-runs make no model calls.
|
|
2
|
+
|
|
3
|
+
The library only caches when it's given a cache. Cached entries hold sources,
|
|
4
|
+
summaries and what was extracted from them, so never cache sensitive data.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import tempfile
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Protocol, cast
|
|
15
|
+
|
|
16
|
+
from groundline_core._core.file_problems import os_reason
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger("groundline.cache")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def stable_hash(value: Any) -> str:
|
|
22
|
+
"""A short hash of any JSON-serializable value, independent of dict order."""
|
|
23
|
+
text = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str)
|
|
24
|
+
return hashlib.sha256(text.encode()).hexdigest()[:24]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Cache(Protocol):
|
|
28
|
+
def get(self, step: str, key: str) -> dict[str, Any] | None: ...
|
|
29
|
+
def put(self, step: str, key: str, entry: dict[str, Any]) -> None: ...
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
SAFE_NAME = re.compile(r"[A-Za-z0-9_-]+")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class FileCache:
|
|
36
|
+
"""Entries stored as `<root>/<step>/<key>.json`.
|
|
37
|
+
|
|
38
|
+
Folders it creates are readable only by their owner (0700), and so is
|
|
39
|
+
every entry (0600), since entries hold sources and summaries."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, root: str | Path) -> None:
|
|
42
|
+
self.root = Path(root)
|
|
43
|
+
self._write_failed = False
|
|
44
|
+
|
|
45
|
+
def _path(self, step: str, key: str) -> Path:
|
|
46
|
+
for name in (step, key):
|
|
47
|
+
if not SAFE_NAME.fullmatch(name):
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"{name!r} is not a safe cache name: use only letters, "
|
|
50
|
+
"digits, '_' and '-'"
|
|
51
|
+
)
|
|
52
|
+
return self.root / step / f"{key}.json"
|
|
53
|
+
|
|
54
|
+
def get(self, step: str, key: str) -> dict[str, Any] | None:
|
|
55
|
+
"""The stored entry, or None if there isn't one or it can't be read.
|
|
56
|
+
The cache is only an optimization, so a damaged entry is a miss, and
|
|
57
|
+
the next `put` replaces it."""
|
|
58
|
+
path = self._path(step, key)
|
|
59
|
+
try:
|
|
60
|
+
entry = json.loads(path.read_text(encoding="utf-8"))
|
|
61
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
62
|
+
return None
|
|
63
|
+
return cast(dict[str, Any], entry) if isinstance(entry, dict) else None
|
|
64
|
+
|
|
65
|
+
def put(self, step: str, key: str, entry: dict[str, Any]) -> None:
|
|
66
|
+
"""Write to a temporary file, then rename it into place, so an
|
|
67
|
+
interrupted write never leaves a half-written entry.
|
|
68
|
+
|
|
69
|
+
The cache is only an optimization, so a write that fails (a full disk,
|
|
70
|
+
a folder that can't be written) leaves the cache as it was, rather
|
|
71
|
+
than failing a check whose model calls are already paid for. The first
|
|
72
|
+
failure is logged as a warning; later ones would only repeat it.
|
|
73
|
+
|
|
74
|
+
It's a log record, not a Python warning, because a warnings filter set
|
|
75
|
+
to "error" would turn a warning into an exception that fails the check."""
|
|
76
|
+
path = self._path(step, key)
|
|
77
|
+
try:
|
|
78
|
+
self._write(path, key, entry)
|
|
79
|
+
except OSError as error:
|
|
80
|
+
if not self._write_failed:
|
|
81
|
+
self._write_failed = True
|
|
82
|
+
logger.warning(
|
|
83
|
+
"couldn't write to the stage cache (%s); "
|
|
84
|
+
"this run's results won't be cached",
|
|
85
|
+
os_reason(error),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def _write(self, path: Path, key: str, entry: dict[str, Any]) -> None:
|
|
89
|
+
self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
90
|
+
path.parent.mkdir(mode=0o700, exist_ok=True)
|
|
91
|
+
# mkstemp creates the file readable and writable by its owner only.
|
|
92
|
+
fd, temp = tempfile.mkstemp(dir=path.parent, prefix=f".{key}.", suffix=".tmp")
|
|
93
|
+
try:
|
|
94
|
+
with os.fdopen(fd, "w", encoding="utf-8") as file:
|
|
95
|
+
json.dump(entry, file, indent=2, ensure_ascii=False)
|
|
96
|
+
os.replace(temp, path)
|
|
97
|
+
except BaseException:
|
|
98
|
+
Path(temp).unlink(missing_ok=True)
|
|
99
|
+
raise
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Records of model calls, and what they cost.
|
|
2
|
+
|
|
3
|
+
Every model call a stage makes is recorded: which model answered, tokens, and
|
|
4
|
+
time. Calls replayed from the cache are marked `cached` and cost nothing.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from collections.abc import Iterable
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Price(BaseModel):
|
|
13
|
+
"""US dollars per million tokens."""
|
|
14
|
+
|
|
15
|
+
input: float
|
|
16
|
+
output: float
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
PRICES: dict[str, Price] = {
|
|
20
|
+
# Anthropic first-party API rates.
|
|
21
|
+
"claude-opus-5-5": Price(input=4.0, output=20.0),
|
|
22
|
+
"claude-sonnet-5-5": Price(input=2.0, output=10.0),
|
|
23
|
+
"claude-haiku-5-5": Price(input=0.1, output=0.5),
|
|
24
|
+
}
|
|
25
|
+
"""Models without an entry are reported as unpriced rather than guessed at."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CallRecord(BaseModel):
|
|
29
|
+
step: str
|
|
30
|
+
model: str
|
|
31
|
+
input_tokens: int
|
|
32
|
+
output_tokens: int
|
|
33
|
+
seconds: float
|
|
34
|
+
cached: bool = False
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Usage(BaseModel):
|
|
38
|
+
calls: int
|
|
39
|
+
cached_calls: int
|
|
40
|
+
input_tokens: int
|
|
41
|
+
"""Live (not cached) calls only, like the other token and cost totals."""
|
|
42
|
+
output_tokens: int
|
|
43
|
+
cost_usd: float
|
|
44
|
+
unpriced_models: list[str]
|
|
45
|
+
"""Models that made live calls but have no price, so aren't in `cost_usd`."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def summarize_usage(
|
|
49
|
+
calls: Iterable[CallRecord], prices: dict[str, Price] = PRICES
|
|
50
|
+
) -> Usage:
|
|
51
|
+
calls = list(calls)
|
|
52
|
+
live = [c for c in calls if not c.cached]
|
|
53
|
+
cost = 0.0
|
|
54
|
+
unpriced: set[str] = set()
|
|
55
|
+
for c in live:
|
|
56
|
+
price = prices.get(c.model)
|
|
57
|
+
if price is None:
|
|
58
|
+
unpriced.add(c.model)
|
|
59
|
+
continue
|
|
60
|
+
cost += (c.input_tokens * price.input + c.output_tokens * price.output) / 1e6
|
|
61
|
+
return Usage(
|
|
62
|
+
calls=len(calls),
|
|
63
|
+
cached_calls=len(calls) - len(live),
|
|
64
|
+
input_tokens=sum(c.input_tokens for c in live),
|
|
65
|
+
output_tokens=sum(c.output_tokens for c in live),
|
|
66
|
+
cost_usd=cost,
|
|
67
|
+
unpriced_models=sorted(unpriced),
|
|
68
|
+
)
|