howhard 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- howhard/__init__.py +9 -0
- howhard/__main__.py +3 -0
- howhard/accounting.py +503 -0
- howhard/cli.py +261 -0
- howhard/config.py +156 -0
- howhard/context.py +199 -0
- howhard/data/context.toml +5 -0
- howhard/data/default_config.toml +75 -0
- howhard/data/semantic.toml +19 -0
- howhard/decisions.py +105 -0
- howhard/engine.py +295 -0
- howhard/files.py +23 -0
- howhard/llm.py +140 -0
- howhard/memory.py +221 -0
- howhard/native.py +115 -0
- howhard/report.py +75 -0
- howhard/retrieval.py +298 -0
- howhard/selector.py +89 -0
- howhard/semantic.py +622 -0
- howhard/state.py +112 -0
- howhard/storage.py +300 -0
- howhard/validation.py +39 -0
- howhard-2.0.0.dist-info/METADATA +201 -0
- howhard-2.0.0.dist-info/RECORD +27 -0
- howhard-2.0.0.dist-info/WHEEL +5 -0
- howhard-2.0.0.dist-info/entry_points.txt +2 -0
- howhard-2.0.0.dist-info/top_level.txt +1 -0
howhard/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""LLM model and effort selection, with a zero-configuration starting point."""
|
|
2
|
+
|
|
3
|
+
__version__ = "2.0.0"
|
|
4
|
+
|
|
5
|
+
from .config import load_config
|
|
6
|
+
from .engine import Engine
|
|
7
|
+
from .retrieval import RetrievalPipeline
|
|
8
|
+
|
|
9
|
+
__all__ = ["Engine", "RetrievalPipeline", "load_config", "__version__"]
|
howhard/__main__.py
ADDED
howhard/accounting.py
ADDED
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
"""Versioned, offline execution usage collection from explicitly owned artifacts."""
|
|
2
|
+
from copy import deepcopy
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import stat
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from .validation import HowHardError, keys, number, text
|
|
11
|
+
|
|
12
|
+
CATEGORIES = ('input', 'output', 'cache_read', 'cache_write', 'cache_write_5m', 'cache_write_1h')
|
|
13
|
+
USAGE_FORMAT = 'howhard-execution-usage-v1'
|
|
14
|
+
RATE_FORMAT = 'howhard-rate-card-v1'
|
|
15
|
+
ADAPTERS = ('codex-exec-v1', 'codex-session-v1', 'claude-sdk-v1', 'claude-session-v1')
|
|
16
|
+
SUPPORTED_HOSTS = {'codex-exec-v1': ('0.166.0',), 'codex-session-v1': ('0.166.0',), 'claude-sdk-v1': ('2.1.318',), 'claude-session-v1': ('2.1.318',)}
|
|
17
|
+
MAX_SOURCE = 32 * 1024 * 1024
|
|
18
|
+
MAX_SPAN = 16 * 1024 * 1024
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def tokens(value, *, forecast=False):
|
|
22
|
+
keys(value, CATEGORIES, where='token_usage')
|
|
23
|
+
for key, count in value.items():
|
|
24
|
+
if count is None and not forecast:
|
|
25
|
+
continue
|
|
26
|
+
if type(count) is int and count > 2**63 - 1:
|
|
27
|
+
raise HowHardError('token_usage: token count exceeds supported integer range')
|
|
28
|
+
number(count, 'token_usage.' + key, nonnegative=True)
|
|
29
|
+
if not forecast and type(count) is not int:
|
|
30
|
+
raise HowHardError('token_usage: measured tokens must be integers or null')
|
|
31
|
+
return value
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def total(value):
|
|
35
|
+
return sum(value.values()) if all(v is not None for v in value.values()) else None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def rate_card(value):
|
|
39
|
+
keys(value, ['format', 'source', 'version', 'effective_at', 'provider', 'model', 'service_tier', 'basis', 'rates'], where='rate_card')
|
|
40
|
+
if value['format'] != RATE_FORMAT or value['basis'] not in ('estimate', 'charge'):
|
|
41
|
+
raise HowHardError('rate_card: unsupported format or basis')
|
|
42
|
+
for key in ('source', 'version', 'provider', 'model'):
|
|
43
|
+
text(value[key], 'rate_card.' + key)
|
|
44
|
+
if value['service_tier'] is not None:
|
|
45
|
+
text(value['service_tier'], 'rate_card.service_tier')
|
|
46
|
+
try:
|
|
47
|
+
if datetime.fromisoformat(value['effective_at']).utcoffset() is None:
|
|
48
|
+
raise ValueError
|
|
49
|
+
except (ValueError, TypeError) as exc:
|
|
50
|
+
raise HowHardError('rate_card.effective_at: expected timezone-aware ISO timestamp') from exc
|
|
51
|
+
keys(value['rates'], CATEGORIES, where='rate_card.rates')
|
|
52
|
+
for key, amount in value['rates'].items():
|
|
53
|
+
if amount is not None:
|
|
54
|
+
number(amount, 'rate_card.rates.' + key, nonnegative=True)
|
|
55
|
+
return value
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def price(counts, card):
|
|
59
|
+
rate_card(card)
|
|
60
|
+
if any(counts[k] is None or (counts[k] != 0 and card['rates'][k] is None) for k in CATEGORIES):
|
|
61
|
+
return None
|
|
62
|
+
return sum(counts[k] * (card['rates'][k] or 0) for k in CATEGORIES) / 1e6
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def money(value, *, billed=False):
|
|
66
|
+
if value is None:
|
|
67
|
+
return
|
|
68
|
+
keys(value, ['usd', 'source', 'version'], ['rate_card', 'authoritative', 'execution_id', 'receipt_id'], where='usage money')
|
|
69
|
+
number(value['usd'], 'usage money.usd', nonnegative=True)
|
|
70
|
+
text(value['source'], 'usage money.source')
|
|
71
|
+
text(value['version'], 'usage money.version')
|
|
72
|
+
if billed and (value.get('authoritative') is not True or not value.get('receipt_id') or not value.get('execution_id')):
|
|
73
|
+
raise HowHardError('billed_cost: explicit authoritative execution receipt required')
|
|
74
|
+
if not billed and any(k in value for k in ('authoritative', 'execution_id', 'receipt_id')):
|
|
75
|
+
raise HowHardError('estimated_cost: billed receipt fields are not estimates')
|
|
76
|
+
if billed:
|
|
77
|
+
for k in ('receipt_id', 'execution_id'):
|
|
78
|
+
text(value[k], 'billed_cost.' + k)
|
|
79
|
+
if 'rate_card' in value:
|
|
80
|
+
if billed:
|
|
81
|
+
raise HowHardError('billed_cost: a rate card is an estimate, not a billed receipt')
|
|
82
|
+
rate_card(value['rate_card'])
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def validate(value):
|
|
86
|
+
keys(value, ['format', 'execution_id', 'status', 'provenance', 'claims', 'segments'], where='execution_usage')
|
|
87
|
+
if value['format'] != USAGE_FORMAT or value['status'] not in ('complete', 'partial', 'unavailable'):
|
|
88
|
+
raise HowHardError('execution_usage: unsupported format or status')
|
|
89
|
+
text(value['execution_id'], 'execution_usage.execution_id')
|
|
90
|
+
p = value['provenance']
|
|
91
|
+
keys(p, ['adapter', 'host_version', 'artifact_sha256', 'session_id', 'start', 'end', 'scope', 'coordinate_id'], ['baseline_sha256', 'agent_id', 'completed'], where='usage provenance')
|
|
92
|
+
if p['adapter'] not in ADAPTERS or p['scope'] not in ('self', 'inclusive'):
|
|
93
|
+
raise HowHardError('usage provenance: unsupported adapter or scope')
|
|
94
|
+
if p['host_version'] not in SUPPORTED_HOSTS[p['adapter']]:
|
|
95
|
+
raise HowHardError('usage provenance.host_version: unsupported host/schema combination')
|
|
96
|
+
if 'agent_id' in p:
|
|
97
|
+
text(p['agent_id'], 'usage provenance.agent_id')
|
|
98
|
+
if 'baseline_sha256' in p and (not isinstance(p['baseline_sha256'], str) or len(p['baseline_sha256']) != 64 or any(c not in '0123456789abcdef' for c in p['baseline_sha256'])):
|
|
99
|
+
raise HowHardError('usage provenance.baseline_sha256: invalid SHA-256')
|
|
100
|
+
if 'completed' in p and type(p['completed']) is not bool:
|
|
101
|
+
raise HowHardError('usage provenance.completed: expected explicit boolean')
|
|
102
|
+
if value['status'] == 'complete' and p.get('completed') is not True:
|
|
103
|
+
raise HowHardError('execution_usage: complete requires a completed source boundary')
|
|
104
|
+
for key in ('host_version', 'session_id', 'coordinate_id'):
|
|
105
|
+
text(p[key], 'usage provenance.' + key)
|
|
106
|
+
if not isinstance(p['artifact_sha256'], str) or len(p['artifact_sha256']) != 64 or any(c not in '0123456789abcdef' for c in p['artifact_sha256']):
|
|
107
|
+
raise HowHardError('usage provenance: invalid artifact hash')
|
|
108
|
+
if any(type(p[k]) is not int or p[k] < 0 for k in ('start', 'end')) or p['end'] < p['start']:
|
|
109
|
+
raise HowHardError('usage provenance: invalid byte span')
|
|
110
|
+
if not isinstance(value['claims'], list) or not value['claims']:
|
|
111
|
+
raise HowHardError('execution_usage.claims: explicit source claims required')
|
|
112
|
+
for c in value['claims']:
|
|
113
|
+
keys(c, ['session_id', 'start', 'end', 'coordinate_id'], where='usage claim')
|
|
114
|
+
text(c['session_id'], 'usage claim.session_id')
|
|
115
|
+
text(c['coordinate_id'], 'usage claim.coordinate_id')
|
|
116
|
+
if type(c['start']) is not int or c['start'] < 0 or (c['end'] is not None and (type(c['end']) is not int or c['end'] <= c['start'])):
|
|
117
|
+
raise HowHardError('usage claim: invalid exclusive span')
|
|
118
|
+
expected_session = p['session_id'] + ':agent:' + p['agent_id'] if 'agent_id' in p else p['session_id']
|
|
119
|
+
expected = dict(session_id=expected_session, start=p['start'], end=p['end'], coordinate_id=p['coordinate_id'])
|
|
120
|
+
if value['claims'][0] != expected:
|
|
121
|
+
raise HowHardError('execution_usage.claims: primary claim must match source provenance')
|
|
122
|
+
if p['scope'] == 'self' and len(value['claims']) != 1:
|
|
123
|
+
raise HowHardError('execution_usage.claims: self scope cannot claim child sessions')
|
|
124
|
+
for claim in value['claims'][1:]:
|
|
125
|
+
if claim['start'] != 0 or claim['end'] is not None or claim['coordinate_id'] != '*':
|
|
126
|
+
raise HowHardError('execution_usage.claims: included child requires a whole-session claim')
|
|
127
|
+
if not isinstance(value['segments'], list):
|
|
128
|
+
raise HowHardError('execution_usage.segments: expected array')
|
|
129
|
+
if value['status'] == 'complete' and not value['segments']:
|
|
130
|
+
raise HowHardError('execution_usage: complete requires measured segments')
|
|
131
|
+
seen, receipts = set(), set()
|
|
132
|
+
for segment in value['segments']:
|
|
133
|
+
keys(segment, ['source_id', 'provider', 'model', 'service_tier', 'tokens', 'estimated_cost', 'billed_cost'], ['reported_tokens'], where='usage segment')
|
|
134
|
+
for key in ('source_id', 'provider', 'model'):
|
|
135
|
+
text(segment[key], 'usage segment.' + key)
|
|
136
|
+
if segment['source_id'] in seen:
|
|
137
|
+
raise HowHardError('usage segment: duplicate source identity')
|
|
138
|
+
seen.add(segment['source_id'])
|
|
139
|
+
if segment['service_tier'] is not None:
|
|
140
|
+
text(segment['service_tier'], 'usage segment.service_tier')
|
|
141
|
+
tokens(segment['tokens'])
|
|
142
|
+
if segment.get('reported_tokens') is not None and (type(segment['reported_tokens']) is not int or segment['reported_tokens'] < 0):
|
|
143
|
+
raise HowHardError('usage segment.reported_tokens: expected nonnegative integer or null')
|
|
144
|
+
if segment.get('reported_tokens') is not None and total(segment['tokens']) is not None and segment['reported_tokens'] != total(segment['tokens']):
|
|
145
|
+
raise HowHardError('usage segment.reported_tokens: does not match token categories')
|
|
146
|
+
if value['status'] == 'complete' and total(segment['tokens']) is None:
|
|
147
|
+
raise HowHardError('execution_usage: complete requires every measured category')
|
|
148
|
+
money(segment['estimated_cost'])
|
|
149
|
+
money(segment['billed_cost'], billed=True)
|
|
150
|
+
if segment['billed_cost'] is not None:
|
|
151
|
+
receipt = segment['billed_cost']['source'], segment['billed_cost']['receipt_id']
|
|
152
|
+
if receipt in receipts:
|
|
153
|
+
raise HowHardError('billed_cost: duplicate authoritative receipt line identity')
|
|
154
|
+
receipts.add(receipt)
|
|
155
|
+
if segment['billed_cost'] is not None and segment['billed_cost']['execution_id'] != value['execution_id']:
|
|
156
|
+
raise HowHardError('billed_cost.execution_id: receipt does not match execution')
|
|
157
|
+
estimate = segment['estimated_cost']
|
|
158
|
+
if estimate is not None and 'rate_card' in estimate:
|
|
159
|
+
card = estimate['rate_card']
|
|
160
|
+
if (card['provider'], card['model'], card['service_tier']) != (segment['provider'], segment['model'], segment['service_tier']) or estimate['source'] != card['source'] or estimate['version'] != card['version']:
|
|
161
|
+
raise HowHardError('estimated_cost: rate-card identity/provenance mismatch')
|
|
162
|
+
amount = price(segment['tokens'], card)
|
|
163
|
+
if amount is None or abs(amount - estimate['usd']) > 1e-9:
|
|
164
|
+
raise HowHardError('estimated_cost: rate-card amount does not match measured categories')
|
|
165
|
+
if not p.get('completed') and segment['estimated_cost'] is not None:
|
|
166
|
+
raise HowHardError('execution_usage: incomplete source boundary cannot claim complete estimate')
|
|
167
|
+
return value
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def duplicates(rows):
|
|
171
|
+
"""Check effective executions, so replacing an execution's own claims is permitted."""
|
|
172
|
+
owners = []
|
|
173
|
+
for row in rows:
|
|
174
|
+
usage = row.get('execution_usage')
|
|
175
|
+
if usage is None:
|
|
176
|
+
continue
|
|
177
|
+
validate(usage)
|
|
178
|
+
if usage['execution_id'] != row['id']:
|
|
179
|
+
raise HowHardError('execution_usage.execution_id: must identify the original observation')
|
|
180
|
+
for old, other in owners:
|
|
181
|
+
if usage['execution_id'] == old['execution_id']:
|
|
182
|
+
continue
|
|
183
|
+
bills = {(s['billed_cost']['source'], s['billed_cost']['receipt_id']) for s in usage['segments'] if s['billed_cost'] is not None}
|
|
184
|
+
old_bills = {(s['billed_cost']['source'], s['billed_cost']['receipt_id']) for s in old['segments'] if s['billed_cost'] is not None}
|
|
185
|
+
if bills & old_bills:
|
|
186
|
+
raise HowHardError('billed_cost: authoritative receipt already belongs to another execution')
|
|
187
|
+
if any(a['source_id'] == b['source_id'] for a in usage['segments'] for b in old['segments']):
|
|
188
|
+
raise HowHardError('execution_usage: source identity already belongs to another execution')
|
|
189
|
+
for a in usage['claims']:
|
|
190
|
+
for b in old['claims']:
|
|
191
|
+
if a['session_id'] == b['session_id'] and (a['coordinate_id'] != b['coordinate_id'] or ((a['end'] is None or b['start'] < a['end']) and (b['end'] is None or a['start'] < b['end']))):
|
|
192
|
+
raise HowHardError('execution_usage: source span overlaps another execution')
|
|
193
|
+
owners.append((usage, row['id']))
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _count(value, key, default=None):
|
|
197
|
+
if not isinstance(value, dict):
|
|
198
|
+
raise HowHardError('source usage: expected object')
|
|
199
|
+
count = value.get(key, default)
|
|
200
|
+
if count is not None and (type(count) is not int or count < 0 or count > 2**63 - 1):
|
|
201
|
+
raise HowHardError('source usage.' + key + ': expected nonnegative integer')
|
|
202
|
+
return count
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _codex(u):
|
|
206
|
+
inp, cached, out = (_count(u, k) for k in ('input_tokens', 'cached_input_tokens', 'output_tokens'))
|
|
207
|
+
writes = _count(u, 'cache_write_input_tokens', 0)
|
|
208
|
+
if inp is not None and cached is not None and ((cached + writes if writes is not None else cached) > inp):
|
|
209
|
+
raise HowHardError('Codex cached input exceeds inclusive input')
|
|
210
|
+
reasoning = _count(u, 'reasoning_output_tokens', 0)
|
|
211
|
+
if out is not None and reasoning is not None and reasoning > out:
|
|
212
|
+
raise HowHardError('Codex reasoning exceeds inclusive output')
|
|
213
|
+
return dict(input=None if inp is None or cached is None or writes is None else inp - cached - writes,
|
|
214
|
+
output=out, cache_read=cached, cache_write=writes, cache_write_5m=0, cache_write_1h=0)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _codex_total(u):
|
|
218
|
+
inp, out = _count(u, 'input_tokens'), _count(u, 'output_tokens')
|
|
219
|
+
count = _count(u, 'total_tokens')
|
|
220
|
+
if count is not None and inp is not None and out is not None and count != inp + out:
|
|
221
|
+
raise HowHardError('Codex total_tokens: does not match inclusive input/output')
|
|
222
|
+
return count if count is not None else inp + out if inp is not None and out is not None else None
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _claude(u, *, sdk=False):
|
|
226
|
+
names = ('inputTokens', 'outputTokens', 'cacheReadInputTokens', 'cacheCreationInputTokens') if sdk else ('input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens')
|
|
227
|
+
inp, out, read, write = (_count(u, k) for k in names)
|
|
228
|
+
ttl = u.get('cache_creation')
|
|
229
|
+
if ttl is not None:
|
|
230
|
+
keys(ttl, ['ephemeral_5m_input_tokens', 'ephemeral_1h_input_tokens'], where='Claude cache creation')
|
|
231
|
+
five, hour = (_count(ttl, k) for k in ('ephemeral_5m_input_tokens', 'ephemeral_1h_input_tokens'))
|
|
232
|
+
if write is None or five is None or hour is None or five + hour > write:
|
|
233
|
+
raise HowHardError('Claude cache TTL counts exceed or lack total')
|
|
234
|
+
write -= five + hour
|
|
235
|
+
else:
|
|
236
|
+
five, hour = 0, 0
|
|
237
|
+
return dict(input=inp, output=out, cache_read=read, cache_write=write, cache_write_5m=five, cache_write_1h=hour)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def collect(manifest):
|
|
241
|
+
keys(manifest, ['format', 'execution_id', 'host_version', 'adapter', 'source', 'session_id', 'scope', 'start', 'end', 'completed', 'provider', 'model', 'service_tier'], ['included_sessions', 'included_agents', 'rate_card', 'baseline', 'coordinate_id', 'agent_id'], where='usage manifest')
|
|
242
|
+
if manifest['format'] != 'howhard-usage-manifest-v1' or manifest['adapter'] not in ADAPTERS:
|
|
243
|
+
raise HowHardError('usage manifest: unsupported format/adapter')
|
|
244
|
+
if manifest['host_version'] not in SUPPORTED_HOSTS[manifest['adapter']]:
|
|
245
|
+
raise HowHardError('usage manifest.host_version: unsupported host/schema combination')
|
|
246
|
+
if type(manifest['completed']) is not bool or manifest['scope'] not in ('self', 'inclusive'):
|
|
247
|
+
raise HowHardError('usage manifest: explicit completion and scope required')
|
|
248
|
+
for key in ('execution_id', 'provider', 'model', 'session_id'):
|
|
249
|
+
text(manifest[key], 'usage manifest.' + key)
|
|
250
|
+
for key in ('agent_id', 'coordinate_id', 'service_tier'):
|
|
251
|
+
if key in manifest and manifest[key] is not None:
|
|
252
|
+
text(manifest[key], 'usage manifest.' + key)
|
|
253
|
+
if ('agent_id' in manifest and manifest['agent_id'] is None) or ('coordinate_id' in manifest and manifest['coordinate_id'] is None):
|
|
254
|
+
raise HowHardError('usage manifest: agent_id/coordinate_id must be explicit nonempty identities')
|
|
255
|
+
start, end = manifest['start'], manifest['end']
|
|
256
|
+
if type(start) is not int or type(end) is not int or start < 0 or end <= start or end - start > MAX_SPAN:
|
|
257
|
+
raise HowHardError('usage manifest: invalid or oversized byte span')
|
|
258
|
+
path = Path(text(manifest['source'], 'usage manifest.source'))
|
|
259
|
+
try:
|
|
260
|
+
if not stat.S_ISREG(path.stat().st_mode):
|
|
261
|
+
raise HowHardError('usage source: bounded regular file required')
|
|
262
|
+
# Nonblocking open plus fstat also closes a regular-file-to-FIFO replacement race.
|
|
263
|
+
descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK)
|
|
264
|
+
with os.fdopen(descriptor, 'rb') as handle:
|
|
265
|
+
if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
|
|
266
|
+
raise HowHardError('usage source: bounded regular file required')
|
|
267
|
+
if not path.is_file() or handle.seek(0, 2) > MAX_SOURCE or handle.tell() < end:
|
|
268
|
+
raise HowHardError('usage source: unavailable span or oversized artifact')
|
|
269
|
+
if start:
|
|
270
|
+
handle.seek(start - 1)
|
|
271
|
+
if handle.read(1) != b'\n':
|
|
272
|
+
raise HowHardError('usage span: start must be a complete record boundary')
|
|
273
|
+
handle.seek(start)
|
|
274
|
+
blob = handle.read(end - start)
|
|
275
|
+
if len(blob) != end - start:
|
|
276
|
+
raise HowHardError('usage source: source changed during bounded read')
|
|
277
|
+
except OSError as exc:
|
|
278
|
+
raise HowHardError('usage source: read failed; outcome remains independently durable') from exc
|
|
279
|
+
if not blob.endswith(b'\n'):
|
|
280
|
+
raise HowHardError('usage span: end must be a complete record boundary')
|
|
281
|
+
try:
|
|
282
|
+
events = [json.loads(line) for line in blob.splitlines()]
|
|
283
|
+
except (ValueError, UnicodeError) as exc:
|
|
284
|
+
raise HowHardError('usage source: invalid JSON records') from exc
|
|
285
|
+
if any(not isinstance(e, dict) for e in events):
|
|
286
|
+
raise HowHardError('usage source: expected event objects')
|
|
287
|
+
adapter, session = manifest['adapter'], manifest['session_id']
|
|
288
|
+
text(session, 'usage session_id')
|
|
289
|
+
if adapter == 'claude-sdk-v1' and manifest['scope'] != 'inclusive':
|
|
290
|
+
raise HowHardError('Claude SDK result modelUsage is child-inclusive; declare inclusive scope')
|
|
291
|
+
if adapter == 'claude-sdk-v1' and 'included_sessions' not in manifest:
|
|
292
|
+
raise HowHardError('Claude inclusive result requires explicit included_sessions (empty only when no children ran)')
|
|
293
|
+
children = manifest.get('included_sessions', [])
|
|
294
|
+
if not isinstance(children, list) or any(not isinstance(s, str) or not s or s == session for s in children) or len(set(children)) != len(children):
|
|
295
|
+
raise HowHardError('usage manifest.included_sessions: distinct child session IDs required')
|
|
296
|
+
agents = manifest.get('included_agents', [])
|
|
297
|
+
if not isinstance(agents, list) or any(not isinstance(a, str) or not a for a in agents) or len(set(agents)) != len(agents):
|
|
298
|
+
raise HowHardError('usage manifest.included_agents: distinct actual Claude agent IDs required')
|
|
299
|
+
if agents and adapter != 'claude-sdk-v1':
|
|
300
|
+
raise HowHardError('usage manifest.included_agents: only Claude SDK inclusive results support shared-parent agents')
|
|
301
|
+
if (children or agents) and manifest['scope'] != 'inclusive':
|
|
302
|
+
raise HowHardError('usage manifest: child claims require inclusive scope')
|
|
303
|
+
# Coordinates must identify a single append-only artifact across captures. Without
|
|
304
|
+
# host binding, cross-source same-session claims are conservatively incompatible.
|
|
305
|
+
coordinate = manifest.get('coordinate_id', str(path.resolve()) + ':' + adapter)
|
|
306
|
+
source_session = session + ':agent:' + manifest['agent_id'] if 'agent_id' in manifest else session
|
|
307
|
+
claims = [{'session_id': source_session, 'start': start, 'end': end, 'coordinate_id': coordinate}] + [{'session_id': s, 'start': 0, 'end': None, 'coordinate_id': '*'} for s in children] + [{'session_id': session + ':agent:' + a, 'start': 0, 'end': None, 'coordinate_id': '*'} for a in agents]
|
|
308
|
+
result = dict(format=USAGE_FORMAT, execution_id=manifest['execution_id'], status='unavailable',
|
|
309
|
+
provenance={k: manifest[k] for k in ('adapter', 'host_version', 'session_id', 'start', 'end', 'scope')}, claims=claims, segments=[])
|
|
310
|
+
result['provenance']['coordinate_id'] = coordinate
|
|
311
|
+
if 'agent_id' in manifest:
|
|
312
|
+
result['provenance']['agent_id'] = manifest['agent_id']
|
|
313
|
+
result['provenance']['artifact_sha256'] = hashlib.sha256(blob).hexdigest()
|
|
314
|
+
records, terminal, seen, previous = [], False, {}, None
|
|
315
|
+
baseline = manifest.get('baseline')
|
|
316
|
+
if adapter == 'claude-sdk-v1' and baseline is not None:
|
|
317
|
+
keys(baseline, ['session_id', 'modelUsage'], where='Claude baseline')
|
|
318
|
+
if baseline['session_id'] != session or not isinstance(baseline['modelUsage'], dict):
|
|
319
|
+
raise HowHardError('Claude baseline: same-session measured starting totals required')
|
|
320
|
+
result['provenance']['baseline_sha256'] = hashlib.sha256(json.dumps(baseline, sort_keys=True).encode()).hexdigest()
|
|
321
|
+
if adapter == 'codex-session-v1' and baseline is not None:
|
|
322
|
+
if not isinstance(baseline, dict):
|
|
323
|
+
raise HowHardError('Codex baseline: expected measured cumulative starting usage')
|
|
324
|
+
_codex(baseline)
|
|
325
|
+
previous = baseline
|
|
326
|
+
result['provenance']['baseline_sha256'] = hashlib.sha256(json.dumps(baseline, sort_keys=True).encode()).hexdigest()
|
|
327
|
+
codex_seen_gap = False
|
|
328
|
+
for event in events:
|
|
329
|
+
if adapter == 'codex-exec-v1':
|
|
330
|
+
if event.get('type') == 'thread.started' and event.get('thread_id') != session:
|
|
331
|
+
raise HowHardError('Codex source session identity mismatch')
|
|
332
|
+
if event.get('type') == 'turn.completed':
|
|
333
|
+
u = event.get('usage')
|
|
334
|
+
if not isinstance(u, dict):
|
|
335
|
+
raise HowHardError('Codex completed turn lacks usage')
|
|
336
|
+
records.append((str(start) + ':' + str(len(records)), manifest['model'], manifest['service_tier'], _codex(u), None, _codex_total(u)))
|
|
337
|
+
terminal = True
|
|
338
|
+
elif event.get('type') in ('turn.started', 'turn.failed', 'error'):
|
|
339
|
+
terminal = False
|
|
340
|
+
elif adapter == 'codex-session-v1':
|
|
341
|
+
p = event.get('payload', {})
|
|
342
|
+
if not isinstance(p, dict):
|
|
343
|
+
raise HowHardError('Codex transcript: invalid payload object')
|
|
344
|
+
if event.get('type') == 'session_meta' and p.get('id') != session:
|
|
345
|
+
raise HowHardError('Codex source session identity mismatch')
|
|
346
|
+
if event.get('type') == 'event_msg' and p.get('type') == 'token_count':
|
|
347
|
+
info = p.get('info')
|
|
348
|
+
if info is None:
|
|
349
|
+
codex_seen_gap = True
|
|
350
|
+
continue
|
|
351
|
+
if not isinstance(info, dict) or not isinstance(info.get('total_token_usage'), dict) or not isinstance(info.get('last_token_usage'), dict):
|
|
352
|
+
raise HowHardError('Codex transcript: unsupported token_count schema')
|
|
353
|
+
current = info['total_token_usage']
|
|
354
|
+
identity = json.dumps(current, sort_keys=True)
|
|
355
|
+
if identity in seen:
|
|
356
|
+
continue
|
|
357
|
+
if previous is not None and any(_count(current, k, 0) is not None and _count(previous, k, 0) is not None and current.get(k, 0) < previous.get(k, 0) for k in ('input_tokens', 'output_tokens')):
|
|
358
|
+
raise HowHardError('Codex cumulative usage reset; split at reset with explicit ownership')
|
|
359
|
+
if previous is not None or start == 0:
|
|
360
|
+
for k in ('input_tokens', 'cached_input_tokens', 'cache_write_input_tokens', 'output_tokens'):
|
|
361
|
+
now, before, last = (_count(d, k, 0) for d in (current, previous if previous is not None else {}, info['last_token_usage']))
|
|
362
|
+
if None in (now, before, last) or now - before != last:
|
|
363
|
+
codex_seen_gap = True
|
|
364
|
+
else:
|
|
365
|
+
codex_seen_gap = True # A resumed span needs its measured cumulative starting point.
|
|
366
|
+
previous = current
|
|
367
|
+
seen[identity] = True
|
|
368
|
+
stamp = text(event.get('timestamp'), 'Codex token_count timestamp')
|
|
369
|
+
records.append((stamp + ':' + hashlib.sha256(identity.encode()).hexdigest(), manifest['model'], manifest['service_tier'], _codex(info['last_token_usage']), None, _codex_total(info['last_token_usage'])))
|
|
370
|
+
terminal = False
|
|
371
|
+
if event.get('type') == 'event_msg' and p.get('type') == 'task_complete':
|
|
372
|
+
terminal = True
|
|
373
|
+
elif event.get('type') == 'event_msg' and p.get('type') in ('task_started', 'turn_aborted'):
|
|
374
|
+
terminal = False
|
|
375
|
+
elif adapter == 'claude-sdk-v1':
|
|
376
|
+
if event.get('type') != 'result':
|
|
377
|
+
continue
|
|
378
|
+
if event.get('session_id') != session:
|
|
379
|
+
raise HowHardError('Claude result session identity mismatch')
|
|
380
|
+
if records:
|
|
381
|
+
raise HowHardError('Claude result: one bounded invocation result required; do not sum resume totals')
|
|
382
|
+
models = event.get('modelUsage')
|
|
383
|
+
if not isinstance(models, dict) or not models:
|
|
384
|
+
raise HowHardError('Claude result: modelUsage required')
|
|
385
|
+
aggregate = event.get('total_cost_usd')
|
|
386
|
+
if aggregate is not None:
|
|
387
|
+
aggregate = number(aggregate, 'Claude result.total_cost_usd', nonnegative=True)
|
|
388
|
+
costs = []
|
|
389
|
+
for u in models.values():
|
|
390
|
+
if not isinstance(u, dict):
|
|
391
|
+
raise HowHardError('Claude result: invalid model usage')
|
|
392
|
+
if u.get('costUSD') is not None:
|
|
393
|
+
costs.append(number(u['costUSD'], 'Claude model estimated cost', nonnegative=True))
|
|
394
|
+
if len(costs) == len(models) and abs(sum(costs) - aggregate) > max(1e-8, aggregate * 1e-9):
|
|
395
|
+
raise HowHardError('Claude result: aggregate client cost contradicts model breakdown; unsupported fee/cost scope')
|
|
396
|
+
if baseline is None:
|
|
397
|
+
terminal = False
|
|
398
|
+
continue # Cumulative totals with unknown starting point are not invocation usage.
|
|
399
|
+
if set(baseline['modelUsage']) - set(models):
|
|
400
|
+
raise HowHardError('Claude cumulative usage reset: baseline model missing')
|
|
401
|
+
for model, u in models.items():
|
|
402
|
+
if not isinstance(u, dict):
|
|
403
|
+
raise HowHardError('Claude result: invalid model usage')
|
|
404
|
+
estimate = u.get('costUSD')
|
|
405
|
+
if estimate is not None:
|
|
406
|
+
number(estimate, 'Claude result estimated cost', nonnegative=True)
|
|
407
|
+
counts = _claude(u, sdk=True)
|
|
408
|
+
prior = baseline['modelUsage'].get(model)
|
|
409
|
+
if prior is not None:
|
|
410
|
+
before = _claude(prior, sdk=True)
|
|
411
|
+
for key in CATEGORIES:
|
|
412
|
+
if counts[key] is None or before[key] is None:
|
|
413
|
+
counts[key] = None
|
|
414
|
+
else:
|
|
415
|
+
counts[key] -= before[key]
|
|
416
|
+
if counts[key] < 0:
|
|
417
|
+
raise HowHardError('Claude cumulative usage reset; split execution at reset')
|
|
418
|
+
if estimate is not None:
|
|
419
|
+
if prior.get('costUSD') is None:
|
|
420
|
+
estimate = None
|
|
421
|
+
else:
|
|
422
|
+
number(prior['costUSD'], 'Claude baseline estimated cost', nonnegative=True)
|
|
423
|
+
estimate -= prior['costUSD']
|
|
424
|
+
if estimate < 0:
|
|
425
|
+
raise HowHardError('Claude cumulative cost reset; split execution at reset')
|
|
426
|
+
result_id = text(event.get('uuid'), 'Claude result UUID')
|
|
427
|
+
records.append((result_id + ':' + model, model, manifest['service_tier'], counts, estimate, total(counts)))
|
|
428
|
+
# Never replay cumulative restored spend as a fresh invocation's estimate.
|
|
429
|
+
if event.get('num_turns') is not None and type(event['num_turns']) is not int:
|
|
430
|
+
raise HowHardError('Claude result: invalid turn count')
|
|
431
|
+
if type(event.get('is_error')) is not bool:
|
|
432
|
+
raise HowHardError('Claude result.is_error: explicit boolean required')
|
|
433
|
+
subtype = event.get('subtype')
|
|
434
|
+
terminal = subtype in ('success', 'error_max_turns', 'error_max_budget_usd')
|
|
435
|
+
if terminal and event.get('is_error', False) != (subtype != 'success'):
|
|
436
|
+
raise HowHardError('Claude result: contradictory terminal failure metadata')
|
|
437
|
+
else:
|
|
438
|
+
if event.get('type') != 'assistant':
|
|
439
|
+
continue
|
|
440
|
+
if event.get('sessionId') != session or event.get('isSidechain', False) != ('agent_id' in manifest) or ('agent_id' in manifest and event.get('agentId') != manifest['agent_id']):
|
|
441
|
+
raise HowHardError('Claude transcript: session/child scope mismatch')
|
|
442
|
+
msg = event.get('message')
|
|
443
|
+
if not isinstance(msg, dict) or not isinstance(msg.get('usage'), dict):
|
|
444
|
+
raise HowHardError('Claude transcript: unsupported assistant usage schema')
|
|
445
|
+
ident = text(msg.get('id'), 'Claude message ID')
|
|
446
|
+
normalized = (ident, text(msg.get('model'), 'Claude message model'), msg['usage'].get('service_tier', manifest['service_tier']), _claude(msg['usage']), None, total(_claude(msg['usage'])))
|
|
447
|
+
if ident in seen:
|
|
448
|
+
# Streaming records can revise usage for the same assistant message.
|
|
449
|
+
records[seen[ident]] = normalized
|
|
450
|
+
else:
|
|
451
|
+
seen[ident] = len(records)
|
|
452
|
+
records.append(normalized)
|
|
453
|
+
terminal = msg.get('stop_reason') == 'end_turn'
|
|
454
|
+
result['provenance']['completed'] = bool(records) and terminal and manifest['completed'] and not codex_seen_gap
|
|
455
|
+
complete = bool(records) and terminal and manifest['completed'] and not codex_seen_gap and all(total(r[3]) is not None for r in records)
|
|
456
|
+
result['status'] = 'complete' if complete else 'partial' if records else 'unavailable'
|
|
457
|
+
for ident, model, tier, counts, estimate, reported in records:
|
|
458
|
+
segment = dict(source_id=source_session + ':' + adapter + ':' + ident, provider=manifest['provider'], model=model,
|
|
459
|
+
service_tier=tier, tokens=counts, reported_tokens=reported, estimated_cost=None, billed_cost=None)
|
|
460
|
+
if result['provenance']['completed'] and estimate is not None:
|
|
461
|
+
segment['estimated_cost'] = dict(usd=estimate, source='claude-client', version=manifest['host_version'])
|
|
462
|
+
card = manifest.get('rate_card')
|
|
463
|
+
if card is not None:
|
|
464
|
+
rate_card(card)
|
|
465
|
+
if (card['provider'], card['model'], card['service_tier']) != (segment['provider'], model, tier):
|
|
466
|
+
raise HowHardError('rate_card: source provider/model/tier mismatch')
|
|
467
|
+
amount = price(counts, card)
|
|
468
|
+
if result['provenance']['completed'] and amount is not None:
|
|
469
|
+
segment['estimated_cost'] = dict(usd=amount, source=card['source'], version=card['version'], rate_card=deepcopy(card))
|
|
470
|
+
result['segments'].append(segment)
|
|
471
|
+
return validate(result)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def summary(rows):
|
|
475
|
+
"""Separate money bases and coverage without turning missing measurements into zero."""
|
|
476
|
+
result = {"runs": len(rows), "statuses": {s: 0 for s in ("complete", "partial", "unavailable", "missing")},
|
|
477
|
+
"tokens": {k: {"known": 0, "unknown_segments": 0, "unknown_runs": 0} for k in CATEGORIES}}
|
|
478
|
+
for field in ("estimated_cost", "billed_cost"):
|
|
479
|
+
known, unknown = 0.0, 0
|
|
480
|
+
for row in rows:
|
|
481
|
+
usage = row.get('execution_usage')
|
|
482
|
+
if not usage or not usage['segments']:
|
|
483
|
+
unknown += 1
|
|
484
|
+
continue
|
|
485
|
+
amounts = [s[field] for s in usage['segments']]
|
|
486
|
+
known += sum(v['usd'] for v in amounts if v is not None)
|
|
487
|
+
unknown += int(any(v is None for v in amounts) or (field == 'estimated_cost' and not usage['provenance'].get('completed')))
|
|
488
|
+
result[field] = dict(known_usd=known, unknown_runs=unknown, total_usd=known if not unknown else None)
|
|
489
|
+
for row in rows:
|
|
490
|
+
usage = row.get('execution_usage')
|
|
491
|
+
result['statuses'][usage['status'] if usage else 'missing'] += 1
|
|
492
|
+
for key in CATEGORIES:
|
|
493
|
+
if not usage or not usage['segments'] or not usage['provenance'].get('completed') or any(s['tokens'][key] is None for s in usage['segments']):
|
|
494
|
+
result['tokens'][key]['unknown_runs'] += 1
|
|
495
|
+
for segment in usage['segments'] if usage else []:
|
|
496
|
+
for key, count in segment['tokens'].items():
|
|
497
|
+
if count is None:
|
|
498
|
+
result['tokens'][key]['unknown_segments'] += 1
|
|
499
|
+
else:
|
|
500
|
+
result['tokens'][key]['known'] += count
|
|
501
|
+
for category in result['tokens'].values():
|
|
502
|
+
category['total'] = category['known'] if not category['unknown_runs'] else None
|
|
503
|
+
return result
|