sessionwiki 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sessionwiki/__init__.py +74 -0
- sessionwiki/__main__.py +6 -0
- sessionwiki/_frontmatter.py +118 -0
- sessionwiki/_system_pages.py +44 -0
- sessionwiki/adapter_config.py +124 -0
- sessionwiki/adapters/__init__.py +134 -0
- sessionwiki/adapters/base.py +191 -0
- sessionwiki/adapters/claude_code.py +77 -0
- sessionwiki/adapters/codex_cli.py +226 -0
- sessionwiki/adapters/contrib/__init__.py +1 -0
- sessionwiki/adapters/contrib/chatgpt.py +252 -0
- sessionwiki/adapters/contrib/copilot_chat.py +121 -0
- sessionwiki/adapters/contrib/copilot_cli.py +115 -0
- sessionwiki/adapters/contrib/cursor.py +100 -0
- sessionwiki/adapters/contrib/gemini_cli.py +74 -0
- sessionwiki/adapters/contrib/obsidian.py +121 -0
- sessionwiki/adapters/opencode.py +169 -0
- sessionwiki/adapters/status.py +58 -0
- sessionwiki/adapters/zcode.py +45 -0
- sessionwiki/audit.py +174 -0
- sessionwiki/backlinks.py +224 -0
- sessionwiki/build.py +2786 -0
- sessionwiki/cache.py +401 -0
- sessionwiki/candidates.py +309 -0
- sessionwiki/categories.py +158 -0
- sessionwiki/changelog_timeline.py +422 -0
- sessionwiki/cli.py +1687 -0
- sessionwiki/compare.py +404 -0
- sessionwiki/completion.py +191 -0
- sessionwiki/confidence.py +179 -0
- sessionwiki/config.py +74 -0
- sessionwiki/config_schedule.py +77 -0
- sessionwiki/context_md.py +182 -0
- sessionwiki/convert.py +1746 -0
- sessionwiki/docs_pages.py +725 -0
- sessionwiki/doctor.py +113 -0
- sessionwiki/exporters.py +680 -0
- sessionwiki/freshness.py +153 -0
- sessionwiki/graph.py +1058 -0
- sessionwiki/graphify_bridge.py +509 -0
- sessionwiki/ingest_queue.py +101 -0
- sessionwiki/lifecycle.py +167 -0
- sessionwiki/link_checker.py +148 -0
- sessionwiki/lint/__init__.py +170 -0
- sessionwiki/lint/rules/__init__.py +69 -0
- sessionwiki/lint/rules/_helpers.py +119 -0
- sessionwiki/lint/rules/claim_verification.py +44 -0
- sessionwiki/lint/rules/content_freshness.py +41 -0
- sessionwiki/lint/rules/contradiction_detection.py +45 -0
- sessionwiki/lint/rules/duplicate_detection.py +164 -0
- sessionwiki/lint/rules/entity_consistency.py +29 -0
- sessionwiki/lint/rules/frontmatter_completeness.py +46 -0
- sessionwiki/lint/rules/frontmatter_count_consistency.py +61 -0
- sessionwiki/lint/rules/frontmatter_validity.py +71 -0
- sessionwiki/lint/rules/index_sync.py +68 -0
- sessionwiki/lint/rules/link_integrity.py +34 -0
- sessionwiki/lint/rules/orphan_detection.py +43 -0
- sessionwiki/lint/rules/stale_candidates.py +59 -0
- sessionwiki/lint/rules/stale_reference_detection.py +42 -0
- sessionwiki/lint/rules/summary_accuracy.py +36 -0
- sessionwiki/lint/rules/tags_topics_convention.py +58 -0
- sessionwiki/lint/rules/tools_consistency.py +65 -0
- sessionwiki/lock.py +126 -0
- sessionwiki/log_reader.py +129 -0
- sessionwiki/manifest.py +132 -0
- sessionwiki/mcp/__init__.py +29 -0
- sessionwiki/mcp/__main__.py +5 -0
- sessionwiki/mcp/server.py +1061 -0
- sessionwiki/models_page.py +291 -0
- sessionwiki/obsidian_output.py +208 -0
- sessionwiki/opencode/__init__.py +1 -0
- sessionwiki/opencode/autosync.py +102 -0
- sessionwiki/opencode/export.py +419 -0
- sessionwiki/opencode/install.py +152 -0
- sessionwiki/pipeline.py +150 -0
- sessionwiki/project_topics.py +213 -0
- sessionwiki/py.typed +0 -0
- sessionwiki/quarantine.py +273 -0
- sessionwiki/queue.py +29 -0
- sessionwiki/references.py +224 -0
- sessionwiki/render/__init__.py +20 -0
- sessionwiki/render/css.py +988 -0
- sessionwiki/render/docs_css.py +463 -0
- sessionwiki/render/js.py +1337 -0
- sessionwiki/schema.py +270 -0
- sessionwiki/search_facets.py +150 -0
- sessionwiki/search_tree.py +184 -0
- sessionwiki/serve.py +91 -0
- sessionwiki/session_export.py +333 -0
- sessionwiki/skill_installer.py +112 -0
- sessionwiki/sync/__init__.py +13 -0
- sessionwiki/sync/status.py +120 -0
- sessionwiki/synth/__init__.py +14 -0
- sessionwiki/synth/agent_delegate.py +327 -0
- sessionwiki/synth/base.py +125 -0
- sessionwiki/synth/cli_helpers.py +98 -0
- sessionwiki/synth/estimate.py +146 -0
- sessionwiki/synth/ollama.py +379 -0
- sessionwiki/synth/openai_compat.py +325 -0
- sessionwiki/synth/pipeline.py +1489 -0
- sessionwiki/tag_utils.py +62 -0
- sessionwiki/tags.py +359 -0
- sessionwiki/vault.py +347 -0
- sessionwiki/viz_heatmap.py +305 -0
- sessionwiki/viz_tokens.py +429 -0
- sessionwiki/viz_tools.py +241 -0
- sessionwiki/watch.py +154 -0
- sessionwiki/zcode/__init__.py +43 -0
- sessionwiki/zcode/autosync.py +150 -0
- sessionwiki/zcode/export.py +568 -0
- sessionwiki/zcode/install.py +386 -0
- sessionwiki-2.0.0.dist-info/METADATA +303 -0
- sessionwiki-2.0.0.dist-info/RECORD +118 -0
- sessionwiki-2.0.0.dist-info/WHEEL +5 -0
- sessionwiki-2.0.0.dist-info/entry_points.txt +2 -0
- sessionwiki-2.0.0.dist-info/licenses/LICENSE +22 -0
- sessionwiki-2.0.0.dist-info/licenses/NOTICE +14 -0
- sessionwiki-2.0.0.dist-info/top_level.txt +1 -0
sessionwiki/cache.py
ADDED
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
"""Prompt caching + batch-API scaffolding (v1.1.0 · #50).
|
|
2
|
+
|
|
3
|
+
Thread-safety
|
|
4
|
+
-------------
|
|
5
|
+
**This module is NOT thread-safe.** ``load_batch_state`` /
|
|
6
|
+
``save_batch_state`` read + write a JSON file with no locking;
|
|
7
|
+
concurrent callers can race on the temp-file rename. The pure-functional
|
|
8
|
+
helpers (``make_cached_block``, ``build_messages``, ``estimate_tokens``,
|
|
9
|
+
``estimate_cost``) are reentrant. Batch-state helpers must be called from
|
|
10
|
+
a single thread or under an external lock. Today's only call sites are
|
|
11
|
+
single-process CLI invocations of ``sessionwiki synthesize`` so this is fine;
|
|
12
|
+
a future MCP path that queues batches concurrently must serialize via its
|
|
13
|
+
own mutex (#py-l7 / #605).
|
|
14
|
+
|
|
15
|
+
Every `/wiki-sync` and `/wiki-ingest` bundles the same stable prefix —
|
|
16
|
+
CLAUDE.md schema, `wiki/index.md`, and `wiki/overview.md` — with every
|
|
17
|
+
source file it asks the model to summarize. On a 500-page wiki that
|
|
18
|
+
prefix is ≈30k tokens per request. sage-wiki reports 50–90% savings by
|
|
19
|
+
marking the prefix as ``cache_control: {type: "ephemeral"}`` so
|
|
20
|
+
Anthropic caches and re-uses it across calls.
|
|
21
|
+
|
|
22
|
+
This module provides **only the plumbing**: header construction, token
|
|
23
|
+
estimation, batch-state persistence. Actual Anthropic API calls land in
|
|
24
|
+
a follow-up PR (v1.2) once the scaffolding lands and is exercised by
|
|
25
|
+
the existing Ollama / Dummy backends for testing.
|
|
26
|
+
|
|
27
|
+
Public surface
|
|
28
|
+
--------------
|
|
29
|
+
- ``make_cached_block(text)`` — wrap a string as a cached content block
|
|
30
|
+
- ``CachedPrompt`` — dataclass holding stable prefix + dynamic suffix
|
|
31
|
+
- ``build_messages(prompt)`` — render a ``CachedPrompt`` into the
|
|
32
|
+
Anthropic ``messages`` array with ``cache_control`` on the prefix
|
|
33
|
+
- ``estimate_tokens(text)`` — char/4 heuristic (fast, no tokenizer dep)
|
|
34
|
+
- ``estimate_cost(...)`` — dollar estimate using the published rate card
|
|
35
|
+
- ``BatchState`` — pending / completed ``message_batches`` IDs on disk
|
|
36
|
+
- ``load_batch_state`` / ``save_batch_state`` — JSON round-trip helpers
|
|
37
|
+
- ``MODEL_PRICING`` — published USD/MTok rates for the models we ship
|
|
38
|
+
|
|
39
|
+
Design notes
|
|
40
|
+
------------
|
|
41
|
+
- **Stdlib-only.** We don't import ``anthropic`` here — the scaffold
|
|
42
|
+
runs anywhere. The real backend will depend on ``anthropic`` and
|
|
43
|
+
re-use this module.
|
|
44
|
+
- **Estimate-first.** ``estimate_cost()`` lets ``sessionwiki sync --estimate``
|
|
45
|
+
print a cached-vs-fresh breakdown *before* spending money.
|
|
46
|
+
- **Batch state file.** ``.llmwiki-batch-state.json`` mirrors the shape
|
|
47
|
+
of ``.llmwiki-synth-state.json``: small, line-oriented, easy to grep.
|
|
48
|
+
- **No implicit cache writes.** Cache-control lives on the block, not
|
|
49
|
+
the request; inserting ``cache_control`` is always opt-in so tests
|
|
50
|
+
can drive pure prefix-vs-suffix logic.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
from __future__ import annotations
|
|
54
|
+
|
|
55
|
+
import json
|
|
56
|
+
from dataclasses import dataclass, field
|
|
57
|
+
from pathlib import Path
|
|
58
|
+
from typing import Any, Optional, TypedDict
|
|
59
|
+
|
|
60
|
+
# ─── Constants ─────────────────────────────────────────────────────────
|
|
61
|
+
|
|
62
|
+
# Anthropic's ephemeral cache block shape.
|
|
63
|
+
CACHE_CONTROL_EPHEMERAL = {"type": "ephemeral"}
|
|
64
|
+
|
|
65
|
+
# Rough token estimator: Anthropic's own guidance is ~4 chars/token for
|
|
66
|
+
# English prose. Close enough for a cost preview; real counts come back
|
|
67
|
+
# in the API response usage block.
|
|
68
|
+
CHARS_PER_TOKEN = 4
|
|
69
|
+
|
|
70
|
+
# Minimum prefix size the Anthropic cache will accept. Below this the
|
|
71
|
+
# ``cache_control`` header is ignored and you pay the full input price.
|
|
72
|
+
# (Value per Anthropic docs; kept here so ``estimate_cost`` can warn
|
|
73
|
+
# when the prefix is too small to benefit.)
|
|
74
|
+
MIN_CACHEABLE_TOKENS = 1024
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ─── Pricing ───────────────────────────────────────────────────────────
|
|
78
|
+
|
|
79
|
+
class ModelRates(TypedDict):
|
|
80
|
+
"""Per-model USD rates per 1 M tokens."""
|
|
81
|
+
|
|
82
|
+
input: float # fresh (un-cached) input tokens
|
|
83
|
+
cached_input: float # cache-hit input tokens (usually 0.1x input)
|
|
84
|
+
cache_write: float # first-time cache-write premium (usually 1.25x input)
|
|
85
|
+
output: float # model output tokens
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# Published USD / MTok rates (as of v1.1.0 · 2026-04). Kept inline so
|
|
89
|
+
# nothing requires a network round-trip to estimate cost.
|
|
90
|
+
MODEL_PRICING: dict[str, ModelRates] = {
|
|
91
|
+
"claude-sonnet-4-6": {
|
|
92
|
+
"input": 3.00,
|
|
93
|
+
"cached_input": 0.30,
|
|
94
|
+
"cache_write": 3.75,
|
|
95
|
+
"output": 15.00,
|
|
96
|
+
},
|
|
97
|
+
"claude-haiku-4": {
|
|
98
|
+
"input": 0.80,
|
|
99
|
+
"cached_input": 0.08,
|
|
100
|
+
"cache_write": 1.00,
|
|
101
|
+
"output": 4.00,
|
|
102
|
+
},
|
|
103
|
+
# #py-m3 (#589): synthesize_overview actually invokes
|
|
104
|
+
# `claude-haiku-4-5-20251001` (the date-suffixed alias). Without
|
|
105
|
+
# this entry, cost-estimate code raises `ValueError: unknown model`.
|
|
106
|
+
# Same rate card as the bare claude-haiku-4 above.
|
|
107
|
+
"claude-haiku-4-5-20251001": {
|
|
108
|
+
"input": 0.80,
|
|
109
|
+
"cached_input": 0.08,
|
|
110
|
+
"cache_write": 1.00,
|
|
111
|
+
"output": 4.00,
|
|
112
|
+
},
|
|
113
|
+
"claude-opus-4": {
|
|
114
|
+
"input": 15.00,
|
|
115
|
+
"cached_input": 1.50,
|
|
116
|
+
"cache_write": 18.75,
|
|
117
|
+
"output": 75.00,
|
|
118
|
+
},
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
DEFAULT_MODEL = "claude-sonnet-4-6"
|
|
122
|
+
BATCH_STATE_FILENAME = ".llmwiki-batch-state.json"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# ─── Cached block / message builders ──────────────────────────────────
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class ContentBlock(TypedDict, total=False):
|
|
129
|
+
"""Anthropic message content block (subset we care about)."""
|
|
130
|
+
|
|
131
|
+
type: str
|
|
132
|
+
text: str
|
|
133
|
+
cache_control: dict[str, str]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def make_cached_block(text: str) -> ContentBlock:
|
|
137
|
+
"""Return a ``text`` content block with ``cache_control: ephemeral``.
|
|
138
|
+
|
|
139
|
+
The Anthropic cache header is placed on the *last* block you want
|
|
140
|
+
cached — everything up to and including that block becomes a single
|
|
141
|
+
cache key.
|
|
142
|
+
"""
|
|
143
|
+
return {
|
|
144
|
+
"type": "text",
|
|
145
|
+
"text": text,
|
|
146
|
+
"cache_control": dict(CACHE_CONTROL_EPHEMERAL),
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def make_plain_block(text: str) -> ContentBlock:
|
|
151
|
+
"""Return an un-cached ``text`` content block."""
|
|
152
|
+
return {"type": "text", "text": text}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@dataclass(frozen=True)
|
|
156
|
+
class CachedPrompt:
|
|
157
|
+
"""A prompt split into a cacheable prefix and a dynamic suffix.
|
|
158
|
+
|
|
159
|
+
``stable_prefix`` is everything that's identical across source files
|
|
160
|
+
(CLAUDE.md schema, current ``wiki/index.md``, current
|
|
161
|
+
``wiki/overview.md``). It gets the cache header.
|
|
162
|
+
|
|
163
|
+
``dynamic_suffix`` is the per-source content that changes every call
|
|
164
|
+
(the session body, slug, date, project). It never carries cache_control.
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
stable_prefix: str
|
|
168
|
+
dynamic_suffix: str
|
|
169
|
+
system: Optional[str] = None
|
|
170
|
+
|
|
171
|
+
def content_blocks(self) -> list[ContentBlock]:
|
|
172
|
+
"""Return the list of content blocks for the user message."""
|
|
173
|
+
blocks: list[ContentBlock] = []
|
|
174
|
+
if self.stable_prefix:
|
|
175
|
+
blocks.append(make_cached_block(self.stable_prefix))
|
|
176
|
+
if self.dynamic_suffix:
|
|
177
|
+
blocks.append(make_plain_block(self.dynamic_suffix))
|
|
178
|
+
return blocks
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def build_messages(prompt: CachedPrompt) -> list[dict[str, Any]]:
|
|
182
|
+
"""Render a :class:`CachedPrompt` into Anthropic's ``messages`` list.
|
|
183
|
+
|
|
184
|
+
Returns a single-message list; the real backend will pass it straight
|
|
185
|
+
to ``client.messages.create(messages=..., system=...)``.
|
|
186
|
+
"""
|
|
187
|
+
return [{"role": "user", "content": prompt.content_blocks()}]
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# ─── Token + cost estimation ──────────────────────────────────────────
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def estimate_tokens(text: str) -> int:
|
|
194
|
+
"""Rough token count via char/4 heuristic.
|
|
195
|
+
|
|
196
|
+
Slightly under-counts emoji-heavy text and over-counts code-heavy
|
|
197
|
+
text, but it's plenty accurate for a pre-spend sanity check. Real
|
|
198
|
+
counts come back in ``usage`` on each API response.
|
|
199
|
+
"""
|
|
200
|
+
if not text:
|
|
201
|
+
return 0
|
|
202
|
+
return max(1, len(text) // CHARS_PER_TOKEN)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@dataclass(frozen=True)
|
|
206
|
+
class CostEstimate:
|
|
207
|
+
"""Dollar + token breakdown for a single Anthropic call."""
|
|
208
|
+
|
|
209
|
+
model: str
|
|
210
|
+
cached_tokens: int # prefix (paid at cached_input rate on hit)
|
|
211
|
+
fresh_tokens: int # dynamic suffix (paid at input rate)
|
|
212
|
+
output_tokens: int # expected completion length
|
|
213
|
+
cache_hit: bool # True = re-use, False = first write
|
|
214
|
+
usd: float # total estimated dollars
|
|
215
|
+
|
|
216
|
+
def breakdown(self) -> dict[str, float]:
|
|
217
|
+
"""Return per-bucket dollar amounts. Useful for ``--estimate``."""
|
|
218
|
+
rates = MODEL_PRICING[self.model]
|
|
219
|
+
prefix_rate = rates["cached_input"] if self.cache_hit else rates["cache_write"]
|
|
220
|
+
return {
|
|
221
|
+
"prefix_usd": self.cached_tokens * prefix_rate / 1_000_000,
|
|
222
|
+
"fresh_usd": self.fresh_tokens * rates["input"] / 1_000_000,
|
|
223
|
+
"output_usd": self.output_tokens * rates["output"] / 1_000_000,
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def estimate_cost(
|
|
228
|
+
*,
|
|
229
|
+
cached_tokens: int,
|
|
230
|
+
fresh_tokens: int,
|
|
231
|
+
output_tokens: int,
|
|
232
|
+
model: str = DEFAULT_MODEL,
|
|
233
|
+
cache_hit: bool = True,
|
|
234
|
+
) -> CostEstimate:
|
|
235
|
+
"""Price out one API call given token counts.
|
|
236
|
+
|
|
237
|
+
Parameters
|
|
238
|
+
----------
|
|
239
|
+
cached_tokens : int
|
|
240
|
+
Tokens in the stable prefix.
|
|
241
|
+
fresh_tokens : int
|
|
242
|
+
Tokens in the per-source dynamic suffix.
|
|
243
|
+
output_tokens : int
|
|
244
|
+
Expected response length.
|
|
245
|
+
model : str
|
|
246
|
+
Model id from :data:`MODEL_PRICING`.
|
|
247
|
+
cache_hit : bool
|
|
248
|
+
``True`` to assume the prefix is already in cache (cheap),
|
|
249
|
+
``False`` to price the first-write premium.
|
|
250
|
+
"""
|
|
251
|
+
if model not in MODEL_PRICING:
|
|
252
|
+
raise ValueError(f"unknown model {model!r}; see MODEL_PRICING")
|
|
253
|
+
if cached_tokens < 0 or fresh_tokens < 0 or output_tokens < 0:
|
|
254
|
+
raise ValueError("token counts must be non-negative")
|
|
255
|
+
|
|
256
|
+
rates = MODEL_PRICING[model]
|
|
257
|
+
prefix_rate = rates["cached_input"] if cache_hit else rates["cache_write"]
|
|
258
|
+
usd = (
|
|
259
|
+
cached_tokens * prefix_rate
|
|
260
|
+
+ fresh_tokens * rates["input"]
|
|
261
|
+
+ output_tokens * rates["output"]
|
|
262
|
+
) / 1_000_000
|
|
263
|
+
|
|
264
|
+
return CostEstimate(
|
|
265
|
+
model=model,
|
|
266
|
+
cached_tokens=cached_tokens,
|
|
267
|
+
fresh_tokens=fresh_tokens,
|
|
268
|
+
output_tokens=output_tokens,
|
|
269
|
+
cache_hit=cache_hit,
|
|
270
|
+
usd=usd,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def format_estimate(est: CostEstimate) -> str:
|
|
275
|
+
"""Pretty-print a :class:`CostEstimate` for the ``--estimate`` flag."""
|
|
276
|
+
bd = est.breakdown()
|
|
277
|
+
hit_label = "cache hit" if est.cache_hit else "first write"
|
|
278
|
+
return (
|
|
279
|
+
f"Model: {est.model} ({hit_label})\n"
|
|
280
|
+
f" Prefix: {est.cached_tokens:>7,} tok ${bd['prefix_usd']:.4f}\n"
|
|
281
|
+
f" Fresh: {est.fresh_tokens:>7,} tok ${bd['fresh_usd']:.4f}\n"
|
|
282
|
+
f" Output: {est.output_tokens:>7,} tok ${bd['output_usd']:.4f}\n"
|
|
283
|
+
f" Total: ${est.usd:.4f}"
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def warn_prefix_too_small(cached_tokens: int) -> Optional[str]:
|
|
288
|
+
"""Return a one-line warning if the prefix is below the cache floor.
|
|
289
|
+
|
|
290
|
+
Anthropic silently ignores ``cache_control`` on prefixes below
|
|
291
|
+
:data:`MIN_CACHEABLE_TOKENS` tokens, so ``--estimate`` should flag
|
|
292
|
+
that the prefix isn't actually being cached.
|
|
293
|
+
"""
|
|
294
|
+
if cached_tokens < MIN_CACHEABLE_TOKENS:
|
|
295
|
+
return (
|
|
296
|
+
f"prefix is {cached_tokens} tok (< {MIN_CACHEABLE_TOKENS} min) — "
|
|
297
|
+
f"Anthropic will not cache it; savings estimate is best-case only."
|
|
298
|
+
)
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
# ─── Batch state persistence ──────────────────────────────────────────
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
@dataclass
|
|
306
|
+
class BatchJob:
|
|
307
|
+
"""One Anthropic ``message_batches`` submission in flight."""
|
|
308
|
+
|
|
309
|
+
batch_id: str
|
|
310
|
+
source_slugs: list[str] = field(default_factory=list)
|
|
311
|
+
submitted_at: str = "" # ISO-8601
|
|
312
|
+
status: str = "pending" # pending | completed | expired | failed
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@dataclass
|
|
316
|
+
class BatchState:
|
|
317
|
+
"""Persistent state for the batch ingest pipeline.
|
|
318
|
+
|
|
319
|
+
Mirrors ``.llmwiki-synth-state.json`` in spirit: small JSON file at
|
|
320
|
+
the repo root, safe to grep / ``cat`` to see what's in flight.
|
|
321
|
+
"""
|
|
322
|
+
|
|
323
|
+
pending: list[BatchJob] = field(default_factory=list)
|
|
324
|
+
completed: list[BatchJob] = field(default_factory=list)
|
|
325
|
+
|
|
326
|
+
def to_json(self) -> dict[str, Any]:
|
|
327
|
+
return {
|
|
328
|
+
"pending": [vars(b) for b in self.pending],
|
|
329
|
+
"completed": [vars(b) for b in self.completed],
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
@classmethod
|
|
333
|
+
def from_json(cls, data: dict[str, Any]) -> "BatchState":
|
|
334
|
+
# #py-l4 (#602): wrap the BatchJob(**b) construction so a
|
|
335
|
+
# corrupt entry (extra/missing keys → TypeError) doesn't leak
|
|
336
|
+
# past the callers that expect this constructor to either
|
|
337
|
+
# succeed or return a deterministic fallback. Mirror the
|
|
338
|
+
# load_batch_state() shape: print a warning, drop the bad
|
|
339
|
+
# entries, return whatever survived.
|
|
340
|
+
import sys as _sys
|
|
341
|
+
def _safe(rows):
|
|
342
|
+
kept = []
|
|
343
|
+
for b in rows:
|
|
344
|
+
try:
|
|
345
|
+
kept.append(BatchJob(**b))
|
|
346
|
+
except TypeError as e:
|
|
347
|
+
print(
|
|
348
|
+
f"warning: dropping malformed batch entry {b!r}: {e}",
|
|
349
|
+
file=_sys.stderr,
|
|
350
|
+
)
|
|
351
|
+
return kept
|
|
352
|
+
return cls(
|
|
353
|
+
pending=_safe(data.get("pending", [])),
|
|
354
|
+
completed=_safe(data.get("completed", [])),
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def batch_state_path(repo_root: Path) -> Path:
|
|
359
|
+
"""Return the on-disk path for the batch state file."""
|
|
360
|
+
return repo_root / BATCH_STATE_FILENAME
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def load_batch_state(repo_root: Path) -> BatchState:
|
|
364
|
+
"""Load batch state from disk, or an empty state if the file is
|
|
365
|
+
missing or corrupt (mirrors ``_load_state`` in ``synth/pipeline.py``)."""
|
|
366
|
+
path = batch_state_path(repo_root)
|
|
367
|
+
if not path.is_file():
|
|
368
|
+
return BatchState()
|
|
369
|
+
try:
|
|
370
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
371
|
+
except ValueError:
|
|
372
|
+
return BatchState()
|
|
373
|
+
return BatchState.from_json(data)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def save_batch_state(repo_root: Path, state: BatchState) -> Path:
|
|
377
|
+
"""Write batch state to disk and return the path."""
|
|
378
|
+
path = batch_state_path(repo_root)
|
|
379
|
+
path.write_text(
|
|
380
|
+
json.dumps(state.to_json(), indent=2, sort_keys=True),
|
|
381
|
+
encoding="utf-8",
|
|
382
|
+
)
|
|
383
|
+
return path
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def add_pending(state: BatchState, job: BatchJob) -> None:
|
|
387
|
+
"""Append a new pending job (dedup by batch_id)."""
|
|
388
|
+
if any(b.batch_id == job.batch_id for b in state.pending):
|
|
389
|
+
return
|
|
390
|
+
state.pending.append(job)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def mark_completed(state: BatchState, batch_id: str, *, status: str = "completed") -> bool:
|
|
394
|
+
"""Move a pending job into the completed list. Returns True if
|
|
395
|
+
the job was found and moved, False otherwise."""
|
|
396
|
+
for i, job in enumerate(state.pending):
|
|
397
|
+
if job.batch_id == batch_id:
|
|
398
|
+
job.status = status
|
|
399
|
+
state.completed.append(state.pending.pop(i))
|
|
400
|
+
return True
|
|
401
|
+
return False
|