specmodule 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm/__init__.py +21 -0
- llm/client.py +654 -0
- llm/config.py +213 -0
- module_harness/__init__.py +201 -0
- module_harness/align.py +39 -0
- module_harness/builtins.py +29 -0
- module_harness/checkpoint.py +336 -0
- module_harness/cli.py +1492 -0
- module_harness/command.py +115 -0
- module_harness/config.py +95 -0
- module_harness/consistency.py +123 -0
- module_harness/entry.py +74 -0
- module_harness/events.py +149 -0
- module_harness/feed.py +197 -0
- module_harness/graph_builder.py +334 -0
- module_harness/harness.py +181 -0
- module_harness/loader.py +215 -0
- module_harness/module.py +452 -0
- module_harness/outputfmt.py +139 -0
- module_harness/prompt.py +84 -0
- module_harness/query.py +216 -0
- module_harness/registry.py +180 -0
- module_harness/scaffold.py +404 -0
- module_harness/spec.py +209 -0
- module_harness/status.py +96 -0
- module_harness/store.py +482 -0
- module_harness/submodule.py +268 -0
- module_harness/templates/builtin/codereview.json +32 -0
- module_harness/templates/builtin/docwrite.json +30 -0
- module_harness/templates/builtin/summarize.json +24 -0
- module_harness/templates/builtin/translate.json +27 -0
- module_harness/translator.py +314 -0
- specmodule-0.1.0.dist-info/METADATA +321 -0
- specmodule-0.1.0.dist-info/RECORD +38 -0
- specmodule-0.1.0.dist-info/WHEEL +5 -0
- specmodule-0.1.0.dist-info/entry_points.txt +2 -0
- specmodule-0.1.0.dist-info/licenses/LICENSE +21 -0
- specmodule-0.1.0.dist-info/top_level.txt +2 -0
module_harness/module.py
ADDED
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
"""Module 编排器 — spec + template/tasklist → tasklist → runner。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import copy
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import os
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from tickflow.async_runner import AsyncRunner
|
|
16
|
+
from tickflow.persistence import NullBackend, SqliteBackend
|
|
17
|
+
|
|
18
|
+
from .spec import Spec, Tasklist
|
|
19
|
+
from .consistency import ConsistencyError, ConsistencyReport, ConsistencyReviewer
|
|
20
|
+
from .translator import Translator, TemplateLoader, TasklistValidator
|
|
21
|
+
from .graph_builder import TasklistTranslator
|
|
22
|
+
from .registry import HarnessRegistry
|
|
23
|
+
from .events import EventBus, ConsistencyReviewed
|
|
24
|
+
from .checkpoint import (
|
|
25
|
+
ModuleInputStore,
|
|
26
|
+
ResumeError,
|
|
27
|
+
check_resume_compat,
|
|
28
|
+
tasklist_from_dict,
|
|
29
|
+
tasklist_to_dict,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
log = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _persist_dir(module_id: str) -> Path:
|
|
36
|
+
"""``<工作目录>/.specmodule/runs/<run_id>/run.sqlite``(D9)。
|
|
37
|
+
|
|
38
|
+
run_id = module_id:一个任务一次运行一个子目录、一个独立 SQLite 数据库。
|
|
39
|
+
"""
|
|
40
|
+
return Path.cwd() / ".specmodule" / "runs" / module_id / "run.sqlite"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _status_path(module_id: str) -> Path:
|
|
44
|
+
"""``<工作目录>/.specmodule/runs/<module_id>/status.json``(roadmap #7)。
|
|
45
|
+
|
|
46
|
+
阶段级运行状态文件:与 run.sqlite 同目录,跨进程查询的轻量通道。
|
|
47
|
+
"""
|
|
48
|
+
return Path.cwd() / ".specmodule" / "runs" / module_id / "status.json"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Module:
|
|
52
|
+
"""SpecModule 的核心编排器。
|
|
53
|
+
|
|
54
|
+
spec + template → 翻译 → tasklist → tickflow Graph + registry → AsyncRunner
|
|
55
|
+
或 spec + tasklist(自定义)→ 校验 + 一致性审核 → AsyncRunner。
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
spec: dict[str, Any],
|
|
61
|
+
*,
|
|
62
|
+
template_name: str | None = None,
|
|
63
|
+
tasklist: Tasklist | None = None,
|
|
64
|
+
llm_client: Any,
|
|
65
|
+
event_bus: EventBus | None = None,
|
|
66
|
+
template_loader: TemplateLoader | None = None,
|
|
67
|
+
module_id: str | None = None,
|
|
68
|
+
registry: HarnessRegistry | None = None,
|
|
69
|
+
review_harness: str | None = "spec_tasklist_review",
|
|
70
|
+
keep_records: bool = True,
|
|
71
|
+
persist: bool = True,
|
|
72
|
+
status_file: bool = True,
|
|
73
|
+
modules: dict[str, Any] | None = None,
|
|
74
|
+
hooks: dict | None = None,
|
|
75
|
+
) -> None:
|
|
76
|
+
if (template_name is None) == (tasklist is None):
|
|
77
|
+
raise ValueError("template_name 与 tasklist 必须且只能传一个")
|
|
78
|
+
self.spec = Spec(spec)
|
|
79
|
+
self.template_name = template_name
|
|
80
|
+
self.tasklist = tasklist
|
|
81
|
+
self.review_harness = review_harness
|
|
82
|
+
self.keep_records = keep_records
|
|
83
|
+
# True(默认):构造 .specmodule/runs/<run_id>/run.sqlite 持久 backend(D9)
|
|
84
|
+
# False:快速模式——NullBackend 全内存,零落盘零 I/O(D7 语义正式化)
|
|
85
|
+
self.persist = persist
|
|
86
|
+
# True(默认):写 .specmodule/runs/<module_id>/status.json
|
|
87
|
+
# (阶段级,跨进程查询通道);False:零残留(快速模式可用)
|
|
88
|
+
self.status_file = status_file
|
|
89
|
+
self.review_result: ConsistencyReport | None = None
|
|
90
|
+
self.module_id = module_id or f"mod_{uuid.uuid4().hex[:8]}"
|
|
91
|
+
self._llm_client = llm_client
|
|
92
|
+
# submodule 引用解析表 {tasklist 名: SubModule 类}:TasklistValidator 校验
|
|
93
|
+
# submodule 节点(T2)与 TasklistTranslator 构建嵌套子图(T6)共用
|
|
94
|
+
self._modules = dict(modules or {})
|
|
95
|
+
# runner hooks 透传(观察通道):{hook名: async/sync 回调},构造
|
|
96
|
+
# runner 后注册。CLI 实时显示使用 on_tick_start/on_fire。
|
|
97
|
+
self._hooks = dict(hooks or {})
|
|
98
|
+
|
|
99
|
+
if registry is not None:
|
|
100
|
+
self._reg = registry
|
|
101
|
+
else:
|
|
102
|
+
self._reg = HarnessRegistry(
|
|
103
|
+
llm_client=llm_client,
|
|
104
|
+
event_bus=event_bus or EventBus.null(),
|
|
105
|
+
)
|
|
106
|
+
self._loader = template_loader or TemplateLoader()
|
|
107
|
+
self._translator = Translator(self._reg)
|
|
108
|
+
self._write_phase("idle")
|
|
109
|
+
|
|
110
|
+
# roadmap #5:runner 由 _build_runner_async 持有;快照/回滚 API 依赖它
|
|
111
|
+
self._runner: AsyncRunner | None = None
|
|
112
|
+
self._last_tasklist: Tasklist | None = None
|
|
113
|
+
self._input_store: ModuleInputStore | None = None
|
|
114
|
+
|
|
115
|
+
# ------------------------------------------------------------------
|
|
116
|
+
# 运行状态(roadmap #7)
|
|
117
|
+
# ------------------------------------------------------------------
|
|
118
|
+
|
|
119
|
+
def _write_phase(self, phase: str, error: str | None = None) -> None:
|
|
120
|
+
"""原子写 status.json(tmp + os.replace)。失败仅 log,不阻断运行。
|
|
121
|
+
|
|
122
|
+
phase 取值:idle/translating/reviewing/building/ready/running/
|
|
123
|
+
done/aborted/cancelled。status_file=False 时不写盘(零残留)。
|
|
124
|
+
"""
|
|
125
|
+
if not self.status_file:
|
|
126
|
+
return
|
|
127
|
+
path = _status_path(self.module_id)
|
|
128
|
+
tmp = path.with_suffix(".json.tmp")
|
|
129
|
+
try:
|
|
130
|
+
tmp.parent.mkdir(parents=True, exist_ok=True)
|
|
131
|
+
tmp.write_text(
|
|
132
|
+
json.dumps({
|
|
133
|
+
"module_id": self.module_id,
|
|
134
|
+
"phase": phase,
|
|
135
|
+
"error": error,
|
|
136
|
+
"updated_at": time.time(),
|
|
137
|
+
}, ensure_ascii=False),
|
|
138
|
+
encoding="utf-8",
|
|
139
|
+
)
|
|
140
|
+
os.replace(tmp, path)
|
|
141
|
+
except OSError:
|
|
142
|
+
log.exception("写 status.json 失败(不阻断运行): %s", path)
|
|
143
|
+
|
|
144
|
+
def build_runner(self) -> AsyncRunner:
|
|
145
|
+
"""执行翻译 → 构建 graph → 返回 AsyncRunner。
|
|
146
|
+
|
|
147
|
+
Note: 这是一个同步方法。在 async 上下文中直接使用
|
|
148
|
+
await module._build_runner_async() 或 await module.run()。
|
|
149
|
+
"""
|
|
150
|
+
loop = asyncio.new_event_loop()
|
|
151
|
+
try:
|
|
152
|
+
return loop.run_until_complete(self._build_runner_async())
|
|
153
|
+
except Exception as e:
|
|
154
|
+
self._write_phase("aborted", error=str(e))
|
|
155
|
+
raise
|
|
156
|
+
finally:
|
|
157
|
+
loop.close()
|
|
158
|
+
|
|
159
|
+
async def _build_runner_async(self) -> AsyncRunner:
|
|
160
|
+
"""异步版 build_runner。"""
|
|
161
|
+
if self.tasklist is not None:
|
|
162
|
+
self._write_phase("reviewing")
|
|
163
|
+
tasklist = self.tasklist
|
|
164
|
+
errors = TasklistValidator.validate(tasklist, self._reg, self._modules)
|
|
165
|
+
if errors:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
"tasklist 校验失败:\n" + "\n".join(f" - {e}" for e in errors)
|
|
168
|
+
)
|
|
169
|
+
if self.review_harness is not None:
|
|
170
|
+
report = await ConsistencyReviewer(
|
|
171
|
+
self._reg, self.review_harness
|
|
172
|
+
).review(self.spec, tasklist)
|
|
173
|
+
self.review_result = report
|
|
174
|
+
self._reg._event_bus.emit(ConsistencyReviewed(
|
|
175
|
+
timestamp=time.monotonic(), node="__review__", tick=0,
|
|
176
|
+
consistent=report.consistent,
|
|
177
|
+
suggestions=report.suggestions,
|
|
178
|
+
raw=report.raw,
|
|
179
|
+
))
|
|
180
|
+
if not report.consistent:
|
|
181
|
+
raise ConsistencyError(report)
|
|
182
|
+
else:
|
|
183
|
+
self._write_phase("translating")
|
|
184
|
+
template = self._loader.get(self.template_name)
|
|
185
|
+
if template is None:
|
|
186
|
+
raise ValueError(f"模板 '{self.template_name}' 未找到")
|
|
187
|
+
tasklist = await self._translator.translate(self.spec, template)
|
|
188
|
+
self._write_phase("building")
|
|
189
|
+
builder = TasklistTranslator(
|
|
190
|
+
self._reg, self.module_id,
|
|
191
|
+
modules=self._modules, llm_client=self._llm_client,
|
|
192
|
+
)
|
|
193
|
+
graph, reg = builder.build(tasklist, spec=self.spec)
|
|
194
|
+
self._last_tasklist = tasklist
|
|
195
|
+
backend = (
|
|
196
|
+
SqliteBackend(_persist_dir(self.module_id))
|
|
197
|
+
if self.persist
|
|
198
|
+
else NullBackend()
|
|
199
|
+
)
|
|
200
|
+
self._write_phase("ready")
|
|
201
|
+
runner = AsyncRunner(
|
|
202
|
+
graph,
|
|
203
|
+
registry=reg,
|
|
204
|
+
keep_records=self.keep_records,
|
|
205
|
+
backend=backend,
|
|
206
|
+
session_id=self.module_id,
|
|
207
|
+
)
|
|
208
|
+
self._runner = runner
|
|
209
|
+
for _hook_name, _cb in self._hooks.items():
|
|
210
|
+
_register = getattr(runner, _hook_name, None)
|
|
211
|
+
if callable(_register):
|
|
212
|
+
_register(_cb)
|
|
213
|
+
else:
|
|
214
|
+
log.warning("Module hooks: 未知 runner hook '%s'(忽略)", _hook_name)
|
|
215
|
+
return runner
|
|
216
|
+
|
|
217
|
+
# ------------------------------------------------------------------
|
|
218
|
+
# 快照/回滚(roadmap #5)
|
|
219
|
+
# ------------------------------------------------------------------
|
|
220
|
+
|
|
221
|
+
def _require_runner(self) -> AsyncRunner:
|
|
222
|
+
"""快照/回滚 API 前置守卫:runner 未构建时抛错。"""
|
|
223
|
+
if self._runner is None:
|
|
224
|
+
raise RuntimeError("尚未构建 runner——请先 build_runner() 或 run()")
|
|
225
|
+
return self._runner
|
|
226
|
+
|
|
227
|
+
def close(self) -> None:
|
|
228
|
+
"""释放 Module 持有的 SQLite 连接(``_input_store``,懒创建)。
|
|
229
|
+
|
|
230
|
+
run()/resume() 结束后可调用;幂等(重复调用安全)。再次 run()/resume()
|
|
231
|
+
会按需重新创建 store(``_archive_module_inputs`` 懒重建)。
|
|
232
|
+
|
|
233
|
+
注:runner 持有的 SqliteBackend 连接属于 runner 生命周期,由调用方
|
|
234
|
+
管理(与既有行为一致),本方法只关闭 Module 自己创建的
|
|
235
|
+
ModuleInputStore 连接,不触碰 runner。
|
|
236
|
+
"""
|
|
237
|
+
if self._input_store is not None:
|
|
238
|
+
self._input_store.close()
|
|
239
|
+
self._input_store = None
|
|
240
|
+
|
|
241
|
+
def snapshot(self) -> dict:
|
|
242
|
+
"""进程内全量快照:{spec, tasklist, runner_snapshot} 三件套。
|
|
243
|
+
|
|
244
|
+
深拷贝语义:修改返回的 dict 不影响 Module 状态。
|
|
245
|
+
"""
|
|
246
|
+
runner = self._require_runner()
|
|
247
|
+
assert self._last_tasklist is not None
|
|
248
|
+
snap = {
|
|
249
|
+
"spec": self.spec.to_dict(),
|
|
250
|
+
"tasklist": tasklist_to_dict(self._last_tasklist),
|
|
251
|
+
"runner": runner.snapshot(),
|
|
252
|
+
}
|
|
253
|
+
# to_dict/runner 快照均为浅拷贝(嵌套结构共享引用)——整体深拷贝
|
|
254
|
+
# 兑现 docstring 的深拷贝承诺(零 tickflow 修改)。
|
|
255
|
+
return copy.deepcopy(snap)
|
|
256
|
+
|
|
257
|
+
def restore(self, snap: dict) -> None:
|
|
258
|
+
"""回滚 runner 到快照,并恢复 spec/tasklist 字段。"""
|
|
259
|
+
runner = self._require_runner()
|
|
260
|
+
self.spec = Spec(snap["spec"])
|
|
261
|
+
self.tasklist = tasklist_from_dict(snap["tasklist"])
|
|
262
|
+
self._last_tasklist = self.tasklist
|
|
263
|
+
# 与 __init__ 的"template/tasklist 二选一"不变量一致:restore 后
|
|
264
|
+
# 走 tasklist 通道,template_name 不再持有
|
|
265
|
+
self.template_name = None
|
|
266
|
+
runner.restore(snap["runner"])
|
|
267
|
+
|
|
268
|
+
def checkpoint(self, label: str) -> None:
|
|
269
|
+
"""手动检查点(backend 表,永久保留)。透传 runner。"""
|
|
270
|
+
runner = self._require_runner()
|
|
271
|
+
if not self.persist:
|
|
272
|
+
raise RuntimeError("检查点需要 persist=True(fast mode 零持久化)")
|
|
273
|
+
runner.checkpoint(label)
|
|
274
|
+
|
|
275
|
+
def rollback_to(self, label: str) -> None:
|
|
276
|
+
"""进程内回退到命名检查点。透传 runner。"""
|
|
277
|
+
runner = self._require_runner()
|
|
278
|
+
if not self.persist:
|
|
279
|
+
raise RuntimeError("检查点需要 persist=True(fast mode 零持久化)")
|
|
280
|
+
runner.rollback_to(label)
|
|
281
|
+
|
|
282
|
+
def list_checkpoints(self) -> list[tuple[int, str | list[str], str]]:
|
|
283
|
+
"""全部检查点 (tick, fired 或 label, kind),按 tick 升序。
|
|
284
|
+
|
|
285
|
+
kind ∈ {"tick", "manual"}:tick = snapshots 表每 tick 最小快照
|
|
286
|
+
(fired 节点列表,历史审阅雏形);manual = checkpoint() 手动检查点
|
|
287
|
+
(label)。不依赖 runner——跨进程场景(新 Module 实例)也可查询。
|
|
288
|
+
"""
|
|
289
|
+
out: list[tuple[int, str | list[str], str]] = []
|
|
290
|
+
if self.persist:
|
|
291
|
+
backend = SqliteBackend(_persist_dir(self.module_id))
|
|
292
|
+
try:
|
|
293
|
+
for tick in backend.list_snapshots(self.module_id):
|
|
294
|
+
snap = backend.load_snapshot(self.module_id, tick)
|
|
295
|
+
if snap is None:
|
|
296
|
+
continue
|
|
297
|
+
out.append((tick, list(snap.get("fired", [])), "tick"))
|
|
298
|
+
out.extend(
|
|
299
|
+
(tick, label, "manual")
|
|
300
|
+
for label, tick in backend.list_checkpoints(self.module_id)
|
|
301
|
+
)
|
|
302
|
+
except Exception:
|
|
303
|
+
log.exception("检查点列表读取失败(忽略)")
|
|
304
|
+
finally:
|
|
305
|
+
backend.close()
|
|
306
|
+
return sorted(out, key=lambda item: item[0])
|
|
307
|
+
|
|
308
|
+
@staticmethod
|
|
309
|
+
def _resolve_target(backend: SqliteBackend, module_id: str, rollback_to: int | str) -> dict | None:
|
|
310
|
+
"""解析回退目标:tick 号 → snapshots 表;manual:xxx → checkpoints 表。
|
|
311
|
+
|
|
312
|
+
其他(非数字、非 manual 前缀)返回 None——调用方抛 KeyError。
|
|
313
|
+
"""
|
|
314
|
+
if isinstance(rollback_to, int) or (
|
|
315
|
+
isinstance(rollback_to, str) and rollback_to.isdigit()
|
|
316
|
+
):
|
|
317
|
+
return backend.load_snapshot(module_id, int(rollback_to))
|
|
318
|
+
if isinstance(rollback_to, str) and rollback_to.startswith("manual:"):
|
|
319
|
+
return backend.load_checkpoint(module_id, rollback_to)
|
|
320
|
+
return None
|
|
321
|
+
|
|
322
|
+
async def run(self, max_ticks: int = 100):
|
|
323
|
+
"""执行翻译 → 构建 → 运行。一步跑完。
|
|
324
|
+
|
|
325
|
+
persist=True 时:每 tick 由 tickflow ``_persist_tick`` 落盘最小快照,
|
|
326
|
+
并归档本次 spec/tasklist 到 module_inputs 表(``_archive_module_inputs``)。
|
|
327
|
+
"""
|
|
328
|
+
try:
|
|
329
|
+
runner = await self._build_runner_async()
|
|
330
|
+
except Exception as e:
|
|
331
|
+
self._write_phase("aborted", error=str(e))
|
|
332
|
+
raise
|
|
333
|
+
return await self._run_with_phases(runner, max_ticks)
|
|
334
|
+
|
|
335
|
+
async def _run_with_phases(self, runner: AsyncRunner, max_ticks: int) -> list:
|
|
336
|
+
"""归档本次输入 → 运行 → 按结果映射终态 phase(run/resume 共用)。"""
|
|
337
|
+
self._archive_module_inputs()
|
|
338
|
+
self._write_phase("running")
|
|
339
|
+
try:
|
|
340
|
+
firings = await runner.run_until_idle(max_ticks=max_ticks)
|
|
341
|
+
except asyncio.CancelledError:
|
|
342
|
+
self._write_phase("cancelled", error="cancelled")
|
|
343
|
+
raise
|
|
344
|
+
except Exception as e:
|
|
345
|
+
self._write_phase("aborted", error=str(e))
|
|
346
|
+
raise
|
|
347
|
+
else:
|
|
348
|
+
self._finalize_phase(runner)
|
|
349
|
+
return firings
|
|
350
|
+
|
|
351
|
+
def _archive_module_inputs(self) -> None:
|
|
352
|
+
"""归档本次运行的 spec/tasklist 到 module_inputs 表(警告 1 对比源)。
|
|
353
|
+
|
|
354
|
+
run()/resume() 共用(_run_with_phases 开头调用);resume 中位于
|
|
355
|
+
兼容性校验与 restore 之后——先读旧存档再覆盖,顺序正确。
|
|
356
|
+
"""
|
|
357
|
+
if not self.persist:
|
|
358
|
+
return
|
|
359
|
+
if self._input_store is None:
|
|
360
|
+
self._input_store = ModuleInputStore(self.module_id)
|
|
361
|
+
assert self._last_tasklist is not None
|
|
362
|
+
self._input_store.save_module_inputs(
|
|
363
|
+
self.spec.to_dict(), self._last_tasklist.to_dict()
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
def _finalize_phase(self, runner: AsyncRunner) -> None:
|
|
367
|
+
"""按 runner.status 映射终态 phase(run/resume 共用)。"""
|
|
368
|
+
from tickflow.runner import RunStatus
|
|
369
|
+
if runner.status == RunStatus.ABORTED:
|
|
370
|
+
self._write_phase("aborted", error=runner.cancel_reason or "aborted")
|
|
371
|
+
elif runner.status == RunStatus.CANCELLED:
|
|
372
|
+
self._write_phase("cancelled", error=runner.cancel_reason or "cancelled")
|
|
373
|
+
elif runner.status == RunStatus.FAILED:
|
|
374
|
+
self._write_phase("aborted", error="all nodes failed")
|
|
375
|
+
elif runner.status == RunStatus.RUNNING:
|
|
376
|
+
self._write_phase("running") # max_ticks 截断:仍在运行
|
|
377
|
+
else:
|
|
378
|
+
self._write_phase("done")
|
|
379
|
+
|
|
380
|
+
async def resume(self, rollback_to: int | str, max_ticks: int = 100):
|
|
381
|
+
"""跨进程续跑:从 tick 号/手动检查点恢复 + 用当前 spec/tasklist 重建未执行部分。
|
|
382
|
+
|
|
383
|
+
流程:回退目标解析(tick 号 → snapshots 表;manual:xxx → checkpoints
|
|
384
|
+
表)→ 新图全量重建 → 兼容性校验(硬错误拒绝,不触碰 runner)→
|
|
385
|
+
restore → 归档新输入 → 续跑。
|
|
386
|
+
|
|
387
|
+
要求 persist=True(快照依赖 SQLite backend)。
|
|
388
|
+
|
|
389
|
+
max_ticks 是绝对 tick 上限:从 restore 的 tick 起继续计数(如
|
|
390
|
+
restore 于 tick 95,则默认 100 只剩 5 个 tick 可跑)。
|
|
391
|
+
"""
|
|
392
|
+
if not self.persist:
|
|
393
|
+
raise RuntimeError(
|
|
394
|
+
"resume 需要 persist=True(快照依赖 SQLite backend)"
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
# 1. 回退目标解析 + 已执行节点(同一连接,避免多次打开 run.sqlite)
|
|
398
|
+
backend = SqliteBackend(_persist_dir(self.module_id))
|
|
399
|
+
try:
|
|
400
|
+
snap = self._resolve_target(backend, self.module_id, rollback_to)
|
|
401
|
+
if snap is None:
|
|
402
|
+
ticks = backend.list_snapshots(self.module_id)
|
|
403
|
+
manual = [label for label, _ in backend.list_checkpoints(self.module_id)]
|
|
404
|
+
raise KeyError(
|
|
405
|
+
f"回退目标 {rollback_to!r} 不存在"
|
|
406
|
+
f"(可用 tick: {ticks or '无'};manual: {manual or '无'})"
|
|
407
|
+
)
|
|
408
|
+
# 已执行节点:firings 表中 tick < 快照 tick 的去重节点(S3 后
|
|
409
|
+
# 快照不再含 edges 窗口)。快照 tick N 在 tick N-1 结束后落盘,
|
|
410
|
+
# tick == N 的 firing 属于 restore 后会被重跑的部分,不算已执行。
|
|
411
|
+
# 注:firings 表按 module_id 累积(跨多次 run),前一轮 run 的
|
|
412
|
+
# 记录也会计入——仅影响提示性警告 1/3 的准确性,不影响硬错误。
|
|
413
|
+
executed_nodes = {
|
|
414
|
+
d["node"] for d in backend.list_firings(self.module_id)
|
|
415
|
+
if d.get("node") and int(d.get("tick", 0)) < int(snap.get("tick", 0))
|
|
416
|
+
}
|
|
417
|
+
finally:
|
|
418
|
+
backend.close()
|
|
419
|
+
|
|
420
|
+
# 2. 旧输入存档(警告 1 对比源;覆盖前读取)
|
|
421
|
+
store = ModuleInputStore(self.module_id)
|
|
422
|
+
try:
|
|
423
|
+
old_inputs = store.load_module_inputs()
|
|
424
|
+
finally:
|
|
425
|
+
store.close()
|
|
426
|
+
|
|
427
|
+
# 3. 新 spec/tasklist 全量重建(含校验 + 一致性审核)
|
|
428
|
+
try:
|
|
429
|
+
runner = await self._build_runner_async()
|
|
430
|
+
except Exception as e:
|
|
431
|
+
self._write_phase("aborted", error=str(e))
|
|
432
|
+
raise
|
|
433
|
+
|
|
434
|
+
# 4. 兼容性校验(构造 runner 后、restore 前;硬错误拒绝且不触碰状态)
|
|
435
|
+
marking = snap.get("marking") or {}
|
|
436
|
+
old_tl = tasklist_from_dict(old_inputs["tasklist"]) if old_inputs else None
|
|
437
|
+
check = check_resume_compat(
|
|
438
|
+
self._last_tasklist, runner.graph, executed_nodes,
|
|
439
|
+
old_tasklist=old_tl,
|
|
440
|
+
marking_slots=marking.get("slots"),
|
|
441
|
+
armed_starts=marking.get("armed_starts"),
|
|
442
|
+
)
|
|
443
|
+
for w in check.warnings:
|
|
444
|
+
log.warning("resume 兼容性警告: %s", w)
|
|
445
|
+
if check.hard_errors:
|
|
446
|
+
self._write_phase("aborted", error="resume 兼容性校验失败")
|
|
447
|
+
raise ResumeError(check.hard_errors)
|
|
448
|
+
|
|
449
|
+
# 5. restore + 续跑(phase 写盘与 run() 共用;不再 remap_graph——
|
|
450
|
+
# restore 已设好 marking,同图 remap 是 no-op,C2)
|
|
451
|
+
runner.restore(snap)
|
|
452
|
+
return await self._run_with_phases(runner, max_ticks)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""OutputFormat 输出格式约束定义 + OutputValidator 校验器。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any, Callable, Literal
|
|
9
|
+
|
|
10
|
+
from tickflow import Failure
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class OutputFormat:
|
|
15
|
+
"""输出格式约束定义。
|
|
16
|
+
|
|
17
|
+
``type`` 为 "json_object" 时要求合法 JSON;
|
|
18
|
+
"json_schema" 时还需 ``schema`` 校验通过;
|
|
19
|
+
"text" 时不做校验,直接返回原文本。
|
|
20
|
+
"""
|
|
21
|
+
type: Literal["json_object", "json_schema", "text"]
|
|
22
|
+
schema: dict[str, Any] | None = None
|
|
23
|
+
instruction: str | None = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# ── 内置提取器 ──────────────────────────────────────────────────
|
|
27
|
+
|
|
28
|
+
def _strip_markdown_fences(raw: str) -> str | None:
|
|
29
|
+
"""去除 ```json ... ``` 包裹。"""
|
|
30
|
+
pattern = r'```(?:json)?\s*\n?(.*?)\n?```'
|
|
31
|
+
m = re.search(pattern, raw, re.DOTALL)
|
|
32
|
+
if m:
|
|
33
|
+
return m.group(1).strip()
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _extract_first_json(raw: str) -> str | None:
|
|
38
|
+
"""匹配第一个完整 JSON 对象或数组。"""
|
|
39
|
+
# 先尝试 match 对象
|
|
40
|
+
m = re.search(r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', raw, re.DOTALL)
|
|
41
|
+
if m:
|
|
42
|
+
return m.group(0)
|
|
43
|
+
# 再尝试 match 数组
|
|
44
|
+
m = re.search(r'\[[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]', raw, re.DOTALL)
|
|
45
|
+
if m:
|
|
46
|
+
return m.group(0)
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _strip_trailing_junk(raw: str) -> str | None:
|
|
51
|
+
"""从末尾逐步截断非 JSON 字符,尝试解析。"""
|
|
52
|
+
s = raw.strip()
|
|
53
|
+
while s:
|
|
54
|
+
try:
|
|
55
|
+
json.loads(s)
|
|
56
|
+
return s
|
|
57
|
+
except json.JSONDecodeError:
|
|
58
|
+
pass
|
|
59
|
+
s = s[:-1]
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ── OutputValidator ──────────────────────────────────────────────
|
|
64
|
+
|
|
65
|
+
class OutputValidator:
|
|
66
|
+
"""输出格式校验器。
|
|
67
|
+
|
|
68
|
+
校验流程:先直接解析,失败则逐个尝试提取器;提取再失败 → Failure(type="llm")。
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(self, fmt: OutputFormat) -> None:
|
|
72
|
+
self.fmt = fmt
|
|
73
|
+
self._extractors: list[Callable[[str], str | None]] = [
|
|
74
|
+
_strip_markdown_fences,
|
|
75
|
+
_extract_first_json,
|
|
76
|
+
_strip_trailing_junk,
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
def prompt_instruction(self) -> str:
|
|
80
|
+
"""生成注入到 user prompt 的格式约束文本。"""
|
|
81
|
+
if self.fmt.instruction is not None:
|
|
82
|
+
return self.fmt.instruction
|
|
83
|
+
if self.fmt.type == "text":
|
|
84
|
+
return ""
|
|
85
|
+
if self.fmt.type == "json_object":
|
|
86
|
+
return "请输出合法 JSON,不要包含任何解释或其他文本。"
|
|
87
|
+
if self.fmt.type == "json_schema":
|
|
88
|
+
schema_text = json.dumps(self.fmt.schema, ensure_ascii=False, indent=2)
|
|
89
|
+
return (
|
|
90
|
+
"请严格按照以下 JSON Schema 输出,不要包含任何解释或其他文本:\n"
|
|
91
|
+
f"```json\n{schema_text}\n```"
|
|
92
|
+
)
|
|
93
|
+
return ""
|
|
94
|
+
|
|
95
|
+
def validate(self, raw: str) -> Any:
|
|
96
|
+
"""校验并返回解析值,或 Failure(type="llm")。"""
|
|
97
|
+
if self.fmt.type == "text":
|
|
98
|
+
return raw
|
|
99
|
+
|
|
100
|
+
# Step 1: 直接解析
|
|
101
|
+
parsed, error = self._try_parse(raw)
|
|
102
|
+
if parsed is not None and error is None:
|
|
103
|
+
return parsed
|
|
104
|
+
|
|
105
|
+
# Step 2: 逐个尝试提取器
|
|
106
|
+
for extractor in self._extractors:
|
|
107
|
+
extracted = extractor(raw)
|
|
108
|
+
if extracted is not None:
|
|
109
|
+
parsed, error = self._try_parse(extracted)
|
|
110
|
+
if parsed is not None and error is None:
|
|
111
|
+
return parsed
|
|
112
|
+
|
|
113
|
+
return Failure(
|
|
114
|
+
f"输出格式校验失败:{error or '所有提取器均未能修复输出'}",
|
|
115
|
+
type="llm",
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
def register_extractor(self, fn: Callable[[str], str | None]) -> None:
|
|
119
|
+
"""注册自定义提取策略。插入到内置提取器之前(优先尝试)。"""
|
|
120
|
+
self._extractors.insert(0, fn)
|
|
121
|
+
|
|
122
|
+
def _try_parse(self, text: str) -> tuple[Any, str | None]:
|
|
123
|
+
"""尝试 json.loads + 可选的 schema 校验。返回 (parsed, error)。"""
|
|
124
|
+
try:
|
|
125
|
+
parsed = json.loads(text)
|
|
126
|
+
except json.JSONDecodeError as e:
|
|
127
|
+
return None, str(e)
|
|
128
|
+
|
|
129
|
+
if self.fmt.type == "json_schema" and self.fmt.schema is not None:
|
|
130
|
+
try:
|
|
131
|
+
import jsonschema
|
|
132
|
+
jsonschema.validate(parsed, self.fmt.schema)
|
|
133
|
+
except ImportError:
|
|
134
|
+
# jsonschema 未安装时跳过 schema 校验,仅保证是 JSON
|
|
135
|
+
pass
|
|
136
|
+
except jsonschema.ValidationError as e:
|
|
137
|
+
return None, str(e)
|
|
138
|
+
|
|
139
|
+
return parsed, None
|
module_harness/prompt.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# module_harness/prompt.py
|
|
2
|
+
"""三层 prompt 渲染 + 关键词替换。"""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from tickflow.views import DictView, Missing
|
|
10
|
+
|
|
11
|
+
from .config import HarnessConfig
|
|
12
|
+
|
|
13
|
+
# 常量正则模块级编译一次(render 每 harness 节点每 tick 调用)。
|
|
14
|
+
_PLACEHOLDER_RE = re.compile(r"\{(\w+)\}")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class PromptRenderer:
|
|
18
|
+
"""三层 prompt 拼接 + 关键词替换。
|
|
19
|
+
|
|
20
|
+
数据来源:
|
|
21
|
+
Layer 1: config.prompt_core — 核心提示词模板,含 {key} 占位符
|
|
22
|
+
Layer 2: config.prompt_modes[mode] — 由 Task promptmode 选出的动态 prompt
|
|
23
|
+
Layer 3: prompt_extra — Task prompt 字段,人工注入部分
|
|
24
|
+
|
|
25
|
+
关键词替换:模板中的 {key} 从 DictView 取值(view.key.value)。
|
|
26
|
+
未匹配的 key 保留原样(不隐藏问题)。
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, config: HarnessConfig) -> None:
|
|
30
|
+
self.config = config
|
|
31
|
+
|
|
32
|
+
def render(
|
|
33
|
+
self,
|
|
34
|
+
view: DictView,
|
|
35
|
+
*,
|
|
36
|
+
promptmode: str | None = None,
|
|
37
|
+
prompt_extra: str | None = None,
|
|
38
|
+
extra_values: dict[str, Any] | None = None,
|
|
39
|
+
) -> str:
|
|
40
|
+
"""渲染最终 user prompt。
|
|
41
|
+
|
|
42
|
+
``extra_values``:占位符兜底值(如 spec 字段常量),
|
|
43
|
+
view 中缺失的 key 从该 dict 取值。
|
|
44
|
+
"""
|
|
45
|
+
parts: list[str] = []
|
|
46
|
+
|
|
47
|
+
# Layer 1: 核心提示词
|
|
48
|
+
parts.append(self.config.prompt_core)
|
|
49
|
+
|
|
50
|
+
# Layer 2: 由 promptmode 选出的动态 prompt
|
|
51
|
+
if promptmode is not None:
|
|
52
|
+
mode_text = self.config.prompt_modes[promptmode]
|
|
53
|
+
parts.append(mode_text)
|
|
54
|
+
|
|
55
|
+
# Layer 3: 人工注入
|
|
56
|
+
if prompt_extra:
|
|
57
|
+
parts.append(prompt_extra)
|
|
58
|
+
|
|
59
|
+
combined = "\n\n".join(parts)
|
|
60
|
+
return self._substitute(combined, view, extra_values)
|
|
61
|
+
|
|
62
|
+
def _substitute(
|
|
63
|
+
self,
|
|
64
|
+
template: str,
|
|
65
|
+
view: DictView,
|
|
66
|
+
extra_values: dict[str, Any] | None = None,
|
|
67
|
+
) -> str:
|
|
68
|
+
"""替换模板中的 {key} 占位符为 view 中的值。"""
|
|
69
|
+
|
|
70
|
+
def _replacer(m: re.Match) -> str:
|
|
71
|
+
key = m.group(1)
|
|
72
|
+
try:
|
|
73
|
+
val = view[key].value
|
|
74
|
+
except (KeyError, AttributeError):
|
|
75
|
+
if extra_values and key in extra_values:
|
|
76
|
+
return str(extra_values[key])
|
|
77
|
+
return m.group(0) # 保留原样
|
|
78
|
+
if val is Missing:
|
|
79
|
+
if extra_values and key in extra_values:
|
|
80
|
+
return str(extra_values[key])
|
|
81
|
+
return m.group(0)
|
|
82
|
+
return str(val)
|
|
83
|
+
|
|
84
|
+
return _PLACEHOLDER_RE.sub(_replacer, template)
|