arga-clone 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
argaclone/store.py ADDED
@@ -0,0 +1,461 @@
1
+ """Run 生命周期与状态的唯一持有者。
2
+
3
+ 设计要点(对应 Arga 的三条硬语义):
4
+ 1. per-run 隔离:每个 run_id 有自己的一份 provider 状态,互不可见。
5
+ 2. 快照式 reset:provision 完成时打一份 baseline,reset 就是回滚到它,
6
+ 不是"重建一个空环境"——所以 reset 之后看起来跟刚 provision 完一模一样。
7
+ 3. 显式 TTL:过期就 410,不静默存活。
8
+
9
+ 共享实体层(shared.people)是复刻品的价值所在:
10
+ 同一个自然人在 Slack 和 Stripe 里必须是同一个人,否则"多系统交叉核对"这类
11
+ 任务根本无从判定。这里用 email 做弱身份键,两端写入时自动关联。
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import copy
17
+ import json
18
+ import fcntl
19
+ import os
20
+ import threading
21
+ import time
22
+ import uuid
23
+ from dataclasses import dataclass, field
24
+ from pathlib import Path
25
+ from typing import Any
26
+
27
+
28
+ class RunExpired(Exception):
29
+ """run 已过期。调用方应回 410 而不是 404——语义不同,别混。"""
30
+
31
+
32
+ class RunNotFound(Exception):
33
+ """run_id 不存在。"""
34
+
35
+
36
+ class StaleRunVersion(Exception):
37
+ """手里的 run 已经过期:盘上的版本号比这份新。
38
+
39
+ 这是**乐观并发**的失败信号,不是错误状态的崩溃。它存在的理由:
40
+ 两个进程各持一份同一 run 时,后写的那个会把对方的改动整段抹掉——
41
+ 而抹掉之后的读数是"世界看起来没问题、只是少了一次操作",
42
+ 这种缺陷在评测里无法与"agent 没做"区分。
43
+ """
44
+
45
+
46
+ def new_id(prefix: str, n: int = 24) -> str:
47
+ """生成 provider 风格的长 id。Stripe 是 `cus_` + 14~24 位,Slack 是 `U` + 大写串。"""
48
+ return prefix + uuid.uuid4().hex[:n]
49
+
50
+
51
+ @dataclass
52
+ class Run:
53
+ run_id: str
54
+ twins: list[str]
55
+ ttl_seconds: int
56
+ created_at: float
57
+ # provider 名 → 该 provider 的全部状态(纯 dict,方便 deepcopy 与 JSON 化)
58
+ state: dict[str, dict[str, Any]] = field(default_factory=dict)
59
+ # provision 完成时打的基线快照;reset 的目标
60
+ baseline: dict[str, dict[str, Any]] = field(default_factory=dict)
61
+ # stub 命中日志:provider → {endpoint_template: hits}
62
+ stub_hits: dict[str, dict[str, int]] = field(default_factory=dict)
63
+ # 其中"twin 自己声明过不打算实现"的那部分。
64
+ # 与 stub_hits 分开记而不是事后按名单过滤:声明与命中必须成对落盘,
65
+ # 否则改一次声明就会让历史报表变样(过去的读数不可复算)。
66
+ declared_stub_hits: dict[str, dict[str, int]] = field(default_factory=dict)
67
+ # 第三桶:请求的端点**真实 API 里根本不存在**(agent 自己编的,或把 A 家的路径
68
+ # 打到了 B 家的 Host 上)。它既不是"我们欠着的活",也不该给假成功 ——
69
+ # 真实 provider 会 404,twin 照做。混进 stub_hits 会让欠账看起来比实际大。
70
+ absent_hits: dict[str, dict[str, int]] = field(default_factory=dict)
71
+ # 命中流水(带时间与完整路径)。GitHub 那套 admin 报告要 recent 20 条,
72
+ # 但流水本身对任何 provider 都有用:只保留计数就无法回答"是哪次调用撞的"。
73
+ stub_recent: list[dict[str, Any]] = field(default_factory=list)
74
+ locked: bool = False
75
+ extras: dict[str, Any] = field(default_factory=dict)
76
+ # 落盘版本号:每次成功写盘 +1。用来做**乐观并发**——两个进程各自拿着一份
77
+ # run 时,后写的那个会发现"我手里的是第 N 版,盘上已经是 N+1 版",
78
+ # 于是报错而不是把对方的改动抹掉。
79
+ version: int = 0
80
+ # 读到这份 run 时盘上的版本号。测试与调试时改它没有意义,所以不序列化。
81
+ loaded_version: int = field(default=0, compare=False)
82
+
83
+ def to_dict(self) -> dict[str, Any]:
84
+ return {
85
+ "run_id": self.run_id,
86
+ "twins": list(self.twins),
87
+ "ttl_seconds": self.ttl_seconds,
88
+ "created_at": self.created_at,
89
+ "state": self.state,
90
+ "baseline": self.baseline,
91
+ "stub_hits": self.stub_hits,
92
+ "declared_stub_hits": self.declared_stub_hits,
93
+ "absent_hits": self.absent_hits,
94
+ "stub_recent": self.stub_recent,
95
+ "locked": self.locked,
96
+ "extras": self.extras,
97
+ "version": self.version,
98
+ }
99
+
100
+ @classmethod
101
+ def from_dict(cls, payload: dict[str, Any]) -> "Run":
102
+ return cls(
103
+ run_id=payload["run_id"],
104
+ twins=list(payload.get("twins") or []),
105
+ ttl_seconds=int(payload.get("ttl_seconds") or 0),
106
+ created_at=float(payload.get("created_at") or 0.0),
107
+ state=payload.get("state") or {},
108
+ baseline=payload.get("baseline") or {},
109
+ stub_hits=payload.get("stub_hits") or {},
110
+ declared_stub_hits=payload.get("declared_stub_hits") or {},
111
+ absent_hits=payload.get("absent_hits") or {},
112
+ stub_recent=payload.get("stub_recent") or [],
113
+ locked=bool(payload.get("locked")),
114
+ extras=payload.get("extras") or {},
115
+ version=int(payload.get("version") or 0),
116
+ # 刚读进来的这份与盘上同版本:第一次写盘不算冲突。
117
+ loaded_version=int(payload.get("version") or 0),
118
+ )
119
+
120
+ @property
121
+ def expires_at(self) -> float:
122
+ return self.created_at + self.ttl_seconds
123
+
124
+ def expired(self, now: float | None = None) -> bool:
125
+ return (now if now is not None else time.time()) >= self.expires_at
126
+
127
+ def remaining(self, now: float | None = None) -> int:
128
+ return max(0, int(self.expires_at - (now if now is not None else time.time())))
129
+
130
+
131
+ class Store:
132
+ """状态库。单锁足够——这是仿真器,吞吐不是它的卖点。
133
+
134
+ 落盘是**可选的镜像**,不是事务日志:
135
+ - 给了 `directory` 就每个 run 存一份 JSON,进程重启后能接着用;
136
+ - 写盘走"临时文件 → 回读校验 → os.replace",所以崩在中间只会留下上一份完好的快照,
137
+ 不会留下半个文件;
138
+ - 没有 `directory` 就退回纯内存(默认),行为与以前完全一致。
139
+
140
+ 为什么不默认落盘:评测要跑几百个 episode,每个都留一份世界状态只会把磁盘塞满,
141
+ 而且那些 run 过期即废。**落盘是给"要跨重启的调试与演示"用的,不是给批量评测用的。**
142
+ """
143
+
144
+ def __init__(self, directory: Path | str | None = None) -> None:
145
+ self._lock = threading.RLock()
146
+ self._runs: dict[str, Run] = {}
147
+ self.directory = Path(directory) if directory is not None else None
148
+ # 启动时读不动的文件记在这里,而不是静默跳过:一个坏文件说明
149
+ # 上一轮写过一半或有人手改坏了,这件事本身要被看见。
150
+ self.load_errors: list[str] = []
151
+ if self.directory is not None:
152
+ self.directory.mkdir(parents=True, exist_ok=True)
153
+ self._load_all()
154
+
155
+ # ---------- 生命周期 ----------
156
+
157
+ def create(
158
+ self, twins: list[str], ttl_seconds: int, strict_tokens: bool = False
159
+ ) -> Run:
160
+ with self._lock:
161
+ run_id = uuid.uuid4().hex[:12]
162
+ run = Run(
163
+ run_id=run_id,
164
+ twins=list(twins),
165
+ ttl_seconds=ttl_seconds,
166
+ created_at=time.time(),
167
+ )
168
+ # 每个 run 初始就带一个 shared 命名空间
169
+ run.state["shared"] = {"people": {}}
170
+ # run-scoped 凭据:与 run_id 绑定,用来让"只替换 base URL 与密钥"
171
+ # 这套接入方式成立。真实产品同样在 provision 时下发凭据,
172
+ # 否则 agent 只能靠猜或者改双份配置。
173
+ run.extras["credentials"] = {
174
+ "stripe": f"sk_test_twin_{run_id}",
175
+ "slack": f"xoxb-twin-{run_id}",
176
+ "github": f"ghp_twin{run_id}",
177
+ # 后加的 provider 也必须有 run-scoped 凭据,否则"只替换 base URL
178
+ # 与密钥"这套接入承诺对它们不成立
179
+ "jira": f"jira-twin-{run_id}",
180
+ "linear": f"lin_api_twin_{run_id}",
181
+ "notion": f"ntn_twin_{run_id}",
182
+ "hubspot": f"pat-hi-twin-{run_id}",
183
+ # Salesforce 的 session token 长这样:`00D...!AQ4AQ...`。
184
+ # 前缀不是装饰——jsforce 与不少客户端会按 `!` 分段解析它。
185
+ "salesforce": f"00Dtw1n000000AAA!AQ4AQtwin{run_id}",
186
+ # Google OAuth access token 形状(ya29. 开头)。gmail/calendar/drive
187
+ # 共用同一个 token —— 真 Google 也是这样,一个 token 覆盖多个 API。
188
+ "gmail": f"ya29.twin-{run_id}",
189
+ "calendar": f"ya29.twin-{run_id}",
190
+ "drive": f"ya29.twin-{run_id}",
191
+ }
192
+ # 默认关闭。开着的含义见 Gateway._strict_credential_check:
193
+ # token 必须逐字等于上面这一张表里的值。
194
+ run.extras["strict_tokens"] = bool(strict_tokens)
195
+ self._runs[run_id] = run
196
+ self._save(run)
197
+ return run
198
+
199
+ def get(self, run_id: str, *, allow_expired: bool = False) -> Run:
200
+ with self._lock:
201
+ run = self._runs.get(run_id)
202
+ if run is None:
203
+ raise RunNotFound(run_id)
204
+ if not allow_expired and run.expired():
205
+ raise RunExpired(run_id)
206
+ return run
207
+
208
+ def drop(self, run_id: str) -> None:
209
+ with self._lock:
210
+ self._runs.pop(run_id, None)
211
+ path = self._path(run_id)
212
+ if path is not None:
213
+ try:
214
+ path.unlink()
215
+ except FileNotFoundError:
216
+ pass
217
+
218
+ def save(self, run: Run) -> None:
219
+ """外部(网关)改完 state 后调它落盘。没配 directory 时是空操作。"""
220
+ self._save(run)
221
+
222
+ def all_runs(self) -> list[Run]:
223
+ with self._lock:
224
+ return list(self._runs.values())
225
+
226
+ def gc(self) -> int:
227
+ """清掉过期的 run。返回清掉的数量。故意不自动调用——过期要能被观测到(410),
228
+ 立刻删掉会让过期变成 404,测不出 TTL 逻辑。"""
229
+ with self._lock:
230
+ dead = [rid for rid, r in self._runs.items() if r.expired()]
231
+ for rid in dead:
232
+ del self._runs[rid]
233
+ return len(dead)
234
+
235
+ # ---------- 快照 ----------
236
+
237
+ def snapshot(self, run_id: str) -> None:
238
+ """把当前状态冻结成 baseline。provision 结束时调用一次。"""
239
+ with self._lock:
240
+ run = self.get(run_id)
241
+ run.baseline = copy.deepcopy(run.state)
242
+ self._save(run)
243
+
244
+ def reset(self, run_id: str) -> None:
245
+ """回到 baseline。注意 stub_hits 不清——它是观测数据不是业务状态。"""
246
+ with self._lock:
247
+ run = self.get(run_id)
248
+ run.state = copy.deepcopy(run.baseline)
249
+ run.created_at = time.time() # reset 重新计时,等同"再来一次"
250
+ self._save(run)
251
+
252
+ def extend(self, run_id: str, ttl_seconds: int) -> None:
253
+ with self._lock:
254
+ run = self.get(run_id)
255
+ run.ttl_seconds = ttl_seconds
256
+ run.created_at = time.time()
257
+ self._save(run)
258
+
259
+ def lock(self, run_id: str) -> None:
260
+ with self._lock:
261
+ run = self.get(run_id)
262
+ run.locked = True
263
+ self._save(run)
264
+
265
+ # ---------- 共享实体层 ----------
266
+
267
+ def link_person(self, run_id: str, email: str, system: str, entity_id: str) -> None:
268
+ """把 (system, entity_id) 挂到 email 这个自然人身上。两端都调,就自动对齐。
269
+
270
+ 同一个 email 在同一个系统里可能有多个档案 —— 那正是"重复账单""重复线索"
271
+ 这类题要考的东西。所以这里 **first-wins + 记 duplicates**,不覆盖:
272
+ 覆盖会让共享实体层在 seed 阶段就把题目要考的不一致抹平,
273
+ agent 再去查时两个系统都只看到一个档案,题目变得不可解而没人知道。
274
+ """
275
+ with self._lock:
276
+ run = self.get(run_id)
277
+ people = run.state["shared"].setdefault("people", {})
278
+ person = people.setdefault(
279
+ email.lower(), {"email": email.lower(), "links": {}, "duplicates": {}}
280
+ )
281
+ links = person["links"]
282
+ dups = person.setdefault("duplicates", {})
283
+ if system not in links:
284
+ links[system] = entity_id
285
+ elif links[system] != entity_id:
286
+ bucket = dups.setdefault(system, [])
287
+ if entity_id not in bucket:
288
+ bucket.append(entity_id)
289
+ self._save(run)
290
+
291
+ def person_for(self, run_id: str, email: str) -> dict[str, Any] | None:
292
+ with self._lock:
293
+ run = self.get(run_id)
294
+ return run.state["shared"].get("people", {}).get(email.lower())
295
+
296
+ # ---------- stub 观测 ----------
297
+
298
+ def record_stub(
299
+ self, run_id: str, provider: str, endpoint: str, planned: bool = False
300
+ ) -> None:
301
+ """记一次 stub 命中。
302
+
303
+ `planned=True` 表示 twin 主动声明过"这个端点我不打算实现"(见
304
+ `Twin.declared_stubs`)。两类命中都算 stub(因为返回的都是假数据),
305
+ 但报表必须能拆开:一类是"还欠着的活",另一类是"画了线的边界"。
306
+ 混在一起看,就会把已经想清楚的取舍当成未完成的工程量。
307
+ """
308
+ with self._lock:
309
+ run = self.get(run_id, allow_expired=True)
310
+ hits = run.stub_hits.setdefault(provider, {})
311
+ hits[endpoint] = hits.get(endpoint, 0) + 1
312
+ if planned:
313
+ dh = run.declared_stub_hits.setdefault(provider, {})
314
+ dh[endpoint] = dh.get(endpoint, 0) + 1
315
+ run.stub_recent.append(
316
+ {
317
+ "at": time.time(),
318
+ "provider": provider,
319
+ "endpoint": endpoint,
320
+ "planned": planned,
321
+ }
322
+ )
323
+ # 只留最近 200 条:这是观测流水不是审计日志,无限增长没有收益
324
+ if len(run.stub_recent) > 200:
325
+ del run.stub_recent[:-200]
326
+ self._save(run)
327
+
328
+ def record_absent(self, run_id: str, provider: str, endpoint: str) -> None:
329
+ """记一次"真实 API 也没有这个端点"。
330
+
331
+ 与 stub 分开记的理由见 `Run.absent_hits` 的注释:这是 agent 的错,
332
+ 不是环境的债。合成一个数会让人得出"还欠 70 个端点"的错误结论。
333
+ """
334
+ with self._lock:
335
+ run = self.get(run_id, allow_expired=True)
336
+ hits = run.absent_hits.setdefault(provider, {})
337
+ hits[endpoint] = hits.get(endpoint, 0) + 1
338
+ self._save(run)
339
+
340
+ def stub_report(self, run_id: str, provider: str | None = None) -> dict[str, Any]:
341
+ with self._lock:
342
+ run = self.get(run_id, allow_expired=True)
343
+ source = (
344
+ {provider: run.stub_hits.get(provider, {})}
345
+ if provider
346
+ else dict(run.stub_hits)
347
+ )
348
+ declared_source = (
349
+ {provider: run.declared_stub_hits.get(provider, {})}
350
+ if provider
351
+ else dict(run.declared_stub_hits)
352
+ )
353
+ total = sum(sum(h.values()) for h in source.values())
354
+ declared_total = sum(sum(h.values()) for h in declared_source.values())
355
+ declared_eps = sorted({ep for h in declared_source.values() for ep in h})
356
+ merged: dict[str, int] = {}
357
+ for hits in source.values():
358
+ for ep, n in hits.items():
359
+ merged[ep] = merged.get(ep, 0) + n
360
+ recent = [
361
+ r for r in run.stub_recent if provider is None or r["provider"] == provider
362
+ ][-20:]
363
+ absent_source = (
364
+ {provider: run.absent_hits.get(provider, {})}
365
+ if provider
366
+ else dict(run.absent_hits)
367
+ )
368
+ absent_merged: dict[str, int] = {}
369
+ for hits in absent_source.values():
370
+ for ep, n in hits.items():
371
+ absent_merged[ep] = absent_merged.get(ep, 0) + n
372
+ return {
373
+ "total_hits": total,
374
+ "absent_hits": sum(absent_merged.values()),
375
+ "absent_endpoints": [
376
+ {"endpoint": ep, "hits": n}
377
+ for ep, n in sorted(absent_merged.items(), key=lambda kv: -kv[1])
378
+ ],
379
+ "unique_endpoints": len(merged),
380
+ # 拆成"欠着的"与"说过不做的"。两列必须都在,否则报表会把
381
+ # 声明的边界算成欠账,或反过来把欠账洗成边界。
382
+ "unplanned_hits": total - declared_total,
383
+ "declared_hits": declared_total,
384
+ "declared_endpoints": declared_eps,
385
+ "summary": [
386
+ {"endpoint": ep, "hits": n}
387
+ for ep, n in sorted(merged.items(), key=lambda kv: -kv[1])
388
+ ],
389
+ "recent": list(reversed(recent)),
390
+ }
391
+
392
+ def clear_stub_hits(self, run_id: str) -> None:
393
+ with self._lock:
394
+ run = self.get(run_id, allow_expired=True)
395
+ run.stub_hits.clear()
396
+ run.declared_stub_hits.clear()
397
+ run.absent_hits.clear()
398
+ run.stub_recent.clear()
399
+ self._save(run)
400
+
401
+ # ---------- 落盘 ----------
402
+
403
+ def _path(self, run_id: str) -> Path | None:
404
+ if self.directory is None:
405
+ return None
406
+ return self.directory / f"{run_id}.json"
407
+
408
+ def _save(self, run: Run) -> None:
409
+ """原子写 + 回读校验 + 跨进程互斥 + 乐观版本检查。
410
+
411
+ 为什么不是直接 write_text:写到一半崩掉会留下一个**语法上看起来没问题**
412
+ 的半个 JSON,下次启动时要么整个服务起不来,要么(更糟)读到一份缺了一半
413
+ 状态的世界,而 agent 会把"这些人怎么不见了"当成真实现象。
414
+
415
+ 为什么还要 flock:同一目录可能被两个进程打开(重放调试、两个 worker 打
416
+ 同一个 run)。没有锁时两个 os.replace 互相覆盖,最后一个赢,输的那次
417
+ 改动消失得无声无息。锁只保证"读盘校验 + 替换"这一小段是串行的。
418
+
419
+ 为什么还要比版本号:锁只解决"同时写",解决不了"我先读、你写、我再写"。
420
+ 那种情况下我手里的 world 已经是旧的,直接覆盖等于把你那次操作删掉。
421
+ """
422
+ path = self._path(run.run_id)
423
+ if path is None:
424
+ return
425
+ lock_path = path.with_suffix(".lock")
426
+ with open(lock_path, "a+") as lock_file:
427
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
428
+ try:
429
+ disk_version = 0
430
+ if path.exists():
431
+ try:
432
+ disk_version = int(
433
+ json.loads(path.read_text(encoding="utf-8")).get("version") or 0
434
+ )
435
+ except Exception: # 坏文件:当成第 0 版,让它被这次写覆盖掉
436
+ disk_version = 0
437
+ if disk_version != run.loaded_version:
438
+ raise StaleRunVersion(
439
+ f"run {run.run_id} 已被其它进程改到 v{disk_version},"
440
+ f"本次改动基于 v{run.loaded_version};请重新读取后再改"
441
+ )
442
+ run.version = disk_version + 1
443
+ run.loaded_version = run.version
444
+ tmp = path.with_name(f".{path.name}.tmp-{os.getpid()}-{threading.get_ident()}")
445
+ data = json.dumps(run.to_dict(), ensure_ascii=False)
446
+ tmp.write_text(data, encoding="utf-8")
447
+ json.loads(tmp.read_text(encoding="utf-8")) # 回读校验:坏 JSON 不许过
448
+ os.replace(tmp, path)
449
+ finally:
450
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
451
+
452
+ def _load_all(self) -> None:
453
+ assert self.directory is not None
454
+ for path in sorted(self.directory.glob("*.json")):
455
+ try:
456
+ payload = json.loads(path.read_text(encoding="utf-8"))
457
+ run = Run.from_dict(payload)
458
+ except Exception as exc: # noqa: BLE001 - 一个坏文件不该让整个服务起不来
459
+ self.load_errors.append(f"{path.name}: {type(exc).__name__}: {exc}")
460
+ continue
461
+ self._runs[run.run_id] = run