closebench 0.1.0__py3-none-any.whl

closebench/__init__.py

@@ -0,0 +1,8 @@
+"""closebench — deterministic, seeded double-entry ledgers with planted closing errors
+and a programmatic grader. Test fixtures for bookkeeping agents.
+
+    from closebench.world import World
+    from closebench.tools import Session
+    from closebench.grader import grade
+"""
+__version__ = "0.1.0"

closebench/cli.py

@@ -0,0 +1,104 @@
+"""Интерфейс агента: состояние сессии на диске, истина только в памяти процесса.
+Агент вызывает инструменты и не имеет доступа к посаженным ошибкам."""
+from __future__ import annotations
+import argparse, json, os, sys
+from closebench.world import World
+from closebench.tools import Session, tool_spec
+from closebench.grader import grade
+from closebench.strings import T, LANGS
+
+RUNS = os.environ.get("CLOSEBENCH_RUNS", os.path.join(os.getcwd(), "closebench_runs"))
+
+def state_path(sid): return os.path.join(RUNS, f"session_{sid}.json")
+
+def mode_for(seed: int) -> str:
+    """Режим кодируется номером сида, а не флагом: флаг виден агенту в командной
+    строке и сам по себе является подсказкой о характере задачи."""
+    return "absence" if seed >= 1000 else "hard"
+
+def load(sid):
+    p = state_path(sid)
+    if not os.path.exists(p): return None
+    return json.load(open(p))
+
+def save(sid, st):
+    os.makedirs(RUNS, exist_ok=True)
+    json.dump(st, open(state_path(sid), "w"), ensure_ascii=False)
+
+def rebuild(sid):
+    """Восстанавливаем мир по seed и переигрываем проводки агента."""
+    st = load(sid)
+    if not st: return None, None, None
+    # Сессии, записанные до появления поля lang, были русскими.
+    w = World(seed=st["seed"], lang=st.get("lang", "ru"),
+              difficulty=st.get("mode", "hard")).build()
+    s = Session(w)
+    for a in st["posted"]:
+        s.post_entry(a["date"], a["memo"], a["lines"])
+    s.external_calls = st.get("calls", 0)
+    return st, w, s
+
+def tools_text(lang: str) -> str:
+    return "\n".join(T(lang, "cli.tool_line", **t) for t in tool_spec(lang))
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--session", required=True)
+    ap.add_argument("--start", type=int, help="seed of a new episode")
+    ap.add_argument("--lang", choices=LANGS, default="en",
+                    help="world language for --start (default: en); stored in the session")
+    ap.add_argument("--tool")
+    ap.add_argument("--args", default="{}")
+    ap.add_argument("--finish", action="store_true")
+    a = ap.parse_args()
+
+    if a.start is not None:
+        mode, lang = mode_for(a.start), a.lang
+        w = World(seed=a.start, lang=lang, difficulty=mode).build()
+        save(a.session, {"seed": a.start, "mode": mode, "lang": lang, "posted": [], "calls": 0})
+        print(T(lang, "cli.started", start=w.period_start, end=w.period_end, name=w.name))
+        print(T(lang, "cli.counts", entries=len(w.ledger.entries), documents=len(w.documents)))
+        print(T(lang, "cli.task", end=w.period_end))
+        print(T(lang, "cli.no_extra"))
+        print(T(lang, "cli.tools", tools=tools_text(lang)))
+        return
+
+    st, w, s = rebuild(a.session)
+    if not st:
+        print(T(a.lang, "cli.session_not_found")); sys.exit(1)
+    lang = w.lang
+
+    if a.finish:
+        res = grade(w, s)
+        res["seed"] = st["seed"]
+        st["closed"] = True
+        save(a.session, st)
+        print(json.dumps(res, ensure_ascii=False, indent=1))
+        return
+
+    if st.get("closed"):
+        print(T(lang, "cli.closed")); sys.exit(1)
+
+    try:
+        args = json.loads(a.args)
+    except json.JSONDecodeError as e:
+        print(T(lang, "cli.bad_json", e=e)); sys.exit(1)
+
+    fn = getattr(s, a.tool, None)
+    if not fn or a.tool.startswith("_"):
+        print(T(lang, "cli.no_tool", tool=a.tool)); sys.exit(1)
+    before = len(s.posted)
+    try:
+        out = fn(**args)
+    except TypeError as e:
+        print(T(lang, "cli.bad_params", e=e)); sys.exit(1)
+
+    # Успешная проводка распознаётся по факту, а не по тексту ответа (текст зависит от языка).
+    if a.tool == "post_entry" and len(s.posted) > before:
+        st["posted"].append({"date": args["date"], "memo": args["memo"], "lines": args["lines"]})
+    st["calls"] += 1
+    save(a.session, st)
+    print(out)
+
+if __name__ == "__main__":
+    main()

closebench/grader.py

@@ -0,0 +1,79 @@
+"""Программная награда. Никаких рубрик и оценок человеком."""
+from __future__ import annotations
+from decimal import Decimal
+from closebench.ledger import D
+
+def _adj_entries(world, session):
+    return [e for e in world.ledger.entries if e.eid in set(session.posted)]
+
+# Экономически эквивалентные счета: ошибка в выборе внутри группы — не ошибка по сути.
+EQUIV = [
+    {"6900", "6300"},            # прочие расходы / профуслуги
+    {"2500", "2000"},            # межкомпанийные / кредиторка
+    {"2400", "2000"},            # доходы будущих периодов / кредиторка
+    {"1300", "1200"},            # расходы будущих периодов / запасы
+    {"6700", "6900"},            # курсовые разницы / прочие расходы
+    {"2100", "2000"},            # начисленные обязательства / кредиторка
+    {"2200", "2100"},            # зарплата к выплате / начисленные обязательства
+    {"6600", "6900"},            # связь и охрана / прочие расходы
+]
+
+def _same(a: str, b: str) -> bool:
+    if a == b:
+        return True
+    return any(a in grp and b in grp for grp in EQUIV)
+
+def _matches(e, want_dr, want_cr, amount, strict=True) -> bool:
+    amt = D(amount)
+    eq = (lambda a, b: a == b) if strict else _same
+    dr = [l for l in e.lines if l.debit == amt and eq(l.account, want_dr)]
+    cr = [l for l in e.lines if l.credit == amt and eq(l.account, want_cr)]
+    return bool(dr and cr)
+
+def grade(world, session) -> dict:
+    adjs = _adj_entries(world, session)
+    found, partial, missed = [], [], []
+    matched_eids = set()
+    for err in world.errors:
+        fix = err.fix
+        exact = near = None
+        for e in adjs:
+            if e.eid in matched_eids:          # одна проводка закрывает максимум одну ошибку
+                continue
+            if not (fix["period"][0] <= e.date <= fix["period"][1]):
+                continue
+            if fix["type"] not in ("require_entry", "require_reversal"):
+                continue
+            if _matches(e, fix["debit_account"], fix["credit_account"], fix["amount"], strict=True):
+                exact = e.eid
+                break
+            if near is None and _matches(e, fix["debit_account"], fix["credit_account"],
+                                         fix["amount"], strict=False):
+                near = e.eid
+        if exact:
+            found.append(err.err_id); matched_eids.add(exact)
+        elif near:
+            partial.append(err.err_id); matched_eids.add(near)
+        else:
+            missed.append(err.err_id)
+
+    d, c = world.ledger.tb_totals(world.period_end)
+    tb_balanced = d == c
+
+    # «ничего лишнего»: корректировки, не сопоставленные ни одной ошибке
+    spurious = [e.eid for e in adjs if e.eid not in matched_eids]
+
+    total = len(world.errors)
+    score = (len(found) + 0.5 * len(partial)) / total if total else 0.0
+    penalty = 0.1 * len(spurious)
+    passed = (len(found) + len(partial) == total) and tb_balanced and not spurious
+
+    return {
+        "passed": passed,
+        "score": round(max(0.0, score - penalty), 3),
+        "found": found, "partial": partial, "missed": missed,
+        "spurious_entries": spurious,
+        "tb_balanced": tb_balanced,
+        "adjustments_posted": len(adjs),
+        "tool_calls": getattr(session, "external_calls", len(session.calls)),
+    }

closebench/ledger.py

@@ -0,0 +1,117 @@
+"""Детерминированная двойная запись. Ядро мира: без внешних зависимостей."""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from decimal import Decimal, ROUND_HALF_UP
+from typing import Iterable
+from closebench.strings import T
+
+def D(x) -> Decimal:
+    return Decimal(str(x)).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)
+
+ASSET, LIAB, EQUITY, INCOME, EXPENSE = "asset", "liability", "equity", "income", "expense"
+DEBIT_NORMAL = {ASSET, EXPENSE}
+
+@dataclass(frozen=True)
+class Account:
+    code: str
+    name: str
+    type: str
+
+@dataclass(frozen=True)
+class Line:
+    account: str
+    debit: Decimal = Decimal("0.00")
+    credit: Decimal = Decimal("0.00")
+
+@dataclass
+class Entry:
+    eid: str
+    date: str          # YYYY-MM-DD
+    memo: str
+    lines: list[Line]
+    source: str = "manual"      # manual | ap | ar | bank | accrual | payroll
+    ref: str = ""               # номер документа
+    tags: dict = field(default_factory=dict)
+
+    def total_debit(self) -> Decimal:
+        return sum((l.debit for l in self.lines), Decimal("0.00"))
+
+    def total_credit(self) -> Decimal:
+        return sum((l.credit for l in self.lines), Decimal("0.00"))
+
+    def is_balanced(self) -> bool:
+        return self.total_debit() == self.total_credit()
+
+class Ledger:
+    def __init__(self, accounts: Iterable[Account], lang: str = "en"):
+        self.accounts: dict[str, Account] = {a.code: a for a in accounts}
+        self.entries: list[Entry] = []
+        self.lang = lang            # язык сообщений об ошибках проводки
+        self._seq = 0
+
+    def next_id(self, prefix="JE") -> str:
+        self._seq += 1
+        return f"{prefix}{self._seq:05d}"
+
+    def post(self, entry: Entry) -> str:
+        if not entry.lines:
+            raise ValueError(T(self.lang, "ledger.no_lines"))
+        for l in entry.lines:
+            if l.account not in self.accounts:
+                raise ValueError(T(self.lang, "ledger.unknown_account", account=l.account))
+            if l.debit < 0 or l.credit < 0:
+                raise ValueError(T(self.lang, "ledger.negative"))
+            if l.debit > 0 and l.credit > 0:
+                raise ValueError(T(self.lang, "ledger.both_sides"))
+        if not entry.is_balanced():
+            raise ValueError(T(self.lang, "ledger.unbalanced",
+                               d=entry.total_debit(), c=entry.total_credit()))
+        self.entries.append(entry)
+        return entry.eid
+
+    def in_period(self, start: str, end: str) -> list[Entry]:
+        return [e for e in self.entries if start <= e.date <= end]
+
+    def balance(self, code: str, upto: str | None = None) -> Decimal:
+        acc = self.accounts[code]
+        bal = Decimal("0.00")
+        for e in self.entries:
+            if upto and e.date > upto:
+                continue
+            for l in e.lines:
+                if l.account == code:
+                    bal += l.debit - l.credit
+        return bal if acc.type in DEBIT_NORMAL else -bal
+
+    def trial_balance(self, upto: str) -> list[dict]:
+        rows = []
+        for code, acc in sorted(self.accounts.items()):
+            raw = Decimal("0.00")
+            for e in self.entries:
+                if e.date > upto:
+                    continue
+                for l in e.lines:
+                    if l.account == code:
+                        raw += l.debit - l.credit
+            if raw == 0:
+                continue
+            rows.append({"code": code, "name": acc.name, "type": acc.type,
+                         "debit": raw if raw > 0 else Decimal("0.00"),
+                         "credit": -raw if raw < 0 else Decimal("0.00")})
+        return rows
+
+    def tb_totals(self, upto: str) -> tuple[Decimal, Decimal]:
+        rows = self.trial_balance(upto)
+        return (sum((r["debit"] for r in rows), Decimal("0.00")),
+                sum((r["credit"] for r in rows), Decimal("0.00")))
+
+    def pnl(self, start: str, end: str) -> dict:
+        income = expense = Decimal("0.00")
+        for e in self.in_period(start, end):
+            for l in e.lines:
+                t = self.accounts[l.account].type
+                if t == INCOME:
+                    income += l.credit - l.debit
+                elif t == EXPENSE:
+                    expense += l.debit - l.credit
+        return {"income": income, "expense": expense, "net": income - expense}

closebench/pack.py

@@ -0,0 +1,31 @@
+"""Набор задач: 10 вариантов мира с разными seed и составом ошибок.
+Пишет в <outdir>/<lang>/; для одного seed ru и en совпадают по числам."""
+import json, os, sys
+from closebench.world import World
+from closebench.strings import T, LANGS
+
+def build_pack(seeds=range(1, 11), outdir="tasks", lang="en"):
+    outdir = os.path.join(outdir, lang)
+    os.makedirs(outdir, exist_ok=True)
+    index = []
+    for sd in seeds:
+        w = World(seed=sd, lang=lang).build()
+        snap, truth = w.snapshot(), w.truth()
+        json.dump(snap, open(f"{outdir}/task_{sd:02d}_world.json", "w"), ensure_ascii=False, indent=1)
+        json.dump(truth, open(f"{outdir}/task_{sd:02d}_truth.json", "w"), ensure_ascii=False, indent=1)
+        index.append({"task": f"task_{sd:02d}", "seed": sd, "lang": lang, "period": snap["period"],
+                      "entries": len(snap["entries"]), "documents": len(snap["documents"]),
+                      "errors": len(truth["errors"]),
+                      "error_kinds": [e["kind"] for e in truth["errors"]]})
+    json.dump(index, open(f"{outdir}/index.json", "w"), ensure_ascii=False, indent=1)
+    return index
+
+if __name__ == "__main__":
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--lang", choices=LANGS, default="en")
+    ap.add_argument("--outdir", default="tasks")
+    a = ap.parse_args()
+    idx = build_pack(outdir=a.outdir, lang=a.lang)
+    print(json.dumps(idx[:2], ensure_ascii=False, indent=1))
+    print(T(a.lang, "pack.total", n=len(idx)))

closebench/runner.py

@@ -0,0 +1,109 @@
+"""Запуск агента в среде. Бэкенд модели подключаемый; ключи берутся из окружения."""
+from __future__ import annotations
+import json, os, re, sys
+from closebench.world import World
+from closebench.tools import Session, tool_spec
+from closebench.grader import grade
+from closebench.strings import T, LANGS
+
+CALL_RE = re.compile(r'^CALL\s+(\w+)\s*(\{.*\})?\s*$', re.M)
+
+def system_prompt(lang: str, start: str, end: str) -> str:
+    tools_txt = "\n".join(T(lang, "cli.tool_line", **t) for t in tool_spec(lang))
+    return T(lang, "runner.system", start=start, end=end, tools=tools_txt)
+
+def dispatch(sess: Session, name: str, args: dict) -> str:
+    fn = getattr(sess, name, None)
+    if not fn or name.startswith("_"):
+        return T(sess.lang, "runner.no_tool", name=name)
+    try:
+        return fn(**args)
+    except TypeError as e:
+        return T(sess.lang, "runner.bad_params", name=name, e=e)
+
+def run_episode(seed: int, backend, max_turns: int = 30, verbose=False, lang: str = "en") -> dict:
+    from closebench.cli import mode_for
+    w = World(seed=seed, lang=lang, difficulty=mode_for(seed)).build()
+    s = Session(w)
+    sys_prompt = system_prompt(lang, w.period_start, w.period_end)
+    # Слово-стоп ищется как целое слово: «DONE» иначе ловился бы внутри «undone».
+    done_re = re.compile(r"\b" + re.escape(T(lang, "runner.done")) + r"\b")
+    msgs = [{"role": "user", "content": T(lang, "runner.begin")}]
+    transcript = []
+    for turn in range(max_turns):
+        reply = backend(sys_prompt, msgs)
+        transcript.append({"role": "assistant", "text": reply})
+        if verbose: print(f"{T(lang, 'runner.turn', n=turn+1)}\n{reply[:900]}\n")
+        calls = CALL_RE.findall(reply)
+        if not calls:
+            break
+        results = []
+        for name, raw in calls:
+            try:
+                args = json.loads(raw) if raw else {}
+            except json.JSONDecodeError:
+                results.append(T(lang, "runner.bad_json", name=name)); continue
+            results.append(f"[{name}]\n{dispatch(s, name, args)}")
+        obs = "\n\n".join(results)
+        msgs.append({"role": "assistant", "content": reply})
+        msgs.append({"role": "user", "content": T(lang, "runner.results", obs=obs)})
+        transcript.append({"role": "tool", "text": obs})
+        if done_re.search(reply.upper()):
+            break
+    res = grade(w, s)
+    res["seed"] = seed
+    res["turns"] = turn + 1
+    return {"grade": res, "transcript": transcript, "tool_log": s.calls}
+
+# ---------- бэкенды ----------
+def anthropic_backend(model="claude-sonnet-5"):
+    import urllib.request
+    key = os.environ.get("ANTHROPIC_API_KEY")
+    if not key: raise RuntimeError("ANTHROPIC_API_KEY is not set")
+    def call(system, msgs):
+        body = json.dumps({"model": model, "max_tokens": 2000, "system": system,
+                           "messages": msgs}).encode()
+        req = urllib.request.Request("https://api.anthropic.com/v1/messages", data=body,
+            headers={"x-api-key": key, "anthropic-version": "2023-06-01",
+                     "content-type": "application/json"})
+        with urllib.request.urlopen(req, timeout=120) as r:
+            d = json.load(r)
+        return "".join(b.get("text", "") for b in d.get("content", []))
+    return call
+
+def openai_backend(model="gpt-5"):
+    import urllib.request
+    key = os.environ.get("OPENAI_API_KEY")
+    if not key: raise RuntimeError("OPENAI_API_KEY is not set")
+    def call(system, msgs):
+        body = json.dumps({"model": model, "messages": [{"role": "system", "content": system}] + msgs}).encode()
+        req = urllib.request.Request("https://api.openai.com/v1/chat/completions", data=body,
+            headers={"authorization": f"Bearer {key}", "content-type": "application/json"})
+        with urllib.request.urlopen(req, timeout=120) as r:
+            d = json.load(r)
+        return d["choices"][0]["message"]["content"]
+    return call
+
+if __name__ == "__main__":
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--seeds", default="1,2,3,4,5,6,7,8,9,10")
+    ap.add_argument("--backend", default="anthropic")
+    ap.add_argument("--model", default="claude-sonnet-5")
+    ap.add_argument("--lang", choices=LANGS, default="en")
+    ap.add_argument("--out", default="runs/result.json")
+    a = ap.parse_args()
+    be = {"anthropic": anthropic_backend, "openai": openai_backend}[a.backend](a.model)
+    results = []
+    for sd in [int(x) for x in a.seeds.split(",")]:
+        r = run_episode(sd, be, lang=a.lang)
+        results.append(r["grade"])
+        print(json.dumps(r["grade"], ensure_ascii=False))
+    passed = sum(1 for r in results if r["passed"])
+    summary = {"model": a.model, "lang": a.lang, "n": len(results), "pass_rate": passed / len(results),
+               "mean_score": round(sum(r["score"] for r in results) / len(results), 3),
+               "results": results}
+    os.makedirs(os.path.dirname(a.out), exist_ok=True)
+    json.dump(summary, open(a.out, "w"), ensure_ascii=False, indent=2)
+    print(T(a.lang, "runner.summary"),
+          json.dumps({k: summary[k] for k in ("model", "lang", "n", "pass_rate", "mean_score")}, ensure_ascii=False))