nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,270 @@
1
+ """LoCoMo's driver for the Nullius standard (JSON lines on stdin and stdout).
2
+
3
+ A capability benchmark: the memory under test is plugged in by Nullius through
4
+ cap.memory/v1, and this driver only says what to feed it and what to ask.
5
+ What LoCoMo itself decides - the reader's prompt and the score - is LoCoMo's
6
+ own code at the tested commit ($NULLIUS_UPSTREAM), imported, not copied: its
7
+ modules import torch, bert_score and the vendors' clients, so the driver runs
8
+ in the environment of its own that standard.json declares (driver.install).
9
+
10
+ units - every question of categories 1 to 4, as <sample_id>:<n> (the
11
+ adversarial category 5 has no reference answer in the file)
12
+ setup - the question's conversation, a session at a time, for ``add``;
13
+ the question, for ``recall``
14
+ produce - LoCoMo's reader prompt over what the memory recalled, as its RAG
15
+ mode builds it (task_eval/gpt_utils.py: the context, a blank line,
16
+ QA_PROMPT; a category-2 question carries the date hint
17
+ get_gpt_answers adds), asked by LoCoMo's own run_chatgpt
18
+ (global_methods.py) as get_gpt_answers calls it for one question
19
+ (num_tokens_request=32, temperature=0, wait_time=2, its model
20
+ argument and use_16k from the model's name), with the openai 0.28
21
+ client it is written for pointed at the relay (openai.api_base and
22
+ openai.api_key, the base_url and key Nullius gives); the answer is
23
+ its reply stripped, as get_gpt_answers keeps it. Its behaviour is
24
+ the upstream's, as published: the message's role and whether a
25
+ model is asked at all follow the model's name (a name with
26
+ "gpt-3.5" is asked as gpt-3.5-turbo, as the system; one with
27
+ "gpt-4", as the user; any other raises), it retries API,
28
+ connection and unavailable errors without end (a rate limit with
29
+ no wait), and a reply with no text fails the question. The run's
30
+ turn_seconds bounds it: past them the question is the reader's
31
+ error and the driver is started again for the next
32
+ score - LoCoMo's own eval_question_answering (task_eval/evaluation.py) on
33
+ the one question: token F1 with its normalisation and Porter
34
+ stemming, category 1 by comma-separated parts, category 3 against
35
+ the reference's part before the first ";"
36
+ answer - a setup's (original, model): LoCoMo's own task_eval/evaluate_qa.py,
37
+ run as published on the one conversation - its questions of
38
+ categories 1 to 4, as its scripts feed it - with --model
39
+ the model the setup asks as and --batch-size as the paper's
40
+ scripts give it (20 for gpt-4-turbo, 10 for the gpt-3.5-turbo
41
+ windows); its openai 0.28 client pointed at the relay through
42
+ OPENAI_API_BASE and OPENAI_API_KEY, which it reads. Its output
43
+ file, with each question's prediction and its own F1, stays in
44
+ the unit's folder for score. With retrieval (the setups' options
45
+ retrieval, retriever, top_k: --use-rag, --rag-mode, --retriever,
46
+ --top-k), as scripts/evaluate_rag_gpts.sh runs it: one question a
47
+ call, the database in the unit's folder (--emb-dir); for
48
+ observation and summary the upstream's own get_facts.py and
49
+ get_session_summaries.py write it first, with the flags
50
+ scripts/generate_observations.sh and generate_session_summaries.sh
51
+ pass (the latter's "---data-file" read as the --data-file it
52
+ means). Its encoders run on a CUDA GPU, as published
53
+ oracle - the reference control: the question's own "answer" field as that
54
+ scorer reads it - for category 3, its first part (``gold``): there
55
+ the field can carry the authors' reasoning after a ";", which
56
+ LoCoMo's evaluation cuts off before scoring, so the whole field
57
+ would lose F1 on its own words
58
+ """
59
+ import ast
60
+ import contextlib
61
+ import inspect
62
+ import json
63
+ import os
64
+ import subprocess
65
+ import sys
66
+ import textwrap
67
+
68
+ sys.path.insert(0, os.environ["NULLIUS_UPSTREAM"])
69
+ with contextlib.redirect_stdout(sys.stderr): # stdout is the protocol's: the upstream's prints go to the log
70
+ from task_eval import evaluation, gpt_utils # noqa: E402 as published
71
+ import openai # noqa: E402 0.28, the client run_chatgpt is written for
72
+
73
+ DATA = {}
74
+
75
+
76
+ def load():
77
+ if not DATA:
78
+ with open(os.environ["NULLIUS_DATASET"]) as f:
79
+ for conv in json.load(f):
80
+ DATA[conv["sample_id"]] = conv
81
+ return DATA
82
+
83
+
84
+ def question(unit):
85
+ sid, n = unit.rsplit(":", 1)
86
+ return load()[sid], load()[sid]["qa"][int(n)]
87
+
88
+
89
+ def units(dataset, setup=None):
90
+ if setup: # a setup's unit is a conversation, as the paper batches its questions
91
+ return {"units": list(load())}
92
+ out = []
93
+ for sid, conv in load().items():
94
+ out += [f"{sid}:{i}" for i, q in enumerate(conv["qa"]) if q.get("category") in (1, 2, 3, 4)]
95
+ return {"units": out}
96
+
97
+
98
+ # What goes into the memory for each session: the run's ``rag_mode`` option, LoCoMo's --rag-mode - dialog
99
+ # (the conversation), or the authors' own released extractions, observation (facts per speaker) and
100
+ # summary (one per session). How a fact or a summary is put as a turn is ours: a fact is its speaker's
101
+ # turn, a summary one turn whose role is "summary".
102
+ RAG_MODE = os.environ.get("NULLIUS_OPTION_RAG_MODE", "dialog")
103
+
104
+
105
+ def sessions(conv):
106
+ c = conv["conversation"]
107
+ n = 1
108
+ out = []
109
+ while f"session_{n}" in c:
110
+ if RAG_MODE == "observation":
111
+ turns = [{"role": spk, "text": fact} for spk, facts in conv["observation"][f"session_{n}_observation"].items()
112
+ for fact, _ in facts]
113
+ elif RAG_MODE == "summary":
114
+ turns = [{"role": "summary", "text": conv["session_summary"][f"session_{n}_summary"]}]
115
+ else:
116
+ turns = [{"role": t["speaker"], "text": t["text"] + (f" [shares a photo: {t['blip_caption']}]" if t.get("blip_caption") else "")}
117
+ for t in c[f"session_{n}"]]
118
+ date = c.get(f"session_{n}_date_time", "")
119
+ out.append({"id": f"session_{n}", "date": date, "turns": turns,
120
+ "text": f"DATE: {date}\n" + "\n".join(f"{t['role']}: {t['text']}" for t in turns)})
121
+ n += 1
122
+ return out
123
+
124
+
125
+ def setup(unit, dir):
126
+ conv, q = question(unit)
127
+ return {"memory": {"add": sessions(conv), "recall": [{"id": unit, "question": q["question"]}]}}
128
+
129
+
130
+ def date_hint():
131
+ """The words LoCoMo's reader adds to a category-2 question. They have no
132
+ name of their own in the upstream: they are read from get_gpt_answers'
133
+ source, through the imported module, at the tested commit."""
134
+ tree = ast.parse(textwrap.dedent(inspect.getsource(gpt_utils.get_gpt_answers)))
135
+ for node in ast.walk(tree):
136
+ if isinstance(node, ast.If) and ast.unparse(node.test).replace('"', "'") == "qa['category'] == 2":
137
+ for sub in ast.walk(node.body[0]):
138
+ if isinstance(sub, ast.BinOp) and isinstance(sub.right, ast.Constant) and isinstance(sub.right.value, str):
139
+ return sub.right.value
140
+ raise RuntimeError("LoCoMo's get_gpt_answers no longer adds a date hint to a category-2 question")
141
+
142
+
143
+ HINT = date_hint()
144
+
145
+
146
+ def produce(unit, dir, recalled, model, base_url, key):
147
+ _, q = question(unit)
148
+ memory = recalled[0]["memory"] if recalled else ""
149
+ asked = q["question"] + (HINT if q["category"] == 2 else "")
150
+ query = (memory or "(nothing recalled)") + "\n\n" + gpt_utils.QA_PROMPT.format(asked)
151
+ openai.api_base, openai.api_key = base_url, key # what run_chatgpt's client reads: the relay, for this question
152
+ with contextlib.redirect_stdout(sys.stderr):
153
+ # get_gpt_answers' own call for one question (batch size 1, as its RAG mode asks)
154
+ answer = gpt_utils.run_chatgpt(query, num_gen=1, num_tokens_request=32,
155
+ model='chatgpt' if 'gpt-3.5' in model else model,
156
+ use_16k=True if any([k in model for k in ['16k', '12k', '8k', '4k']]) else False,
157
+ temperature=0, wait_time=2)
158
+ return {"answer": answer.strip()}
159
+
160
+
161
+ def gold(q):
162
+ """The reference as LoCoMo's evaluation scores against it: the answer
163
+ field, for category 3 only its part before the first ";"."""
164
+ ref = str(q["answer"])
165
+ return ref.split(";")[0].strip() if q["category"] == 3 else ref
166
+
167
+
168
+ # The paper's batch per reader, as its scripts run evaluate_qa.py (scripts/evaluate_gpts.sh): every reader
169
+ # with a gpt-3.5-turbo window is asked 10 questions a call, gpt-4-turbo 20.
170
+ BATCH = {"gpt-4-turbo": 20}
171
+ ASKED = "asked.json" # in the unit's folder: the conversation as fed, and what came back
172
+ ANSWERS = "answers.json"
173
+
174
+
175
+ # The setups' retrieval (none: the whole conversation in the prompt), as evaluate_qa.py's flags name it.
176
+ RETRIEVAL = os.environ.get("NULLIUS_OPTION_RETRIEVAL", "none")
177
+ RETRIEVER = os.environ.get("NULLIUS_OPTION_RETRIEVER", "dragon")
178
+ TOP_K = os.environ.get("NULLIUS_OPTION_TOP_K", "5")
179
+ # what writes the database first, for a mode whose database is written by a script of its own: its output file
180
+ BUILD = {"observation": ("get_facts.py", "observation.json", ["--use-date", "--overwrite"]),
181
+ "summary": ("get_session_summaries.py", "session_summary.json", ["--use-date"])}
182
+
183
+
184
+ def model_key(model):
185
+ """The name evaluate_qa.py files a reader's predictions and F1 under."""
186
+ return model if RETRIEVAL == "none" else f"{model}_{RETRIEVAL}_top_{TOP_K}"
187
+
188
+
189
+ def upstream(script, argv, dir, env):
190
+ """One of the upstream's task_eval scripts, run as published; its output in the unit's folder."""
191
+ up = os.environ["NULLIUS_UPSTREAM"]
192
+ logname = os.path.join(dir, script.replace(".py", ".log"))
193
+ with open(logname, "w") as log:
194
+ rc = subprocess.run([sys.executable, os.path.join(up, "task_eval", script)] + argv, cwd=os.path.join(up, "task_eval"),
195
+ env=env, stdout=log, stderr=subprocess.STDOUT).returncode
196
+ if rc != 0:
197
+ with open(logname) as f:
198
+ tail = [ln for ln in f.read().splitlines() if ln.strip()][-3:]
199
+ raise RuntimeError(f"{script} exited {rc}: " + " | ".join(tail))
200
+
201
+
202
+ def answer(unit, dir, setup, model, base_url, key):
203
+ conv = load()[unit]
204
+ fed = dict(conv, qa=[q for q in conv["qa"] if q.get("category") in (1, 2, 3, 4)])
205
+ asked = os.path.join(dir, ASKED)
206
+ with open(asked, "w") as f:
207
+ json.dump([fed], f)
208
+ out = os.path.join(dir, ANSWERS)
209
+ env = dict(os.environ, OPENAI_API_BASE=base_url, OPENAI_API_KEY=key)
210
+ up = os.environ["NULLIUS_UPSTREAM"]
211
+ argv = ["--data-file", asked, "--out-file", out, "--model", model]
212
+ if RETRIEVAL == "none":
213
+ argv += ["--batch-size", str(BATCH.get(model, 10))]
214
+ else:
215
+ if RETRIEVAL in BUILD: # its database, written by the upstream's own script; named as evaluate_qa.py reads it
216
+ script, name, flags = BUILD[RETRIEVAL]
217
+ upstream(script, ["--data-file", asked, "--out-file", os.path.join(dir, "asked_" + name),
218
+ "--prompt-dir", os.path.join(up, "prompt_examples"), "--emb-dir", dir,
219
+ "--retriever", RETRIEVER] + flags, dir, env)
220
+ argv += ["--batch-size", "1", "--use-rag", "--retriever", RETRIEVER, "--top-k", TOP_K,
221
+ "--emb-dir", dir, "--rag-mode", RETRIEVAL]
222
+ upstream("evaluate_qa.py", argv, dir, env)
223
+ with open(out) as f:
224
+ qa = json.load(f)[0]["qa"]
225
+ k = model_key(model)
226
+ return {"answer": json.dumps({str(i): q.get(f"{k}_prediction", "") for i, q in enumerate(qa)}), "model": model}
227
+
228
+
229
+ def score(unit, dir, transcript, answer, setup=None, **_):
230
+ if setup: # the upstream's own F1 per question, as evaluate_qa.py wrote it
231
+ with open(os.path.join(dir, ANSWERS)) as f:
232
+ qa = json.load(f)[0]["qa"]
233
+ model = next((t["asked"] for t in transcript if t.get("asked")), "")
234
+ k = model_key(model)
235
+ f1 = [q.get(f"{k}_f1") for q in qa]
236
+ got = [x for x in f1 if isinstance(x, (int, float))]
237
+ rag = {} if RETRIEVAL == "none" else {"retrieval": RETRIEVAL, "retriever": RETRIEVER, "top_k": int(TOP_K)}
238
+ recall = [q[f"{k}_recall"] for q in qa if isinstance(q.get(f"{k}_recall"), (int, float))]
239
+ if recall: # the upstream's own: the share of a question's evidence among what was retrieved
240
+ rag["recall"] = sum(recall) / len(recall)
241
+ # the paper's number is over questions, not conversations: the job weighs this one by its own
242
+ parts = {"n": len(got), "by": "category",
243
+ "parts": [{"n": i + 1, "text": q["question"], "group": q["category"], "score": q.get(f"{k}_f1")}
244
+ for i, q in enumerate(qa) if isinstance(q.get(f"{k}_f1"), (int, float))]}
245
+ return {"reward": sum(got) / len(got) if got else 0.0, **({"parts": parts} if got else {}),
246
+ "detail": dict({"questions": len(qa), "scored": len(got), "model": model,
247
+ "per_question": [dict({"question": q["question"], "reference": q["answer"], "category": q["category"],
248
+ "answer": q.get(f"{k}_prediction"), "f1": q.get(f"{k}_f1")},
249
+ **({"retrieved": q.get(f"{k}_prediction_context"), "recall": q.get(f"{k}_recall")} if rag else {}))
250
+ for q in qa]}, **rag)}
251
+ _, q = question(unit)
252
+ with contextlib.redirect_stdout(sys.stderr):
253
+ values, _, _ = evaluation.eval_question_answering([dict(q, prediction=answer or "")], "prediction")
254
+ return {"reward": float(values[0]), "detail": {"question": q["question"], "reference": q["answer"], "category": q["category"],
255
+ "answer": answer, "evidence": q.get("evidence")}}
256
+
257
+
258
+ def oracle(unit, dir):
259
+ _, q = question(unit)
260
+ return {"answer": gold(q)}
261
+
262
+
263
+ HANDLERS = {"units": units, "setup": setup, "produce": produce, "score": score, "oracle": oracle, "answer": answer}
264
+ for line in sys.stdin:
265
+ try:
266
+ req = json.loads(line)
267
+ out = HANDLERS[req.pop("op")](**req)
268
+ except Exception as e:
269
+ out = {"error": f"{type(e).__name__}: {e}"}
270
+ print(json.dumps(out), flush=True)
@@ -0,0 +1,265 @@
1
+ {
2
+ "schema": "nullius.benchmark/v1",
3
+ "key": "locomo",
4
+ "label": "LoCoMo",
5
+ "summary": "Each question is a unit: its conversation's sessions go into the memory under test, the question is recalled against it, and LoCoMo's reader prompt answers from what came back; the answer is scored by token F1.",
6
+ "family": "capability",
7
+ "needs": [
8
+ "cap.memory/v1"
9
+ ],
10
+ "upstream": {
11
+ "repository": "https://github.com/snap-research/locomo",
12
+ "tested": "3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376",
13
+ "tested_date": "2024-08-13",
14
+ "license": "CC-BY-NC-4.0"
15
+ },
16
+ "driver": {
17
+ "command": [
18
+ "{install}/.venv/bin/python",
19
+ "{here}/driver.py"
20
+ ],
21
+ "install": {
22
+ "python": "3.9",
23
+ "pip": [
24
+ "torch==2.0.1",
25
+ "numpy==1.26.0",
26
+ "regex==2022.10.31",
27
+ "nltk==3.8.1",
28
+ "bert-score==0.3.13",
29
+ "transformers==4.35.0",
30
+ "tokenizers==0.14.1",
31
+ "huggingface-hub==0.17.3",
32
+ "pandas==2.1.2",
33
+ "matplotlib==3.7.0",
34
+ "requests==2.31.0",
35
+ "tqdm==4.64.1",
36
+ "tiktoken==0.5.2",
37
+ "openai==0.28.0",
38
+ "anthropic==0.32.0",
39
+ "google-generativeai==0.7.2",
40
+ "scipy==1.11.3",
41
+ "httpx==0.25.0",
42
+ "pydantic==1.10.13"
43
+ ]
44
+ }
45
+ },
46
+ "datasets": [
47
+ {
48
+ "id": "locomo10",
49
+ "label": "LoCoMo-10",
50
+ "path": "data/locomo10.json",
51
+ "sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
52
+ "units": 1540
53
+ }
54
+ ],
55
+ "units": {
56
+ "noun": "question",
57
+ "nouns": "questions"
58
+ },
59
+ "controls": {
60
+ "nop": {
61
+ "source": "nullius",
62
+ "from": "Nullius: no capability runs and nothing is recalled; the answer is empty."
63
+ },
64
+ "oracle": {
65
+ "source": "nullius",
66
+ "from": "Nullius, from the authors' answers: the question's answer field in the authors' locomo10.json, as their evaluation scores against it (task_eval/evaluation.py): for a category-3 question, the part before the first ';' - in 11 of them the field goes on with the authors' reasoning, which their scorer cuts off. Given to the scorer as the answer.",
67
+ "in_prompt": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
68
+ }
69
+ },
70
+ "score": {
71
+ "kind": "score",
72
+ "says": "token F1 of the answer against the reference, by LoCoMo's own eval_question_answering (task_eval/evaluation.py): its normalisation and Porter stemming, category 1 by comma-separated parts, category 3 against the reference's part before the first ';' (categories 1 to 4)"
73
+ },
74
+ "options": {
75
+ "rag_mode": {
76
+ "label": "RAG mode",
77
+ "summary": "Whether the memory stores conversation turns, extracted facts, or session summaries.",
78
+ "values": [
79
+ "dialog",
80
+ "observation",
81
+ "summary"
82
+ ],
83
+ "default": "dialog",
84
+ "about": {
85
+ "dialog": "Each session's turns, speaker by speaker, with its date. What every earlier run fed.",
86
+ "observation": "Each session's observations: the facts about each speaker the authors extracted, one turn a fact, its speaker's.",
87
+ "summary": "Each session's summary, written by the authors, one turn a session (its role: summary)."
88
+ }
89
+ },
90
+ "retrieval": {
91
+ "summary": "Whether the model receives the whole conversation or retrieves relevant turns, extracted facts, or session summaries.",
92
+ "values": [
93
+ "none",
94
+ "dialog",
95
+ "observation",
96
+ "summary"
97
+ ],
98
+ "default": "none",
99
+ "about": {
100
+ "none": "The whole conversation in the prompt, as the paper's long-context rows.",
101
+ "dialog": "Each turn embedded, with its date; the reader gets the top-k turns.",
102
+ "observation": "The upstream's get_facts.py first writes each session's observations (gpt-3.5-turbo, as published), embedded with their dates; the reader gets the top-k.",
103
+ "summary": "The upstream's get_session_summaries.py first writes a summary of each session (gpt-3.5-turbo, temperature 1, as published); the reader gets the top-k summaries."
104
+ },
105
+ "needs": {
106
+ "dialog": {
107
+ "host": [
108
+ "cuda_gpu"
109
+ ]
110
+ },
111
+ "observation": {
112
+ "host": [
113
+ "cuda_gpu"
114
+ ]
115
+ },
116
+ "summary": {
117
+ "host": [
118
+ "cuda_gpu"
119
+ ]
120
+ }
121
+ },
122
+ "setups": [
123
+ "original",
124
+ "model"
125
+ ]
126
+ },
127
+ "retriever": {
128
+ "when": {
129
+ "retrieval": [
130
+ "dialog",
131
+ "observation",
132
+ "summary"
133
+ ]
134
+ },
135
+ "summary": "The embedding model used to find relevant entries when retrieval is enabled.",
136
+ "values": [
137
+ "dragon",
138
+ "dpr"
139
+ ],
140
+ "default": "dragon",
141
+ "about": {
142
+ "dragon": "facebook/dragon-plus context and query encoders, as scripts/evaluate_rag_gpts.sh runs.",
143
+ "dpr": "facebook/dpr-ctx_encoder and question_encoder, single-nq-base."
144
+ },
145
+ "setups": [
146
+ "original",
147
+ "model"
148
+ ]
149
+ },
150
+ "top_k": {
151
+ "label": "Top-k",
152
+ "when": {
153
+ "retrieval": [
154
+ "dialog",
155
+ "observation",
156
+ "summary"
157
+ ]
158
+ },
159
+ "summary": "The maximum number of entries the model receives per question when retrieval is enabled.",
160
+ "values": [
161
+ "2",
162
+ "5",
163
+ "10",
164
+ "25",
165
+ "50"
166
+ ],
167
+ "default": "5",
168
+ "setups": [
169
+ "original",
170
+ "model"
171
+ ]
172
+ }
173
+ },
174
+ "host": {
175
+ "cuda_gpu": {
176
+ "summary": "An NVIDIA GPU with CUDA, for LoCoMo's retrievers: the upstream loads their encoders with .cuda(). Its driver's nvidia-smi on PATH; the driver's own torch (2.0.1, CUDA 11.7) brings the rest.",
177
+ "check": [
178
+ "nvidia-smi"
179
+ ]
180
+ }
181
+ },
182
+ "setups": {
183
+ "original": {
184
+ "summary": "LoCoMo's question answering as the paper ran it: its own task_eval/evaluate_qa.py on a whole conversation - the conversation in the prompt, cut to the reader's window, its questions 20 a call (10 for the gpt-3.5-turbo windows), or, with retrieval, the top-k of a database its own code builds, one question a call, as its RAG rows ran with gpt-3.5-turbo - its prompts, temperature 0 and token budget, scored by its own eval_question_answering, the mean token F1 over the conversation's questions of categories 1 to 4.",
185
+ "models": [
186
+ "gpt-4-turbo",
187
+ "gpt-3.5-turbo",
188
+ "gpt-3.5-turbo-16k",
189
+ "gpt-3.5-turbo-12k",
190
+ "gpt-3.5-turbo-8k",
191
+ "gpt-3.5-turbo-4k"
192
+ ],
193
+ "default": "gpt-4-turbo",
194
+ "served_as": {
195
+ "gpt-3.5-turbo-16k": "gpt-3.5-turbo",
196
+ "gpt-3.5-turbo-12k": "gpt-3.5-turbo",
197
+ "gpt-3.5-turbo-8k": "gpt-3.5-turbo",
198
+ "gpt-3.5-turbo-4k": "gpt-3.5-turbo"
199
+ },
200
+ "providers": {
201
+ "openai": "",
202
+ "openrouter": "openai/"
203
+ },
204
+ "units": {
205
+ "noun": "conversation",
206
+ "nouns": "conversations"
207
+ },
208
+ "turn_seconds": 3600,
209
+ "side_models": [
210
+ "gpt-3.5-turbo"
211
+ ]
212
+ },
213
+ "model": {
214
+ "summary": "The same pipeline with any model in the reader's place: LoCoMo's code asks for gpt-4-turbo - its 128k window, 20 questions a call, its token budget - and the relay sends the model chosen; with retrieval, the observations and summaries its code writes are asked of the model chosen too. Its scores compare with other runs of this setup, not with the paper.",
215
+ "runs_as": "gpt-4-turbo"
216
+ }
217
+ },
218
+ "proven": {
219
+ "locomo10": {
220
+ "units": {
221
+ "units": 1540,
222
+ "passed": "1540 ids, the same twice, unique",
223
+ "on": "2026-11-02"
224
+ },
225
+ "leak": {
226
+ "units": 1540,
227
+ "looked_for": 1144,
228
+ "short": 396,
229
+ "passed": "1540/1540",
230
+ "in_prompt": {
231
+ "units": 2,
232
+ "first": [
233
+ "conv-49:10",
234
+ "conv-50:39"
235
+ ],
236
+ "why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
237
+ },
238
+ "note": "396 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 2 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
239
+ "on": "2026-11-02"
240
+ },
241
+ "nop": {
242
+ "passed": "0/1540",
243
+ "value": 0.0,
244
+ "expect": {
245
+ "at_most": 0.0
246
+ },
247
+ "units": 1540,
248
+ "job": "2026-11-02__15-03-51__locomo__locomo10__prove-nop",
249
+ "source": "nullius",
250
+ "on": "2026-11-02"
251
+ },
252
+ "oracle": {
253
+ "passed": "1540/1540",
254
+ "value": 1.0,
255
+ "expect": {
256
+ "at_least": 1.0
257
+ },
258
+ "units": 1540,
259
+ "job": "2026-11-02__15-03-51__locomo__locomo10__prove-oracle",
260
+ "source": "nullius",
261
+ "on": "2026-11-02"
262
+ }
263
+ }
264
+ }
265
+ }