nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,733 @@
1
+ {
2
+ "schema": "nullius.benchmark/v1",
3
+ "key": "longmemeval",
4
+ "label": "LongMemEval",
5
+ "summary": "Each question is a unit: its chat history goes into the memory under test a session at a time, the question is recalled against it, LongMemEval's reader prompt answers from what came back, and LongMemEval's own judge says whether the answer is right.",
6
+ "family": "capability",
7
+ "needs": [
8
+ "cap.memory/v1"
9
+ ],
10
+ "upstream": {
11
+ "repository": "https://github.com/xiaowu0162/LongMemEval",
12
+ "tested": "9e0b455f4ef0e2ab8f2e582289761153549043fc",
13
+ "tested_date": "2026-05-11",
14
+ "license": "MIT"
15
+ },
16
+ "driver": {
17
+ "command": [
18
+ "{install}/.venv/bin/python",
19
+ "{here}/driver.py"
20
+ ],
21
+ "install": {
22
+ "python": "3.9",
23
+ "pip": [
24
+ "torch==2.3.1",
25
+ "torchvision==0.18.1",
26
+ "openai==1.35.1",
27
+ "httpx==0.27.0",
28
+ "tqdm==4.66.4",
29
+ "backoff==2.2.1",
30
+ "numpy==1.26.3",
31
+ "nltk==3.9.1",
32
+ "transformers==4.43.3",
33
+ "tokenizers==0.19.1",
34
+ "tiktoken==0.7.0",
35
+ "huggingface-hub==0.23.4",
36
+ "rank-bm25==0.2.2",
37
+ "sentence-transformers==2.2.2",
38
+ "scikit-learn==1.5.1"
39
+ ]
40
+ }
41
+ },
42
+ "datasets": [
43
+ {
44
+ "id": "longmemeval_s",
45
+ "label": "LongMemEval-S · cleaned",
46
+ "url": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_s_cleaned.json",
47
+ "sha256": "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",
48
+ "bytes": 277383467,
49
+ "units": 500
50
+ },
51
+ {
52
+ "id": "longmemeval_m",
53
+ "label": "LongMemEval-M · cleaned",
54
+ "summary": "About 500 sessions of history per question. Needs about 5.5 GB of free disk.",
55
+ "url": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_m_cleaned.json",
56
+ "sha256": "9d79e5524794a2e6900a3aa9cb7d9152c5a3e8319c9a87c25494ba1eacee495f",
57
+ "bytes": 2737100077,
58
+ "units": 500
59
+ },
60
+ {
61
+ "id": "longmemeval_oracle",
62
+ "label": "LongMemEval · evidence sessions only",
63
+ "summary": "Only the sessions that hold each question's evidence.",
64
+ "url": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_oracle.json",
65
+ "sha256": "821a2034d219ab45846873dd14c14f12cfe7776e73527a483f9dac095d38620c",
66
+ "bytes": 15388478,
67
+ "units": 500
68
+ },
69
+ {
70
+ "id": "longmemeval_s_original",
71
+ "label": "LongMemEval-S · as first released",
72
+ "summary": "The S file as first released in 2024, before the authors cleaned it. The paper's numbers come from it.",
73
+ "url": "https://huggingface.co/datasets/xiaowu0162/longmemeval/resolve/2ec2a557f339b6c0369619b1ed5793734cc87533/longmemeval_s",
74
+ "sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894",
75
+ "bytes": 278025796,
76
+ "units": 500
77
+ },
78
+ {
79
+ "id": "longmemeval_m_original",
80
+ "label": "LongMemEval-M · as first released",
81
+ "summary": "The M file as first released in 2024. Needs about 5.5 GB of free disk.",
82
+ "url": "https://huggingface.co/datasets/xiaowu0162/longmemeval/resolve/2ec2a557f339b6c0369619b1ed5793734cc87533/longmemeval_m",
83
+ "sha256": "fb5413e3b077c62927daab794836991a2fcfa61ceacab57dc679fb02daaff2d9",
84
+ "bytes": 2745274681,
85
+ "units": 500
86
+ }
87
+ ],
88
+ "units": {
89
+ "noun": "question",
90
+ "nouns": "questions"
91
+ },
92
+ "controls": {
93
+ "nop": {
94
+ "source": "nullius",
95
+ "from": "Nullius: no capability runs and nothing is recalled; the answer is empty. The 30 abstention questions in each dataset are right when the answer says it cannot answer, so the judge may pass an empty one; it also passes an empty answer by mistake now and then.",
96
+ "expect": {
97
+ "at_most": 0.07
98
+ }
99
+ },
100
+ "oracle": {
101
+ "source": "nullius",
102
+ "from": "Nullius, from the authors' answers: the question's answer field in the authors' file (for an abstention question, their explanation of why it cannot be answered), given to the judge as the answer. The judge, a model, rejects about one in twenty of them.",
103
+ "in_prompt": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
104
+ "expect": {
105
+ "at_least": 0.92
106
+ }
107
+ }
108
+ },
109
+ "score": {
110
+ "kind": "pass",
111
+ "says": "LongMemEval's own judge (src/evaluation/evaluate_qa.py): its prompt for the question's type (abstention questions: whether the answer says it cannot be answered), sent by its own client call, answered yes or no",
112
+ "judge": {
113
+ "model": "gpt-4o-2024-08-06",
114
+ "says": "the paper's judge, gpt-4o, at temperature 0, ten tokens"
115
+ }
116
+ },
117
+ "limits": {
118
+ "turn_seconds": 900
119
+ },
120
+ "options": {
121
+ "question_type": {
122
+ "label": "Question type",
123
+ "summary": "Which memory ability the run measures: every question, or only the questions of one type.",
124
+ "values": [
125
+ "all",
126
+ "knowledge-update",
127
+ "temporal-reasoning",
128
+ "multi-session",
129
+ "single-session-user",
130
+ "single-session-assistant",
131
+ "single-session-preference"
132
+ ],
133
+ "default": "all",
134
+ "setups": [
135
+ "original",
136
+ "model",
137
+ "yours"
138
+ ]
139
+ },
140
+ "reading": {
141
+ "summary": "Whether the model answers from recalled memory directly or first identifies relevant facts and reasons through them.",
142
+ "values": [
143
+ "con",
144
+ "direct"
145
+ ],
146
+ "default": "con",
147
+ "about": {
148
+ "con": "Chain-of-note in one call: prepare_prompt's history-chats template with cot, gen_length 800. The upstream's default.",
149
+ "direct": "The template without cot, gen_length 500: the answer at once."
150
+ },
151
+ "setups": [
152
+ "original",
153
+ "model",
154
+ "yours"
155
+ ]
156
+ },
157
+ "retriever": {
158
+ "summary": "Whether the model receives the full conversation history, no history, or sessions or turns selected by a retrieval method.",
159
+ "values": [
160
+ "full-history-session",
161
+ "full-history-turn",
162
+ "no-retrieval",
163
+ "flat-bm25-session",
164
+ "flat-bm25-turn",
165
+ "flat-contriever-session",
166
+ "flat-contriever-turn",
167
+ "flat-stella-session",
168
+ "flat-stella-turn"
169
+ ],
170
+ "default": "full-history-session",
171
+ "setups": [
172
+ "original",
173
+ "model"
174
+ ],
175
+ "about": {
176
+ "full-history-session": "--retriever_type orig-session: every session, latest kept first when the history is cut to the window.",
177
+ "full-history-turn": "--retriever_type orig-turn: every turn, each its own chunk.",
178
+ "no-retrieval": "--retriever_type no-retrieval: the question alone.",
179
+ "flat-bm25-session": "run_retrieval.py flat-bm25 at session granularity (a session's user turns are its key), then --retriever_type flat-session: the top-k sessions.",
180
+ "flat-bm25-turn": "run_retrieval.py flat-bm25 at turn granularity (a user turn is the key), then --retriever_type flat-turn: the top-k rounds.",
181
+ "flat-contriever-session": "run_retrieval.py flat-contriever (facebook/contriever, mean pooling) at session granularity, then --retriever_type flat-session.",
182
+ "flat-contriever-turn": "run_retrieval.py flat-contriever at turn granularity, then --retriever_type flat-turn.",
183
+ "flat-stella-session": "run_retrieval.py flat-stella (Stella V5 1.5B, its 1024-dimension head) at session granularity; the model fetched into the upstream's model_cache first, as its README has it downloaded by hand. Its fixed batch of 64 needs at least 12000 MiB of GPU memory.",
184
+ "flat-stella-turn": "run_retrieval.py flat-stella at turn granularity, then --retriever_type flat-turn."
185
+ },
186
+ "needs": {
187
+ "flat-contriever-session": {
188
+ "host": [
189
+ "cuda_gpu"
190
+ ]
191
+ },
192
+ "flat-contriever-turn": {
193
+ "host": [
194
+ "cuda_gpu"
195
+ ]
196
+ },
197
+ "flat-stella-session": {
198
+ "host": [
199
+ "stella_gpu"
200
+ ]
201
+ },
202
+ "flat-stella-turn": {
203
+ "host": [
204
+ "stella_gpu"
205
+ ]
206
+ }
207
+ }
208
+ },
209
+ "topk": {
210
+ "label": "Top-k",
211
+ "summary": "The maximum number of conversation sessions or turns the model receives when answering a question.",
212
+ "values": [
213
+ "1",
214
+ "3",
215
+ "5",
216
+ "10",
217
+ "20",
218
+ "30",
219
+ "50",
220
+ "100",
221
+ "1000"
222
+ ],
223
+ "default": "1000",
224
+ "setups": [
225
+ "original",
226
+ "model"
227
+ ]
228
+ },
229
+ "history_format": {
230
+ "summary": "Whether conversation history is shown to the model as JSON or as lines of text labelled by speaker.",
231
+ "values": [
232
+ "json",
233
+ "nl"
234
+ ],
235
+ "default": "json",
236
+ "setups": [
237
+ "original",
238
+ "model"
239
+ ]
240
+ },
241
+ "useronly": {
242
+ "label": "User turns only",
243
+ "summary": "Whether conversation history includes only user messages or both user and assistant messages.",
244
+ "values": [
245
+ "false",
246
+ "true"
247
+ ],
248
+ "default": "false",
249
+ "setups": [
250
+ "original",
251
+ "model"
252
+ ]
253
+ },
254
+ "merge_key_expansion": {
255
+ "summary": "Whether retrieved conversations are shown in full, replaced by their user text, or combined with that text.",
256
+ "values": [
257
+ "none",
258
+ "merge",
259
+ "replace"
260
+ ],
261
+ "default": "none",
262
+ "setups": [
263
+ "original",
264
+ "model"
265
+ ]
266
+ },
267
+ "separate_notes": {
268
+ "label": "Separate notes",
269
+ "summary": "Write a separate note for each retrieved session or turn before reasoning. Original memory only.",
270
+ "values": [
271
+ "false",
272
+ "true"
273
+ ],
274
+ "default": "false",
275
+ "setups": [
276
+ "original",
277
+ "model"
278
+ ],
279
+ "when": {
280
+ "reading": [
281
+ "con"
282
+ ]
283
+ }
284
+ },
285
+ "session_expansion": {
286
+ "label": "Session index expansion",
287
+ "summary": "Expand retrieval keys with the authors' cached summaries, keyphrases or user facts. No expansion by default.",
288
+ "values": [
289
+ "none",
290
+ "session-summ",
291
+ "session-keyphrase",
292
+ "session-userfact"
293
+ ],
294
+ "default": "none",
295
+ "setups": [
296
+ "original",
297
+ "model"
298
+ ],
299
+ "when": {
300
+ "retriever": [
301
+ "flat-bm25-session",
302
+ "flat-contriever-session",
303
+ "flat-stella-session"
304
+ ]
305
+ },
306
+ "needs": {
307
+ "session-summ": {
308
+ "files": [
309
+ "index_expansions"
310
+ ]
311
+ },
312
+ "session-keyphrase": {
313
+ "files": [
314
+ "index_expansions"
315
+ ]
316
+ },
317
+ "session-userfact": {
318
+ "files": [
319
+ "index_expansions"
320
+ ]
321
+ }
322
+ }
323
+ },
324
+ "turn_expansion": {
325
+ "label": "Turn index expansion",
326
+ "summary": "Expand retrieval keys with the authors' cached summaries, keyphrases or user facts. No expansion by default.",
327
+ "values": [
328
+ "none",
329
+ "turn-keyphrase",
330
+ "turn-userfact"
331
+ ],
332
+ "default": "none",
333
+ "setups": [
334
+ "original",
335
+ "model"
336
+ ],
337
+ "when": {
338
+ "retriever": [
339
+ "flat-bm25-turn",
340
+ "flat-contriever-turn",
341
+ "flat-stella-turn"
342
+ ]
343
+ },
344
+ "needs": {
345
+ "turn-keyphrase": {
346
+ "files": [
347
+ "index_expansions"
348
+ ]
349
+ },
350
+ "turn-userfact": {
351
+ "files": [
352
+ "index_expansions"
353
+ ]
354
+ }
355
+ }
356
+ },
357
+ "expansion_join": {
358
+ "label": "Join index expansion",
359
+ "summary": "How cached expansions join retrieval keys when expansion is enabled: separate keys, merged text or replacement text. Split modes separate sentences or phrases first.",
360
+ "values": [
361
+ "separate",
362
+ "split-separate",
363
+ "merge",
364
+ "split-merge",
365
+ "replace",
366
+ "split-replace"
367
+ ],
368
+ "default": "separate",
369
+ "setups": [
370
+ "original",
371
+ "model"
372
+ ],
373
+ "when": {
374
+ "retriever": [
375
+ "flat-bm25-session",
376
+ "flat-bm25-turn",
377
+ "flat-contriever-session",
378
+ "flat-contriever-turn",
379
+ "flat-stella-session",
380
+ "flat-stella-turn"
381
+ ]
382
+ },
383
+ "needs": {
384
+ "split-separate": {
385
+ "files": [
386
+ "punkt_tab"
387
+ ],
388
+ "when": {
389
+ "session_expansion": [
390
+ "session-summ"
391
+ ],
392
+ "retriever": [
393
+ "flat-bm25-session",
394
+ "flat-contriever-session",
395
+ "flat-stella-session"
396
+ ]
397
+ }
398
+ },
399
+ "split-merge": {
400
+ "files": [
401
+ "punkt_tab"
402
+ ],
403
+ "when": {
404
+ "session_expansion": [
405
+ "session-summ"
406
+ ],
407
+ "retriever": [
408
+ "flat-bm25-session",
409
+ "flat-contriever-session",
410
+ "flat-stella-session"
411
+ ]
412
+ }
413
+ },
414
+ "split-replace": {
415
+ "files": [
416
+ "punkt_tab"
417
+ ],
418
+ "when": {
419
+ "session_expansion": [
420
+ "session-summ"
421
+ ],
422
+ "retriever": [
423
+ "flat-bm25-session",
424
+ "flat-contriever-session",
425
+ "flat-stella-session"
426
+ ]
427
+ }
428
+ }
429
+ }
430
+ }
431
+ },
432
+ "host": {
433
+ "cuda_gpu": {
434
+ "summary": "An NVIDIA GPU with CUDA, for LongMemEval's Contriever retrievers.",
435
+ "check": [
436
+ "nvidia-smi"
437
+ ]
438
+ },
439
+ "stella_gpu": {
440
+ "summary": "An NVIDIA GPU with CUDA and at least 12000 MiB of GPU memory, for Stella's fixed batch of 64.",
441
+ "check": [
442
+ "nvidia-smi"
443
+ ],
444
+ "gpu_memory_mb": 12000
445
+ }
446
+ },
447
+ "setups": {
448
+ "original": {
449
+ "summary": "LongMemEval's long-context reader as the paper ran it: its own src/generation/run_generation.py on one question, as run_generation.sh runs it - the history in the prompt (or what the paper's BM25 retrieval ranks), cut to the reader's 128k window, its prompts, temperature 0 and answer length - scored by its own src/evaluation/evaluate_qa.py with its judge, gpt-4o.",
450
+ "models": [
451
+ "gpt-4o",
452
+ "gpt-4o-mini"
453
+ ],
454
+ "default": "gpt-4o",
455
+ "served_as": {
456
+ "gpt-4o": "gpt-4o-2024-08-06",
457
+ "gpt-4o-mini": "gpt-4o-mini-2024-07-18"
458
+ },
459
+ "providers": {
460
+ "openai": "",
461
+ "openrouter": "openai/"
462
+ },
463
+ "turn_seconds": 3600,
464
+ "paper": "Table 2 (long-context reading) and Figure 5 onward (retrieval-augmented reading)"
465
+ },
466
+ "model": {
467
+ "summary": "The same pipeline with any model in the reader's place: LongMemEval's code asks for gpt-4o - its 128k window, the history cut to fit it, its answer length - and the relay sends the model chosen. Its scores compare with other runs of this setup, not with the paper.",
468
+ "runs_as": "gpt-4o"
469
+ }
470
+ },
471
+ "proven": {
472
+ "longmemeval_s": {
473
+ "units": {
474
+ "units": 500,
475
+ "passed": "500 ids, the same twice, unique",
476
+ "on": "2026-11-03"
477
+ },
478
+ "leak": {
479
+ "units": 500,
480
+ "looked_for": 248,
481
+ "short": 252,
482
+ "passed": "500/500",
483
+ "in_prompt": {
484
+ "units": 11,
485
+ "first": [
486
+ "gpt4_4929293a",
487
+ "gpt4_2487a7cb",
488
+ "gpt4_2312f94c",
489
+ "gpt4_2d58bcd6",
490
+ "gpt4_483dd43c"
491
+ ],
492
+ "why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
493
+ },
494
+ "note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
495
+ "on": "2026-11-03"
496
+ },
497
+ "nop": {
498
+ "passed": "20/500",
499
+ "value": 0.04,
500
+ "expect": {
501
+ "at_most": 0.07
502
+ },
503
+ "units": 500,
504
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_s__prove-nop",
505
+ "source": "nullius",
506
+ "judge": "gpt-4o-2024-08-06",
507
+ "on": "2026-11-03"
508
+ },
509
+ "oracle": {
510
+ "passed": "471/500",
511
+ "value": 0.942,
512
+ "expect": {
513
+ "at_least": 0.92
514
+ },
515
+ "units": 500,
516
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_s__prove-oracle",
517
+ "source": "nullius",
518
+ "judge": "gpt-4o-2024-08-06",
519
+ "on": "2026-11-03"
520
+ }
521
+ },
522
+ "longmemeval_m": {
523
+ "units": {
524
+ "units": 500,
525
+ "passed": "500 ids, the same twice, unique",
526
+ "on": "2026-11-03"
527
+ },
528
+ "leak": {
529
+ "units": 500,
530
+ "looked_for": 248,
531
+ "short": 252,
532
+ "passed": "500/500",
533
+ "in_prompt": {
534
+ "units": 11,
535
+ "first": [
536
+ "gpt4_4929293a",
537
+ "gpt4_2487a7cb",
538
+ "gpt4_2312f94c",
539
+ "gpt4_2d58bcd6",
540
+ "gpt4_483dd43c"
541
+ ],
542
+ "why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
543
+ },
544
+ "note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
545
+ "on": "2026-11-03"
546
+ },
547
+ "nop": {
548
+ "passed": "24/500",
549
+ "value": 0.048,
550
+ "expect": {
551
+ "at_most": 0.07
552
+ },
553
+ "units": 500,
554
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_m__prove-nop",
555
+ "source": "nullius",
556
+ "judge": "gpt-4o-2024-08-06",
557
+ "on": "2026-11-03"
558
+ },
559
+ "oracle": {
560
+ "passed": "470/500",
561
+ "value": 0.94,
562
+ "expect": {
563
+ "at_least": 0.92
564
+ },
565
+ "units": 500,
566
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_m__prove-oracle",
567
+ "source": "nullius",
568
+ "judge": "gpt-4o-2024-08-06",
569
+ "on": "2026-11-03"
570
+ }
571
+ },
572
+ "longmemeval_oracle": {
573
+ "units": {
574
+ "units": 500,
575
+ "passed": "500 ids, the same twice, unique",
576
+ "on": "2026-11-03"
577
+ },
578
+ "leak": {
579
+ "units": 500,
580
+ "looked_for": 248,
581
+ "short": 252,
582
+ "passed": "500/500",
583
+ "in_prompt": {
584
+ "units": 11,
585
+ "first": [
586
+ "gpt4_2487a7cb",
587
+ "gpt4_2312f94c",
588
+ "gpt4_2d58bcd6",
589
+ "gpt4_483dd43c",
590
+ "gpt4_93f6379c"
591
+ ],
592
+ "why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
593
+ },
594
+ "note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
595
+ "on": "2026-11-03"
596
+ },
597
+ "nop": {
598
+ "passed": "22/500",
599
+ "value": 0.044,
600
+ "expect": {
601
+ "at_most": 0.07
602
+ },
603
+ "units": 500,
604
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_oracle__prove-nop",
605
+ "source": "nullius",
606
+ "judge": "gpt-4o-2024-08-06",
607
+ "on": "2026-11-03"
608
+ },
609
+ "oracle": {
610
+ "passed": "469/500",
611
+ "value": 0.938,
612
+ "expect": {
613
+ "at_least": 0.92
614
+ },
615
+ "units": 500,
616
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_oracle__prove-oracle",
617
+ "source": "nullius",
618
+ "judge": "gpt-4o-2024-08-06",
619
+ "on": "2026-11-03"
620
+ }
621
+ },
622
+ "longmemeval_s_original": {
623
+ "units": {
624
+ "units": 500,
625
+ "passed": "500 ids, the same twice, unique",
626
+ "on": "2026-11-03"
627
+ },
628
+ "leak": {
629
+ "units": 500,
630
+ "looked_for": 248,
631
+ "short": 252,
632
+ "passed": "500/500",
633
+ "in_prompt": {
634
+ "units": 11,
635
+ "first": [
636
+ "gpt4_4929293a",
637
+ "gpt4_2487a7cb",
638
+ "gpt4_2312f94c",
639
+ "gpt4_2d58bcd6",
640
+ "gpt4_483dd43c"
641
+ ],
642
+ "why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
643
+ },
644
+ "note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
645
+ "on": "2026-11-03"
646
+ },
647
+ "nop": {
648
+ "passed": "21/500",
649
+ "value": 0.042,
650
+ "expect": {
651
+ "at_most": 0.07
652
+ },
653
+ "units": 500,
654
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_s_original__prove-nop",
655
+ "source": "nullius",
656
+ "judge": "gpt-4o-2024-08-06",
657
+ "on": "2026-11-03"
658
+ },
659
+ "oracle": {
660
+ "passed": "469/500",
661
+ "value": 0.938,
662
+ "expect": {
663
+ "at_least": 0.92
664
+ },
665
+ "units": 500,
666
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_s_original__prove-oracle",
667
+ "source": "nullius",
668
+ "judge": "gpt-4o-2024-08-06",
669
+ "on": "2026-11-03"
670
+ }
671
+ },
672
+ "longmemeval_m_original": {
673
+ "units": {
674
+ "units": 500,
675
+ "passed": "500 ids, the same twice, unique",
676
+ "on": "2026-11-03"
677
+ },
678
+ "leak": {
679
+ "units": 500,
680
+ "looked_for": 248,
681
+ "short": 252,
682
+ "passed": "500/500",
683
+ "in_prompt": {
684
+ "units": 11,
685
+ "first": [
686
+ "gpt4_4929293a",
687
+ "gpt4_2487a7cb",
688
+ "gpt4_2312f94c",
689
+ "gpt4_2d58bcd6",
690
+ "gpt4_483dd43c"
691
+ ],
692
+ "why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
693
+ },
694
+ "note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
695
+ "on": "2026-11-03"
696
+ },
697
+ "nop": {
698
+ "passed": "27/500",
699
+ "value": 0.054,
700
+ "expect": {
701
+ "at_most": 0.07
702
+ },
703
+ "units": 500,
704
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_m_original__prove-nop",
705
+ "source": "nullius",
706
+ "judge": "gpt-4o-2024-08-06",
707
+ "on": "2026-11-03"
708
+ },
709
+ "oracle": {
710
+ "passed": "469/500",
711
+ "value": 0.938,
712
+ "expect": {
713
+ "at_least": 0.92
714
+ },
715
+ "units": 500,
716
+ "job": "2026-11-03__01-26-22__longmemeval__longmemeval_m_original__prove-oracle",
717
+ "source": "nullius",
718
+ "judge": "gpt-4o-2024-08-06",
719
+ "on": "2026-11-03"
720
+ }
721
+ }
722
+ },
723
+ "files": {
724
+ "index_expansions": {
725
+ "url": "https://drive.usercontent.google.com/download?id=1Y6vImnr_WQZXv4bKzpqPdHMbyoat8b_P&export=download&confirm=t",
726
+ "sha256": "2c87f279962968e86ff5a1b64be8c83c2c21077e0cf5fc76fb9c84db1d8ce7c1"
727
+ },
728
+ "punkt_tab": {
729
+ "url": "https://raw.githubusercontent.com/nltk/nltk_data/550b6625bcef1f2abff2ff770a5a0d272c9c6b2a/packages/tokenizers/punkt_tab.zip",
730
+ "sha256": "e57f64187974277726a3417ca6f181ec5403676c717672eef6a748a7b20e0106"
731
+ }
732
+ }
733
+ }