nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,76 @@
1
+ [
2
+ {"entry": "data (Hugging Face)", "setting": "longmemeval-cleaned/longmemeval_s_cleaned.json", "values": "500 questions, about 115k tokens of history each", "default": "", "nullius": "dataset:longmemeval_s"},
3
+ {"entry": "data (Hugging Face)", "setting": "longmemeval-cleaned/longmemeval_m_cleaned.json", "values": "500 questions, about 500 sessions each", "default": "", "nullius": "dataset:longmemeval_m"},
4
+ {"entry": "data (Hugging Face)", "setting": "longmemeval-cleaned/longmemeval_oracle.json", "values": "500 questions, evidence sessions only (the same file in both releases)", "default": "", "nullius": "dataset:longmemeval_oracle"},
5
+ {"entry": "data (Hugging Face)", "setting": "longmemeval/longmemeval_s (first release, uncleaned)", "values": "500 questions", "default": "", "nullius": "dataset:longmemeval_s_original"},
6
+ {"entry": "data (Hugging Face)", "setting": "longmemeval/longmemeval_m (first release, uncleaned)", "values": "500 questions", "default": "", "nullius": "dataset:longmemeval_m_original"},
7
+ {"entry": "data (Hugging Face)", "setting": "question_type (a field of every question)", "values": "knowledge-update, temporal-reasoning, multi-session, single-session-user, single-session-assistant, single-session-preference; an abstention question (its question_id ends in _abs) keeps its type", "default": "", "nullius": "option:question_type"},
8
+ {"entry": "data/custom_history/sample_haystack_and_timestamp.py", "setting": "task n_questions min_n_haystack_filler max_n_haystack_filler enforce_json_length", "values": "a task name, counts, a length (80 / 500 sessions, 115000 for S)", "default": "", "nullius": "not offered - it compiles new histories from a corpus released on Google Drive without a digest; the released files are the paper's histories, offered as datasets"},
9
+ {"entry": "src/generation/run_generation.sh", "setting": "$1 in_file", "values": "a released file, or a retrieval log", "default": "required", "nullius": "fixed:the unit's question, written as a one-question file of the run's dataset (or run_retrieval.py's log of it) - one question a unit"},
10
+ {"entry": "src/generation/run_generation.sh", "setting": "$2 model_alias gpt-4o, gpt-4o-mini", "values": "gpt-4o (gpt-4o-2024-08-06), gpt-4o-mini (gpt-4o-mini-2024-07-18)", "default": "required", "nullius": "setup:original"},
11
+ {"entry": "src/generation/run_generation.sh", "setting": "$2 model_alias, another model in the reader's place", "values": "any", "default": "", "nullius": "setup:model"},
12
+ {"entry": "src/generation/run_generation.sh", "setting": "$2 model_alias, open-weight readers", "values": "llama-3-8b/70b-instruct, llama-3.1-8b/70b-instruct, mistral-7b-instruct-v0.2/v0.3, film-7b, phi-3-medium-128k-instruct, phi-3.5-mini-instruct, phi-4", "default": "", "nullius": "not offered - served by vLLM (src/utils/serve_vllm.sh) and tokenized with their Hugging Face tokenizers (gated weights): local weights, not a model the run's relay carries, and the 8B ones in full precision pass an 11 GB GPU. The model setup puts any served model in gpt-4o's place"},
13
+ {"entry": "src/generation/run_generation.sh", "setting": "$3 retriever_alias full-history-session, full-history-turn, no-retrieval, flat-bm25-session, flat-bm25-turn", "values": "as listed", "default": "required", "nullius": "option:retriever"},
14
+ {"entry": "src/generation/run_generation.sh", "setting": "$3 retriever_alias flat-contriever-*, flat-stella-*", "values": "session, turn", "default": "", "nullius": "option:retriever"},
15
+ {"entry": "src/generation/run_generation.sh", "setting": "$3 retriever_alias flat-gtr-*", "values": "session, turn", "default": "", "nullius": "not offered - run_retrieval.py has no gtr retriever (its dense one is flat-gte, gte-Qwen2-7B-instruct), so the alias reads a log no script writes; flat-gte itself loads a 7B model in full precision, about 30 GB of GPU memory"},
16
+ {"entry": "src/generation/run_generation.sh", "setting": "$4 topk_context", "values": "an integer", "default": "50", "nullius": "option:topk"},
17
+ {"entry": "src/generation/run_generation.sh", "setting": "$5 history_format", "values": "json, nl", "default": "json", "nullius": "option:history_format"},
18
+ {"entry": "src/generation/run_generation.sh", "setting": "$6 useronly", "values": "false, true", "default": "false", "nullius": "option:useronly"},
19
+ {"entry": "src/generation/run_generation.sh", "setting": "$7 reading_method", "values": "direct, con, con-separate", "default": "con", "nullius": "option:reading"},
20
+ {"entry": "src/generation/run_generation.sh", "setting": "$8 merge_key_expansion_into_value", "values": "none, merge, replace", "default": "none", "nullius": "option:merge_key_expansion"},
21
+ {"entry": "src/generation/run_generation.sh", "setting": "$9 suffix", "values": "a string", "default": "none", "nullius": "fixed:not passed - the output is the unit's own folder, so no suffix tells runs apart"},
22
+ {"entry": "src/generation/run_generation.sh", "setting": "openai_key, openai_org_flag", "values": "YOUR_API_KEY, --openai_organization YOUR_ORGANIZATION", "default": "", "nullius": "fixed:the relay's placeholder key and no organization - the real key stays in the relay"},
23
+ {"entry": "src/generation/run_generation.sh", "setting": "openai_base_url_flag", "values": "none for gpt-4o and gpt-4o-mini; http://localhost:8001/v1 for the others", "default": "", "nullius": "fixed:the unit's relay (--openai_base_url) - it sends the setup's model as the provider names it"},
24
+ {"entry": "src/generation/run_generation.sh", "setting": "out_dir", "values": "generation_logs/<retriever>/<model>/<reading>/", "default": "", "nullius": "fixed:generation/ in the unit's folder - not the shared upstream checkout"},
25
+ {"entry": "src/generation/run_generation.sh", "setting": "HF_HOME, PYTHONIOENCODING", "values": "model_cache/, utf-8", "default": "", "nullius": "fixed:PYTHONIOENCODING utf-8; HF_HOME unused - the GPT readers count tokens with tiktoken"},
26
+ {"entry": "src/generation/run_generation.py", "setting": "--in_file, --out_dir, --out_file_suffix", "values": "paths, a string", "default": "required, required, empty", "nullius": "fixed:the unit's one-question file and folder, no suffix - one question a unit"},
27
+ {"entry": "src/generation/run_generation.py", "setting": "--model_name, --model_alias", "values": "the served name and its alias", "default": "required", "nullius": "setup:original"},
28
+ {"entry": "src/generation/run_generation.py", "setting": "--openai_base_url, --openai_key, --openai_organization", "values": "a URL, a key, an organization", "default": "None, required, None", "nullius": "fixed:the unit's relay, its placeholder key, no organization - the real key stays in the relay"},
29
+ {"entry": "src/generation/run_generation.py", "setting": "--retriever_type", "values": "orig-session, orig-turn, flat-session, flat-turn, no-retrieval", "default": "required", "nullius": "option:retriever"},
30
+ {"entry": "src/generation/run_generation.py", "setting": "--retriever_type oracle-session, oracle-turn", "values": "as listed", "default": "", "nullius": "not offered - no alias of run_generation.sh passes them, and they read the history as orig-session and orig-turn do; the paper's oracle reading is the oracle file with full-history-session (dataset longmemeval_oracle)"},
31
+ {"entry": "src/generation/run_generation.py", "setting": "--topk_context", "values": "an integer", "default": "required", "nullius": "option:topk"},
32
+ {"entry": "src/generation/run_generation.py", "setting": "--history_format", "values": "json, nl", "default": "required", "nullius": "option:history_format"},
33
+ {"entry": "src/generation/run_generation.py", "setting": "--useronly", "values": "true, false", "default": "required", "nullius": "option:useronly"},
34
+ {"entry": "src/generation/run_generation.py", "setting": "--con", "values": "true, false", "default": "false", "nullius": "option:separate_notes"},
35
+ {"entry": "src/generation/run_generation.py", "setting": "--cot", "values": "true, false", "default": "required", "nullius": "option:reading"},
36
+ {"entry": "src/generation/run_generation.py", "setting": "--merge_key_expansion_into_value", "values": "merge, replace, none", "default": "none", "nullius": "option:merge_key_expansion"},
37
+ {"entry": "src/generation/run_generation.py", "setting": "--gen_length", "values": "an integer", "default": "None (500, or 800 with --cot)", "nullius": "fixed:None - run_generation.sh never passes it, so the answer length follows the reading method"},
38
+ {"entry": "src/generation/run_generation.py", "setting": "model2maxlength, max_retrieval_length", "values": "128000 for the GPT readers; the window less the answer length less 1000", "default": "", "nullius": "fixed:as published - the history is cut to 126,200 tokens (con) by o200k_base, the start kept"},
39
+ {"entry": "src/generation/run_generation.py", "setting": "temperature, n", "values": "0, 1", "default": "", "nullius": "fixed:0, 1 - as published"},
40
+ {"entry": "src/generation/run_generation.py", "setting": "chat_completions_with_backoff", "values": "a rate limit retried every 5 s, without end", "default": "", "nullius": "fixed:as published - the unit's turn_seconds (3600 s) bounds it"},
41
+ {"entry": "src/generation/run_generation.py", "setting": "con-separate note call", "values": "max_tokens 500, temperature 0, the reader's model", "default": "", "nullius": "fixed:as published - its calls go through the same relay route, to the setup's model"},
42
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$1 in_file", "values": "a released file", "default": "required", "nullius": "fixed:the unit's one-question file - one question a unit"},
43
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$2 retriever flat-bm25", "values": "flat-bm25", "default": "required", "nullius": "option:retriever"},
44
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$2 retriever flat-contriever, flat-stella", "values": "as listed", "default": "", "nullius": "option:retriever"},
45
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$2 retriever flat-gte", "values": "gte-Qwen2-7B-instruct", "default": "", "nullius": "not offered - run_retrieval.py loads the 7B model in full precision (no dtype), about 30 GB of GPU memory, with its batch of 1; not proven on a GPU that holds it"},
46
+ {"entry": "src/retrieval/run_retrieval.py", "setting": "model2bsz (encoding batch)", "values": "contriever 128, stella 64, gte 1", "default": "", "nullius": "fixed:as published - not a flag; Stella's 64 texts of up to 512 tokens in full precision run out of memory on an 11 GB GPU, said in Good to know"},
47
+ {"entry": "src/retrieval/run_retrieval.py", "setting": "--retriever oracle", "values": "oracle", "default": "", "nullius": "not offered - it ranks the evidence first from the ids' answer marks, a check of the retrieval metrics; no alias of run_generation.sh reads its log, and the oracle file is the paper's oracle reading"},
48
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$3 granularity", "values": "session, turn", "default": "required", "nullius": "option:retriever"},
49
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$4 index_expansion session methods", "values": "none, session-summ, session-keyphrase, session-userfact", "default": "none", "nullius": "option:session_expansion"},
50
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$4 index_expansion turn methods", "values": "none, turn-keyphrase, turn-userfact", "default": "none", "nullius": "option:turn_expansion"},
51
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$5 index_expansion_join_mode", "values": "separate, split-separate, merge, split-merge, replace, split-replace, none", "default": "none", "nullius": "option:expansion_join"},
52
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$6 index_expansion_cache", "values": "a path", "default": "none", "nullius": "fixed:the method's exact cache member - from the authors' digest-pinned index_expansions archive"},
53
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$7 aux_model_alias", "values": "none, llama-3.1-8b-instruct, llama-3.1-8b-instruct-ICL, llama-3.1-70b-instruct", "default": "none", "nullius": "fixed:none - only labels output folders; expansions come from the released cache"},
54
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "$8 outfile_prefix", "values": "a string", "default": "none", "nullius": "fixed:none - the log is named after the unit's file"},
55
+ {"entry": "src/retrieval/run_retrieval.sh", "setting": "HF_HOME, cache_dir", "values": "model_cache/", "default": "", "nullius": "fixed:the repository's own model_cache/, as run_retrieval.sh sets it - shared by every question, so a model downloads once; Stella fetched into it first, as the README has it downloaded by hand"},
56
+ {"entry": "src/retrieval/run_retrieval.py", "setting": "num_processes", "values": "10 for bm25, else the GPU count", "default": "", "nullius": "fixed:as published - ten processes for one question with BM25, about 330 MB each while they start; one a GPU for a dense retriever"},
57
+ {"entry": "src/retrieval/run_retrieval.py", "setting": "retrieval metrics", "values": "recall_any, recall_all, ndcg_any at 1, 3, 5, 10, 30, 50", "default": "", "nullius": "fixed:as published - written into the unit's retrieval log, not scored: the job scores the answer"},
58
+ {"entry": "src/evaluation/evaluate_qa.py", "setting": "argv[1] metric model gpt-4o", "values": "gpt-4o (gpt-4o-2024-08-06)", "default": "required", "nullius": "fixed:gpt-4o - the paper's judge; the judge's relay route sends the run's judge model (gpt-4o-2024-08-06 unless another is asked) in its place"},
59
+ {"entry": "src/evaluation/evaluate_qa.py", "setting": "argv[1] metric model gpt-4o-mini, llama-3.1-70b-instruct", "values": "as listed", "default": "", "nullius": "not offered - not the paper's judge; another judge model is the run's choice (the builder's judge model), sent in gpt-4o's place, and llama-3.1-70b-instruct is served by a local vLLM"},
60
+ {"entry": "src/evaluation/evaluate_qa.py", "setting": "argv[2] hyp_file, argv[3] ref_file", "values": "paths", "default": "required", "nullius": "fixed:run_generation.py's test log and the unit's one-question file - one question a unit"},
61
+ {"entry": "src/evaluation/evaluate_qa.py", "setting": "OPENAI_API_KEY, OPENAI_ORGANIZATION", "values": "a key, an organization", "default": "", "nullius": "fixed:the relay's placeholder key, the judge's route as OPENAI_BASE_URL, no organization - the real key stays in the relay"},
62
+ {"entry": "src/evaluation/evaluate_qa.py", "setting": "temperature, max_tokens, n", "values": "0, 10, 1", "default": "", "nullius": "fixed:0, 10, 1 - as published"},
63
+ {"entry": "src/evaluation/evaluate_qa.py", "setting": "chat_completions_with_backoff", "values": "rate limits and API errors retried, exponentially, without end", "default": "", "nullius": "fixed:as published - the score's 600 s bound it"},
64
+ {"entry": "src/evaluation/evaluate_qa.py", "setting": "get_anscheck_prompt", "values": "a prompt per question type, abstention by the _abs id", "default": "", "nullius": "fixed:its prompt per question type, abstention by the _abs id - as published"},
65
+ {"entry": "src/evaluation/print_qa_metrics.py", "setting": "in_file ref_file", "values": "the eval log, the reference file", "default": "", "nullius": "not offered - as the README runs it (three arguments: gpt-4o first) it prints its usage and exits; its overall accuracy is the job's pass rate, and each trial carries its question_type for the task-averaged and abstention accuracies"},
66
+ {"entry": "src/evaluation/print_retrieval_metrics.py", "setting": "in_file", "values": "a retrieval log", "default": "", "nullius": "not offered - retrieval metrics, not a score of the answer; each flat-bm25 unit keeps run_retrieval.py's log with them"},
67
+ {"entry": "src/index_expansion/batch_expansion_session_summ.py", "setting": "model_name, in_file, cache_file, client", "values": "Meta-Llama-3.1-8B-Instruct; the authors' own paths; http://localhost:8001 or 8002 (vLLM)", "default": "", "nullius": "not offered - hard-coded to the authors' file paths (/home/diwu/..., /local2/diwu/...) and a local vLLM; the released index expansion caches are used instead"},
68
+ {"entry": "src/index_expansion/batch_expansion_session_keyphrases.py", "setting": "model_name, in_file, cache_file, client", "values": "Meta-Llama-3.1-8B-Instruct; the authors' own paths; http://localhost:8001 or 8002 (vLLM)", "default": "", "nullius": "not offered - hard-coded to the authors' file paths (/home/diwu/..., /local2/diwu/...) and a local vLLM; the released index expansion caches are used instead"},
69
+ {"entry": "src/index_expansion/batch_expansion_session_userfact.py", "setting": "model_name, in_file, cache_file, client", "values": "Meta-Llama-3.1-8B-Instruct; the authors' own paths; http://localhost:8001 or 8002 (vLLM)", "default": "", "nullius": "not offered - hard-coded to the authors' file paths (/home/diwu/..., /local2/diwu/...) and a local vLLM; the released index expansion caches are used instead"},
70
+ {"entry": "src/index_expansion/batch_expansion_turn_keyphrases.py", "setting": "model_name, in_file, cache_file, client", "values": "Meta-Llama-3.1-8B-Instruct; the authors' own paths; http://localhost:8001 or 8002 (vLLM)", "default": "", "nullius": "not offered - hard-coded to the authors' file paths (/home/diwu/..., /local2/diwu/...) and a local vLLM; the released index expansion caches are used instead"},
71
+ {"entry": "src/index_expansion/batch_expansion_turn_userfact.py", "setting": "model_name, in_file, cache_file, client", "values": "Meta-Llama-3.1-8B-Instruct; the authors' own paths; http://localhost:8001 or 8002 (vLLM)", "default": "", "nullius": "not offered - hard-coded to the authors' file paths (/home/diwu/..., /local2/diwu/...) and a local vLLM; the released index expansion caches are used instead"},
72
+ {"entry": "src/index_expansion/batch_expansion_session_temp_event.py", "setting": "model_name, in_file, cache_file, client", "values": "Meta-Llama-3.1-8B-Instruct; the authors' own paths; http://localhost:8001 or 8002 (vLLM)", "default": "", "nullius": "not offered - hard-coded to the authors' file paths (/home/diwu/..., /local2/diwu/...) and a local vLLM; the released index expansion caches are used instead"},
73
+ {"entry": "src/index_expansion/temp_query_search_pruning.py", "setting": "TIMESTAMP_EVENT_FILE RETRIEVAL_LOG GRANULARITY, model gpt-4o", "values": "a downloaded event file, a retrieval log, session or turn", "default": "", "nullius": "not offered - time-aware query expansion: it reads timestamped events released on Google Drive without a digest, and as published it stops at once (json.load is given the file's name, not the file)"},
74
+ {"entry": "src/utils/serve_vllm.sh, serve_vllm_with_maxlen.sh", "setting": "GPU MODEL [MAXLEN] PORT TP_SIZE", "values": "the open-weight readers", "default": "", "nullius": "not offered - serves the open-weight readers on local GPUs, not offered"},
75
+ {"entry": "requirements-full.txt", "setting": "torch 2.3.1 (CUDA 12.1), vllm, flash-attn", "values": "", "default": "", "nullius": "fixed:torch 2.3.1 - its CPU build in version 3, its CUDA 12.1 build in version 4 (the dense retrievers) - with rank-bm25, sentence-transformers 2.2.2 and the rest as pinned, in the driver's Python 3.9 - vllm and flash-attn serve GPU readers, not offered"}
76
+ ]
@@ -0,0 +1,87 @@
1
+ {
2
+ "key": "swe-bench-verified",
3
+ "title": "SWE-bench Verified",
4
+ "category": "Coding",
5
+ "domain": "Software engineering: fix a real GitHub issue so the failing tests pass.",
6
+ "scoring": "Tests",
7
+ "harness": [
8
+ "standard",
9
+ "harbor"
10
+ ],
11
+ "origin": "official",
12
+ "published": {
13
+ "agent": "swe-agent",
14
+ "says": "SWE-agent, the SWE-bench authors' own agent and the paper's agent baseline, which Harbor registers; a model of your choice runs it."
15
+ },
16
+ "upstream_limitations": [
17
+ "django__django-10097's graded tests pass without the fix, and the URL tests its fix is for are not among them. An empty patch is rejected, but these tests cannot distinguish the intended fix from an unrelated patch."
18
+ ],
19
+ "limitations": [
20
+ "On a shared API key, rate limits cause most failures: run few trials at once.",
21
+ "gpt-5.x models need the Responses API to reason while using tools.",
22
+ "On Nullius, the agent works in the instance's own container and the tests run there after it. Test files are reset first, so editing them does not help.",
23
+ "On Nullius, the agent can reach only its model. The tests get internet access after the agent ends.",
24
+ "The prompt is the issue's title and body, without the hints, as on the leaderboard.",
25
+ "Each instance's image is pinned. An image the authors rebuild is not used until the pin is refreshed.",
26
+ "SWE-agent, the authors' own agent, runs only on Harbor."
27
+ ],
28
+ "datasets": [
29
+ {
30
+ "key": "swe-bench-verified",
31
+ "title": "SWE-bench Verified",
32
+ "tasks": 500,
33
+ "official": "https://github.com/SWE-bench/SWE-bench",
34
+ "registry": "swebench-verified@1.0",
35
+ "hub": "",
36
+ "verifier": "The failing tests must pass and the passing ones keep passing.",
37
+ "notes": "500 human-verified tasks from 12 Python projects.",
38
+ "source": "Harbor registry"
39
+ }
40
+ ],
41
+ "report": {
42
+ "sources": [
43
+ "swe-bench",
44
+ "swebench",
45
+ "swebench-verified",
46
+ "swe-bench-verified"
47
+ ],
48
+ "file": "verifier/report.json",
49
+ "required": true,
50
+ "under_task": true,
51
+ "succeeded": "resolved",
52
+ "produced": "patch_exists",
53
+ "produced_unless": "patch_is_None",
54
+ "usable": "patch_successfully_applied",
55
+ "gained": {
56
+ "pass": [
57
+ "tests_status.FAIL_TO_PASS.success"
58
+ ],
59
+ "fail": [
60
+ "tests_status.FAIL_TO_PASS.failure"
61
+ ]
62
+ },
63
+ "held": {
64
+ "pass": [
65
+ "tests_status.PASS_TO_PASS.success"
66
+ ],
67
+ "fail": [
68
+ "tests_status.PASS_TO_PASS.failure",
69
+ "tests_status.PASS_TO_FAIL.success",
70
+ "tests_status.PASS_TO_FAIL.failure",
71
+ "tests_status.FAIL_TO_FAIL.success",
72
+ "tests_status.FAIL_TO_FAIL.failure"
73
+ ]
74
+ },
75
+ "words": {
76
+ "gained": "Fail → Pass",
77
+ "gained_abbr": "F→P",
78
+ "gained_term": "FAIL_TO_PASS",
79
+ "held": "Pass → Pass",
80
+ "held_abbr": "P→P",
81
+ "held_term": "PASS_TO_PASS",
82
+ "has_patch": true
83
+ }
84
+ },
85
+ "citation": "@inproceedings{\n jimenez2024swebench,\n title={{SWE}-bench: Can Language Models Resolve Real-world Github Issues?},\n author={Carlos E Jimenez and John Yang and Alexander Wettig and Shunyu Yao and Kexin Pei and Ofir Press and Karthik R Narasimhan},\n booktitle={The Twelfth International Conference on Learning Representations},\n year={2024},\n url={https://openreview.net/forum?id=VTF8yNQM66}\n}",
86
+ "citation_source": "https://github.com/SWE-bench/SWE-bench"
87
+ }
@@ -0,0 +1,116 @@
1
+ """SWE-bench Verified's driver for the Nullius standard (JSON lines on stdin and stdout).
2
+
3
+ Each instance is a GitHub issue of a Python project, and a container the
4
+ SWE-bench authors built for it: the repository at the issue's base commit in
5
+ ``/testbed``, its dependencies installed. The dataset (the SWE-bench
6
+ organisation's own, on Hugging Face, at a pinned revision) carries, per
7
+ instance, the problem statement, the gold patch, the image's name, the eval
8
+ script the harness runs and the name of the log parser that reads its output.
9
+
10
+ units - the instance ids, in the dataset's order
11
+ setup - the problem statement as the prompt, verbatim; the instance's own
12
+ image (``image``: the authors' prebuilt image, pulled by the digest
13
+ its tag had when pinned here, ``images.json``), ``/testbed`` as the
14
+ working directory, and the check (``verify``: the harness's eval
15
+ script for the instance, written into the container once the agent
16
+ has ended and run there with a way out to download what it installs
17
+ - it resets the test files to the base commit, applies the test
18
+ patch and runs the FAIL_TO_PASS and PASS_TO_PASS tests). The agent's
19
+ changes are tested where it made them; the harness would re-apply
20
+ them as a patch in a fresh container.
21
+ score - the harness's own grading (``swebench.harness.grading.get_eval_report``)
22
+ of the patch and eval script's output: resolved when a patch exists,
23
+ every FAIL_TO_PASS test passes and every PASS_TO_PASS test still passes. Its report is kept
24
+ as ``verifier/report.json``, in the harness's own shape; the diff
25
+ from the instance's base commit, captured before testing, is kept
26
+ as ``verifier/model.patch`` and given to the grader.
27
+ oracle - the reference control: the gold patch applied in the container with
28
+ ``git apply``, as the harness applies a prediction.
29
+
30
+ The driver never runs docker: the run does each step in the unit's container.
31
+ """
32
+ import json
33
+ import os
34
+ import shlex
35
+ import sys
36
+
37
+ import pyarrow.parquet as pq
38
+ from swebench.harness.grading import get_eval_report
39
+ from swebench.harness.utils import make_test_spec
40
+
41
+ HERE = os.path.dirname(os.path.abspath(__file__))
42
+ WORKDIR = "/testbed"
43
+ EVAL_SECONDS = 1800 # the harness's own --timeout default for one instance's tests
44
+ SOLVE_SECONDS = 300
45
+ RESULTS = "/tmp/nullius-swebench-verifier"
46
+
47
+ with open(os.path.join(HERE, "images.json")) as f:
48
+ IMAGES = json.load(f)
49
+
50
+ _rows = None
51
+
52
+
53
+ def rows() -> dict:
54
+ """The dataset's instances by id, read once from the parquet file the run checked."""
55
+ global _rows
56
+ if _rows is None:
57
+ _rows = {r["instance_id"]: r for r in pq.read_table(os.environ["NULLIUS_DATASET"]).to_pylist()}
58
+ return _rows
59
+
60
+
61
+ def row(unit: str) -> dict:
62
+ if unit not in rows():
63
+ raise ValueError(f"{unit} is not an instance of this dataset")
64
+ return rows()[unit]
65
+
66
+
67
+ def units(dataset):
68
+ return {"units": list(rows())}
69
+
70
+
71
+ def setup(unit, dir):
72
+ r = row(unit)
73
+ capture = (f"cd {WORKDIR} && git add -A && "
74
+ f"git diff --cached --binary --no-ext-diff --no-textconv {shlex.quote(r['base_commit'])} -- "
75
+ f"> {RESULTS}/model.patch && exec /bin/bash /eval.sh")
76
+ return {"prompt": r["problem_statement"], "environment": "task", "image": {"pull": IMAGES[unit]}, "workdir": WORKDIR,
77
+ "env": {"PATH": "/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin",
78
+ "CONDA_DEFAULT_ENV": "testbed", "CONDA_PREFIX": "/opt/miniconda3/envs/testbed",
79
+ "CONDA_SHLVL": "1", "CONDA_EXE": "/opt/miniconda3/bin/conda",
80
+ "CONDA_PYTHON_EXE": "/opt/miniconda3/bin/python", "_CE_CONDA": "", "_CE_M": "",
81
+ "CONDA_PROMPT_MODIFIER": "(testbed) "},
82
+ "verify": {"write": {"/eval.sh": make_test_spec(r).eval_script}, "run": ["/bin/bash", "-c", capture],
83
+ "collect": RESULTS, "seconds": EVAL_SECONDS, "network": True}}
84
+
85
+
86
+ def score(unit, dir, transcript, answer, verified=None):
87
+ if not verified:
88
+ return {"reward": 0.0, "passed": False, "detail": {"why": "the instance's tests did not run"}}
89
+ log = os.path.join(verified["dir"], "test-stdout.txt")
90
+ with open(os.path.join(verified["dir"], "model.patch")) as f:
91
+ patch = f.read()
92
+ report = get_eval_report(make_test_spec(row(unit)), {"instance_id": unit, "model_patch": patch or None}, log, True)
93
+ with open(os.path.join(verified["dir"], "report.json"), "w") as f:
94
+ json.dump(report, f, indent=1)
95
+ r = report[unit]
96
+ tests = {k: {kk: len(vv) for kk, vv in v.items()} for k, v in (r.get("tests_status") or {}).items()}
97
+ detail = {"exit": verified.get("exit"), "timed_out": verified.get("timed_out"), "resolved": r["resolved"],
98
+ "patch_successfully_applied": r["patch_successfully_applied"], "tests": tests}
99
+ if r.get("infra_failure_reason"):
100
+ detail["infra_failure"] = r["infra_failure_reason"]
101
+ return {"reward": 1.0 if r["resolved"] else 0.0, "passed": bool(r["resolved"]), "detail": detail}
102
+
103
+
104
+ def oracle(unit, dir):
105
+ return {"answer": "", "solve": {"write": {"/tmp/gold.patch": row(unit)["patch"]},
106
+ "run": ["git", "-C", WORKDIR, "apply", "-v", "/tmp/gold.patch"], "seconds": SOLVE_SECONDS}}
107
+
108
+
109
+ HANDLERS = {"units": units, "setup": setup, "score": score, "oracle": oracle}
110
+ for line in sys.stdin:
111
+ try:
112
+ req = json.loads(line)
113
+ out = HANDLERS[req.pop("op")](**req)
114
+ except Exception as e:
115
+ out = {"error": f"{type(e).__name__}: {e}"}
116
+ print(json.dumps(out), flush=True)