nullius-eval 0.1.0rc2__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (313) hide show
  1. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/PKG-INFO +7 -6
  2. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/README.md +22 -20
  3. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/.env.example +14 -5
  4. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/codex/agent.json +20 -0
  5. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/goose/agent.json +12 -1
  6. nullius_eval-0.1.1/app/nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  7. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/jiuwenswarm/agent.py +45 -6
  8. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/opencode/agent.json +12 -1
  9. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  10. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  11. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  12. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  13. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  14. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  15. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/gsm8k/benchmark.json +15 -2
  16. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/gsm8k/driver.py +25 -10
  17. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/gsm8k/standard.json +49 -1
  18. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +7 -0
  19. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/humaneval/benchmark.json +0 -1
  20. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/humaneval/driver.py +42 -7
  21. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/humaneval/standard.json +1 -0
  22. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/locomo/benchmark.json +2 -1
  23. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/locomo/upstream-settings.json +2 -2
  24. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/longmemeval/benchmark.json +4 -3
  25. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/longmemeval/driver.py +63 -7
  26. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/longmemeval/standard.json +186 -4
  27. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +13 -11
  28. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/agents.json +393 -0
  29. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/benchmark.json +48 -0
  30. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/driver.py +332 -0
  31. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/nop.py +46 -0
  32. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/standard.json +274 -0
  33. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json +3061 -0
  34. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +1 -1
  35. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/driver.py +20 -9
  36. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +1 -1
  37. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/benchmark.json +2 -2
  38. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/driver.py +5 -2
  39. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +3 -0
  40. nullius_eval-0.1.1/app/nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  41. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/standard.json +58 -6
  42. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +10 -10
  43. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/terminal-bench/benchmark.json +5 -6
  44. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/terminal-bench/driver.py +84 -6
  45. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/terminal-bench/standard.json +6 -0
  46. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +5 -5
  47. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/capabilities/mem0/capability.py +6 -3
  48. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/catalog.py +26 -13
  49. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/harbor_entry.py +2 -2
  50. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/sources.py +28 -6
  51. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/cli.py +2 -0
  52. nullius_eval-0.1.1/app/nullius/constraints.py +713 -0
  53. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/execution.py +86 -55
  54. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/hosts.py +91 -13
  55. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/client.py +37 -20
  56. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/reconcile.py +4 -0
  57. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/onboarding.py +10 -6
  58. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/paths.py +6 -1
  59. nullius_eval-0.1.1/app/nullius/prerequisites.py +167 -0
  60. nullius_eval-0.1.1/app/nullius/prices.py +283 -0
  61. nullius_eval-0.1.1/app/nullius/providers.py +335 -0
  62. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/classify.py +30 -3
  63. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/launches.py +64 -3
  64. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/live.py +14 -4
  65. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/parse.py +26 -1
  66. nullius_eval-0.1.1/app/nullius/sandbox.py +109 -0
  67. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/secrets.py +30 -4
  68. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/settings.py +127 -17
  69. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/acp.py +10 -6
  70. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/cli.py +100 -14
  71. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/conform.py +2 -2
  72. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/container.py +799 -273
  73. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/definitions.py +104 -30
  74. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/fidelity.py +2 -1
  75. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/forward.py +57 -10
  76. nullius_eval-0.1.1/app/nullius/standard/hostreq.py +115 -0
  77. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/install.py +120 -48
  78. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/job.py +28 -12
  79. nullius_eval-0.1.1/app/nullius/standard/kits/agent-search-v1/driver.py +171 -0
  80. nullius_eval-0.1.1/app/nullius/standard/kits/agent-search-v1/standard.json +99 -0
  81. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/cap-memory-v1/driver.py +1 -1
  82. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/cap-memory-v1/standard.json +2 -2
  83. nullius_eval-0.1.1/app/nullius/standard/ledger.py +326 -0
  84. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/mcp_bridge.py +1 -1
  85. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/mcp_machine.py +4 -4
  86. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/merge.py +80 -20
  87. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/names.py +8 -1
  88. nullius_eval-0.1.1/app/nullius/standard/network_requirements.py +44 -0
  89. nullius_eval-0.1.1/app/nullius/standard/pace.py +53 -0
  90. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/payload.py +24 -1
  91. nullius_eval-0.1.1/app/nullius/standard/pins.py +130 -0
  92. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/ports.py +12 -0
  93. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/prove.py +22 -6
  94. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/recipe.py +136 -46
  95. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/relay.py +184 -28
  96. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/run.py +666 -129
  97. nullius_eval-0.1.1/app/nullius/standard/setup_container.py +210 -0
  98. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/shard.py +2 -0
  99. nullius_eval-0.1.1/app/nullius/standard/web_services.py +30 -0
  100. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/workspace.py +12 -4
  101. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/packaging/PYPI.md +6 -5
  102. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/README.md +4 -0
  103. nullius_eval-0.1.1/app/static/brand/docker.svg +1 -0
  104. nullius_eval-0.1.1/app/static/brand/huggingface.svg +37 -0
  105. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/builder/command-builder-core.js +30 -11
  106. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/builder/command-builder.css +1 -1
  107. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/builder/command-builder.js +87 -78
  108. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/builder/model-picker.js +1 -1
  109. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/catalogue.css +5 -35
  110. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/marks.svg +8 -0
  111. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/new.js +1 -1
  112. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/controls.js +8 -0
  113. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/notifications.js +15 -0
  114. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/onboarding.js +1 -1
  115. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/board.css +5 -1
  116. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/launches.css +22 -4
  117. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/live-board.css +1 -1
  118. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/live-board.js +16 -9
  119. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/path-suggestions.js +1 -1
  120. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/autosave.js +26 -6
  121. nullius_eval-0.1.1/app/static/settings/constraints.js +130 -0
  122. nullius_eval-0.1.1/app/static/settings/custom-keys.js +100 -0
  123. nullius_eval-0.1.1/app/static/settings/data-providers.js +111 -0
  124. nullius_eval-0.1.1/app/static/settings/hosts.js +115 -0
  125. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/judges.js +2 -2
  126. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/navigation.js +1 -1
  127. nullius_eval-0.1.1/app/static/settings/prices.js +68 -0
  128. nullius_eval-0.1.1/app/static/settings/providers.js +141 -0
  129. nullius_eval-0.1.1/app/static/settings/sandbox.js +20 -0
  130. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/settings.css +124 -11
  131. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/style.css +68 -28
  132. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/traces/conversation.js +1 -1
  133. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/agents/entry.html +2 -2
  134. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/base.html +8 -1
  135. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/benchmark.html +7 -38
  136. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/benchmarks.html +2 -3
  137. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/new.html +1 -1
  138. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/compare/compare.html +2 -2
  139. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/_execution.html +4 -4
  140. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/index.html +97 -40
  141. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/job.html +17 -3
  142. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/setup.html +1 -1
  143. nullius_eval-0.1.1/app/templates/settings/custom_keys.html +82 -0
  144. nullius_eval-0.1.1/app/templates/settings/data_providers.html +28 -0
  145. nullius_eval-0.1.1/app/templates/settings/machine_info.html +22 -0
  146. nullius_eval-0.1.1/app/templates/settings/settings.html +612 -0
  147. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/_judgement.html +1 -1
  148. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/trial.html +1 -1
  149. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/agents.py +7 -7
  150. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/builder.py +42 -22
  151. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/catalogue.py +19 -23
  152. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/execute.py +38 -20
  153. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/filters.py +8 -1
  154. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/runs.py +106 -17
  155. nullius_eval-0.1.1/app/web/settings.py +480 -0
  156. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/shared.py +41 -2
  157. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/PKG-INFO +7 -6
  158. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/SOURCES.txt +33 -0
  159. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/pyproject.toml +2 -2
  160. nullius_eval-0.1.0rc2/app/nullius/catalogue/agents/jiuwenswarm/agent.json +0 -115
  161. nullius_eval-0.1.0rc2/app/nullius/prerequisites.py +0 -111
  162. nullius_eval-0.1.0rc2/app/nullius/providers.py +0 -115
  163. nullius_eval-0.1.0rc2/app/nullius/sandbox.py +0 -146
  164. nullius_eval-0.1.0rc2/app/nullius/standard/hostreq.py +0 -65
  165. nullius_eval-0.1.0rc2/app/static/settings/providers.js +0 -78
  166. nullius_eval-0.1.0rc2/app/static/settings/sandbox.js +0 -83
  167. nullius_eval-0.1.0rc2/app/templates/settings/settings.html +0 -508
  168. nullius_eval-0.1.0rc2/app/web/settings.py +0 -210
  169. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/LICENSE +0 -0
  170. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/app.py +0 -0
  171. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/__init__.py +0 -0
  172. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/__main__.py +0 -0
  173. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/__init__.py +0 -0
  174. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/codex/config.toml +0 -0
  175. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/goose/config.yaml +0 -0
  176. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/mini-swe-agent/agent.json +0 -0
  177. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/mini-swe-agent/run.py +0 -0
  178. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/opencode/opencode.json +0 -0
  179. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/reference/agent.json +0 -0
  180. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/reference/agent.py +0 -0
  181. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/terminus-2/agent.json +0 -0
  182. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/agents/terminus-2/run.py +0 -0
  183. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/humaneval/upstream-settings.json +0 -0
  184. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/locomo/driver.py +0 -0
  185. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/locomo/standard.json +0 -0
  186. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/images.json +0 -0
  187. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/benchmarks/swe-bench-verified/standard.json +0 -0
  188. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/capabilities/keyword-memory/capability.json +0 -0
  189. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/capabilities/keyword-memory/capability.py +0 -0
  190. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/capabilities/mem0/capability.json +0 -0
  191. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/citation.py +0 -0
  192. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/catalogue/storage.py +0 -0
  193. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/compare/__init__.py +0 -0
  194. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/compare/cli.py +0 -0
  195. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/compare/comparison.py +0 -0
  196. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/gitrefs.py +0 -0
  197. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/harbor/__init__.py +0 -0
  198. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/harbor/feed.py +0 -0
  199. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/harbor/source.py +0 -0
  200. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/importers/__init__.py +0 -0
  201. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/importers/atif.py +0 -0
  202. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/importers/cli.py +0 -0
  203. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/importers/otel.py +0 -0
  204. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/__init__.py +0 -0
  205. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/batch.py +0 -0
  206. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/floors.json +0 -0
  207. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/lanes.py +0 -0
  208. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/questions.py +0 -0
  209. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/state.py +0 -0
  210. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/judge/store.py +0 -0
  211. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/migrate.py +0 -0
  212. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/progress.py +0 -0
  213. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/recipes/__init__.py +0 -0
  214. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/recipes/launch.py +0 -0
  215. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/redact.py +0 -0
  216. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/__init__.py +0 -0
  217. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/artifacts.py +0 -0
  218. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/cli.py +0 -0
  219. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/feasibility.py +0 -0
  220. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/runs/verifier.py +0 -0
  221. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/__init__.py +0 -0
  222. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/cancellation.py +0 -0
  223. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/egress.py +0 -0
  224. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/howto.py +0 -0
  225. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/jsonl.py +0 -0
  226. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-conversation-v1/driver.py +0 -0
  227. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-conversation-v1/standard.json +0 -0
  228. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-task-v1/driver.py +0 -0
  229. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-task-v1/standard.json +0 -0
  230. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-tools-v1/driver.py +0 -0
  231. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-tools-v1/mcp_kit.py +0 -0
  232. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-tools-v1/standard.json +0 -0
  233. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-workspace-v1/driver.py +0 -0
  234. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/kits/agent-workspace-v1/standard.json +0 -0
  235. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/standard/match.py +0 -0
  236. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/__init__.py +0 -0
  237. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/behavior.py +0 -0
  238. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/charts.py +0 -0
  239. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/conversation.py +0 -0
  240. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/otel.py +0 -0
  241. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/trace.py +0 -0
  242. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/traces/trajectory.py +0 -0
  243. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/nullius/workspace_cli.py +0 -0
  244. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/requirements.txt +0 -0
  245. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/favicon.svg +0 -0
  246. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/mark-dark.svg +0 -0
  247. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/marks.svg +0 -0
  248. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/tokens.json +0 -0
  249. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/typesafe.png +0 -0
  250. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/brand/wordmark-dark.svg +0 -0
  251. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/add.js +0 -0
  252. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/README.md +0 -0
  253. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/goose.svg +0 -0
  254. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/jiuwenswarm.svg +0 -0
  255. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/marks.json +0 -0
  256. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/mini-swe-agent.svg +0 -0
  257. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/agents/opencode.svg +0 -0
  258. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/catalogue.js +0 -0
  259. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/catalogue/versions.js +0 -0
  260. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/compare/comparison.css +0 -0
  261. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/disclosures.js +0 -0
  262. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/0569c822cab6c510.ttf +0 -0
  263. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/DMSans-Medium.woff2 +0 -0
  264. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/DMSans-Regular.woff2 +0 -0
  265. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  266. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  267. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  268. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  269. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/README.md +0 -0
  270. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  271. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/dmsans-OFL.txt +0 -0
  272. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/ibmplexmono-OFL.txt +0 -0
  273. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts/instrumentserif-OFL.txt +0 -0
  274. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/fonts.css +0 -0
  275. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/onboarding.css +0 -0
  276. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/board.js +0 -0
  277. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/runs/start.css +0 -0
  278. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/modal.css +0 -0
  279. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/settings/modal.js +0 -0
  280. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/static/traces/conversation.css +0 -0
  281. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/agents/kits.html +0 -0
  282. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/agents/standard.html +0 -0
  283. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/builder/new_run.html +0 -0
  284. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/add_dialog.html +0 -0
  285. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/catalogue/use_bundle.html +0 -0
  286. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/compare/_detail.html +0 -0
  287. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/compare/compare_picker.html +0 -0
  288. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/compare/compare_task.html +0 -0
  289. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/_runtime.html +0 -0
  290. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/ascii-mark.html +0 -0
  291. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/feasibility.html +0 -0
  292. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/preparing.html +0 -0
  293. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/reconcile.html +0 -0
  294. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/runs/start.html +0 -0
  295. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/_charts.html +0 -0
  296. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/_conversation.html +0 -0
  297. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/traces/artifact.html +0 -0
  298. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/templates/workspaces/workspaces.html +0 -0
  299. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/__init__.py +0 -0
  300. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/compare.py +0 -0
  301. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/core.py +0 -0
  302. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/feasibility.py +0 -0
  303. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/judge.py +0 -0
  304. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/judge_tasks.py +0 -0
  305. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/onboarding.py +0 -0
  306. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/reconcile.py +0 -0
  307. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/traces.py +0 -0
  308. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/app/web/workspaces.py +0 -0
  309. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/dependency_links.txt +0 -0
  310. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/entry_points.txt +0 -0
  311. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/requires.txt +0 -0
  312. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/nullius_eval.egg-info/top_level.txt +0 -0
  313. {nullius_eval-0.1.0rc2 → nullius_eval-0.1.1}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nullius-eval
3
- Version: 0.1.0rc2
3
+ Version: 0.1.1
4
4
  Summary: An evaluation framework for AI agents. Bring any benchmark, agent or capability, run it on your machines, and inspect the evidence behind every score. Bring anything, run it, prove it.
5
5
  Maintainer-email: ThinkFlowLab <nullius.thinkflowlab@proton.me>
6
6
  License-Expression: Apache-2.0
@@ -27,7 +27,9 @@ Dynamic: license-file
27
27
  Bring anything. Run it. Prove it.
28
28
 
29
29
  Nullius runs AI evaluations on your machines and opens each score into the
30
- evidence behind it: what failed, the agent or the benchmark.
30
+ evidence behind it, so you know whether the agent failed or the benchmark did.\
31
+ Add your own benchmark, agent or capability with a single prompt and iterate
32
+ faster.
31
33
 
32
34
  ## Install and launch
33
35
 
@@ -43,10 +45,9 @@ the application and command are `nullius`.
43
45
 
44
46
  Python 3.12 or later is required. Reading existing results needs no model key.
45
47
  Running benchmarks requires the appropriate provider credentials and benchmark
46
- prerequisites; Docker is needed for Harbor and the standard's docker backend.
48
+ prerequisites; Docker is needed for Harbor and the standard's container backend.
47
49
 
48
- This is a release candidate for testing before version `0.1.0`. Release
49
- validation targets Linux with Python 3.12. The wheel is a Python package with
50
+ Release validation targets Linux with Python 3.12. The wheel is a Python package with
50
51
  a local web UI, not a standalone desktop installer.
51
52
 
52
53
  ## What it does
@@ -72,7 +73,7 @@ Update to the latest version with:
72
73
  uv tool upgrade nullius-eval
73
74
  ```
74
75
 
75
- To select a specific version, use `uv tool install nullius-eval==0.1.0rc2`.
76
+ To select a specific version, use `uv tool install nullius-eval==0.1.1`.
76
77
  Add `--force` when replacing an existing installation. Explicit version pins
77
78
  are retained by `uv tool upgrade`; use `uv tool install --force nullius-eval`
78
79
  to return to the latest version without a pin.
@@ -8,29 +8,31 @@
8
8
  </p>
9
9
 
10
10
  Nullius runs AI evaluations on your machines and opens every score into the
11
- evidence behind it: what failed, the agent or the benchmark.
11
+ evidence behind it, so you know whether the agent failed or the benchmark did.\
12
+ Add your own benchmark, agent or capability with a single prompt and iterate
13
+ faster.
12
14
 
13
15
  ## Quickstart
14
16
 
15
17
  Python 3.12+ and [uv](https://docs.astral.sh/uv/); a model key to run
16
- benchmarks; Docker for the docker backend and for Harbor.
17
-
18
- Install the latest version with `uv`. The PyPI package is `nullius-eval`;
19
- the application and its command remain `nullius`:
18
+ benchmarks; Docker or Podman for the container backend; Docker for Harbor.
20
19
 
21
20
  ```bash
22
21
  uv tool install nullius-eval
23
22
  nullius serve
24
23
  ```
25
24
 
26
- To select a specific version, use `uv tool install nullius-eval==0.1.0rc2`.
27
- The [Quickstart guide](docs/getting-started.md) covers upgrades, wheel and
28
- source installation, and migration from a checkout.
25
+ A specific version: `uv tool install nullius-eval==0.1.1`.\
26
+ The main branch, from source:
27
+
28
+ ```bash
29
+ git clone https://github.com/thinkflowlab/nullius.git
30
+ cd nullius
31
+ uv run --no-project --with-requirements app/requirements.txt app/app.py
32
+ ```
29
33
 
30
- Open **[127.0.0.1:5057](http://127.0.0.1:5057)** to go straight to **Runs**. A first open offers a tour
31
- and a *First steps* checklist; Settings, *General*, replays it. The
32
- [Quickstart guide](docs/getting-started.md): open results, a first run, the app
33
- on another machine.
34
+ Open **[127.0.0.1:5057](http://127.0.0.1:5057)** to go straight to **Runs**.\
35
+ The [Quickstart guide](docs/getting-started.md) covers open results, a first run, the app on another machine, upgrades, wheel installation and migration from a checkout.
34
36
 
35
37
  ```bash
36
38
  # open job folders from the command line
@@ -48,9 +50,9 @@ profile. Provider keys can be saved in Settings without creating an `.env` file.
48
50
 
49
51
  ### Bring anything
50
52
 
51
- *+ Add* on Benchmarks, Agents or Capabilities writes a prompt for your coding assistant. Its import skill pins the upstream, wraps the upstream's own code, proves it and reports what fits; it asks before any code of ours. Additions go in your user catalogue and survive application upgrades.
53
+ *+ Add* on Benchmarks, Agents or Capabilities writes a prompt for your coding assistant. Its import skill pins the upstream, wraps the upstream's own code, proves it and reports what fits; it asks before adding any code of its own. Additions go in your user catalogue and survive application upgrades.
52
54
 
53
- The import skills: `/import-benchmark` (asks how it runs, Nullius or Harbor),
55
+ The import skills: `/import-benchmark`,
54
56
  `/import-standard`, `/import-agent`, `/import-capability`.
55
57
 
56
58
  <img src="docs/assets/nullius/readme/bring.svg" alt="Benchmarks, agents and capabilities come into one catalogue." width="800">
@@ -69,7 +71,7 @@ With no agent, the nop control must score nothing and the oracle everything; the
69
71
 
70
72
  ### Run anywhere
71
73
 
72
- Several agents and datasets in one run. This machine, a host over ssh, or several machines, one shard each, merged into one job by `cli merge`. The docker backend gives each unit's agent or capability its own container. On the Nullius standard a relay per unit holds the key, which the agent never sees, and records every model call.
74
+ Several agents and datasets in one run. This machine, a host over ssh, or several machines, one shard each, merged into one job by `cli merge`. The container backend gives each unit's agent or capability its own container. On the Nullius standard a relay per unit holds the key, which the agent never sees, and records every model call.
73
75
 
74
76
  The Run builder: **Runner → Benchmark → Dataset → Agent → Model → Limits → Where →
75
77
  Review**. Review checks each machine before Run; follow each trial in the job's
@@ -156,7 +158,7 @@ uv run --no-project --with-requirements app/requirements.txt \
156
158
  ```
157
159
 
158
160
  The [tests workflow](https://github.com/thinkflowlab/nullius/actions/workflows/tests.yml)
159
- runs both suites on pull requests and pushes to `main`; they can also be run locally.
161
+ runs both suites on pull requests; they can also be run locally.
160
162
 
161
163
  ## License
162
164
 
@@ -173,10 +175,10 @@ Built by the Nullius contributors. To cite it:
173
175
 
174
176
  ```bibtex
175
177
  @software{Nullius,
176
- author = {{Nullius contributors}},
177
- title = {{Nullius: the run behind a benchmark score}},
178
- year = {2026},
179
- url = {https://github.com/thinkflowlab/nullius}
178
+ author = {{Nullius contributors}},
179
+ title = {{Nullius: the run behind a benchmark score}},
180
+ year = {2026},
181
+ url = {https://github.com/thinkflowlab/nullius}
180
182
  }
181
183
  ```
182
184
 
@@ -6,6 +6,7 @@
6
6
  # Start-up options, not on the Settings page:
7
7
  # NULLIUS_HOME: a separate profile with config/, data/ and cache/ directories
8
8
  # NULLIUS_CATALOGUE: the user catalogue directory (default catalogue/ in the user data directory)
9
+ # NULLIUS_PRICES: the model price list file (default prices.json in the user configuration directory); set for a run on a host
9
10
  # APP_HOST: the address the server binds (default 127.0.0.1, or --host)
10
11
  # APP_PORT: the port the server listens on (default 5057, or --port)
11
12
  # JOB_ROOTS: job roots to open at start (or the command-line argument)
@@ -17,13 +18,13 @@
17
18
  # HARBOR_PINS: where the Harbor a recipe pins is built and kept on this machine (default harbor-pins/ beside the runs)
18
19
  # STANDARD_DIR: where the Nullius standard installs agents, capabilities and benchmark upstreams (default standard/ in the user data directory)
19
20
  # STANDARD_DATA_DIR: where the Nullius standard keeps the datasets it downloads, which can be gigabytes (default datasets/ in STANDARD_DIR)
20
- # SECRETS_DIR: where each benchmark's licences are kept, a folder per benchmark, set on its page (default secrets/ in the user configuration directory)
21
+ # SECRETS_DIR: where each benchmark's licences (Settings → Custom keys) and the Hugging Face token (Settings → Data providers) are kept (default secrets/ in the user configuration directory)
21
22
 
22
23
  # --- Model credentials ---
23
- # The model API key for a local or self-hosted model: a recipe whose provider is an OpenAI-compatible endpoint of your own is sent there, at MODEL_API_URL, with this key. Optional, for a server that checks none: without it the run is sent the word 'local' as its key. A run gets it as API_TOKEN, OPENAI_API_KEY and LLM_API_KEY in its environment - over stdin to a host - and never in a recipe or a log.
24
+ # The key of the OpenAI-compatible endpoint at MODEL_API_URL: a recipe on that endpoint is sent there with this key. Optional, for a server that checks none: without it the run is sent the word 'local' as its key. A run gets it as API_TOKEN, OPENAI_API_KEY and LLM_API_KEY in its environment - over stdin to a host - and never in a recipe or a log.
24
25
  # e.g. sk-…
25
26
  MODEL_API_KEY=
26
- # The endpoint of that local or self-hosted model, required for those recipes: there is no default to fall back on, and the builder offers the local provider only once it is set. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL.
27
+ # The URL of an OpenAI-compatible endpoint, API path included, required for a recipe on it: there is no default to fall back on, and the builder offers the endpoint only once it is set. It is shown on the page, so it holds no credential: no user or password, no key in its query. More endpoints, each with a name, are MODEL_<ID>_NAME, MODEL_<ID>_API_URL and MODEL_<ID>_API_KEY. A run gets it as API_URL, OPENAI_BASE_URL and LLM_BASE_URL.
27
28
  # e.g. https://…/v1
28
29
  MODEL_API_URL=
29
30
  # An OpenAI key, for a recipe whose provider is OpenAI: that run is sent to OpenAI's own endpoint, https://api.openai.com/v1, with this key - never with the model API key or its endpoint. It reaches the run as the model key does, and never a recipe or a log.
@@ -39,7 +40,7 @@ TYPESAFE_API_KEY=
39
40
  SYSTEM_ONE_PROVIDER=
40
41
  # The System One model, as the provider names it: on TypeSafe jev-latest, on OpenRouter ~typesafe/jev-latest, typesafe/jev-router, jaredpalmer/kev-4b... Unset: the provider's Jev.
41
42
  SYSTEM_ONE_MODEL=
42
- # The provider of the LLM judgement, the second judgement beside System One: openrouter, openai or local, asked with that provider's own key and endpoint from the model providers above. Unset: the first of them that is set.
43
+ # The provider of the LLM judgement, the second judgement beside System One: openrouter, openai, local (the endpoint at MODEL_API_URL) or local:<ID> (the endpoint MODEL_<ID>_API_URL), asked with that provider's own key and endpoint from the model providers above. Unset: the first of them that is set.
43
44
  LLM_JUDGE_PROVIDER=
44
45
  # The model of the LLM judgement, as its provider names it (on OpenRouter, vendor/model). It is asked the same questions as System One and answers them as JSON; its numbers are what it states, so a choice or a score carries no confidence.
45
46
  LLM_JUDGE_MODEL=
@@ -48,7 +49,15 @@ LLM_JUDGE_MODEL=
48
49
  # Machines a recipe can run on, over ssh: an alias from ~/.ssh/config, or user@host[:port][:/path/to/key]. Only the name is stored; keys stay where they are.
49
50
  # e.g. user@host[:port][:/path/to/key]
50
51
  HOSTS=
52
+ # What Nullius may use on a machine: a JSON object from local (this machine) or a host's entry to its constraints (cpus, memory_mb, processes, concurrency, cpu_set, gpu_ids). A constraint not set is no constraint.
53
+ # e.g. {"local": {"cpus": 4, "memory_mb": 8192}}
54
+ HOST_CONSTRAINTS=
51
55
 
52
- # --- Agent keys ---
56
+ # --- Custom keys ---
53
57
  # The names of the agent keys this file holds, comma-separated: the keys an agent needs of its own vendor, beside the model's (Cursor's CLI agent a CURSOR_API_KEY, Copilot's a GH_TOKEN). Each is a line of its own in this file. A Harbor run gets every one of them in its environment - over stdin to a host - and a Nullius standard run's never; none is ever written into a recipe or a log.
54
58
  AGENT_KEYS=
59
+
60
+ # --- Data providers ---
61
+ # Where Hugging Face downloads go: https://huggingface.co when unset, or a mirror's address. A run's dataset downloads, installs and drivers get it as HF_ENDPOINT before they load huggingface_hub; the environment wins over this file, a run on a host is sent it.
62
+ # e.g. https://hf-mirror.com
63
+ HF_ENDPOINT=
@@ -128,6 +128,16 @@
128
128
  "ok": true
129
129
  }
130
130
  }
131
+ },
132
+ "agent.search/v1": {
133
+ "passed": "3/3",
134
+ "on": "2026-11-12",
135
+ "model": "meta/muse-spark-1.3-contributor",
136
+ "provider": "openrouter",
137
+ "version": "2.2.2",
138
+ "kit_version": 1,
139
+ "backend": "docker",
140
+ "job": "2026-11-12__10-24-26__kit__agent-search-v1__codex-2.2.2"
131
141
  }
132
142
  }
133
143
  },
@@ -191,6 +201,16 @@
191
201
  "ok": true
192
202
  }
193
203
  }
204
+ },
205
+ "agent.search/v1": {
206
+ "passed": "3/3",
207
+ "on": "2026-11-12",
208
+ "model": "meta/muse-spark-1.3-contributor",
209
+ "provider": "openrouter",
210
+ "version": "0.16.0",
211
+ "kit_version": 1,
212
+ "backend": "docker",
213
+ "job": "2026-11-12__11-08-33__kit__agent-search-v1__codex-0.16.0"
194
214
  }
195
215
  },
196
216
  "upstream": {
@@ -59,7 +59,8 @@
59
59
  "agent.task/v1",
60
60
  "agent.workspace/v1",
61
61
  "agent.tools/v1",
62
- "agent.conversation/v1"
62
+ "agent.conversation/v1",
63
+ "agent.search/v1"
63
64
  ],
64
65
  "proven": {
65
66
  "agent.workspace/v1": {
@@ -115,6 +116,16 @@
115
116
  "kit_version": 1,
116
117
  "backend": "docker",
117
118
  "job": "2026-11-02__13-27-36__kit__agent-conversation-v1__goose"
119
+ },
120
+ "agent.search/v1": {
121
+ "passed": "3/3",
122
+ "on": "2026-11-12",
123
+ "model": "meta/muse-spark-1.3-contributor",
124
+ "provider": "openrouter",
125
+ "version": "1.53.0",
126
+ "kit_version": 1,
127
+ "backend": "docker",
128
+ "job": "2026-11-12__11-09-30__kit__agent-search-v1__goose"
118
129
  }
119
130
  }
120
131
  }
@@ -0,0 +1,197 @@
1
+ {
2
+ "schema": "nullius.agent/v1",
3
+ "key": "jiuwenswarm",
4
+ "label": "JiuwenSwarm",
5
+ "summary": "openJiuwen's agent.",
6
+ "start": {
7
+ "acp": [
8
+ "{install}/.venv/bin/python",
9
+ "{here}/agent.py"
10
+ ]
11
+ },
12
+ "model": {
13
+ "apis": [
14
+ "chat"
15
+ ],
16
+ "env": {
17
+ "NULLIUS_BASE_URL": "{base_url}",
18
+ "NULLIUS_KEY": "{key}",
19
+ "NULLIUS_MODEL": "{model}",
20
+ "NULLIUS_REASONING": "{opt_reasoning_effort}"
21
+ }
22
+ },
23
+ "options": {
24
+ "reasoning_effort": {
25
+ "summary": "How much reasoning the model uses before answering, with `default` leaving the choice to the provider.",
26
+ "values": [
27
+ "default",
28
+ "off",
29
+ "minimal",
30
+ "low",
31
+ "medium",
32
+ "high",
33
+ "xhigh",
34
+ "max"
35
+ ],
36
+ "default": "default"
37
+ }
38
+ },
39
+ "default": "0.2.8",
40
+ "versions": {
41
+ "0.2.7": {
42
+ "upstream": {
43
+ "repository": "https://github.com/openJiuwen-ai/jiuwenswarm",
44
+ "tested": "2aa7fea7e55fbe1edaae0d468ef666b088c76fa9",
45
+ "tested_date": "2026-10-10",
46
+ "package": "workswarm",
47
+ "license": "Apache-2.0"
48
+ },
49
+ "install": {
50
+ "python": "3.12",
51
+ "pip": [
52
+ "workswarm==0.2.7"
53
+ ]
54
+ },
55
+ "proven": {
56
+ "agent.task/v1": {
57
+ "passed": "3/3",
58
+ "on": "2026-11-02",
59
+ "model": "openai/gpt-5-mini",
60
+ "provider": "openrouter",
61
+ "version": "0.2.7",
62
+ "kit_version": 1,
63
+ "backend": "docker",
64
+ "job": "2026-11-02__13-28-04__kit__agent-task-v1__jiuwenswarm"
65
+ },
66
+ "agent.conversation/v1": {
67
+ "passed": "2/2",
68
+ "on": "2026-11-02",
69
+ "model": "openai/gpt-5-mini",
70
+ "provider": "openrouter",
71
+ "version": "0.2.7",
72
+ "kit_version": 1,
73
+ "backend": "docker",
74
+ "job": "2026-11-02__13-36-28__kit__agent-conversation-v1__jiuwenswarm"
75
+ },
76
+ "agent.workspace/v1": {
77
+ "passed": "2/2",
78
+ "on": "2026-11-02",
79
+ "model": "openai/gpt-5-mini",
80
+ "provider": "openrouter",
81
+ "version": "0.2.7",
82
+ "kit_version": 1,
83
+ "backend": "docker",
84
+ "job": "2026-11-02__13-41-01__kit__agent-workspace-v1__jiuwenswarm"
85
+ },
86
+ "agent.tools/v1": {
87
+ "passed": "2/2",
88
+ "on": "2026-11-02",
89
+ "model": "openai/gpt-5-mini",
90
+ "provider": "openrouter",
91
+ "version": "0.2.7",
92
+ "kit_version": 2,
93
+ "backend": "docker",
94
+ "job": "2026-11-02__13-33-45__kit__agent-tools-v1__jiuwenswarm",
95
+ "skills": {
96
+ "decoys": {
97
+ "passed": "1/1",
98
+ "ok": true
99
+ },
100
+ "chain": {
101
+ "passed": "1/1",
102
+ "ok": true
103
+ },
104
+ "retry": {
105
+ "passed": "1/1",
106
+ "ok": true
107
+ }
108
+ }
109
+ }
110
+ }
111
+ },
112
+ "0.2.8": {
113
+ "upstream": {
114
+ "repository": "https://github.com/openJiuwen-ai/jiuwenswarm",
115
+ "tag": "v0.2.8",
116
+ "tested": "068f148b59e140a179e31c9fe58eed8ea1700c1e",
117
+ "tested_date": "2026-11-12",
118
+ "package": "workswarm",
119
+ "license": "Apache-2.0",
120
+ "core": "openjiuwen 0.1.20 at 79f6ec656f513dadc412aaafa13f60ecf7f0fc04, pinned in install"
121
+ },
122
+ "install": {
123
+ "python": "3.12",
124
+ "pip": [
125
+ "workswarm==0.2.8",
126
+ "openjiuwen @ git+https://github.com/openJiuwen-ai/agent-core.git@79f6ec656f513dadc412aaafa13f60ecf7f0fc04"
127
+ ]
128
+ },
129
+ "proven": {
130
+ "agent.task/v1": {
131
+ "passed": "3/3",
132
+ "on": "2026-11-12",
133
+ "model": "meta/muse-spark-1.3-contributor",
134
+ "provider": "openrouter",
135
+ "version": "0.2.8",
136
+ "kit_version": 1,
137
+ "backend": "docker",
138
+ "job": "2026-11-12__14-14-10__kit__agent-task-v1__jiuwenswarm"
139
+ },
140
+ "agent.workspace/v1": {
141
+ "passed": "2/2",
142
+ "on": "2026-11-12",
143
+ "model": "meta/muse-spark-1.3-contributor",
144
+ "provider": "openrouter",
145
+ "version": "0.2.8",
146
+ "kit_version": 1,
147
+ "backend": "docker",
148
+ "job": "2026-11-12__14-15-36__kit__agent-workspace-v1__jiuwenswarm"
149
+ },
150
+ "agent.tools/v1": {
151
+ "passed": "2/2",
152
+ "on": "2026-11-12",
153
+ "model": "meta/muse-spark-1.3-contributor",
154
+ "provider": "openrouter",
155
+ "version": "0.2.8",
156
+ "kit_version": 2,
157
+ "backend": "docker",
158
+ "job": "2026-11-12__14-16-36__kit__agent-tools-v1__jiuwenswarm",
159
+ "skills": {
160
+ "decoys": {
161
+ "passed": "1/1",
162
+ "ok": true
163
+ },
164
+ "chain": {
165
+ "passed": "1/1",
166
+ "ok": true
167
+ },
168
+ "retry": {
169
+ "passed": "1/1",
170
+ "ok": true
171
+ }
172
+ }
173
+ },
174
+ "agent.conversation/v1": {
175
+ "passed": "2/2",
176
+ "on": "2026-11-12",
177
+ "model": "meta/muse-spark-1.3-contributor",
178
+ "provider": "openrouter",
179
+ "version": "0.2.8",
180
+ "kit_version": 1,
181
+ "backend": "docker",
182
+ "job": "2026-11-12__14-19-05__kit__agent-conversation-v1__jiuwenswarm"
183
+ },
184
+ "agent.search/v1": {
185
+ "passed": "3/3",
186
+ "on": "2026-11-12",
187
+ "model": "meta/muse-spark-1.3-contributor",
188
+ "provider": "openrouter",
189
+ "version": "0.2.8",
190
+ "kit_version": 1,
191
+ "backend": "docker",
192
+ "job": "2026-11-12__14-20-04__kit__agent-search-v1__jiuwenswarm"
193
+ }
194
+ }
195
+ }
196
+ }
197
+ }
@@ -30,8 +30,11 @@ changes, each taken from the protocol itself:
30
30
  that offers no files (Nullius on docker, where this wrapper runs in the
31
31
  agent's container) gets none: ``initialize`` offers them to the bridge, and
32
32
  this wrapper answers each request itself, inside the session's cwd.
33
+ Requests for the unit's own JiuwenSwarm home are served by this wrapper
34
+ on both backends, so its config and todo files remain accessible.
33
35
 
34
- When the bridge ends, the servers are stopped.
36
+ The resident worker uses a short, private socket path. When the bridge ends,
37
+ the servers and worker are stopped.
35
38
  """
36
39
  import json
37
40
  import os
@@ -39,6 +42,7 @@ import signal
39
42
  import socket
40
43
  import subprocess
41
44
  import sys
45
+ import tempfile
42
46
  import threading
43
47
  import time
44
48
 
@@ -168,11 +172,17 @@ class Sessions:
168
172
  self.agent_in.flush()
169
173
 
170
174
 
175
+ def in_home(path):
176
+ home = os.path.realpath(JHOME)
177
+ path = os.path.realpath(path)
178
+ return path == home or path.startswith(home + os.sep)
179
+
180
+
171
181
  def serve_fs(msg, cwd):
172
- """An fs/ request answered here, in the container: a path inside the session's cwd only."""
182
+ """Serve files in the session workspace or this unit's JiuwenSwarm home."""
173
183
  params, root = msg["params"], os.path.realpath(cwd or "/nonexistent")
174
184
  path = os.path.realpath(str(params.get("path") or ""))
175
- if path != root and not path.startswith(root + os.sep):
185
+ if path != root and not path.startswith(root + os.sep) and not in_home(path):
176
186
  return {"code": -32602, "message": f"{params.get('path')} is outside the session's directory"}, None
177
187
  try:
178
188
  if msg["method"] == "fs/write_text_file":
@@ -246,7 +256,8 @@ def to_client(src, dst, ses):
246
256
  path, cwd = params.get("path"), ses.of(params.get("sessionId"))
247
257
  if isinstance(path, str) and path and not os.path.isabs(path) and cwd:
248
258
  params["path"] = os.path.normpath(os.path.join(cwd, path))
249
- if ses.serve_fs and "id" in msg:
259
+ own_home = isinstance(params.get("path"), str) and in_home(params["path"])
260
+ if (ses.serve_fs or own_home) and "id" in msg:
250
261
  err, res = serve_fs(msg, ses.of(params.get("sessionId")))
251
262
  reply = {"jsonrpc": "2.0", "id": msg["id"], **({"error": err} if err else {"result": res})}
252
263
  ses.to_bridge((json.dumps(reply) + "\n").encode())
@@ -256,7 +267,26 @@ def to_client(src, dst, ses):
256
267
  dst.flush()
257
268
 
258
269
 
259
- def main():
270
+ def stop_worker(env):
271
+ """0.2.8's resident worker outlives the frontend; use its shutdown API."""
272
+ path = env["JIUWENSWARM_RUNTIME_SOCKET"]
273
+ if not os.path.exists(path):
274
+ return
275
+ code = ("import asyncio, os, sys\n"
276
+ "from jiuwenswarm.server.supervisor.worker_process import request_resident_worker_shutdown\n"
277
+ "async def stop():\n"
278
+ " await request_resident_worker_shutdown(sys.argv[1])\n"
279
+ " while os.path.exists(sys.argv[1]):\n"
280
+ " await asyncio.sleep(0.05)\n"
281
+ "asyncio.run(asyncio.wait_for(stop(), 25))\n")
282
+ try:
283
+ subprocess.run([PY, "-c", code, path], env=env, stdin=subprocess.DEVNULL,
284
+ stdout=sys.stderr, stderr=sys.stderr, timeout=30, check=True)
285
+ except (subprocess.SubprocessError, OSError) as e:
286
+ say("runtime worker shutdown failed:", e)
287
+
288
+
289
+ def run(runtime_socket):
260
290
  os.makedirs(HOME, exist_ok=True)
261
291
  if not os.path.isfile(os.path.join(CONFIG, "config.yaml")):
262
292
  r = subprocess.run([PY, "-c", "from jiuwenswarm.common.utils import prepare_workspace\n"
@@ -269,7 +299,8 @@ def main():
269
299
  reasoning_level()
270
300
  ports = free_ports(4)
271
301
  write_env(ports)
272
- env = dict(os.environ, PYTHONIOENCODING="utf-8", PYTHONUNBUFFERED="1")
302
+ env = dict(os.environ, PYTHONIOENCODING="utf-8", PYTHONUNBUFFERED="1",
303
+ JIUWENSWARM_RUNTIME_SOCKET=runtime_socket)
273
304
  server = subprocess.Popen([PY, "-m", "jiuwenswarm.app"], stdin=subprocess.DEVNULL,
274
305
  stdout=sys.stderr, stderr=sys.stderr, cwd=HOME, env=env)
275
306
  bridge = None
@@ -310,6 +341,14 @@ def main():
310
341
  return code
311
342
  finally:
312
343
  stop()
344
+ stop_worker(env)
345
+
346
+
347
+ def main():
348
+ # A long job path exceeds the Unix socket limit. Each unit owns a short,
349
+ # private directory, including on the local backend where /tmp is shared.
350
+ with tempfile.TemporaryDirectory(prefix="nj-", dir="/tmp") as runtime:
351
+ return run(os.path.join(runtime, "worker.sock"))
313
352
 
314
353
 
315
354
  if __name__ == "__main__":
@@ -63,7 +63,8 @@
63
63
  "agent.task/v1",
64
64
  "agent.workspace/v1",
65
65
  "agent.tools/v1",
66
- "agent.conversation/v1"
66
+ "agent.conversation/v1",
67
+ "agent.search/v1"
67
68
  ],
68
69
  "proven": {
69
70
  "agent.task/v1": {
@@ -119,6 +120,16 @@
119
120
  "kit_version": 1,
120
121
  "backend": "docker",
121
122
  "job": "2026-11-02__14-14-07__kit__agent-conversation-v1__opencode"
123
+ },
124
+ "agent.search/v1": {
125
+ "passed": "3/3",
126
+ "on": "2026-11-12",
127
+ "model": "meta/muse-spark-1.3-contributor",
128
+ "provider": "openrouter",
129
+ "version": "1.18.35",
130
+ "kit_version": 1,
131
+ "backend": "docker",
132
+ "job": "2026-11-12__11-10-20__kit__agent-search-v1__opencode"
122
133
  }
123
134
  }
124
135
  }
@@ -0,0 +1,72 @@
1
+ {
2
+ "key": "gaia",
3
+ "title": "GAIA",
4
+ "category": "General assistance",
5
+ "domain": "Real-world questions needing reasoning, web research and file understanding.",
6
+ "scoring": "Answer accuracy",
7
+ "harness": [
8
+ "standard"
9
+ ],
10
+ "origin": "official",
11
+ "good_to_know": [
12
+ "Agents need public internet access.",
13
+ "Accept GAIA's access conditions on Hugging Face, then add a token in Settings → Data providers. Do not redistribute its questions or answers in a crawlable form.",
14
+ "Levels 1, 2 and 3 (53, 86 and 26 questions) split the 165 validation questions.",
15
+ "The authors' scorer gets the text after the last FINAL ANSWER: marker, or the whole reply without one.",
16
+ "Original is the paper's no-tools API baseline; choose an agent for files and web research. The paper ran it 3 times: 3 attempts per question."
17
+ ],
18
+ "upstream_limitations": [
19
+ "The 301 test questions have private answers and are excluded from local scoring.",
20
+ "GAIA uses the live web; some evidence may have changed or become unavailable."
21
+ ],
22
+ "limitations": [],
23
+ "datasets": [
24
+ {
25
+ "key": "gaia-validation",
26
+ "title": "Validation · All levels",
27
+ "tasks": 165,
28
+ "official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
29
+ "source": "Hugging Face",
30
+ "registry": "",
31
+ "hub": "",
32
+ "verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
33
+ "notes": "Official 2023 validation split."
34
+ },
35
+ {
36
+ "key": "gaia-validation-level1",
37
+ "title": "Validation · Level 1",
38
+ "tasks": 53,
39
+ "official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
40
+ "source": "Hugging Face",
41
+ "registry": "",
42
+ "hub": "",
43
+ "verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
44
+ "notes": "Official 2023 validation split."
45
+ },
46
+ {
47
+ "key": "gaia-validation-level2",
48
+ "title": "Validation · Level 2",
49
+ "tasks": 86,
50
+ "official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
51
+ "source": "Hugging Face",
52
+ "registry": "",
53
+ "hub": "",
54
+ "verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
55
+ "notes": "Official 2023 validation split."
56
+ },
57
+ {
58
+ "key": "gaia-validation-level3",
59
+ "title": "Validation · Level 3",
60
+ "tasks": 26,
61
+ "official": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
62
+ "source": "Hugging Face",
63
+ "registry": "",
64
+ "hub": "",
65
+ "verifier": "The authors' scorer at the pinned commit, after Nullius extracts the final answer.",
66
+ "notes": "Official 2023 validation split."
67
+ }
68
+ ],
69
+ "citation": "@misc{mialon2023gaia,\n title={GAIA: a benchmark for General AI Assistants},\n author={Grégoire Mialon and Clémentine Fourrier and Craig Swift and Thomas Wolf and Yann LeCun and Thomas Scialom},\n year={2023},\n eprint={2311.12983},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}",
70
+ "citation_source": "https://huggingface.co/spaces/gaia-benchmark/leaderboard/blob/9f133d71362e77b3539f1514f31b9c101a545fec/content.py",
71
+ "task_count": 165
72
+ }