nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,135 @@
1
+ {
2
+ "schema": "nullius.capability/v1",
3
+ "key": "mem0",
4
+ "label": "Mem0",
5
+ "summary": "Mem0's memory layer: an LLM extracts facts from conversations, recalled by hybrid search, optionally reranked.",
6
+ "kind": "memory",
7
+ "upstream": {
8
+ "repository": "https://github.com/mem0ai/mem0",
9
+ "tag": "v2.2.1",
10
+ "tested": "94c3fe9f238f3dbf29c9ce98643bd71eb13077cd",
11
+ "tested_date": "2026-11-02",
12
+ "package": "mem0ai",
13
+ "version": "2.2.1",
14
+ "license": "Apache-2.0"
15
+ },
16
+ "install": {
17
+ "python": "3.12",
18
+ "pip": [
19
+ "mem0ai==2.2.1",
20
+ "fastembed==0.9.1",
21
+ "spacy==3.8.18"
22
+ ],
23
+ "wheels": [
24
+ {
25
+ "url": "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl",
26
+ "sha256": "1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85"
27
+ }
28
+ ],
29
+ "prefetch": [
30
+ "Qdrant/bm25"
31
+ ]
32
+ },
33
+ "env": {
34
+ "MEM0_TELEMETRY": "False",
35
+ "FASTEMBED_CACHE_PATH": "{install}/hf"
36
+ },
37
+ "model": {
38
+ "apis": [
39
+ "chat"
40
+ ],
41
+ "env": {
42
+ "OPENAI_BASE_URL": "{base_url}",
43
+ "OPENAI_API_KEY": "{key}",
44
+ "MEM0_LLM_MODEL": "{model}",
45
+ "MEM0_REASONING": "{opt_reasoning_effort}",
46
+ "MEM0_INFER": "{opt_infer}",
47
+ "MEM0_EMBEDDER": "{opt_embedder}",
48
+ "MEM0_TOP_K": "{opt_top_k}",
49
+ "MEM0_THRESHOLD": "{opt_threshold}",
50
+ "MEM0_RERANK": "{opt_rerank}"
51
+ },
52
+ "side_models": [
53
+ "text-embedding-3-small",
54
+ "text-embedding-3-large"
55
+ ]
56
+ },
57
+ "options": {
58
+ "reasoning_effort": {
59
+ "summary": "How much reasoning the model uses before answering, with `default` leaving the choice to the provider.",
60
+ "values": [
61
+ "default",
62
+ "minimal",
63
+ "low",
64
+ "medium",
65
+ "high"
66
+ ],
67
+ "default": "default"
68
+ },
69
+ "infer": {
70
+ "summary": "Whether Mem0's LLM extracts facts from each session (`on`, Mem0's default) or the messages are stored as they are (`off`).",
71
+ "values": [
72
+ "on",
73
+ "off"
74
+ ],
75
+ "default": "on"
76
+ },
77
+ "embedder": {
78
+ "summary": "The embedding model Mem0 indexes and searches with; Mem0's default is text-embedding-3-small.",
79
+ "values": [
80
+ "text-embedding-3-small",
81
+ "text-embedding-3-large"
82
+ ],
83
+ "default": "text-embedding-3-small"
84
+ },
85
+ "top_k": {
86
+ "summary": "How many memories a search returns at most; Mem0's default is 20.",
87
+ "values": [
88
+ "5",
89
+ "10",
90
+ "20",
91
+ "50"
92
+ ],
93
+ "default": "20"
94
+ },
95
+ "threshold": {
96
+ "summary": "The lowest score a memory needs to be returned; Mem0's default is 0.1.",
97
+ "values": [
98
+ "0",
99
+ "0.1",
100
+ "0.3",
101
+ "0.5"
102
+ ],
103
+ "default": "0.1"
104
+ },
105
+ "rerank": {
106
+ "summary": "Whether search results are reranked by Mem0's LLM reranker, with the run's model (`llm`); Mem0 does not rerank by default.",
107
+ "values": [
108
+ "off",
109
+ "llm"
110
+ ],
111
+ "default": "off"
112
+ }
113
+ },
114
+ "start": {
115
+ "command": [
116
+ "{install}/.venv/bin/python",
117
+ "{here}/capability.py"
118
+ ]
119
+ },
120
+ "offers": [
121
+ "cap.memory/v1"
122
+ ],
123
+ "proven": {
124
+ "cap.memory/v1": {
125
+ "passed": "2/2",
126
+ "on": "2026-11-02",
127
+ "model": "openai/gpt-5-mini",
128
+ "provider": "openrouter",
129
+ "version": "2.2.1",
130
+ "kit_version": 1,
131
+ "backend": "docker",
132
+ "job": "2026-11-02__18-28-27__kit__cap-memory-v1__mem0"
133
+ }
134
+ }
135
+ }
@@ -0,0 +1,173 @@
1
+ #!/usr/bin/env python3
2
+ """Mem0 behind cap.memory/v1 (JSON lines on stdin and stdout).
3
+
4
+ It runs in the environment its definition installs - ``mem0ai`` at the pinned
5
+ release - and calls Mem0's own ``Memory`` as its open-source quickstart does:
6
+ its OpenAI LLM extracts facts (``add`` with ``infer``), its OpenAI embedder and
7
+ its local Qdrant store (Mem0's default vector store, the one with its keyword
8
+ search, here in a folder of this process's own: Nullius starts a process per
9
+ unit) index them, and its ``search`` recalls them, optionally reranked by its
10
+ LLM reranker. Every model call goes to ``OPENAI_BASE_URL``, the run's relay.
11
+ Nothing of Mem0 is changed.
12
+
13
+ ``add {session}`` - the session's turns as Mem0 messages (a role other than
14
+ user or assistant is a user turn naming its speaker),
15
+ added for one user, the session's id and date as metadata,
16
+ the date also as ``timestamp``
17
+ ``recall {question}`` - ``{memory}``: what Mem0's search returns, best first,
18
+ each memory headed by its session's date
19
+
20
+ Each reply also carries ``trace``, what Mem0 did, for the run's trajectory (the
21
+ benchmark reads only ``memory``): for ``add``, the memories it wrote with their
22
+ events, how many it holds, and its warnings - an extraction that failed is
23
+ logged by Mem0 and stores nothing, so it shows here and nowhere else; for
24
+ ``recall``, each hit with its id, score, score details (``explain``) and dates.
25
+
26
+ The settings, each Mem0's own default unless a run sets it:
27
+ ``MEM0_LLM_MODEL`` the run's model; ``MEM0_REASONING`` a reasoning effort
28
+ (``default``: the provider's); ``MEM0_INFER`` ``on`` (facts extracted by the
29
+ LLM) or ``off`` (the messages stored as they are); ``MEM0_EMBEDDER`` the
30
+ embedding model; ``MEM0_TOP_K`` and ``MEM0_THRESHOLD`` the search's;
31
+ ``MEM0_RERANK`` ``off`` or ``llm`` (Mem0's LLM reranker, the run's model).
32
+ """
33
+ import json
34
+ import logging
35
+ import os
36
+ import shutil
37
+ import sys
38
+ import tempfile
39
+ import time
40
+
41
+ os.environ.setdefault("MEM0_TELEMETRY", "False") # read when mem0 is imported: no calls home
42
+
43
+ USER = "nullius"
44
+ DIMS = {"text-embedding-3-small": 1536, "text-embedding-3-large": 3072}
45
+
46
+
47
+ def setting(name, default):
48
+ v = os.environ.get(name) or ""
49
+ return default if v in ("", "default") else v
50
+
51
+
52
+ def config(work):
53
+ llm = {"model": os.environ["MEM0_LLM_MODEL"]}
54
+ effort = setting("MEM0_REASONING", "")
55
+ if effort:
56
+ llm["reasoning_effort"] = effort
57
+ embedder = setting("MEM0_EMBEDDER", "text-embedding-3-small")
58
+ out = {
59
+ "llm": {"provider": "openai", "config": llm},
60
+ "embedder": {"provider": "openai", "config": {"model": embedder}},
61
+ "vector_store": {"provider": "qdrant", "config": {"path": os.path.join(work, "qdrant"), "collection_name": "nullius",
62
+ "embedding_model_dims": DIMS.get(embedder, 1536)}},
63
+ "history_db_path": os.path.join(work, "history.db"),
64
+ }
65
+ if setting("MEM0_RERANK", "off") == "llm":
66
+ out["reranker"] = {"provider": "llm_reranker", "config": {"provider": "openai", "model": llm["model"]}}
67
+ return out
68
+
69
+
70
+ class Warnings(logging.Handler):
71
+ """What Mem0 logs at warning and above while one request runs."""
72
+ def __init__(self):
73
+ super().__init__(logging.WARNING)
74
+ self.got = []
75
+
76
+ def emit(self, record):
77
+ self.got.append(f"{record.levelname.lower()}: {record.getMessage()}"[:500])
78
+
79
+ def take(self):
80
+ got, self.got = self.got, []
81
+ return got
82
+
83
+
84
+ class Memory:
85
+ def __init__(self):
86
+ self.mem, self.work, self.dates = None, None, {}
87
+ self.warnings = Warnings()
88
+ logging.getLogger("mem0").addHandler(self.warnings)
89
+
90
+ def open(self):
91
+ from mem0 import Memory as Mem0
92
+ self.work = tempfile.mkdtemp(prefix="mem0-")
93
+ self.mem = Mem0.from_config(config(self.work))
94
+
95
+ def held(self):
96
+ got = self.mem.get_all(filters={"user_id": USER}, top_k=100000)
97
+ return len(got.get("results") if isinstance(got, dict) else got or [])
98
+
99
+ def add(self, s):
100
+ if self.mem is None:
101
+ self.open()
102
+ messages = []
103
+ for t in s.get("turns") or []:
104
+ role, text = str(t.get("role") or "user"), str(t.get("text") or "")
105
+ messages.append({"role": role, "content": text} if role in ("user", "assistant")
106
+ else {"role": "user", "content": f"{role}: {text}"})
107
+ if not messages and str(s.get("text") or "").strip():
108
+ messages = [{"role": "user", "content": str(s["text"])}]
109
+ if not messages:
110
+ return {"trace": {"skipped": "an empty session"}}
111
+ sid = str(s.get("id") or f"s{len(self.dates) + 1}")
112
+ self.dates[sid] = str(s.get("date") or "")
113
+ self.warnings.take()
114
+ t0 = time.time()
115
+ meta = {"session": sid, "date": self.dates[sid]}
116
+ if self.dates[sid]:
117
+ meta["timestamp"] = self.dates[sid]
118
+ got = self.mem.add(messages, user_id=USER, metadata=meta, infer=setting("MEM0_INFER", "on") != "off")
119
+ items = got.get("results") if isinstance(got, dict) else got
120
+ return {"trace": {"messages": len(messages), "seconds": round(time.time() - t0, 3),
121
+ "written": [{k: it.get(k) for k in ("id", "event", "memory", "previous_memory") if it.get(k) is not None}
122
+ for it in items or []],
123
+ "held": self.held(), "warnings": self.warnings.take()}}
124
+
125
+ def recall(self, question):
126
+ if self.mem is None:
127
+ return {"memory": "", "trace": {"hits": [], "held": 0}}
128
+ self.warnings.take()
129
+ t0 = time.time()
130
+ got = self.mem.search(str(question), filters={"user_id": USER}, top_k=int(setting("MEM0_TOP_K", "20")),
131
+ threshold=float(setting("MEM0_THRESHOLD", "0.1")),
132
+ rerank=setting("MEM0_RERANK", "off") != "off", explain=True)
133
+ items = got.get("results") if isinstance(got, dict) else got
134
+ lines, hits = [], []
135
+ for it in items or []:
136
+ date = ((it.get("metadata") or {}).get("date") or "").strip()
137
+ lines.append(f"[{date}] {it.get('memory') or ''}" if date else str(it.get("memory") or ""))
138
+ hits.append({k: it.get(k) for k in ("id", "score", "rerank_score", "score_details", "created_at", "updated_at")
139
+ if it.get(k) is not None} | {"session": (it.get("metadata") or {}).get("session")})
140
+ return {"memory": "\n".join(lines),
141
+ "trace": {"seconds": round(time.time() - t0, 3), "hits": hits, "held": self.held(),
142
+ "warnings": self.warnings.take()}}
143
+
144
+ def close(self):
145
+ if self.work:
146
+ shutil.rmtree(self.work, ignore_errors=True)
147
+
148
+
149
+ OUT = sys.stdout # the replies; whatever Mem0 and its libraries print goes to stderr, the capability's log
150
+ sys.stdout = sys.stderr
151
+
152
+
153
+ def main():
154
+ m = Memory()
155
+ try:
156
+ for line in iter(sys.stdin.readline, ""):
157
+ try:
158
+ req = json.loads(line)
159
+ if req["op"] == "add":
160
+ out = m.add(req["session"])
161
+ elif req["op"] == "recall":
162
+ out = m.recall(req["question"])
163
+ else:
164
+ out = {"error": f"unknown op {req['op']!r}"}
165
+ except Exception as e:
166
+ out = {"error": f"{type(e).__name__}: {e}"}
167
+ print(json.dumps(out, default=str), file=OUT, flush=True)
168
+ finally:
169
+ m.close()
170
+
171
+
172
+ if __name__ == "__main__":
173
+ main()