nullius-eval 0.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +54 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +16 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +744 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +994 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +453 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +25 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +34 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +354 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +87 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +387 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +145 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/providers.js +78 -0
- _nullius_app/static/settings/sandbox.js +83 -0
- _nullius_app/static/settings/settings.css +229 -0
- _nullius_app/static/style.css +1751 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +477 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +301 -0
- _nullius_app/templates/catalogue/benchmarks.html +92 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +446 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/settings.html +508 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +392 -0
- _nullius_app/web/catalogue.py +374 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +205 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +224 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +461 -0
- _nullius_app/web/settings.py +210 -0
- _nullius_app/web/shared.py +468 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +202 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +120 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +115 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +316 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +124 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +37 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +208 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +114 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +42 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +64 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +323 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +551 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +74 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +105 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +605 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +111 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +495 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +70 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +189 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +160 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +170 -0
- nullius/catalogue/catalog.py +413 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +85 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +83 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1624 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +339 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +416 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +172 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +112 -0
- nullius/paths.py +84 -0
- nullius/prerequisites.py +111 -0
- nullius/progress.py +294 -0
- nullius/providers.py +115 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +189 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +139 -0
- nullius/runs/live.py +705 -0
- nullius/runs/parse.py +991 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +146 -0
- nullius/secrets.py +192 -0
- nullius/settings.py +456 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +301 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +650 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1381 -0
- nullius/standard/definitions.py +1577 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +255 -0
- nullius/standard/forward.py +92 -0
- nullius/standard/hostreq.py +65 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +397 -0
- nullius/standard/job.py +541 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +209 -0
- nullius/standard/names.py +30 -0
- nullius/standard/payload.py +255 -0
- nullius/standard/ports.py +64 -0
- nullius/standard/prove.py +357 -0
- nullius/standard/recipe.py +663 -0
- nullius/standard/relay.py +629 -0
- nullius/standard/run.py +1367 -0
- nullius/standard/shard.py +71 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +409 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0rc1.dist-info/METADATA +93 -0
- nullius_eval-0.1.0rc1.dist-info/RECORD +268 -0
- nullius_eval-0.1.0rc1.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0rc1.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "locomo",
|
|
4
|
+
"label": "LoCoMo",
|
|
5
|
+
"summary": "Each question is a unit: its conversation's sessions go into the memory under test, the question is recalled against it, and LoCoMo's reader prompt answers from what came back; the answer is scored by token F1.",
|
|
6
|
+
"family": "capability",
|
|
7
|
+
"needs": [
|
|
8
|
+
"cap.memory/v1"
|
|
9
|
+
],
|
|
10
|
+
"upstream": {
|
|
11
|
+
"repository": "https://github.com/snap-research/locomo",
|
|
12
|
+
"tested": "3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376",
|
|
13
|
+
"tested_date": "2024-08-13",
|
|
14
|
+
"license": "CC-BY-NC-4.0"
|
|
15
|
+
},
|
|
16
|
+
"driver": {
|
|
17
|
+
"command": [
|
|
18
|
+
"{install}/.venv/bin/python",
|
|
19
|
+
"{here}/driver.py"
|
|
20
|
+
],
|
|
21
|
+
"install": {
|
|
22
|
+
"python": "3.9",
|
|
23
|
+
"pip": [
|
|
24
|
+
"torch==2.0.1",
|
|
25
|
+
"numpy==1.26.0",
|
|
26
|
+
"regex==2022.10.31",
|
|
27
|
+
"nltk==3.8.1",
|
|
28
|
+
"bert-score==0.3.13",
|
|
29
|
+
"transformers==4.35.0",
|
|
30
|
+
"tokenizers==0.14.1",
|
|
31
|
+
"huggingface-hub==0.17.3",
|
|
32
|
+
"pandas==2.1.2",
|
|
33
|
+
"matplotlib==3.7.0",
|
|
34
|
+
"requests==2.31.0",
|
|
35
|
+
"tqdm==4.64.1",
|
|
36
|
+
"tiktoken==0.5.2",
|
|
37
|
+
"openai==0.28.0",
|
|
38
|
+
"anthropic==0.32.0",
|
|
39
|
+
"google-generativeai==0.7.2",
|
|
40
|
+
"scipy==1.11.3",
|
|
41
|
+
"httpx==0.25.0",
|
|
42
|
+
"pydantic==1.10.13"
|
|
43
|
+
]
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"datasets": [
|
|
47
|
+
{
|
|
48
|
+
"id": "locomo10",
|
|
49
|
+
"label": "LoCoMo-10",
|
|
50
|
+
"path": "data/locomo10.json",
|
|
51
|
+
"sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
|
|
52
|
+
"units": 1540
|
|
53
|
+
}
|
|
54
|
+
],
|
|
55
|
+
"units": {
|
|
56
|
+
"noun": "question",
|
|
57
|
+
"nouns": "questions"
|
|
58
|
+
},
|
|
59
|
+
"controls": {
|
|
60
|
+
"nop": {
|
|
61
|
+
"source": "nullius",
|
|
62
|
+
"from": "Nullius: no capability runs and nothing is recalled; the answer is empty."
|
|
63
|
+
},
|
|
64
|
+
"oracle": {
|
|
65
|
+
"source": "nullius",
|
|
66
|
+
"from": "Nullius, from the authors' answers: the question's answer field in the authors' locomo10.json, as their evaluation scores against it (task_eval/evaluation.py): for a category-3 question, the part before the first ';' - in 11 of them the field goes on with the authors' reasoning, which their scorer cuts off. Given to the scorer as the answer.",
|
|
67
|
+
"in_prompt": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
"score": {
|
|
71
|
+
"kind": "score",
|
|
72
|
+
"says": "token F1 of the answer against the reference, by LoCoMo's own eval_question_answering (task_eval/evaluation.py): its normalisation and Porter stemming, category 1 by comma-separated parts, category 3 against the reference's part before the first ';' (categories 1 to 4)"
|
|
73
|
+
},
|
|
74
|
+
"options": {
|
|
75
|
+
"rag_mode": {
|
|
76
|
+
"label": "RAG mode",
|
|
77
|
+
"summary": "Whether the memory stores conversation turns, extracted facts, or session summaries.",
|
|
78
|
+
"values": [
|
|
79
|
+
"dialog",
|
|
80
|
+
"observation",
|
|
81
|
+
"summary"
|
|
82
|
+
],
|
|
83
|
+
"default": "dialog",
|
|
84
|
+
"about": {
|
|
85
|
+
"dialog": "Each session's turns, speaker by speaker, with its date. What every earlier run fed.",
|
|
86
|
+
"observation": "Each session's observations: the facts about each speaker the authors extracted, one turn a fact, its speaker's.",
|
|
87
|
+
"summary": "Each session's summary, written by the authors, one turn a session (its role: summary)."
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
"retrieval": {
|
|
91
|
+
"summary": "Whether the model receives the whole conversation or retrieves relevant turns, extracted facts, or session summaries.",
|
|
92
|
+
"values": [
|
|
93
|
+
"none",
|
|
94
|
+
"dialog",
|
|
95
|
+
"observation",
|
|
96
|
+
"summary"
|
|
97
|
+
],
|
|
98
|
+
"default": "none",
|
|
99
|
+
"about": {
|
|
100
|
+
"none": "The whole conversation in the prompt, as the paper's long-context rows.",
|
|
101
|
+
"dialog": "Each turn embedded, with its date; the reader gets the top-k turns.",
|
|
102
|
+
"observation": "The upstream's get_facts.py first writes each session's observations (gpt-3.5-turbo, as published), embedded with their dates; the reader gets the top-k.",
|
|
103
|
+
"summary": "The upstream's get_session_summaries.py first writes a summary of each session (gpt-3.5-turbo, temperature 1, as published); the reader gets the top-k summaries."
|
|
104
|
+
},
|
|
105
|
+
"needs": {
|
|
106
|
+
"dialog": {
|
|
107
|
+
"host": [
|
|
108
|
+
"cuda_gpu"
|
|
109
|
+
]
|
|
110
|
+
},
|
|
111
|
+
"observation": {
|
|
112
|
+
"host": [
|
|
113
|
+
"cuda_gpu"
|
|
114
|
+
]
|
|
115
|
+
},
|
|
116
|
+
"summary": {
|
|
117
|
+
"host": [
|
|
118
|
+
"cuda_gpu"
|
|
119
|
+
]
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
"setups": [
|
|
123
|
+
"original",
|
|
124
|
+
"model"
|
|
125
|
+
]
|
|
126
|
+
},
|
|
127
|
+
"retriever": {
|
|
128
|
+
"when": {
|
|
129
|
+
"retrieval": [
|
|
130
|
+
"dialog",
|
|
131
|
+
"observation",
|
|
132
|
+
"summary"
|
|
133
|
+
]
|
|
134
|
+
},
|
|
135
|
+
"summary": "The embedding model used to find relevant entries when retrieval is enabled.",
|
|
136
|
+
"values": [
|
|
137
|
+
"dragon",
|
|
138
|
+
"dpr"
|
|
139
|
+
],
|
|
140
|
+
"default": "dragon",
|
|
141
|
+
"about": {
|
|
142
|
+
"dragon": "facebook/dragon-plus context and query encoders, as scripts/evaluate_rag_gpts.sh runs.",
|
|
143
|
+
"dpr": "facebook/dpr-ctx_encoder and question_encoder, single-nq-base."
|
|
144
|
+
},
|
|
145
|
+
"setups": [
|
|
146
|
+
"original",
|
|
147
|
+
"model"
|
|
148
|
+
]
|
|
149
|
+
},
|
|
150
|
+
"top_k": {
|
|
151
|
+
"label": "Top-k",
|
|
152
|
+
"when": {
|
|
153
|
+
"retrieval": [
|
|
154
|
+
"dialog",
|
|
155
|
+
"observation",
|
|
156
|
+
"summary"
|
|
157
|
+
]
|
|
158
|
+
},
|
|
159
|
+
"summary": "The maximum number of entries the model receives per question when retrieval is enabled.",
|
|
160
|
+
"values": [
|
|
161
|
+
"2",
|
|
162
|
+
"5",
|
|
163
|
+
"10",
|
|
164
|
+
"25",
|
|
165
|
+
"50"
|
|
166
|
+
],
|
|
167
|
+
"default": "5",
|
|
168
|
+
"setups": [
|
|
169
|
+
"original",
|
|
170
|
+
"model"
|
|
171
|
+
]
|
|
172
|
+
}
|
|
173
|
+
},
|
|
174
|
+
"host": {
|
|
175
|
+
"cuda_gpu": {
|
|
176
|
+
"summary": "An NVIDIA GPU with CUDA, for LoCoMo's retrievers: the upstream loads their encoders with .cuda(). Its driver's nvidia-smi on PATH; the driver's own torch (2.0.1, CUDA 11.7) brings the rest.",
|
|
177
|
+
"check": [
|
|
178
|
+
"nvidia-smi"
|
|
179
|
+
]
|
|
180
|
+
}
|
|
181
|
+
},
|
|
182
|
+
"setups": {
|
|
183
|
+
"original": {
|
|
184
|
+
"summary": "LoCoMo's question answering as the paper ran it: its own task_eval/evaluate_qa.py on a whole conversation - the conversation in the prompt, cut to the reader's window, its questions 20 a call (10 for the gpt-3.5-turbo windows), or, with retrieval, the top-k of a database its own code builds, one question a call, as its RAG rows ran with gpt-3.5-turbo - its prompts, temperature 0 and token budget, scored by its own eval_question_answering, the mean token F1 over the conversation's questions of categories 1 to 4.",
|
|
185
|
+
"models": [
|
|
186
|
+
"gpt-4-turbo",
|
|
187
|
+
"gpt-3.5-turbo",
|
|
188
|
+
"gpt-3.5-turbo-16k",
|
|
189
|
+
"gpt-3.5-turbo-12k",
|
|
190
|
+
"gpt-3.5-turbo-8k",
|
|
191
|
+
"gpt-3.5-turbo-4k"
|
|
192
|
+
],
|
|
193
|
+
"default": "gpt-4-turbo",
|
|
194
|
+
"served_as": {
|
|
195
|
+
"gpt-3.5-turbo-16k": "gpt-3.5-turbo",
|
|
196
|
+
"gpt-3.5-turbo-12k": "gpt-3.5-turbo",
|
|
197
|
+
"gpt-3.5-turbo-8k": "gpt-3.5-turbo",
|
|
198
|
+
"gpt-3.5-turbo-4k": "gpt-3.5-turbo"
|
|
199
|
+
},
|
|
200
|
+
"providers": {
|
|
201
|
+
"openai": "",
|
|
202
|
+
"openrouter": "openai/"
|
|
203
|
+
},
|
|
204
|
+
"units": {
|
|
205
|
+
"noun": "conversation",
|
|
206
|
+
"nouns": "conversations"
|
|
207
|
+
},
|
|
208
|
+
"turn_seconds": 3600,
|
|
209
|
+
"side_models": [
|
|
210
|
+
"gpt-3.5-turbo"
|
|
211
|
+
]
|
|
212
|
+
},
|
|
213
|
+
"model": {
|
|
214
|
+
"summary": "The same pipeline with any model in the reader's place: LoCoMo's code asks for gpt-4-turbo - its 128k window, 20 questions a call, its token budget - and the relay sends the model chosen; with retrieval, the observations and summaries its code writes are asked of the model chosen too. Its scores compare with other runs of this setup, not with the paper.",
|
|
215
|
+
"runs_as": "gpt-4-turbo"
|
|
216
|
+
}
|
|
217
|
+
},
|
|
218
|
+
"proven": {
|
|
219
|
+
"locomo10": {
|
|
220
|
+
"units": {
|
|
221
|
+
"units": 1540,
|
|
222
|
+
"passed": "1540 ids, the same twice, unique",
|
|
223
|
+
"on": "2026-11-02"
|
|
224
|
+
},
|
|
225
|
+
"leak": {
|
|
226
|
+
"units": 1540,
|
|
227
|
+
"looked_for": 1144,
|
|
228
|
+
"short": 396,
|
|
229
|
+
"passed": "1540/1540",
|
|
230
|
+
"in_prompt": {
|
|
231
|
+
"units": 2,
|
|
232
|
+
"first": [
|
|
233
|
+
"conv-49:10",
|
|
234
|
+
"conv-50:39"
|
|
235
|
+
],
|
|
236
|
+
"why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
|
|
237
|
+
},
|
|
238
|
+
"note": "396 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 2 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
|
|
239
|
+
"on": "2026-11-02"
|
|
240
|
+
},
|
|
241
|
+
"nop": {
|
|
242
|
+
"passed": "0/1540",
|
|
243
|
+
"value": 0.0,
|
|
244
|
+
"expect": {
|
|
245
|
+
"at_most": 0.0
|
|
246
|
+
},
|
|
247
|
+
"units": 1540,
|
|
248
|
+
"job": "2026-11-02__15-03-51__locomo__locomo10__prove-nop",
|
|
249
|
+
"source": "nullius",
|
|
250
|
+
"on": "2026-11-02"
|
|
251
|
+
},
|
|
252
|
+
"oracle": {
|
|
253
|
+
"passed": "1540/1540",
|
|
254
|
+
"value": 1.0,
|
|
255
|
+
"expect": {
|
|
256
|
+
"at_least": 1.0
|
|
257
|
+
},
|
|
258
|
+
"units": 1540,
|
|
259
|
+
"job": "2026-11-02__15-03-51__locomo__locomo10__prove-oracle",
|
|
260
|
+
"source": "nullius",
|
|
261
|
+
"on": "2026-11-02"
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"entry": "scripts/evaluate_gpts.sh", "setting": "--model gpt-4-turbo --batch-size 20", "values": "gpt-4-turbo", "default": "",
|
|
3
|
+
"nullius": "setup:original"},
|
|
4
|
+
{"entry": "scripts/evaluate_gpts.sh", "setting": "--model gpt-3.5-turbo-{4k,8k,12k,16k} --batch-size 10; --model gpt-3.5-turbo (scripts/evaluate_rag_gpts.sh)", "values": "gpt-3.5-turbo-4k, gpt-3.5-turbo-8k, gpt-3.5-turbo-12k, gpt-3.5-turbo-16k (each asked as gpt-3.5-turbo, the window from MAX_LENGTH)", "default": "",
|
|
5
|
+
"nullius": "setup:original"},
|
|
6
|
+
{"entry": "scripts/evaluate_gpts.sh", "setting": "another model in the reader's place", "values": "any", "default": "",
|
|
7
|
+
"nullius": "setup:model"},
|
|
8
|
+
{"entry": "scripts/env.sh", "setting": "DATA_FILE_PATH", "values": "./data/locomo10.json", "default": "./data/locomo10.json",
|
|
9
|
+
"nullius": "dataset:locomo10"},
|
|
10
|
+
{"entry": "scripts/env.sh", "setting": "OUT_DIR, EMB_DIR, QA_OUTPUT_FILE", "values": "./outputs, locomo10_qa.json", "default": "./outputs",
|
|
11
|
+
"nullius": "fixed:the unit's own folder (answers.json, answers_stats.json) - one conversation a unit, so runs never share an output file"},
|
|
12
|
+
{"entry": "scripts/env.sh", "setting": "OPENAI_API_KEY", "values": "a key", "default": "",
|
|
13
|
+
"nullius": "fixed:the relay's placeholder, with OPENAI_API_BASE the unit's relay - the real key stays in the relay"},
|
|
14
|
+
{"entry": "scripts/env.sh", "setting": "ANTHROPIC_API_KEY, GOOGLE_API_KEY, HF_TOKEN", "values": "keys", "default": "",
|
|
15
|
+
"nullius": "not offered - only their readers read them (claude-sonnet, gemini-pro-1.0, the Hugging Face models), and none of those is offered: see their rows"},
|
|
16
|
+
{"entry": "scripts/evaluate_claude.sh", "setting": "--model claude-sonnet --batch-size 10", "values": "claude-sonnet (run_claude: claude-3-sonnet-20240229)", "default": "",
|
|
17
|
+
"nullius": "not offered - run_claude asks claude-3-sonnet-20240229 by that dated name, which Anthropic retired in July 2025: the call cannot be made as published"},
|
|
18
|
+
{"entry": "global_methods.py", "setting": "run_claude model claude-haiku", "values": "claude-3-haiku-20240307", "default": "",
|
|
19
|
+
"nullius": "not offered - no script of the paper runs it"},
|
|
20
|
+
{"entry": "scripts/evaluate_gemini.sh", "setting": "--model gemini-pro-1.0 --batch-size 20", "values": "gemini-pro-1.0 (evaluate_qa.py: models/gemini-1.0-pro-latest)", "default": "",
|
|
21
|
+
"nullius": "not offered - Google retired Gemini 1.0 Pro in 2025, and google-generativeai calls Google's own API, which the relay (OpenAI and Anthropic shapes) does not carry"},
|
|
22
|
+
{"entry": "task_eval/gemini_utils.py", "setting": "pause before each gemini-pro-1.0 call", "values": "30 s", "default": "30 s",
|
|
23
|
+
"nullius": "not offered - gemini-pro-1.0's, not offered"},
|
|
24
|
+
{"entry": "scripts/evaluate_hf_llm.sh", "setting": "--model llama3-chat-70b --use-4bit --batch-size 1", "values": "meta-llama/Meta-Llama-3-70B-Instruct (listed twice in the script's loop)", "default": "",
|
|
25
|
+
"nullius": "not offered - local weights loaded in 4 bits (bitsandbytes, which the upstream's requirements.txt does not list) with device_map auto, from a gated repository (HF_TOKEN): 70B in 4 bits is about 35 GB, past an 11 GB GPU, and the reader is not a model the run's relay carries"},
|
|
26
|
+
{"entry": "scripts/evaluate_hf_llm.sh", "setting": "--model mistral-instruct-7b-32k-v2 --use-4bit --batch-size 1", "values": "mistralai/Mistral-7B-Instruct-v0.2", "default": "",
|
|
27
|
+
"nullius": "not offered - local weights in 4 bits from a gated repository (HF_TOKEN, bitsandbytes not in requirements.txt); 7B fits a GPU of 11 GB, but the reader would be weights on the machine, not a model the run's relay carries"},
|
|
28
|
+
{"entry": "task_eval/hf_llm_utils.py", "setting": "other Hugging Face readers", "values": "llama2, llama2-70b, llama2-chat, llama2-chat-70b, mistral-7b-*, mistral-instruct-7b-*, gemma-7b-it", "default": "",
|
|
29
|
+
"nullius": "not offered - no script of the paper runs them, and they are local weights as the two it runs"},
|
|
30
|
+
{"entry": "task_eval/hf_llm_utils.py", "setting": "temperature, ANS_TOKENS_PER_QUES, MAX_LENGTH", "values": "0.4, 50, 4096 to 128000 by model", "default": "",
|
|
31
|
+
"nullius": "not offered - the Hugging Face readers' settings, not offered"},
|
|
32
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--model", "values": "a name holding gpt, claude, gemini, or gemma, llama, mistral", "default": "required",
|
|
33
|
+
"nullius": "setup:original"},
|
|
34
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--data-file", "values": "a LoCoMo file", "default": "required",
|
|
35
|
+
"nullius": "dataset:locomo10"},
|
|
36
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--out-file", "values": "a path", "default": "required",
|
|
37
|
+
"nullius": "fixed:answers.json in the unit's folder - read back by score"},
|
|
38
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--batch-size", "values": "an integer", "default": "1",
|
|
39
|
+
"nullius": "fixed:20 for gpt-4-turbo, 10 for the gpt-3.5-turbo windows - as scripts/evaluate_gpts.sh runs each; the model setup takes gpt-4-turbo's 20"},
|
|
40
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--overwrite", "values": "on, off", "default": "off",
|
|
41
|
+
"nullius": "fixed:off - each unit writes a new file, so there is nothing to overwrite"},
|
|
42
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--use-4bit", "values": "on, off", "default": "off",
|
|
43
|
+
"nullius": "not offered - the Hugging Face readers' only, not offered"},
|
|
44
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--use-rag", "values": "on, off", "default": "off",
|
|
45
|
+
"nullius": "option:retrieval"},
|
|
46
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--rag-mode", "values": "dialog, observation, summary", "default": "(none)",
|
|
47
|
+
"nullius": "option:retrieval"},
|
|
48
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--rag-mode (a memory under test)", "values": "dialog, observation, summary", "default": "dialog",
|
|
49
|
+
"nullius": "option:rag_mode"},
|
|
50
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--retriever", "values": "dragon, contriever, dpr, openai", "default": "contriever",
|
|
51
|
+
"nullius": "option:retriever"},
|
|
52
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--top-k", "values": "an integer", "default": "5",
|
|
53
|
+
"nullius": "option:top_k"},
|
|
54
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--emb-dir", "values": "a folder", "default": "(the working directory)",
|
|
55
|
+
"nullius": "fixed:the unit's own folder - the database pickles beside its answers, one conversation a unit"},
|
|
56
|
+
{"entry": "scripts/evaluate_rag_gpts.sh", "setting": "--model gpt-3.5-turbo --batch-size 1 --use-rag --retriever dragon", "values": "gpt-3.5-turbo, dragon", "default": "",
|
|
57
|
+
"nullius": "option:retrieval"},
|
|
58
|
+
{"entry": "scripts/evaluate_rag_gpts.sh", "setting": "--top-k by --rag-mode", "values": "dialog and observation: 5, 10, 25, 50; summary: 2, 5, 10", "default": "",
|
|
59
|
+
"nullius": "option:top_k"},
|
|
60
|
+
{"entry": "task_eval/rag_utils.py", "setting": "retriever dragon", "values": "facebook/dragon-plus-query-encoder, facebook/dragon-plus-context-encoder", "default": "",
|
|
61
|
+
"nullius": "option:retriever"},
|
|
62
|
+
{"entry": "task_eval/rag_utils.py", "setting": "retriever dpr", "values": "facebook/dpr-ctx_encoder-single-nq-base, facebook/dpr-question_encoder-single-nq-base", "default": "",
|
|
63
|
+
"nullius": "option:retriever"},
|
|
64
|
+
{"entry": "task_eval/rag_utils.py", "setting": "retriever contriever", "values": "facebook/contriever", "default": "",
|
|
65
|
+
"nullius": "not offered - as published it fails: init_query_model reads context_tokenizer, which it never defines (NameError), and get_embeddings pools with the input list's attention_mask"},
|
|
66
|
+
{"entry": "task_eval/rag_utils.py", "setting": "retriever openai", "values": "text-embedding-ada-002 (global_methods.get_openai_embedding)", "default": "",
|
|
67
|
+
"nullius": "not offered - get_embeddings passes the whole input list, not the batch, for each batch of 24, and get_openai_embedding asks for all of it once per text: n texts make ceil(n/24) x n calls of n texts, and ceil(n/24) x n rows that prepare_for_rag's assert refuses once n > 24. Tried on 2026-10-16 through the relay: conv-26's 419 turns sent 710 calls of 419 texts (16,883 tokens each, about $1.20) in 10 minutes before it was stopped, of the 7,542 the context alone would take before failing"},
|
|
68
|
+
{"entry": "task_eval/rag_utils.py", "setting": "batch_size", "values": "24", "default": "24",
|
|
69
|
+
"nullius": "fixed:24 - get_embeddings' own"},
|
|
70
|
+
{"entry": "scripts/generate_observations.sh", "setting": "task_eval/get_facts.py --use-date --overwrite (--retriever dragon by default) - retrieval=observation", "values": "gpt-3.5-turbo extraction (run_chatgpt_with_examples, 500 tokens, temperature 1), then embeddings", "default": "",
|
|
71
|
+
"nullius": "option:retrieval"},
|
|
72
|
+
{"entry": "scripts/generate_session_summaries.sh", "setting": "task_eval/get_session_summaries.py --use-date (--retriever dragon by default) - retrieval=summary, run with --data-file, the flag the script misspells", "values": "gpt-3.5-turbo summaries (256 tokens, temperature 1), then embeddings", "default": "",
|
|
73
|
+
"nullius": "option:retrieval"},
|
|
74
|
+
{"entry": "scripts/generate_conversations.sh", "setting": "generative_agents/generate_conversations.py", "values": "--num-sessions, --num-days, --num-events, --max-turns-per-session, --persona, --blip-caption, ...", "default": "",
|
|
75
|
+
"nullius": "not offered - it makes new conversations, not an evaluation of these"},
|
|
76
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "MAX_LENGTH", "values": "gpt-4-turbo 128000, gpt-3.5-turbo-16k 16000, -12k 12000, -8k 8000, -4k 4000, gpt-3.5-turbo 4096, gpt-4 4096, gpt-4-32k 320000", "default": "",
|
|
77
|
+
"nullius": "fixed:the upstream's table, by the model's name - the setups' windows (the model setup takes gpt-4-turbo's)"},
|
|
78
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "PER_QA_TOKEN_BUDGET", "values": "50", "default": "50",
|
|
79
|
+
"nullius": "fixed:50 - the upstream's constant: max_tokens is 50 a question of the batch, and the window is cut to leave it"},
|
|
80
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "temperature", "values": "0", "default": "0",
|
|
81
|
+
"nullius": "fixed:0 - as get_gpt_answers asks"},
|
|
82
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "num_tokens_request for one question", "values": "32", "default": "32",
|
|
83
|
+
"nullius": "fixed:32 - get_gpt_answers' batch-of-one call, which the memory path makes for each question"},
|
|
84
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "prompts (CONV_START_PROMPT, QA_PROMPT_BATCH, QA_PROMPT)", "values": "the upstream's text", "default": "",
|
|
85
|
+
"nullius": "fixed:the upstream's, imported - QA_PROMPT over what the memory recalled on the memory path"},
|
|
86
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "reply parsing (process_ouput), 3 trials", "values": "", "default": "",
|
|
87
|
+
"nullius": "fixed:as published - a reply it cannot parse stops the conversation (the setups), said in the limitations"},
|
|
88
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "pause before each gpt-4 call", "values": "5 s", "default": "5 s",
|
|
89
|
+
"nullius": "fixed:5 s in the setups, evaluate_qa.py run whole - the memory path's one-question call does not make it"},
|
|
90
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "date hint for category 2", "values": "' Use DATE of CONVERSATION to answer with an approximate date.'", "default": "",
|
|
91
|
+
"nullius": "fixed:the upstream's words - read from get_gpt_answers' source, never copied"},
|
|
92
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "tiktoken encoding", "values": "gpt-3.5-turbo-16k's for the windows, else the model's", "default": "",
|
|
93
|
+
"nullius": "fixed:as published - the upstream's own call"},
|
|
94
|
+
{"entry": "global_methods.py", "setting": "run_chatgpt role and retries", "values": "gpt-3.5 as system, gpt-4 as user; API, connection and unavailable errors retried without end, a rate limit with no wait", "default": "",
|
|
95
|
+
"nullius": "fixed:as published - the run's turn_seconds bounds a unit, said in the limitations"},
|
|
96
|
+
{"entry": "data/locomo10.json", "setting": "question categories 1 to 4", "values": "1,540 questions: multi-hop, temporal, open-domain, single-hop", "default": "",
|
|
97
|
+
"nullius": "dataset:locomo10"},
|
|
98
|
+
{"entry": "data/locomo10.json", "setting": "question category 5 (adversarial)", "values": "446 questions", "default": "",
|
|
99
|
+
"nullius": "not offered - the published file gives them adversarial_answer and no answer, and get_gpt_answers formats qa['answer'] into the question (a KeyError), so as published they cannot be asked; its option order is also drawn by an unseeded random.random"},
|
|
100
|
+
{"entry": "data/locomo10.json", "setting": "event_summary (event summarization task)", "values": "annotated events per speaker and session", "default": "",
|
|
101
|
+
"nullius": "not offered - the README's event summarization evaluation is 'Coming soon!': there is no code to run"},
|
|
102
|
+
{"entry": "README.MD", "setting": "multimodal dialog generation (MiniGPT-5)", "values": "", "default": "",
|
|
103
|
+
"nullius": "not offered - 'Coming soon!' in the README: there is no code to run"},
|
|
104
|
+
{"entry": "task_eval/evaluation.py", "setting": "eval_question_answering (F1)", "values": "token F1, normalisation and Porter stemming; category 1 by comma-separated parts; category 3 against the part before the first ';'", "default": "",
|
|
105
|
+
"nullius": "fixed:the upstream's, imported - every path scores with it"},
|
|
106
|
+
{"entry": "task_eval/evaluation_stats.py", "setting": "analyze_aggr_acc", "values": "per-category accuracy and memory-distance breakdowns", "default": "",
|
|
107
|
+
"nullius": "fixed:written by evaluate_qa.py beside the answers (answers_stats.json) in the setups - the run's score is the mean F1 the answers carry"},
|
|
108
|
+
{"entry": "task_eval/rag_utils.py", "setting": "the encoders' device", "values": ".cuda()", "default": "",
|
|
109
|
+
"nullius": "fixed:a CUDA GPU, as published - the host requirement cuda_gpu, checked by nvidia-smi; the driver's torch 2.0.1 is its CUDA 11.7 build (version 3)"},
|
|
110
|
+
{"entry": "task_eval/get_facts.py, get_session_summaries.py", "setting": "the extraction model", "values": "gpt-3.5-turbo (run_chatgpt_with_examples, run_chatgpt 'chatgpt')", "default": "",
|
|
111
|
+
"nullius": "fixed:gpt-3.5-turbo in the original, whatever the reader - sent as the provider names it (the setup's side_models); the model chosen, in the model setup"}
|
|
112
|
+
]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "longmemeval",
|
|
3
|
+
"title": "LongMemEval",
|
|
4
|
+
"category": "Memory",
|
|
5
|
+
"domain": "Long-term chat memory: questions over timestamped chat histories.",
|
|
6
|
+
"scoring": "GPT-4o judge",
|
|
7
|
+
"harness": [
|
|
8
|
+
"standard"
|
|
9
|
+
],
|
|
10
|
+
"origin": "official",
|
|
11
|
+
"good_to_know": [
|
|
12
|
+
"The S, M and oracle files run cleaned, and S and M also as first released. The paper's numbers come from the files as first released.",
|
|
13
|
+
"An M file is 2.7 GB and needs about 5.5 GB of free disk. STANDARD_DATA_DIR moves the datasets to a disk with room.",
|
|
14
|
+
"Reading is chain-of-note by default, or direct. Separate notes work with Original memory only.",
|
|
15
|
+
"Original memory offers the paper's Contriever and Stella retrievers and needs a CUDA GPU. Stella needs more than 11 GB of GPU memory."
|
|
16
|
+
],
|
|
17
|
+
"upstream_limitations": [
|
|
18
|
+
"Scores compare with the paper only with its reader, GPT-4o or GPT-4o-mini, and its judge, GPT-4o.",
|
|
19
|
+
"A reader you choose must take about 126,200 tokens. The history is not cut to fit, so a longer recall fails at the model.",
|
|
20
|
+
"An empty reply from the reader fails the question.",
|
|
21
|
+
"Three combinations fail as published: merge expansion with a full-history retriever, user-only with flat-bm25-turn, and separate notes with chain-of-note reading and nl history.",
|
|
22
|
+
"The judge, a model, rejects about one right answer in twenty, and passes an empty answer now and then."
|
|
23
|
+
],
|
|
24
|
+
"limitations": [
|
|
25
|
+
"The GTE retriever, the index and query expansions, and the open-weight readers are not offered."
|
|
26
|
+
],
|
|
27
|
+
"datasets": [
|
|
28
|
+
{
|
|
29
|
+
"key": "longmemeval-s",
|
|
30
|
+
"title": "LongMemEval-S (cleaned)",
|
|
31
|
+
"tasks": 500,
|
|
32
|
+
"official": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned",
|
|
33
|
+
"source": "Hugging Face",
|
|
34
|
+
"registry": "",
|
|
35
|
+
"hub": "",
|
|
36
|
+
"verifier": "LongMemEval's own judge, GPT-4o (gpt-4o-2024-08-06): yes or no per answer.",
|
|
37
|
+
"notes": "About 40 sessions and 115k tokens of history per question."
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"key": "longmemeval-m",
|
|
41
|
+
"title": "LongMemEval-M (cleaned)",
|
|
42
|
+
"tasks": 500,
|
|
43
|
+
"official": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned",
|
|
44
|
+
"source": "Hugging Face",
|
|
45
|
+
"registry": "",
|
|
46
|
+
"hub": "",
|
|
47
|
+
"verifier": "LongMemEval's own judge, GPT-4o (gpt-4o-2024-08-06): yes or no per answer.",
|
|
48
|
+
"notes": "About 500 sessions per question."
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"key": "longmemeval-oracle",
|
|
52
|
+
"title": "LongMemEval oracle \u00b7 evidence sessions only",
|
|
53
|
+
"tasks": 500,
|
|
54
|
+
"official": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned",
|
|
55
|
+
"source": "Hugging Face",
|
|
56
|
+
"registry": "",
|
|
57
|
+
"hub": "",
|
|
58
|
+
"verifier": "LongMemEval's own judge, GPT-4o (gpt-4o-2024-08-06): yes or no per answer.",
|
|
59
|
+
"notes": "Only the evidence sessions: the ceiling a memory layer is read against."
|
|
60
|
+
}
|
|
61
|
+
],
|
|
62
|
+
"citation": "@article{wu2024longmemeval,\n title={LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory},\n author={Di Wu and Hongwei Wang and Wenhao Yu and Yuwei Zhang and Kai-Wei Chang and Dong Yu},\n year={2024},\n eprint={2410.10813},\n archivePrefix={arXiv},\n primaryClass={cs.CL},\n url={https://arxiv.org/abs/2410.10813},\n}",
|
|
63
|
+
"citation_source": "https://github.com/xiaowu0162/LongMemEval"
|
|
64
|
+
}
|