nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,733 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "nullius.benchmark/v1",
|
|
3
|
+
"key": "longmemeval",
|
|
4
|
+
"label": "LongMemEval",
|
|
5
|
+
"summary": "Each question is a unit: its chat history goes into the memory under test a session at a time, the question is recalled against it, LongMemEval's reader prompt answers from what came back, and LongMemEval's own judge says whether the answer is right.",
|
|
6
|
+
"family": "capability",
|
|
7
|
+
"needs": [
|
|
8
|
+
"cap.memory/v1"
|
|
9
|
+
],
|
|
10
|
+
"upstream": {
|
|
11
|
+
"repository": "https://github.com/xiaowu0162/LongMemEval",
|
|
12
|
+
"tested": "9e0b455f4ef0e2ab8f2e582289761153549043fc",
|
|
13
|
+
"tested_date": "2026-05-11",
|
|
14
|
+
"license": "MIT"
|
|
15
|
+
},
|
|
16
|
+
"driver": {
|
|
17
|
+
"command": [
|
|
18
|
+
"{install}/.venv/bin/python",
|
|
19
|
+
"{here}/driver.py"
|
|
20
|
+
],
|
|
21
|
+
"install": {
|
|
22
|
+
"python": "3.9",
|
|
23
|
+
"pip": [
|
|
24
|
+
"torch==2.3.1",
|
|
25
|
+
"torchvision==0.18.1",
|
|
26
|
+
"openai==1.35.1",
|
|
27
|
+
"httpx==0.27.0",
|
|
28
|
+
"tqdm==4.66.4",
|
|
29
|
+
"backoff==2.2.1",
|
|
30
|
+
"numpy==1.26.3",
|
|
31
|
+
"nltk==3.9.1",
|
|
32
|
+
"transformers==4.43.3",
|
|
33
|
+
"tokenizers==0.19.1",
|
|
34
|
+
"tiktoken==0.7.0",
|
|
35
|
+
"huggingface-hub==0.23.4",
|
|
36
|
+
"rank-bm25==0.2.2",
|
|
37
|
+
"sentence-transformers==2.2.2",
|
|
38
|
+
"scikit-learn==1.5.1"
|
|
39
|
+
]
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
"datasets": [
|
|
43
|
+
{
|
|
44
|
+
"id": "longmemeval_s",
|
|
45
|
+
"label": "LongMemEval-S · cleaned",
|
|
46
|
+
"url": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_s_cleaned.json",
|
|
47
|
+
"sha256": "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",
|
|
48
|
+
"bytes": 277383467,
|
|
49
|
+
"units": 500
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"id": "longmemeval_m",
|
|
53
|
+
"label": "LongMemEval-M · cleaned",
|
|
54
|
+
"summary": "About 500 sessions of history per question. Needs about 5.5 GB of free disk.",
|
|
55
|
+
"url": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_m_cleaned.json",
|
|
56
|
+
"sha256": "9d79e5524794a2e6900a3aa9cb7d9152c5a3e8319c9a87c25494ba1eacee495f",
|
|
57
|
+
"bytes": 2737100077,
|
|
58
|
+
"units": 500
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"id": "longmemeval_oracle",
|
|
62
|
+
"label": "LongMemEval · evidence sessions only",
|
|
63
|
+
"summary": "Only the sessions that hold each question's evidence.",
|
|
64
|
+
"url": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_oracle.json",
|
|
65
|
+
"sha256": "821a2034d219ab45846873dd14c14f12cfe7776e73527a483f9dac095d38620c",
|
|
66
|
+
"bytes": 15388478,
|
|
67
|
+
"units": 500
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "longmemeval_s_original",
|
|
71
|
+
"label": "LongMemEval-S · as first released",
|
|
72
|
+
"summary": "The S file as first released in 2024, before the authors cleaned it. The paper's numbers come from it.",
|
|
73
|
+
"url": "https://huggingface.co/datasets/xiaowu0162/longmemeval/resolve/2ec2a557f339b6c0369619b1ed5793734cc87533/longmemeval_s",
|
|
74
|
+
"sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894",
|
|
75
|
+
"bytes": 278025796,
|
|
76
|
+
"units": 500
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"id": "longmemeval_m_original",
|
|
80
|
+
"label": "LongMemEval-M · as first released",
|
|
81
|
+
"summary": "The M file as first released in 2024. Needs about 5.5 GB of free disk.",
|
|
82
|
+
"url": "https://huggingface.co/datasets/xiaowu0162/longmemeval/resolve/2ec2a557f339b6c0369619b1ed5793734cc87533/longmemeval_m",
|
|
83
|
+
"sha256": "fb5413e3b077c62927daab794836991a2fcfa61ceacab57dc679fb02daaff2d9",
|
|
84
|
+
"bytes": 2745274681,
|
|
85
|
+
"units": 500
|
|
86
|
+
}
|
|
87
|
+
],
|
|
88
|
+
"units": {
|
|
89
|
+
"noun": "question",
|
|
90
|
+
"nouns": "questions"
|
|
91
|
+
},
|
|
92
|
+
"controls": {
|
|
93
|
+
"nop": {
|
|
94
|
+
"source": "nullius",
|
|
95
|
+
"from": "Nullius: no capability runs and nothing is recalled; the answer is empty. The 30 abstention questions in each dataset are right when the answer says it cannot answer, so the judge may pass an empty one; it also passes an empty answer by mistake now and then.",
|
|
96
|
+
"expect": {
|
|
97
|
+
"at_most": 0.07
|
|
98
|
+
}
|
|
99
|
+
},
|
|
100
|
+
"oracle": {
|
|
101
|
+
"source": "nullius",
|
|
102
|
+
"from": "Nullius, from the authors' answers: the question's answer field in the authors' file (for an abstention question, their explanation of why it cannot be answered), given to the judge as the answer. The judge, a model, rejects about one in twenty of them.",
|
|
103
|
+
"in_prompt": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
|
|
104
|
+
"expect": {
|
|
105
|
+
"at_least": 0.92
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
"score": {
|
|
110
|
+
"kind": "pass",
|
|
111
|
+
"says": "LongMemEval's own judge (src/evaluation/evaluate_qa.py): its prompt for the question's type (abstention questions: whether the answer says it cannot be answered), sent by its own client call, answered yes or no",
|
|
112
|
+
"judge": {
|
|
113
|
+
"model": "gpt-4o-2024-08-06",
|
|
114
|
+
"says": "the paper's judge, gpt-4o, at temperature 0, ten tokens"
|
|
115
|
+
}
|
|
116
|
+
},
|
|
117
|
+
"limits": {
|
|
118
|
+
"turn_seconds": 900
|
|
119
|
+
},
|
|
120
|
+
"options": {
|
|
121
|
+
"question_type": {
|
|
122
|
+
"label": "Question type",
|
|
123
|
+
"summary": "Which memory ability the run measures: every question, or only the questions of one type.",
|
|
124
|
+
"values": [
|
|
125
|
+
"all",
|
|
126
|
+
"knowledge-update",
|
|
127
|
+
"temporal-reasoning",
|
|
128
|
+
"multi-session",
|
|
129
|
+
"single-session-user",
|
|
130
|
+
"single-session-assistant",
|
|
131
|
+
"single-session-preference"
|
|
132
|
+
],
|
|
133
|
+
"default": "all",
|
|
134
|
+
"setups": [
|
|
135
|
+
"original",
|
|
136
|
+
"model",
|
|
137
|
+
"yours"
|
|
138
|
+
]
|
|
139
|
+
},
|
|
140
|
+
"reading": {
|
|
141
|
+
"summary": "Whether the model answers from recalled memory directly or first identifies relevant facts and reasons through them.",
|
|
142
|
+
"values": [
|
|
143
|
+
"con",
|
|
144
|
+
"direct"
|
|
145
|
+
],
|
|
146
|
+
"default": "con",
|
|
147
|
+
"about": {
|
|
148
|
+
"con": "Chain-of-note in one call: prepare_prompt's history-chats template with cot, gen_length 800. The upstream's default.",
|
|
149
|
+
"direct": "The template without cot, gen_length 500: the answer at once."
|
|
150
|
+
},
|
|
151
|
+
"setups": [
|
|
152
|
+
"original",
|
|
153
|
+
"model",
|
|
154
|
+
"yours"
|
|
155
|
+
]
|
|
156
|
+
},
|
|
157
|
+
"retriever": {
|
|
158
|
+
"summary": "Whether the model receives the full conversation history, no history, or sessions or turns selected by a retrieval method.",
|
|
159
|
+
"values": [
|
|
160
|
+
"full-history-session",
|
|
161
|
+
"full-history-turn",
|
|
162
|
+
"no-retrieval",
|
|
163
|
+
"flat-bm25-session",
|
|
164
|
+
"flat-bm25-turn",
|
|
165
|
+
"flat-contriever-session",
|
|
166
|
+
"flat-contriever-turn",
|
|
167
|
+
"flat-stella-session",
|
|
168
|
+
"flat-stella-turn"
|
|
169
|
+
],
|
|
170
|
+
"default": "full-history-session",
|
|
171
|
+
"setups": [
|
|
172
|
+
"original",
|
|
173
|
+
"model"
|
|
174
|
+
],
|
|
175
|
+
"about": {
|
|
176
|
+
"full-history-session": "--retriever_type orig-session: every session, latest kept first when the history is cut to the window.",
|
|
177
|
+
"full-history-turn": "--retriever_type orig-turn: every turn, each its own chunk.",
|
|
178
|
+
"no-retrieval": "--retriever_type no-retrieval: the question alone.",
|
|
179
|
+
"flat-bm25-session": "run_retrieval.py flat-bm25 at session granularity (a session's user turns are its key), then --retriever_type flat-session: the top-k sessions.",
|
|
180
|
+
"flat-bm25-turn": "run_retrieval.py flat-bm25 at turn granularity (a user turn is the key), then --retriever_type flat-turn: the top-k rounds.",
|
|
181
|
+
"flat-contriever-session": "run_retrieval.py flat-contriever (facebook/contriever, mean pooling) at session granularity, then --retriever_type flat-session.",
|
|
182
|
+
"flat-contriever-turn": "run_retrieval.py flat-contriever at turn granularity, then --retriever_type flat-turn.",
|
|
183
|
+
"flat-stella-session": "run_retrieval.py flat-stella (Stella V5 1.5B, its 1024-dimension head) at session granularity; the model fetched into the upstream's model_cache first, as its README has it downloaded by hand. Its fixed batch of 64 needs at least 12000 MiB of GPU memory.",
|
|
184
|
+
"flat-stella-turn": "run_retrieval.py flat-stella at turn granularity, then --retriever_type flat-turn."
|
|
185
|
+
},
|
|
186
|
+
"needs": {
|
|
187
|
+
"flat-contriever-session": {
|
|
188
|
+
"host": [
|
|
189
|
+
"cuda_gpu"
|
|
190
|
+
]
|
|
191
|
+
},
|
|
192
|
+
"flat-contriever-turn": {
|
|
193
|
+
"host": [
|
|
194
|
+
"cuda_gpu"
|
|
195
|
+
]
|
|
196
|
+
},
|
|
197
|
+
"flat-stella-session": {
|
|
198
|
+
"host": [
|
|
199
|
+
"stella_gpu"
|
|
200
|
+
]
|
|
201
|
+
},
|
|
202
|
+
"flat-stella-turn": {
|
|
203
|
+
"host": [
|
|
204
|
+
"stella_gpu"
|
|
205
|
+
]
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
},
|
|
209
|
+
"topk": {
|
|
210
|
+
"label": "Top-k",
|
|
211
|
+
"summary": "The maximum number of conversation sessions or turns the model receives when answering a question.",
|
|
212
|
+
"values": [
|
|
213
|
+
"1",
|
|
214
|
+
"3",
|
|
215
|
+
"5",
|
|
216
|
+
"10",
|
|
217
|
+
"20",
|
|
218
|
+
"30",
|
|
219
|
+
"50",
|
|
220
|
+
"100",
|
|
221
|
+
"1000"
|
|
222
|
+
],
|
|
223
|
+
"default": "1000",
|
|
224
|
+
"setups": [
|
|
225
|
+
"original",
|
|
226
|
+
"model"
|
|
227
|
+
]
|
|
228
|
+
},
|
|
229
|
+
"history_format": {
|
|
230
|
+
"summary": "Whether conversation history is shown to the model as JSON or as lines of text labelled by speaker.",
|
|
231
|
+
"values": [
|
|
232
|
+
"json",
|
|
233
|
+
"nl"
|
|
234
|
+
],
|
|
235
|
+
"default": "json",
|
|
236
|
+
"setups": [
|
|
237
|
+
"original",
|
|
238
|
+
"model"
|
|
239
|
+
]
|
|
240
|
+
},
|
|
241
|
+
"useronly": {
|
|
242
|
+
"label": "User turns only",
|
|
243
|
+
"summary": "Whether conversation history includes only user messages or both user and assistant messages.",
|
|
244
|
+
"values": [
|
|
245
|
+
"false",
|
|
246
|
+
"true"
|
|
247
|
+
],
|
|
248
|
+
"default": "false",
|
|
249
|
+
"setups": [
|
|
250
|
+
"original",
|
|
251
|
+
"model"
|
|
252
|
+
]
|
|
253
|
+
},
|
|
254
|
+
"merge_key_expansion": {
|
|
255
|
+
"summary": "Whether retrieved conversations are shown in full, replaced by their user text, or combined with that text.",
|
|
256
|
+
"values": [
|
|
257
|
+
"none",
|
|
258
|
+
"merge",
|
|
259
|
+
"replace"
|
|
260
|
+
],
|
|
261
|
+
"default": "none",
|
|
262
|
+
"setups": [
|
|
263
|
+
"original",
|
|
264
|
+
"model"
|
|
265
|
+
]
|
|
266
|
+
},
|
|
267
|
+
"separate_notes": {
|
|
268
|
+
"label": "Separate notes",
|
|
269
|
+
"summary": "Write a separate note for each retrieved session or turn before reasoning. Original memory only.",
|
|
270
|
+
"values": [
|
|
271
|
+
"false",
|
|
272
|
+
"true"
|
|
273
|
+
],
|
|
274
|
+
"default": "false",
|
|
275
|
+
"setups": [
|
|
276
|
+
"original",
|
|
277
|
+
"model"
|
|
278
|
+
],
|
|
279
|
+
"when": {
|
|
280
|
+
"reading": [
|
|
281
|
+
"con"
|
|
282
|
+
]
|
|
283
|
+
}
|
|
284
|
+
},
|
|
285
|
+
"session_expansion": {
|
|
286
|
+
"label": "Session index expansion",
|
|
287
|
+
"summary": "Expand retrieval keys with the authors' cached summaries, keyphrases or user facts. No expansion by default.",
|
|
288
|
+
"values": [
|
|
289
|
+
"none",
|
|
290
|
+
"session-summ",
|
|
291
|
+
"session-keyphrase",
|
|
292
|
+
"session-userfact"
|
|
293
|
+
],
|
|
294
|
+
"default": "none",
|
|
295
|
+
"setups": [
|
|
296
|
+
"original",
|
|
297
|
+
"model"
|
|
298
|
+
],
|
|
299
|
+
"when": {
|
|
300
|
+
"retriever": [
|
|
301
|
+
"flat-bm25-session",
|
|
302
|
+
"flat-contriever-session",
|
|
303
|
+
"flat-stella-session"
|
|
304
|
+
]
|
|
305
|
+
},
|
|
306
|
+
"needs": {
|
|
307
|
+
"session-summ": {
|
|
308
|
+
"files": [
|
|
309
|
+
"index_expansions"
|
|
310
|
+
]
|
|
311
|
+
},
|
|
312
|
+
"session-keyphrase": {
|
|
313
|
+
"files": [
|
|
314
|
+
"index_expansions"
|
|
315
|
+
]
|
|
316
|
+
},
|
|
317
|
+
"session-userfact": {
|
|
318
|
+
"files": [
|
|
319
|
+
"index_expansions"
|
|
320
|
+
]
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
},
|
|
324
|
+
"turn_expansion": {
|
|
325
|
+
"label": "Turn index expansion",
|
|
326
|
+
"summary": "Expand retrieval keys with the authors' cached summaries, keyphrases or user facts. No expansion by default.",
|
|
327
|
+
"values": [
|
|
328
|
+
"none",
|
|
329
|
+
"turn-keyphrase",
|
|
330
|
+
"turn-userfact"
|
|
331
|
+
],
|
|
332
|
+
"default": "none",
|
|
333
|
+
"setups": [
|
|
334
|
+
"original",
|
|
335
|
+
"model"
|
|
336
|
+
],
|
|
337
|
+
"when": {
|
|
338
|
+
"retriever": [
|
|
339
|
+
"flat-bm25-turn",
|
|
340
|
+
"flat-contriever-turn",
|
|
341
|
+
"flat-stella-turn"
|
|
342
|
+
]
|
|
343
|
+
},
|
|
344
|
+
"needs": {
|
|
345
|
+
"turn-keyphrase": {
|
|
346
|
+
"files": [
|
|
347
|
+
"index_expansions"
|
|
348
|
+
]
|
|
349
|
+
},
|
|
350
|
+
"turn-userfact": {
|
|
351
|
+
"files": [
|
|
352
|
+
"index_expansions"
|
|
353
|
+
]
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
},
|
|
357
|
+
"expansion_join": {
|
|
358
|
+
"label": "Join index expansion",
|
|
359
|
+
"summary": "How cached expansions join retrieval keys when expansion is enabled: separate keys, merged text or replacement text. Split modes separate sentences or phrases first.",
|
|
360
|
+
"values": [
|
|
361
|
+
"separate",
|
|
362
|
+
"split-separate",
|
|
363
|
+
"merge",
|
|
364
|
+
"split-merge",
|
|
365
|
+
"replace",
|
|
366
|
+
"split-replace"
|
|
367
|
+
],
|
|
368
|
+
"default": "separate",
|
|
369
|
+
"setups": [
|
|
370
|
+
"original",
|
|
371
|
+
"model"
|
|
372
|
+
],
|
|
373
|
+
"when": {
|
|
374
|
+
"retriever": [
|
|
375
|
+
"flat-bm25-session",
|
|
376
|
+
"flat-bm25-turn",
|
|
377
|
+
"flat-contriever-session",
|
|
378
|
+
"flat-contriever-turn",
|
|
379
|
+
"flat-stella-session",
|
|
380
|
+
"flat-stella-turn"
|
|
381
|
+
]
|
|
382
|
+
},
|
|
383
|
+
"needs": {
|
|
384
|
+
"split-separate": {
|
|
385
|
+
"files": [
|
|
386
|
+
"punkt_tab"
|
|
387
|
+
],
|
|
388
|
+
"when": {
|
|
389
|
+
"session_expansion": [
|
|
390
|
+
"session-summ"
|
|
391
|
+
],
|
|
392
|
+
"retriever": [
|
|
393
|
+
"flat-bm25-session",
|
|
394
|
+
"flat-contriever-session",
|
|
395
|
+
"flat-stella-session"
|
|
396
|
+
]
|
|
397
|
+
}
|
|
398
|
+
},
|
|
399
|
+
"split-merge": {
|
|
400
|
+
"files": [
|
|
401
|
+
"punkt_tab"
|
|
402
|
+
],
|
|
403
|
+
"when": {
|
|
404
|
+
"session_expansion": [
|
|
405
|
+
"session-summ"
|
|
406
|
+
],
|
|
407
|
+
"retriever": [
|
|
408
|
+
"flat-bm25-session",
|
|
409
|
+
"flat-contriever-session",
|
|
410
|
+
"flat-stella-session"
|
|
411
|
+
]
|
|
412
|
+
}
|
|
413
|
+
},
|
|
414
|
+
"split-replace": {
|
|
415
|
+
"files": [
|
|
416
|
+
"punkt_tab"
|
|
417
|
+
],
|
|
418
|
+
"when": {
|
|
419
|
+
"session_expansion": [
|
|
420
|
+
"session-summ"
|
|
421
|
+
],
|
|
422
|
+
"retriever": [
|
|
423
|
+
"flat-bm25-session",
|
|
424
|
+
"flat-contriever-session",
|
|
425
|
+
"flat-stella-session"
|
|
426
|
+
]
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
},
|
|
432
|
+
"host": {
|
|
433
|
+
"cuda_gpu": {
|
|
434
|
+
"summary": "An NVIDIA GPU with CUDA, for LongMemEval's Contriever retrievers.",
|
|
435
|
+
"check": [
|
|
436
|
+
"nvidia-smi"
|
|
437
|
+
]
|
|
438
|
+
},
|
|
439
|
+
"stella_gpu": {
|
|
440
|
+
"summary": "An NVIDIA GPU with CUDA and at least 12000 MiB of GPU memory, for Stella's fixed batch of 64.",
|
|
441
|
+
"check": [
|
|
442
|
+
"nvidia-smi"
|
|
443
|
+
],
|
|
444
|
+
"gpu_memory_mb": 12000
|
|
445
|
+
}
|
|
446
|
+
},
|
|
447
|
+
"setups": {
|
|
448
|
+
"original": {
|
|
449
|
+
"summary": "LongMemEval's long-context reader as the paper ran it: its own src/generation/run_generation.py on one question, as run_generation.sh runs it - the history in the prompt (or what the paper's BM25 retrieval ranks), cut to the reader's 128k window, its prompts, temperature 0 and answer length - scored by its own src/evaluation/evaluate_qa.py with its judge, gpt-4o.",
|
|
450
|
+
"models": [
|
|
451
|
+
"gpt-4o",
|
|
452
|
+
"gpt-4o-mini"
|
|
453
|
+
],
|
|
454
|
+
"default": "gpt-4o",
|
|
455
|
+
"served_as": {
|
|
456
|
+
"gpt-4o": "gpt-4o-2024-08-06",
|
|
457
|
+
"gpt-4o-mini": "gpt-4o-mini-2024-07-18"
|
|
458
|
+
},
|
|
459
|
+
"providers": {
|
|
460
|
+
"openai": "",
|
|
461
|
+
"openrouter": "openai/"
|
|
462
|
+
},
|
|
463
|
+
"turn_seconds": 3600,
|
|
464
|
+
"paper": "Table 2 (long-context reading) and Figure 5 onward (retrieval-augmented reading)"
|
|
465
|
+
},
|
|
466
|
+
"model": {
|
|
467
|
+
"summary": "The same pipeline with any model in the reader's place: LongMemEval's code asks for gpt-4o - its 128k window, the history cut to fit it, its answer length - and the relay sends the model chosen. Its scores compare with other runs of this setup, not with the paper.",
|
|
468
|
+
"runs_as": "gpt-4o"
|
|
469
|
+
}
|
|
470
|
+
},
|
|
471
|
+
"proven": {
|
|
472
|
+
"longmemeval_s": {
|
|
473
|
+
"units": {
|
|
474
|
+
"units": 500,
|
|
475
|
+
"passed": "500 ids, the same twice, unique",
|
|
476
|
+
"on": "2026-11-03"
|
|
477
|
+
},
|
|
478
|
+
"leak": {
|
|
479
|
+
"units": 500,
|
|
480
|
+
"looked_for": 248,
|
|
481
|
+
"short": 252,
|
|
482
|
+
"passed": "500/500",
|
|
483
|
+
"in_prompt": {
|
|
484
|
+
"units": 11,
|
|
485
|
+
"first": [
|
|
486
|
+
"gpt4_4929293a",
|
|
487
|
+
"gpt4_2487a7cb",
|
|
488
|
+
"gpt4_2312f94c",
|
|
489
|
+
"gpt4_2d58bcd6",
|
|
490
|
+
"gpt4_483dd43c"
|
|
491
|
+
],
|
|
492
|
+
"why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
|
|
493
|
+
},
|
|
494
|
+
"note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
|
|
495
|
+
"on": "2026-11-03"
|
|
496
|
+
},
|
|
497
|
+
"nop": {
|
|
498
|
+
"passed": "20/500",
|
|
499
|
+
"value": 0.04,
|
|
500
|
+
"expect": {
|
|
501
|
+
"at_most": 0.07
|
|
502
|
+
},
|
|
503
|
+
"units": 500,
|
|
504
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_s__prove-nop",
|
|
505
|
+
"source": "nullius",
|
|
506
|
+
"judge": "gpt-4o-2024-08-06",
|
|
507
|
+
"on": "2026-11-03"
|
|
508
|
+
},
|
|
509
|
+
"oracle": {
|
|
510
|
+
"passed": "471/500",
|
|
511
|
+
"value": 0.942,
|
|
512
|
+
"expect": {
|
|
513
|
+
"at_least": 0.92
|
|
514
|
+
},
|
|
515
|
+
"units": 500,
|
|
516
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_s__prove-oracle",
|
|
517
|
+
"source": "nullius",
|
|
518
|
+
"judge": "gpt-4o-2024-08-06",
|
|
519
|
+
"on": "2026-11-03"
|
|
520
|
+
}
|
|
521
|
+
},
|
|
522
|
+
"longmemeval_m": {
|
|
523
|
+
"units": {
|
|
524
|
+
"units": 500,
|
|
525
|
+
"passed": "500 ids, the same twice, unique",
|
|
526
|
+
"on": "2026-11-03"
|
|
527
|
+
},
|
|
528
|
+
"leak": {
|
|
529
|
+
"units": 500,
|
|
530
|
+
"looked_for": 248,
|
|
531
|
+
"short": 252,
|
|
532
|
+
"passed": "500/500",
|
|
533
|
+
"in_prompt": {
|
|
534
|
+
"units": 11,
|
|
535
|
+
"first": [
|
|
536
|
+
"gpt4_4929293a",
|
|
537
|
+
"gpt4_2487a7cb",
|
|
538
|
+
"gpt4_2312f94c",
|
|
539
|
+
"gpt4_2d58bcd6",
|
|
540
|
+
"gpt4_483dd43c"
|
|
541
|
+
],
|
|
542
|
+
"why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
|
|
543
|
+
},
|
|
544
|
+
"note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
|
|
545
|
+
"on": "2026-11-03"
|
|
546
|
+
},
|
|
547
|
+
"nop": {
|
|
548
|
+
"passed": "24/500",
|
|
549
|
+
"value": 0.048,
|
|
550
|
+
"expect": {
|
|
551
|
+
"at_most": 0.07
|
|
552
|
+
},
|
|
553
|
+
"units": 500,
|
|
554
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_m__prove-nop",
|
|
555
|
+
"source": "nullius",
|
|
556
|
+
"judge": "gpt-4o-2024-08-06",
|
|
557
|
+
"on": "2026-11-03"
|
|
558
|
+
},
|
|
559
|
+
"oracle": {
|
|
560
|
+
"passed": "470/500",
|
|
561
|
+
"value": 0.94,
|
|
562
|
+
"expect": {
|
|
563
|
+
"at_least": 0.92
|
|
564
|
+
},
|
|
565
|
+
"units": 500,
|
|
566
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_m__prove-oracle",
|
|
567
|
+
"source": "nullius",
|
|
568
|
+
"judge": "gpt-4o-2024-08-06",
|
|
569
|
+
"on": "2026-11-03"
|
|
570
|
+
}
|
|
571
|
+
},
|
|
572
|
+
"longmemeval_oracle": {
|
|
573
|
+
"units": {
|
|
574
|
+
"units": 500,
|
|
575
|
+
"passed": "500 ids, the same twice, unique",
|
|
576
|
+
"on": "2026-11-03"
|
|
577
|
+
},
|
|
578
|
+
"leak": {
|
|
579
|
+
"units": 500,
|
|
580
|
+
"looked_for": 248,
|
|
581
|
+
"short": 252,
|
|
582
|
+
"passed": "500/500",
|
|
583
|
+
"in_prompt": {
|
|
584
|
+
"units": 11,
|
|
585
|
+
"first": [
|
|
586
|
+
"gpt4_2487a7cb",
|
|
587
|
+
"gpt4_2312f94c",
|
|
588
|
+
"gpt4_2d58bcd6",
|
|
589
|
+
"gpt4_483dd43c",
|
|
590
|
+
"gpt4_93f6379c"
|
|
591
|
+
],
|
|
592
|
+
"why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
|
|
593
|
+
},
|
|
594
|
+
"note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
|
|
595
|
+
"on": "2026-11-03"
|
|
596
|
+
},
|
|
597
|
+
"nop": {
|
|
598
|
+
"passed": "22/500",
|
|
599
|
+
"value": 0.044,
|
|
600
|
+
"expect": {
|
|
601
|
+
"at_most": 0.07
|
|
602
|
+
},
|
|
603
|
+
"units": 500,
|
|
604
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_oracle__prove-nop",
|
|
605
|
+
"source": "nullius",
|
|
606
|
+
"judge": "gpt-4o-2024-08-06",
|
|
607
|
+
"on": "2026-11-03"
|
|
608
|
+
},
|
|
609
|
+
"oracle": {
|
|
610
|
+
"passed": "469/500",
|
|
611
|
+
"value": 0.938,
|
|
612
|
+
"expect": {
|
|
613
|
+
"at_least": 0.92
|
|
614
|
+
},
|
|
615
|
+
"units": 500,
|
|
616
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_oracle__prove-oracle",
|
|
617
|
+
"source": "nullius",
|
|
618
|
+
"judge": "gpt-4o-2024-08-06",
|
|
619
|
+
"on": "2026-11-03"
|
|
620
|
+
}
|
|
621
|
+
},
|
|
622
|
+
"longmemeval_s_original": {
|
|
623
|
+
"units": {
|
|
624
|
+
"units": 500,
|
|
625
|
+
"passed": "500 ids, the same twice, unique",
|
|
626
|
+
"on": "2026-11-03"
|
|
627
|
+
},
|
|
628
|
+
"leak": {
|
|
629
|
+
"units": 500,
|
|
630
|
+
"looked_for": 248,
|
|
631
|
+
"short": 252,
|
|
632
|
+
"passed": "500/500",
|
|
633
|
+
"in_prompt": {
|
|
634
|
+
"units": 11,
|
|
635
|
+
"first": [
|
|
636
|
+
"gpt4_4929293a",
|
|
637
|
+
"gpt4_2487a7cb",
|
|
638
|
+
"gpt4_2312f94c",
|
|
639
|
+
"gpt4_2d58bcd6",
|
|
640
|
+
"gpt4_483dd43c"
|
|
641
|
+
],
|
|
642
|
+
"why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
|
|
643
|
+
},
|
|
644
|
+
"note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
|
|
645
|
+
"on": "2026-11-03"
|
|
646
|
+
},
|
|
647
|
+
"nop": {
|
|
648
|
+
"passed": "21/500",
|
|
649
|
+
"value": 0.042,
|
|
650
|
+
"expect": {
|
|
651
|
+
"at_most": 0.07
|
|
652
|
+
},
|
|
653
|
+
"units": 500,
|
|
654
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_s_original__prove-nop",
|
|
655
|
+
"source": "nullius",
|
|
656
|
+
"judge": "gpt-4o-2024-08-06",
|
|
657
|
+
"on": "2026-11-03"
|
|
658
|
+
},
|
|
659
|
+
"oracle": {
|
|
660
|
+
"passed": "469/500",
|
|
661
|
+
"value": 0.938,
|
|
662
|
+
"expect": {
|
|
663
|
+
"at_least": 0.92
|
|
664
|
+
},
|
|
665
|
+
"units": 500,
|
|
666
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_s_original__prove-oracle",
|
|
667
|
+
"source": "nullius",
|
|
668
|
+
"judge": "gpt-4o-2024-08-06",
|
|
669
|
+
"on": "2026-11-03"
|
|
670
|
+
}
|
|
671
|
+
},
|
|
672
|
+
"longmemeval_m_original": {
|
|
673
|
+
"units": {
|
|
674
|
+
"units": 500,
|
|
675
|
+
"passed": "500 ids, the same twice, unique",
|
|
676
|
+
"on": "2026-11-03"
|
|
677
|
+
},
|
|
678
|
+
"leak": {
|
|
679
|
+
"units": 500,
|
|
680
|
+
"looked_for": 248,
|
|
681
|
+
"short": 252,
|
|
682
|
+
"passed": "500/500",
|
|
683
|
+
"in_prompt": {
|
|
684
|
+
"units": 11,
|
|
685
|
+
"first": [
|
|
686
|
+
"gpt4_4929293a",
|
|
687
|
+
"gpt4_2487a7cb",
|
|
688
|
+
"gpt4_2312f94c",
|
|
689
|
+
"gpt4_2d58bcd6",
|
|
690
|
+
"gpt4_483dd43c"
|
|
691
|
+
],
|
|
692
|
+
"why": "A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it."
|
|
693
|
+
},
|
|
694
|
+
"note": "252 reference(s) under 12 characters not looked for: too short to tell a leak from chance; 11 prompt(s) name their answer, as the benchmark says they may: A comparison question names its answer among the options it offers ('A or B?'), with the other option beside it.",
|
|
695
|
+
"on": "2026-11-03"
|
|
696
|
+
},
|
|
697
|
+
"nop": {
|
|
698
|
+
"passed": "27/500",
|
|
699
|
+
"value": 0.054,
|
|
700
|
+
"expect": {
|
|
701
|
+
"at_most": 0.07
|
|
702
|
+
},
|
|
703
|
+
"units": 500,
|
|
704
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_m_original__prove-nop",
|
|
705
|
+
"source": "nullius",
|
|
706
|
+
"judge": "gpt-4o-2024-08-06",
|
|
707
|
+
"on": "2026-11-03"
|
|
708
|
+
},
|
|
709
|
+
"oracle": {
|
|
710
|
+
"passed": "469/500",
|
|
711
|
+
"value": 0.938,
|
|
712
|
+
"expect": {
|
|
713
|
+
"at_least": 0.92
|
|
714
|
+
},
|
|
715
|
+
"units": 500,
|
|
716
|
+
"job": "2026-11-03__01-26-22__longmemeval__longmemeval_m_original__prove-oracle",
|
|
717
|
+
"source": "nullius",
|
|
718
|
+
"judge": "gpt-4o-2024-08-06",
|
|
719
|
+
"on": "2026-11-03"
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
},
|
|
723
|
+
"files": {
|
|
724
|
+
"index_expansions": {
|
|
725
|
+
"url": "https://drive.usercontent.google.com/download?id=1Y6vImnr_WQZXv4bKzpqPdHMbyoat8b_P&export=download&confirm=t",
|
|
726
|
+
"sha256": "2c87f279962968e86ff5a1b64be8c83c2c21077e0cf5fc76fb9c84db1d8ce7c1"
|
|
727
|
+
},
|
|
728
|
+
"punkt_tab": {
|
|
729
|
+
"url": "https://raw.githubusercontent.com/nltk/nltk_data/550b6625bcef1f2abff2ff770a5a0d272c9c6b2a/packages/tokenizers/punkt_tab.zip",
|
|
730
|
+
"sha256": "e57f64187974277726a3417ca6f181ec5403676c717672eef6a748a7b20e0106"
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
}
|