nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"entry": "scripts/evaluate_gpts.sh", "setting": "--model gpt-4-turbo --batch-size 20", "values": "gpt-4-turbo", "default": "",
|
|
3
|
+
"nullius": "setup:original"},
|
|
4
|
+
{"entry": "scripts/evaluate_gpts.sh", "setting": "--model gpt-3.5-turbo-{4k,8k,12k,16k} --batch-size 10; --model gpt-3.5-turbo (scripts/evaluate_rag_gpts.sh)", "values": "gpt-3.5-turbo-4k, gpt-3.5-turbo-8k, gpt-3.5-turbo-12k, gpt-3.5-turbo-16k (each asked as gpt-3.5-turbo, the window from MAX_LENGTH)", "default": "",
|
|
5
|
+
"nullius": "setup:original"},
|
|
6
|
+
{"entry": "scripts/evaluate_gpts.sh", "setting": "another model in the reader's place", "values": "any", "default": "",
|
|
7
|
+
"nullius": "setup:model"},
|
|
8
|
+
{"entry": "scripts/env.sh", "setting": "DATA_FILE_PATH", "values": "./data/locomo10.json", "default": "./data/locomo10.json",
|
|
9
|
+
"nullius": "dataset:locomo10"},
|
|
10
|
+
{"entry": "scripts/env.sh", "setting": "OUT_DIR, EMB_DIR, QA_OUTPUT_FILE", "values": "./outputs, locomo10_qa.json", "default": "./outputs",
|
|
11
|
+
"nullius": "fixed:the unit's own folder (answers.json, answers_stats.json) - one conversation a unit, so runs never share an output file"},
|
|
12
|
+
{"entry": "scripts/env.sh", "setting": "OPENAI_API_KEY", "values": "a key", "default": "",
|
|
13
|
+
"nullius": "fixed:the relay's placeholder, with OPENAI_API_BASE the unit's relay - the real key stays in the relay"},
|
|
14
|
+
{"entry": "scripts/env.sh", "setting": "ANTHROPIC_API_KEY, GOOGLE_API_KEY, HF_TOKEN", "values": "keys", "default": "",
|
|
15
|
+
"nullius": "not offered - only their readers read them (claude-sonnet, gemini-pro-1.0, the Hugging Face models), and none of those is offered: see their rows"},
|
|
16
|
+
{"entry": "scripts/evaluate_claude.sh", "setting": "--model claude-sonnet --batch-size 10", "values": "claude-sonnet (run_claude: claude-3-sonnet-20240229)", "default": "",
|
|
17
|
+
"nullius": "not offered - run_claude asks claude-3-sonnet-20240229 by that dated name, which Anthropic retired in July 2025: the call cannot be made as published"},
|
|
18
|
+
{"entry": "global_methods.py", "setting": "run_claude model claude-haiku", "values": "claude-3-haiku-20240307", "default": "",
|
|
19
|
+
"nullius": "not offered - no script of the paper runs it"},
|
|
20
|
+
{"entry": "scripts/evaluate_gemini.sh", "setting": "--model gemini-pro-1.0 --batch-size 20", "values": "gemini-pro-1.0 (evaluate_qa.py: models/gemini-1.0-pro-latest)", "default": "",
|
|
21
|
+
"nullius": "not offered - Google retired Gemini 1.0 Pro in 2025, and google-generativeai calls Google's own API, which the relay (OpenAI and Anthropic shapes) does not carry"},
|
|
22
|
+
{"entry": "task_eval/gemini_utils.py", "setting": "pause before each gemini-pro-1.0 call", "values": "30 s", "default": "30 s",
|
|
23
|
+
"nullius": "not offered - gemini-pro-1.0's, not offered"},
|
|
24
|
+
{"entry": "scripts/evaluate_hf_llm.sh", "setting": "--model llama3-chat-70b --use-4bit --batch-size 1", "values": "meta-llama/Meta-Llama-3-70B-Instruct (listed twice in the script's loop)", "default": "",
|
|
25
|
+
"nullius": "not offered - local weights loaded in 4 bits (bitsandbytes, which the upstream's requirements.txt does not list) with device_map auto, from a gated repository (HF_TOKEN): 70B in 4 bits is about 35 GB, past an 11 GB GPU, and the reader is not a model the run's relay carries"},
|
|
26
|
+
{"entry": "scripts/evaluate_hf_llm.sh", "setting": "--model mistral-instruct-7b-32k-v2 --use-4bit --batch-size 1", "values": "mistralai/Mistral-7B-Instruct-v0.2", "default": "",
|
|
27
|
+
"nullius": "not offered - local weights in 4 bits from a gated repository (HF_TOKEN, bitsandbytes not in requirements.txt); 7B fits a GPU of 11 GB, but the reader would be weights on the machine, not a model the run's relay carries"},
|
|
28
|
+
{"entry": "task_eval/hf_llm_utils.py", "setting": "other Hugging Face readers", "values": "llama2, llama2-70b, llama2-chat, llama2-chat-70b, mistral-7b-*, mistral-instruct-7b-*, gemma-7b-it", "default": "",
|
|
29
|
+
"nullius": "not offered - no script of the paper runs them, and they are local weights as the two it runs"},
|
|
30
|
+
{"entry": "task_eval/hf_llm_utils.py", "setting": "temperature, ANS_TOKENS_PER_QUES, MAX_LENGTH", "values": "0.4, 50, 4096 to 128000 by model", "default": "",
|
|
31
|
+
"nullius": "not offered - the Hugging Face readers' settings, not offered"},
|
|
32
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--model", "values": "a name holding gpt, claude, gemini, or gemma, llama, mistral", "default": "required",
|
|
33
|
+
"nullius": "setup:original"},
|
|
34
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--data-file", "values": "a LoCoMo file", "default": "required",
|
|
35
|
+
"nullius": "dataset:locomo10"},
|
|
36
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--out-file", "values": "a path", "default": "required",
|
|
37
|
+
"nullius": "fixed:answers.json in the unit's folder - read back by score"},
|
|
38
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--batch-size", "values": "an integer", "default": "1",
|
|
39
|
+
"nullius": "fixed:20 for gpt-4-turbo, 10 for the gpt-3.5-turbo windows - as scripts/evaluate_gpts.sh runs each; the model setup takes gpt-4-turbo's 20"},
|
|
40
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--overwrite", "values": "on, off", "default": "off",
|
|
41
|
+
"nullius": "fixed:off - each unit writes a new file, so there is nothing to overwrite"},
|
|
42
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--use-4bit", "values": "on, off", "default": "off",
|
|
43
|
+
"nullius": "not offered - the Hugging Face readers' only, not offered"},
|
|
44
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--use-rag", "values": "on, off", "default": "off",
|
|
45
|
+
"nullius": "option:retrieval"},
|
|
46
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--rag-mode", "values": "dialog, observation, summary", "default": "(none)",
|
|
47
|
+
"nullius": "option:retrieval"},
|
|
48
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--rag-mode (a memory under test)", "values": "dialog, observation, summary", "default": "dialog",
|
|
49
|
+
"nullius": "option:rag_mode"},
|
|
50
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--retriever", "values": "dragon, contriever, dpr, openai", "default": "contriever",
|
|
51
|
+
"nullius": "option:retriever"},
|
|
52
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--top-k", "values": "an integer", "default": "5",
|
|
53
|
+
"nullius": "option:top_k"},
|
|
54
|
+
{"entry": "task_eval/evaluate_qa.py", "setting": "--emb-dir", "values": "a folder", "default": "(the working directory)",
|
|
55
|
+
"nullius": "fixed:the unit's own folder - the database pickles beside its answers, one conversation a unit"},
|
|
56
|
+
{"entry": "scripts/evaluate_rag_gpts.sh", "setting": "--model gpt-3.5-turbo --batch-size 1 --use-rag --retriever dragon", "values": "gpt-3.5-turbo, dragon", "default": "",
|
|
57
|
+
"nullius": "option:retrieval"},
|
|
58
|
+
{"entry": "scripts/evaluate_rag_gpts.sh", "setting": "--top-k by --rag-mode", "values": "dialog and observation: 5, 10, 25, 50; summary: 2, 5, 10", "default": "",
|
|
59
|
+
"nullius": "option:top_k"},
|
|
60
|
+
{"entry": "task_eval/rag_utils.py", "setting": "retriever dragon", "values": "facebook/dragon-plus-query-encoder, facebook/dragon-plus-context-encoder", "default": "",
|
|
61
|
+
"nullius": "option:retriever"},
|
|
62
|
+
{"entry": "task_eval/rag_utils.py", "setting": "retriever dpr", "values": "facebook/dpr-ctx_encoder-single-nq-base, facebook/dpr-question_encoder-single-nq-base", "default": "",
|
|
63
|
+
"nullius": "option:retriever"},
|
|
64
|
+
{"entry": "task_eval/rag_utils.py", "setting": "retriever contriever", "values": "facebook/contriever", "default": "",
|
|
65
|
+
"nullius": "not offered - as published it fails: init_query_model reads context_tokenizer, which only its DRAGON branch assigns (UnboundLocalError), and get_embeddings pools with the input list's attention_mask"},
|
|
66
|
+
{"entry": "task_eval/rag_utils.py", "setting": "retriever openai", "values": "text-embedding-ada-002 (global_methods.get_openai_embedding)", "default": "",
|
|
67
|
+
"nullius": "not offered - get_embeddings passes the whole input list, not the batch, for each batch of 24, and get_openai_embedding asks for all of it once per text: n texts make ceil(n/24) x n calls of n texts, and ceil(n/24) x n rows that prepare_for_rag's assert refuses once n > 24. Tried on 2026-10-16 through the relay: conv-26's 419 turns sent 710 calls of 419 texts (16,883 tokens each, about $1.20) in 10 minutes before it was stopped, of the 7,542 the context alone would take before failing"},
|
|
68
|
+
{"entry": "task_eval/rag_utils.py", "setting": "batch_size", "values": "24", "default": "24",
|
|
69
|
+
"nullius": "fixed:24 - get_embeddings' own"},
|
|
70
|
+
{"entry": "scripts/generate_observations.sh", "setting": "task_eval/get_facts.py --use-date --overwrite (--retriever dragon by default) - retrieval=observation", "values": "gpt-3.5-turbo extraction (run_chatgpt_with_examples, 500 tokens, temperature 1), then embeddings", "default": "",
|
|
71
|
+
"nullius": "option:retrieval"},
|
|
72
|
+
{"entry": "scripts/generate_session_summaries.sh", "setting": "task_eval/get_session_summaries.py --use-date (--retriever dragon by default) - retrieval=summary, run with --data-file, the flag the script misspells", "values": "gpt-3.5-turbo summaries (256 tokens, temperature 1), then embeddings", "default": "",
|
|
73
|
+
"nullius": "option:retrieval"},
|
|
74
|
+
{"entry": "scripts/generate_conversations.sh", "setting": "generative_agents/generate_conversations.py", "values": "--num-sessions, --num-days, --num-events, --max-turns-per-session, --persona, --blip-caption, ...", "default": "",
|
|
75
|
+
"nullius": "not offered - it makes new conversations, not an evaluation of these"},
|
|
76
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "MAX_LENGTH", "values": "gpt-4-turbo 128000, gpt-3.5-turbo-16k 16000, -12k 12000, -8k 8000, -4k 4000, gpt-3.5-turbo 4096, gpt-4 4096, gpt-4-32k 320000", "default": "",
|
|
77
|
+
"nullius": "fixed:the upstream's table, by the model's name - the setups' windows (the model setup takes gpt-4-turbo's)"},
|
|
78
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "PER_QA_TOKEN_BUDGET", "values": "50", "default": "50",
|
|
79
|
+
"nullius": "fixed:50 - the upstream's constant: max_tokens is 50 a question of the batch, and the window is cut to leave it"},
|
|
80
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "temperature", "values": "0", "default": "0",
|
|
81
|
+
"nullius": "fixed:0 - as get_gpt_answers asks"},
|
|
82
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "num_tokens_request for one question", "values": "32", "default": "32",
|
|
83
|
+
"nullius": "fixed:32 - get_gpt_answers' batch-of-one call, which the memory path makes for each question"},
|
|
84
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "prompts (CONV_START_PROMPT, QA_PROMPT_BATCH, QA_PROMPT)", "values": "the upstream's text", "default": "",
|
|
85
|
+
"nullius": "fixed:the upstream's, imported - QA_PROMPT over what the memory recalled on the memory path"},
|
|
86
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "reply parsing (process_ouput), 3 trials", "values": "", "default": "",
|
|
87
|
+
"nullius": "fixed:as published - a reply it cannot parse stops the conversation (the setups), said in the limitations"},
|
|
88
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "pause before each gpt-4 call", "values": "5 s", "default": "5 s",
|
|
89
|
+
"nullius": "fixed:5 s in the setups, evaluate_qa.py run whole - the memory path's one-question call does not make it"},
|
|
90
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "date hint for category 2", "values": "' Use DATE of CONVERSATION to answer with an approximate date.'", "default": "",
|
|
91
|
+
"nullius": "fixed:the upstream's words - read from get_gpt_answers' source, never copied"},
|
|
92
|
+
{"entry": "task_eval/gpt_utils.py", "setting": "tiktoken encoding", "values": "gpt-3.5-turbo-16k's for the windows, else the model's", "default": "",
|
|
93
|
+
"nullius": "fixed:as published - the upstream's own call"},
|
|
94
|
+
{"entry": "global_methods.py", "setting": "run_chatgpt role and retries", "values": "gpt-3.5 as system, gpt-4 as user; API, connection and unavailable errors retried without end, a rate limit with no wait", "default": "",
|
|
95
|
+
"nullius": "fixed:as published - the run's turn_seconds bounds a unit"},
|
|
96
|
+
{"entry": "data/locomo10.json", "setting": "question categories 1 to 4", "values": "1,540 questions: multi-hop, temporal, open-domain, single-hop", "default": "",
|
|
97
|
+
"nullius": "dataset:locomo10"},
|
|
98
|
+
{"entry": "data/locomo10.json", "setting": "question category 5 (adversarial)", "values": "446 questions", "default": "",
|
|
99
|
+
"nullius": "not offered - the published file gives them adversarial_answer and no answer, and get_gpt_answers formats qa['answer'] into the question (a KeyError), so as published they cannot be asked; its option order is also drawn by an unseeded random.random"},
|
|
100
|
+
{"entry": "data/locomo10.json", "setting": "event_summary (event summarization task)", "values": "annotated events per speaker and session", "default": "",
|
|
101
|
+
"nullius": "not offered - the README's event summarization evaluation is 'Coming soon!': there is no code to run"},
|
|
102
|
+
{"entry": "README.MD", "setting": "multimodal dialog generation (MiniGPT-5)", "values": "", "default": "",
|
|
103
|
+
"nullius": "not offered - 'Coming soon!' in the README: there is no code to run"},
|
|
104
|
+
{"entry": "task_eval/evaluation.py", "setting": "eval_question_answering (F1)", "values": "token F1, normalisation and Porter stemming; category 1 by comma-separated parts; category 3 against the part before the first ';'", "default": "",
|
|
105
|
+
"nullius": "fixed:the upstream's, imported - every path scores with it"},
|
|
106
|
+
{"entry": "task_eval/evaluation_stats.py", "setting": "analyze_aggr_acc", "values": "per-category accuracy and memory-distance breakdowns", "default": "",
|
|
107
|
+
"nullius": "fixed:written by evaluate_qa.py beside the answers (answers_stats.json) in the setups - the run's score is the mean F1 the answers carry"},
|
|
108
|
+
{"entry": "task_eval/rag_utils.py", "setting": "the encoders' device", "values": ".cuda()", "default": "",
|
|
109
|
+
"nullius": "fixed:a CUDA GPU, as published - the host requirement cuda_gpu, checked by nvidia-smi; the driver's torch 2.0.1 is its CUDA 11.7 build (version 3)"},
|
|
110
|
+
{"entry": "task_eval/get_facts.py, get_session_summaries.py", "setting": "the extraction model", "values": "gpt-3.5-turbo (run_chatgpt_with_examples, run_chatgpt 'chatgpt')", "default": "",
|
|
111
|
+
"nullius": "fixed:gpt-3.5-turbo in the original, whatever the reader - sent as the provider names it (the setup's side_models); the model chosen, in the model setup"}
|
|
112
|
+
]
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "longmemeval",
|
|
3
|
+
"title": "LongMemEval",
|
|
4
|
+
"category": "Memory",
|
|
5
|
+
"domain": "Long-term chat memory: questions over timestamped chat histories.",
|
|
6
|
+
"scoring": "GPT-4o judge",
|
|
7
|
+
"harness": [
|
|
8
|
+
"standard"
|
|
9
|
+
],
|
|
10
|
+
"origin": "official",
|
|
11
|
+
"good_to_know": [
|
|
12
|
+
"The S, M and oracle files run cleaned, and S and M also as first released. The paper's numbers come from the files as first released.",
|
|
13
|
+
"An M file is 2.7 GB and needs about 5.5 GB of free disk. STANDARD_DATA_DIR moves the datasets to a disk with room.",
|
|
14
|
+
"Reading is chain-of-note by default, or direct. Separate notes work with Original memory only.",
|
|
15
|
+
"Original memory offers the paper's Contriever and Stella retrievers and needs a CUDA GPU. Stella requires at least 12000 MiB of GPU memory; the run checks it before starting."
|
|
16
|
+
],
|
|
17
|
+
"upstream_limitations": [
|
|
18
|
+
"Scores compare with the paper only with its reader, GPT-4o or GPT-4o-mini, and its judge, GPT-4o.",
|
|
19
|
+
"A reader you choose must take about 126,200 tokens. The history is not cut to fit, so a longer recall fails at the model.",
|
|
20
|
+
"An empty reply from the reader fails the question.",
|
|
21
|
+
"Three combinations fail as published: merge expansion with a full-history retriever, user-only with flat-bm25-turn, and separate notes with chain-of-note reading and nl history.",
|
|
22
|
+
"The judge, a model, rejects about one right answer in twenty, and passes an empty answer now and then.",
|
|
23
|
+
"Time-aware query expansion fails as published."
|
|
24
|
+
],
|
|
25
|
+
"limitations": [
|
|
26
|
+
"The GTE retriever, time-aware query expansion and open-weight readers are not offered. Index expansion uses the authors' released caches with flat retrievers at the matching session or turn granularity."
|
|
27
|
+
],
|
|
28
|
+
"datasets": [
|
|
29
|
+
{
|
|
30
|
+
"key": "longmemeval-s",
|
|
31
|
+
"title": "LongMemEval-S (cleaned)",
|
|
32
|
+
"tasks": 500,
|
|
33
|
+
"official": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned",
|
|
34
|
+
"source": "Hugging Face",
|
|
35
|
+
"registry": "",
|
|
36
|
+
"hub": "",
|
|
37
|
+
"verifier": "LongMemEval's own judge, GPT-4o (gpt-4o-2024-08-06): yes or no per answer.",
|
|
38
|
+
"notes": "About 40 sessions and 115k tokens of history per question."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"key": "longmemeval-m",
|
|
42
|
+
"title": "LongMemEval-M (cleaned)",
|
|
43
|
+
"tasks": 500,
|
|
44
|
+
"official": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned",
|
|
45
|
+
"source": "Hugging Face",
|
|
46
|
+
"registry": "",
|
|
47
|
+
"hub": "",
|
|
48
|
+
"verifier": "LongMemEval's own judge, GPT-4o (gpt-4o-2024-08-06): yes or no per answer.",
|
|
49
|
+
"notes": "About 500 sessions per question."
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"key": "longmemeval-oracle",
|
|
53
|
+
"title": "LongMemEval oracle \u00b7 evidence sessions only",
|
|
54
|
+
"tasks": 500,
|
|
55
|
+
"official": "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned",
|
|
56
|
+
"source": "Hugging Face",
|
|
57
|
+
"registry": "",
|
|
58
|
+
"hub": "",
|
|
59
|
+
"verifier": "LongMemEval's own judge, GPT-4o (gpt-4o-2024-08-06): yes or no per answer.",
|
|
60
|
+
"notes": "Only the evidence sessions: the ceiling a memory layer is read against."
|
|
61
|
+
}
|
|
62
|
+
],
|
|
63
|
+
"citation": "@article{wu2024longmemeval,\n title={LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory},\n author={Di Wu and Hongwei Wang and Wenhao Yu and Yuwei Zhang and Kai-Wei Chang and Dong Yu},\n year={2024},\n eprint={2410.10813},\n archivePrefix={arXiv},\n primaryClass={cs.CL},\n url={https://arxiv.org/abs/2410.10813},\n}",
|
|
64
|
+
"citation_source": "https://github.com/xiaowu0162/LongMemEval"
|
|
65
|
+
}
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
"""Run LongMemEval over JSON lines using its pinned upstream code and client.
|
|
2
|
+
|
|
3
|
+
For cap.memory/v1, setup feeds history sessions to add and the question to
|
|
4
|
+
recall. produce uses the upstream history-chats prompt and reader call on
|
|
5
|
+
recalled text. The reading option selects chain-of-note (800 tokens) or
|
|
6
|
+
direct answers (500). Recalled text is not truncated: upstream truncation
|
|
7
|
+
requires structured sessions. Reader calls retain upstream's unbounded
|
|
8
|
+
five-second rate-limit retries; an empty reply fails the question.
|
|
9
|
+
|
|
10
|
+
For Original, answer runs retrieval when selected, then run_generation.py
|
|
11
|
+
on one question using the shell script's model and option mappings. The
|
|
12
|
+
retrieval log is converted to a JSON list for the reader. Hypotheses stay
|
|
13
|
+
in the unit directory; a skipped question becomes an error. score runs
|
|
14
|
+
evaluate_qa.py on that hypothesis; capability runs call its judge directly.
|
|
15
|
+
Both paths use the run's relay routes for reader and judge model calls.
|
|
16
|
+
The oracle control supplies the dataset's answer, including its explanation
|
|
17
|
+
for abstention questions.
|
|
18
|
+
|
|
19
|
+
Questions retain dataset order and question_id; the question_type option
|
|
20
|
+
keeps one type's questions, abstention ones included. Large datasets are streamed
|
|
21
|
+
once into a line-per-question index so workers need not load the whole file.
|
|
22
|
+
"""
|
|
23
|
+
import ast
|
|
24
|
+
import contextlib
|
|
25
|
+
import glob
|
|
26
|
+
import inspect
|
|
27
|
+
import json
|
|
28
|
+
import os
|
|
29
|
+
import re
|
|
30
|
+
import shutil
|
|
31
|
+
import stat
|
|
32
|
+
import subprocess
|
|
33
|
+
import tarfile
|
|
34
|
+
import sys
|
|
35
|
+
import textwrap
|
|
36
|
+
import zipfile
|
|
37
|
+
|
|
38
|
+
sys.path.insert(0, os.path.join(os.environ["NULLIUS_UPSTREAM"], "src"))
|
|
39
|
+
with contextlib.redirect_stdout(sys.stderr): # stdout is the protocol's: the upstream's prints go to the log
|
|
40
|
+
from evaluation import evaluate_qa # noqa: E402 as published
|
|
41
|
+
from generation import run_generation # noqa: E402
|
|
42
|
+
from openai import OpenAI # noqa: E402 the client evaluate_qa is written for, pinned beside it
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# Match the configured reading default to the upstream chain-of-note mode.
|
|
46
|
+
COT = os.environ.get("NULLIUS_OPTION_READING", "con") == "con"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def reader_template():
|
|
50
|
+
"""prepare_prompt's template for retrieved history chats (``merge_key_expansion_into_value``
|
|
51
|
+
none), with chain of thought or without (``cot``, by the run's reading): a local name in the
|
|
52
|
+
upstream, so read from its source through the imported module."""
|
|
53
|
+
tree = ast.parse(textwrap.dedent(inspect.getsource(run_generation.prepare_prompt)))
|
|
54
|
+
for node in ast.walk(tree):
|
|
55
|
+
if isinstance(node, ast.If) and "merge_key_expansion_into_value is None" in ast.unparse(node.test):
|
|
56
|
+
for inner in node.body:
|
|
57
|
+
if isinstance(inner, ast.If) and ast.unparse(inner.test) == "cot":
|
|
58
|
+
for st in (inner.body if COT else inner.orelse):
|
|
59
|
+
if isinstance(st, ast.Assign) and isinstance(st.value, ast.Constant) \
|
|
60
|
+
and [ast.unparse(t) for t in st.targets] == ["answer_prompt_template"]:
|
|
61
|
+
return st.value.value
|
|
62
|
+
raise RuntimeError("LongMemEval's prepare_prompt no longer holds its history-chats template where the driver reads it")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def gen_length():
|
|
66
|
+
"""main's answer length (``gen_length = 500 if not args.cot else 800``), by the run's reading:
|
|
67
|
+
a local name in the upstream, so read from its source through the imported module."""
|
|
68
|
+
tree = ast.parse(textwrap.dedent(inspect.getsource(run_generation.main)))
|
|
69
|
+
for node in ast.walk(tree):
|
|
70
|
+
if isinstance(node, ast.Assign) and [ast.unparse(t) for t in node.targets] == ["gen_length"] \
|
|
71
|
+
and isinstance(node.value, ast.IfExp) and ast.unparse(node.value.test) == "not args.cot" \
|
|
72
|
+
and isinstance(node.value.body, ast.Constant) and isinstance(node.value.orelse, ast.Constant):
|
|
73
|
+
return (node.value.orelse if COT else node.value.body).value
|
|
74
|
+
raise RuntimeError("LongMemEval's main no longer sets gen_length where the driver reads it")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
READER = reader_template()
|
|
78
|
+
GEN_LENGTH = gen_length()
|
|
79
|
+
INDEX = {}
|
|
80
|
+
TYPES = {} # question_id -> question_type, read with the index
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def entries(path, size=1 << 20):
|
|
84
|
+
"""The objects of the JSON list at ``path``, one at a time."""
|
|
85
|
+
decoder = json.JSONDecoder()
|
|
86
|
+
with open(path, encoding="utf-8") as f:
|
|
87
|
+
buf = f.read(size)
|
|
88
|
+
pos = buf.index("[") + 1
|
|
89
|
+
while True:
|
|
90
|
+
while True:
|
|
91
|
+
while pos < len(buf) and buf[pos] in " \t\r\n,":
|
|
92
|
+
pos += 1
|
|
93
|
+
if pos < len(buf):
|
|
94
|
+
break
|
|
95
|
+
more = f.read(size)
|
|
96
|
+
if not more:
|
|
97
|
+
return
|
|
98
|
+
buf, pos = buf[pos:] + more, 0
|
|
99
|
+
if buf[pos] == "]":
|
|
100
|
+
return
|
|
101
|
+
while True:
|
|
102
|
+
try:
|
|
103
|
+
obj, end = decoder.raw_decode(buf, pos)
|
|
104
|
+
break
|
|
105
|
+
except json.JSONDecodeError:
|
|
106
|
+
more = f.read(size)
|
|
107
|
+
if not more:
|
|
108
|
+
raise
|
|
109
|
+
buf, pos = buf[pos:] + more, 0
|
|
110
|
+
yield obj
|
|
111
|
+
buf, pos = buf[end:], 0
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def lines_path():
|
|
115
|
+
src = os.environ["NULLIUS_DATASET"]
|
|
116
|
+
out = src + ".lines"
|
|
117
|
+
if not os.path.isfile(out): # once per file: a copy, a line per question, written whole then renamed
|
|
118
|
+
part = f"{out}.{os.getpid()}.part"
|
|
119
|
+
with open(part, "w", encoding="utf-8") as f:
|
|
120
|
+
for e in entries(src):
|
|
121
|
+
f.write(json.dumps(e, ensure_ascii=False) + "\n")
|
|
122
|
+
os.replace(part, out)
|
|
123
|
+
return out
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def index():
|
|
127
|
+
if not INDEX:
|
|
128
|
+
with open(lines_path(), "rb") as f:
|
|
129
|
+
at = 0
|
|
130
|
+
for line in f:
|
|
131
|
+
q = json.loads(line)
|
|
132
|
+
INDEX[q["question_id"]] = at
|
|
133
|
+
TYPES[q["question_id"]] = q.get("question_type")
|
|
134
|
+
at += len(line)
|
|
135
|
+
return INDEX
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def entry(qid):
|
|
139
|
+
with open(lines_path(), "rb") as f:
|
|
140
|
+
f.seek(index()[qid])
|
|
141
|
+
return json.loads(f.readline())
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def units(dataset, setup=None):
|
|
145
|
+
# a setup's unit is a question too, as run_generation reads them; one type keeps its abstention
|
|
146
|
+
# questions too (their question_type is the type's), in dataset order, before any --limit
|
|
147
|
+
kind = os.environ.get("NULLIUS_OPTION_QUESTION_TYPE", "all")
|
|
148
|
+
return {"units": [q for q in index() if kind == "all" or TYPES[q] == kind]}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def setup(unit, dir):
|
|
152
|
+
q = entry(unit)
|
|
153
|
+
dates = q.get("haystack_dates") or [""] * len(q["haystack_session_ids"])
|
|
154
|
+
sessions = []
|
|
155
|
+
for sid, turns, date in zip(q["haystack_session_ids"], q["haystack_sessions"], dates):
|
|
156
|
+
ts = [{"role": t.get("role") or "user", "text": str(t.get("content") or "")} for t in turns]
|
|
157
|
+
sessions.append({"id": str(sid), "date": date or "", "turns": ts,
|
|
158
|
+
"text": f"Session Date: {date}\n" + "\n".join(f"{t['role']}: {t['text']}" for t in ts)})
|
|
159
|
+
return {"memory": {"add": sessions, "recall": [{"id": unit, "question": q["question"]}]}}
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def produce(unit, dir, recalled, model, base_url, key):
|
|
163
|
+
q = entry(unit)
|
|
164
|
+
memory = recalled[0]["memory"] if recalled else ""
|
|
165
|
+
prompt = READER.format(memory or "(nothing recalled)", q.get("question_date") or "", q["question"])
|
|
166
|
+
# run_generation.main's own request for one question: its client, its kwargs, its call, its reading of the reply
|
|
167
|
+
client = OpenAI(api_key=key, base_url=base_url)
|
|
168
|
+
kwargs = {"model": model, "messages": [{"role": "user", "content": prompt}], "n": 1, "temperature": 0, "max_tokens": GEN_LENGTH}
|
|
169
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
170
|
+
completion = run_generation.chat_completions_with_backoff(client, **kwargs)
|
|
171
|
+
return {"answer": completion.choices[0].message.content.strip()}
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def score(unit, dir, transcript, answer, judge=None, setup=None, **_):
|
|
175
|
+
if not judge:
|
|
176
|
+
raise ValueError("LongMemEval is graded by a judge model, and none was given")
|
|
177
|
+
if setup:
|
|
178
|
+
return score_setup(unit, dir, judge)
|
|
179
|
+
q = entry(unit)
|
|
180
|
+
# evaluate_qa's own loop, for one question: its prompt, its client call (backoff included), its reading of the reply
|
|
181
|
+
prompt = evaluate_qa.get_anscheck_prompt(q["question_type"], q["question"], q["answer"], answer or "", abstention="_abs" in unit)
|
|
182
|
+
client = OpenAI(api_key=judge["key"], base_url=judge["base_url"])
|
|
183
|
+
completion = evaluate_qa.chat_completions_with_backoff(client, model=judge["model"], messages=[{"role": "user", "content": prompt}],
|
|
184
|
+
n=1, temperature=0, max_tokens=10)
|
|
185
|
+
said = completion.choices[0].message.content.strip()
|
|
186
|
+
ok = "yes" in said.lower()
|
|
187
|
+
return {"reward": 1.0 if ok else 0.0, "passed": ok,
|
|
188
|
+
"detail": {"question_type": q["question_type"], "judge": judge["model"], "judge_said": said, "reference": q["answer"]}}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def oracle(unit, dir):
|
|
192
|
+
return {"answer": str(entry(unit)["answer"])}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# --- a setup: the paper's pipeline, run as its scripts run it ----------------------------------------
|
|
196
|
+
|
|
197
|
+
def _opt(name, default):
|
|
198
|
+
return os.environ.get("NULLIUS_OPTION_" + name.upper(), default)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _script(*path):
|
|
202
|
+
with open(os.path.join(os.environ["NULLIUS_UPSTREAM"], *path), encoding="utf-8") as f:
|
|
203
|
+
return f.read()
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def model_zoo():
|
|
207
|
+
"""run_generation.sh's model_zoo: the alias a run names -> the name run_generation.py is given."""
|
|
208
|
+
return dict(re.findall(r'^model_zoo\["([^"]+)"\]="([^"]+)"', _script("src", "generation", "run_generation.sh"), re.M))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def alias_table(var, target):
|
|
212
|
+
"""One of run_generation.sh's if-chains, ``$var == "a" || ...`` -> ``target="value"``: the alias -> its value."""
|
|
213
|
+
out = {}
|
|
214
|
+
for cond, value in re.findall(r'if \[\[ (.*?) \]\]; then\s+' + target + r'="([^"]*)"', _script("src", "generation", "run_generation.sh")):
|
|
215
|
+
for a in re.findall(r'\$' + var + r' == "([^"]+)"', cond):
|
|
216
|
+
out[a] = value
|
|
217
|
+
return out
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
RETRIEVED = "retrieval" # in the unit's folder: run_retrieval.py's log, for a retriever of the paper's
|
|
221
|
+
|
|
222
|
+
# Stella V5, which run_retrieval.py reads from the cache folder rather than the hub - the README has it downloaded by
|
|
223
|
+
# hand: the model, its code and its 1024-dimension head (2_Dense_1024), the one run_retrieval.py loads. The weights
|
|
224
|
+
# once, as model.safetensors, which AutoModel reads first: the repository's pytorch_model.bin is the same 6 GB again.
|
|
225
|
+
STELLA = ("dunzhang/stella_en_1.5B_v5", "dunzhang_stella_en_1.5B_v5",
|
|
226
|
+
["*.json", "*.py", "*.txt", "*.model", "model*.safetensors", "2_Dense_1024/*"])
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def model_cache(up, retriever):
|
|
230
|
+
"""run_retrieval.sh's cache - the repository's own model_cache, HF_HOME and --cache_dir both - shared by every
|
|
231
|
+
question, so a retriever's model is downloaded once; Stella fetched into it first, as the README has it."""
|
|
232
|
+
cache = os.path.join(up, "model_cache")
|
|
233
|
+
os.makedirs(cache, exist_ok=True)
|
|
234
|
+
need = ("config.json", "model.safetensors", "2_Dense_1024/pytorch_model.bin") # each renamed in only once whole
|
|
235
|
+
if retriever == "flat-stella" and not all(os.path.exists(os.path.join(cache, STELLA[1], f)) for f in need):
|
|
236
|
+
from huggingface_hub import snapshot_download # the driver's environment's, pinned
|
|
237
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
238
|
+
snapshot_download(STELLA[0], local_dir=os.path.join(cache, STELLA[1]), allow_patterns=STELLA[2])
|
|
239
|
+
return cache
|
|
240
|
+
GENERATED = "generation" # run_generation.py's test log
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _run(argv, cwd, env, log, what):
|
|
244
|
+
with open(log, "w") as f:
|
|
245
|
+
rc = subprocess.run(argv, cwd=cwd, env=env, stdout=f, stderr=subprocess.STDOUT).returncode
|
|
246
|
+
if rc != 0:
|
|
247
|
+
with open(log, encoding="utf-8", errors="replace") as f:
|
|
248
|
+
tail = [ln for ln in f.read().splitlines() if ln.strip()][-3:]
|
|
249
|
+
raise RuntimeError(f"{what} exited {rc}: " + " | ".join(tail))
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
# Exact members of the authors' released archive; no search by suffix and no archive-wide extraction.
|
|
253
|
+
EXPANSIONS = {
|
|
254
|
+
"session-summ": "merged_cache_session-summ_llama-3.1-8b-instruct_zeroshot.json",
|
|
255
|
+
"session-keyphrase": "merged_cache_session-keyphrase_llama-3.1-8b-instruct_zeroshot.json",
|
|
256
|
+
"session-userfact": "merged_cache_session-userfact_llama-3.1-8b-instruct_ICL.json",
|
|
257
|
+
"turn-keyphrase": "merged_cache_turn-keyphrase_llama-3.1-8b-instruct_zeroshot.json",
|
|
258
|
+
"turn-userfact": "merged_cache_turn-userfact_llama-3.1-8b-instruct_ICL.json",
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def expansion_cache(method, dir):
|
|
263
|
+
member = EXPANSIONS[method]
|
|
264
|
+
path = os.path.join(dir, member)
|
|
265
|
+
with tarfile.open(os.environ["NULLIUS_FILE_INDEX_EXPANSIONS"]) as archive:
|
|
266
|
+
info = archive.getmember(member)
|
|
267
|
+
if not info.isfile():
|
|
268
|
+
raise ValueError(f"index expansion cache {member} is not a regular file")
|
|
269
|
+
with archive.extractfile(info) as source, open(path, "wb") as target:
|
|
270
|
+
shutil.copyfileobj(source, target)
|
|
271
|
+
return path
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def summary_tokenizer(dir):
|
|
275
|
+
root = os.path.join(dir, "nltk_data")
|
|
276
|
+
target_dir = os.path.join(root, "tokenizers", "punkt_tab", "english")
|
|
277
|
+
os.makedirs(target_dir, exist_ok=True)
|
|
278
|
+
with zipfile.ZipFile(os.environ["NULLIUS_FILE_PUNKT_TAB"]) as archive:
|
|
279
|
+
for name in ("collocations.tab", "sent_starters.txt", "abbrev_types.txt", "ortho_context.tab"):
|
|
280
|
+
info = archive.getinfo("punkt_tab/english/" + name)
|
|
281
|
+
if info.is_dir() or stat.S_ISLNK(info.external_attr >> 16):
|
|
282
|
+
raise ValueError(f"tokenizer member {name} is not a regular file")
|
|
283
|
+
with archive.open(info) as source, open(os.path.join(target_dir, name), "wb") as target:
|
|
284
|
+
shutil.copyfileobj(source, target)
|
|
285
|
+
return root
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def answer(unit, dir, setup, model, base_url, key, judge=None, simulator=None, attempt=None):
|
|
289
|
+
up = os.environ["NULLIUS_UPSTREAM"]
|
|
290
|
+
retriever = _opt("retriever", "full-history-session")
|
|
291
|
+
reading = _opt("reading", "con")
|
|
292
|
+
if reading == "con" and _opt("separate_notes", "false") == "true":
|
|
293
|
+
reading = "con-separate"
|
|
294
|
+
retriever_type = alias_table("retriever_alias", "retriever_type").get(retriever)
|
|
295
|
+
flags = alias_table("reading_method", "reading_flags").get(reading)
|
|
296
|
+
served = model_zoo().get(model)
|
|
297
|
+
if retriever_type is None or flags is None or served is None:
|
|
298
|
+
raise ValueError(f"run_generation.sh does not know {retriever!r}, {reading!r} or {model!r}")
|
|
299
|
+
in_file = os.path.join(dir, f"{unit}.json") # the one question, as the released files hold it
|
|
300
|
+
with open(in_file, "w", encoding="utf-8") as f:
|
|
301
|
+
json.dump([entry(unit)], f)
|
|
302
|
+
env = dict(os.environ, PYTHONIOENCODING="utf-8")
|
|
303
|
+
m = re.match(r"(flat-[a-z0-9]+)-(session|turn)$", retriever)
|
|
304
|
+
if m: # the paper's retrieval first, as run_retrieval.sh runs it
|
|
305
|
+
out = os.path.join(dir, RETRIEVED)
|
|
306
|
+
os.makedirs(out, exist_ok=True)
|
|
307
|
+
cache = model_cache(up, m.group(1))
|
|
308
|
+
expansion = _opt(m.group(2) + "_expansion", "none")
|
|
309
|
+
if expansion != "none" and not expansion.startswith(m.group(2) + "-"):
|
|
310
|
+
raise ValueError("index expansion must match the retriever's granularity")
|
|
311
|
+
expansion_file = expansion_cache(expansion, dir) if expansion != "none" else "none"
|
|
312
|
+
join = _opt("expansion_join", "separate") if expansion != "none" else "none"
|
|
313
|
+
retrieval_env = dict(env, PYTHONPATH=up, HF_HOME=cache)
|
|
314
|
+
if expansion == "session-summ" and join.startswith("split-"):
|
|
315
|
+
retrieval_env["NLTK_DATA"] = summary_tokenizer(dir)
|
|
316
|
+
_run([sys.executable, "run_retrieval.py", "--in_file", in_file, "--retriever", m.group(1), "--granularity", m.group(2),
|
|
317
|
+
"--index_expansion_method", expansion, "--index_expansion_result_join_mode", join,
|
|
318
|
+
"--index_expansion_llm", "none", "--index_expansion_result_cache", expansion_file,
|
|
319
|
+
"--out_dir", out, "--outfile_prefix", "none", "--cache_dir", cache],
|
|
320
|
+
os.path.join(up, "src", "retrieval"), retrieval_env,
|
|
321
|
+
os.path.join(dir, "run_retrieval.log"), "run_retrieval.py")
|
|
322
|
+
# its log is JSON lines, which run_generation.py reads only when json.load fails on it - and a log of one
|
|
323
|
+
# question is one line, which json.load reads as that question's object: the same entries are handed
|
|
324
|
+
# over as a list, the form the released files have and run_generation.py reads first
|
|
325
|
+
with open(glob.glob(os.path.join(out, "*_retrievallog_*"))[0], encoding="utf-8") as f:
|
|
326
|
+
ranked = [json.loads(ln) for ln in f if ln.strip()]
|
|
327
|
+
in_file = os.path.join(dir, f"{unit}.retrieved.json")
|
|
328
|
+
with open(in_file, "w", encoding="utf-8") as f:
|
|
329
|
+
json.dump(ranked, f)
|
|
330
|
+
out = os.path.join(dir, GENERATED)
|
|
331
|
+
os.makedirs(out, exist_ok=True)
|
|
332
|
+
argv = [sys.executable, "run_generation.py", "--in_file", in_file, "--out_dir", out,
|
|
333
|
+
"--model_name", served, "--model_alias", model, "--openai_key", key, "--openai_base_url", base_url,
|
|
334
|
+
"--retriever_type", retriever_type, "--topk_context", _opt("topk", "1000"),
|
|
335
|
+
"--merge_key_expansion_into_value", _opt("merge_key_expansion", "none"),
|
|
336
|
+
"--history_format", _opt("history_format", "json"), "--useronly", _opt("useronly", "false")] + flags.split()
|
|
337
|
+
log = os.path.join(dir, "run_generation.log")
|
|
338
|
+
_run(argv, os.path.join(up, "src", "generation"), env, log, "run_generation.py")
|
|
339
|
+
hyp = [json.loads(ln) for p in glob.glob(os.path.join(out, "*_testlog_*")) for ln in open(p, encoding="utf-8") if ln.strip()]
|
|
340
|
+
if not hyp: # main skipped the question: its own words say why
|
|
341
|
+
with open(log, encoding="utf-8", errors="replace") as f:
|
|
342
|
+
said = [ln for ln in f.read().splitlines() if ln.startswith("One exception captured")]
|
|
343
|
+
raise RuntimeError("run_generation.py skipped the question: " + (said[-1] if said else "no hypothesis written"))
|
|
344
|
+
return {"answer": hyp[0]["hypothesis"], "model": model}
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def score_setup(unit, dir, judge):
|
|
348
|
+
hyp = glob.glob(os.path.join(dir, GENERATED, "*_testlog_*"))
|
|
349
|
+
hyp = [p for p in hyp if ".eval-results-" not in p]
|
|
350
|
+
if not hyp:
|
|
351
|
+
raise RuntimeError("no hypothesis to score: run_generation.py wrote none")
|
|
352
|
+
up = os.environ["NULLIUS_UPSTREAM"]
|
|
353
|
+
env = dict(os.environ, PYTHONIOENCODING="utf-8", OPENAI_API_KEY=judge["key"], OPENAI_BASE_URL=judge["base_url"])
|
|
354
|
+
env.pop("OPENAI_ORGANIZATION", None)
|
|
355
|
+
_run([sys.executable, "evaluate_qa.py", "gpt-4o", hyp[0], os.path.join(dir, f"{unit}.json")],
|
|
356
|
+
os.path.join(up, "src", "evaluation"), env, os.path.join(dir, "evaluate_qa.log"), "evaluate_qa.py")
|
|
357
|
+
with open(hyp[0] + ".eval-results-gpt-4o", encoding="utf-8") as f:
|
|
358
|
+
rows = [json.loads(ln) for ln in f if ln.strip()]
|
|
359
|
+
if not rows:
|
|
360
|
+
raise RuntimeError("evaluate_qa.py wrote no verdict")
|
|
361
|
+
ok = bool(rows[0]["autoeval_label"]["label"])
|
|
362
|
+
q = entry(unit)
|
|
363
|
+
return {"reward": 1.0 if ok else 0.0, "passed": ok,
|
|
364
|
+
"detail": {"question_type": q["question_type"], "judge": judge["model"], "asked_as": rows[0]["autoeval_label"]["model"],
|
|
365
|
+
"reference": q["answer"], "answer": rows[0]["hypothesis"],
|
|
366
|
+
"options": {n: _opt(n, d) for n, d in (("retriever", "full-history-session"), ("topk", "1000"),
|
|
367
|
+
("reading", "con"), ("separate_notes", "false"), ("history_format", "json"), ("useronly", "false"),
|
|
368
|
+
("merge_key_expansion", "none"), ("session_expansion", "none"), ("turn_expansion", "none"),
|
|
369
|
+
("expansion_join", "separate"))}}}
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
HANDLERS = {"units": units, "setup": setup, "produce": produce, "score": score, "oracle": oracle, "answer": answer}
|
|
373
|
+
for line in sys.stdin:
|
|
374
|
+
try:
|
|
375
|
+
req = json.loads(line)
|
|
376
|
+
out = HANDLERS[req.pop("op")](**req)
|
|
377
|
+
except Exception as e:
|
|
378
|
+
out = {"error": f"{type(e).__name__}: {e}"}
|
|
379
|
+
print(json.dumps(out), flush=True)
|