nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
"""The benchmark catalogue: which benchmarks exist, in what state, and where
|
|
2
|
+
each dataset comes from.
|
|
3
|
+
|
|
4
|
+
Bundled JSON files beside this module, overlaid by the user's catalogue,
|
|
5
|
+
one folder per benchmark. The app renders them as ``/benchmarks`` - a card
|
|
6
|
+
per benchmark - and
|
|
7
|
+
``/benchmarks/<key>``, one page per folder: how it runs, its datasets, its
|
|
8
|
+
limitations (the upstream's, then Nullius's), its citation. ``validate`` gates it.
|
|
9
|
+
The knowledge itself — what runs a benchmark, how its dataset is fetched —
|
|
10
|
+
is written once, here::
|
|
11
|
+
|
|
12
|
+
benchmarks/<key>/ one folder per benchmark, named by its key
|
|
13
|
+
benchmark.json the entry, its datasets nested in it
|
|
14
|
+
standard.json its definition on the Nullius standard, when
|
|
15
|
+
it runs there (nullius.standard)
|
|
16
|
+
|
|
17
|
+
Everything about a benchmark is in its folder. A dataset's ``benchmark`` is
|
|
18
|
+
the folder it sits in, never written; its key is unique across the whole
|
|
19
|
+
catalogue. ``load`` reads the folders in name order, so the catalogue's
|
|
20
|
+
order is the folders', and nothing depends on it: the page and the builder's
|
|
21
|
+
benchmark list sort by title, a lookup goes by key, and a benchmark's
|
|
22
|
+
datasets keep the order its entry writes them in. ``layout`` is the check on
|
|
23
|
+
the tree itself - a folder named after its key - and ``validate`` runs it.
|
|
24
|
+
|
|
25
|
+
A dataset's ``source`` is a short phrase (Harbor registry, Harbor hub,
|
|
26
|
+
GitHub, …). Each benchmark carries a ``category`` (Coding, Terminal, Customer
|
|
27
|
+
service, …) used as a tag and a filter.
|
|
28
|
+
|
|
29
|
+
An optional ``task_count`` gives the number of distinct tasks when datasets
|
|
30
|
+
overlap, such as a full split beside difficulty subsets. It controls the
|
|
31
|
+
benchmark headline only; every dataset keeps its own count.
|
|
32
|
+
|
|
33
|
+
Every entry says where it comes from (``origin``): ``official``, the
|
|
34
|
+
publisher's own benchmark, or
|
|
35
|
+
``community``, adapted by the community. A card and a page show it as a tag
|
|
36
|
+
beside the category, and the Benchmarks page shelves by it. Whether anyone
|
|
37
|
+
has run it is not part of an entry.
|
|
38
|
+
|
|
39
|
+
Routes: a benchmark reaches a run through a harness, and ``Benchmark.routes``
|
|
40
|
+
says, for each harness in ``HARNESSES``, whether that route is usable here.
|
|
41
|
+
A harness the entry names is usable: the Nullius standard, or Harbor. The page
|
|
42
|
+
lists a benchmark with a usable route, and says why for one without. A
|
|
43
|
+
dataset is shown inside its
|
|
44
|
+
benchmark's entry, so it is listed exactly when its benchmark is.
|
|
45
|
+
"""
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import json
|
|
49
|
+
import os
|
|
50
|
+
from dataclasses import dataclass, field
|
|
51
|
+
|
|
52
|
+
from nullius import paths
|
|
53
|
+
from . import storage
|
|
54
|
+
|
|
55
|
+
# Where an entry comes from: the publisher's benchmark (official),
|
|
56
|
+
# or adapted by the community (community).
|
|
57
|
+
ORIGINS = ("official", "community")
|
|
58
|
+
ROOT = os.path.dirname(os.path.abspath(__file__))
|
|
59
|
+
BENCHMARKS = "benchmarks" # the folder of benchmark folders, under ROOT
|
|
60
|
+
ENTRY = "benchmark.json" # a benchmark's entry, in its folder
|
|
61
|
+
|
|
62
|
+
# The harnesses a benchmark can be run through, in the order the Benchmarks
|
|
63
|
+
# page lists them: the Nullius standard first (benchmarks/<key>/standard.json,
|
|
64
|
+
# nullius.standard), then Harbor, each a route of the entries that name it. A
|
|
65
|
+
# benchmark's own scripts run as the standard's Original setup.
|
|
66
|
+
HARNESSES = (("standard", "Nullius"), ("harbor", "Harbor"))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass(frozen=True)
|
|
71
|
+
class Route:
|
|
72
|
+
"""One harness route of a benchmark: usable, or unavailable with a reason.
|
|
73
|
+
|
|
74
|
+
``usable`` is the one thing a selector may act on. ``reason`` is filled
|
|
75
|
+
only when it is not — the text a page shows greyed beside the route, so a
|
|
76
|
+
reader learns *why* rather than only that it cannot be picked."""
|
|
77
|
+
|
|
78
|
+
harness: str # a key of HARNESSES: harbor or standard
|
|
79
|
+
label: str # its label: Harbor
|
|
80
|
+
usable: bool
|
|
81
|
+
reason: str = ""
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def summary(self) -> str:
|
|
85
|
+
"""The route as one phrase: `Harbor · runnable` or `Harbor · unavailable — …`."""
|
|
86
|
+
return f"{self.label} · runnable" if self.usable else f"{self.label} · unavailable — {self.reason}"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass
|
|
90
|
+
class Benchmark:
|
|
91
|
+
key: str
|
|
92
|
+
title: str
|
|
93
|
+
category: str = "" # one tag: Coding, Terminal, Customer service, …
|
|
94
|
+
domain: str = ""
|
|
95
|
+
#: how a task is scored, in two or three words for its card; each dataset's verifier says it in full
|
|
96
|
+
scoring: str = ""
|
|
97
|
+
harness: list = field(default_factory=list)
|
|
98
|
+
good_to_know: list = field(default_factory=list)
|
|
99
|
+
#: what the upstream itself gets wrong or leaves out, as published: not Nullius's to fix
|
|
100
|
+
upstream_limitations: list = field(default_factory=list)
|
|
101
|
+
#: Nullius's own: how it runs the benchmark differently, what it does not offer yet
|
|
102
|
+
limitations: list = field(default_factory=list)
|
|
103
|
+
datasets: list = field(default_factory=list)
|
|
104
|
+
#: declared per-phase timeouts in seconds (env_setup, agent_setup,
|
|
105
|
+
#: agent_run, verify) as recorded from a real run; empty when no run has
|
|
106
|
+
#: recorded them
|
|
107
|
+
timeouts: dict = field(default_factory=dict)
|
|
108
|
+
#: the publisher's benchmark or a community adaptation (ORIGINS)
|
|
109
|
+
origin: str = "official"
|
|
110
|
+
#: how it runs as published when Harbor runs it: the upstream's own agent, by
|
|
111
|
+
#: the name Harbor registers it (``agent``), and why it is the published one
|
|
112
|
+
#: (``says``); a benchmark on the standard says it in its ``setups`` instead
|
|
113
|
+
published: dict = field(default_factory=dict)
|
|
114
|
+
#: how its verifier's own report is read (``nullius.runs.verifier``):
|
|
115
|
+
#: the file, the fields that say pass or fail, the words the pages use; empty
|
|
116
|
+
#: when the reward is all there is
|
|
117
|
+
report: dict = field(default_factory=dict)
|
|
118
|
+
#: how to cite it: the BibTeX its official repository gives, else its paper's
|
|
119
|
+
citation: str = ""
|
|
120
|
+
#: where that citation was taken from
|
|
121
|
+
citation_source: str = ""
|
|
122
|
+
#: Distinct tasks across dataset views, when known; otherwise show their count summary.
|
|
123
|
+
task_count: int | None = None
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def label(self) -> str:
|
|
127
|
+
return self.origin.capitalize()
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def can_run(self) -> bool:
|
|
131
|
+
"""Listed in the catalogue: a harness it names is here."""
|
|
132
|
+
return bool(self.usable_routes)
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def routes(self) -> list:
|
|
136
|
+
"""Every harness, as a `Route`: usable when the entry names it, since
|
|
137
|
+
a harness in `HARNESSES` is here; Harbor, named or not, says so when
|
|
138
|
+
it is not this entry's."""
|
|
139
|
+
out = []
|
|
140
|
+
for harness, label in HARNESSES:
|
|
141
|
+
if harness in self.harness:
|
|
142
|
+
out.append(Route(harness, label, True))
|
|
143
|
+
elif harness == "harbor":
|
|
144
|
+
out.append(Route(harness, label, False, "no Harbor dataset"))
|
|
145
|
+
return out
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def usable_routes(self) -> list:
|
|
149
|
+
return [r for r in self.routes if r.usable]
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def blocker(self) -> str:
|
|
153
|
+
"""Why nothing is usable, in one phrase — empty when a route is: the
|
|
154
|
+
reason a benchmark's page shows in place of Build a run."""
|
|
155
|
+
if self.usable_routes:
|
|
156
|
+
return ""
|
|
157
|
+
missing = [h for h in self.harness if h not in dict(HARNESSES)]
|
|
158
|
+
if missing:
|
|
159
|
+
return "no harness here for " + ", ".join(missing)
|
|
160
|
+
return "it names no harness"
|
|
161
|
+
|
|
162
|
+
@property
|
|
163
|
+
def route_case(self) -> str:
|
|
164
|
+
"""Which of the four discovery cases this entry is — what a test and a
|
|
165
|
+
walk-through name: `harbor-only`, `multi-route`, `non-harbor` (usable
|
|
166
|
+
routes, none of them Harbor) or `none`. A benchmark served only by the
|
|
167
|
+
Nullius standard is `non-harbor`; one with both a Harbor adapter and a
|
|
168
|
+
definition on the standard is `multi-route`."""
|
|
169
|
+
usable = [r.harness for r in self.usable_routes]
|
|
170
|
+
if not usable:
|
|
171
|
+
return "none"
|
|
172
|
+
if usable == ["harbor"]:
|
|
173
|
+
return "harbor-only"
|
|
174
|
+
return "multi-route" if "harbor" in usable else "non-harbor"
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass
|
|
178
|
+
class Dataset:
|
|
179
|
+
key: str
|
|
180
|
+
title: str
|
|
181
|
+
benchmark: str = ""
|
|
182
|
+
tasks: int = 0
|
|
183
|
+
official: str = ""
|
|
184
|
+
source: str = "" # where it comes from, in a phrase: Harbor hub, GitHub …
|
|
185
|
+
registry: str = ""
|
|
186
|
+
hub: str = ""
|
|
187
|
+
#: how its tasks are scored
|
|
188
|
+
verifier: str = ""
|
|
189
|
+
notes: str = ""
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def sort_key(title: str) -> str:
|
|
193
|
+
"""Alphabetical as a reader expects: a title starting with τ files under T."""
|
|
194
|
+
return title.replace("τ", "t").replace("³", "3").lower()
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@dataclass
|
|
198
|
+
class Catalog:
|
|
199
|
+
benchmarks: list
|
|
200
|
+
datasets: list
|
|
201
|
+
path: str = ROOT # explicit root, or the writable user catalogue
|
|
202
|
+
_folders: dict = field(default_factory=dict, repr=False)
|
|
203
|
+
|
|
204
|
+
def entry_dir(self, key: str) -> str:
|
|
205
|
+
"""The selected folder: all files come from the same entry copy."""
|
|
206
|
+
return self._folders.get(key, os.path.join(self.path, BENCHMARKS, key))
|
|
207
|
+
|
|
208
|
+
def benchmark(self, key: str):
|
|
209
|
+
return next((b for b in self.benchmarks if b.key == key), None)
|
|
210
|
+
|
|
211
|
+
def dataset(self, key: str):
|
|
212
|
+
return next((d for d in self.datasets if d.key == key), None)
|
|
213
|
+
|
|
214
|
+
def datasets_of(self, b: "Benchmark") -> list:
|
|
215
|
+
"""A benchmark's datasets, as `Dataset`, in the order its entry writes them."""
|
|
216
|
+
return [d for d in (self.dataset(k) for k in b.datasets) if d]
|
|
217
|
+
|
|
218
|
+
def tasks_of(self, b: "Benchmark") -> tuple:
|
|
219
|
+
"""Its tasks as a number and a label, for a card: ``("500", "tasks")``,
|
|
220
|
+
``("500", "tasks each")`` when its datasets agree, ``("10–89", "tasks")``
|
|
221
|
+
when they do not, ``("—", "tasks")`` when none is recorded.
|
|
222
|
+
An explicit distinct task count takes precedence over those summaries."""
|
|
223
|
+
if type(b.task_count) is int and b.task_count > 0:
|
|
224
|
+
return f"{b.task_count:,}", "tasks"
|
|
225
|
+
counts = [d.tasks for d in self.datasets_of(b) if d.tasks]
|
|
226
|
+
if not counts:
|
|
227
|
+
return "—", "tasks"
|
|
228
|
+
if len(set(counts)) == 1:
|
|
229
|
+
return f"{counts[0]:,}", "tasks each" if len(counts) > 1 else "tasks"
|
|
230
|
+
return f"{min(counts):,}–{max(counts):,}", "tasks"
|
|
231
|
+
|
|
232
|
+
def files(self, key: str) -> list:
|
|
233
|
+
"""The files in a benchmark's folder, by name: what its page shows it is made of."""
|
|
234
|
+
base = self.entry_dir(key)
|
|
235
|
+
try:
|
|
236
|
+
return sorted(n for n in os.listdir(base)
|
|
237
|
+
if os.path.isfile(os.path.join(base, n)) and not n.startswith("."))
|
|
238
|
+
except OSError:
|
|
239
|
+
return []
|
|
240
|
+
|
|
241
|
+
@property
|
|
242
|
+
def listed_benchmarks(self) -> list:
|
|
243
|
+
"""What the catalogue shows (the Benchmarks tab, the rail): the
|
|
244
|
+
benchmarks with a usable route, alphabetically."""
|
|
245
|
+
return sorted((b for b in self.benchmarks if b.can_run),
|
|
246
|
+
key=lambda b: sort_key(b.title))
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def categories(self) -> list:
|
|
250
|
+
"""Every category the listed benchmarks use, alphabetically."""
|
|
251
|
+
return sorted({b.category for b in self.listed_benchmarks if b.category}, key=sort_key)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _pick(cls, data: dict):
|
|
255
|
+
known = {f for f in cls.__dataclass_fields__}
|
|
256
|
+
return cls(**{k: v for k, v in data.items() if k in known})
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _read(path: str):
|
|
260
|
+
with open(path, encoding="utf-8") as fh:
|
|
261
|
+
return json.load(fh)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def folders(root: str | None = None) -> list:
|
|
265
|
+
"""The benchmark folders under ``root``, by name: the catalogue's order.
|
|
266
|
+
A folder holding nothing but Python's ``__pycache__`` is left out: git
|
|
267
|
+
does not track the cache, so a checkout that once had a package there
|
|
268
|
+
keeps the folder after the package moves."""
|
|
269
|
+
return list(storage.folders(BENCHMARKS, ROOT, root))
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def entry_dir(key: str, root: str | None = None) -> str | None:
|
|
273
|
+
return storage.entry_dir(BENCHMARKS, key, ROOT, root)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def writable_dir(key: str, root: str | None = None) -> str:
|
|
277
|
+
return storage.writable_dir(BENCHMARKS, key, ROOT, root)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def signature(root: str | None = None) -> tuple:
|
|
281
|
+
"""What changes when any file ``load`` reads changes, or a benchmark
|
|
282
|
+
folder comes or goes: the app's cache reloads on it."""
|
|
283
|
+
entries = [os.path.join(folder, ENTRY) for folder in storage.folders(BENCHMARKS, ROOT, root).values()]
|
|
284
|
+
out = [(base, None) for base in storage.roots(ROOT, root)]
|
|
285
|
+
for p in entries:
|
|
286
|
+
try:
|
|
287
|
+
info = os.stat(p)
|
|
288
|
+
out.append((p, (info.st_mtime_ns, info.st_size)))
|
|
289
|
+
except OSError:
|
|
290
|
+
out.append((p, None))
|
|
291
|
+
return tuple(out)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def load(root: str | None = None) -> Catalog:
|
|
295
|
+
"""Every ``benchmarks/<key>/benchmark.json`` under
|
|
296
|
+
``root``, the folders in name order. A benchmark's datasets are nested in
|
|
297
|
+
its entry; each comes out as a `Dataset` naming the folder it sits in,
|
|
298
|
+
and the entry's ``datasets`` as their keys, in the order written.
|
|
299
|
+
|
|
300
|
+
With no root, user folders replace bundled folders of the same name.
|
|
301
|
+
An explicit root reads that tree alone, without the user overlay.
|
|
302
|
+
"""
|
|
303
|
+
benchmarks, datasets = [], []
|
|
304
|
+
selected = storage.folders(BENCHMARKS, ROOT, root)
|
|
305
|
+
for name, folder in selected.items():
|
|
306
|
+
path = os.path.join(folder, ENTRY)
|
|
307
|
+
if not os.path.isfile(path):
|
|
308
|
+
continue # layout() names it; nothing to list
|
|
309
|
+
try:
|
|
310
|
+
benchmark, nested = _entry(path, name)
|
|
311
|
+
except (OSError, ValueError, TypeError, AttributeError):
|
|
312
|
+
continue # a hand-edited entry that does not read: layout() names it, every page still opens
|
|
313
|
+
benchmarks.append(benchmark)
|
|
314
|
+
datasets.extend(nested)
|
|
315
|
+
return Catalog(
|
|
316
|
+
benchmarks=benchmarks,
|
|
317
|
+
datasets=datasets,
|
|
318
|
+
path=os.fspath(root) if root is not None else paths.catalogue_dir(),
|
|
319
|
+
_folders=selected,
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _entry(path: str, name: str) -> tuple:
|
|
324
|
+
"""(Benchmark, [Dataset]) of one ``benchmark.json``; ValueError or
|
|
325
|
+
TypeError when it is not JSON, not an object or lacks a field."""
|
|
326
|
+
entry = _read(path)
|
|
327
|
+
if not isinstance(entry, dict):
|
|
328
|
+
raise ValueError("not a JSON object")
|
|
329
|
+
if entry.get("experiment") or entry.get("origin") == "experiment":
|
|
330
|
+
entry = {**entry, "origin": "official"} # experiment is now official
|
|
331
|
+
nested = [d for d in entry.get("datasets", []) if isinstance(d, dict)]
|
|
332
|
+
return (_pick(Benchmark, {**entry, "datasets": [d.get("key") for d in nested]}),
|
|
333
|
+
[_pick(Dataset, {**d, "benchmark": name}) for d in nested])
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def layout(root: str | None = None) -> list:
|
|
337
|
+
"""What is wrong with the tree itself, which no loaded object can show:
|
|
338
|
+
a folder without its entry, an entry whose key is not its folder's name,
|
|
339
|
+
a dataset that names another benchmark than the folder it sits in.
|
|
340
|
+
Empty means the folders say what they hold."""
|
|
341
|
+
return _layout(storage.folders(BENCHMARKS, ROOT, root))
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _layout(selected: dict) -> list:
|
|
345
|
+
bad = []
|
|
346
|
+
for name, folder in selected.items():
|
|
347
|
+
where = f"{BENCHMARKS}/{name}/"
|
|
348
|
+
entry = {}
|
|
349
|
+
if not os.path.isfile(os.path.join(folder, ENTRY)):
|
|
350
|
+
bad.append(f"{where} has no {ENTRY}")
|
|
351
|
+
else:
|
|
352
|
+
try:
|
|
353
|
+
entry = _read(os.path.join(folder, ENTRY))
|
|
354
|
+
_entry(os.path.join(folder, ENTRY), name)
|
|
355
|
+
except (OSError, ValueError, TypeError, AttributeError) as e:
|
|
356
|
+
bad.append(f"{where}{ENTRY}: {e}")
|
|
357
|
+
entry = entry if isinstance(entry, dict) else {}
|
|
358
|
+
if entry and entry.get("key") != name:
|
|
359
|
+
bad.append(f"{where}{ENTRY}: key {entry.get('key')!r} is not its folder's name")
|
|
360
|
+
if "standard" in entry.get("harness", []) and not os.path.isfile(os.path.join(folder, "standard.json")):
|
|
361
|
+
bad.append(f"{where} names the Nullius standard but has no standard.json")
|
|
362
|
+
for d in entry.get("datasets", []):
|
|
363
|
+
if not isinstance(d, dict):
|
|
364
|
+
bad.append(f"{where}{ENTRY}: a dataset is a JSON object")
|
|
365
|
+
elif d.get("benchmark", name) != name:
|
|
366
|
+
bad.append(f"{where}{ENTRY}: dataset {d.get('key')!r} names benchmark "
|
|
367
|
+
f"{d['benchmark']!r}, and sits in {name!r}")
|
|
368
|
+
return bad
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def validate(cat: Catalog) -> list:
|
|
372
|
+
"""Everything that would make a tab lie. Empty means consistent.
|
|
373
|
+
|
|
374
|
+
The selected folders are checked too (`layout`): a folder named after
|
|
375
|
+
its key, whether from the bundle, the user catalogue or an explicit root."""
|
|
376
|
+
bad = []
|
|
377
|
+
if cat._folders:
|
|
378
|
+
bad.extend(_layout(cat._folders))
|
|
379
|
+
elif os.path.isdir(os.path.join(cat.path, BENCHMARKS)):
|
|
380
|
+
bad.extend(layout(cat.path))
|
|
381
|
+
keys = [b.key for b in cat.benchmarks]
|
|
382
|
+
if len(set(keys)) != len(keys):
|
|
383
|
+
bad.append("duplicate benchmark keys")
|
|
384
|
+
every = [d.key for d in cat.datasets]
|
|
385
|
+
for k in sorted({k for k in every if every.count(k) > 1}):
|
|
386
|
+
bad.append(f"dataset key {k!r} used twice: a dataset key is unique across the catalogue")
|
|
387
|
+
dkeys = {d.key for d in cat.datasets}
|
|
388
|
+
# a report block's sources name the job's dataset: one name, one way to read it
|
|
389
|
+
claimed = {}
|
|
390
|
+
for b in cat.benchmarks:
|
|
391
|
+
for s in (b.report or {}).get("sources") or []:
|
|
392
|
+
if s.strip().lower() in claimed:
|
|
393
|
+
bad.append(f"{b.key}: report source {s!r} is {claimed[s.strip().lower()]}'s too")
|
|
394
|
+
claimed.setdefault(s.strip().lower(), b.key)
|
|
395
|
+
for b in cat.benchmarks:
|
|
396
|
+
if b.origin not in ORIGINS:
|
|
397
|
+
bad.append(f"{b.key}: origin is official or community")
|
|
398
|
+
if b.task_count is not None:
|
|
399
|
+
if type(b.task_count) is not int or b.task_count < 1:
|
|
400
|
+
bad.append(f"{b.key}: task_count must be a positive integer")
|
|
401
|
+
elif any(d.tasks > b.task_count for d in cat.datasets_of(b)):
|
|
402
|
+
bad.append(f"{b.key}: task_count cannot be smaller than a dataset")
|
|
403
|
+
if b.published and not (isinstance(b.published, dict) and isinstance(b.published.get("agent"), str) and b.published["agent"]
|
|
404
|
+
and isinstance(b.published.get("says"), str) and b.published["says"]):
|
|
405
|
+
bad.append(f"{b.key}: published is {{agent: a Harbor agent's name, says: why it is the published one}}")
|
|
406
|
+
if not b.category:
|
|
407
|
+
bad.append(f"{b.key}: no category")
|
|
408
|
+
for h in b.harness:
|
|
409
|
+
if h not in dict(HARNESSES):
|
|
410
|
+
bad.append(f"{b.key}: harness {h!r} is not one of {[k for k, _ in HARNESSES]}")
|
|
411
|
+
for d in b.datasets:
|
|
412
|
+
if d not in dkeys:
|
|
413
|
+
bad.append(f"{b.key}: dataset {d!r} is not in the catalogue")
|
|
414
|
+
for d in cat.datasets:
|
|
415
|
+
# A dataset is shown on its benchmark's page and
|
|
416
|
+
# nowhere else, so one its benchmark does not list would never be seen.
|
|
417
|
+
owner = cat.benchmark(d.benchmark) if d.benchmark else None
|
|
418
|
+
if not d.benchmark:
|
|
419
|
+
bad.append(f"dataset {d.key}: names no benchmark")
|
|
420
|
+
elif owner is None:
|
|
421
|
+
bad.append(f"dataset {d.key}: benchmark {d.benchmark!r} is not in the catalogue")
|
|
422
|
+
elif d.key not in owner.datasets:
|
|
423
|
+
bad.append(f"dataset {d.key}: benchmark {d.benchmark!r} does not list it")
|
|
424
|
+
if not d.source:
|
|
425
|
+
bad.append(f"dataset {d.key}: no source")
|
|
426
|
+
return bad
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""A benchmark's citation, read: its ``citation`` field is BibTeX, as the
|
|
2
|
+
benchmark's authors publish it; this turns each entry into the parts a page
|
|
3
|
+
shows - the title, the people, the venue and year, the link - beside the
|
|
4
|
+
entry itself, which is what a reader copies. Nothing is fetched and nothing
|
|
5
|
+
is rewritten: a field the entry lacks is left empty.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
_ACCENTS = {'"': {"o": "ö", "a": "ä", "u": "ü", "e": "ë", "i": "ï", "O": "Ö", "A": "Ä", "U": "Ü"},
|
|
12
|
+
"'": {"e": "é", "a": "á", "o": "ó", "i": "í", "u": "ú", "E": "É"},
|
|
13
|
+
"`": {"e": "è", "a": "à"}, "^": {"o": "ô", "e": "ê"}, "~": {"n": "ñ"}, "c": {"c": "ç"}}
|
|
14
|
+
_SHORT = [(re.compile(r"International Conference on Learning Representations"), "ICLR"),
|
|
15
|
+
(re.compile(r"Empirical Methods in Natural Language Processing"), "EMNLP"),
|
|
16
|
+
(re.compile(r"Neural Information Processing Systems"), "NeurIPS"),
|
|
17
|
+
(re.compile(r"International Conference on Machine Learning"), "ICML")]
|
|
18
|
+
KINDS = {"conference": "Conference paper", "preprint": "Preprint", "workshop": "Workshop paper"}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def delatex(s: str) -> str:
|
|
22
|
+
"""BibTeX's markup as text: the τ of τ-bench, dashes, accents; braces gone."""
|
|
23
|
+
s = s.replace("$\\tau^2$", "τ²").replace("$\\tau^3$", "τ³").replace("$\\tau$", "τ")
|
|
24
|
+
s = re.sub(r"\{?\\textemdash\}?", "—", s)
|
|
25
|
+
s = re.sub(r"\{?\\textendash\}?", "–", s)
|
|
26
|
+
s = re.sub(r"\{?\\([\"'`^~c])\{?([A-Za-z])\}?\}?", lambda m: _ACCENTS.get(m.group(1), {}).get(m.group(2), m.group(2)), s)
|
|
27
|
+
return re.sub(r"\s+", " ", s.replace("{", "").replace("}", "")).strip()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _value(s: str, j: int) -> tuple:
|
|
31
|
+
if j < len(s) and s[j] == "{":
|
|
32
|
+
d, k = 1, j + 1
|
|
33
|
+
while k < len(s) and d:
|
|
34
|
+
d += 1 if s[k] == "{" else -1 if s[k] == "}" else 0
|
|
35
|
+
k += 1
|
|
36
|
+
return s[j + 1:k - 1], k
|
|
37
|
+
if j < len(s) and s[j] == '"':
|
|
38
|
+
d, k = 0, j + 1
|
|
39
|
+
while k < len(s) and not (s[k] == '"' and d == 0):
|
|
40
|
+
d += 1 if s[k] == "{" else -1 if s[k] == "}" else 0
|
|
41
|
+
k += 1
|
|
42
|
+
return s[j + 1:k], k + 1
|
|
43
|
+
k = j
|
|
44
|
+
while k < len(s) and s[k] != ",":
|
|
45
|
+
k += 1
|
|
46
|
+
return s[j:k].strip(), k
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _name(n: str) -> str:
|
|
50
|
+
n = delatex(n)
|
|
51
|
+
if "," in n:
|
|
52
|
+
last, first = (x.strip() for x in n.split(",", 1))
|
|
53
|
+
return f"{first} {last}".strip()
|
|
54
|
+
return n
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _paper(kind: str, key: str, f: dict, raw: str) -> dict:
|
|
58
|
+
authors = [_name(a) for a in re.split(r"\s+and\s+", f.get("author", "")) if a.strip()]
|
|
59
|
+
journal = delatex(f.get("journal", ""))
|
|
60
|
+
m = re.search(r"arXiv:(\d{4}\.\d{4,5})", journal)
|
|
61
|
+
arxiv = f.get("eprint", "") or (m.group(1) if m else "")
|
|
62
|
+
venue = delatex(f["booktitle"]) if f.get("booktitle") else journal
|
|
63
|
+
if not venue and arxiv:
|
|
64
|
+
venue = "arXiv preprint arXiv:" + arxiv
|
|
65
|
+
short = venue
|
|
66
|
+
for pattern, name in _SHORT:
|
|
67
|
+
if pattern.search(venue) and "Workshop" not in venue:
|
|
68
|
+
short = name
|
|
69
|
+
if venue.startswith("arXiv"):
|
|
70
|
+
short = "arXiv"
|
|
71
|
+
if "Workshop" in venue:
|
|
72
|
+
short = re.sub(r" on .*", "", venue)
|
|
73
|
+
year = delatex(f.get("year", ""))
|
|
74
|
+
kind_of = "workshop" if "Workshop" in venue else "preprint" if short == "arXiv" else "conference"
|
|
75
|
+
shown = len(authors) <= 4
|
|
76
|
+
return {"key": key, "type": kind, "bibtex": raw, "title": delatex(f.get("title", "")), "authors": authors,
|
|
77
|
+
"authors_short": (", ".join(authors[:-1]) + (" and " if len(authors) > 1 else "") + (authors[-1] if authors else "")) if shown
|
|
78
|
+
else f"{', '.join(authors[:3])} and {len(authors) - 3} others",
|
|
79
|
+
"year": year, "venue": venue, "venue_short": "arXiv" if short == "arXiv" else re.sub(r" \d{4}.*", "", short),
|
|
80
|
+
"kind": kind_of, "kind_label": KINDS[kind_of], "arxiv": arxiv,
|
|
81
|
+
# a link is an http(s) one or none: an entry is text anyone may have written, and a page links it
|
|
82
|
+
"url": (f.get("url", "").strip() if re.match(r"^https?://", f.get("url", "").strip(), re.I) else "")
|
|
83
|
+
or (f"https://arxiv.org/abs/{arxiv}" if re.fullmatch(r"\d{4}\.\d{4,5}(v\d+)?", arxiv) else "")}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def parse(text: str) -> list:
|
|
87
|
+
"""Every entry of a BibTeX text, in order: a dict of its parts (see
|
|
88
|
+
:func:`_paper`) and ``bibtex``, the entry as written."""
|
|
89
|
+
text = (text or "").replace("\r\n", "\n")
|
|
90
|
+
out, i = [], 0
|
|
91
|
+
while True:
|
|
92
|
+
i = text.find("@", i)
|
|
93
|
+
if i < 0:
|
|
94
|
+
break
|
|
95
|
+
m = re.match(r"@(\w+)\s*\{", text[i:])
|
|
96
|
+
if not m:
|
|
97
|
+
i += 1
|
|
98
|
+
continue
|
|
99
|
+
j = start = i + m.end()
|
|
100
|
+
d = 1
|
|
101
|
+
while j < len(text) and d:
|
|
102
|
+
d += 1 if text[j] == "{" else -1 if text[j] == "}" else 0
|
|
103
|
+
j += 1
|
|
104
|
+
body, raw = text[start:j - 1], text[i:j].strip()
|
|
105
|
+
c = body.find(",")
|
|
106
|
+
key, fields, k = body[:c].strip(), {}, c + 1
|
|
107
|
+
while k < len(body):
|
|
108
|
+
nm = re.match(r"[\s,]*([A-Za-z][\w-]*)\s*=\s*", body[k:])
|
|
109
|
+
if not nm:
|
|
110
|
+
break
|
|
111
|
+
k += nm.end()
|
|
112
|
+
value, k = _value(body, k)
|
|
113
|
+
fields[nm.group(1).lower()] = value
|
|
114
|
+
out.append(_paper(m.group(1).lower(), key, fields, raw))
|
|
115
|
+
i = j
|
|
116
|
+
return out
|