nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/gitrefs.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""What a git repository offers - its default branch, its branches and its
|
|
2
|
+
tags, each at its commit - read with ``git ls-remote``.
|
|
3
|
+
|
|
4
|
+
One lister for the picker that names a version of a repository: the Harbor a
|
|
5
|
+
recipe pins (``harbor/source.py``). It asks the repository itself, as this machine's git
|
|
6
|
+
would: with the machine's own credentials - an ssh key, a credential helper -
|
|
7
|
+
and never a token of its own, so a private repository lists when this
|
|
8
|
+
machine can clone it, and there is no API rate limit to meet. Nothing is
|
|
9
|
+
prompted for (``GIT_TERMINAL_PROMPT=0``, ssh in batch mode) and a repository
|
|
10
|
+
that does not answer within the timeout is said to, not waited on.
|
|
11
|
+
|
|
12
|
+
``ls-remote`` carries no dates, so the order is the names': tags by version,
|
|
13
|
+
newest first, an annotated tag read through to the commit it points at
|
|
14
|
+
(its ``^{}`` line); branches with the default first, then by name.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import subprocess
|
|
21
|
+
from typing import Callable, List, Optional
|
|
22
|
+
|
|
23
|
+
GIT = "git" # the program; a test or a probe points it elsewhere
|
|
24
|
+
TIMEOUT = 30.0
|
|
25
|
+
LISTED = 50 # the most of each kind a picker is given; the totals say how many there are
|
|
26
|
+
COMMIT = re.compile(r"^[0-9a-f]{40}$")
|
|
27
|
+
_SLUG = re.compile(r"^([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)$")
|
|
28
|
+
_URL = re.compile(r"^(?:https?|ssh|git|file)://\S+$")
|
|
29
|
+
_SCP = re.compile(r"^[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+:\S+$")
|
|
30
|
+
_GITHUB = re.compile(r"^(?:https://github\.com/|ssh://git@github\.com/|git@github\.com:)([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?)(?:\.git)?/?$")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def url_of(text: str) -> Optional[str]:
|
|
34
|
+
"""The URL git is given for what a person typed: ``owner/name`` is that
|
|
35
|
+
repository on GitHub over https; an https, ssh, git or file URL, or an
|
|
36
|
+
scp-like ``user@host:path``, is taken as it is. Anything else - an
|
|
37
|
+
option, a word with spaces - is None."""
|
|
38
|
+
text = (text or "").strip()
|
|
39
|
+
if not text or text.startswith("-") or any(c.isspace() or ord(c) < 32 for c in text):
|
|
40
|
+
return None
|
|
41
|
+
m = _SLUG.match(text)
|
|
42
|
+
if m and not text.startswith("."):
|
|
43
|
+
name = m.group(2)[:-4] if m.group(2).endswith(".git") else m.group(2)
|
|
44
|
+
return f"https://github.com/{m.group(1)}/{name}.git"
|
|
45
|
+
return text if _URL.match(text) or _SCP.match(text) else None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def label_of(url: str) -> str:
|
|
49
|
+
"""How a repository is named on a page: ``owner/name`` on GitHub, else its URL."""
|
|
50
|
+
m = _GITHUB.match(url or "")
|
|
51
|
+
return m.group(1) if m else url
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def env() -> dict:
|
|
55
|
+
"""The environment git runs in: this process's, with no prompt possible."""
|
|
56
|
+
out = dict(os.environ, GIT_TERMINAL_PROMPT="0")
|
|
57
|
+
if not out.get("GIT_SSH_COMMAND") and not out.get("GIT_SSH"):
|
|
58
|
+
out["GIT_SSH_COMMAND"] = "ssh -o BatchMode=yes"
|
|
59
|
+
return out
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def run_git(args: List[str], timeout: float = TIMEOUT) -> tuple:
|
|
63
|
+
"""``git <args>``: (returncode, stdout, stderr); a missing git and a
|
|
64
|
+
timeout are return codes of their own."""
|
|
65
|
+
try:
|
|
66
|
+
p = subprocess.run([GIT, *args], capture_output=True, text=True, timeout=timeout, env=env(), stdin=subprocess.DEVNULL)
|
|
67
|
+
except OSError as e:
|
|
68
|
+
return 127, "", f"{GIT}: {e.strerror or e}"
|
|
69
|
+
except subprocess.TimeoutExpired:
|
|
70
|
+
return 124, "", f"no answer within {int(timeout)} s"
|
|
71
|
+
return p.returncode, p.stdout, p.stderr
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def version_key(name: str):
|
|
75
|
+
"""A tag or a version name as something to sort by: its numbers as
|
|
76
|
+
numbers, from the first digit on (``v1.10`` after ``v1.9``, ``harbor-0.3``
|
|
77
|
+
beside ``0.3``), a pre-release before its release; a name with no digit
|
|
78
|
+
sorts below every one that has one."""
|
|
79
|
+
m = re.search(r"\d", name)
|
|
80
|
+
if not m:
|
|
81
|
+
return (0, [], name)
|
|
82
|
+
parts = [(1, int(p)) if p.isdigit() else (0, p.lower()) for p in re.findall(r"\d+|[A-Za-z]+", name[m.start():])]
|
|
83
|
+
return (1, parts + [(0.5, "")], name)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def parse(text: str) -> dict:
|
|
87
|
+
"""``git ls-remote --symref`` output as the default branch, the branches
|
|
88
|
+
and the tags, each ``{name, commit}``, in the order a picker shows them."""
|
|
89
|
+
default, branches, tags, peeled = "", {}, {}, {}
|
|
90
|
+
for line in text.splitlines():
|
|
91
|
+
left, _, ref = line.partition("\t")
|
|
92
|
+
if left.startswith("ref: ") and ref.strip() == "HEAD":
|
|
93
|
+
target = left[5:].strip()
|
|
94
|
+
if target.startswith("refs/heads/"):
|
|
95
|
+
default = target[len("refs/heads/"):]
|
|
96
|
+
continue
|
|
97
|
+
sha, ref = left.strip(), ref.strip()
|
|
98
|
+
if not COMMIT.match(sha):
|
|
99
|
+
continue
|
|
100
|
+
if ref.startswith("refs/heads/"):
|
|
101
|
+
branches[ref[len("refs/heads/"):]] = sha
|
|
102
|
+
elif ref.startswith("refs/tags/"):
|
|
103
|
+
name = ref[len("refs/tags/"):]
|
|
104
|
+
if name.endswith("^{}"):
|
|
105
|
+
peeled[name[:-3]] = sha # an annotated tag: the commit it names
|
|
106
|
+
else:
|
|
107
|
+
tags[name] = sha
|
|
108
|
+
tags.update({k: v for k, v in peeled.items() if k in tags})
|
|
109
|
+
order = sorted(branches, key=lambda n: (n != default, n))
|
|
110
|
+
return {"default_branch": default,
|
|
111
|
+
"branches": [{"name": n, "commit": branches[n]} for n in order],
|
|
112
|
+
"tags": [{"name": n, "commit": tags[n]} for n in sorted(tags, key=version_key, reverse=True)]}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def ls_remote(url: str, run: Optional[Callable] = None, listed: int = LISTED) -> dict:
|
|
116
|
+
"""What ``url`` offers: ``{default_branch, branches, branches_total,
|
|
117
|
+
tags, tags_total, error}``, at most ``listed`` of each kind. On any
|
|
118
|
+
failure the lists are empty and ``error`` is git's own last word."""
|
|
119
|
+
out = {"default_branch": "", "branches": [], "branches_total": 0, "tags": [], "tags_total": 0, "error": ""}
|
|
120
|
+
if url_of(url) is None:
|
|
121
|
+
return dict(out, error="not a repository URL")
|
|
122
|
+
rc, text, err = (run or run_git)(["ls-remote", "--symref", url, "HEAD", "refs/heads/*", "refs/tags/*"])
|
|
123
|
+
if rc != 0:
|
|
124
|
+
said = [l.strip() for l in (err or text or "").splitlines() if l.strip()]
|
|
125
|
+
return dict(out, error=(said[-1] if said else f"git exited {rc}").removeprefix("fatal: "))
|
|
126
|
+
got = parse(text)
|
|
127
|
+
return dict(out, default_branch=got["default_branch"], branches=got["branches"][:listed], branches_total=len(got["branches"]),
|
|
128
|
+
tags=got["tags"][:listed], tags_total=len(got["tags"]))
|
|
File without changes
|
nullius/harbor/feed.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The live feed for any Harbor: ``events.jsonl`` written from Harbor's own trial hooks.
|
|
2
|
+
|
|
3
|
+
A run's ``harbor`` script (``source.wrapper``) runs this file, sent beside it
|
|
4
|
+
as ``harbor-feed.py``, with the pinned
|
|
5
|
+
Harbor's Python instead of the ``harbor`` command: it registers the hooks
|
|
6
|
+
every Harbor ``Job`` offers (``on_trial_started`` … ``on_trial_ended``), then
|
|
7
|
+
hands its arguments to Harbor's own command line, unchanged. Each job then
|
|
8
|
+
writes the feed ``nullius.runs.live`` folds into the job page's Live view -
|
|
9
|
+
the same lines, in the same schema - whatever Harbor it is: a PyPI release or
|
|
10
|
+
a repository at a commit.
|
|
11
|
+
|
|
12
|
+
What a Harbor without its own feed cannot give is left out, never made up:
|
|
13
|
+
no agent-setup phase (its hooks go from the environment to the agent), no
|
|
14
|
+
step or token counts while the agent runs; the trial's own result fills
|
|
15
|
+
the totals at its end. A phase's limit is the trial's own (its
|
|
16
|
+
``*_timeout_sec`` properties, multipliers applied), read from the trial
|
|
17
|
+
object the feed keeps by name since the hooks do not carry it. A Harbor that writes the feed itself
|
|
18
|
+
(its ``Job`` has a live feed of its own) is left alone, so a job never has two
|
|
19
|
+
writers.
|
|
20
|
+
|
|
21
|
+
Nothing here imports Harbor until :func:`main` runs inside Harbor's
|
|
22
|
+
environment, so this module loads in this application like any other; a
|
|
23
|
+
failure of the feed is said once on stderr and never stops the run.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
import os
|
|
29
|
+
import sys
|
|
30
|
+
import threading
|
|
31
|
+
from datetime import datetime, timezone
|
|
32
|
+
|
|
33
|
+
FEED = "events.jsonl"
|
|
34
|
+
TRIALS: dict = {} # trial name -> the Harbor trial object, kept as each is made, for its limits
|
|
35
|
+
LIMITS = {"env_setup": "environment_build_timeout_sec", "agent_run": "agent_timeout_sec", "verify": "verifier_timeout_sec"}
|
|
36
|
+
# an exception type naming the infrastructure, not the agent: the board's infra_error - the provider's
|
|
37
|
+
# (Api*, a refused key, a model name it does not serve) as the classifier counts them (nullius.runs.classify)
|
|
38
|
+
INFRA = ("Api", "Authentication", "ModelNotFound", "RateLimit", "Setup", "Environment", "Docker", "Build", "Connection", "Network")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Writer:
|
|
42
|
+
"""One job's feed: a line per event, ``seq`` strictly increasing, each
|
|
43
|
+
line flushed as written so a reader folds it as it grows."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, path: str, job: str):
|
|
46
|
+
self.path, self.job, self.seq = path, job, 0
|
|
47
|
+
self.lock = threading.Lock()
|
|
48
|
+
|
|
49
|
+
def emit(self, event: str, trial=None, **data) -> None:
|
|
50
|
+
with self.lock:
|
|
51
|
+
self.seq += 1
|
|
52
|
+
line = {"seq": self.seq, "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
53
|
+
"job": self.job, "event": event, "trial": trial}
|
|
54
|
+
line.update({k: v for k, v in data.items() if v is not None})
|
|
55
|
+
with open(self.path, "a", encoding="utf-8") as fh:
|
|
56
|
+
fh.write(json.dumps(line, default=str) + "\n")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def outcome_of(result) -> tuple:
|
|
60
|
+
"""``(outcome, reward, evidence)`` of a finished trial, in the feed's words:
|
|
61
|
+
the reward when the verifier gave one, else what the exception says."""
|
|
62
|
+
rewards = getattr(getattr(result, "verifier_result", None), "rewards", None)
|
|
63
|
+
reward = rewards.get("reward") if isinstance(rewards, dict) else None
|
|
64
|
+
info = getattr(result, "exception_info", None)
|
|
65
|
+
kind = (getattr(info, "exception_type", "") or "") if info is not None else ""
|
|
66
|
+
if "Cancel" in kind:
|
|
67
|
+
return "cancelled", reward, kind
|
|
68
|
+
if reward is not None:
|
|
69
|
+
return ("resolved" if reward >= 1 else "unresolved"), reward, f"reward {reward}"
|
|
70
|
+
if "Timeout" in kind:
|
|
71
|
+
return "timeout", None, kind
|
|
72
|
+
if kind:
|
|
73
|
+
return ("infra_error" if any(w in kind for w in INFRA) else "agent_error"), None, kind
|
|
74
|
+
return "unresolved", None, "no reward"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _seconds(timing):
|
|
78
|
+
start, end = getattr(timing, "started_at", None), getattr(timing, "finished_at", None)
|
|
79
|
+
return (end - start).total_seconds() if start is not None and end is not None else None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _agent_name(agent) -> str:
|
|
83
|
+
return getattr(agent, "name", None) or getattr(agent, "import_path", None) or "agent"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _task_name(config) -> str:
|
|
87
|
+
task = getattr(config, "task", None)
|
|
88
|
+
try:
|
|
89
|
+
return task.get_task_id().get_name()
|
|
90
|
+
except Exception:
|
|
91
|
+
return str(getattr(task, "path", "") or "")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _said_once(e) -> None:
|
|
95
|
+
if not _SAID:
|
|
96
|
+
_SAID.append(e)
|
|
97
|
+
print(f"nullius: the live feed stopped: {e!r}; the run goes on", file=sys.stderr)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
_SAID: list = []
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _safe(hook):
|
|
104
|
+
"""A hook whose failure is said once and never reaches Harbor's trial."""
|
|
105
|
+
async def run(event):
|
|
106
|
+
try:
|
|
107
|
+
await hook(event)
|
|
108
|
+
except Exception as e:
|
|
109
|
+
_said_once(e)
|
|
110
|
+
return run
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def attach(job, writer: Writer) -> None:
|
|
114
|
+
"""Register the feed's hooks on a Harbor ``Job`` and say the job started:
|
|
115
|
+
its trials, each queued in the order Harbor will run them."""
|
|
116
|
+
def who(event) -> dict:
|
|
117
|
+
agent = getattr(event.config, "agent", None)
|
|
118
|
+
return {"task": event.task_name, "agent": _agent_name(agent), "model": getattr(agent, "model_name", None)}
|
|
119
|
+
|
|
120
|
+
def limit(trial_name, name):
|
|
121
|
+
try:
|
|
122
|
+
v = getattr(TRIALS.get(trial_name), LIMITS[name], None)
|
|
123
|
+
return float(v) if isinstance(v, (int, float)) and v > 0 else None
|
|
124
|
+
except Exception:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
def phase(name):
|
|
128
|
+
async def hook(event):
|
|
129
|
+
writer.emit("trial.phase", event.trial_name, phase=name, timeout_sec=limit(event.trial_name, name), **who(event))
|
|
130
|
+
return hook
|
|
131
|
+
|
|
132
|
+
async def ended(event):
|
|
133
|
+
result = event.result
|
|
134
|
+
outcome, reward, evidence = outcome_of(result)
|
|
135
|
+
usage = getattr(result, "agent_result", None)
|
|
136
|
+
writer.emit("trial.end", event.trial_name, outcome=outcome, reward=reward, evidence=evidence,
|
|
137
|
+
exception_type=getattr(getattr(result, "exception_info", None), "exception_type", None),
|
|
138
|
+
durations={"env_setup": _seconds(result.environment_setup), "agent_setup": _seconds(result.agent_setup),
|
|
139
|
+
"agent_run": _seconds(result.agent_execution), "verify": _seconds(result.verifier)},
|
|
140
|
+
n_input_tokens=getattr(usage, "n_input_tokens", None), n_cache_tokens=getattr(usage, "n_cache_tokens", None),
|
|
141
|
+
n_output_tokens=getattr(usage, "n_output_tokens", None), cost_usd=getattr(usage, "cost_usd", None),
|
|
142
|
+
**who(event))
|
|
143
|
+
|
|
144
|
+
async def cancelled(event):
|
|
145
|
+
writer.emit("trial.progress", event.trial_name, activity="cancellation requested", **who(event))
|
|
146
|
+
|
|
147
|
+
job.on_environment_started(_safe(phase("env_setup")))
|
|
148
|
+
job.on_agent_started(_safe(phase("agent_run")))
|
|
149
|
+
job.on_verification_started(_safe(phase("verify")))
|
|
150
|
+
job.on_trial_ended(_safe(ended))
|
|
151
|
+
job.on_trial_cancelled(_safe(cancelled))
|
|
152
|
+
configs = list(getattr(job, "_remaining_trial_configs", None) or getattr(job, "_trial_configs", None) or [])
|
|
153
|
+
writer.emit("job.start", n_total=len(getattr(job, "_trial_configs", None) or configs),
|
|
154
|
+
n_concurrent=getattr(job.config, "n_concurrent_trials", None),
|
|
155
|
+
agents=sorted({_agent_name(c.agent) for c in configs}), tasks=sorted({_task_name(c) for c in configs}))
|
|
156
|
+
for position, c in enumerate(configs):
|
|
157
|
+
writer.emit("trial.queued", c.trial_name, task=_task_name(c), agent=_agent_name(c.agent),
|
|
158
|
+
model=getattr(c.agent, "model_name", None), position=position)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def keep_trials(Trial) -> None:
|
|
162
|
+
"""Wrap ``Trial.__init__`` so every trial is kept by its name: the hooks
|
|
163
|
+
name a trial, and its limits are on the object."""
|
|
164
|
+
made = Trial.__init__
|
|
165
|
+
|
|
166
|
+
def init(self, *args, **kwargs):
|
|
167
|
+
made(self, *args, **kwargs)
|
|
168
|
+
try:
|
|
169
|
+
TRIALS[self.config.trial_name] = self
|
|
170
|
+
except Exception as e:
|
|
171
|
+
_said_once(e)
|
|
172
|
+
|
|
173
|
+
Trial.__init__ = init
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def install(Job, Trial=None) -> bool:
|
|
177
|
+
"""Wrap ``Job.run`` so each job writes the feed (and ``Trial`` so its
|
|
178
|
+
phases carry their limits). False, and nothing changed, when this Harbor
|
|
179
|
+
writes its own."""
|
|
180
|
+
if hasattr(Job, "_init_live_feed"):
|
|
181
|
+
return False
|
|
182
|
+
if Trial is not None:
|
|
183
|
+
keep_trials(Trial)
|
|
184
|
+
run = Job.run
|
|
185
|
+
warn = _said_once
|
|
186
|
+
|
|
187
|
+
async def run_with_feed(self, *args, **kwargs):
|
|
188
|
+
writer = None
|
|
189
|
+
try:
|
|
190
|
+
os.makedirs(str(self.job_dir), exist_ok=True)
|
|
191
|
+
writer = Writer(os.path.join(str(self.job_dir), FEED), self.config.job_name)
|
|
192
|
+
attach(self, writer)
|
|
193
|
+
except Exception as e: # the feed is a view: never the run's undoing
|
|
194
|
+
warn(e)
|
|
195
|
+
result = await run(self, *args, **kwargs)
|
|
196
|
+
if writer is not None:
|
|
197
|
+
try:
|
|
198
|
+
stats = getattr(result, "stats", None)
|
|
199
|
+
writer.emit("job.end", n_completed=getattr(stats, "n_completed_trials", None),
|
|
200
|
+
n_errored=getattr(stats, "n_errored_trials", None), n_cancelled=getattr(stats, "n_cancelled_trials", None))
|
|
201
|
+
except Exception as e:
|
|
202
|
+
warn(e)
|
|
203
|
+
return result
|
|
204
|
+
|
|
205
|
+
Job.run = run_with_feed
|
|
206
|
+
return True
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def main() -> None:
|
|
210
|
+
"""``python feed.py <harbor arguments>``: Harbor's command line, with the feed."""
|
|
211
|
+
try:
|
|
212
|
+
from harbor.job import Job
|
|
213
|
+
try:
|
|
214
|
+
from harbor.trial.trial import Trial
|
|
215
|
+
except Exception:
|
|
216
|
+
Trial = None
|
|
217
|
+
install(Job, Trial)
|
|
218
|
+
except Exception as e:
|
|
219
|
+
print(f"nullius: no live feed with this Harbor: {e!r}; the run goes on", file=sys.stderr)
|
|
220
|
+
from importlib.metadata import entry_points
|
|
221
|
+
# the `harbor` command as this Harbor declares it, whatever module it lives in
|
|
222
|
+
command = next(iter(entry_points(group="console_scripts", name="harbor"))).load()
|
|
223
|
+
sys.argv = ["harbor"] + sys.argv[1:]
|
|
224
|
+
sys.exit(command())
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
if __name__ == "__main__":
|
|
228
|
+
main()
|