nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/harbor/source.py
ADDED
|
@@ -0,0 +1,582 @@
|
|
|
1
|
+
"""The Harbor a recipe pins: a release from PyPI, or a repository at a commit
|
|
2
|
+
- built here, kept here, scanned for its agents.
|
|
3
|
+
|
|
4
|
+
A Harbor recipe names its Harbor (the recipe's ``harbor`` field, checked by
|
|
5
|
+
:func:`pin_of`):
|
|
6
|
+
|
|
7
|
+
- ``{"kind": "release", "version": "0.24.0", "extras": []}`` - the official
|
|
8
|
+
package, ``harbor`` on PyPI, at that version; nothing is built;
|
|
9
|
+
- ``{"kind": "repo", "repo": "owner/name" or a git URL, "revisionKind":
|
|
10
|
+
"branch" | "tag" | "commit", "revision": ..., "commit": <40 hex>,
|
|
11
|
+
"extras": []}`` - a repository, a private fork included, at the commit the
|
|
12
|
+
picker resolved the branch or tag to: the commit is the pin.
|
|
13
|
+
|
|
14
|
+
A repository pin is **built on this machine**, once, with this machine's own
|
|
15
|
+
git credentials (an ssh key, a credential helper): the commit is fetched,
|
|
16
|
+
Harbor's wheel built with ``uv build``, and so is a wheel for each dependency
|
|
17
|
+
it declares that PyPI does not serve - one its ``[tool.uv.sources]`` names (a
|
|
18
|
+
git repository, a path, a workspace member, a URL) or a direct-URL
|
|
19
|
+
requirement - since a fork often pins a private package. The wheels and a
|
|
20
|
+
record (``pin.json``) go into a directory of their own under the pins root
|
|
21
|
+
(:func:`root`: ``HARBOR_PINS``, else ``harbor-pins/`` beside the runs), built
|
|
22
|
+
aside and renamed into place, so a pin is ready or absent. A host is sent
|
|
23
|
+
that directory and resolves everything else from PyPI: only this machine
|
|
24
|
+
needs access to a private fork.
|
|
25
|
+
|
|
26
|
+
Harbor runs from the pin through ``uvx`` (:func:`wrapper`), its command
|
|
27
|
+
line started by ``nullius.harbor.feed`` so every job writes the live feed
|
|
28
|
+
the job page folds: ``harbor==X`` for
|
|
29
|
+
a release, the built wheel with ``--find-links`` beside it for a repository
|
|
30
|
+
(and an override for a dependency named by URL, which a find-links
|
|
31
|
+
directory cannot satisfy). The scan (:func:`scan`) asks that same command
|
|
32
|
+
``harbor agents list --json`` - Harbor's own registry - and keeps the answer
|
|
33
|
+
beside the pin (``agents.json``), so it is asked once per pin. Building and
|
|
34
|
+
scanning run on an explicit action - the builder's scan, or a Run - never on
|
|
35
|
+
a page load; two asks for one pin build it once.
|
|
36
|
+
|
|
37
|
+
Nothing here imports Flask; every command goes through a ``run`` seam and
|
|
38
|
+
PyPI through a ``fetch`` seam.
|
|
39
|
+
"""
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import glob
|
|
43
|
+
import json
|
|
44
|
+
import os
|
|
45
|
+
import re
|
|
46
|
+
import shlex
|
|
47
|
+
import shutil
|
|
48
|
+
import subprocess
|
|
49
|
+
import tempfile
|
|
50
|
+
import threading
|
|
51
|
+
import time
|
|
52
|
+
import tomllib
|
|
53
|
+
import urllib.parse
|
|
54
|
+
import zipfile
|
|
55
|
+
from datetime import datetime, timezone
|
|
56
|
+
from email.parser import Parser
|
|
57
|
+
from typing import Callable, Dict, List, Optional
|
|
58
|
+
from urllib import error, request
|
|
59
|
+
|
|
60
|
+
from nullius import gitrefs
|
|
61
|
+
|
|
62
|
+
PACKAGE = "harbor"
|
|
63
|
+
PYPI = "https://pypi.org/pypi"
|
|
64
|
+
RECORD = "pin.json"
|
|
65
|
+
AGENTS = "agents.json"
|
|
66
|
+
WHEELS = "wheels"
|
|
67
|
+
BUILD_TIMEOUT = 1800.0
|
|
68
|
+
SCAN_TIMEOUT = 1800.0
|
|
69
|
+
KINDS = ("release", "repo")
|
|
70
|
+
REVISION_KINDS = ("branch", "tag", "commit")
|
|
71
|
+
_VERSION = re.compile(r"^[0-9][0-9A-Za-z.+!-]{0,63}$")
|
|
72
|
+
_EXTRA = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
|
73
|
+
_REVISION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,199}$")
|
|
74
|
+
_WHEEL = re.compile(r"^[A-Za-z0-9_.+!-]+\.whl$")
|
|
75
|
+
_AGENT = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
|
|
76
|
+
_PRE = re.compile(r"(a|b|rc|dev)\d*$")
|
|
77
|
+
_UV_SAYS = re.compile(r"^\s*(Installed|Uninstalled|Downloading|Downloaded|Resolved|Prepared|Audited|Built|Building) ")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class BuildError(Exception):
|
|
81
|
+
"""A pin that could not be made ready here, and why."""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _norm(name: str) -> str:
|
|
85
|
+
return re.sub(r"[-_.]+", "-", name).lower()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _now() -> str:
|
|
89
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def pin_of(raw) -> tuple:
|
|
94
|
+
"""``(pin, problem)`` for a recipe's ``harbor`` field: the pin
|
|
95
|
+
normalised, or None with no problem when the recipe names none (a recipe
|
|
96
|
+
from before the field, or a release not chosen yet) - that recipe runs
|
|
97
|
+
the harbor on the target's PATH - or None and what is wrong."""
|
|
98
|
+
if not isinstance(raw, dict) or raw.get("kind") not in KINDS:
|
|
99
|
+
return None, None
|
|
100
|
+
extras = raw.get("extras") or []
|
|
101
|
+
if not isinstance(extras, list) or not all(isinstance(x, str) and _EXTRA.match(x) for x in extras):
|
|
102
|
+
return None, "the Harbor extras must be names of the package's optional dependencies"
|
|
103
|
+
extras = sorted(set(extras))
|
|
104
|
+
if raw["kind"] == "release":
|
|
105
|
+
version = str(raw.get("version") or "").strip()
|
|
106
|
+
if not version:
|
|
107
|
+
return None, None
|
|
108
|
+
if not _VERSION.match(version):
|
|
109
|
+
return None, f"{version!r} is not a Harbor release version"
|
|
110
|
+
return {"kind": "release", "version": version, "extras": extras}, None
|
|
111
|
+
url = gitrefs.url_of(str(raw.get("repo") or ""))
|
|
112
|
+
if not url:
|
|
113
|
+
return None, "the Harbor repository must be owner/name on GitHub, or a git URL"
|
|
114
|
+
commit = str(raw.get("commit") or "")
|
|
115
|
+
if not gitrefs.COMMIT.match(commit):
|
|
116
|
+
return None, f"choose a branch, a tag or a full commit of {gitrefs.label_of(url)}: the pin is a commit"
|
|
117
|
+
kind = raw.get("revisionKind") if raw.get("revisionKind") in REVISION_KINDS else "commit"
|
|
118
|
+
revision = str(raw.get("revision") or commit)
|
|
119
|
+
if not _REVISION.match(revision):
|
|
120
|
+
return None, f"{revision!r} is not a branch, a tag or a commit"
|
|
121
|
+
return {"kind": "repo", "repo": str(raw.get("repo")).strip(), "url": url, "revisionKind": kind, "revision": revision,
|
|
122
|
+
"commit": commit, "extras": extras}, None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def key_of(pin: dict) -> str:
|
|
126
|
+
"""The pin's directory name: ``release-<version>``, ``git-<commit>``."""
|
|
127
|
+
return f"release-{pin['version']}" if pin["kind"] == "release" else f"git-{pin['commit']}"
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def label(pin: dict) -> str:
|
|
131
|
+
"""The pin as a sentence fragment: ``harbor 0.24.0 from PyPI``,
|
|
132
|
+
``harbor from owner/name, branch main at 0123456789ab``."""
|
|
133
|
+
extras = f" [{','.join(pin['extras'])}]" if pin.get("extras") else ""
|
|
134
|
+
if pin["kind"] == "release":
|
|
135
|
+
return f"harbor {pin['version']}{extras} from PyPI"
|
|
136
|
+
at = pin["commit"][:12]
|
|
137
|
+
how = f"commit {at}" if pin["revisionKind"] == "commit" else f"{pin['revisionKind']} {pin['revision']} at {at}"
|
|
138
|
+
return f"harbor{extras} from {gitrefs.label_of(pin['url'])}, {how}"
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def root(runs: str, env=os.environ) -> str:
|
|
142
|
+
"""Where pins are built and kept on this machine: ``HARBOR_PINS``, else
|
|
143
|
+
``harbor-pins/`` beside the runs."""
|
|
144
|
+
return env.get("HARBOR_PINS") or os.path.join(os.path.dirname(os.path.abspath(runs)), "harbor-pins")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def pypi_fetch(url: str, timeout: float = 20.0) -> bytes:
|
|
149
|
+
req = request.Request(url, headers={"Accept": "application/json", "User-Agent": "nullius"})
|
|
150
|
+
with request.urlopen(req, timeout=timeout) as resp:
|
|
151
|
+
return resp.read()
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def releases(fetch: Optional[Callable] = None) -> dict:
|
|
155
|
+
"""The official Harbor's releases on PyPI, newest first: ``{versions:
|
|
156
|
+
[{version, date, requires_python}], error}``. Pre-releases and versions
|
|
157
|
+
with no wheel are left out; on any failure, no versions and why."""
|
|
158
|
+
try:
|
|
159
|
+
data = json.loads((fetch or pypi_fetch)(f"{PYPI}/{PACKAGE}/json"))
|
|
160
|
+
rels = data["releases"]
|
|
161
|
+
if not isinstance(rels, dict):
|
|
162
|
+
raise ValueError("no releases")
|
|
163
|
+
except error.HTTPError as e:
|
|
164
|
+
return {"versions": [], "error": f"PyPI answered {e.code} for {PACKAGE}"}
|
|
165
|
+
except (error.URLError, OSError, TimeoutError) as e:
|
|
166
|
+
return {"versions": [], "error": f"PyPI could not be reached: {getattr(e, 'reason', e)}"}
|
|
167
|
+
except (ValueError, KeyError, TypeError):
|
|
168
|
+
return {"versions": [], "error": "PyPI's answer names no releases"}
|
|
169
|
+
out = []
|
|
170
|
+
for v, files in rels.items():
|
|
171
|
+
if not _VERSION.match(v) or _PRE.search(v) or not isinstance(files, list):
|
|
172
|
+
continue
|
|
173
|
+
wheels = [f for f in files if isinstance(f, dict) and str(f.get("filename", "")).endswith(".whl") and not f.get("yanked")]
|
|
174
|
+
if wheels:
|
|
175
|
+
out.append({"version": v, "date": str(wheels[0].get("upload_time_iso_8601") or "")[:10],
|
|
176
|
+
"requires_python": str(wheels[0].get("requires_python") or "")})
|
|
177
|
+
out.sort(key=lambda r: gitrefs.version_key(r["version"]), reverse=True)
|
|
178
|
+
return {"versions": out, "error": ""}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _release_record(pin: dict, fetch: Optional[Callable]) -> dict:
|
|
182
|
+
"""A release's record from PyPI: its Python requirement and its extras.
|
|
183
|
+
A version PyPI does not have is an error; PyPI out of reach is not - the
|
|
184
|
+
record is then the version alone, and not kept."""
|
|
185
|
+
record = {"kind": "release", "version": pin["version"], "requires_python": "", "extras": [], "wheels": [], "private": []}
|
|
186
|
+
try:
|
|
187
|
+
info = json.loads((fetch or pypi_fetch)(f"{PYPI}/{PACKAGE}/{pin['version']}/json"))["info"]
|
|
188
|
+
except error.HTTPError as e:
|
|
189
|
+
if e.code == 404:
|
|
190
|
+
raise BuildError(f"harbor {pin['version']} is not a release on PyPI")
|
|
191
|
+
return dict(record, partial=True)
|
|
192
|
+
except (error.URLError, OSError, TimeoutError, ValueError, KeyError, TypeError):
|
|
193
|
+
return dict(record, partial=True)
|
|
194
|
+
return dict(record, requires_python=str(info.get("requires_python") or ""),
|
|
195
|
+
extras=sorted(x for x in (info.get("provides_extra") or []) if isinstance(x, str) and _EXTRA.match(x)))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def run_command(argv: List[str], timeout: float = 600.0, stop: Optional[threading.Event] = None) -> tuple:
|
|
200
|
+
"""(returncode, combined output) of one command, with git unable to
|
|
201
|
+
prompt; a timeout is 124, a stop (``stop`` set) is 130."""
|
|
202
|
+
try:
|
|
203
|
+
proc = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=gitrefs.env())
|
|
204
|
+
except OSError as e:
|
|
205
|
+
return 127, f"{argv[0]}: {e.strerror or e}"
|
|
206
|
+
deadline = time.monotonic() + timeout
|
|
207
|
+
while True:
|
|
208
|
+
try:
|
|
209
|
+
out, _ = proc.communicate(timeout=0.5)
|
|
210
|
+
return proc.returncode, out or ""
|
|
211
|
+
except subprocess.TimeoutExpired:
|
|
212
|
+
if (stop is not None and stop.is_set()) or time.monotonic() >= deadline:
|
|
213
|
+
proc.kill(); proc.communicate()
|
|
214
|
+
return (130, "stopped") if stop is not None and stop.is_set() else (124, f"no answer within {int(timeout)} s")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _tail(out: str, n: int = 6) -> str:
|
|
218
|
+
lines = [l for l in (out or "").strip().splitlines() if l.strip()]
|
|
219
|
+
return " / ".join(lines[-n:])
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _check(run: Callable, argv: List[str], what: str, timeout: float = BUILD_TIMEOUT) -> str:
|
|
223
|
+
rc, out = run(argv, timeout)
|
|
224
|
+
if rc != 0:
|
|
225
|
+
raise BuildError(f"{what} failed: {_tail(out) or f'exit {rc}'}")
|
|
226
|
+
return out
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _fetch_commit(run: Callable, url: str, rev: str, dest: str) -> None:
|
|
230
|
+
"""``dest`` checked out at ``rev`` (a commit, a branch or a tag) of
|
|
231
|
+
``url``: a shallow fetch of that one revision, else the whole repository."""
|
|
232
|
+
os.makedirs(dest, exist_ok=True)
|
|
233
|
+
git = gitrefs.GIT
|
|
234
|
+
_check(run, [git, "init", "-q", dest], "git init")
|
|
235
|
+
rc, _ = run([git, "-C", dest, "fetch", "-q", "--depth", "1", url, rev], BUILD_TIMEOUT)
|
|
236
|
+
if rc == 0:
|
|
237
|
+
_check(run, [git, "-C", dest, "-c", "advice.detachedHead=false", "checkout", "-q", "FETCH_HEAD"], "git checkout")
|
|
238
|
+
return
|
|
239
|
+
_check(run, [git, "-C", dest, "fetch", "-q", url, "+refs/heads/*:refs/remotes/upstream/*", "+refs/tags/*:refs/tags/*"],
|
|
240
|
+
f"fetching {gitrefs.label_of(url)}")
|
|
241
|
+
_check(run, [git, "-C", dest, "-c", "advice.detachedHead=false", "checkout", "-q", rev], f"checking out {rev}")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _wheel_metadata(path: str):
|
|
245
|
+
with zipfile.ZipFile(path) as z:
|
|
246
|
+
name = next(n for n in z.namelist() if n.count("/") == 1 and n.endswith(".dist-info/METADATA"))
|
|
247
|
+
return Parser().parsestr(z.read(name).decode("utf-8"))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _built(before: set, wheels: str, what: str) -> str:
|
|
251
|
+
new = sorted(set(os.listdir(wheels)) - before)
|
|
252
|
+
found = [n for n in new if n.endswith(".whl")]
|
|
253
|
+
if not found:
|
|
254
|
+
raise BuildError(f"{what}: no wheel was built")
|
|
255
|
+
return found[0]
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _build_dir(run: Callable, src: str, wheels: str, what: str) -> str:
|
|
259
|
+
before = set(os.listdir(wheels))
|
|
260
|
+
_check(run, ["uv", "build", "--wheel", "--out-dir", wheels, src], f"building {what}")
|
|
261
|
+
return _built(before, wheels, what)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _requirement(text: str) -> tuple:
|
|
265
|
+
"""``(name, url or None)`` of one Requires-Dist line."""
|
|
266
|
+
body = text.split(";", 1)[0].strip()
|
|
267
|
+
if " @ " in body:
|
|
268
|
+
name, url = body.split(" @ ", 1)
|
|
269
|
+
return _norm(re.sub(r"\[.*\]", "", name).strip()), url.strip()
|
|
270
|
+
m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)", body)
|
|
271
|
+
return (_norm(m.group(1)) if m else "", None)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _git_url(text: str) -> tuple:
|
|
275
|
+
"""``git+URL@rev#subdirectory=sub`` as (url, rev, sub)."""
|
|
276
|
+
url = text[4:]
|
|
277
|
+
sub = ""
|
|
278
|
+
if "#" in url:
|
|
279
|
+
url, frag = url.split("#", 1)
|
|
280
|
+
sub = urllib.parse.parse_qs(frag).get("subdirectory", [""])[0]
|
|
281
|
+
rev = ""
|
|
282
|
+
tail = url.rsplit("/", 1)[-1]
|
|
283
|
+
if "@" in tail:
|
|
284
|
+
url, rev = url.rsplit("@", 1)
|
|
285
|
+
return url, rev, sub
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _download(url: str, dest_dir: str) -> str:
|
|
289
|
+
name = os.path.basename(urllib.parse.urlparse(url).path) or "download"
|
|
290
|
+
path = os.path.join(dest_dir, name)
|
|
291
|
+
req = request.Request(url, headers={"User-Agent": "nullius"})
|
|
292
|
+
with request.urlopen(req, timeout=300) as resp, open(path, "wb") as out:
|
|
293
|
+
shutil.copyfileobj(resp, out)
|
|
294
|
+
return path
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _build_private(run: Callable, name: str, how: dict, src: str, work: str, wheels: str, log: Callable) -> str:
|
|
298
|
+
"""One dependency PyPI does not serve, as a wheel in ``wheels``. ``how``
|
|
299
|
+
is its ``[tool.uv.sources]`` entry, or ``{"url": ...}`` for a
|
|
300
|
+
direct-URL requirement."""
|
|
301
|
+
if how.get("git"):
|
|
302
|
+
# the pinned repository's own metadata says where: checked as the pin is, before git is given any of it
|
|
303
|
+
url, rev, sub = gitrefs.url_of(str(how["git"])), str(how.get("rev") or how.get("tag") or how.get("branch") or "HEAD"), str(how.get("subdirectory") or "")
|
|
304
|
+
if not url:
|
|
305
|
+
raise BuildError(f"{name}: {how['git']!r} is not a git URL")
|
|
306
|
+
if not _REVISION.match(rev):
|
|
307
|
+
raise BuildError(f"{name}: {rev!r} is not a branch, a tag or a commit")
|
|
308
|
+
if sub and (os.path.isabs(sub) or ".." in sub.replace("\\", "/").split("/")):
|
|
309
|
+
raise BuildError(f"{name}: the subdirectory {sub!r} is outside its repository")
|
|
310
|
+
log(f" building {name} from {gitrefs.label_of(url)} at {rev}{' (' + sub + ')' if sub else ''}")
|
|
311
|
+
dest = os.path.join(work, _norm(name))
|
|
312
|
+
_fetch_commit(run, url, rev, dest)
|
|
313
|
+
return _build_dir(run, os.path.join(dest, sub) if sub else dest, wheels, name)
|
|
314
|
+
if how.get("url", "").startswith("git+"):
|
|
315
|
+
url, rev, sub = _git_url(how["url"])
|
|
316
|
+
return _build_private(run, name, {"git": url, "rev": rev, "subdirectory": sub}, src, work, wheels, log)
|
|
317
|
+
if how.get("path") or how.get("workspace"):
|
|
318
|
+
if how.get("workspace"):
|
|
319
|
+
members = []
|
|
320
|
+
for pattern in (_project(src).get("tool", {}).get("uv", {}).get("workspace", {}).get("members") or []):
|
|
321
|
+
members += glob.glob(os.path.join(src, pattern))
|
|
322
|
+
found = [m for m in members if _norm(_project(m).get("project", {}).get("name", "")) == _norm(name)]
|
|
323
|
+
if not found:
|
|
324
|
+
raise BuildError(f"{name} is a workspace member the repository does not hold")
|
|
325
|
+
path = found[0]
|
|
326
|
+
else:
|
|
327
|
+
path = os.path.normpath(os.path.join(src, how["path"]))
|
|
328
|
+
if not path.startswith(os.path.abspath(src) + os.sep):
|
|
329
|
+
raise BuildError(f"{name}'s path leaves the repository")
|
|
330
|
+
log(f" building {name} from {os.path.relpath(path, src)} in the repository")
|
|
331
|
+
if path.endswith(".whl"):
|
|
332
|
+
shutil.copy(path, wheels)
|
|
333
|
+
return os.path.basename(path)
|
|
334
|
+
return _build_dir(run, path, wheels, name)
|
|
335
|
+
url = how.get("url", "")
|
|
336
|
+
if url.startswith(("https://", "http://")):
|
|
337
|
+
log(f" fetching {name} from {url}")
|
|
338
|
+
try:
|
|
339
|
+
got = _download(url, work)
|
|
340
|
+
except (error.URLError, OSError) as e:
|
|
341
|
+
raise BuildError(f"{name} could not be downloaded from {url}: {getattr(e, 'reason', e)}")
|
|
342
|
+
if got.endswith(".whl"):
|
|
343
|
+
shutil.move(got, os.path.join(wheels, os.path.basename(got)))
|
|
344
|
+
return os.path.basename(got)
|
|
345
|
+
return _build_dir(run, got, wheels, name)
|
|
346
|
+
raise BuildError(f"{name} comes from {url or how}, which cannot be built here")
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _project(directory: str) -> dict:
|
|
350
|
+
try:
|
|
351
|
+
with open(os.path.join(directory, "pyproject.toml"), "rb") as fh:
|
|
352
|
+
return tomllib.load(fh)
|
|
353
|
+
except (OSError, tomllib.TOMLDecodeError):
|
|
354
|
+
return {}
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _build_repo(pin: dict, tmp: str, run: Callable, log: Callable) -> dict:
|
|
358
|
+
src, work, wheels = os.path.join(tmp, "src"), os.path.join(tmp, "work"), os.path.join(tmp, WHEELS)
|
|
359
|
+
os.makedirs(work); os.makedirs(wheels)
|
|
360
|
+
log(f" fetching {gitrefs.label_of(pin['url'])} at {pin['commit'][:12]}")
|
|
361
|
+
_fetch_commit(run, pin["url"], pin["commit"], src)
|
|
362
|
+
head = _check(run, [gitrefs.GIT, "-C", src, "rev-parse", "HEAD"], "git rev-parse").strip()
|
|
363
|
+
if head != pin["commit"]:
|
|
364
|
+
raise BuildError(f"{gitrefs.label_of(pin['url'])} checked out at {head[:12]}, not {pin['commit'][:12]}")
|
|
365
|
+
project = _project(src)
|
|
366
|
+
named = _norm(project.get("project", {}).get("name", ""))
|
|
367
|
+
if named != PACKAGE:
|
|
368
|
+
raise BuildError(f"{gitrefs.label_of(pin['url'])} at {pin['commit'][:12]} is the project {named or '(none)'}, not harbor")
|
|
369
|
+
log(" building harbor")
|
|
370
|
+
wheel = _build_dir(run, src, wheels, "harbor")
|
|
371
|
+
meta = _wheel_metadata(os.path.join(wheels, wheel))
|
|
372
|
+
sources = project.get("tool", {}).get("uv", {}).get("sources", {}) or {}
|
|
373
|
+
sources = {_norm(k): (v[0] if isinstance(v, list) and v else v) for k, v in sources.items()}
|
|
374
|
+
private, queue, seen = [], list(meta.get_all("Requires-Dist") or []), {PACKAGE}
|
|
375
|
+
while queue:
|
|
376
|
+
name, url = _requirement(queue.pop(0))
|
|
377
|
+
if not name or name in seen:
|
|
378
|
+
continue
|
|
379
|
+
how = {"url": url} if url else sources.get(name) if isinstance(sources.get(name), dict) else None
|
|
380
|
+
if not how or not any(how.get(k) for k in ("git", "path", "url", "workspace")):
|
|
381
|
+
continue # served by an index (PyPI, or one a source names): not built here
|
|
382
|
+
seen.add(name)
|
|
383
|
+
built = _build_private(run, name, how, src, work, wheels, log)
|
|
384
|
+
private.append({"name": name, "wheel": built, "direct": bool(url), "from": url or {k: str(v) for k, v in how.items()}})
|
|
385
|
+
queue += list(_wheel_metadata(os.path.join(wheels, built)).get_all("Requires-Dist") or []) # what it names by URL, in turn
|
|
386
|
+
for name in os.listdir(wheels):
|
|
387
|
+
if not name.endswith(".whl"):
|
|
388
|
+
os.remove(os.path.join(wheels, name)) # uv build's .gitignore
|
|
389
|
+
if not all(_WHEEL.match(n) for n in os.listdir(wheels)):
|
|
390
|
+
raise BuildError("a built wheel has a name this cannot pass on safely")
|
|
391
|
+
return {"kind": "repo", "url": pin["url"], "repo": pin["repo"], "commit": pin["commit"], "version": meta.get("Version", ""),
|
|
392
|
+
"requires_python": meta.get("Requires-Python") or "", "wheel": wheel, "private": private,
|
|
393
|
+
"extras": sorted(x for x in (meta.get_all("Provides-Extra") or []) if _EXTRA.match(x)), "built_at": _now()}
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
_GUARD = threading.Lock()
|
|
397
|
+
_LOCKS: Dict[str, threading.RLock] = {}
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _lock(key: str) -> threading.RLock:
|
|
401
|
+
with _GUARD:
|
|
402
|
+
return _LOCKS.setdefault(key, threading.RLock())
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def record_of(pin: dict, pins: str) -> Optional[dict]:
|
|
406
|
+
"""The pin's record when it is ready here, else None."""
|
|
407
|
+
try:
|
|
408
|
+
with open(os.path.join(pins, key_of(pin), RECORD), encoding="utf-8") as fh:
|
|
409
|
+
rec = json.load(fh)
|
|
410
|
+
except (OSError, ValueError):
|
|
411
|
+
return None
|
|
412
|
+
return rec if isinstance(rec, dict) and rec.get("kind") == pin["kind"] else None
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def ensure(pin: dict, pins: str, run: Optional[Callable] = None, fetch: Optional[Callable] = None,
|
|
416
|
+
log: Optional[Callable] = None) -> dict:
|
|
417
|
+
"""The pin ready on this machine; its record. A repository is built once
|
|
418
|
+
(see the module docstring), a release is read from PyPI once; a ready pin
|
|
419
|
+
is read back. Raises :class:`BuildError` with the reason."""
|
|
420
|
+
run, log = run or run_command, log or (lambda line: None)
|
|
421
|
+
key = key_of(pin)
|
|
422
|
+
with _lock(key):
|
|
423
|
+
rec = record_of(pin, pins)
|
|
424
|
+
if rec is not None:
|
|
425
|
+
return rec
|
|
426
|
+
if pin["kind"] == "release":
|
|
427
|
+
rec = _release_record(pin, fetch)
|
|
428
|
+
if not rec.get("partial"):
|
|
429
|
+
_write(os.path.join(pins, key), RECORD, rec)
|
|
430
|
+
return rec
|
|
431
|
+
os.makedirs(pins, exist_ok=True)
|
|
432
|
+
tmp = tempfile.mkdtemp(prefix=f".{key}-", dir=pins)
|
|
433
|
+
try:
|
|
434
|
+
rec = _build_repo(pin, tmp, run, log)
|
|
435
|
+
for name in ("src", "work"):
|
|
436
|
+
shutil.rmtree(os.path.join(tmp, name), ignore_errors=True)
|
|
437
|
+
_write(tmp, RECORD, rec)
|
|
438
|
+
dest = os.path.join(pins, key)
|
|
439
|
+
shutil.rmtree(dest, ignore_errors=True)
|
|
440
|
+
os.replace(tmp, dest)
|
|
441
|
+
return rec
|
|
442
|
+
finally:
|
|
443
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _write(directory: str, name: str, data: dict) -> None:
|
|
447
|
+
os.makedirs(directory, exist_ok=True)
|
|
448
|
+
tmp = os.path.join(directory, name + ".tmp")
|
|
449
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
450
|
+
json.dump(data, fh, indent=1)
|
|
451
|
+
os.replace(tmp, os.path.join(directory, name))
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def spec(record: dict, extras=(), wheels_sh: str = "") -> str:
|
|
456
|
+
"""What ``uvx --from`` is given, as shell text: ``'harbor==0.24.0'``, or
|
|
457
|
+
the built wheel under ``wheels_sh`` (a shell expression)."""
|
|
458
|
+
ex = f"[{','.join(sorted(extras))}]" if extras else ""
|
|
459
|
+
if record["kind"] == "release":
|
|
460
|
+
return shlex.quote(f"{PACKAGE}{ex}=={record['version']}")
|
|
461
|
+
return f'"{PACKAGE}{ex} @ {wheels_sh}/"{shlex.quote(record["wheel"])}'
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
FEED_SUFFIX = "-feed.py" # the live feed's program, beside the harbor script it serves
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def feed_source() -> str:
|
|
468
|
+
"""``nullius.harbor.feed`` as a file to send: what the harbor script runs."""
|
|
469
|
+
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "feed.py"), encoding="utf-8") as fh:
|
|
470
|
+
return fh.read()
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _harbor_call() -> str:
|
|
474
|
+
"""The pinned Harbor's command line, as uvx is given it: its Python running
|
|
475
|
+
the feed beside this script, which registers the live feed and runs ``harbor``."""
|
|
476
|
+
return f'python "$0{FEED_SUFFIX}" "$@"'
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def wrapper(record: dict, pin_dir_sh: str, extras=()) -> str:
|
|
480
|
+
"""A ``harbor`` that runs the pinned Harbor: a small sh script for
|
|
481
|
+
``$NULLIUS_HARBOR``. ``pin_dir_sh`` is the pin's directory on the target
|
|
482
|
+
as shell text (quoted, or ``"$HOME/…"``); a release needs none."""
|
|
483
|
+
extras = sorted(x for x in extras if _EXTRA.match(x))
|
|
484
|
+
py = f" --python {shlex.quote(record['requires_python'])}" if record.get("requires_python") else ""
|
|
485
|
+
lines = ["#!/bin/sh", "# Written by Nullius: " + (f"harbor {record['version']} from PyPI" if record["kind"] == "release"
|
|
486
|
+
else f"harbor {record.get('version', '')} built from {gitrefs.label_of(record['url'])} at {record['commit']}") + ", as the recipe pins it.",
|
|
487
|
+
'command -v uvx >/dev/null 2>&1 || PATH="$HOME/.local/bin:$PATH"']
|
|
488
|
+
if record["kind"] == "release":
|
|
489
|
+
lines.append(f"exec uvx --isolated{py} --from {spec(record, extras)} {_harbor_call()}")
|
|
490
|
+
return "\n".join(lines) + "\n"
|
|
491
|
+
lines += [f"w={pin_dir_sh}/{WHEELS}",
|
|
492
|
+
f'[ -f "$w"/{shlex.quote(record["wheel"])} ] || {{ echo "harbor: the wheels built for this pin are not in $w" >&2; exit 2; }}']
|
|
493
|
+
direct = [p for p in record.get("private") or [] if p.get("direct")]
|
|
494
|
+
call = f"uvx --isolated{py} --find-links \"$w\"" + (' --overrides "$o"' if direct else "") + f" --from {spec(record, extras, '$w')} {_harbor_call()}"
|
|
495
|
+
if not direct:
|
|
496
|
+
return "\n".join(lines + ["exec " + call]) + "\n"
|
|
497
|
+
# a dependency named by URL is not satisfied by a find-links directory: an override names the wheel built for it
|
|
498
|
+
lines += ['o=$(mktemp "${TMPDIR:-/tmp}/nullius-harbor.XXXXXX") || exit 1',
|
|
499
|
+
"u=$(printf %s \"$w\" | sed 's/%/%25/g; s/ /%20/g')",
|
|
500
|
+
": >\"$o\""]
|
|
501
|
+
lines += [f'printf \'%s\\n\' {shlex.quote(p["name"] + " @ file://")}"$u/"{shlex.quote(p["wheel"])} >>"$o"' for p in direct]
|
|
502
|
+
lines += [call, 'rc=$?; rm -f "$o"; exit $rc']
|
|
503
|
+
return "\n".join(lines) + "\n"
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def write_wrapper(path: str, record: dict, pin_dir_sh: str, extras=()) -> str:
|
|
507
|
+
with open(path + FEED_SUFFIX, "w", encoding="utf-8") as fh:
|
|
508
|
+
fh.write(feed_source())
|
|
509
|
+
with open(path, "w", encoding="utf-8") as fh:
|
|
510
|
+
fh.write(wrapper(record, pin_dir_sh, extras))
|
|
511
|
+
os.chmod(path, 0o755)
|
|
512
|
+
return path
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def scanned(pin: dict, pins: str) -> Optional[dict]:
|
|
517
|
+
"""The scan kept for this pin, or None."""
|
|
518
|
+
try:
|
|
519
|
+
with open(os.path.join(pins, key_of(pin), AGENTS), encoding="utf-8") as fh:
|
|
520
|
+
data = json.load(fh)
|
|
521
|
+
except (OSError, ValueError):
|
|
522
|
+
return None
|
|
523
|
+
return data if isinstance(data, dict) and isinstance(data.get("agents"), list) else None
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def _answer(state: str, pin: dict, rec: Optional[dict] = None, agents=None, error: str = "", at: str = "") -> dict:
|
|
527
|
+
rec = rec or {}
|
|
528
|
+
return {"state": state, "key": key_of(pin), "label": label(pin), "agents": agents, "error": error, "at": at,
|
|
529
|
+
"version": rec.get("version", pin.get("version", "")), "extras": rec.get("extras", [])}
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def scan(pin: dict, pins: str, build: bool = True, run: Optional[Callable] = None, fetch: Optional[Callable] = None,
|
|
533
|
+
log: Optional[Callable] = None) -> dict:
|
|
534
|
+
"""The agents the pinned Harbor registers - ``harbor agents list --json``
|
|
535
|
+
through the same command a run uses - kept beside the pin. With ``build``
|
|
536
|
+
False only what is kept is read (``state`` ``absent`` when nothing is);
|
|
537
|
+
else the pin is made ready and asked once. A failure is said and not
|
|
538
|
+
kept, so asking again tries again."""
|
|
539
|
+
run = run or run_command
|
|
540
|
+
rec, kept = record_of(pin, pins), scanned(pin, pins)
|
|
541
|
+
if kept is not None:
|
|
542
|
+
return _answer("ready", pin, rec or kept, kept["agents"], at=kept.get("at", ""))
|
|
543
|
+
if not build:
|
|
544
|
+
return _answer("absent", pin, rec)
|
|
545
|
+
with _lock(key_of(pin)):
|
|
546
|
+
kept = scanned(pin, pins)
|
|
547
|
+
if kept is not None:
|
|
548
|
+
return _answer("ready", pin, record_of(pin, pins) or kept, kept["agents"], at=kept.get("at", ""))
|
|
549
|
+
try:
|
|
550
|
+
rec = ensure(pin, pins, run=run, fetch=fetch, log=log)
|
|
551
|
+
except BuildError as e:
|
|
552
|
+
return _answer("failed", pin, None, error=str(e))
|
|
553
|
+
d = os.path.join(pins, key_of(pin))
|
|
554
|
+
os.makedirs(d, exist_ok=True)
|
|
555
|
+
exe = write_wrapper(os.path.join(d, "harbor"), rec, shlex.quote(d))
|
|
556
|
+
rc, out = run([exe, "agents", "list", "--json"], SCAN_TIMEOUT)
|
|
557
|
+
names, why = _agents_of(rc, out)
|
|
558
|
+
if names is None:
|
|
559
|
+
return _answer("failed", pin, rec, error=why)
|
|
560
|
+
at = _now()
|
|
561
|
+
if not rec.get("partial"):
|
|
562
|
+
_write(d, AGENTS, {"agents": names, "at": at, "version": rec.get("version", ""), "extras": rec.get("extras", [])})
|
|
563
|
+
return _answer("ready", pin, rec, names, at=at)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _agents_of(rc: int, out: str) -> tuple:
|
|
567
|
+
"""(names, None) from what ``harbor agents list --json`` printed, or
|
|
568
|
+
(None, why). Only a list of agent names is a list."""
|
|
569
|
+
if rc != 0:
|
|
570
|
+
said = "\n".join(l for l in (out or "").splitlines() if not _UV_SAYS.match(l)) # uv's narration is not the reason
|
|
571
|
+
return None, f"harbor agents list failed: {_tail(said, 4) or f'exit {rc}'}"
|
|
572
|
+
text, names = out or "", None
|
|
573
|
+
# uv says what it installed on the same stream: the list is the first line that starts one
|
|
574
|
+
for m in re.finditer(r"(?m)^\s*\[", text):
|
|
575
|
+
try:
|
|
576
|
+
names = json.JSONDecoder().raw_decode(text[m.start():].lstrip())[0]
|
|
577
|
+
break
|
|
578
|
+
except ValueError:
|
|
579
|
+
continue
|
|
580
|
+
if not isinstance(names, list) or not all(isinstance(n, str) and _AGENT.match(n) for n in names):
|
|
581
|
+
return None, "harbor agents list printed something other than a list of agent names"
|
|
582
|
+
return sorted(set(names)), None
|