hivekit 0.8.1__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hivekit-0.8.1 → hivekit-0.8.2}/PKG-INFO +1 -1
- {hivekit-0.8.1 → hivekit-0.8.2}/pyproject.toml +1 -1
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/config.py +69 -73
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/experiment.py +0 -2
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/main.py +27 -4
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/skills/hive-setup/SKILL.md +9 -1
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/skills/hive-setup/references/configuration.md +20 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/archive.py +35 -2
- hivekit-0.8.2/src/cli/utils/attachments.py +225 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/config_paths.py +0 -8
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/upload.py +19 -9
- {hivekit-0.8.1 → hivekit-0.8.2}/src/hivekit.egg-info/PKG-INFO +1 -1
- {hivekit-0.8.1 → hivekit-0.8.2}/src/hivekit.egg-info/SOURCES.txt +2 -0
- hivekit-0.8.2/tests/test_attachments.py +171 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_config.py +209 -116
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_experiment.py +8 -1
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_main.py +293 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/LICENSE +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/README.md +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/setup.cfg +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/__init__.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/auth/__init__.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/auth/auth_utils.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/auth/credential_store.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/auth/login_page.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/auth/logo.svg +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/auth/oidc_flow.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/auth/session_manager.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/auth/token_revoker.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/completers.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/http_client.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/skills/hive-setup/references/gpu-hardware.md +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/skills/hive-setup/references/multi-evaluator.md +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/skills_install.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/update_check.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/__init__.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/config_sync.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/docker.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/logger.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/terminal.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/time.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/utils/url_utils.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/cli/version.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/hivekit.egg-info/dependency_links.txt +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/hivekit.egg-info/entry_points.txt +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/hivekit.egg-info/requires.txt +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/src/hivekit.egg-info/top_level.txt +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_completers.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_config_sync.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_http_client.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_login.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_logout.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_overrides.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_push_image.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_time_utils.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_update_check.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_upload.py +0 -0
- {hivekit-0.8.1 → hivekit-0.8.2}/tests/test_version.py +0 -0
|
@@ -11,6 +11,7 @@ import pydantic
|
|
|
11
11
|
import yaml
|
|
12
12
|
from kubernetes.utils import parse_quantity
|
|
13
13
|
from pydantic import (
|
|
14
|
+
AnyHttpUrl,
|
|
14
15
|
ConfigDict,
|
|
15
16
|
Field,
|
|
16
17
|
PrivateAttr,
|
|
@@ -19,7 +20,6 @@ from pydantic import (
|
|
|
19
20
|
model_validator,
|
|
20
21
|
)
|
|
21
22
|
|
|
22
|
-
from cli.utils.config_paths import get_coordinator_overlay_path
|
|
23
23
|
from cli.utils.logger import set_log_level
|
|
24
24
|
from cli.utils.time import unique_suffix
|
|
25
25
|
|
|
@@ -256,11 +256,67 @@ class PromptConfig(BaseModel):
|
|
|
256
256
|
default=None,
|
|
257
257
|
description="A list of ideas which will be randomly sampled to inject into the Hive.",
|
|
258
258
|
)
|
|
259
|
-
|
|
259
|
+
attachments: Optional[list[str]] = Field(
|
|
260
260
|
default=None,
|
|
261
|
-
description=
|
|
261
|
+
description=(
|
|
262
|
+
"A list of local PDF files/directories or HTTP/HTTPS PDF URLs to provide as "
|
|
263
|
+
"reference material. Directories are searched recursively for files with a "
|
|
264
|
+
".pdf extension (case-insensitive). Local relative paths resolve from the "
|
|
265
|
+
"directory where hive is run."
|
|
266
|
+
),
|
|
262
267
|
)
|
|
263
268
|
|
|
269
|
+
@model_validator(mode="before")
|
|
270
|
+
@classmethod
|
|
271
|
+
def migrate_arxiv_ids(cls, values):
|
|
272
|
+
"""Convert legacy IDs into attachment URLs without mutating the input."""
|
|
273
|
+
if not isinstance(values, dict) or "arxiv_ids" not in values:
|
|
274
|
+
return values
|
|
275
|
+
values = dict(values)
|
|
276
|
+
# Preserve the legacy list's type coercion while folding it into attachments.
|
|
277
|
+
string_list = pydantic.TypeAdapter(Optional[list[str]], config=cls.model_config)
|
|
278
|
+
legacy = string_list.validate_python(values.pop("arxiv_ids"))
|
|
279
|
+
converted_urls = [f"https://arxiv.org/pdf/{arxiv_id}" for arxiv_id in legacy or []]
|
|
280
|
+
if converted_urls:
|
|
281
|
+
attachments = string_list.validate_python(
|
|
282
|
+
cls.validate_attachments_list(values.get("attachments"))
|
|
283
|
+
)
|
|
284
|
+
values["attachments"] = [*(attachments or []), *converted_urls]
|
|
285
|
+
warning = (
|
|
286
|
+
"prompt.arxiv_ids is deprecated and is converted to PDF URLs in "
|
|
287
|
+
"prompt.attachments. Use PDF URLs in prompt.attachments and remove "
|
|
288
|
+
"prompt.arxiv_ids to silence this warning."
|
|
289
|
+
)
|
|
290
|
+
if converted_urls:
|
|
291
|
+
warning += "\nConverted PDF URLs:\n" + "\n".join(f" - {url}" for url in converted_urls)
|
|
292
|
+
logger.warning(warning)
|
|
293
|
+
return values
|
|
294
|
+
|
|
295
|
+
@field_validator("attachments", mode="before")
|
|
296
|
+
@classmethod
|
|
297
|
+
def validate_attachments_list(cls, value):
|
|
298
|
+
if isinstance(value, str):
|
|
299
|
+
raise ValueError(
|
|
300
|
+
"Must be a YAML list, not a plain string. Example:\n attachments:\n - paper.pdf"
|
|
301
|
+
)
|
|
302
|
+
return value
|
|
303
|
+
|
|
304
|
+
@field_validator("attachments")
|
|
305
|
+
@classmethod
|
|
306
|
+
def validate_attachment_syntax(cls, value: Optional[list[str]]) -> Optional[list[str]]:
|
|
307
|
+
"""Check attachment syntax without accessing local files or the network."""
|
|
308
|
+
if not value:
|
|
309
|
+
return value
|
|
310
|
+
http_url = pydantic.TypeAdapter(AnyHttpUrl)
|
|
311
|
+
for source in value:
|
|
312
|
+
if not source.strip():
|
|
313
|
+
raise ValueError("Attachment paths and URLs must not be empty")
|
|
314
|
+
# Recognize URL prefixes without treating colons in local filenames as schemes.
|
|
315
|
+
if re.match(r"(?i)^(?:[a-z][a-z0-9+.-]*://|https?:)", source.lstrip()):
|
|
316
|
+
http_url.validate_python(source, strict=True)
|
|
317
|
+
# Keep configured paths and URLs unchanged, including URL query strings.
|
|
318
|
+
return value
|
|
319
|
+
|
|
264
320
|
|
|
265
321
|
def _extract_path(entry: str, strip_exclude: bool = False) -> str:
|
|
266
322
|
"""Reduce a file-list entry to the path it references."""
|
|
@@ -630,10 +686,9 @@ class HiveConfig(BaseModel):
|
|
|
630
686
|
coordinator_image_tag: Optional[str] = Field(
|
|
631
687
|
default=None,
|
|
632
688
|
description="The image tag to run the coordinator at, e.g. 'v0.6.0'. Optional — when "
|
|
633
|
-
"omitted,
|
|
634
|
-
"
|
|
635
|
-
"
|
|
636
|
-
"ignored in production environments.",
|
|
689
|
+
"omitted, the backend's default coordinator image is used. Set this in the experiment "
|
|
690
|
+
"config or with a command-line override. This field is ignored outside development "
|
|
691
|
+
"environments.",
|
|
637
692
|
)
|
|
638
693
|
runtime: RuntimeConfig = Field(
|
|
639
694
|
default_factory=RuntimeConfig, description="Runtime configuration for the experiment."
|
|
@@ -862,71 +917,12 @@ def _reconcile_num_sandboxes_override(cfg: dict, overridden_keys: set[tuple[str,
|
|
|
862
917
|
del runtime[loser]
|
|
863
918
|
|
|
864
919
|
|
|
865
|
-
def
|
|
866
|
-
"""
|
|
867
|
-
|
|
868
|
-
A broken overlay must not stop an experiment the user asked for, so anything
|
|
869
|
-
unusable is warned about and ignored. An absent, empty or non-scalar tag
|
|
870
|
-
records nothing; a bool is not a scalar here, being an int subclass.
|
|
871
|
-
"""
|
|
872
|
-
if not os.path.exists(overlay_path):
|
|
873
|
-
return None
|
|
874
|
-
try:
|
|
875
|
-
with open(overlay_path, "r") as file:
|
|
876
|
-
overlay = yaml.safe_load(file)
|
|
877
|
-
except (OSError, yaml.YAMLError) as e:
|
|
878
|
-
logger.warning("Ignoring %s, which could not be read: %s", overlay_path, e)
|
|
879
|
-
return None
|
|
880
|
-
tag = overlay.get("coordinator_image_tag") if isinstance(overlay, dict) else None
|
|
881
|
-
if tag is None or tag == "":
|
|
882
|
-
return None
|
|
883
|
-
if isinstance(tag, bool) or not isinstance(tag, (str, int, float)):
|
|
884
|
-
logger.warning(
|
|
885
|
-
"Ignoring coordinator_image_tag: %r in %s, which must be a string or number.",
|
|
886
|
-
tag,
|
|
887
|
-
overlay_path,
|
|
888
|
-
)
|
|
889
|
-
return None
|
|
890
|
-
return str(tag)
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
def _resolve_coordinator_image_tag(config_data: dict, overridden: bool) -> None:
|
|
894
|
-
"""Fall back to the coordinator image tag `hivectl generate config` recorded.
|
|
895
|
-
|
|
896
|
-
A tag the experiment asked for wins, even an empty one asking for the backend
|
|
897
|
-
default. Whichever tag is used is logged with where it came from, as is any
|
|
898
|
-
recorded tag it discards, so which coordinator image runs is never a surprise.
|
|
899
|
-
"""
|
|
900
|
-
overlay_path = get_coordinator_overlay_path()
|
|
901
|
-
recorded = _recorded_coordinator_image_tag(overlay_path)
|
|
902
|
-
if "coordinator_image_tag" not in config_data:
|
|
903
|
-
if recorded is None:
|
|
904
|
-
return
|
|
905
|
-
config_data["coordinator_image_tag"] = recorded
|
|
906
|
-
logger.warning(
|
|
907
|
-
"Using coordinator_image_tag=%s recorded in %s by `hivectl generate config`; "
|
|
908
|
-
"set coordinator_image_tag in the experiment config to override it.",
|
|
909
|
-
recorded,
|
|
910
|
-
overlay_path,
|
|
911
|
-
)
|
|
912
|
-
return
|
|
913
|
-
asked_for = config_data["coordinator_image_tag"]
|
|
920
|
+
def _log_coordinator_image_tag(config_data: dict, overridden: bool) -> None:
|
|
921
|
+
"""Report the tag explicitly selected by the experiment config or CLI."""
|
|
922
|
+
asked_for = config_data.get("coordinator_image_tag")
|
|
914
923
|
source = "a command-line override" if overridden else "the experiment config"
|
|
915
|
-
|
|
916
|
-
# an explicit null would match the tag "None" rather than discard it.
|
|
917
|
-
if recorded is not None and (asked_for is None or str(asked_for) != recorded):
|
|
918
|
-
logger.warning(
|
|
919
|
-
"Using coordinator_image_tag=%s from %s, discarding coordinator_image_tag=%s "
|
|
920
|
-
"recorded in %s.",
|
|
921
|
-
asked_for,
|
|
922
|
-
source,
|
|
923
|
-
recorded,
|
|
924
|
-
overlay_path,
|
|
925
|
-
)
|
|
926
|
-
elif asked_for is not None and asked_for != "":
|
|
924
|
+
if asked_for is not None and asked_for != "":
|
|
927
925
|
logger.warning("Using coordinator_image_tag=%s from %s.", asked_for, source)
|
|
928
|
-
# An empty tag with nothing to discard asks for the backend's default image,
|
|
929
|
-
# exactly as setting no tag at all does, so there is nothing to report.
|
|
930
926
|
|
|
931
927
|
|
|
932
928
|
def load_config(
|
|
@@ -936,8 +932,8 @@ def load_config(
|
|
|
936
932
|
) -> HiveConfig:
|
|
937
933
|
"""Load configuration from a YAML file.
|
|
938
934
|
|
|
939
|
-
Pass ``coordinator=False`` when no coordinator will run, so
|
|
940
|
-
coordinator image tag is
|
|
935
|
+
Pass ``coordinator=False`` when no coordinator will run, so an explicit
|
|
936
|
+
coordinator image tag is not reported as being used.
|
|
941
937
|
"""
|
|
942
938
|
with open(file_path, "r") as file:
|
|
943
939
|
raw_text = file.read()
|
|
@@ -948,7 +944,7 @@ def load_config(
|
|
|
948
944
|
overridden_keys = apply_overrides(config_data, overrides)
|
|
949
945
|
|
|
950
946
|
if coordinator and isinstance(config_data, dict):
|
|
951
|
-
|
|
947
|
+
_log_coordinator_image_tag(config_data, ("coordinator_image_tag",) in overridden_keys)
|
|
952
948
|
|
|
953
949
|
try:
|
|
954
950
|
config = HiveConfig(**config_data)
|
|
@@ -124,8 +124,6 @@ def build_experiment_crd(config: HiveConfig) -> Dict[str, Any]:
|
|
|
124
124
|
experiment["spec"]["prompt"]["context"] = config.prompt.context
|
|
125
125
|
if config.prompt.ideas:
|
|
126
126
|
experiment["spec"]["prompt"]["ideas"] = config.prompt.ideas
|
|
127
|
-
if config.prompt.arxiv_ids:
|
|
128
|
-
experiment["spec"]["prompt"]["arxivIds"] = config.prompt.arxiv_ids
|
|
129
127
|
if config.prompt.enable_evolution:
|
|
130
128
|
experiment["spec"]["prompt"]["enableEvolution"] = config.prompt.enable_evolution
|
|
131
129
|
|
|
@@ -42,10 +42,11 @@ from cli.skills_install import skills_install
|
|
|
42
42
|
from cli.update_check import check_for_update, print_update_footer, warn_if_outdated
|
|
43
43
|
from cli.utils import docker
|
|
44
44
|
from cli.utils.archive import resolve_file_list
|
|
45
|
+
from cli.utils.attachments import AttachmentError, prepare_attachments
|
|
45
46
|
from cli.utils.config_paths import get_config_dir, get_config_path, load_organization_id
|
|
46
47
|
from cli.utils.config_sync import sync_base_image
|
|
47
48
|
from cli.utils.time import humanize_time
|
|
48
|
-
from cli.utils.upload import streaming_upload
|
|
49
|
+
from cli.utils.upload import streaming_upload, streaming_upload_entries
|
|
49
50
|
from cli.utils.url_utils import (
|
|
50
51
|
build_oidc_endpoints,
|
|
51
52
|
derive_identity_base_url,
|
|
@@ -253,6 +254,18 @@ def _upload_source_if_any(
|
|
|
253
254
|
return patch_id
|
|
254
255
|
|
|
255
256
|
|
|
257
|
+
def _upload_attachments_if_any(
|
|
258
|
+
client, attachments: list[str] | None, *, dry_run: bool = False
|
|
259
|
+
) -> str | None:
|
|
260
|
+
"""Prepare and upload PDFs, keeping downloads only for the upload's lifetime."""
|
|
261
|
+
with prepare_attachments(attachments or []) as entries:
|
|
262
|
+
if not entries or dry_run:
|
|
263
|
+
return None
|
|
264
|
+
resource_id = streaming_upload_entries(client, entries)
|
|
265
|
+
logger.debug("Resource upload complete. resource_id=%s", resource_id)
|
|
266
|
+
return resource_id
|
|
267
|
+
|
|
268
|
+
|
|
256
269
|
@contextlib.contextmanager
|
|
257
270
|
def _resolve_source(config):
|
|
258
271
|
"""Yield a local directory path for repo.source, cloning if remote."""
|
|
@@ -293,14 +306,24 @@ def create_experiment(args, overrides) -> None:
|
|
|
293
306
|
)
|
|
294
307
|
sys.exit(1)
|
|
295
308
|
|
|
309
|
+
attachments = config.prompt.attachments if config.prompt is not None else []
|
|
310
|
+
try:
|
|
311
|
+
resource_id = _upload_attachments_if_any(client, attachments, dry_run=args.dry_run)
|
|
312
|
+
except AttachmentError as e:
|
|
313
|
+
console.print(f"[bold red]Error:[/bold red] {e}", highlight=False)
|
|
314
|
+
sys.exit(1)
|
|
315
|
+
|
|
296
316
|
experiment_crd = experiment.build_experiment_crd(config)
|
|
297
317
|
|
|
298
318
|
patch_id = _upload_source_if_any(
|
|
299
319
|
client, config, dry_run=args.dry_run, allow_missing=args.allow_missing_files
|
|
300
320
|
)
|
|
301
|
-
if patch_id:
|
|
302
|
-
experiment_crd.setdefault("metadata", {}).setdefault("annotations", {})
|
|
303
|
-
|
|
321
|
+
if patch_id or resource_id:
|
|
322
|
+
annotations = experiment_crd.setdefault("metadata", {}).setdefault("annotations", {})
|
|
323
|
+
if patch_id:
|
|
324
|
+
annotations["hiverge.ai/patch-key"] = patch_id
|
|
325
|
+
if resource_id:
|
|
326
|
+
annotations["hiverge.ai/resource-key"] = resource_id
|
|
304
327
|
|
|
305
328
|
if args.dry_run:
|
|
306
329
|
console.print(
|
|
@@ -307,7 +307,13 @@ This is the agents' primary steer. Write it as an onboarding document — imagin
|
|
|
307
307
|
|
|
308
308
|
- **Describe the problem, not the solution.** Do NOT suggest optimization approaches, likely bottlenecks, or directions to explore — that biases the Hive and narrows its search.
|
|
309
309
|
- **Never describe the current code.** This context is fixed while the target evolves every iteration, so anything about how the present implementation works is stale and misleading at once.
|
|
310
|
-
- **Distill useful background from the repo's README or design docs** into this prose —
|
|
310
|
+
- **Distill useful background from the repo's README or design docs** into this prose — `repo.additional_context` is for code files only. PDF references can be attached separately (see below).
|
|
311
|
+
|
|
312
|
+
### PDF reference material
|
|
313
|
+
|
|
314
|
+
When the user supplies research papers, technical manuals, or other PDF references, add them under `prompt.attachments`. They provide reference material for agents and are not included in candidate code. Keep the task and metric in `prompt.context`, with a short note on which references are relevant.
|
|
315
|
+
|
|
316
|
+
Use local PDF files, local directories, or PDF URLs. For arXiv papers, use the `/pdf/` URL, such as `https://arxiv.org/pdf/2605.10327`. See [PDF attachments](references/configuration.md#pdf-attachments) for an example, path handling, and limits.
|
|
311
317
|
|
|
312
318
|
### Key field notes
|
|
313
319
|
|
|
@@ -336,6 +342,8 @@ These drive cost and feasibility, so **when unsure about sandbox count, runtime,
|
|
|
336
342
|
|
|
337
343
|
After writing `hive.yaml`, you **must** run `hive create exp -c hive.yaml --dry-run` to confirm the config is valid. Fix any errors before moving on.
|
|
338
344
|
|
|
345
|
+
With `prompt.attachments`, the dry-run also checks local PDFs and downloads PDF URLs for validation. Relative attachment paths resolve from the directory where you run `hive`; use absolute paths if the launch directory may differ.
|
|
346
|
+
|
|
339
347
|
Add `--allow-missing-files` only when a `target_code`/`additional_context` path is deliberately absent from the upload — a prebuilt image with `repo.files` narrowed, `source: null`, or a file `setup_script` generates. It's accepted by `hive create exp` and `hive shell` alike; when the whole source directory is uploaded, omit it so a mistyped path still errors.
|
|
340
348
|
|
|
341
349
|
|
|
@@ -22,6 +22,7 @@ Source: https://docs.hiverge.ai/gettingstarted/cli/configuration
|
|
|
22
22
|
| `apiversion` | string | `v1alpha1` | Schema version |
|
|
23
23
|
| `experiment_name` | string | required | Valid DNS label (`[a-z0-9-]`, max 51 chars, no leading `-`); trailing `-` appends a random 7-char unique suffix, so a name ending in `-` may have at most 43 chars before the `-` |
|
|
24
24
|
| `coordinator_config_name` | string | `default-coordinator-config` | |
|
|
25
|
+
| `coordinator_image_tag` | string | `null` | Development environments only. Set explicitly here or via `coordinator_image_tag=<tag>` on the command line; omitted or empty uses the backend default. |
|
|
25
26
|
|
|
26
27
|
## `repo`
|
|
27
28
|
| Field | Type | Default | Notes |
|
|
@@ -108,6 +109,23 @@ Optional — omit for defaults. Steers the agents' search.
|
|
|
108
109
|
|---|---|---|---|
|
|
109
110
|
| `context` | string | — | Experiment-specific guidance, multi-line |
|
|
110
111
|
| `ideas` | list[string] | — | Distinct directions; one randomly sampled and injected each iteration |
|
|
112
|
+
| `attachments` | list[string] | `null` | Local PDF files, local directories, or PDF URLs. See [PDF attachments](#pdf-attachments). |
|
|
113
|
+
|
|
114
|
+
### PDF attachments
|
|
115
|
+
|
|
116
|
+
PDFs provide reference material for agents and are not included in candidate code.
|
|
117
|
+
|
|
118
|
+
```yaml
|
|
119
|
+
prompt:
|
|
120
|
+
attachments:
|
|
121
|
+
- papers/reference.pdf
|
|
122
|
+
- background/
|
|
123
|
+
- https://arxiv.org/pdf/2605.10327
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
- Relative paths resolve from the directory where you run `hive`, not from the location of `hive.yaml` or `repo.source`. Directories are searched recursively for PDFs.
|
|
127
|
+
- Each PDF can be up to **20 MiB**. URLs must return a PDF; use a paper's PDF download URL rather than its abstract or landing page.
|
|
128
|
+
- The CLI validates attachments before creating the experiment. `hive create exp -c hive.yaml --dry-run` also checks attachments, including downloading PDF URLs.
|
|
111
129
|
|
|
112
130
|
## Evaluator output contract
|
|
113
131
|
`evaluate.py` must print a JSON object on the **final line** of stdout, and must **exit 0 even when reporting a failure** — a non-zero exit code is a crashed evaluator, not a failed candidate. Report invalid candidates with `{"status": "failed", ...}` and exit cleanly.
|
|
@@ -181,6 +199,8 @@ prompt:
|
|
|
181
199
|
ideas:
|
|
182
200
|
- "Try batching database writes"
|
|
183
201
|
- "Consider async I/O for network calls"
|
|
202
|
+
attachments:
|
|
203
|
+
- https://arxiv.org/pdf/2605.10327
|
|
184
204
|
```
|
|
185
205
|
|
|
186
206
|
## CLI cheat-sheet
|
|
@@ -1,23 +1,37 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
3
|
import tarfile
|
|
4
|
+
from dataclasses import dataclass
|
|
4
5
|
from pathlib import PurePath
|
|
5
6
|
|
|
6
7
|
logger = logging.getLogger("hivekit")
|
|
7
8
|
|
|
8
9
|
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class ArchiveEntry:
|
|
12
|
+
"""One physical file and the relative name it should have in an archive."""
|
|
13
|
+
|
|
14
|
+
source_path: str
|
|
15
|
+
archive_name: str
|
|
16
|
+
|
|
17
|
+
|
|
9
18
|
def _is_glob(pattern: str) -> bool:
|
|
10
19
|
return any(c in pattern for c in "*?[")
|
|
11
20
|
|
|
12
21
|
|
|
22
|
+
def _validate_relative_path(rel_path: str) -> None:
|
|
23
|
+
"""Ensure a path is a safe relative archive name."""
|
|
24
|
+
if os.path.isabs(rel_path) or ".." in rel_path.split(os.sep):
|
|
25
|
+
raise ValueError(f"Unsafe relative path: '{rel_path}'")
|
|
26
|
+
|
|
27
|
+
|
|
13
28
|
def _validate_path(base_dir: str, rel_path: str) -> None:
|
|
14
29
|
"""Ensure rel_path resolves within base_dir and is a safe archive name."""
|
|
30
|
+
_validate_relative_path(rel_path)
|
|
15
31
|
real_base = os.path.realpath(base_dir)
|
|
16
32
|
real_full = os.path.realpath(os.path.join(base_dir, rel_path))
|
|
17
33
|
if not real_full.startswith(real_base + os.sep) and real_full != real_base:
|
|
18
34
|
raise ValueError(f"Path escapes base directory: '{rel_path}'")
|
|
19
|
-
if os.path.isabs(rel_path) or ".." in rel_path.split(os.sep):
|
|
20
|
-
raise ValueError(f"Unsafe relative path: '{rel_path}'")
|
|
21
35
|
|
|
22
36
|
|
|
23
37
|
def _walk_non_hidden(base_dir: str):
|
|
@@ -125,6 +139,25 @@ def add_to_tar(tar: tarfile.TarFile, base_dir: str, files: list[str]) -> None:
|
|
|
125
139
|
raise ValueError(f"Path is neither file nor directory: '{entry}'")
|
|
126
140
|
|
|
127
141
|
|
|
142
|
+
def add_entries_to_tar(tar: tarfile.TarFile, entries: list[ArchiveEntry]) -> None:
|
|
143
|
+
"""Add explicitly mapped regular files to an open tar archive."""
|
|
144
|
+
seen: set[str] = set()
|
|
145
|
+
for entry in entries:
|
|
146
|
+
_validate_relative_path(entry.archive_name)
|
|
147
|
+
if entry.archive_name in {"", "."}:
|
|
148
|
+
raise ValueError(f"Unsafe archive name: '{entry.archive_name}'")
|
|
149
|
+
if entry.archive_name in seen:
|
|
150
|
+
raise ValueError(f"Duplicate archive name: '{entry.archive_name}'")
|
|
151
|
+
seen.add(entry.archive_name)
|
|
152
|
+
|
|
153
|
+
source_path = os.path.expanduser(entry.source_path)
|
|
154
|
+
if os.path.islink(source_path):
|
|
155
|
+
raise ValueError(f"Symlinks are not supported: '{source_path}'")
|
|
156
|
+
if not os.path.isfile(source_path):
|
|
157
|
+
raise ValueError(f"Archive source is not a file: '{source_path}'")
|
|
158
|
+
tar.add(source_path, arcname=entry.archive_name, recursive=False)
|
|
159
|
+
|
|
160
|
+
|
|
128
161
|
def _add_directory(tar: tarfile.TarFile, base_dir: str, rel_dir: str) -> None:
|
|
129
162
|
"""Recursively add a directory, rejecting symlinks at any level."""
|
|
130
163
|
full_dir = os.path.join(base_dir, rel_dir)
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# Copyright (C) 2026 Hiverge
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
"""Prepare local and remote PDF attachments for upload."""
|
|
6
|
+
|
|
7
|
+
import contextlib
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import tempfile
|
|
11
|
+
from collections.abc import Iterator
|
|
12
|
+
from pathlib import Path, PurePosixPath
|
|
13
|
+
from urllib.parse import unquote, urlparse
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
from rich import filesize
|
|
17
|
+
from rich.markup import escape
|
|
18
|
+
from rich.progress import FileSizeColumn, Progress, SpinnerColumn, TextColumn, TransferSpeedColumn
|
|
19
|
+
|
|
20
|
+
from cli.utils.archive import ArchiveEntry
|
|
21
|
+
|
|
22
|
+
MAX_PDF_ATTACHMENT_BYTES = 20 * 1024 * 1024
|
|
23
|
+
MAX_PDF_ATTACHMENT_MIB = MAX_PDF_ATTACHMENT_BYTES // (1024 * 1024)
|
|
24
|
+
RESOURCES_DIR = ".hive/resources"
|
|
25
|
+
|
|
26
|
+
_DOWNLOAD_CHUNK_SIZE = 64 * 1024
|
|
27
|
+
_PDF_HEADER_SCAN_BYTES = 1024
|
|
28
|
+
_REQUEST_TIMEOUT = (10, 120)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class AttachmentError(ValueError):
|
|
32
|
+
"""A configured attachment could not be prepared for upload."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _validate_pdf(path: Path, configured_source: str) -> None:
|
|
36
|
+
"""Check the PDF header and size, rejecting symlinks and unreadable files."""
|
|
37
|
+
if path.is_symlink():
|
|
38
|
+
raise AttachmentError(
|
|
39
|
+
f"attachment {configured_source!r} is a symlink; symlinks are not supported"
|
|
40
|
+
)
|
|
41
|
+
try:
|
|
42
|
+
size = path.stat().st_size
|
|
43
|
+
except OSError as error:
|
|
44
|
+
raise AttachmentError(
|
|
45
|
+
f"attachment {configured_source!r} could not be read: {error}"
|
|
46
|
+
) from error
|
|
47
|
+
if size > MAX_PDF_ATTACHMENT_BYTES:
|
|
48
|
+
raise AttachmentError(
|
|
49
|
+
f"attachment {configured_source!r} exceeds the {MAX_PDF_ATTACHMENT_MIB} MiB PDF limit"
|
|
50
|
+
)
|
|
51
|
+
try:
|
|
52
|
+
with path.open("rb") as file:
|
|
53
|
+
header = file.read(_PDF_HEADER_SCAN_BYTES)
|
|
54
|
+
except OSError as error:
|
|
55
|
+
raise AttachmentError(
|
|
56
|
+
f"attachment {configured_source!r} could not be read: {error}"
|
|
57
|
+
) from error
|
|
58
|
+
if b"%PDF-" not in header:
|
|
59
|
+
raise AttachmentError(f"attachment {configured_source!r} is not a PDF")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _safe_stem(value: str) -> str:
|
|
63
|
+
"""Return a non-empty filename stem without path components or control characters."""
|
|
64
|
+
name = PurePosixPath(value.replace("\\", "/")).name
|
|
65
|
+
name = re.sub(r"[\x00-\x1f\x7f]", "", name).strip()
|
|
66
|
+
if name.lower().endswith(".pdf"):
|
|
67
|
+
name = name[:-4]
|
|
68
|
+
name = name.strip().strip(".")
|
|
69
|
+
return name[:200] or "attachment"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _allocate_filename(source_name: str, used_names: set[str]) -> str:
|
|
73
|
+
"""Choose an unused PDF basename and add it to used_names."""
|
|
74
|
+
stem = _safe_stem(source_name)
|
|
75
|
+
candidate = f"{stem}.pdf"
|
|
76
|
+
duplicate = 0
|
|
77
|
+
while candidate in used_names:
|
|
78
|
+
duplicate += 1
|
|
79
|
+
candidate = f"{stem} ({duplicate}).pdf"
|
|
80
|
+
used_names.add(candidate)
|
|
81
|
+
return candidate
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _download_pdf(url: str, destination: Path) -> None:
|
|
85
|
+
"""Download a PDF with filename, byte-count, and transfer-speed feedback."""
|
|
86
|
+
filename = f"{_safe_stem(unquote(urlparse(url).path))}.pdf"
|
|
87
|
+
try:
|
|
88
|
+
with Progress(
|
|
89
|
+
SpinnerColumn(),
|
|
90
|
+
TextColumn("[bold blue]Downloading"),
|
|
91
|
+
TextColumn("{task.description}", markup=False),
|
|
92
|
+
FileSizeColumn(),
|
|
93
|
+
TransferSpeedColumn(),
|
|
94
|
+
transient=True,
|
|
95
|
+
) as progress:
|
|
96
|
+
if not progress.console.is_interactive:
|
|
97
|
+
progress.console.print(f"Downloading {filename}", markup=False, highlight=False)
|
|
98
|
+
task = progress.add_task(filename, total=None)
|
|
99
|
+
with requests.get(
|
|
100
|
+
url,
|
|
101
|
+
stream=True,
|
|
102
|
+
allow_redirects=True,
|
|
103
|
+
timeout=_REQUEST_TIMEOUT,
|
|
104
|
+
) as response:
|
|
105
|
+
response.raise_for_status()
|
|
106
|
+
content_length = response.headers.get("content-length")
|
|
107
|
+
if content_length is not None:
|
|
108
|
+
try:
|
|
109
|
+
declared_size = int(content_length)
|
|
110
|
+
except ValueError:
|
|
111
|
+
declared_size = None
|
|
112
|
+
if declared_size is not None and declared_size > MAX_PDF_ATTACHMENT_BYTES:
|
|
113
|
+
raise AttachmentError(
|
|
114
|
+
f"attachment {url!r} exceeds the {MAX_PDF_ATTACHMENT_MIB} MiB PDF limit"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
downloaded = 0
|
|
118
|
+
with destination.open("wb") as file:
|
|
119
|
+
for chunk in response.iter_content(chunk_size=_DOWNLOAD_CHUNK_SIZE):
|
|
120
|
+
if not chunk:
|
|
121
|
+
continue
|
|
122
|
+
downloaded += len(chunk)
|
|
123
|
+
if downloaded > MAX_PDF_ATTACHMENT_BYTES:
|
|
124
|
+
raise AttachmentError(
|
|
125
|
+
f"attachment {url!r} exceeds the {MAX_PDF_ATTACHMENT_MIB} MiB PDF limit"
|
|
126
|
+
)
|
|
127
|
+
file.write(chunk)
|
|
128
|
+
progress.update(task, completed=downloaded)
|
|
129
|
+
except AttachmentError:
|
|
130
|
+
raise
|
|
131
|
+
except (OSError, requests.RequestException) as error:
|
|
132
|
+
raise AttachmentError(f"attachment {url!r} could not be downloaded: {error}") from error
|
|
133
|
+
|
|
134
|
+
progress.console.print(
|
|
135
|
+
f"[bold green]✓[/bold green] Downloaded {escape(filename)} "
|
|
136
|
+
f"({filesize.decimal(downloaded)})",
|
|
137
|
+
highlight=False,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _local_pdfs(configured_path: str) -> list[Path]:
|
|
142
|
+
"""Resolve a local file or recursively select directory PDFs in path order."""
|
|
143
|
+
expanded = os.path.expandvars(os.path.expanduser(configured_path))
|
|
144
|
+
path = Path(os.path.abspath(expanded))
|
|
145
|
+
if path.is_file():
|
|
146
|
+
return [path]
|
|
147
|
+
if path.is_dir():
|
|
148
|
+
pdfs = sorted(
|
|
149
|
+
(
|
|
150
|
+
candidate
|
|
151
|
+
for candidate in path.rglob("*")
|
|
152
|
+
if candidate.is_file() and candidate.suffix.lower() == ".pdf"
|
|
153
|
+
),
|
|
154
|
+
key=lambda candidate: candidate.relative_to(path).as_posix(),
|
|
155
|
+
)
|
|
156
|
+
if pdfs:
|
|
157
|
+
return pdfs
|
|
158
|
+
raise AttachmentError(f"attachment directory {configured_path!r} contains no PDF files")
|
|
159
|
+
raise AttachmentError(f"attachment path {configured_path!r} does not exist")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@contextlib.contextmanager
|
|
163
|
+
def prepare_attachments(configured_sources: list[str]) -> Iterator[list[ArchiveEntry]]:
|
|
164
|
+
"""Yield validated PDF archive entries and clean up temporary downloads.
|
|
165
|
+
|
|
166
|
+
Sources have already passed syntax validation in PromptConfig.
|
|
167
|
+
Local files are read in place and left unchanged. All source failures are
|
|
168
|
+
reported together before yielding entries to the uploader.
|
|
169
|
+
"""
|
|
170
|
+
if not configured_sources:
|
|
171
|
+
yield []
|
|
172
|
+
return
|
|
173
|
+
|
|
174
|
+
with tempfile.TemporaryDirectory(prefix="hive-attachments-") as temp_dir:
|
|
175
|
+
temp_path = Path(temp_dir)
|
|
176
|
+
entries: list[ArchiveEntry] = []
|
|
177
|
+
used_names: set[str] = set()
|
|
178
|
+
failures: list[str] = []
|
|
179
|
+
|
|
180
|
+
for source_index, configured_source in enumerate(configured_sources):
|
|
181
|
+
try:
|
|
182
|
+
if configured_source.lower().startswith(("http://", "https://")):
|
|
183
|
+
downloaded_path = temp_path / f"download-{source_index}.pdf"
|
|
184
|
+
_download_pdf(configured_source, downloaded_path)
|
|
185
|
+
source_paths = [downloaded_path]
|
|
186
|
+
source_names = [unquote(urlparse(configured_source).path)]
|
|
187
|
+
validation_sources = [configured_source]
|
|
188
|
+
else:
|
|
189
|
+
source_paths = _local_pdfs(configured_source)
|
|
190
|
+
source_names = [path.name for path in source_paths]
|
|
191
|
+
expanded = Path(
|
|
192
|
+
os.path.abspath(os.path.expandvars(os.path.expanduser(configured_source)))
|
|
193
|
+
)
|
|
194
|
+
if expanded.is_dir():
|
|
195
|
+
validation_sources = [
|
|
196
|
+
f"{configured_source.rstrip('/')}/{path.relative_to(expanded).as_posix()}"
|
|
197
|
+
for path in source_paths
|
|
198
|
+
]
|
|
199
|
+
else:
|
|
200
|
+
validation_sources = [configured_source]
|
|
201
|
+
except AttachmentError as error:
|
|
202
|
+
failures.append(str(error))
|
|
203
|
+
continue
|
|
204
|
+
|
|
205
|
+
for source_path, source_name, validation_source in zip(
|
|
206
|
+
source_paths, source_names, validation_sources, strict=True
|
|
207
|
+
):
|
|
208
|
+
try:
|
|
209
|
+
_validate_pdf(source_path, validation_source)
|
|
210
|
+
except AttachmentError as error:
|
|
211
|
+
failures.append(str(error))
|
|
212
|
+
continue
|
|
213
|
+
filename = _allocate_filename(source_name, used_names)
|
|
214
|
+
entries.append(
|
|
215
|
+
ArchiveEntry(
|
|
216
|
+
source_path=str(source_path),
|
|
217
|
+
archive_name=f"{RESOURCES_DIR}/{filename}",
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
if failures:
|
|
222
|
+
details = "\n".join(f" - {failure}" for failure in failures)
|
|
223
|
+
raise AttachmentError(f"The following attachments could not be prepared:\n{details}")
|
|
224
|
+
|
|
225
|
+
yield entries
|
|
@@ -29,14 +29,6 @@ def get_config_path() -> str:
|
|
|
29
29
|
return os.path.join(get_config_dir(), "config.yaml")
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def get_coordinator_overlay_path() -> str:
|
|
33
|
-
"""
|
|
34
|
-
Return the path of the file ``hivectl generate config`` records the
|
|
35
|
-
coordinator image tag in, which experiment configs fall back to.
|
|
36
|
-
"""
|
|
37
|
-
return os.path.join(get_config_dir(), "coordinator.yaml")
|
|
38
|
-
|
|
39
|
-
|
|
40
32
|
def load_organization_id() -> str:
|
|
41
33
|
"""Load the organization ID from ~/.hive/config.yaml."""
|
|
42
34
|
config_path = get_config_path()
|