world-model-optimizer 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_waterfall/LICENSE +21 -0
- llm_waterfall/__init__.py +53 -0
- llm_waterfall/adapters/__init__.py +36 -0
- llm_waterfall/adapters/anthropic.py +105 -0
- llm_waterfall/adapters/aws_mantle.py +47 -0
- llm_waterfall/adapters/azure_openai.py +71 -0
- llm_waterfall/adapters/base.py +51 -0
- llm_waterfall/adapters/bedrock.py +309 -0
- llm_waterfall/adapters/openai.py +130 -0
- llm_waterfall/classify.py +184 -0
- llm_waterfall/pricing.py +110 -0
- llm_waterfall/py.typed +0 -0
- llm_waterfall/types.py +295 -0
- llm_waterfall/waterfall.py +255 -0
- wmo/__init__.py +38 -0
- wmo/agents/__init__.py +7 -0
- wmo/agents/default.py +29 -0
- wmo/agents/meta.py +55 -0
- wmo/agents/optimizer.py +55 -0
- wmo/agents/project.py +928 -0
- wmo/cli/__init__.py +5 -0
- wmo/cli/agent_session.py +1123 -0
- wmo/cli/app.py +2489 -0
- wmo/cli/e2b_cmds.py +212 -0
- wmo/cli/eval_closed_loop.py +207 -0
- wmo/cli/harness_app.py +1147 -0
- wmo/cli/harness_distill.py +659 -0
- wmo/cli/hosted_session.py +880 -0
- wmo/cli/ingest_cmd.py +165 -0
- wmo/cli/model_roles.py +82 -0
- wmo/cli/platform_cmds.py +372 -0
- wmo/cli/route_app.py +274 -0
- wmo/cli/session_state.py +243 -0
- wmo/cli/ui.py +1107 -0
- wmo/cli/workspace_sync.py +504 -0
- wmo/config/__init__.py +60 -0
- wmo/config/card.py +129 -0
- wmo/config/config.py +367 -0
- wmo/config/dotenv.py +67 -0
- wmo/config/settings.py +128 -0
- wmo/config/store.py +177 -0
- wmo/conftest.py +19 -0
- wmo/connect/__init__.py +88 -0
- wmo/connect/apps.py +78 -0
- wmo/connect/brave.py +284 -0
- wmo/connect/connector.py +79 -0
- wmo/connect/credentials.py +164 -0
- wmo/connect/github.py +321 -0
- wmo/connect/google.py +627 -0
- wmo/connect/notion.py +790 -0
- wmo/connect/oauth.py +461 -0
- wmo/connect/slack.py +555 -0
- wmo/connect/store.py +199 -0
- wmo/connect/types.py +156 -0
- wmo/core/__init__.py +21 -0
- wmo/core/parsing.py +281 -0
- wmo/core/render.py +271 -0
- wmo/core/text.py +40 -0
- wmo/core/types.py +116 -0
- wmo/distill/__init__.py +14 -0
- wmo/distill/agents.py +140 -0
- wmo/distill/config.py +1006 -0
- wmo/distill/cost.py +437 -0
- wmo/distill/data.py +921 -0
- wmo/distill/deadlines.py +254 -0
- wmo/distill/fake_tinker.py +734 -0
- wmo/distill/gate.py +122 -0
- wmo/distill/loop.py +3499 -0
- wmo/distill/renderers.py +399 -0
- wmo/distill/rendering.py +620 -0
- wmo/distill/rollouts.py +726 -0
- wmo/distill/samples.py +195 -0
- wmo/distill/store.py +829 -0
- wmo/distill/teacher.py +714 -0
- wmo/distill/tokens.py +535 -0
- wmo/distill/tracking.py +552 -0
- wmo/distill/tripwire.py +411 -0
- wmo/distill/xtoken/byte_offsets.py +152 -0
- wmo/distill/xtoken/chunks.py +457 -0
- wmo/distill/xtoken/prompt_logprobs.py +475 -0
- wmo/distill/xtoken/teacher_render.py +346 -0
- wmo/engine/__init__.py +28 -0
- wmo/engine/autoconfig.py +367 -0
- wmo/engine/build.py +346 -0
- wmo/engine/demo.py +77 -0
- wmo/engine/eval_suites.py +245 -0
- wmo/engine/grounding.py +491 -0
- wmo/engine/knowledge.py +291 -0
- wmo/engine/loader.py +36 -0
- wmo/engine/play.py +92 -0
- wmo/engine/prompts.py +99 -0
- wmo/engine/replay.py +443 -0
- wmo/engine/reporting.py +58 -0
- wmo/engine/workspace.py +468 -0
- wmo/engine/world_model.py +568 -0
- wmo/env/__init__.py +22 -0
- wmo/env/base.py +121 -0
- wmo/env/closed_loop.py +229 -0
- wmo/env/episode.py +107 -0
- wmo/env/llm_agent.py +93 -0
- wmo/env/scenarios.py +73 -0
- wmo/evals/__init__.py +52 -0
- wmo/evals/agreement.py +110 -0
- wmo/evals/base.py +45 -0
- wmo/evals/closed_loop.py +480 -0
- wmo/evals/failover.py +96 -0
- wmo/evals/gold.py +127 -0
- wmo/evals/grid.py +394 -0
- wmo/evals/grid_plot.py +205 -0
- wmo/evals/harbor/__init__.py +27 -0
- wmo/evals/harbor/agent.py +573 -0
- wmo/evals/harbor/ctrf.py +171 -0
- wmo/evals/harbor/e2b_environment.py +587 -0
- wmo/evals/harbor/e2b_template_policy.py +144 -0
- wmo/evals/harbor/scorer.py +875 -0
- wmo/evals/harbor/tasks.py +140 -0
- wmo/evals/open_loop.py +194 -0
- wmo/evals/tasks.py +53 -0
- wmo/harness/__init__.py +51 -0
- wmo/harness/code_runtime.py +288 -0
- wmo/harness/create.py +1191 -0
- wmo/harness/delta.py +220 -0
- wmo/harness/doc.py +556 -0
- wmo/harness/e2b_ledger.py +342 -0
- wmo/harness/e2b_reap.py +476 -0
- wmo/harness/e2b_sandbox.py +350 -0
- wmo/harness/environment.py +35 -0
- wmo/harness/live_session.py +543 -0
- wmo/harness/mutate.py +343 -0
- wmo/harness/pi_e2b.py +1710 -0
- wmo/harness/pi_entry/entry.ts +268 -0
- wmo/harness/pi_entry/runner_frames.ts +92 -0
- wmo/harness/pi_entry/runner_live.ts +587 -0
- wmo/harness/pi_entry/runner_service.ts +270 -0
- wmo/harness/pi_entry/runner_stdio.ts +374 -0
- wmo/harness/pi_entry/runner_termination.ts +142 -0
- wmo/harness/pi_local.py +262 -0
- wmo/harness/pi_runtime.py +495 -0
- wmo/harness/pi_vendor.py +65 -0
- wmo/harness/population.py +509 -0
- wmo/harness/project_proposer.py +569 -0
- wmo/harness/proposer.py +977 -0
- wmo/harness/runner_link.py +619 -0
- wmo/harness/runtime.py +389 -0
- wmo/harness/scoring.py +247 -0
- wmo/harness/skills.py +116 -0
- wmo/harness/source_tree.py +319 -0
- wmo/harness/store.py +176 -0
- wmo/harness/tools.py +105 -0
- wmo/harness/vendor/manifest.sha256 +58 -0
- wmo/harness/vendor/pi-agent/CHANGELOG.md +556 -0
- wmo/harness/vendor/pi-agent/LICENSE +21 -0
- wmo/harness/vendor/pi-agent/README.md +488 -0
- wmo/harness/vendor/pi-agent/VENDOR.md +39 -0
- wmo/harness/vendor/pi-agent/docs/agent-harness.md +486 -0
- wmo/harness/vendor/pi-agent/docs/durable-harness.md +212 -0
- wmo/harness/vendor/pi-agent/docs/hooks.md +445 -0
- wmo/harness/vendor/pi-agent/docs/models.md +966 -0
- wmo/harness/vendor/pi-agent/docs/observability.md +376 -0
- wmo/harness/vendor/pi-agent/package.json +60 -0
- wmo/harness/vendor/pi-agent/src/agent-loop.ts +748 -0
- wmo/harness/vendor/pi-agent/src/agent.ts +575 -0
- wmo/harness/vendor/pi-agent/src/harness/agent-harness.ts +1029 -0
- wmo/harness/vendor/pi-agent/src/harness/compaction/branch-summarization.ts +261 -0
- wmo/harness/vendor/pi-agent/src/harness/compaction/compaction.ts +747 -0
- wmo/harness/vendor/pi-agent/src/harness/compaction/utils.ts +144 -0
- wmo/harness/vendor/pi-agent/src/harness/env/nodejs.ts +550 -0
- wmo/harness/vendor/pi-agent/src/harness/messages.ts +164 -0
- wmo/harness/vendor/pi-agent/src/harness/prompt-templates.ts +267 -0
- wmo/harness/vendor/pi-agent/src/harness/session/jsonl-repo.ts +177 -0
- wmo/harness/vendor/pi-agent/src/harness/session/jsonl-storage.ts +293 -0
- wmo/harness/vendor/pi-agent/src/harness/session/memory-repo.ts +50 -0
- wmo/harness/vendor/pi-agent/src/harness/session/memory-storage.ts +131 -0
- wmo/harness/vendor/pi-agent/src/harness/session/repo-utils.ts +51 -0
- wmo/harness/vendor/pi-agent/src/harness/session/session.ts +267 -0
- wmo/harness/vendor/pi-agent/src/harness/session/uuid.ts +54 -0
- wmo/harness/vendor/pi-agent/src/harness/skills.ts +375 -0
- wmo/harness/vendor/pi-agent/src/harness/system-prompt.ts +34 -0
- wmo/harness/vendor/pi-agent/src/harness/types.ts +836 -0
- wmo/harness/vendor/pi-agent/src/harness/utils/shell-output.ts +135 -0
- wmo/harness/vendor/pi-agent/src/harness/utils/truncate.ts +344 -0
- wmo/harness/vendor/pi-agent/src/index.ts +44 -0
- wmo/harness/vendor/pi-agent/src/node.ts +2 -0
- wmo/harness/vendor/pi-agent/src/proxy.ts +367 -0
- wmo/harness/vendor/pi-agent/src/types.ts +428 -0
- wmo/harness/vendor/pi-agent/test/agent-loop.test.ts +1351 -0
- wmo/harness/vendor/pi-agent/test/agent.test.ts +699 -0
- wmo/harness/vendor/pi-agent/test/e2e.test.ts +404 -0
- wmo/harness/vendor/pi-agent/test/harness/agent-harness-stream.test.ts +213 -0
- wmo/harness/vendor/pi-agent/test/harness/agent-harness.test.ts +608 -0
- wmo/harness/vendor/pi-agent/test/harness/compaction.test.ts +655 -0
- wmo/harness/vendor/pi-agent/test/harness/nodejs-env.test.ts +321 -0
- wmo/harness/vendor/pi-agent/test/harness/prompt-templates.test.ts +90 -0
- wmo/harness/vendor/pi-agent/test/harness/repo.test.ts +68 -0
- wmo/harness/vendor/pi-agent/test/harness/resource-formatting.test.ts +24 -0
- wmo/harness/vendor/pi-agent/test/harness/session-test-utils.ts +55 -0
- wmo/harness/vendor/pi-agent/test/harness/session-uuid.test.ts +50 -0
- wmo/harness/vendor/pi-agent/test/harness/session.test.ts +156 -0
- wmo/harness/vendor/pi-agent/test/harness/skills.test.ts +116 -0
- wmo/harness/vendor/pi-agent/test/harness/storage.test.ts +299 -0
- wmo/harness/vendor/pi-agent/test/harness/system-prompt.test.ts +66 -0
- wmo/harness/vendor/pi-agent/test/harness/truncate.test.ts +169 -0
- wmo/harness/vendor/pi-agent/test/scratch/simple.ts +72 -0
- wmo/harness/vendor/pi-agent/test/utils/calculate.ts +32 -0
- wmo/harness/vendor/pi-agent/test/utils/get-current-time.ts +46 -0
- wmo/harness/vendor/pi-agent/tsconfig.build.json +13 -0
- wmo/harness/vendor/pi-agent/vitest.config.ts +19 -0
- wmo/harness/vendor/pi-agent/vitest.harness.config.ts +28 -0
- wmo/harness/vendor/vendor_pi.sh +59 -0
- wmo/harness/workspace_patch.py +270 -0
- wmo/ingest/__init__.py +47 -0
- wmo/ingest/adapter.py +72 -0
- wmo/ingest/base.py +114 -0
- wmo/ingest/braintrust.py +339 -0
- wmo/ingest/detect.py +126 -0
- wmo/ingest/langfuse.py +291 -0
- wmo/ingest/langsmith.py +444 -0
- wmo/ingest/mastra.py +330 -0
- wmo/ingest/messages.py +170 -0
- wmo/ingest/normalize.py +679 -0
- wmo/ingest/otel_genai.py +69 -0
- wmo/ingest/otel_writer.py +100 -0
- wmo/ingest/phoenix.py +150 -0
- wmo/ingest/postgres.py +246 -0
- wmo/ingest/posthog.py +320 -0
- wmo/ingest/quality.py +28 -0
- wmo/ingest/stream.py +209 -0
- wmo/ingest/testdata/sample_otlp.json +60 -0
- wmo/ingest/testdata/sample_spans.jsonl +3 -0
- wmo/optimize/__init__.py +25 -0
- wmo/optimize/base.py +143 -0
- wmo/optimize/gepa.py +806 -0
- wmo/optimize/judge.py +262 -0
- wmo/optimize/judge_quality.py +359 -0
- wmo/optimize/knn.py +468 -0
- wmo/optimize/numeric.py +152 -0
- wmo/optimize/outcomes.py +103 -0
- wmo/optimize/policy.py +669 -0
- wmo/optimize/report.py +231 -0
- wmo/optimize/reward.py +129 -0
- wmo/optimize/routing.py +373 -0
- wmo/platform/__init__.py +6 -0
- wmo/platform/auth.py +115 -0
- wmo/platform/client.py +551 -0
- wmo/platform/credentials.py +126 -0
- wmo/platform/transfer.py +158 -0
- wmo/providers/__init__.py +40 -0
- wmo/providers/_bedrock_chat.py +155 -0
- wmo/providers/_openai_common.py +182 -0
- wmo/providers/_responses_common.py +472 -0
- wmo/providers/anthropic.py +134 -0
- wmo/providers/azure_openai.py +296 -0
- wmo/providers/base.py +300 -0
- wmo/providers/bedrock.py +312 -0
- wmo/providers/models.py +205 -0
- wmo/providers/openai.py +143 -0
- wmo/providers/openai_responses.py +240 -0
- wmo/providers/pool.py +170 -0
- wmo/providers/registry.py +73 -0
- wmo/providers/retry.py +151 -0
- wmo/providers/tinker.py +936 -0
- wmo/providers/waterfall.py +336 -0
- wmo/research/__init__.py +81 -0
- wmo/research/ablation.py +133 -0
- wmo/research/concurrency_plot.py +523 -0
- wmo/research/concurrency_run.py +240 -0
- wmo/research/concurrency_scaling.py +270 -0
- wmo/research/gepa_scaling.py +274 -0
- wmo/research/pipeline.py +198 -0
- wmo/research/scaling_split.py +82 -0
- wmo/research/scenario_fidelity.py +198 -0
- wmo/research/scenario_recovery.py +92 -0
- wmo/research/seed_stability.py +90 -0
- wmo/research/trace_scaling.py +348 -0
- wmo/retrieval/__init__.py +6 -0
- wmo/retrieval/embedders.py +105 -0
- wmo/retrieval/leakfree.py +52 -0
- wmo/retrieval/retriever.py +173 -0
- wmo/scenarios/__init__.py +58 -0
- wmo/scenarios/builder.py +152 -0
- wmo/scenarios/mining/__init__.py +27 -0
- wmo/scenarios/mining/clustering.py +171 -0
- wmo/scenarios/mining/facets.py +226 -0
- wmo/scenarios/mining/selection.py +220 -0
- wmo/scenarios/synthesis/__init__.py +6 -0
- wmo/scenarios/synthesis/scenario_set.py +63 -0
- wmo/scenarios/synthesis/synthesizer.py +85 -0
- wmo/scenarios/verification/__init__.py +17 -0
- wmo/scenarios/verification/judge.py +97 -0
- wmo/scenarios/verification/verify.py +135 -0
- wmo/serving/__init__.py +5 -0
- wmo/serving/builds.py +451 -0
- wmo/serving/chat.py +878 -0
- wmo/serving/endpoint_config.py +64 -0
- wmo/serving/savings.py +250 -0
- wmo/serving/server.py +553 -0
- wmo/serving/traces_source.py +206 -0
- wmo/telemetry.py +213 -0
- wmo/tracking/__init__.py +36 -0
- wmo/tracking/clock.py +24 -0
- wmo/tracking/metered.py +125 -0
- wmo/tracking/pricing.py +99 -0
- wmo/tracking/store.py +31 -0
- wmo/tracking/tracker.py +149 -0
- world_model_optimizer-0.2.0.dist-info/METADATA +203 -0
- world_model_optimizer-0.2.0.dist-info/RECORD +308 -0
- world_model_optimizer-0.2.0.dist-info/WHEEL +4 -0
- world_model_optimizer-0.2.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,504 @@
|
|
|
1
|
+
# Copyright (c) 2026 Experiential Labs. All rights reserved.
|
|
2
|
+
|
|
3
|
+
"""Safe local snapshot and three-way sync for hosted E2B agent workspaces."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import contextlib
|
|
8
|
+
import hashlib
|
|
9
|
+
import io
|
|
10
|
+
import os
|
|
11
|
+
import shutil
|
|
12
|
+
import stat
|
|
13
|
+
import tarfile
|
|
14
|
+
import tempfile
|
|
15
|
+
import uuid
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from pathlib import Path, PurePosixPath
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
|
|
20
|
+
from wmo.harness.workspace_patch import PatchFileState, parse_workspace_patch
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from collections.abc import Iterable
|
|
24
|
+
|
|
25
|
+
MAX_WORKSPACE_ARCHIVE_BYTES = 50 * 1024 * 1024
|
|
26
|
+
MAX_WORKSPACE_UNPACKED_BYTES = 512 * 1024 * 1024
|
|
27
|
+
MAX_WORKSPACE_ENTRIES = 100_000
|
|
28
|
+
|
|
29
|
+
# Dependency trees, VCS internals, caches, and WMO recovery artifacts are not
|
|
30
|
+
# useful source inputs and can turn a small repository into a multi-GB upload.
|
|
31
|
+
EXCLUDED_DIRECTORY_NAMES = frozenset(
|
|
32
|
+
{
|
|
33
|
+
".cache",
|
|
34
|
+
".git",
|
|
35
|
+
".mypy_cache",
|
|
36
|
+
".pytest_cache",
|
|
37
|
+
".ruff_cache",
|
|
38
|
+
".tox",
|
|
39
|
+
".venv",
|
|
40
|
+
".wmo-conflicts",
|
|
41
|
+
"__pycache__",
|
|
42
|
+
"node_modules",
|
|
43
|
+
"venv",
|
|
44
|
+
}
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class WorkspaceSyncError(RuntimeError):
|
|
49
|
+
"""A workspace cannot be safely archived or synchronized."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class FileState:
|
|
54
|
+
"""Content and executable-mode identity used by the three-way merge."""
|
|
55
|
+
|
|
56
|
+
sha256: str
|
|
57
|
+
mode: int
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class WorkspaceSnapshot:
|
|
62
|
+
"""Initial upload archive plus its regular-file manifest."""
|
|
63
|
+
|
|
64
|
+
archive: bytes
|
|
65
|
+
files: dict[str, FileState]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class SyncResult:
|
|
70
|
+
"""Paths applied automatically and paths preserved as local conflicts."""
|
|
71
|
+
|
|
72
|
+
applied: tuple[str, ...]
|
|
73
|
+
conflicts: tuple[str, ...]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def snapshot_workspace(root: Path) -> WorkspaceSnapshot:
|
|
77
|
+
"""Archive regular files under ``root`` and capture their initial identities."""
|
|
78
|
+
resolved = root.resolve()
|
|
79
|
+
files = _manifest(resolved)
|
|
80
|
+
try:
|
|
81
|
+
total = sum(path.stat().st_size for path in _paths_for_manifest(resolved, files))
|
|
82
|
+
except OSError as error:
|
|
83
|
+
msg = "workspace changed while it was being inspected; retry the run"
|
|
84
|
+
raise WorkspaceSyncError(msg) from error
|
|
85
|
+
if total > MAX_WORKSPACE_UNPACKED_BYTES:
|
|
86
|
+
msg = f"workspace files exceed {MAX_WORKSPACE_UNPACKED_BYTES} uncompressed bytes"
|
|
87
|
+
raise WorkspaceSyncError(msg)
|
|
88
|
+
buffer = io.BytesIO()
|
|
89
|
+
try:
|
|
90
|
+
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
|
|
91
|
+
for relative in sorted(files):
|
|
92
|
+
source = resolved / relative
|
|
93
|
+
info = tarfile.TarInfo(relative)
|
|
94
|
+
file_stat = source.stat()
|
|
95
|
+
info.size = file_stat.st_size
|
|
96
|
+
info.mode = stat.S_IMODE(file_stat.st_mode)
|
|
97
|
+
info.mtime = int(file_stat.st_mtime)
|
|
98
|
+
with source.open("rb") as handle:
|
|
99
|
+
archive.addfile(info, handle)
|
|
100
|
+
except OSError as error:
|
|
101
|
+
msg = "workspace changed while it was being archived; retry the run"
|
|
102
|
+
raise WorkspaceSyncError(msg) from error
|
|
103
|
+
if _manifest(resolved) != files:
|
|
104
|
+
msg = "workspace changed while it was being archived; retry the run"
|
|
105
|
+
raise WorkspaceSyncError(msg)
|
|
106
|
+
content = buffer.getvalue()
|
|
107
|
+
if len(content) > MAX_WORKSPACE_ARCHIVE_BYTES:
|
|
108
|
+
msg = f"workspace archive exceeds {MAX_WORKSPACE_ARCHIVE_BYTES} compressed bytes"
|
|
109
|
+
raise WorkspaceSyncError(msg)
|
|
110
|
+
return WorkspaceSnapshot(archive=content, files=files)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def snapshot_from_archive(content: bytes) -> WorkspaceSnapshot:
|
|
114
|
+
"""Rehydrate a snapshot (archive plus manifest) from persisted archive bytes.
|
|
115
|
+
|
|
116
|
+
Detached sessions persist their last synchronized archive between CLI
|
|
117
|
+
invocations; the manifest is recomputed from the archive itself so the
|
|
118
|
+
checkpoint has a single source of truth.
|
|
119
|
+
|
|
120
|
+
Raises:
|
|
121
|
+
WorkspaceSyncError: If the bytes are not a safe regular-file archive.
|
|
122
|
+
"""
|
|
123
|
+
files = {
|
|
124
|
+
path: FileState(
|
|
125
|
+
sha256=hashlib.sha256(body, usedforsecurity=False).hexdigest(),
|
|
126
|
+
mode=stat.S_IMODE(mode),
|
|
127
|
+
)
|
|
128
|
+
for path, (body, mode) in _archive_files(content).items()
|
|
129
|
+
}
|
|
130
|
+
return WorkspaceSnapshot(archive=content, files=files)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def apply_patch_to_snapshot(
|
|
134
|
+
snapshot: WorkspaceSnapshot, content: bytes, *, conflicts: Iterable[str] = ()
|
|
135
|
+
) -> WorkspaceSnapshot:
|
|
136
|
+
"""Advance a synchronized base by the patch operations that landed locally.
|
|
137
|
+
|
|
138
|
+
The base must reflect only synchronized content. Re-reading the local
|
|
139
|
+
directory here would absorb not-yet-uploaded local edits into the base,
|
|
140
|
+
so they would never upload and the final sync could even delete them.
|
|
141
|
+
Conflicted paths keep their base state: the disagreement stays visible to
|
|
142
|
+
both sides until the terminal reconciliation.
|
|
143
|
+
|
|
144
|
+
Raises:
|
|
145
|
+
WorkspaceSyncError: If the patch or the rebuilt base violates limits.
|
|
146
|
+
"""
|
|
147
|
+
try:
|
|
148
|
+
patch = parse_workspace_patch(content)
|
|
149
|
+
except Exception as error: # noqa: BLE001 - normalize to the sync error surface
|
|
150
|
+
raise WorkspaceSyncError(f"workspace patch is invalid: {error}") from error
|
|
151
|
+
rejected = set(conflicts)
|
|
152
|
+
files = _archive_files(snapshot.archive)
|
|
153
|
+
for operation in patch.operations:
|
|
154
|
+
if operation.path in rejected:
|
|
155
|
+
continue
|
|
156
|
+
if operation.after is None:
|
|
157
|
+
files.pop(operation.path, None)
|
|
158
|
+
else:
|
|
159
|
+
files[operation.path] = (patch.files[operation.path], operation.after.mode)
|
|
160
|
+
return _snapshot_from_files(files)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def advance_snapshot_paths(
|
|
164
|
+
base: WorkspaceSnapshot, current: WorkspaceSnapshot, paths: Iterable[str]
|
|
165
|
+
) -> WorkspaceSnapshot:
|
|
166
|
+
"""Advance only ``paths`` of the base to their state in ``current``.
|
|
167
|
+
|
|
168
|
+
A partially-accepted upload moves the sandbox for the accepted paths only.
|
|
169
|
+
Advancing the whole base would hide the rejected paths' divergence, while
|
|
170
|
+
advancing nothing re-pushes the accepted paths against a stale base later,
|
|
171
|
+
which can manufacture conflicts if they change again locally in between.
|
|
172
|
+
"""
|
|
173
|
+
files = _archive_files(base.archive)
|
|
174
|
+
replacements = _archive_files(current.archive)
|
|
175
|
+
for path in paths:
|
|
176
|
+
if path in replacements:
|
|
177
|
+
files[path] = replacements[path]
|
|
178
|
+
else:
|
|
179
|
+
files.pop(path, None)
|
|
180
|
+
return _snapshot_from_files(files)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _archive_files(content: bytes) -> dict[str, tuple[bytes, int]]:
|
|
184
|
+
"""Read a checkpoint archive's regular files as ``{path: (bytes, mode)}``."""
|
|
185
|
+
if len(content) > MAX_WORKSPACE_ARCHIVE_BYTES:
|
|
186
|
+
msg = f"workspace archive exceeds {MAX_WORKSPACE_ARCHIVE_BYTES} compressed bytes"
|
|
187
|
+
raise WorkspaceSyncError(msg)
|
|
188
|
+
files: dict[str, tuple[bytes, int]] = {}
|
|
189
|
+
total = 0
|
|
190
|
+
try:
|
|
191
|
+
with tarfile.open(fileobj=io.BytesIO(content), mode="r:gz") as archive:
|
|
192
|
+
members = archive.getmembers()
|
|
193
|
+
if len(members) > MAX_WORKSPACE_ENTRIES:
|
|
194
|
+
msg = f"workspace archive has more than {MAX_WORKSPACE_ENTRIES} entries"
|
|
195
|
+
raise WorkspaceSyncError(msg)
|
|
196
|
+
for member in members:
|
|
197
|
+
relative = _normalized_name(member.name)
|
|
198
|
+
if member.isdir():
|
|
199
|
+
continue
|
|
200
|
+
if not member.isfile():
|
|
201
|
+
msg = f"workspace entry must be a regular file or directory: {member.name}"
|
|
202
|
+
raise WorkspaceSyncError(msg)
|
|
203
|
+
if relative in files:
|
|
204
|
+
raise WorkspaceSyncError(f"duplicate workspace path: {relative}")
|
|
205
|
+
total += member.size
|
|
206
|
+
if total > MAX_WORKSPACE_UNPACKED_BYTES:
|
|
207
|
+
msg = f"workspace expands beyond {MAX_WORKSPACE_UNPACKED_BYTES} bytes"
|
|
208
|
+
raise WorkspaceSyncError(msg)
|
|
209
|
+
source = archive.extractfile(member)
|
|
210
|
+
if source is None:
|
|
211
|
+
raise WorkspaceSyncError(f"workspace file has no content: {member.name}")
|
|
212
|
+
with source:
|
|
213
|
+
body = source.read()
|
|
214
|
+
files[relative] = (body, stat.S_IMODE(member.mode))
|
|
215
|
+
except WorkspaceSyncError:
|
|
216
|
+
raise
|
|
217
|
+
except (tarfile.TarError, OSError, EOFError) as error:
|
|
218
|
+
msg = "workspace must be a valid gzip tar archive"
|
|
219
|
+
raise WorkspaceSyncError(msg) from error
|
|
220
|
+
return files
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _snapshot_from_files(files: dict[str, tuple[bytes, int]]) -> WorkspaceSnapshot:
|
|
224
|
+
"""Build a deterministic snapshot archive plus manifest from file contents."""
|
|
225
|
+
total = sum(len(body) for body, _mode in files.values())
|
|
226
|
+
if total > MAX_WORKSPACE_UNPACKED_BYTES:
|
|
227
|
+
msg = f"workspace files exceed {MAX_WORKSPACE_UNPACKED_BYTES} uncompressed bytes"
|
|
228
|
+
raise WorkspaceSyncError(msg)
|
|
229
|
+
buffer = io.BytesIO()
|
|
230
|
+
manifest: dict[str, FileState] = {}
|
|
231
|
+
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
|
|
232
|
+
for path in sorted(files):
|
|
233
|
+
body, mode = files[path]
|
|
234
|
+
info = tarfile.TarInfo(path)
|
|
235
|
+
info.size = len(body)
|
|
236
|
+
info.mode = stat.S_IMODE(mode)
|
|
237
|
+
info.mtime = 0
|
|
238
|
+
archive.addfile(info, io.BytesIO(body))
|
|
239
|
+
manifest[path] = FileState(
|
|
240
|
+
sha256=hashlib.sha256(body, usedforsecurity=False).hexdigest(),
|
|
241
|
+
mode=stat.S_IMODE(mode),
|
|
242
|
+
)
|
|
243
|
+
content = buffer.getvalue()
|
|
244
|
+
if len(content) > MAX_WORKSPACE_ARCHIVE_BYTES:
|
|
245
|
+
msg = f"workspace archive exceeds {MAX_WORKSPACE_ARCHIVE_BYTES} compressed bytes"
|
|
246
|
+
raise WorkspaceSyncError(msg)
|
|
247
|
+
return WorkspaceSnapshot(archive=content, files=manifest)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def sync_workspace(
|
|
251
|
+
root: Path,
|
|
252
|
+
initial: WorkspaceSnapshot,
|
|
253
|
+
final_archive: bytes,
|
|
254
|
+
*,
|
|
255
|
+
protected_paths: frozenset[str] = frozenset(),
|
|
256
|
+
) -> SyncResult:
|
|
257
|
+
"""Apply remote changes unless the same path changed locally since upload."""
|
|
258
|
+
resolved = root.resolve()
|
|
259
|
+
if len(final_archive) > MAX_WORKSPACE_ARCHIVE_BYTES:
|
|
260
|
+
msg = f"final workspace archive exceeds {MAX_WORKSPACE_ARCHIVE_BYTES} bytes"
|
|
261
|
+
raise WorkspaceSyncError(msg)
|
|
262
|
+
with tempfile.TemporaryDirectory(prefix="wmo-workspace-") as staging_name:
|
|
263
|
+
staging = Path(staging_name)
|
|
264
|
+
_extract_archive(final_archive, staging)
|
|
265
|
+
remote = _manifest(staging)
|
|
266
|
+
current = _manifest(resolved)
|
|
267
|
+
applied: list[str] = []
|
|
268
|
+
conflicts: list[str] = []
|
|
269
|
+
for relative in sorted(set(initial.files) | set(remote)):
|
|
270
|
+
before = initial.files.get(relative)
|
|
271
|
+
after = remote.get(relative)
|
|
272
|
+
if before == after:
|
|
273
|
+
continue
|
|
274
|
+
target = resolved / relative
|
|
275
|
+
now = current.get(relative)
|
|
276
|
+
# A directory/link/special file occupying the path is always a
|
|
277
|
+
# conflict: the manifests cannot see it, so no equality below is
|
|
278
|
+
# trustworthy.
|
|
279
|
+
if _has_non_file_collision(target, now):
|
|
280
|
+
conflicts.append(relative)
|
|
281
|
+
continue
|
|
282
|
+
# Local and remote agreeing (content and mode) is synchronization,
|
|
283
|
+
# not a conflict, even for a path protected by an earlier live
|
|
284
|
+
# disagreement that has since reconverged.
|
|
285
|
+
if now == after:
|
|
286
|
+
continue
|
|
287
|
+
if relative in protected_paths or now != before:
|
|
288
|
+
conflicts.append(relative)
|
|
289
|
+
continue
|
|
290
|
+
try:
|
|
291
|
+
if after is None:
|
|
292
|
+
target.unlink(missing_ok=True)
|
|
293
|
+
_remove_empty_parents(target.parent, resolved)
|
|
294
|
+
else:
|
|
295
|
+
_atomic_copy(staging / relative, target, root=resolved, mode=after.mode)
|
|
296
|
+
except OSError:
|
|
297
|
+
conflicts.append(relative)
|
|
298
|
+
continue
|
|
299
|
+
applied.append(relative)
|
|
300
|
+
return SyncResult(applied=tuple(applied), conflicts=tuple(conflicts))
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def apply_workspace_patch(root: Path, content: bytes) -> SyncResult:
|
|
304
|
+
"""Apply an incremental remote patch when each local path still matches its base."""
|
|
305
|
+
resolved = root.resolve()
|
|
306
|
+
patch = parse_workspace_patch(content)
|
|
307
|
+
current = _manifest(resolved)
|
|
308
|
+
applied: list[str] = []
|
|
309
|
+
conflicts: list[str] = []
|
|
310
|
+
for operation in patch.operations:
|
|
311
|
+
target = resolved / operation.path
|
|
312
|
+
now = current.get(operation.path)
|
|
313
|
+
before = _file_state(operation.before)
|
|
314
|
+
after = _file_state(operation.after)
|
|
315
|
+
if _has_non_file_collision(target, now) or (now != before and now != after):
|
|
316
|
+
conflicts.append(operation.path)
|
|
317
|
+
continue
|
|
318
|
+
if now == after:
|
|
319
|
+
continue
|
|
320
|
+
try:
|
|
321
|
+
if after is None:
|
|
322
|
+
target.unlink(missing_ok=True)
|
|
323
|
+
_remove_empty_parents(target.parent, resolved)
|
|
324
|
+
else:
|
|
325
|
+
_atomic_write(patch.files[operation.path], target, root=resolved, mode=after.mode)
|
|
326
|
+
except OSError:
|
|
327
|
+
conflicts.append(operation.path)
|
|
328
|
+
continue
|
|
329
|
+
applied.append(operation.path)
|
|
330
|
+
return SyncResult(applied=tuple(applied), conflicts=tuple(conflicts))
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def write_conflict_archive(root: Path, session_id: str, content: bytes) -> Path:
|
|
334
|
+
"""Preserve a downloaded result archive when automatic reconciliation conflicts.
|
|
335
|
+
|
|
336
|
+
The archive is the only remaining copy of the agent's work once the
|
|
337
|
+
handoff is acknowledged, so it lands atomically: a crash mid-write leaves
|
|
338
|
+
the previous file (or nothing), never a truncated archive.
|
|
339
|
+
"""
|
|
340
|
+
directory = root.resolve() / ".wmo-conflicts"
|
|
341
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
342
|
+
path = directory / f"{session_id}.tar.gz"
|
|
343
|
+
fd, tmp_name = tempfile.mkstemp(dir=directory, prefix=f"{path.name}.")
|
|
344
|
+
try:
|
|
345
|
+
with os.fdopen(fd, "wb") as handle:
|
|
346
|
+
handle.write(content)
|
|
347
|
+
os.replace(tmp_name, path)
|
|
348
|
+
except BaseException:
|
|
349
|
+
# The replace may already have consumed the temp file; a missing file
|
|
350
|
+
# must not mask the original exception.
|
|
351
|
+
with contextlib.suppress(OSError):
|
|
352
|
+
os.unlink(tmp_name)
|
|
353
|
+
raise
|
|
354
|
+
return path
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _manifest(root: Path) -> dict[str, FileState]:
|
|
358
|
+
"""Hash regular files without following symlinks or entering excluded trees."""
|
|
359
|
+
manifest: dict[str, FileState] = {}
|
|
360
|
+
entries = 0
|
|
361
|
+
for directory, dirnames, filenames in os.walk(root, topdown=True, followlinks=False):
|
|
362
|
+
base = Path(directory)
|
|
363
|
+
dirnames[:] = sorted(
|
|
364
|
+
name
|
|
365
|
+
for name in dirnames
|
|
366
|
+
if name not in EXCLUDED_DIRECTORY_NAMES and not (base / name).is_symlink()
|
|
367
|
+
)
|
|
368
|
+
for name in sorted(filenames):
|
|
369
|
+
path = base / name
|
|
370
|
+
try:
|
|
371
|
+
file_stat = path.lstat()
|
|
372
|
+
except OSError as error:
|
|
373
|
+
raise WorkspaceSyncError(f"could not inspect workspace file: {path}") from error
|
|
374
|
+
if not stat.S_ISREG(file_stat.st_mode):
|
|
375
|
+
continue
|
|
376
|
+
relative = path.relative_to(root).as_posix()
|
|
377
|
+
manifest[relative] = FileState(
|
|
378
|
+
sha256=_sha256(path), mode=stat.S_IMODE(file_stat.st_mode)
|
|
379
|
+
)
|
|
380
|
+
entries += 1
|
|
381
|
+
if entries > MAX_WORKSPACE_ENTRIES:
|
|
382
|
+
msg = f"workspace has more than {MAX_WORKSPACE_ENTRIES} files"
|
|
383
|
+
raise WorkspaceSyncError(msg)
|
|
384
|
+
return manifest
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _paths_for_manifest(root: Path, manifest: dict[str, FileState]) -> list[Path]:
|
|
388
|
+
"""Resolve manifest paths for aggregate-size accounting."""
|
|
389
|
+
return [root / relative for relative in manifest]
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _sha256(path: Path) -> str:
|
|
393
|
+
"""Hash one regular file without loading it all into memory."""
|
|
394
|
+
digest = hashlib.sha256(usedforsecurity=False)
|
|
395
|
+
try:
|
|
396
|
+
with path.open("rb") as handle:
|
|
397
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
398
|
+
digest.update(chunk)
|
|
399
|
+
except OSError as error:
|
|
400
|
+
raise WorkspaceSyncError(f"could not read workspace file: {path}") from error
|
|
401
|
+
return digest.hexdigest()
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _extract_archive(content: bytes, destination: Path) -> None:
|
|
405
|
+
"""Validate and extract regular files only into an isolated staging directory."""
|
|
406
|
+
total_size = 0
|
|
407
|
+
seen: set[str] = set()
|
|
408
|
+
try:
|
|
409
|
+
with tarfile.open(fileobj=io.BytesIO(content), mode="r:gz") as archive:
|
|
410
|
+
members = archive.getmembers()
|
|
411
|
+
if len(members) > MAX_WORKSPACE_ENTRIES:
|
|
412
|
+
msg = f"workspace archive has more than {MAX_WORKSPACE_ENTRIES} entries"
|
|
413
|
+
raise WorkspaceSyncError(msg)
|
|
414
|
+
for member in members:
|
|
415
|
+
relative = _normalized_name(member.name)
|
|
416
|
+
if relative in seen:
|
|
417
|
+
raise WorkspaceSyncError(f"duplicate workspace path: {relative}")
|
|
418
|
+
seen.add(relative)
|
|
419
|
+
if not (member.isfile() or member.isdir()):
|
|
420
|
+
raise WorkspaceSyncError(
|
|
421
|
+
f"workspace entry must be a regular file or directory: {member.name}"
|
|
422
|
+
)
|
|
423
|
+
total_size += member.size
|
|
424
|
+
if total_size > MAX_WORKSPACE_UNPACKED_BYTES:
|
|
425
|
+
msg = f"workspace expands beyond {MAX_WORKSPACE_UNPACKED_BYTES} bytes"
|
|
426
|
+
raise WorkspaceSyncError(msg)
|
|
427
|
+
target = destination if relative == "." else destination / relative
|
|
428
|
+
if member.isdir():
|
|
429
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
430
|
+
continue
|
|
431
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
432
|
+
source = archive.extractfile(member)
|
|
433
|
+
if source is None:
|
|
434
|
+
raise WorkspaceSyncError(f"workspace file has no content: {member.name}")
|
|
435
|
+
with source, target.open("wb") as output:
|
|
436
|
+
shutil.copyfileobj(source, output)
|
|
437
|
+
target.chmod(member.mode & 0o777)
|
|
438
|
+
except WorkspaceSyncError:
|
|
439
|
+
raise
|
|
440
|
+
except (tarfile.TarError, OSError, EOFError) as error:
|
|
441
|
+
msg = "workspace must be a valid gzip tar archive"
|
|
442
|
+
raise WorkspaceSyncError(msg) from error
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _normalized_name(name: str) -> str:
|
|
446
|
+
"""Normalize one tar name and reject absolute or traversing entries."""
|
|
447
|
+
path = PurePosixPath(name)
|
|
448
|
+
if path.is_absolute() or ".." in path.parts:
|
|
449
|
+
raise WorkspaceSyncError(f"unsafe workspace path: {name}")
|
|
450
|
+
parts = tuple(part for part in path.parts if part not in {"", "."})
|
|
451
|
+
return PurePosixPath(*parts).as_posix() if parts else "."
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _has_non_file_collision(target: Path, state: FileState | None) -> bool:
|
|
455
|
+
"""Return true when a manifest-absent target still exists as a dir/link/special file."""
|
|
456
|
+
return state is None and (target.exists() or target.is_symlink())
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _atomic_copy(source: Path, target: Path, *, root: Path, mode: int) -> None:
|
|
460
|
+
"""Copy through a sibling temporary file after proving the parent stays in ``root``."""
|
|
461
|
+
relative_parent = target.parent.relative_to(root)
|
|
462
|
+
current = root
|
|
463
|
+
for part in relative_parent.parts:
|
|
464
|
+
current /= part
|
|
465
|
+
if current.is_symlink():
|
|
466
|
+
raise OSError(f"workspace path crosses a symlink: {target}")
|
|
467
|
+
if current.exists() and not current.is_dir():
|
|
468
|
+
raise OSError(f"workspace parent is not a directory: {current}")
|
|
469
|
+
current.mkdir(exist_ok=True)
|
|
470
|
+
if target.is_symlink() or (target.exists() and not target.is_file()):
|
|
471
|
+
raise OSError(f"workspace target is not a regular file: {target}")
|
|
472
|
+
temporary = target.parent / f".{target.name}.wmo-{uuid.uuid4().hex}"
|
|
473
|
+
try:
|
|
474
|
+
shutil.copyfile(source, temporary)
|
|
475
|
+
temporary.chmod(mode)
|
|
476
|
+
os.replace(temporary, target)
|
|
477
|
+
finally:
|
|
478
|
+
temporary.unlink(missing_ok=True)
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _atomic_write(content: bytes, target: Path, *, root: Path, mode: int) -> None:
|
|
482
|
+
"""Write patch bytes through the same collision-safe sibling replacement path."""
|
|
483
|
+
with tempfile.TemporaryDirectory(prefix="wmo-patch-") as staging_name:
|
|
484
|
+
source = Path(staging_name) / "content"
|
|
485
|
+
source.write_bytes(content)
|
|
486
|
+
_atomic_copy(source, target, root=root, mode=mode)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _file_state(state: PatchFileState | None) -> FileState | None:
|
|
490
|
+
"""Translate the shared transport state into the CLI merge state."""
|
|
491
|
+
if state is None:
|
|
492
|
+
return None
|
|
493
|
+
return FileState(sha256=state.sha256, mode=state.mode)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _remove_empty_parents(directory: Path, root: Path) -> None:
|
|
497
|
+
"""Remove newly empty parents after a remote deletion, stopping at the sync root."""
|
|
498
|
+
current = directory
|
|
499
|
+
while current != root:
|
|
500
|
+
try:
|
|
501
|
+
current.rmdir()
|
|
502
|
+
except OSError:
|
|
503
|
+
return
|
|
504
|
+
current = current.parent
|
wmo/config/__init__.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Project config + the `.wmo/` artifact layout."""
|
|
2
|
+
|
|
3
|
+
from wmo.config.config import (
|
|
4
|
+
ARTIFACT_DIR,
|
|
5
|
+
FIDELITY_TIERS,
|
|
6
|
+
PROVIDER_ENV_VARS,
|
|
7
|
+
ArtifactPaths,
|
|
8
|
+
FidelityTier,
|
|
9
|
+
HarnessConfig,
|
|
10
|
+
TierSpec,
|
|
11
|
+
load_config,
|
|
12
|
+
save_config,
|
|
13
|
+
)
|
|
14
|
+
from wmo.config.dotenv import load_env_file, upsert_env_var
|
|
15
|
+
from wmo.config.settings import (
|
|
16
|
+
ModelRole,
|
|
17
|
+
ModelsSettings,
|
|
18
|
+
ProjectSettings,
|
|
19
|
+
TelemetrySettings,
|
|
20
|
+
ensure_telemetry_anonymous_id,
|
|
21
|
+
load_settings,
|
|
22
|
+
save_settings,
|
|
23
|
+
set_telemetry_enabled,
|
|
24
|
+
settings_path,
|
|
25
|
+
)
|
|
26
|
+
from wmo.config.store import (
|
|
27
|
+
DEFAULT_MODEL_NAME,
|
|
28
|
+
ModelInfo,
|
|
29
|
+
WorldModelStore,
|
|
30
|
+
normalize_name,
|
|
31
|
+
validate_name,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"ARTIFACT_DIR",
|
|
36
|
+
"DEFAULT_MODEL_NAME",
|
|
37
|
+
"FIDELITY_TIERS",
|
|
38
|
+
"FidelityTier",
|
|
39
|
+
"TierSpec",
|
|
40
|
+
"PROVIDER_ENV_VARS",
|
|
41
|
+
"ArtifactPaths",
|
|
42
|
+
"HarnessConfig",
|
|
43
|
+
"ModelInfo",
|
|
44
|
+
"ModelRole",
|
|
45
|
+
"ModelsSettings",
|
|
46
|
+
"ProjectSettings",
|
|
47
|
+
"TelemetrySettings",
|
|
48
|
+
"WorldModelStore",
|
|
49
|
+
"ensure_telemetry_anonymous_id",
|
|
50
|
+
"load_config",
|
|
51
|
+
"load_env_file",
|
|
52
|
+
"load_settings",
|
|
53
|
+
"normalize_name",
|
|
54
|
+
"save_config",
|
|
55
|
+
"save_settings",
|
|
56
|
+
"set_telemetry_enabled",
|
|
57
|
+
"settings_path",
|
|
58
|
+
"upsert_env_var",
|
|
59
|
+
"validate_name",
|
|
60
|
+
]
|
wmo/config/card.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""The v0 ModelCard: descriptive metadata a built world model carries in its artifact.
|
|
2
|
+
|
|
3
|
+
`card.json` sits in the artifact root next to `config.toml` and is what distribution surfaces
|
|
4
|
+
(the website gallery, `GET /world_models`, future `wmo export`/`pull`) render. The shape follows
|
|
5
|
+
the registry contract sketch in the coordination plan (PLAN.md §2.1); WS-A4 owns and extends it.
|
|
6
|
+
A model without a card still loads everywhere - cards are additive metadata, never required.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, Field, ValidationError
|
|
15
|
+
|
|
16
|
+
CARD_FILENAME = "card.json"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class CardCorpus(BaseModel):
|
|
20
|
+
"""Size and origin of the trace corpus the model was built from.
|
|
21
|
+
|
|
22
|
+
`traces` is optional: for a model whose build predates card support the trace count may not
|
|
23
|
+
be reconstructable, while the indexed step count always is.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
traces: int | None = None
|
|
27
|
+
steps: int
|
|
28
|
+
source: str | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class CardFidelity(BaseModel):
|
|
32
|
+
"""Headline fidelity: which eval suite produced it and the run it came from."""
|
|
33
|
+
|
|
34
|
+
suite: str
|
|
35
|
+
score: float
|
|
36
|
+
std: float | None = None
|
|
37
|
+
run_id: str | None = None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class TracesSource(BaseModel):
|
|
41
|
+
"""Where this model's trace corpus lives on the Hugging Face Hub.
|
|
42
|
+
|
|
43
|
+
The traces are the raw agent sessions (`traces.otel.jsonl`), which are large and need not be
|
|
44
|
+
committed: when they are absent locally, the serve backend fetches them from here on demand
|
|
45
|
+
over the public resolve URL (no auth, no client-side Hub API). A local copy always supersedes.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
repo: str # e.g. "experientiallabs/wmo-tau-bench"
|
|
49
|
+
path: str = "traces.otel.jsonl" # file within the repo
|
|
50
|
+
revision: str = "main"
|
|
51
|
+
kind: str = "dataset" # "dataset" or "model" repo namespace on the Hub
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class ModelCard(BaseModel):
|
|
55
|
+
"""Machine-readable description of one built world model (see module docstring)."""
|
|
56
|
+
|
|
57
|
+
schema_version: int = 1
|
|
58
|
+
name: str
|
|
59
|
+
title: str
|
|
60
|
+
description: str = ""
|
|
61
|
+
task: str | None = None
|
|
62
|
+
corpus: CardCorpus
|
|
63
|
+
provider: str
|
|
64
|
+
model_id: str
|
|
65
|
+
fidelity: CardFidelity | None = None
|
|
66
|
+
cost_per_step_usd: float | None = None
|
|
67
|
+
latency_per_step_s: float | None = None
|
|
68
|
+
built_at: str | None = None # ISO-8601 UTC
|
|
69
|
+
license: str | None = None
|
|
70
|
+
tags: list[str] = Field(default_factory=list)
|
|
71
|
+
traces_hf: TracesSource | None = None # Hub source for on-demand trace download
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def make_build_card(
|
|
75
|
+
*,
|
|
76
|
+
name: str,
|
|
77
|
+
provider: str,
|
|
78
|
+
model_id: str,
|
|
79
|
+
traces: int | None,
|
|
80
|
+
steps: int,
|
|
81
|
+
built_at: str,
|
|
82
|
+
source: str | None = None,
|
|
83
|
+
title: str = "",
|
|
84
|
+
description: str = "",
|
|
85
|
+
tags: list[str] | None = None,
|
|
86
|
+
) -> ModelCard:
|
|
87
|
+
"""Assemble the card a completed build writes.
|
|
88
|
+
|
|
89
|
+
The single card-construction site for both build paths (`wmo build` and serve-side builds),
|
|
90
|
+
so their cards never drift. `fidelity`/cost/latency stay unset here - they are stamped later
|
|
91
|
+
from eval results, not known at build time.
|
|
92
|
+
"""
|
|
93
|
+
return ModelCard(
|
|
94
|
+
name=name,
|
|
95
|
+
title=title or name,
|
|
96
|
+
description=description,
|
|
97
|
+
corpus=CardCorpus(traces=traces, steps=steps, source=source),
|
|
98
|
+
provider=provider,
|
|
99
|
+
model_id=model_id,
|
|
100
|
+
built_at=built_at,
|
|
101
|
+
tags=tags or [],
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def card_path(model_dir: str | Path) -> Path:
|
|
106
|
+
return Path(model_dir) / CARD_FILENAME
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def save_card(card: ModelCard, model_dir: str | Path) -> Path:
|
|
110
|
+
"""Write `card.json` into `model_dir`, returning the path written."""
|
|
111
|
+
path = card_path(model_dir)
|
|
112
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
113
|
+
path.write_text(card.model_dump_json(indent=2) + "\n", encoding="utf-8")
|
|
114
|
+
return path
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def load_card(model_dir: str | Path) -> ModelCard | None:
|
|
118
|
+
"""Read a model dir's `card.json`, or None when the model has no card.
|
|
119
|
+
|
|
120
|
+
A present-but-broken card raises (with the offending path) rather than silently hiding the
|
|
121
|
+
model's metadata: the card was written intentionally, so corruption is a real error.
|
|
122
|
+
"""
|
|
123
|
+
path = card_path(model_dir)
|
|
124
|
+
if not path.exists():
|
|
125
|
+
return None
|
|
126
|
+
try:
|
|
127
|
+
return ModelCard.model_validate(json.loads(path.read_text(encoding="utf-8")))
|
|
128
|
+
except (json.JSONDecodeError, ValidationError) as exc:
|
|
129
|
+
raise ValueError(f"malformed card.json at {path}: {exc}") from exc
|