deltaleague 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- delta_cli/__init__.py +0 -0
- delta_cli/artifacts.py +163 -0
- delta_cli/local.py +193 -0
- delta_cli/local_commands.py +107 -0
- delta_cli/main.py +282 -0
- delta_cli/scoring.py +35 -0
- deltaleague-0.1.0.dist-info/METADATA +83 -0
- deltaleague-0.1.0.dist-info/RECORD +11 -0
- deltaleague-0.1.0.dist-info/WHEEL +5 -0
- deltaleague-0.1.0.dist-info/entry_points.txt +2 -0
- deltaleague-0.1.0.dist-info/top_level.txt +1 -0
delta_cli/__init__.py
ADDED
|
File without changes
|
delta_cli/artifacts.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Portable weight artifact contract. This module never imports the platform."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
import re
|
|
6
|
+
import uuid
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def canonical_json(value):
|
|
11
|
+
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def file_hash(path):
|
|
15
|
+
with open(path, "rb") as f:
|
|
16
|
+
return hashlib.file_digest(f, "sha256").hexdigest()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def validate_manifest(manifest):
|
|
20
|
+
if not isinstance(manifest, dict):
|
|
21
|
+
raise ValueError("invalid_manifest")
|
|
22
|
+
if manifest.get("schema_version") != "delta.submission.v1":
|
|
23
|
+
raise ValueError("unsupported_schema")
|
|
24
|
+
if manifest.get("format") not in {"sparse-diff", "dense-diff", "lora", "full"}:
|
|
25
|
+
raise ValueError("unsupported_format")
|
|
26
|
+
if manifest.get("reconstruction_version") != "delta-reconstruct-v1":
|
|
27
|
+
raise ValueError("unsupported_reconstruction")
|
|
28
|
+
try:
|
|
29
|
+
uuid.UUID(str(manifest["parent_version_id"]))
|
|
30
|
+
except (ValueError, KeyError):
|
|
31
|
+
raise ValueError("invalid_parent_version")
|
|
32
|
+
if not re.fullmatch(r"[0-9a-f]{64}", str(manifest.get("parent_content_hash", ""))):
|
|
33
|
+
raise ValueError("invalid_parent_hash")
|
|
34
|
+
if not re.fullmatch(r"[-a-zA-Z0-9_]{1,100}", str(manifest.get("model_config_id", ""))):
|
|
35
|
+
raise ValueError("invalid_model_config")
|
|
36
|
+
files = manifest.get("artifacts", [])
|
|
37
|
+
if not isinstance(files, list) or not all(isinstance(f, dict) for f in files):
|
|
38
|
+
raise ValueError("invalid_artifact_inventory")
|
|
39
|
+
if len(files) != 1 or files[0].get("name") != "weights.safetensors":
|
|
40
|
+
raise ValueError("expected_weights_safetensors")
|
|
41
|
+
for f in files:
|
|
42
|
+
if not re.fullmatch(r"[0-9a-f]{64}", f.get("sha256", "")):
|
|
43
|
+
raise ValueError("invalid_hash")
|
|
44
|
+
if type(f.get("size_bytes")) is not int or f["size_bytes"] <= 0:
|
|
45
|
+
raise ValueError("invalid_size")
|
|
46
|
+
if manifest.get("operation", "replace") not in {"replace", "add"}:
|
|
47
|
+
raise ValueError("unsupported_operation")
|
|
48
|
+
mapping = manifest.get("tensors", {})
|
|
49
|
+
if not isinstance(mapping, dict) or not all(isinstance(k, str) and isinstance(v, dict) for k, v in mapping.items()):
|
|
50
|
+
raise ValueError("invalid_tensor_mapping")
|
|
51
|
+
for target, spec in mapping.items():
|
|
52
|
+
if len(target) > 1024:
|
|
53
|
+
raise ValueError("invalid_tensor_name")
|
|
54
|
+
required = {"a", "b", "alpha"} if manifest["format"] == "lora" else ({"indices", "values"} if manifest["format"] == "sparse-diff" else {"values"})
|
|
55
|
+
if set(spec) != required or any(not isinstance(v, str) for k, v in spec.items() if k != "alpha"):
|
|
56
|
+
raise ValueError("invalid_tensor_mapping")
|
|
57
|
+
if len(mapping) > 10000:
|
|
58
|
+
raise ValueError("too_many_tensors")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def content_hash(tensors, identity_hash):
|
|
62
|
+
import torch
|
|
63
|
+
h = hashlib.sha256(canonical_json({"schema": "delta-content-v1", "identity": identity_hash}))
|
|
64
|
+
for name, tensor in sorted(tensors.items()):
|
|
65
|
+
t = tensor.detach().cpu().contiguous()
|
|
66
|
+
header = canonical_json([name, list(t.shape), str(t.dtype)])
|
|
67
|
+
h.update(len(header).to_bytes(8, "little"))
|
|
68
|
+
h.update(header)
|
|
69
|
+
h.update(t.reshape(-1).view(torch.uint8).numpy().tobytes())
|
|
70
|
+
return h.hexdigest()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def reconstruct(parent_path, artifact_path, manifest, allowed_prefix=""):
|
|
74
|
+
import torch
|
|
75
|
+
from safetensors.torch import load_file
|
|
76
|
+
validate_manifest(manifest)
|
|
77
|
+
base = load_file(str(parent_path))
|
|
78
|
+
patch = load_file(str(artifact_path))
|
|
79
|
+
if any(not torch.isfinite(t).all() for t in patch.values()):
|
|
80
|
+
raise ValueError("nonfinite_patch")
|
|
81
|
+
result = {k: v.clone() for k, v in base.items()}
|
|
82
|
+
used = set()
|
|
83
|
+
if manifest["format"] == "full":
|
|
84
|
+
if set(patch) != set(base):
|
|
85
|
+
raise ValueError("tensor_inventory")
|
|
86
|
+
mapping = {k: {"values": k} for k in patch}
|
|
87
|
+
else:
|
|
88
|
+
mapping = manifest.get("tensors", {})
|
|
89
|
+
if not mapping:
|
|
90
|
+
raise ValueError("empty_patch")
|
|
91
|
+
for name, spec in mapping.items():
|
|
92
|
+
if name not in base:
|
|
93
|
+
raise ValueError("unknown_tensor")
|
|
94
|
+
original = base[name]
|
|
95
|
+
if not name.startswith(allowed_prefix):
|
|
96
|
+
# Full checkpoints may carry unchanged frozen tensors.
|
|
97
|
+
if manifest["format"] == "full" and torch.equal(patch[name], original):
|
|
98
|
+
used.add(name)
|
|
99
|
+
continue
|
|
100
|
+
raise ValueError("frozen_tensor")
|
|
101
|
+
kind = manifest["format"]
|
|
102
|
+
if kind == "lora":
|
|
103
|
+
a, b = patch[spec["a"]], patch[spec["b"]]
|
|
104
|
+
used.update([spec["a"], spec["b"]])
|
|
105
|
+
if a.ndim != 2 or b.ndim != 2 or original.ndim != 2 or a.shape[0] != b.shape[1] or not 1 <= a.shape[0] <= 256:
|
|
106
|
+
raise ValueError("invalid_lora_rank")
|
|
107
|
+
alpha = float(spec["alpha"])
|
|
108
|
+
if not math.isfinite(alpha) or (b.shape[0], a.shape[1]) != tuple(original.shape):
|
|
109
|
+
raise ValueError("invalid_lora_shape")
|
|
110
|
+
updated = original.float() + (b.float() @ a.float()) * (alpha / a.shape[0])
|
|
111
|
+
else:
|
|
112
|
+
values = patch[spec["values"]]
|
|
113
|
+
used.add(spec["values"])
|
|
114
|
+
if kind == "sparse-diff":
|
|
115
|
+
indices = patch[spec["indices"]]
|
|
116
|
+
used.add(spec["indices"])
|
|
117
|
+
if indices.dtype != torch.int64 or indices.ndim != 1 or values.ndim != 1 or indices.numel() != values.numel():
|
|
118
|
+
raise ValueError("invalid_sparse_shape")
|
|
119
|
+
if indices.numel() and (indices[0] < 0 or indices[-1] >= original.numel() or not (indices[1:] > indices[:-1]).all()):
|
|
120
|
+
raise ValueError("invalid_sparse_indices")
|
|
121
|
+
updated = original.float().clone().reshape(-1)
|
|
122
|
+
updated[indices] = values.float() if manifest.get("operation", "replace") == "replace" else updated[indices] + values.float()
|
|
123
|
+
updated = updated.reshape(original.shape)
|
|
124
|
+
else:
|
|
125
|
+
if values.shape != original.shape:
|
|
126
|
+
raise ValueError("tensor_shape")
|
|
127
|
+
updated = values if kind == "full" or manifest.get("operation", "replace") == "replace" else original.float() + values.float()
|
|
128
|
+
result[name] = updated.to(original.dtype).contiguous()
|
|
129
|
+
if not torch.isfinite(result[name]).all():
|
|
130
|
+
raise ValueError("nonfinite_result")
|
|
131
|
+
if used != set(patch):
|
|
132
|
+
raise ValueError("unlisted_tensors")
|
|
133
|
+
return result
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def build_sparse(parent, modified, output, parent_id, parent_hash, model_id):
|
|
137
|
+
import torch
|
|
138
|
+
from safetensors.torch import load_file, save_file
|
|
139
|
+
output = Path(output)
|
|
140
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
141
|
+
before, after = load_file(str(parent)), load_file(str(modified))
|
|
142
|
+
if set(before) != set(after):
|
|
143
|
+
raise ValueError("tensor_inventory")
|
|
144
|
+
tensors, mapping = {}, {}
|
|
145
|
+
for number, name in enumerate(sorted(before)):
|
|
146
|
+
a, b = before[name], after[name]
|
|
147
|
+
if a.shape != b.shape or a.dtype != b.dtype:
|
|
148
|
+
raise ValueError("tensor_shape_or_dtype")
|
|
149
|
+
idx = torch.nonzero(a.reshape(-1) != b.reshape(-1)).flatten()
|
|
150
|
+
if idx.numel():
|
|
151
|
+
ik, vk = f"t{number}.indices", f"t{number}.values"
|
|
152
|
+
tensors[ik], tensors[vk] = idx, b.reshape(-1)[idx]
|
|
153
|
+
mapping[name] = {"indices": ik, "values": vk}
|
|
154
|
+
if not tensors:
|
|
155
|
+
raise ValueError("empty_patch")
|
|
156
|
+
path = output / "weights.safetensors"
|
|
157
|
+
save_file(tensors, str(path))
|
|
158
|
+
manifest = {"schema_version": "delta.submission.v1", "format": "sparse-diff", "operation": "replace",
|
|
159
|
+
"reconstruction_version": "delta-reconstruct-v1", "parent_version_id": str(parent_id),
|
|
160
|
+
"parent_content_hash": parent_hash, "model_config_id": model_id, "tensors": mapping,
|
|
161
|
+
"artifacts": [{"name": path.name, "sha256": file_hash(path), "size_bytes": path.stat().st_size}]}
|
|
162
|
+
(output / "manifest.json").write_text(json.dumps(manifest, indent=2))
|
|
163
|
+
return manifest
|
delta_cli/local.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""Offline text inference and public validation; no platform or private-data imports."""
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import statistics
|
|
8
|
+
import tempfile
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
from .artifacts import file_hash, reconstruct, validate_manifest
|
|
12
|
+
from .scoring import score_answer
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def read_workspace(workspace):
|
|
16
|
+
workspace = Path(workspace).resolve()
|
|
17
|
+
identity = json.loads((workspace / 'delta-workspace.json').read_text())
|
|
18
|
+
config = json.loads((workspace / 'evaluation.json').read_text())
|
|
19
|
+
metadata = json.loads((workspace / 'model/delta-model.json').read_text())
|
|
20
|
+
if (identity['config_id'] != config['id'] or identity['model_id'] != config['model_id']
|
|
21
|
+
or identity['parent_version_id'] != metadata['id'] or identity['model_id'] != metadata['model_id']):
|
|
22
|
+
raise ValueError('Workspace model, parent, or evaluation configuration mismatch. Initialize a fresh workspace.')
|
|
23
|
+
return workspace, identity, config, metadata
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@contextmanager
|
|
27
|
+
def model_directory(workspace, metadata, weights=None, patch=None, parent=None):
|
|
28
|
+
"""Overlay a candidate without mutating the downloaded parent or tokenizer."""
|
|
29
|
+
if weights and patch:
|
|
30
|
+
raise ValueError('Choose either --weights or --patch.')
|
|
31
|
+
parent = parent or workspace / 'model'
|
|
32
|
+
for artifact in metadata['files']:
|
|
33
|
+
name = artifact['name']
|
|
34
|
+
if Path(name).name != name:
|
|
35
|
+
raise ValueError('Invalid model artifact filename.')
|
|
36
|
+
if name == 'model.safetensors' and weights:
|
|
37
|
+
continue
|
|
38
|
+
path = parent / name
|
|
39
|
+
if path.stat().st_size != artifact['size_bytes'] or file_hash(path) != artifact['sha256']:
|
|
40
|
+
raise ValueError(f'Parent artifact changed: {name}. Keep the parent untouched and use --weights or --patch.')
|
|
41
|
+
if not weights and not patch:
|
|
42
|
+
yield parent, {'kind': 'parent', 'weights_sha256': file_hash(parent / 'model.safetensors')}
|
|
43
|
+
return
|
|
44
|
+
with tempfile.TemporaryDirectory(prefix='delta-local-') as temp:
|
|
45
|
+
target = Path(temp)
|
|
46
|
+
for artifact in metadata['files']:
|
|
47
|
+
if artifact['name'] != 'model.safetensors':
|
|
48
|
+
(target / artifact['name']).symlink_to((parent / artifact['name']).resolve())
|
|
49
|
+
if patch:
|
|
50
|
+
from safetensors.torch import save_file
|
|
51
|
+
patch = Path(patch).resolve()
|
|
52
|
+
manifest = json.loads((patch / 'manifest.json').read_text())
|
|
53
|
+
validate_manifest(manifest)
|
|
54
|
+
if (manifest['parent_version_id'] != metadata['id'] or manifest['parent_content_hash'] != metadata['content_hash']
|
|
55
|
+
or manifest['model_config_id'] != metadata['model_id']):
|
|
56
|
+
raise ValueError('Patch does not reference this workspace parent.')
|
|
57
|
+
for artifact in manifest['artifacts']:
|
|
58
|
+
path = patch / artifact['name']
|
|
59
|
+
if path.stat().st_size != artifact['size_bytes'] or file_hash(path) != artifact['sha256']:
|
|
60
|
+
raise ValueError('Patch artifact integrity failure.')
|
|
61
|
+
tensors = reconstruct(parent / 'model.safetensors', patch / manifest['artifacts'][0]['name'], manifest)
|
|
62
|
+
save_file(tensors, str(target / 'model.safetensors'))
|
|
63
|
+
del tensors
|
|
64
|
+
else:
|
|
65
|
+
weights = Path(weights).resolve()
|
|
66
|
+
if not weights.is_file() or weights.suffix != '.safetensors':
|
|
67
|
+
raise ValueError('--weights must be a full .safetensors checkpoint.')
|
|
68
|
+
(target / 'model.safetensors').symlink_to(weights)
|
|
69
|
+
yield target, {'kind': 'patch' if patch else 'weights', 'weights_sha256': file_hash(target / 'model.safetensors')}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def select_device(torch, requested):
|
|
73
|
+
if requested not in {'auto', 'cpu', 'cuda', 'mps'}:
|
|
74
|
+
raise ValueError('--device must be auto, cpu, cuda, or mps.')
|
|
75
|
+
if requested == 'auto':
|
|
76
|
+
return 'cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
|
|
77
|
+
if requested == 'cuda' and not torch.cuda.is_available():
|
|
78
|
+
raise ValueError('CUDA is unavailable. Install a CUDA-enabled PyTorch build or use --device cpu.')
|
|
79
|
+
if requested == 'mps' and not torch.backends.mps.is_available():
|
|
80
|
+
raise ValueError('MPS is unavailable. Use a supported Apple Silicon Mac or --device cpu.')
|
|
81
|
+
return requested
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class Runner:
|
|
85
|
+
def __init__(self, model_path, device='auto'):
|
|
86
|
+
import torch
|
|
87
|
+
from transformers import AutoConfig, AutoTokenizer, AutoModelForCausalLM, AutoModelForImageTextToText
|
|
88
|
+
self.torch = torch
|
|
89
|
+
self.device = select_device(torch, device)
|
|
90
|
+
# Float32 is the portable CPU/MPS path; CUDA uses BF16 when supported.
|
|
91
|
+
dtype = torch.bfloat16 if self.device == 'cuda' and torch.cuda.is_bf16_supported() else torch.float32
|
|
92
|
+
self.tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True, trust_remote_code=False)
|
|
93
|
+
config = AutoConfig.from_pretrained(model_path, local_files_only=True, trust_remote_code=False)
|
|
94
|
+
auto = AutoModelForImageTextToText if type(config) in AutoModelForImageTextToText._model_mapping else AutoModelForCausalLM
|
|
95
|
+
self.model, info = auto.from_pretrained(model_path, config=config, local_files_only=True, trust_remote_code=False,
|
|
96
|
+
use_safetensors=True, dtype=dtype, output_loading_info=True)
|
|
97
|
+
if info.get('missing_keys') or info.get('unexpected_keys') or info.get('mismatched_keys') or info.get('error_msgs'):
|
|
98
|
+
raise ValueError('Checkpoint tensors do not match the model configuration; refusing partially initialized weights.')
|
|
99
|
+
self.model.to(self.device).eval()
|
|
100
|
+
self.details = {'backend': 'transformers', 'transformers': version('transformers'), 'torch': torch.__version__,
|
|
101
|
+
'device': self.device, 'dtype': str(dtype),
|
|
102
|
+
'mps_cpu_fallback': self.device == 'mps' and os.getenv('PYTORCH_ENABLE_MPS_FALLBACK') == '1'}
|
|
103
|
+
|
|
104
|
+
def generate(self, prompt, generation, seed):
|
|
105
|
+
from transformers import LogitsProcessor, LogitsProcessorList, set_seed
|
|
106
|
+
torch = self.torch
|
|
107
|
+
rendered = self.tokenizer.apply_chat_template([{'role': 'user', 'content': prompt}], tokenize=False,
|
|
108
|
+
add_generation_prompt=True, enable_thinking=generation.get('enable_thinking', True))
|
|
109
|
+
inputs = self.tokenizer(rendered, return_tensors='pt', add_special_tokens=False).to(self.device)
|
|
110
|
+
prompt_length = inputs['input_ids'].shape[1]
|
|
111
|
+
max_tokens = generation.get('max_tokens', 32768)
|
|
112
|
+
if prompt_length + max_tokens > generation.get('max_model_len', 34816):
|
|
113
|
+
raise ValueError('Prompt plus output budget exceeds max_model_len; reduce --max-new-tokens.')
|
|
114
|
+
penalty = generation.get('presence_penalty', 1.5)
|
|
115
|
+
|
|
116
|
+
class PresencePenalty(LogitsProcessor):
|
|
117
|
+
def __call__(self, input_ids, scores):
|
|
118
|
+
# Match vLLM: presence penalty applies to generated tokens, not the prompt.
|
|
119
|
+
for row in range(input_ids.shape[0]):
|
|
120
|
+
ids = input_ids[row, prompt_length:].unique()
|
|
121
|
+
scores[row, ids] -= penalty
|
|
122
|
+
return scores
|
|
123
|
+
|
|
124
|
+
temperature = generation.get('temperature', .6)
|
|
125
|
+
kwargs = {'max_new_tokens': max_tokens, 'do_sample': temperature > 0,
|
|
126
|
+
'logits_processor': LogitsProcessorList([PresencePenalty()])}
|
|
127
|
+
if temperature > 0:
|
|
128
|
+
kwargs.update(temperature=temperature, top_p=generation.get('top_p', .95), top_k=generation.get('top_k', 20))
|
|
129
|
+
set_seed(seed)
|
|
130
|
+
start = time.monotonic()
|
|
131
|
+
with torch.inference_mode():
|
|
132
|
+
tokens = self.model.generate(**inputs, **kwargs)[0, prompt_length:]
|
|
133
|
+
count = tokens.numel()
|
|
134
|
+
# Preserve </think> so the same scorer can remove reasoning traces as on the server.
|
|
135
|
+
text = self.tokenizer.decode(tokens, skip_special_tokens=False)
|
|
136
|
+
for token in [self.tokenizer.eos_token, self.tokenizer.pad_token]:
|
|
137
|
+
if token:
|
|
138
|
+
text = text.replace(token, '')
|
|
139
|
+
return {'text': text, 'tokens': count, 'truncated': count == max_tokens,
|
|
140
|
+
'generation_seconds': time.monotonic() - start}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def public_examples(workspace, config, benchmark=None, limit=None):
|
|
144
|
+
path = workspace / 'validation.json'
|
|
145
|
+
if file_hash(path) != config['validation_sha256']:
|
|
146
|
+
raise ValueError('Public validation data changed; initialize a fresh workspace.')
|
|
147
|
+
data = json.loads(path.read_text())
|
|
148
|
+
names = config['validation_names']
|
|
149
|
+
if set(data) != set(names) or not names:
|
|
150
|
+
raise ValueError('Public validation benchmark inventory mismatch.')
|
|
151
|
+
if benchmark and benchmark not in names:
|
|
152
|
+
raise ValueError('Unknown benchmark; choose from: ' + ', '.join(names))
|
|
153
|
+
selected = [benchmark] if benchmark else names
|
|
154
|
+
rows = []
|
|
155
|
+
for name in selected:
|
|
156
|
+
if not data[name]:
|
|
157
|
+
raise ValueError(f'Empty benchmark: {name}')
|
|
158
|
+
for index, example in enumerate(data[name][:limit] if limit else data[name]):
|
|
159
|
+
scorer = example.get('scorer', 'exact')
|
|
160
|
+
if scorer not in {'exact', 'choice', 'math', 'ifeval'}:
|
|
161
|
+
raise ValueError(f'Unsupported scorer: {scorer}')
|
|
162
|
+
# Fail on missing scoring dependencies before spending time on inference.
|
|
163
|
+
if scorer == 'math':
|
|
164
|
+
__import__('math_verify')
|
|
165
|
+
elif scorer == 'ifeval':
|
|
166
|
+
__import__('lm_eval.tasks.ifeval.instructions_registry')
|
|
167
|
+
samples = example.get('samples', 1)
|
|
168
|
+
if type(samples) is not int or not 1 <= samples <= 16:
|
|
169
|
+
raise ValueError('Invalid sample count.')
|
|
170
|
+
for sample in range(samples):
|
|
171
|
+
rows.append({'benchmark': name, 'example_index': index, 'sample': sample,
|
|
172
|
+
'seed': 1000 + sample, 'example': example})
|
|
173
|
+
return rows, selected, not benchmark and all(limit is None or limit >= len(v) for v in data.values())
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def evaluate(runner, tasks, generation, stream, progress):
|
|
177
|
+
scores, tokens, seconds, truncated = {}, 0, 0, 0
|
|
178
|
+
for index, task in enumerate(tasks, 1):
|
|
179
|
+
result = runner.generate(task['example']['prompt'], generation, task['seed'])
|
|
180
|
+
score = score_answer(task['example'], result['text'])
|
|
181
|
+
scores.setdefault(task['benchmark'], {}).setdefault(task['example_index'], []).append(score)
|
|
182
|
+
row = {k: v for k, v in task.items() if k != 'example'}
|
|
183
|
+
row.update(example_id=task['example']['id'], score=score, **result)
|
|
184
|
+
stream.write(json.dumps(row) + '\n')
|
|
185
|
+
stream.flush()
|
|
186
|
+
tokens += result['tokens']
|
|
187
|
+
seconds += result['generation_seconds']
|
|
188
|
+
truncated += result['truncated']
|
|
189
|
+
progress(index, len(tasks), task['benchmark'])
|
|
190
|
+
components = {name: statistics.mean(statistics.mean(samples) for samples in examples.values()) * 100
|
|
191
|
+
for name, examples in scores.items()}
|
|
192
|
+
return {'validation_score': statistics.mean(components.values()), 'validation_benchmarks': components,
|
|
193
|
+
'output_tokens': tokens, 'generation_seconds': seconds, 'truncation_rate': truncated / len(tasks)}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""CLI wrappers keep optional inference dependencies out of upload/discovery commands."""
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import uuid
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
|
|
12
|
+
from . import local
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def fail(exc):
|
|
16
|
+
if isinstance(exc, ImportError):
|
|
17
|
+
typer.echo('Local inference dependencies are missing. Install the CLI with pip install ".[local]"; IFEval also needs ".[ifeval]".', err=True)
|
|
18
|
+
else:
|
|
19
|
+
typer.echo(f'Local run failed: {exc}', err=True)
|
|
20
|
+
typer.echo('For device/memory errors, try --device cpu or a smaller --max-new-tokens value.', err=True)
|
|
21
|
+
raise typer.Exit(1)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@contextmanager
|
|
25
|
+
def experiment(workspace, version_id, weights, patch, device, max_new_tokens, thinking):
|
|
26
|
+
from .main import API, call, pull_model
|
|
27
|
+
workspace, identity, config, metadata = local.read_workspace(workspace)
|
|
28
|
+
parent = workspace / 'model'
|
|
29
|
+
if version_id:
|
|
30
|
+
if identity.get("api", API) != API:
|
|
31
|
+
raise ValueError("Workspace belongs to another API. Set delta --api WORKSPACE_API before the command.")
|
|
32
|
+
version_id = str(uuid.UUID(version_id))
|
|
33
|
+
remote = call('GET', f'model-versions/{version_id}')
|
|
34
|
+
if remote['model_id'] != identity['model_id']:
|
|
35
|
+
raise ValueError('Leaderboard version belongs to another model. Initialize that competition first.')
|
|
36
|
+
parent = workspace / '.delta/models' / version_id
|
|
37
|
+
metadata = pull_model(version_id, parent)
|
|
38
|
+
generation = dict(config['generation'])
|
|
39
|
+
if max_new_tokens is not None:
|
|
40
|
+
generation['max_tokens'] = max_new_tokens
|
|
41
|
+
if thinking is not None:
|
|
42
|
+
generation['enable_thinking'] = thinking
|
|
43
|
+
if sys.platform == 'darwin' and device in {'auto', 'mps'}:
|
|
44
|
+
os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1')
|
|
45
|
+
typer.echo('Checking model artifacts…', err=True)
|
|
46
|
+
with local.model_directory(workspace, metadata, weights, patch, parent=parent) as (directory, artifact):
|
|
47
|
+
typer.echo(f'Loading model on {device} (first load may take a while)…', err=True)
|
|
48
|
+
runner = local.Runner(directory, device)
|
|
49
|
+
details = {'local_only': True, 'model_id': identity['model_id'], 'config_id': config['id'],
|
|
50
|
+
'source_version_id': metadata['id'], 'source_content_hash': metadata['content_hash'],
|
|
51
|
+
'artifact': artifact, 'runtime': runner.details, 'generation': generation,
|
|
52
|
+
'generation_overridden': generation != config['generation'],
|
|
53
|
+
'evaluation_sha256': hashlib.sha256((workspace / 'evaluation.json').read_bytes()).hexdigest()}
|
|
54
|
+
typer.echo(f"Running on {runner.details['device']}; local results are advisory.", err=True)
|
|
55
|
+
yield runner, details
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def run(prompt: str, workspace: Path = Path('.'), version: str | None = None,
|
|
59
|
+
weights: Path | None = None, patch: Path | None = None, device: str = 'auto',
|
|
60
|
+
max_new_tokens: int = typer.Option(256, min=1), seed: int = 1000,
|
|
61
|
+
thinking: bool | None = typer.Option(None, '--thinking/--no-thinking'),
|
|
62
|
+
json_output: bool = typer.Option(False, '--json')):
|
|
63
|
+
"""Run a prompt locally on the workspace parent, a leaderboard --version, or candidate weights."""
|
|
64
|
+
try:
|
|
65
|
+
with experiment(workspace, version, weights, patch, device, max_new_tokens, thinking) as (runner, details):
|
|
66
|
+
result = runner.generate(prompt, details['generation'], seed)
|
|
67
|
+
if json_output:
|
|
68
|
+
typer.echo(json.dumps({**details, 'seed': seed, **result}, indent=2))
|
|
69
|
+
else:
|
|
70
|
+
typer.echo(result['text'])
|
|
71
|
+
if result['truncated']:
|
|
72
|
+
typer.echo('Output reached the token limit. Increase --max-new-tokens for a complete answer.', err=True)
|
|
73
|
+
except (ImportError, OSError, ValueError, KeyError, RuntimeError) as exc:
|
|
74
|
+
fail(exc)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def evaluate(workspace: Path = Path('.'), version: str | None = None,
|
|
78
|
+
weights: Path | None = None, patch: Path | None = None, device: str = 'auto',
|
|
79
|
+
benchmark: str | None = None, limit: int | None = typer.Option(None, min=1),
|
|
80
|
+
max_new_tokens: int | None = typer.Option(None, min=1),
|
|
81
|
+
thinking: bool | None = typer.Option(None, '--thinking/--no-thinking'),
|
|
82
|
+
output: Path = Path('local-results.json')):
|
|
83
|
+
"""Evaluate public validation. --limit caps examples per benchmark; output is never an official score."""
|
|
84
|
+
try:
|
|
85
|
+
root, _, config, _ = local.read_workspace(workspace)
|
|
86
|
+
tasks, names, full = local.public_examples(root, config, benchmark, limit)
|
|
87
|
+
output = output.resolve()
|
|
88
|
+
generations = output.with_suffix('.generations.jsonl')
|
|
89
|
+
if output.exists() or generations.exists():
|
|
90
|
+
raise ValueError('Output already exists. Choose a new --output path to preserve previous results.')
|
|
91
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
92
|
+
budget = max_new_tokens or config['generation'].get('max_tokens', 32768)
|
|
93
|
+
typer.echo(f'{len(tasks)} generations; up to {len(tasks) * budget:,} output tokens. Use --limit and --max-new-tokens for a quick check.', err=True)
|
|
94
|
+
with experiment(workspace, version, weights, patch, device, max_new_tokens, thinking) as (runner, details):
|
|
95
|
+
with generations.open('x') as stream:
|
|
96
|
+
result = local.evaluate(runner, tasks, details['generation'], stream,
|
|
97
|
+
lambda index, total, name: typer.echo(f'[{index}/{total}] {name}', err=True))
|
|
98
|
+
report = {**details, **result, 'complete': True, 'full_validation_set': full,
|
|
99
|
+
'selected_benchmarks': names, 'examples_per_benchmark_limit': limit,
|
|
100
|
+
'validation_sha256': config['validation_sha256'], 'generations_file': str(generations),
|
|
101
|
+
'note': 'Advisory local score. Hardware/backend differences affect results; this is not a verified leaderboard score.'}
|
|
102
|
+
with output.open('x') as stream:
|
|
103
|
+
json.dump(report, stream, indent=2)
|
|
104
|
+
typer.echo(json.dumps(report, indent=2))
|
|
105
|
+
typer.echo(f'Saved {output}', err=True)
|
|
106
|
+
except (ImportError, OSError, ValueError, KeyError, RuntimeError) as exc:
|
|
107
|
+
fail(exc)
|
delta_cli/main.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import time
|
|
6
|
+
import uuid
|
|
7
|
+
import webbrowser
|
|
8
|
+
import httpx
|
|
9
|
+
import typer
|
|
10
|
+
from .local_commands import run, evaluate
|
|
11
|
+
from .artifacts import build_sparse, validate_manifest, file_hash
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(help="Delta League: submit a patch, prove an improvement.", no_args_is_help=True)
|
|
14
|
+
models_app = typer.Typer()
|
|
15
|
+
model_app = typer.Typer()
|
|
16
|
+
patch_app = typer.Typer()
|
|
17
|
+
submission_app = typer.Typer()
|
|
18
|
+
competition_app = typer.Typer(help="Discover model competitions and initialize a reproducible workspace.")
|
|
19
|
+
app.add_typer(models_app, name="models")
|
|
20
|
+
app.add_typer(model_app, name="model")
|
|
21
|
+
app.add_typer(patch_app, name="patch")
|
|
22
|
+
app.add_typer(submission_app, name="submission")
|
|
23
|
+
app.add_typer(competition_app, name="competition")
|
|
24
|
+
API = "https://deltaleague.ai"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def config_path():
|
|
28
|
+
return Path(os.getenv("DELTA_CONFIG", str(Path.home() / ".config/deltaleague/auth.json")))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def credentials():
|
|
32
|
+
p = config_path()
|
|
33
|
+
return json.loads(p.read_text()) if p.exists() else {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def client():
|
|
37
|
+
saved = credentials()
|
|
38
|
+
token = os.getenv("DELTA_TOKEN") or (saved.get("token") if saved.get("api") == API else None)
|
|
39
|
+
return httpx.Client(base_url=API, headers={"Authorization": f"Bearer {token}"} if token else {}, timeout=60)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def call(method, path, **kwargs):
|
|
43
|
+
try:
|
|
44
|
+
with client() as c:
|
|
45
|
+
r = c.request(method, "/api/v1/" + path, **kwargs)
|
|
46
|
+
except httpx.HTTPError:
|
|
47
|
+
typer.echo("Could not reach the Delta League API. Check the connection and --api URL, then retry.", err=True)
|
|
48
|
+
raise typer.Exit(1)
|
|
49
|
+
if not r.is_success:
|
|
50
|
+
if r.status_code in (401, 403):
|
|
51
|
+
typer.echo("Authentication or permission required. Run `delta login`, then check your account access.", err=True)
|
|
52
|
+
else:
|
|
53
|
+
typer.echo(f"API request failed (HTTP {r.status_code}).", err=True)
|
|
54
|
+
try:
|
|
55
|
+
show(r.json())
|
|
56
|
+
except ValueError:
|
|
57
|
+
pass
|
|
58
|
+
raise typer.Exit(1)
|
|
59
|
+
return r.json()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def show(value):
|
|
63
|
+
typer.echo(json.dumps(value, indent=2))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@app.callback()
|
|
67
|
+
def options(api: str = typer.Option(os.getenv("DELTA_API", "https://deltaleague.ai"), help="Delta League API origin")):
|
|
68
|
+
global API
|
|
69
|
+
API = api.rstrip("/")
|
|
70
|
+
if not API.startswith("https://") and not API.startswith(("http://127.0.0.1:", "http://localhost:")):
|
|
71
|
+
raise typer.BadParameter("Use HTTPS outside localhost")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@app.command()
|
|
75
|
+
def login(no_browser: bool = typer.Option(False, "--no-browser", help="Print the authorization URL without opening a browser.")):
|
|
76
|
+
data = call("POST", "cli-auth")
|
|
77
|
+
typer.echo(f"Authorize this CLI: {data['verification_url']}")
|
|
78
|
+
if not no_browser:
|
|
79
|
+
webbrowser.open(data["verification_url"])
|
|
80
|
+
for _ in range(120):
|
|
81
|
+
time.sleep(data["interval"])
|
|
82
|
+
result = call("POST", f"cli-auth/{data['id']}", json={"secret": data["secret"]})
|
|
83
|
+
if "token" in result:
|
|
84
|
+
path = config_path()
|
|
85
|
+
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
86
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
87
|
+
os.fchmod(fd, 0o600)
|
|
88
|
+
with os.fdopen(fd, "w") as f:
|
|
89
|
+
json.dump({"api": API, "token": result["token"]}, f)
|
|
90
|
+
typer.echo("Signed in.")
|
|
91
|
+
return
|
|
92
|
+
raise typer.Exit(1)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@app.command()
|
|
96
|
+
def logout():
|
|
97
|
+
call("POST", "logout")
|
|
98
|
+
config_path().unlink(missing_ok=True)
|
|
99
|
+
typer.echo("Signed out.")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@app.command()
|
|
103
|
+
def whoami():
|
|
104
|
+
show(call("GET", "me"))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@app.command()
|
|
108
|
+
def quota():
|
|
109
|
+
show(call("GET", "me"))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@models_app.command("list")
|
|
113
|
+
def models_list():
|
|
114
|
+
show(call("GET", "models"))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def download_artifact(artifact, output, force=False):
|
|
118
|
+
name = artifact["name"]
|
|
119
|
+
if Path(name).name != name:
|
|
120
|
+
raise typer.BadParameter("Invalid artifact filename from server")
|
|
121
|
+
path = output / name
|
|
122
|
+
if path.exists():
|
|
123
|
+
if path.stat().st_size == artifact["size_bytes"] and file_hash(path) == artifact["sha256"]:
|
|
124
|
+
typer.echo(f"Reusing verified {name}", err=True)
|
|
125
|
+
return
|
|
126
|
+
if not force:
|
|
127
|
+
raise typer.BadParameter(f"{path} exists with different contents. Use a new directory or --force to replace it.")
|
|
128
|
+
temporary = path.with_name(path.name + ".part")
|
|
129
|
+
typer.echo(f"Downloading {name} ({artifact['size_bytes'] / 1024**2:.1f} MiB)", err=True)
|
|
130
|
+
try:
|
|
131
|
+
with httpx.stream("GET", artifact["url"], timeout=300) as response:
|
|
132
|
+
response.raise_for_status()
|
|
133
|
+
total = 0
|
|
134
|
+
with temporary.open("wb") as target, typer.progressbar(length=artifact["size_bytes"], label=name, file=sys.stderr) as progress:
|
|
135
|
+
for chunk in response.iter_bytes():
|
|
136
|
+
total += len(chunk)
|
|
137
|
+
if total > artifact["size_bytes"]:
|
|
138
|
+
raise ValueError("download_size")
|
|
139
|
+
target.write(chunk)
|
|
140
|
+
progress.update(len(chunk))
|
|
141
|
+
if temporary.stat().st_size != artifact["size_bytes"] or file_hash(temporary) != artifact["sha256"]:
|
|
142
|
+
raise ValueError("download_integrity")
|
|
143
|
+
temporary.replace(path)
|
|
144
|
+
except (httpx.HTTPError, OSError, ValueError):
|
|
145
|
+
typer.echo(f"Download failed or integrity check failed for {name}. Retry the command; verified files are reused.", err=True)
|
|
146
|
+
raise typer.Exit(1)
|
|
147
|
+
finally:
|
|
148
|
+
temporary.unlink(missing_ok=True)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def pull_model(version, output, force=False):
|
|
152
|
+
data = call("GET", f"model-versions/{version}")
|
|
153
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
154
|
+
metadata_path = output / "delta-model.json"
|
|
155
|
+
if metadata_path.exists():
|
|
156
|
+
previous = json.loads(metadata_path.read_text())
|
|
157
|
+
if previous.get("id") != data["id"]:
|
|
158
|
+
raise typer.BadParameter("This directory contains a different parent version. Choose a new output directory.")
|
|
159
|
+
for artifact in data["files"]:
|
|
160
|
+
download_artifact(artifact, output, force=force)
|
|
161
|
+
# Signed download URLs are temporary credentials, not reproducibility metadata.
|
|
162
|
+
data["files"] = [{k: f[k] for k in ("name", "sha256", "size_bytes")} for f in data["files"]]
|
|
163
|
+
metadata_path.write_text(json.dumps(data, indent=2))
|
|
164
|
+
return data
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@model_app.command("pull")
|
|
168
|
+
def model_pull(version: str, output: Path = typer.Option(Path("model")), force: bool = False):
|
|
169
|
+
"""Download a parent checkpoint. Reuses hash-verified files; --force replaces damaged/modified files."""
|
|
170
|
+
pull_model(version, output, force=force)
|
|
171
|
+
typer.echo(str(output))
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@competition_app.command("list")
|
|
175
|
+
def competitions_list():
|
|
176
|
+
rows = []
|
|
177
|
+
for model in call("GET", "models"):
|
|
178
|
+
board = call("GET", "leaderboard", params={"model": model["id"]})
|
|
179
|
+
parent = next((r["version_id"] for r in board["rows"] if r["parent_id"] is None), None)
|
|
180
|
+
rows.append({"model_id": model["id"], "name": model["name"], "config_id": board["config_id"],
|
|
181
|
+
"default_parent_version_id": parent, "ready": bool(board["config_id"] and parent)})
|
|
182
|
+
show(rows)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@competition_app.command("init")
|
|
186
|
+
def competition_init(model: str, output: Path = typer.Option(Path("experiment")), parent: str | None = None):
|
|
187
|
+
"""Pull a model parent, public validation data, and the active evaluation configuration."""
|
|
188
|
+
board = call("GET", "leaderboard", params={"model": model})
|
|
189
|
+
if not board["config_id"]:
|
|
190
|
+
raise typer.BadParameter("This model has no active competition. Run `delta competition list` to find available models.")
|
|
191
|
+
parent = parent or next((r["version_id"] for r in board["rows"] if r["parent_id"] is None), None)
|
|
192
|
+
if not parent:
|
|
193
|
+
raise typer.BadParameter("No original checkpoint is registered yet. Choose --parent VERSION_ID or wait for setup.")
|
|
194
|
+
version = call("GET", f"model-versions/{parent}")
|
|
195
|
+
if version["model_id"] != model:
|
|
196
|
+
raise typer.BadParameter("The selected parent belongs to a different model.")
|
|
197
|
+
config = call("GET", f"evaluation-configs/{board['config_id']}")
|
|
198
|
+
if not config.get("validation_download") or not config.get("validation_sha256"):
|
|
199
|
+
raise typer.BadParameter("The server does not provide a verifiable public validation download for this configuration yet.")
|
|
200
|
+
identity = {"api": API, "model_id": model, "parent_version_id": parent, "config_id": config["id"]}
|
|
201
|
+
workspace_path = output / "delta-workspace.json"
|
|
202
|
+
if workspace_path.exists():
|
|
203
|
+
previous = json.loads(workspace_path.read_text())
|
|
204
|
+
if any(previous.get(k) != value for k, value in identity.items()):
|
|
205
|
+
raise typer.BadParameter("This workspace belongs to another model, parent, API, or evaluation version. Use a new output directory.")
|
|
206
|
+
pull_model(parent, output / "model")
|
|
207
|
+
download_artifact({"name": "validation.json", "url": config["validation_download"],
|
|
208
|
+
"sha256": config["validation_sha256"], "size_bytes": config["validation_size_bytes"]}, output)
|
|
209
|
+
public_config = {k: v for k, v in config.items() if k != "validation_download"}
|
|
210
|
+
(output / "evaluation.json").write_text(json.dumps(public_config, indent=2))
|
|
211
|
+
workspace_path.write_text(json.dumps(identity, indent=2))
|
|
212
|
+
show({**identity, "directory": str(output), "parent_directory": str(output / "model"),
|
|
213
|
+
"validation_file": str(output / "validation.json"), "validation_benchmarks": config["validation_names"]})
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@patch_app.command("build")
|
|
217
|
+
def patch_build(parent: Path = typer.Option(...), modified: Path = typer.Option(...), output: Path = typer.Option(Path("submission"))):
|
|
218
|
+
"""Build an exact sparse replacement patch. Parent is a `delta model pull` directory."""
|
|
219
|
+
metadata = json.loads((parent / "delta-model.json").read_text())
|
|
220
|
+
manifest = build_sparse(parent / "model.safetensors", modified, output, metadata["id"], metadata["content_hash"], metadata["model_id"])
|
|
221
|
+
typer.echo(f"Built {len(manifest['tensors'])} modified tensors in {output}")
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@patch_app.command("inspect")
|
|
225
|
+
def inspect(path: Path):
|
|
226
|
+
show(json.loads((path / "manifest.json").read_text()))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@app.command()
|
|
230
|
+
def validate(path: Path):
|
|
231
|
+
manifest = json.loads((path / "manifest.json").read_text())
|
|
232
|
+
validate_manifest(manifest)
|
|
233
|
+
for f in manifest["artifacts"]:
|
|
234
|
+
if file_hash(path / f["name"]) != f["sha256"] or (path / f["name"]).stat().st_size != f["size_bytes"]:
|
|
235
|
+
raise typer.BadParameter("Artifact integrity failure")
|
|
236
|
+
typer.echo("Manifest and artifact hashes are valid. Server checks parent compatibility and tensors.")
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@app.command()
|
|
240
|
+
def submit(path: Path, title: str = "Weight modification", pr: str | None = None):
|
|
241
|
+
validate(path)
|
|
242
|
+
manifest = json.loads((path / "manifest.json").read_text())
|
|
243
|
+
if pr:
|
|
244
|
+
manifest["research"] = {"pr_url": pr}
|
|
245
|
+
# Persist intent before posting: retries after a lost response reuse the same idempotency key.
|
|
246
|
+
state_path = path / ".delta-upload.json"
|
|
247
|
+
state = json.loads(state_path.read_text()) if state_path.exists() else {}
|
|
248
|
+
fingerprint = __import__("hashlib").sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest()
|
|
249
|
+
if state.get("fingerprint") != fingerprint or state.get("api") != API:
|
|
250
|
+
upload = call("POST", "upload-sessions", json={"manifest": manifest})
|
|
251
|
+
for f in upload["files"]:
|
|
252
|
+
file = path / f["name"]
|
|
253
|
+
with file.open("rb") as stream:
|
|
254
|
+
response = httpx.put(f["url"], content=stream, headers={"Content-Length": str(file.stat().st_size)}, timeout=600)
|
|
255
|
+
response.raise_for_status()
|
|
256
|
+
state = {"upload_id": upload["id"], "idempotency_key": str(uuid.uuid4()), "fingerprint": fingerprint, "api": API}
|
|
257
|
+
state_path.write_text(json.dumps(state))
|
|
258
|
+
show(call("POST", "submissions", json={"upload_id": state["upload_id"], "title": title}, headers={"Idempotency-Key": state["idempotency_key"]}))
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
@submission_app.command("status")
|
|
262
|
+
def status(submission: str, watch: bool = False):
|
|
263
|
+
while True:
|
|
264
|
+
data = call("GET", f"submissions/{submission}")
|
|
265
|
+
show(data)
|
|
266
|
+
if not watch or data["state"] in {"verified", "rejected", "failed", "canceled"}:
|
|
267
|
+
return
|
|
268
|
+
time.sleep(10)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@submission_app.command("results")
|
|
272
|
+
def results(submission: str, json_output: bool = typer.Option(False, "--json")):
|
|
273
|
+
show(call("GET", f"submissions/{submission}")["result"])
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
@app.command()
|
|
277
|
+
def leaderboard(model: str | None = None):
|
|
278
|
+
show(call("GET", "leaderboard", params={"model": model} if model else {}))
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
app.command("run")(run)
|
|
282
|
+
app.command("evaluate")(evaluate)
|
delta_cli/scoring.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Public scoring rules shared by local validation and the trusted evaluator."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def score_answer(example, text):
|
|
6
|
+
# Thinking traces do not count as the final answer.
|
|
7
|
+
answer = text.split("</think>")[-1].strip()
|
|
8
|
+
scorer = example.get("scorer", "exact")
|
|
9
|
+
if scorer == "exact":
|
|
10
|
+
def normalize(s):
|
|
11
|
+
return re.sub(r"\s+", " ", str(s).strip()).casefold()
|
|
12
|
+
return float(normalize(answer) == normalize(example["answer"]))
|
|
13
|
+
if scorer == "choice":
|
|
14
|
+
boxed = re.findall(r"\\boxed\{([A-E1-5])\}", answer)
|
|
15
|
+
matches = boxed or re.findall(r"(?:^|\n)\s*([A-E1-5])[.)]?\s*$", answer)
|
|
16
|
+
return float(bool(matches) and matches[-1] == str(example["answer"]))
|
|
17
|
+
if scorer == "math":
|
|
18
|
+
from math_verify import parse, verify
|
|
19
|
+
from math_verify.parser import LatexExtractionConfig, ExprExtractionConfig
|
|
20
|
+
gold = parse(str(example["answer"]), extraction_config=[LatexExtractionConfig(), ExprExtractionConfig()])
|
|
21
|
+
pred = parse(answer, extraction_config=[LatexExtractionConfig(), ExprExtractionConfig()])
|
|
22
|
+
return float(bool(gold) and bool(pred) and verify(gold, pred))
|
|
23
|
+
if scorer == "ifeval":
|
|
24
|
+
from lm_eval.tasks.ifeval.instructions_registry import INSTRUCTION_DICT
|
|
25
|
+
passed = []
|
|
26
|
+
for instruction_id, kwargs in zip(example["instruction_id_list"], example["kwargs"], strict=True):
|
|
27
|
+
instruction = INSTRUCTION_DICT[instruction_id](instruction_id)
|
|
28
|
+
instruction.build_description(**{k: v for k, v in kwargs.items() if v is not None})
|
|
29
|
+
args = instruction.get_instruction_args()
|
|
30
|
+
if args and "prompt" in args:
|
|
31
|
+
instruction.build_description(prompt=example["prompt"])
|
|
32
|
+
passed.append(bool(answer) and instruction.check_following(answer))
|
|
33
|
+
return float(bool(passed) and all(passed))
|
|
34
|
+
raise ValueError("unknown_scorer")
|
|
35
|
+
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deltaleague
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CLI for Delta League model-weight competitions, submissions, and local evaluation
|
|
5
|
+
Author: Delta League
|
|
6
|
+
Project-URL: Homepage, https://deltaleague.ai
|
|
7
|
+
Project-URL: Repository, https://github.com/ilblackdragon/deltaleague
|
|
8
|
+
Project-URL: Documentation, https://github.com/ilblackdragon/deltaleague/blob/main/README.md
|
|
9
|
+
Project-URL: Issues, https://github.com/ilblackdragon/deltaleague/issues
|
|
10
|
+
Keywords: llm,benchmark,model-weights,leaderboard
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: httpx<1,>=0.28
|
|
19
|
+
Requires-Dist: typer<1,>=0.16
|
|
20
|
+
Requires-Dist: safetensors<1,>=0.5
|
|
21
|
+
Requires-Dist: torch>=2.6
|
|
22
|
+
Provides-Extra: local
|
|
23
|
+
Requires-Dist: transformers==5.22.1; extra == "local"
|
|
24
|
+
Requires-Dist: math-verify==0.9.0; extra == "local"
|
|
25
|
+
Provides-Extra: ifeval
|
|
26
|
+
Requires-Dist: lm-eval<0.5,>=0.4.9; extra == "ifeval"
|
|
27
|
+
|
|
28
|
+
# Delta League CLI
|
|
29
|
+
|
|
30
|
+
The CLI and submission tools for [Delta League](https://deltaleague.ai): the open competition to improve intelligence.
|
|
31
|
+
|
|
32
|
+
Submit a patch. Prove an improvement. Advance the model.
|
|
33
|
+
|
|
34
|
+
Repository: [ilblackdragon/deltaleague](https://github.com/ilblackdragon/deltaleague). Private during launch preparation; intended to become public. The platform is maintained separately. This package contains no private test data.
|
|
35
|
+
|
|
36
|
+
## Quick start
|
|
37
|
+
|
|
38
|
+
Requires Python 3.12+. Clone this repository, then install `pip install .` in a virtual environment. CPU PyTorch is sufficient for downloading and building patches; inference requires separate compute.
|
|
39
|
+
|
|
40
|
+
```sh
|
|
41
|
+
delta competition list
|
|
42
|
+
delta competition init qwen35-08b-v1 --output experiment
|
|
43
|
+
cd experiment
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
This selects a model competition and downloads its original checkpoint into `model/`, public examples into `validation.json`, and scoring/generation settings into `evaluation.json`. `delta-workspace.json` records the API, immutable parent version, and evaluation version. Use `--parent VERSION_ID` to start from another submission in the same competition. Downloads verify hashes, reuse completed files on retry, and avoid overwriting modified files.
|
|
47
|
+
|
|
48
|
+
Have your agent edit `model/model.safetensors` into a **separate** `edited.safetensors` file. Preserve tensor names, shapes, and dtypes. Keep the parent unchanged. Then:
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
delta patch build --parent model --modified edited.safetensors --output submission
|
|
52
|
+
delta validate submission
|
|
53
|
+
delta login --no-browser
|
|
54
|
+
# Open the research PR first; replace PR_URL with its actual URL.
|
|
55
|
+
delta submit submission --title "My weight edit" --pr PR_URL
|
|
56
|
+
delta submission status SUBMISSION_ID --watch
|
|
57
|
+
delta submission results SUBMISSION_ID --json
|
|
58
|
+
delta leaderboard --model qwen35-08b-v1
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Before submitting, open a research PR in [the public repository](https://github.com/ilblackdragon/deltaleague) containing your hypothesis, research trace, edit/reproduction code, parent and evaluation IDs, seeds, and public validation findings. Attach its actual URL with `--pr`. The CLI records the link; it does not create the PR. Source-free submissions remain supported by omitting `--pr`.
|
|
62
|
+
|
|
63
|
+
Open the authorization URL in a browser and sign in with GitHub. `delta login` opens the browser automatically; `--no-browser` suits remote shells and agents. Authentication is only required for submission/account actions. Credentials are stored with mode 0600 and scoped to the API origin. Put `--api URL` before a command for another deployment.
|
|
64
|
+
|
|
65
|
+
## Run and evaluate on your computer
|
|
66
|
+
|
|
67
|
+
Install with `pip install ".[local]"` from this repository, then run these inside an initialized competition workspace:
|
|
68
|
+
|
|
69
|
+
```sh
|
|
70
|
+
delta run "What is 2 + 2?" --no-thinking
|
|
71
|
+
delta evaluate --limit 2 --max-new-tokens 256 --output baseline-quick.json
|
|
72
|
+
delta evaluate --weights edited.safetensors --limit 2 --max-new-tokens 256 --output edit-quick.json
|
|
73
|
+
# Any entry from `delta leaderboard` can be run or evaluated by version_id:
|
|
74
|
+
delta run "What is 2 + 2?" --version VERSION_ID --no-thinking
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Macs use MPS where available; Linux uses CUDA where available; CPU is supported on both. Local scores are advisory. See [LOCAL.md](LOCAL.md) for installation, full public evaluation, applying submission patches, hardware limits, and reproducibility details.
|
|
78
|
+
|
|
79
|
+
## Agent starting prompt
|
|
80
|
+
|
|
81
|
+
> Work on Delta League using its CLI. Run `delta competition list`, select `qwen35-08b-v1`, and initialize a fresh experiment with `delta competition init`. Read evaluation.json and validation.json. Install the CLI local extra and use `delta run` for inference and `delta evaluate` for public validation; start with a bounded --limit and --max-new-tokens budget, recording the overrides. Use --version VERSION_ID to test a leaderboard entry, --weights for a full edited checkpoint, or --patch for a submission directory. Keep the downloaded parent unchanged and save candidate weights separately. Record your hypothesis, changes, and public validation results. Build and validate a patch. Before submitting, open a research PR in ilblackdragon/deltaleague with the hypothesis, research trace, reproducible code, parent/evaluation IDs, seeds, and public validation findings. Exclude credentials, private test data, and weight artifacts. Ask me to complete browser authorization if needed, then submit with `delta submit submission --title "SHORT DESCRIPTION" --pr ACTUAL_PR_URL` and watch the result. The CLI does not create the PR automatically. If I explicitly choose a source-free submission, omit `--pr`. Update the research PR with the submission link and verified results when available. Never request private test examples or credentials. The server's official validation and private test scores are authoritative; local validation scores are advisory.
|
|
82
|
+
|
|
83
|
+
See [the published-edit reproduction recipe](examples/blog-reproduction/README.md) for a concrete first experiment and [ARTIFACTS.md](ARTIFACTS.md) for LoRA, sparse/dense diff, and full-checkpoint formats. `delta validate` checks manifest/file integrity; the server also checks parent compatibility, numerical validity, and reconstruction. JSON output supports agent use. Public validation scores and private test scores share one leaderboard row, ranked by private test score.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
delta_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
delta_cli/artifacts.py,sha256=m1w39pJ7nw9rsyPf4NwqdKy3Jpp8ye1Uj-VfValKrYQ,7936
|
|
3
|
+
delta_cli/local.py,sha256=cr18wByp4-w_cLO4FDUsroH2Xd3kp_WF0e7wyDjZwlA,11198
|
|
4
|
+
delta_cli/local_commands.py,sha256=7oycfJzmYcVYdLB5p-lpQR1SX0iL8kkgHKfGGnNsYNo,6225
|
|
5
|
+
delta_cli/main.py,sha256=RlE05doXy0smxCmqLxmAU50vNY70w8ytgYCyRdYju5E,12480
|
|
6
|
+
delta_cli/scoring.py,sha256=NNKrYtrUU2cJuJAvsWzXbd0ORvr1ADm4pDla2MxjsiQ,1892
|
|
7
|
+
deltaleague-0.1.0.dist-info/METADATA,sha256=nv5oPWK31cuQ0Le7sydOo272PEdH2eSSmdwIorOY6tU,6391
|
|
8
|
+
deltaleague-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
9
|
+
deltaleague-0.1.0.dist-info/entry_points.txt,sha256=fwKJG5qCs2UuHeivBgOs32JvZshy5r-JYCGVfbqCxVs,45
|
|
10
|
+
deltaleague-0.1.0.dist-info/top_level.txt,sha256=_eXO5uRYbxjf3wdMrjPjKjQOKQx-pHFP1_5UFQJas8A,10
|
|
11
|
+
deltaleague-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
delta_cli
|