lone-ane 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lone/__init__.py +5 -0
- lone/cli.py +228 -0
- lone/cli_options.py +19 -0
- lone/config.py +180 -0
- lone/convert.py +320 -0
- lone/layout.py +130 -0
- lone/metrics.py +261 -0
- lone/repl.py +197 -0
- lone/runtime.py +506 -0
- lone/server.py +284 -0
- lone/spec.py +253 -0
- lone/torch_model.py +259 -0
- lone/verify.py +262 -0
- lone_ane-0.1.0.dist-info/METADATA +294 -0
- lone_ane-0.1.0.dist-info/RECORD +18 -0
- lone_ane-0.1.0.dist-info/WHEEL +4 -0
- lone_ane-0.1.0.dist-info/entry_points.txt +2 -0
- lone_ane-0.1.0.dist-info/licenses/LICENSE +201 -0
lone/__init__.py
ADDED
lone/cli.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""`lone` command line: convert, run, serve, inspect, verify.
|
|
2
|
+
|
|
3
|
+
Typer wraps the same core functions; option names and defaults are unchanged from the argparse
|
|
4
|
+
version. `main()` is kept as the console-script target.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from .cli_options import ComputeUnits, Precision
|
|
17
|
+
|
|
18
|
+
app = typer.Typer(
|
|
19
|
+
add_completion=False,
|
|
20
|
+
no_args_is_help=True,
|
|
21
|
+
help="LFM models on the Apple Neural Engine",
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _units(cpu_only: bool, compute_units: ComputeUnits) -> str:
|
|
26
|
+
return "cpu_only" if cpu_only else compute_units.value
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _report_wired(wired_bytes: int) -> None:
|
|
30
|
+
if wired_bytes:
|
|
31
|
+
print(f"wired {wired_bytes / 2**20:.0f} MiB of caches", file=sys.stderr)
|
|
32
|
+
else:
|
|
33
|
+
print("cache wiring unavailable (needs macOS mlock); continuing unwired", file=sys.stderr)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@app.command(help="convert an HF checkpoint into a LoNE bundle")
|
|
37
|
+
def convert(
|
|
38
|
+
model: str = typer.Option(..., "--model", help="HF checkpoint directory"),
|
|
39
|
+
out: str = typer.Option(..., "--out", help="bundle output directory"),
|
|
40
|
+
ctx: int = typer.Option(2048, "--ctx"),
|
|
41
|
+
prefill: int = typer.Option(64, "--prefill"),
|
|
42
|
+
layers_per_model: int = typer.Option(
|
|
43
|
+
4, "--layers-per-model", help="decoder layers per Core ML graph; fewer graphs are faster, 4 verifies at parity"
|
|
44
|
+
),
|
|
45
|
+
precision: Precision = typer.Option(Precision.fp16, "--precision"),
|
|
46
|
+
head_chunks: int = typer.Option(
|
|
47
|
+
0, "--head-chunks", help="vocabulary slices for the LM head; 0 sizes each graph to ~128 MB"
|
|
48
|
+
),
|
|
49
|
+
ring: bool = typer.Option(
|
|
50
|
+
True, "--ring/--no-ring",
|
|
51
|
+
help="ring-buffer sliding layers to their window instead of the full ctx (default on)",
|
|
52
|
+
),
|
|
53
|
+
decode_only: bool = typer.Option(False, "--decode-only", help="skip the prefill models (slower prompts)"),
|
|
54
|
+
prefill_only: bool = typer.Option(False, "--prefill-only", help="skip the decode models (prefill timing only)"),
|
|
55
|
+
spec_drafts: int = typer.Option(
|
|
56
|
+
0, "--spec-drafts",
|
|
57
|
+
help="convert a verify shape for n-gram speculative decoding: prefill becomes drafts+1 and "
|
|
58
|
+
"the head is traced at that length (0 = off)",
|
|
59
|
+
),
|
|
60
|
+
plan: bool = typer.Option(True, "--plan/--no-plan", help="report the MLComputePlan placement per model"),
|
|
61
|
+
compute_units: ComputeUnits = typer.Option(ComputeUnits.cpu_and_ne, "--compute-units"),
|
|
62
|
+
) -> None:
|
|
63
|
+
from .convert import convert_model
|
|
64
|
+
|
|
65
|
+
convert_model(
|
|
66
|
+
model_dir=model,
|
|
67
|
+
out_dir=out,
|
|
68
|
+
ctx=ctx,
|
|
69
|
+
prefill=prefill,
|
|
70
|
+
layers_per_model=layers_per_model,
|
|
71
|
+
precision=precision.value,
|
|
72
|
+
head_chunks=head_chunks,
|
|
73
|
+
ring=ring,
|
|
74
|
+
decode_only=decode_only,
|
|
75
|
+
prefill_only=prefill_only,
|
|
76
|
+
spec_drafts=spec_drafts,
|
|
77
|
+
check_plan=plan,
|
|
78
|
+
compute_units=compute_units.value,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@app.command(help="generate from a bundle")
|
|
83
|
+
def run(
|
|
84
|
+
bundle: str = typer.Option(..., "--bundle"),
|
|
85
|
+
prompt: str = typer.Option(..., "--prompt"),
|
|
86
|
+
chat: bool = typer.Option(False, "--chat", help="render the prompt through the chat template"),
|
|
87
|
+
system: str | None = typer.Option(None, "--system"),
|
|
88
|
+
max_new_tokens: int = typer.Option(256, "--max-new-tokens"),
|
|
89
|
+
temperature: float = typer.Option(0.0, "--temperature"),
|
|
90
|
+
top_p: float = typer.Option(1.0, "--top-p"),
|
|
91
|
+
top_k: int = typer.Option(0, "--top-k"),
|
|
92
|
+
seed: int | None = typer.Option(None, "--seed"),
|
|
93
|
+
stream: bool = typer.Option(False, "--stream"),
|
|
94
|
+
cpu_only: bool = typer.Option(False, "--cpu-only"),
|
|
95
|
+
compute_units: ComputeUnits = typer.Option(ComputeUnits.cpu_and_ne, "--compute-units"),
|
|
96
|
+
wire_caches: bool = typer.Option(
|
|
97
|
+
False, "--wire-caches", help="wire the KV and conv caches with mlock so they cannot be paged out (macOS)"
|
|
98
|
+
),
|
|
99
|
+
spec_ngram: bool = typer.Option(
|
|
100
|
+
False, "--spec-ngram/--no-spec-ngram",
|
|
101
|
+
help="n-gram speculative decoding (llama.cpp ngram-simple); needs a bundle converted with --spec-drafts",
|
|
102
|
+
),
|
|
103
|
+
spec_n: int = typer.Option(12, "--spec-n", help="lookup n-gram length for --spec-ngram"),
|
|
104
|
+
spec_m: int = typer.Option(16, "--spec-m", help="maximum draft length for --spec-ngram"),
|
|
105
|
+
spec_drafts: int = typer.Option(
|
|
106
|
+
0, "--spec-drafts", help="cap on verified drafts per step for --spec-ngram (0 = the bundle's)"
|
|
107
|
+
),
|
|
108
|
+
) -> None:
|
|
109
|
+
from .runtime import LoneModel
|
|
110
|
+
from .spec import NgramProposer
|
|
111
|
+
|
|
112
|
+
model = LoneModel(bundle, _units(cpu_only, compute_units), wire_caches=wire_caches)
|
|
113
|
+
if wire_caches:
|
|
114
|
+
_report_wired(model.wired_bytes)
|
|
115
|
+
if chat:
|
|
116
|
+
messages = ([{"role": "system", "content": system}] if system else []) + [
|
|
117
|
+
{"role": "user", "content": prompt}
|
|
118
|
+
]
|
|
119
|
+
prompt_ids = model.tokenizer.chat(messages)
|
|
120
|
+
else:
|
|
121
|
+
prompt_ids = model.tokenizer.encode(prompt)
|
|
122
|
+
|
|
123
|
+
first = {"at": None}
|
|
124
|
+
emitted = []
|
|
125
|
+
started = time.perf_counter()
|
|
126
|
+
|
|
127
|
+
def on_token(token: int) -> None:
|
|
128
|
+
if first["at"] is None:
|
|
129
|
+
first["at"] = time.perf_counter()
|
|
130
|
+
emitted.append(token)
|
|
131
|
+
if stream:
|
|
132
|
+
sys.stdout.write(model.tokenizer.decode([token]))
|
|
133
|
+
sys.stdout.flush()
|
|
134
|
+
|
|
135
|
+
proposer = NgramProposer(spec_n, spec_m) if spec_ngram else None
|
|
136
|
+
ids = model.generate(
|
|
137
|
+
prompt_ids,
|
|
138
|
+
max_new_tokens=max_new_tokens,
|
|
139
|
+
temperature=temperature,
|
|
140
|
+
top_p=top_p,
|
|
141
|
+
top_k=top_k,
|
|
142
|
+
seed=seed,
|
|
143
|
+
on_token=on_token,
|
|
144
|
+
proposer=proposer,
|
|
145
|
+
max_drafts=spec_drafts,
|
|
146
|
+
)
|
|
147
|
+
wall = time.perf_counter() - started
|
|
148
|
+
text = model.tokenizer.decode(ids)
|
|
149
|
+
if stream:
|
|
150
|
+
sys.stdout.write("\n")
|
|
151
|
+
else:
|
|
152
|
+
print(text)
|
|
153
|
+
prefill_s = (first["at"] or started) - started
|
|
154
|
+
decode_s = max(wall - prefill_s, 1e-9)
|
|
155
|
+
decode_tokens = max(len(ids) - 1, 0)
|
|
156
|
+
rate = decode_tokens / decode_s if decode_tokens else 0.0
|
|
157
|
+
spec = ""
|
|
158
|
+
stats = model.spec_stats
|
|
159
|
+
if proposer is not None:
|
|
160
|
+
drafted = stats.get("drafted", 0)
|
|
161
|
+
hit = f" ({stats.get('accepted', 0) / drafted:.0%})" if drafted else ""
|
|
162
|
+
spec = (f" | spec {stats.get('accepted', 0)}/{drafted} drafts{hit}"
|
|
163
|
+
f" in {stats.get('passes', 0)} passes")
|
|
164
|
+
print(f"\n[{len(ids)} tokens | {rate:.1f} tok/s | first token {prefill_s * 1e3:.0f} ms | "
|
|
165
|
+
f"{len(prompt_ids)} prompt tokens{spec}]", file=sys.stderr)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@app.command(help="OpenAI-compatible HTTP server")
|
|
169
|
+
def serve(
|
|
170
|
+
bundle: str = typer.Option(..., "--bundle"),
|
|
171
|
+
host: str = typer.Option("127.0.0.1", "--host"),
|
|
172
|
+
port: int = typer.Option(8082, "--port"),
|
|
173
|
+
cpu_only: bool = typer.Option(False, "--cpu-only"),
|
|
174
|
+
compute_units: ComputeUnits = typer.Option(ComputeUnits.cpu_and_ne, "--compute-units"),
|
|
175
|
+
wire_caches: bool = typer.Option(
|
|
176
|
+
False, "--wire-caches", help="wire the KV and conv caches with mlock so they cannot be paged out (macOS)"
|
|
177
|
+
),
|
|
178
|
+
) -> None:
|
|
179
|
+
import uvicorn
|
|
180
|
+
|
|
181
|
+
from .server import create_app
|
|
182
|
+
|
|
183
|
+
app = create_app(bundle, _units(cpu_only, compute_units), wire_caches=wire_caches)
|
|
184
|
+
if wire_caches:
|
|
185
|
+
_report_wired(app.state.wired_bytes)
|
|
186
|
+
uvicorn.run(app, host=host, port=port, log_level="info")
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@app.command("inspect", help="print a bundle's spec and placement")
|
|
190
|
+
def inspect_cmd(bundle: str = typer.Option(..., "--bundle")) -> None:
|
|
191
|
+
with open(os.path.join(bundle, "manifest.json")) as f:
|
|
192
|
+
manifest = json.load(f)
|
|
193
|
+
spec = manifest["spec"]
|
|
194
|
+
print(f"{spec['model_type']}: {spec['num_layers']} layers, hidden {spec['hidden_size']}, "
|
|
195
|
+
f"{spec['num_heads']}/{spec['num_kv_heads']} heads, head_dim {spec['head_dim']}, "
|
|
196
|
+
f"conv {spec['conv_kernel']}, vocab {spec['vocab_size']}")
|
|
197
|
+
print(f"shapes: ctx {manifest['shapes']['ctx']}, prefill {manifest['shapes']['prefill']}, "
|
|
198
|
+
f"precision {manifest['precision']}")
|
|
199
|
+
drafts = manifest.get("speculative", {}).get("drafts", 0)
|
|
200
|
+
if drafts:
|
|
201
|
+
print(f"spec: n-gram verify shape, {drafts} drafts/pass, head {manifest['head'].get('seq')} rows")
|
|
202
|
+
window = spec.get("sliding_window")
|
|
203
|
+
if manifest.get("ring") and window:
|
|
204
|
+
print(f"ring: sliding cache {min(window, manifest['shapes']['ctx'])} rows (window {window})")
|
|
205
|
+
for chunk in manifest["chunks"]:
|
|
206
|
+
tail = ", ".join(
|
|
207
|
+
f"{tag}: {meta.get('plan', {}).get('devices', 'n/a')}" for tag, meta in chunk["models"].items()
|
|
208
|
+
)
|
|
209
|
+
print(f" {chunk['name']} layers {chunk['layers']} -> {tail}")
|
|
210
|
+
print(f" head: {len(manifest['head']['chunks'])} chunks")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
from .repl import register_chat # noqa: E402 (needs `app` above)
|
|
214
|
+
from .verify import register_verify # noqa: E402 (needs `app` above)
|
|
215
|
+
|
|
216
|
+
register_chat(app)
|
|
217
|
+
register_verify(app)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def main() -> None:
|
|
221
|
+
if sys.platform != "darwin":
|
|
222
|
+
print(f"lone needs macOS (Apple Neural Engine); this is {sys.platform}.", file=sys.stderr)
|
|
223
|
+
raise typer.Exit(1)
|
|
224
|
+
app()
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
if __name__ == "__main__":
|
|
228
|
+
main()
|
lone/cli_options.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""CLI option enums shared by `cli.py` and `verify.py`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from enum import Enum
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ComputeUnits(str, Enum):
|
|
9
|
+
cpu_and_ne = "cpu_and_ne"
|
|
10
|
+
cpu_and_gpu = "cpu_and_gpu"
|
|
11
|
+
cpu_only = "cpu_only"
|
|
12
|
+
all = "all"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Precision(str, Enum):
|
|
16
|
+
fp16 = "fp16"
|
|
17
|
+
lut4 = "lut4"
|
|
18
|
+
lut6 = "lut6"
|
|
19
|
+
lut8 = "lut8"
|
lone/config.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Normalised model description shared by the converter and the runtime.
|
|
2
|
+
|
|
3
|
+
The converter never imports `transformers`: everything it needs is in `config.json`, and the
|
|
4
|
+
architecture differences between LFM3 and LFM2.5 are captured by three flags — `sandwich_norm`,
|
|
5
|
+
`attention_gate` and per-layer RoPE parameters.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from dataclasses import asdict, dataclass, field
|
|
13
|
+
|
|
14
|
+
ATTENTION_KINDS = ("full_attention", "sliding_attention")
|
|
15
|
+
CONV = "conv"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SpecError(ValueError):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class RopeSpec:
|
|
24
|
+
theta: float
|
|
25
|
+
partial: float = 1.0
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class ModelSpec:
|
|
30
|
+
model_type: str
|
|
31
|
+
hidden_size: int
|
|
32
|
+
intermediate_size: int
|
|
33
|
+
num_layers: int
|
|
34
|
+
layer_types: list[str]
|
|
35
|
+
num_heads: int
|
|
36
|
+
num_kv_heads: int
|
|
37
|
+
head_dim: int
|
|
38
|
+
conv_kernel: int
|
|
39
|
+
norm_eps: float
|
|
40
|
+
vocab_size: int
|
|
41
|
+
tie_word_embeddings: bool
|
|
42
|
+
sandwich_norm: bool
|
|
43
|
+
attention_gate: bool
|
|
44
|
+
qk_norm_prefix: str
|
|
45
|
+
rope: dict[str, RopeSpec]
|
|
46
|
+
sliding_window: int | None = None
|
|
47
|
+
eos_token_ids: list[int] = field(default_factory=list)
|
|
48
|
+
bos_token_id: int | None = None
|
|
49
|
+
conv_bias: bool = False
|
|
50
|
+
|
|
51
|
+
# ---------------------------------------------------------------- helpers
|
|
52
|
+
@property
|
|
53
|
+
def kv_groups(self) -> int:
|
|
54
|
+
return self.num_heads // self.num_kv_heads
|
|
55
|
+
|
|
56
|
+
def kind(self, layer: int) -> str:
|
|
57
|
+
return self.layer_types[layer]
|
|
58
|
+
|
|
59
|
+
def is_attention(self, layer: int) -> bool:
|
|
60
|
+
return self.layer_types[layer] in ATTENTION_KINDS
|
|
61
|
+
|
|
62
|
+
def rope_for(self, kind: str) -> RopeSpec:
|
|
63
|
+
try:
|
|
64
|
+
return self.rope[kind]
|
|
65
|
+
except KeyError: # pragma: no cover - guarded at load time
|
|
66
|
+
raise SpecError(f"no RoPE parameters for layer kind {kind!r}") from None
|
|
67
|
+
|
|
68
|
+
def layer_ids(self, kind: str) -> list[int]:
|
|
69
|
+
return [i for i, t in enumerate(self.layer_types) if t == kind]
|
|
70
|
+
|
|
71
|
+
def to_dict(self) -> dict:
|
|
72
|
+
d = asdict(self)
|
|
73
|
+
d["rope"] = {k: asdict(v) for k, v in self.rope.items()}
|
|
74
|
+
return d
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def from_dict(cls, d: dict) -> "ModelSpec":
|
|
78
|
+
d = dict(d)
|
|
79
|
+
d["rope"] = {k: RopeSpec(**v) for k, v in d["rope"].items()}
|
|
80
|
+
return cls(**d)
|
|
81
|
+
|
|
82
|
+
# ------------------------------------------------------------------ load
|
|
83
|
+
@classmethod
|
|
84
|
+
def from_dir(cls, model_dir: str) -> "ModelSpec":
|
|
85
|
+
with open(os.path.join(model_dir, "config.json")) as f:
|
|
86
|
+
cfg = json.load(f)
|
|
87
|
+
|
|
88
|
+
family = cfg.get("model_type")
|
|
89
|
+
if family not in ("lfm3", "lfm2"):
|
|
90
|
+
raise SpecError(
|
|
91
|
+
f"unsupported model_type {family!r}; LoNE converts LFM3 (lfm3) and LFM2.5 (lfm2)"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
layer_types = cfg.get("layer_types")
|
|
95
|
+
if not layer_types:
|
|
96
|
+
raise SpecError("config.json has no layer_types; cannot place conv/attention layers")
|
|
97
|
+
bad = set(layer_types) - {CONV, *ATTENTION_KINDS}
|
|
98
|
+
if bad:
|
|
99
|
+
raise SpecError(f"unknown layer types {sorted(bad)}")
|
|
100
|
+
|
|
101
|
+
num_heads = int(cfg["num_attention_heads"])
|
|
102
|
+
num_kv_heads = int(cfg.get("num_key_value_heads", num_heads))
|
|
103
|
+
head_dim = int(cfg.get("head_dim") or cfg["hidden_size"] // num_heads)
|
|
104
|
+
|
|
105
|
+
if family == "lfm3":
|
|
106
|
+
sandwich = bool(cfg.get("use_sandwich_norm", True))
|
|
107
|
+
gate = True
|
|
108
|
+
prefix = "q_norm"
|
|
109
|
+
rope_raw = cfg.get("rope_parameters") or {}
|
|
110
|
+
rope = {}
|
|
111
|
+
for kind in sorted({t for t in layer_types if t in ATTENTION_KINDS}):
|
|
112
|
+
params = rope_raw.get(kind)
|
|
113
|
+
if params is None:
|
|
114
|
+
raise SpecError(f"rope_parameters has no entry for {kind}")
|
|
115
|
+
rope[kind] = RopeSpec(
|
|
116
|
+
theta=float(params.get("rope_theta", 10000.0)),
|
|
117
|
+
partial=float(params.get("partial_rotary_factor", 1.0)),
|
|
118
|
+
)
|
|
119
|
+
sliding = cfg.get("sliding_window")
|
|
120
|
+
else: # lfm2
|
|
121
|
+
sandwich = bool(cfg.get("use_sandwich_norm", False))
|
|
122
|
+
gate = False
|
|
123
|
+
prefix = "q_layernorm"
|
|
124
|
+
rope_raw = cfg.get("rope_parameters") or {}
|
|
125
|
+
theta = float(rope_raw.get("rope_theta", cfg.get("rope_theta", 10000.0)))
|
|
126
|
+
partial = float(rope_raw.get("partial_rotary_factor", 1.0))
|
|
127
|
+
rope = {kind: RopeSpec(theta=theta, partial=partial) for kind in ATTENTION_KINDS}
|
|
128
|
+
sliding = cfg.get("sliding_window")
|
|
129
|
+
|
|
130
|
+
if any(t == "sliding_attention" for t in layer_types) and not sliding:
|
|
131
|
+
raise SpecError("sliding_attention layers present but config has no sliding_window")
|
|
132
|
+
if cfg.get("conv_bias"):
|
|
133
|
+
raise SpecError("conv_bias=true is not implemented")
|
|
134
|
+
|
|
135
|
+
eos = []
|
|
136
|
+
gen_path = os.path.join(model_dir, "generation_config.json")
|
|
137
|
+
if os.path.exists(gen_path):
|
|
138
|
+
with open(gen_path) as f:
|
|
139
|
+
gen = json.load(f)
|
|
140
|
+
eos = gen.get("eos_token_id") or []
|
|
141
|
+
if isinstance(eos, int):
|
|
142
|
+
eos = [eos]
|
|
143
|
+
cfg_eos = cfg.get("eos_token_id")
|
|
144
|
+
if isinstance(cfg_eos, int):
|
|
145
|
+
cfg_eos = [cfg_eos]
|
|
146
|
+
eos = list(dict.fromkeys([*(eos or []), *(cfg_eos or [])]))
|
|
147
|
+
|
|
148
|
+
spec = cls(
|
|
149
|
+
model_type=family,
|
|
150
|
+
hidden_size=int(cfg["hidden_size"]),
|
|
151
|
+
intermediate_size=int(cfg.get("intermediate_size") or cfg.get("block_ff_dim")),
|
|
152
|
+
num_layers=len(layer_types),
|
|
153
|
+
layer_types=list(layer_types),
|
|
154
|
+
num_heads=num_heads,
|
|
155
|
+
num_kv_heads=num_kv_heads,
|
|
156
|
+
head_dim=head_dim,
|
|
157
|
+
conv_kernel=int(cfg["conv_L_cache"]),
|
|
158
|
+
norm_eps=float(cfg.get("norm_eps", cfg.get("block_norm_eps", 1e-5))),
|
|
159
|
+
vocab_size=int(cfg["vocab_size"]),
|
|
160
|
+
tie_word_embeddings=bool(cfg.get("tie_word_embeddings", cfg.get("tie_embedding", True))),
|
|
161
|
+
sandwich_norm=sandwich,
|
|
162
|
+
attention_gate=gate,
|
|
163
|
+
qk_norm_prefix=prefix,
|
|
164
|
+
rope=rope,
|
|
165
|
+
sliding_window=sliding,
|
|
166
|
+
eos_token_ids=eos,
|
|
167
|
+
bos_token_id=cfg.get("bos_token_id"),
|
|
168
|
+
conv_bias=bool(cfg.get("conv_bias", False)),
|
|
169
|
+
)
|
|
170
|
+
spec.validate()
|
|
171
|
+
return spec
|
|
172
|
+
|
|
173
|
+
def validate(self) -> None:
|
|
174
|
+
if self.num_heads % self.num_kv_heads:
|
|
175
|
+
raise SpecError("num_attention_heads must be a multiple of num_key_value_heads")
|
|
176
|
+
for i in range(self.num_layers):
|
|
177
|
+
if self.is_attention(i) and self.kind(i) not in self.rope:
|
|
178
|
+
raise SpecError(f"layer {i} kind {self.kind(i)} has no RoPE spec")
|
|
179
|
+
if self.conv_kernel < 2:
|
|
180
|
+
raise SpecError(f"conv kernel {self.conv_kernel} is not causal")
|