lone-ane 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
lone/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """LoNE — LFM models on the Apple Neural Engine."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ MANIFEST_FORMAT = "lone-1"
lone/cli.py ADDED
@@ -0,0 +1,228 @@
1
+ """`lone` command line: convert, run, serve, inspect, verify.
2
+
3
+ Typer wraps the same core functions; option names and defaults are unchanged from the argparse
4
+ version. `main()` is kept as the console-script target.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import os
11
+ import sys
12
+ import time
13
+
14
+ import typer
15
+
16
+ from .cli_options import ComputeUnits, Precision
17
+
18
+ app = typer.Typer(
19
+ add_completion=False,
20
+ no_args_is_help=True,
21
+ help="LFM models on the Apple Neural Engine",
22
+ )
23
+
24
+
25
+ def _units(cpu_only: bool, compute_units: ComputeUnits) -> str:
26
+ return "cpu_only" if cpu_only else compute_units.value
27
+
28
+
29
+ def _report_wired(wired_bytes: int) -> None:
30
+ if wired_bytes:
31
+ print(f"wired {wired_bytes / 2**20:.0f} MiB of caches", file=sys.stderr)
32
+ else:
33
+ print("cache wiring unavailable (needs macOS mlock); continuing unwired", file=sys.stderr)
34
+
35
+
36
+ @app.command(help="convert an HF checkpoint into a LoNE bundle")
37
+ def convert(
38
+ model: str = typer.Option(..., "--model", help="HF checkpoint directory"),
39
+ out: str = typer.Option(..., "--out", help="bundle output directory"),
40
+ ctx: int = typer.Option(2048, "--ctx"),
41
+ prefill: int = typer.Option(64, "--prefill"),
42
+ layers_per_model: int = typer.Option(
43
+ 4, "--layers-per-model", help="decoder layers per Core ML graph; fewer graphs are faster, 4 verifies at parity"
44
+ ),
45
+ precision: Precision = typer.Option(Precision.fp16, "--precision"),
46
+ head_chunks: int = typer.Option(
47
+ 0, "--head-chunks", help="vocabulary slices for the LM head; 0 sizes each graph to ~128 MB"
48
+ ),
49
+ ring: bool = typer.Option(
50
+ True, "--ring/--no-ring",
51
+ help="ring-buffer sliding layers to their window instead of the full ctx (default on)",
52
+ ),
53
+ decode_only: bool = typer.Option(False, "--decode-only", help="skip the prefill models (slower prompts)"),
54
+ prefill_only: bool = typer.Option(False, "--prefill-only", help="skip the decode models (prefill timing only)"),
55
+ spec_drafts: int = typer.Option(
56
+ 0, "--spec-drafts",
57
+ help="convert a verify shape for n-gram speculative decoding: prefill becomes drafts+1 and "
58
+ "the head is traced at that length (0 = off)",
59
+ ),
60
+ plan: bool = typer.Option(True, "--plan/--no-plan", help="report the MLComputePlan placement per model"),
61
+ compute_units: ComputeUnits = typer.Option(ComputeUnits.cpu_and_ne, "--compute-units"),
62
+ ) -> None:
63
+ from .convert import convert_model
64
+
65
+ convert_model(
66
+ model_dir=model,
67
+ out_dir=out,
68
+ ctx=ctx,
69
+ prefill=prefill,
70
+ layers_per_model=layers_per_model,
71
+ precision=precision.value,
72
+ head_chunks=head_chunks,
73
+ ring=ring,
74
+ decode_only=decode_only,
75
+ prefill_only=prefill_only,
76
+ spec_drafts=spec_drafts,
77
+ check_plan=plan,
78
+ compute_units=compute_units.value,
79
+ )
80
+
81
+
82
+ @app.command(help="generate from a bundle")
83
+ def run(
84
+ bundle: str = typer.Option(..., "--bundle"),
85
+ prompt: str = typer.Option(..., "--prompt"),
86
+ chat: bool = typer.Option(False, "--chat", help="render the prompt through the chat template"),
87
+ system: str | None = typer.Option(None, "--system"),
88
+ max_new_tokens: int = typer.Option(256, "--max-new-tokens"),
89
+ temperature: float = typer.Option(0.0, "--temperature"),
90
+ top_p: float = typer.Option(1.0, "--top-p"),
91
+ top_k: int = typer.Option(0, "--top-k"),
92
+ seed: int | None = typer.Option(None, "--seed"),
93
+ stream: bool = typer.Option(False, "--stream"),
94
+ cpu_only: bool = typer.Option(False, "--cpu-only"),
95
+ compute_units: ComputeUnits = typer.Option(ComputeUnits.cpu_and_ne, "--compute-units"),
96
+ wire_caches: bool = typer.Option(
97
+ False, "--wire-caches", help="wire the KV and conv caches with mlock so they cannot be paged out (macOS)"
98
+ ),
99
+ spec_ngram: bool = typer.Option(
100
+ False, "--spec-ngram/--no-spec-ngram",
101
+ help="n-gram speculative decoding (llama.cpp ngram-simple); needs a bundle converted with --spec-drafts",
102
+ ),
103
+ spec_n: int = typer.Option(12, "--spec-n", help="lookup n-gram length for --spec-ngram"),
104
+ spec_m: int = typer.Option(16, "--spec-m", help="maximum draft length for --spec-ngram"),
105
+ spec_drafts: int = typer.Option(
106
+ 0, "--spec-drafts", help="cap on verified drafts per step for --spec-ngram (0 = the bundle's)"
107
+ ),
108
+ ) -> None:
109
+ from .runtime import LoneModel
110
+ from .spec import NgramProposer
111
+
112
+ model = LoneModel(bundle, _units(cpu_only, compute_units), wire_caches=wire_caches)
113
+ if wire_caches:
114
+ _report_wired(model.wired_bytes)
115
+ if chat:
116
+ messages = ([{"role": "system", "content": system}] if system else []) + [
117
+ {"role": "user", "content": prompt}
118
+ ]
119
+ prompt_ids = model.tokenizer.chat(messages)
120
+ else:
121
+ prompt_ids = model.tokenizer.encode(prompt)
122
+
123
+ first = {"at": None}
124
+ emitted = []
125
+ started = time.perf_counter()
126
+
127
+ def on_token(token: int) -> None:
128
+ if first["at"] is None:
129
+ first["at"] = time.perf_counter()
130
+ emitted.append(token)
131
+ if stream:
132
+ sys.stdout.write(model.tokenizer.decode([token]))
133
+ sys.stdout.flush()
134
+
135
+ proposer = NgramProposer(spec_n, spec_m) if spec_ngram else None
136
+ ids = model.generate(
137
+ prompt_ids,
138
+ max_new_tokens=max_new_tokens,
139
+ temperature=temperature,
140
+ top_p=top_p,
141
+ top_k=top_k,
142
+ seed=seed,
143
+ on_token=on_token,
144
+ proposer=proposer,
145
+ max_drafts=spec_drafts,
146
+ )
147
+ wall = time.perf_counter() - started
148
+ text = model.tokenizer.decode(ids)
149
+ if stream:
150
+ sys.stdout.write("\n")
151
+ else:
152
+ print(text)
153
+ prefill_s = (first["at"] or started) - started
154
+ decode_s = max(wall - prefill_s, 1e-9)
155
+ decode_tokens = max(len(ids) - 1, 0)
156
+ rate = decode_tokens / decode_s if decode_tokens else 0.0
157
+ spec = ""
158
+ stats = model.spec_stats
159
+ if proposer is not None:
160
+ drafted = stats.get("drafted", 0)
161
+ hit = f" ({stats.get('accepted', 0) / drafted:.0%})" if drafted else ""
162
+ spec = (f" | spec {stats.get('accepted', 0)}/{drafted} drafts{hit}"
163
+ f" in {stats.get('passes', 0)} passes")
164
+ print(f"\n[{len(ids)} tokens | {rate:.1f} tok/s | first token {prefill_s * 1e3:.0f} ms | "
165
+ f"{len(prompt_ids)} prompt tokens{spec}]", file=sys.stderr)
166
+
167
+
168
+ @app.command(help="OpenAI-compatible HTTP server")
169
+ def serve(
170
+ bundle: str = typer.Option(..., "--bundle"),
171
+ host: str = typer.Option("127.0.0.1", "--host"),
172
+ port: int = typer.Option(8082, "--port"),
173
+ cpu_only: bool = typer.Option(False, "--cpu-only"),
174
+ compute_units: ComputeUnits = typer.Option(ComputeUnits.cpu_and_ne, "--compute-units"),
175
+ wire_caches: bool = typer.Option(
176
+ False, "--wire-caches", help="wire the KV and conv caches with mlock so they cannot be paged out (macOS)"
177
+ ),
178
+ ) -> None:
179
+ import uvicorn
180
+
181
+ from .server import create_app
182
+
183
+ app = create_app(bundle, _units(cpu_only, compute_units), wire_caches=wire_caches)
184
+ if wire_caches:
185
+ _report_wired(app.state.wired_bytes)
186
+ uvicorn.run(app, host=host, port=port, log_level="info")
187
+
188
+
189
+ @app.command("inspect", help="print a bundle's spec and placement")
190
+ def inspect_cmd(bundle: str = typer.Option(..., "--bundle")) -> None:
191
+ with open(os.path.join(bundle, "manifest.json")) as f:
192
+ manifest = json.load(f)
193
+ spec = manifest["spec"]
194
+ print(f"{spec['model_type']}: {spec['num_layers']} layers, hidden {spec['hidden_size']}, "
195
+ f"{spec['num_heads']}/{spec['num_kv_heads']} heads, head_dim {spec['head_dim']}, "
196
+ f"conv {spec['conv_kernel']}, vocab {spec['vocab_size']}")
197
+ print(f"shapes: ctx {manifest['shapes']['ctx']}, prefill {manifest['shapes']['prefill']}, "
198
+ f"precision {manifest['precision']}")
199
+ drafts = manifest.get("speculative", {}).get("drafts", 0)
200
+ if drafts:
201
+ print(f"spec: n-gram verify shape, {drafts} drafts/pass, head {manifest['head'].get('seq')} rows")
202
+ window = spec.get("sliding_window")
203
+ if manifest.get("ring") and window:
204
+ print(f"ring: sliding cache {min(window, manifest['shapes']['ctx'])} rows (window {window})")
205
+ for chunk in manifest["chunks"]:
206
+ tail = ", ".join(
207
+ f"{tag}: {meta.get('plan', {}).get('devices', 'n/a')}" for tag, meta in chunk["models"].items()
208
+ )
209
+ print(f" {chunk['name']} layers {chunk['layers']} -> {tail}")
210
+ print(f" head: {len(manifest['head']['chunks'])} chunks")
211
+
212
+
213
+ from .repl import register_chat # noqa: E402 (needs `app` above)
214
+ from .verify import register_verify # noqa: E402 (needs `app` above)
215
+
216
+ register_chat(app)
217
+ register_verify(app)
218
+
219
+
220
+ def main() -> None:
221
+ if sys.platform != "darwin":
222
+ print(f"lone needs macOS (Apple Neural Engine); this is {sys.platform}.", file=sys.stderr)
223
+ raise typer.Exit(1)
224
+ app()
225
+
226
+
227
+ if __name__ == "__main__":
228
+ main()
lone/cli_options.py ADDED
@@ -0,0 +1,19 @@
1
+ """CLI option enums shared by `cli.py` and `verify.py`."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from enum import Enum
6
+
7
+
8
+ class ComputeUnits(str, Enum):
9
+ cpu_and_ne = "cpu_and_ne"
10
+ cpu_and_gpu = "cpu_and_gpu"
11
+ cpu_only = "cpu_only"
12
+ all = "all"
13
+
14
+
15
+ class Precision(str, Enum):
16
+ fp16 = "fp16"
17
+ lut4 = "lut4"
18
+ lut6 = "lut6"
19
+ lut8 = "lut8"
lone/config.py ADDED
@@ -0,0 +1,180 @@
1
+ """Normalised model description shared by the converter and the runtime.
2
+
3
+ The converter never imports `transformers`: everything it needs is in `config.json`, and the
4
+ architecture differences between LFM3 and LFM2.5 are captured by three flags — `sandwich_norm`,
5
+ `attention_gate` and per-layer RoPE parameters.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import os
12
+ from dataclasses import asdict, dataclass, field
13
+
14
+ ATTENTION_KINDS = ("full_attention", "sliding_attention")
15
+ CONV = "conv"
16
+
17
+
18
+ class SpecError(ValueError):
19
+ pass
20
+
21
+
22
+ @dataclass
23
+ class RopeSpec:
24
+ theta: float
25
+ partial: float = 1.0
26
+
27
+
28
+ @dataclass
29
+ class ModelSpec:
30
+ model_type: str
31
+ hidden_size: int
32
+ intermediate_size: int
33
+ num_layers: int
34
+ layer_types: list[str]
35
+ num_heads: int
36
+ num_kv_heads: int
37
+ head_dim: int
38
+ conv_kernel: int
39
+ norm_eps: float
40
+ vocab_size: int
41
+ tie_word_embeddings: bool
42
+ sandwich_norm: bool
43
+ attention_gate: bool
44
+ qk_norm_prefix: str
45
+ rope: dict[str, RopeSpec]
46
+ sliding_window: int | None = None
47
+ eos_token_ids: list[int] = field(default_factory=list)
48
+ bos_token_id: int | None = None
49
+ conv_bias: bool = False
50
+
51
+ # ---------------------------------------------------------------- helpers
52
+ @property
53
+ def kv_groups(self) -> int:
54
+ return self.num_heads // self.num_kv_heads
55
+
56
+ def kind(self, layer: int) -> str:
57
+ return self.layer_types[layer]
58
+
59
+ def is_attention(self, layer: int) -> bool:
60
+ return self.layer_types[layer] in ATTENTION_KINDS
61
+
62
+ def rope_for(self, kind: str) -> RopeSpec:
63
+ try:
64
+ return self.rope[kind]
65
+ except KeyError: # pragma: no cover - guarded at load time
66
+ raise SpecError(f"no RoPE parameters for layer kind {kind!r}") from None
67
+
68
+ def layer_ids(self, kind: str) -> list[int]:
69
+ return [i for i, t in enumerate(self.layer_types) if t == kind]
70
+
71
+ def to_dict(self) -> dict:
72
+ d = asdict(self)
73
+ d["rope"] = {k: asdict(v) for k, v in self.rope.items()}
74
+ return d
75
+
76
+ @classmethod
77
+ def from_dict(cls, d: dict) -> "ModelSpec":
78
+ d = dict(d)
79
+ d["rope"] = {k: RopeSpec(**v) for k, v in d["rope"].items()}
80
+ return cls(**d)
81
+
82
+ # ------------------------------------------------------------------ load
83
+ @classmethod
84
+ def from_dir(cls, model_dir: str) -> "ModelSpec":
85
+ with open(os.path.join(model_dir, "config.json")) as f:
86
+ cfg = json.load(f)
87
+
88
+ family = cfg.get("model_type")
89
+ if family not in ("lfm3", "lfm2"):
90
+ raise SpecError(
91
+ f"unsupported model_type {family!r}; LoNE converts LFM3 (lfm3) and LFM2.5 (lfm2)"
92
+ )
93
+
94
+ layer_types = cfg.get("layer_types")
95
+ if not layer_types:
96
+ raise SpecError("config.json has no layer_types; cannot place conv/attention layers")
97
+ bad = set(layer_types) - {CONV, *ATTENTION_KINDS}
98
+ if bad:
99
+ raise SpecError(f"unknown layer types {sorted(bad)}")
100
+
101
+ num_heads = int(cfg["num_attention_heads"])
102
+ num_kv_heads = int(cfg.get("num_key_value_heads", num_heads))
103
+ head_dim = int(cfg.get("head_dim") or cfg["hidden_size"] // num_heads)
104
+
105
+ if family == "lfm3":
106
+ sandwich = bool(cfg.get("use_sandwich_norm", True))
107
+ gate = True
108
+ prefix = "q_norm"
109
+ rope_raw = cfg.get("rope_parameters") or {}
110
+ rope = {}
111
+ for kind in sorted({t for t in layer_types if t in ATTENTION_KINDS}):
112
+ params = rope_raw.get(kind)
113
+ if params is None:
114
+ raise SpecError(f"rope_parameters has no entry for {kind}")
115
+ rope[kind] = RopeSpec(
116
+ theta=float(params.get("rope_theta", 10000.0)),
117
+ partial=float(params.get("partial_rotary_factor", 1.0)),
118
+ )
119
+ sliding = cfg.get("sliding_window")
120
+ else: # lfm2
121
+ sandwich = bool(cfg.get("use_sandwich_norm", False))
122
+ gate = False
123
+ prefix = "q_layernorm"
124
+ rope_raw = cfg.get("rope_parameters") or {}
125
+ theta = float(rope_raw.get("rope_theta", cfg.get("rope_theta", 10000.0)))
126
+ partial = float(rope_raw.get("partial_rotary_factor", 1.0))
127
+ rope = {kind: RopeSpec(theta=theta, partial=partial) for kind in ATTENTION_KINDS}
128
+ sliding = cfg.get("sliding_window")
129
+
130
+ if any(t == "sliding_attention" for t in layer_types) and not sliding:
131
+ raise SpecError("sliding_attention layers present but config has no sliding_window")
132
+ if cfg.get("conv_bias"):
133
+ raise SpecError("conv_bias=true is not implemented")
134
+
135
+ eos = []
136
+ gen_path = os.path.join(model_dir, "generation_config.json")
137
+ if os.path.exists(gen_path):
138
+ with open(gen_path) as f:
139
+ gen = json.load(f)
140
+ eos = gen.get("eos_token_id") or []
141
+ if isinstance(eos, int):
142
+ eos = [eos]
143
+ cfg_eos = cfg.get("eos_token_id")
144
+ if isinstance(cfg_eos, int):
145
+ cfg_eos = [cfg_eos]
146
+ eos = list(dict.fromkeys([*(eos or []), *(cfg_eos or [])]))
147
+
148
+ spec = cls(
149
+ model_type=family,
150
+ hidden_size=int(cfg["hidden_size"]),
151
+ intermediate_size=int(cfg.get("intermediate_size") or cfg.get("block_ff_dim")),
152
+ num_layers=len(layer_types),
153
+ layer_types=list(layer_types),
154
+ num_heads=num_heads,
155
+ num_kv_heads=num_kv_heads,
156
+ head_dim=head_dim,
157
+ conv_kernel=int(cfg["conv_L_cache"]),
158
+ norm_eps=float(cfg.get("norm_eps", cfg.get("block_norm_eps", 1e-5))),
159
+ vocab_size=int(cfg["vocab_size"]),
160
+ tie_word_embeddings=bool(cfg.get("tie_word_embeddings", cfg.get("tie_embedding", True))),
161
+ sandwich_norm=sandwich,
162
+ attention_gate=gate,
163
+ qk_norm_prefix=prefix,
164
+ rope=rope,
165
+ sliding_window=sliding,
166
+ eos_token_ids=eos,
167
+ bos_token_id=cfg.get("bos_token_id"),
168
+ conv_bias=bool(cfg.get("conv_bias", False)),
169
+ )
170
+ spec.validate()
171
+ return spec
172
+
173
+ def validate(self) -> None:
174
+ if self.num_heads % self.num_kv_heads:
175
+ raise SpecError("num_attention_heads must be a multiple of num_key_value_heads")
176
+ for i in range(self.num_layers):
177
+ if self.is_attention(i) and self.kind(i) not in self.rope:
178
+ raise SpecError(f"layer {i} kind {self.kind(i)} has no RoPE spec")
179
+ if self.conv_kernel < 2:
180
+ raise SpecError(f"conv kernel {self.conv_kernel} is not causal")