open-jet 0.4.32__py3-none-win_amd64.whl

open_jet-0.4.32.data/purelib/megakernel/Makefile

@@ -0,0 +1,28 @@
+LLAMA ?= ../llama.cpp
+NVCC ?= /usr/local/cuda/bin/nvcc
+# the first GPU's architecture (sm_86 RTX 30, sm_89 RTX 40, sm_120 RTX 50); sm_86 if nvidia-smi is missing
+ARCH ?= $(or $(shell nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -n1 | tr -d '. '),86)
+ARCH := sm_$(patsubst sm_%,%,$(ARCH))
+NVFLAGS = -O3 -std=c++17 -arch=$(ARCH) -lineinfo -Xptxas -dlcm=cg -Xptxas -v -Xcompiler -fPIC
+
+all: qwen38_mk libqwen38mk.so tools/vocab_freq
+
+# ptxas register / spill report for every phase function goes to ptxas.log
+qwen38_mk.o: qwen38_mk.cu qwen38_mk.h gguf.h prefill.cuh prefill_gemm.cuh
+	$(NVCC) $(NVFLAGS) -c qwen38_mk.cu -o $@ 2> ptxas.log || (cat ptxas.log; false)
+
+qwen38_mk: qwen38_mk.o main.cu
+	$(NVCC) -O3 -std=c++17 -arch=$(ARCH) -I$(LLAMA)/include -I$(LLAMA)/ggml/include main.cu qwen38_mk.o \
+		-L$(LLAMA)/build/bin -lllama -Xlinker -rpath -Xlinker $(abspath $(LLAMA)/build/bin) -o $@
+
+# engine + tokenizer as a shared library with a C API (capi.cpp), used by server/mk_server.py
+libqwen38mk.so: qwen38_mk.o capi.cpp qwen38_mk.h
+	$(NVCC) -O3 -std=c++17 -arch=$(ARCH) -shared -Xcompiler -fPIC -I$(LLAMA)/include -I$(LLAMA)/ggml/include capi.cpp qwen38_mk.o \
+		-L$(LLAMA)/build/bin -lllama -Xlinker -rpath -Xlinker $(abspath $(LLAMA)/build/bin) -o $@
+
+tools/vocab_freq: tools/vocab_freq.cpp
+	g++ -O2 -std=c++17 -I$(LLAMA)/include -I$(LLAMA)/ggml/include $< -L$(LLAMA)/build/bin -lllama \
+		-Wl,-rpath,$(abspath $(LLAMA)/build/bin) -o $@
+
+clean:
+	rm -f qwen38_mk libqwen38mk.so qwen38_mk.o ptxas.log tools/vocab_freq

open_jet-0.4.32.data/purelib/megakernel/README.md

@@ -0,0 +1,203 @@
+# Qwen3.8-27B megakernel (RTX 30/40/50 series)
+
+A specialised CUDA inference engine for `Qwen3.8-27B-Q4_K_M.gguf` (llama.cpp arch `qwen35`:
+48 Gated-DeltaNet + 16 gated-attention layers, one MTP block) on RTX 30, 40 and 50-series GPUs
+with 24 GB+ (sm_86, sm_89, sm_120; tuned and benchmarked on an RTX 3090), with an OpenAI-compatible server that open-jet uses in place of `llama-server`
+(`llama_backend: auto`, see [docs/runtimes/llama-cpp.md](../docs/runtimes/llama-cpp.md)).
+
+Decoding runs an entire speculative cycle in one persistent cooperative kernel launch:
+
+```
+main model over T = 1 + n_draft tokens  ->  lm_head + verification on device (greedy, or exact
+speculative sampling)  ->  MTP catch-up rows for the accepted tokens  ->  n_draft MTP draft steps
+```
+
+The ~340 phases of a cycle (projections, DeltaNet core, attention, FFN, lm_head, MTP) are separated
+by a software grid barrier; the host only reads back the accepted tokens. Prompts go through
+batched tensor-core kernels instead (see Prefill).
+
+## Results
+
+RTX 3090, against `llama-server` with open-jet's flags (`--spec-type draft-mtp --spec-draft-n-max 2
+-fa on -ctk q8_0 -ctv q8_0`). Back-to-back on an otherwise idle GPU; numbers vary with prompt
+predictability (draft acceptance) and other GPU load.
+
+**Reasoning mode as open-jet runs it** (thinking on, xhigh effort, temperature 1.0, top_p 0.95,
+top_k 20; 5 coding prompts x 2 seeds, up to 1024 tokens each):
+
+| | llama.cpp | megakernel | |
+|---|---|---|---|
+| decode through the HTTP servers (`/v1/chat/completions`) | 50.9 tok/s | **78.0 tok/s** | **1.53x** |
+| decode, CLI (`bench/think_bench.py`, `-d 4`) | 50.7 tok/s | 77.4 tok/s | 1.53x |
+
+**Greedy decode, 4 chat prompts x 512 tokens** (`bench/code_bench.py`):
+
+| prompt | llama.cpp | megakernel `-d 3` | megakernel `-d 4` |
+|---|---|---|---|
+| Python function (code) | 72.5 | 126.3 | **139.7** (1.93x) |
+| hash map explanation | 64.1 | 107.4 | 106.5 |
+| short story | 52.8 | 66.8 | 70.7 |
+| Flask to FastAPI plan | 63.2 | 102.6 | 109.8 |
+| **mean** | 63.1 | 100.8 | **106.7** (1.69x) |
+
+Without speculation: llama.cpp 40.4 tok/s, megakernel 42.9 (`-d 0`): the decode itself is close to
+the memory-bandwidth limit for both; the lead comes from verifying 4-5 drafted tokens per pass at
+almost no extra cost.
+
+**A source file in the prompt, then a coding task** (greedy, 256 tokens; prompt = batched prefill of
+the whole prompt incl. the first token):
+
+| context | decode llama.cpp | decode megakernel | prompt llama.cpp | prompt megakernel |
+|---|---|---|---|---|
+| 1.8K | 57.4 | 84.8 | 1050 | 1595 |
+| 3.6K | 56.3 | 95.3 | 1124 | 1595 |
+| 7.8K | 54.8 | 85.8 | 1140 | 1533 |
+| 15.9K | 55.7 | 86.5 | 1126 | 1466 |
+| 29.5K | 52.3 | 72.7 | 1062 | 1236 |
+
+(tok/s; measured before the sequence-token draft vocabulary and double-buffered attention below,
+which add 1-8% to the megakernel's decode here.)
+
+**Pasting a file into the next turn of a conversation** (prompt processing time):
+
+| new tokens | llama.cpp (prompt cache) | megakernel |
+|---|---|---|
+| 2.7K | 2.41 s | 1.68 s |
+| 12K | 10.45 s | 8.10 s |
+
+With the q8_0 KV cache (chosen automatically for contexts above ~80K tokens, e.g. open-jet's 100K)
+decode is ~1-2% and prefill 2-7% slower than with fp16.
+
+**Correctness.** Teacher-forced on llama.cpp's own outputs, the megakernel's greedy prediction
+agrees at 99.3-100% of positions (`--score`: 99.6-100%); the few disagreements are near-ties from
+different float rounding. Speculative verification is exact: greedy output is the main model's
+greedy choice, and with sampling the token distribution is the same as without drafts (2000-sample
+test per position). Verifying 1 vs 4 tokens per pass rounds slightly differently, so on near-ties
+the output can differ between `-d` settings.
+
+## Build / run
+
+Requires CUDA 12+ (12.8+ for RTX 50 series; built with 13.2), an sm_86, sm_89 or sm_120 GPU with 24 GB, and the repo's llama.cpp build (only
+`libllama`, for tokenization, loaded vocab-only). `openjet setup` builds `libqwen38mk.so` automatically
+(this source ships in the open-jet wheel); building by hand:
+
+```sh
+cd megakernel
+make        # qwen38_mk (CLI), libqwen38mk.so (server library), tools/vocab_freq; ~2 min
+./qwen38_mk -m ../models/Qwen3.8-27B-Q4_K_M.gguf -p "Write quicksort in C" -n 512 \
+            -d 4 --draft-vocab draft_vocab_32k.bin --think --temp 1.0 --top-p 0.95 --top-k 20
+```
+
+CLI flags: `-d` drafts per cycle (0-4; with 4 the depth drops to 3 for a cycle after a rejected
+draft), `-c` KV capacity, `--kv f16|q8|auto` (KV cache format), `--think [xhigh|medium|low]`
+(reasoning mode; the chat template's default effort is xhigh), `--temp` / `--top-p` / `--top-k` /
+`--min-p` / `--seed` (sampling; default greedy), `--raw` (no chat template), `--ids`,
+`--prompt-ids f` (prompt as token ids), `--turn2 text` / `--turn2-file f` (a second user turn
+appended to the conversation), `--score ids.txt` (teacher-forced agreement), `MK_PROFILE=1`
+(per-phase timing, including a "min-composed" estimate that filters out time-slicing by other GPU
+users), `MK_PF_PROFILE=1` (per-kernel prefill timing), `MK_PREFILL=mk` (prefill through the decode
+megakernel instead).
+
+Benchmarks: `bench/think_bench.py` (reasoning mode, both engines), `bench/code_bench.py` (greedy
+decode, context sweep, second turn), `bench/bench.py` (decode on the 4 chat prompts);
+`bench/*_bench.cu` are GEMV / GEMM microbenchmarks.
+
+## Server
+
+`server/mk_server.py` serves the engine (through `libqwen38mk.so`, the C API in `capi.cpp`) with an
+OpenAI-compatible `/v1/chat/completions`, streaming or not, plus `/health` and `/v1/models`.
+open-jet starts it itself; standalone:
+
+```sh
+python3 server/mk_server.py -m ../models/Qwen3.8-27B-Q4_K_M.gguf --port 18080 -c 100000
+```
+
+- Prompts are rendered with the chat template stored in the GGUF (jinja2): `chat_template_kwargs`
+  `enable_thinking` / `preserve_thinking` / `reasoning_effort`, OpenAI `tools`. Reasoning is streamed
+  as `reasoning_content`, Qwen's XML tool calls are returned as structured `tool_calls`.
+- Successive requests of a conversation only prefill what is new: the prompt is matched against the
+  engine's sequence (prompts + emitted tokens), and when the re-rendered history diverges after the
+  end of the previous prompt (e.g. a reformatted tool call) the engine rewinds to a checkpoint
+  saved there. Anything else starts over (batched prefill, ~1500 tok/s).
+- Sampling: temperature, top_p, top_k (<= 32), min_p, seed; presence / repetition penalties are
+  ignored (with a warning). One request at a time.
+- KV cache: fp16, or q8_0 when fp16 does not fit the requested context (`--kv auto|f16|q8`).
+
+## How it gets its speed
+
+- **Weights stay in their GGUF k-quant encoding** (Q4_K / Q5_K / Q6_K / Q8_0), read once per
+  cycle regardless of T. Q6_K and Q8_0 are re-laid-out at load time (values unchanged) so every
+  lane does aligned 16/8-byte loads; Q6_K blocks carry their scales and `d` in one 8-byte word.
+- **dp4a GEMV with shared activations**: activations are int8 (per-16/32 scales, as llama.cpp's
+  q8_1), each warp streams 2-4 weight rows at once and reuses every shared-memory activation
+  load across them, so T = 4 costs almost the same memory time as T = 1. Weight loads bypass L1
+  and are marked L2 evict-first.
+- **No launches, few round trips**: activations are re-quantised by the producing phase
+  (including the next RMS norm's weight), so consumers only async-copy (`cp.async`) them.
+- **Gated-DeltaNet rollback by lazy replay**: the recurrent state is committed only through the
+  surely-accepted token; accepted drafts are replayed from a small (k, v, g, beta) ring at the
+  start of the next cycle, so rejected drafts cost no state snapshots or extra state traffic.
+- **Attention**: split-K flash decoding with tensor-core MMAs (Q.K^T and P.V in fp16 with fp32
+  accumulation, online softmax), double-buffered 32-position K/V tiles in shared memory (the next
+  tile streams in while the current one is processed: ~5% faster cycles at 30K context).
+- **MTP drafting on device**, with the draft lm_head restricted to the 32,768 most frequent
+  tokens (`draft_vocab_32k.bin`, built by `tools/vocab_freq` from local code + docs; covers
+  ~99.4% of tokens), plus the tokens of the current sequence that are missing from it, added on the
+  fly (identifiers of pasted code: +3-8% decode speed with code in context). Verification always
+  uses the full vocabulary, so this affects only the acceptance rate, never the output.
+- **Sampling** keeps the 32 best logits of each verified row; a draft is accepted with probability
+  p(draft) and a rejected one is resampled from the rest of the distribution, which keeps the
+  sampled distribution exact.
+- **L2 warm-up**: before each barrier, blocks prefetch (evict-last) what their next latency-bound
+  phase reads first (DeltaNet state, conv weights, norm weights, RoPE rows, first K/V tile).
+
+## Prefill
+
+A prompt (all but its last token) is processed in chunks of up to 2048 tokens by ordinary kernel
+launches (`prefill.cuh`), which leave exactly the state the megakernel expects (trunk and MTP K/V
+caches, committed DeltaNet state, conv ring, hidden-state history); the last prompt token then goes
+through the megakernel, which produces the first token and the first MTP drafts. The same path
+appends to a running conversation (`MKEngine::prefill` on a non-empty sequence, e.g. the next user
+turn): it starts at the committed DeltaNet position and also re-feeds the last emitted token and any
+accepted drafts not yet folded into the recurrent state; the output is token-identical to
+processing the whole conversation as one fresh prompt.
+
+- **GEMMs on fp16 tensor cores straight from the k-quants** (`prefill_gemm.cuh`, ~85% of prefill
+  time): 128 x 128 tiles, weights stay in their GGUF blocks and are dequantised to fp16 in
+  registers (exact small integers via the `0x6400` trick, then one fp16 scale and min per
+  sub-block); activations are fp16 with the elements of each 16-group permuted so that a 4-byte
+  weight word is directly an MMA A fragment. Products accumulate in fp16 over k = 128, then in fp32.
+  ~100-108 TFLOPS on all four quant types (`bench/mmq_bench.cu`). FFN gate and up share one GEMM
+  whose epilogue writes `silu(gate) * up` as the down projection's fp16 input.
+- **DeltaNet in chunked (WY) form on tensor cores**: per 32-token chunk, `A = tril(beta K K^T)`
+  with the gate decays, `T = (I + A)^-1` (one warp, forward substitution), then
+  `U = T beta V - T beta gamma (K S)`, `O = gamma Q S + P U` and `S <- gamma_last S + K^T (decay U)`.
+  Value columns are independent, so a block owns (head, 64 columns) and each warp keeps its 128 x 8
+  slice of the state in fp32 MMA accumulators, turning it (and K S, U) into fp16 MMA operands
+  in registers with `movmatrix`; gates are carried as log g, so tiny or zero gates are exact.
+  ~2% of prefill time; 3.5e-4 relative error against a float64 reference of the recurrence.
+- **Attention**: causal flash attention over the KV cache on tensor cores.
+
+## Limitations
+
+- Specialised to this model's shapes and quant mix (27B, Q4_K_M as published by unsloth); the loader
+  rejects any other model. Tuned on sm_86 (RTX 3090); open-jet enables it on sm_86, sm_89 and sm_120
+  GPUs with 24 GB or more. `make` builds for the installed GPU (`make ARCH=sm_89` to override).
+- Sampling supports temperature, top-k (<= 32), top-p and min-p (llama.cpp's order); no presence /
+  repetition penalties yet (open-jet's non-thinking preset uses a presence penalty of 1.5).
+- Text only; one sequence and one checkpoint: a conversation is reused while it grows (or diverges
+  only after the end of the previous prompt); editing an earlier part, e.g. context compaction,
+  re-prefills it. No slot save / restore (open-jet's low-memory shell swap falls back to
+  re-prompting).
+
+## Files
+
+- `qwen38_mk.cu` / `qwen38_mk.h` - loader, repacking, the megakernel and the `MKEngine` API
+- `prefill.cuh` / `prefill_gemm.cuh` - batched prefill kernels and their quantised fp16 GEMM
+- `capi.cpp` - C API (engine + libllama tokenizer) for `libqwen38mk.so`
+- `server/mk_server.py` - OpenAI-compatible HTTP server
+- `main.cu` - CLI (tokenization via libllama)
+- `gguf.h` - minimal mmap GGUF reader
+- `draft_vocab_32k.bin` - MTP draft-head token subset (int32 ids)
+- `tools/vocab_freq.cpp` - builds a draft vocabulary from a list of text files
+- `bench/` - benchmarks (see Build / run)

open_jet-0.4.32.data/purelib/megakernel/capi.cpp

@@ -0,0 +1,140 @@
+// C API over MKEngine plus libllama's tokenizer (vocab only), for the HTTP server (server/mk_server.py).
+#include <algorithm>
+#include <cstring>
+#include <exception>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "llama.h"
+#include "qwen38_mk.h"
+
+namespace {
+struct Handle {
+    MKEngine * eng = nullptr;
+    llama_model * lm = nullptr;
+    const llama_vocab * vocab = nullptr;
+    std::string err;
+};
+thread_local std::string g_err;
+}
+
+extern "C" {
+
+const char * mk_last_error() { return g_err.c_str(); }
+
+void * mk_new(const char * model, int ctx, int n_draft, const char * draft_vocab, int kv_mode) {
+    try {
+        llama_backend_init();
+        llama_model_params mp = llama_model_default_params();
+        mp.vocab_only = true;
+        auto * h = new Handle;
+        h->lm = llama_model_load_from_file(model, mp);
+        if (!h->lm) throw std::runtime_error("failed to load the vocabulary");
+        h->vocab = llama_model_get_vocab(h->lm);
+        h->eng = new MKEngine(model, ctx, n_draft, draft_vocab ? draft_vocab : "", kv_mode);
+        return h;
+    } catch (const std::exception & e) {
+        g_err = e.what();
+        return nullptr;
+    }
+}
+
+void mk_free(void * hp) {
+    auto * h = static_cast<Handle *>(hp);
+    delete h->eng;
+    llama_model_free(h->lm);
+    delete h;
+}
+
+// chat template stored in the GGUF (tokenizer.chat_template); returns its length, copies up to cap
+int mk_chat_template(void * hp, char * buf, int cap) {
+    auto * h = static_cast<Handle *>(hp);
+    const char * t = llama_model_chat_template(h->lm, nullptr);
+    if (!t) return -1;
+    const int n = (int) strlen(t);
+    if (buf && cap > 0) memcpy(buf, t, std::min(n, cap));
+    return n;
+}
+
+// text -> token ids (special tokens such as <|im_start|> parsed); returns the count (negative: -needed)
+int mk_tokenize(void * hp, const char * text, int len, int * out, int cap) {
+    auto * h = static_cast<Handle *>(hp);
+    return llama_tokenize(h->vocab, text, len, out, cap, false, true);
+}
+
+// token -> its bytes (special tokens rendered as text); returns the byte count
+int mk_piece(void * hp, int tok, char * buf, int cap) {
+    auto * h = static_cast<Handle *>(hp);
+    return llama_token_to_piece(h->vocab, tok, buf, cap, 0, true);
+}
+
+int mk_is_eog(void * hp, int tok) { return llama_vocab_is_eog(static_cast<Handle *>(hp)->vocab, tok) ? 1 : 0; }
+
+int mk_set_sampling(void * hp, float temp, float top_p, int top_k, float min_p, unsigned long long seed) {
+    try {
+        static_cast<Handle *>(hp)->eng->set_sampling(temp, top_p, top_k, min_p, seed);
+        return 0;
+    } catch (const std::exception & e) {
+        g_err = e.what();
+        return -1;
+    }
+}
+
+// appends tokens to the sequence; returns the first generated token (negative on error)
+int mk_prefill(void * hp, const int * toks, int n) {
+    try {
+        return static_cast<Handle *>(hp)->eng->prefill(std::vector<int>(toks, toks + n));
+    } catch (const std::exception & e) {
+        g_err = e.what();
+        return -1;
+    }
+}
+
+// one decode cycle; writes 1..n_draft+1 tokens; returns the count (negative on error)
+int mk_step(void * hp, int * out, int cap) {
+    try {
+        std::vector<int> v;
+        static_cast<Handle *>(hp)->eng->step(v);
+        const int n = std::min((int) v.size(), cap);
+        for (int i = 0; i < n; i++) out[i] = v[i];
+        return n;
+    } catch (const std::exception & e) {
+        g_err = e.what();
+        return -1;
+    }
+}
+
+void mk_reset(void * hp) { static_cast<Handle *>(hp)->eng->reset(); }
+
+int mk_truncate(void * hp, int len) {
+    try {
+        static_cast<Handle *>(hp)->eng->truncate(len);
+        return 0;
+    } catch (const std::exception & e) {
+        g_err = e.what();
+        return -1;
+    }
+}
+
+// copies the sequence so far (up to cap tokens); returns its length
+int mk_sequence(void * hp, int * out, int cap) {
+    const auto & s = static_cast<Handle *>(hp)->eng->sequence();
+    const int n = (int) s.size();
+    if (out)
+        for (int i = 0; i < std::min(n, cap); i++) out[i] = s[i];
+    return n;
+}
+
+int mk_committed(void * hp) { return static_cast<Handle *>(hp)->eng->committed(); }
+int mk_checkpoint_pos(void * hp) { return static_cast<Handle *>(hp)->eng->checkpoint_pos(); }
+int mk_rewind_to_checkpoint(void * hp) { return static_cast<Handle *>(hp)->eng->rewind_to_checkpoint() ? 0 : -1; }
+
+void mk_stats(void * hp, long * cycles, long * drafted, long * accepted) {
+    const auto & s = static_cast<Handle *>(hp)->eng->stats;
+    *cycles = s.cycles;
+    *drafted = s.drafted;
+    *accepted = s.accepted;
+}
+
+} // extern "C"

open_jet-0.4.32.data/purelib/megakernel/gguf.h

@@ -0,0 +1,129 @@
+// Minimal read-only GGUF parser (mmap based). Only what the megakernel loader needs.
+#pragma once
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+namespace gguf {
+
+enum GType : uint32_t { F32 = 0, F16 = 1, Q8_0 = 8, Q4_K = 12, Q5_K = 13, Q6_K = 14 };
+
+struct Tensor {
+    std::string name;
+    uint32_t type = 0;
+    std::vector<uint64_t> ne;
+    uint64_t offset = 0;
+    const uint8_t * data = nullptr;
+    uint64_t nbytes = 0;
+};
+
+inline uint64_t type_row_bytes(uint32_t type, uint64_t n) {
+    switch (type) {
+        case F32: return n * 4;
+        case F16: return n * 2;
+        case Q8_0: return n / 32 * 34;
+        case Q4_K: return n / 256 * 144;
+        case Q5_K: return n / 256 * 176;
+        case Q6_K: return n / 256 * 210;
+        default: throw std::runtime_error("unsupported ggml type " + std::to_string(type));
+    }
+}
+
+struct File {
+    const uint8_t * base = nullptr;
+    size_t size = 0;
+    std::map<std::string, Tensor> tensors;
+    std::map<std::string, uint64_t> ints;
+    std::map<std::string, std::string> strs;
+
+    explicit File(const std::string & path) {
+        int fd = open(path.c_str(), O_RDONLY);
+        if (fd < 0) throw std::runtime_error("cannot open " + path);
+        struct stat st;
+        fstat(fd, &st);
+        size = (size_t) st.st_size;
+        void * p = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0);
+        close(fd);
+        if (p == MAP_FAILED) throw std::runtime_error("mmap failed");
+        base = (const uint8_t *) p;
+        parse();
+    }
+    ~File() { if (base) munmap((void *) base, size); }
+
+    const Tensor & get(const std::string & name) const {
+        auto it = tensors.find(name);
+        if (it == tensors.end()) throw std::runtime_error("missing tensor " + name);
+        return it->second;
+    }
+
+  private:
+    size_t pos = 0;
+    template <class T> T rd() { T v; memcpy(&v, base + pos, sizeof(T)); pos += sizeof(T); return v; }
+    std::string rds() { uint64_t n = rd<uint64_t>(); std::string s((const char *) base + pos, n); pos += n; return s; }
+
+    void skip_val(uint32_t t, const std::string & key, bool top) {
+        switch (t) {
+            case 0: case 1: case 7: { uint8_t v = rd<uint8_t>(); if (top) ints[key] = v; break; }
+            case 2: case 3: pos += 2; break;
+            case 4: { uint32_t v = rd<uint32_t>(); if (top) ints[key] = v; break; }
+            case 5: { int32_t v = rd<int32_t>(); if (top) ints[key] = (uint64_t) v; break; }
+            case 6: pos += 4; break;
+            case 8: { std::string s = rds(); if (top) strs[key] = s; break; }
+            case 9: {
+                uint32_t et = rd<uint32_t>();
+                uint64_t n = rd<uint64_t>();
+                for (uint64_t i = 0; i < n; i++) skip_val(et, key, false);
+                break;
+            }
+            case 10: { uint64_t v = rd<uint64_t>(); if (top) ints[key] = v; break; }
+            case 11: { int64_t v = rd<int64_t>(); if (top) ints[key] = (uint64_t) v; break; }
+            case 12: pos += 8; break;
+            default: throw std::runtime_error("bad gguf kv type");
+        }
+    }
+
+    void parse() {
+        if (memcmp(base, "GGUF", 4) != 0) throw std::runtime_error("not a gguf file");
+        pos = 4;
+        rd<uint32_t>();
+        uint64_t n_tensors = rd<uint64_t>();
+        uint64_t n_kv = rd<uint64_t>();
+        for (uint64_t i = 0; i < n_kv; i++) {
+            std::string key = rds();
+            uint32_t t = rd<uint32_t>();
+            skip_val(t, key, true);
+        }
+        std::vector<Tensor> infos;
+        for (uint64_t i = 0; i < n_tensors; i++) {
+            Tensor t;
+            t.name = rds();
+            uint32_t nd = rd<uint32_t>();
+            for (uint32_t d = 0; d < nd; d++) t.ne.push_back(rd<uint64_t>());
+            t.type = rd<uint32_t>();
+            t.offset = rd<uint64_t>();
+            infos.push_back(t);
+        }
+        uint64_t align = ints.count("general.alignment") ? ints["general.alignment"] : 32;
+        size_t data_start = (pos + align - 1) / align * align;
+        for (auto & t : infos) {
+            t.data = base + data_start + t.offset;
+            uint64_t rows = 1;
+            for (size_t d = 1; d < t.ne.size(); d++) rows *= t.ne[d];
+            t.nbytes = type_row_bytes(t.type, t.ne[0]) * rows;
+            tensors[t.name] = t;
+        }
+    }
+};
+
+} // namespace gguf