glassllm 0.1.0__tar.gz

glassllm-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+dist/
+build/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+
+# Node / Astro
+node_modules/
+site/dist/
+site/.astro/
+site/src/generated/
+site/public/py/
+
+# Notebooks
+.ipynb_checkpoints/
+
+# Editors and OS
+.vscode/
+.idea/
+.DS_Store
+
+# Corpus (regenerate with scripts/prepare_corpus.py)
+data/

glassllm-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Oséias Dias de Farias
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

glassllm-0.1.0/PKG-INFO

@@ -0,0 +1,137 @@
+Metadata-Version: 2.5
+Name: glassllm
+Version: 0.1.0
+Summary: A small Transformer written from scratch in NumPy, built to be looked at.
+Project-URL: Homepage, https://oseiasdfarias.github.io/glassllm/
+Project-URL: Documentation, https://oseiasdfarias.github.io/glassllm/en/docs/
+Project-URL: Source, https://github.com/Oseiasdfarias/glassllm
+Project-URL: Issues, https://github.com/Oseiasdfarias/glassllm/issues
+Author: Oséias Dias de Farias
+License-Expression: MIT
+License-File: LICENSE
+Keywords: attention,autograd,education,llm,numpy,transformer
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.26
+Description-Content-Type: text/markdown
+
+![glassllm — see every step of a small language model](https://raw.githubusercontent.com/Oseiasdfarias/glassllm/main/docs/assets/banner-light.svg)
+
+# glassllm
+
+A small Transformer written from scratch in NumPy, built to be looked at. Follow a token through
+the model and inspect the equation, source code and values produced at each step.
+
+[![PyPI version](https://img.shields.io/pypi/v/glassllm)](https://pypi.org/project/glassllm/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/)
+[![NumPy only](https://img.shields.io/badge/runtime-NumPy%20only-4fb3c2)](https://numpy.org/)
+[![MIT license](https://img.shields.io/badge/license-MIT-4fb3c2)](https://github.com/Oseiasdfarias/glassllm/blob/main/LICENSE)
+
+**English** · [Português](https://github.com/Oseiasdfarias/glassllm/blob/main/README.pt-BR.md) ·
+[Documentation](https://oseiasdfarias.github.io/glassllm/)
+
+glassllm is an educational language model and Python library. Its decoder-only Transformer,
+tokenizer, automatic differentiation engine, training code and tracing tools are implemented
+with NumPy. Pretrained Portuguese and English models are included in the package.
+
+## Install
+
+```bash
+python -m pip install glassllm
+```
+
+The library supports Python 3.10 and later. NumPy is its only runtime dependency. The pretrained
+weights and tokenizers are included, so inference works immediately after installation.
+
+## Generate text
+
+The `glassllm` command uses the included English model by default. Select `pt` for Portuguese.
+
+```bash
+glassllm --lang en generate "Alice was beginning to" --n 40
+glassllm --lang pt generate "Capitu olhou para" --n 40
+```
+
+Use greedy decoding for repeatable output, or inspect the available model operations:
+
+```bash
+glassllm --lang en generate "Alice was beginning to" --n 40 --greedy
+glassllm steps
+glassllm trace "Alice was beginning to" --step attn.softmax --layer 0 --head 1
+```
+
+## Inspect a calculation
+
+Pass a `Tracer` to the model to record its actual intermediate values. Each record includes the
+equation, tensor shape and source location for a tagged operation.
+
+```python
+from glassllm import Tracer, load_pretrained
+
+model, tokenizer = load_pretrained("en")
+token_ids = tokenizer.encode("Alice was beginning to")
+
+tracer = Tracer()
+model.forward(token_ids, trace=tracer)
+
+step = tracer.get("attn.softmax", layer=0, head=1)
+print(step.equation)
+print(step.shape)
+print(step.value)
+```
+
+List all traceable operations with `glassllm steps`. The same tagged calculations are explained
+in the [library documentation](https://oseiasdfarias.github.io/glassllm/en/docs/biblioteca/api-publica/).
+
+## What happens inside the model
+
+Each prediction follows the same sequence of real calculations. The website can replay all 77
+recorded operations for a prediction and lets you inspect their equations, tensors and source.
+
+```text
+text → BPE tokens → embeddings → 4 Transformer blocks → logits → probabilities → next token
+                                  attention + SwiGLU
+```
+
+The model uses causal self-attention, RoPE, grouped-query attention, an attention KV cache,
+RMSNorm, SwiGLU and tied input/output embeddings. It has 852,608 parameters and runs on CPU.
+It is deliberately small for learning; it is not intended for production text generation.
+
+## Interactive site and documentation
+
+Try the [live bilingual lab](https://oseiasdfarias.github.io/glassllm/). It runs the same Python
+wheel in a Pyodide worker in your browser. The network view shows the complete model architecture,
+and slow motion steps through each recorded calculation at a readable pace.
+
+The documentation includes 35 pages in each language, with interface guides, inference and
+training flowcharts, mathematical foundations, API reference, examples and research references:
+
+- [Documentation in English](https://oseiasdfarias.github.io/glassllm/en/docs/)
+- [Documentação em português](https://oseiasdfarias.github.io/glassllm/pt/docs/)
+
+## Training and data
+
+The included models were trained on public-domain books by Machado de Assis (Portuguese) and
+Lewis Carroll (English), sourced from [Project Gutenberg](https://www.gutenberg.org/). Training
+scripts and dataset preparation live in the repository. PyTorch is used only for development
+and parity checks; it is not installed when users install the library.
+
+## Contributing
+
+The source code, tests, design notes and issue tracker are on
+[GitHub](https://github.com/Oseiasdfarias/glassllm). See the
+[contribution guide](https://github.com/Oseiasdfarias/glassllm/blob/main/CONTRIBUTING.md) before
+opening a pull request.
+
+The interactive lab and complete library documentation are available now. The guided chapter
+experience is planned for a later phase.
+
+## License
+
+The Python library is licensed under the [MIT License](https://github.com/Oseiasdfarias/glassllm/blob/main/LICENSE).
+Documentation text is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).

glassllm-0.1.0/README.md

@@ -0,0 +1,115 @@
+![glassllm — see every step of a small language model](https://raw.githubusercontent.com/Oseiasdfarias/glassllm/main/docs/assets/banner-light.svg)
+
+# glassllm
+
+A small Transformer written from scratch in NumPy, built to be looked at. Follow a token through
+the model and inspect the equation, source code and values produced at each step.
+
+[![PyPI version](https://img.shields.io/pypi/v/glassllm)](https://pypi.org/project/glassllm/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/)
+[![NumPy only](https://img.shields.io/badge/runtime-NumPy%20only-4fb3c2)](https://numpy.org/)
+[![MIT license](https://img.shields.io/badge/license-MIT-4fb3c2)](https://github.com/Oseiasdfarias/glassllm/blob/main/LICENSE)
+
+**English** · [Português](https://github.com/Oseiasdfarias/glassllm/blob/main/README.pt-BR.md) ·
+[Documentation](https://oseiasdfarias.github.io/glassllm/)
+
+glassllm is an educational language model and Python library. Its decoder-only Transformer,
+tokenizer, automatic differentiation engine, training code and tracing tools are implemented
+with NumPy. Pretrained Portuguese and English models are included in the package.
+
+## Install
+
+```bash
+python -m pip install glassllm
+```
+
+The library supports Python 3.10 and later. NumPy is its only runtime dependency. The pretrained
+weights and tokenizers are included, so inference works immediately after installation.
+
+## Generate text
+
+The `glassllm` command uses the included English model by default. Select `pt` for Portuguese.
+
+```bash
+glassllm --lang en generate "Alice was beginning to" --n 40
+glassllm --lang pt generate "Capitu olhou para" --n 40
+```
+
+Use greedy decoding for repeatable output, or inspect the available model operations:
+
+```bash
+glassllm --lang en generate "Alice was beginning to" --n 40 --greedy
+glassllm steps
+glassllm trace "Alice was beginning to" --step attn.softmax --layer 0 --head 1
+```
+
+## Inspect a calculation
+
+Pass a `Tracer` to the model to record its actual intermediate values. Each record includes the
+equation, tensor shape and source location for a tagged operation.
+
+```python
+from glassllm import Tracer, load_pretrained
+
+model, tokenizer = load_pretrained("en")
+token_ids = tokenizer.encode("Alice was beginning to")
+
+tracer = Tracer()
+model.forward(token_ids, trace=tracer)
+
+step = tracer.get("attn.softmax", layer=0, head=1)
+print(step.equation)
+print(step.shape)
+print(step.value)
+```
+
+List all traceable operations with `glassllm steps`. The same tagged calculations are explained
+in the [library documentation](https://oseiasdfarias.github.io/glassllm/en/docs/biblioteca/api-publica/).
+
+## What happens inside the model
+
+Each prediction follows the same sequence of real calculations. The website can replay all 77
+recorded operations for a prediction and lets you inspect their equations, tensors and source.
+
+```text
+text → BPE tokens → embeddings → 4 Transformer blocks → logits → probabilities → next token
+                                  attention + SwiGLU
+```
+
+The model uses causal self-attention, RoPE, grouped-query attention, an attention KV cache,
+RMSNorm, SwiGLU and tied input/output embeddings. It has 852,608 parameters and runs on CPU.
+It is deliberately small for learning; it is not intended for production text generation.
+
+## Interactive site and documentation
+
+Try the [live bilingual lab](https://oseiasdfarias.github.io/glassllm/). It runs the same Python
+wheel in a Pyodide worker in your browser. The network view shows the complete model architecture,
+and slow motion steps through each recorded calculation at a readable pace.
+
+The documentation includes 35 pages in each language, with interface guides, inference and
+training flowcharts, mathematical foundations, API reference, examples and research references:
+
+- [Documentation in English](https://oseiasdfarias.github.io/glassllm/en/docs/)
+- [Documentação em português](https://oseiasdfarias.github.io/glassllm/pt/docs/)
+
+## Training and data
+
+The included models were trained on public-domain books by Machado de Assis (Portuguese) and
+Lewis Carroll (English), sourced from [Project Gutenberg](https://www.gutenberg.org/). Training
+scripts and dataset preparation live in the repository. PyTorch is used only for development
+and parity checks; it is not installed when users install the library.
+
+## Contributing
+
+The source code, tests, design notes and issue tracker are on
+[GitHub](https://github.com/Oseiasdfarias/glassllm). See the
+[contribution guide](https://github.com/Oseiasdfarias/glassllm/blob/main/CONTRIBUTING.md) before
+opening a pull request.
+
+The interactive lab and complete library documentation are available now. The guided chapter
+experience is planned for a later phase.
+
+## License
+
+The Python library is licensed under the [MIT License](https://github.com/Oseiasdfarias/glassllm/blob/main/LICENSE).
+Documentation text is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).

glassllm-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "glassllm"
+version = "0.1.0"
+description = "A small Transformer written from scratch in NumPy, built to be looked at."
+readme = { file = "README.md", content-type = "text/markdown" }
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Oséias Dias de Farias" }]
+requires-python = ">=3.10"
+dependencies = ["numpy>=1.26"]
+keywords = ["llm", "transformer", "education", "numpy", "attention", "autograd"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Education",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.urls]
+Homepage = "https://oseiasdfarias.github.io/glassllm/"
+Documentation = "https://oseiasdfarias.github.io/glassllm/en/docs/"
+Source = "https://github.com/Oseiasdfarias/glassllm"
+Issues = "https://github.com/Oseiasdfarias/glassllm/issues"
+
+[project.scripts]
+glassllm = "glassllm.cli:main"
+
+[dependency-groups]
+dev = ["pytest>=8", "torch>=2.2"]
+
+[build-system]
+requires = ["hatchling>=1.26"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/glassllm"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/glassllm", "tests", "scripts", "README.md", "LICENSE", "pyproject.toml"]
+
+[tool.uv.sources]
+torch = { index = "pytorch-cpu" }
+
+[[tool.uv.index]]
+name = "pytorch-cpu"
+url = "https://download.pytorch.org/whl/cpu"
+explicit = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

glassllm-0.1.0/scripts/build_api_inventory.py

@@ -0,0 +1,67 @@
+"""Snapshot actual declarations for reviewing the documentation's API coverage.
+
+This uses AST only: it does not import NumPy, load models or execute library code.
+Regenerate with `python scripts/build_api_inventory.py` after an API change.
+"""
+from __future__ import annotations
+
+import ast
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+PACKAGE = ROOT / "src/glassllm"
+OUTPUT = ROOT / "site/src/data/api-inventory.json"
+
+
+def visible(name: str) -> bool:
+    return not name.startswith("_") or (name.startswith("__") and name.endswith("__") and name not in {"__repr__", "__post_init__"})
+
+
+def function(node: ast.FunctionDef) -> dict:
+    decorators = {ast.unparse(item) for item in node.decorator_list}
+    returns = f" -> {ast.unparse(node.returns)}" if node.returns else ""
+    return {"name": node.name, "kind": "property" if "property" in decorators else "function",
+            "signature": f"{node.name}({ast.unparse(node.args)}){returns}", "line": node.lineno}
+
+
+def members(node: ast.ClassDef) -> list[dict]:
+    result = []
+    for item in node.body:
+        if isinstance(item, ast.FunctionDef) and visible(item.name):
+            result.append(function(item))
+        elif isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name) and visible(item.target.id):
+            result.append({"name": item.target.id, "kind": "field", "signature": f"{item.target.id}: {ast.unparse(item.annotation)}", "line": item.lineno})
+        elif isinstance(item, ast.Assign):
+            for target in item.targets:
+                if isinstance(target, ast.Name) and visible(target.id):
+                    result.append({"name": target.id, "kind": "alias", "signature": f"{target.id} = {ast.unparse(item.value)}", "line": item.lineno})
+    return result
+
+
+def main() -> None:
+    modules = []
+    root_exports = []
+    for path in sorted(PACKAGE.glob("*.py")):
+        tree = ast.parse(path.read_text(encoding="utf-8"))
+        symbols = []
+        for node in tree.body:
+            if isinstance(node, ast.FunctionDef) and not node.name.startswith("_"):
+                symbols.append(function(node))
+            elif isinstance(node, ast.ClassDef) and not node.name.startswith("_"):
+                symbols.append({"name": node.name, "kind": "class", "signature": node.name,
+                                "line": node.lineno, "members": members(node)})
+            elif path.name == "__init__.py" and isinstance(node, ast.Assign):
+                for target in node.targets:
+                    if isinstance(target, ast.Name) and target.id == "__all__":
+                        root_exports = ast.literal_eval(node.value)
+        modules.append({"module": "glassllm" if path.stem == "__init__" else f"glassllm.{path.stem}",
+                        "file": path.relative_to(ROOT).as_posix(), "symbols": symbols})
+    OUTPUT.parent.mkdir(parents=True, exist_ok=True)
+    OUTPUT.write_text(json.dumps({"rootExports": root_exports, "modules": modules}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    count = sum(len(module["symbols"]) for module in modules)
+    print(f"API inventory: {len(root_exports)} root exports; {count} declarations in {len(modules)} modules")
+
+
+if __name__ == "__main__":
+    main()

glassllm-0.1.0/scripts/build_step_index.py

@@ -0,0 +1,14 @@
+"""Write src/glassllm/step_index.json from source comment markers."""
+import json
+
+from glassllm.steps import INDEX_PATH, scan_markers
+
+
+def main() -> None:
+    index = scan_markers()
+    INDEX_PATH.write_text(json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(f"{len(index)} steps -> {INDEX_PATH}")
+
+
+if __name__ == "__main__":
+    main()

glassllm-0.1.0/scripts/prepare_corpus.py

@@ -0,0 +1,62 @@
+"""Download eight public domain books and split each book's final 5% for validation."""
+from __future__ import annotations
+
+import argparse
+import re
+import unicodedata
+import urllib.request
+from pathlib import Path
+
+
+BOOKS = {
+    "pt": {  # Machado de Assis
+        55752: "Dom Casmurro",
+        54829: "Memorias Posthumas de Braz Cubas",
+        55682: "Quincas Borba",
+        56737: "Esau e Jacob",
+        55797: "Memorial de Ayres",
+    },
+    "en": {  # Lewis Carroll
+        11: "Alice's Adventures in Wonderland",
+        12: "Through the Looking-Glass",
+        620: "Sylvie and Bruno",
+    },
+}
+URL = "https://www.gutenberg.org/cache/epub/{id}/pg{id}.txt"
+START = re.compile(r"\*\*\* ?START OF (THE|THIS) PROJECT GUTENBERG EBOOK.*?\*\*\*", re.S)
+END = re.compile(r"\*\*\* ?END OF (THE|THIS) PROJECT GUTENBERG EBOOK")
+
+
+def clean(raw: str) -> str:
+    """Remove Gutenberg delimiters and normalize whitespace and Unicode to NFC."""
+    text = raw.replace("\r\n", "\n")
+    start, end = START.search(text), END.search(text)
+    if start is None or end is None or start.end() >= end.start():
+        raise ValueError("missing or invalid Project Gutenberg header/footer markers")
+    text = unicodedata.normalize("NFC", text[start.end():end.start()])
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip() + "\n"
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--out", default="data")
+    args = parser.parse_args(argv)
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    for lang, books in BOOKS.items():
+        train_parts, val_parts = [], []
+        for book_id, title in books.items():
+            with urllib.request.urlopen(URL.format(id=book_id), timeout=30) as response:
+                text = clean(response.read().decode("utf-8-sig"))
+            cut = text.index("\n", int(len(text) * 0.95)) + 1
+            train_parts.append(text[:cut])
+            val_parts.append(text[cut:])
+            print(f"{lang}: {title}: {len(text):,} characters")
+        (out / f"{lang}_train.txt").write_text("\n".join(train_parts), encoding="utf-8")
+        (out / f"{lang}_val.txt").write_text("\n".join(val_parts), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()

glassllm-0.1.0/scripts/train_model.py

@@ -0,0 +1,72 @@
+"""Train one language with glassllm, restore its best checkpoint, and validate fp16 export."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import time
+
+import numpy as np
+
+from glassllm import Config, GlassLM, LANGUAGES, Tokenizer
+from glassllm.train import evaluate, train
+
+
+MAX_SECONDS = 2 * 60 * 60
+EVAL_SEED = 1234
+EVAL_N_BATCHES = 10
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--lang", required=True, choices=LANGUAGES)
+    parser.add_argument("--steps", type=int, default=3000)
+    parser.add_argument("--batch-size", type=int, default=16)
+    parser.add_argument("--lr", type=float, default=3e-3)
+    parser.add_argument("--data", default="data")
+    parser.add_argument("--out", default="src/glassllm/weights")
+    args = parser.parse_args(argv)
+    deadline = time.monotonic() + MAX_SECONDS
+
+    def check_deadline() -> None:
+        if time.monotonic() >= deadline:
+            raise TimeoutError("pretrained model run exceeded the two-hour deadline")
+
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    tok = Tokenizer.load(out / f"{args.lang}.tokenizer.json")
+
+    def read(split: str) -> np.ndarray:
+        text = (Path(args.data) / f"{args.lang}_{split}.txt").read_text(encoding="utf-8")
+        return np.array(tok.encode(text), dtype=np.int64)
+
+    train_ids, val_ids = read("train"), read("val")
+    cfg = Config(vocab_size=tok.vocab_size)
+    model = GlassLM(cfg, seed=0)
+    check_deadline()
+    train(model, train_ids, steps=args.steps, batch_size=args.batch_size, seq_len=cfg.max_len,
+          lr=args.lr, warmup=100, seed=0, val_data=val_ids, eval_every=100, keep_best=True)
+
+    # Measure the restored checkpoint and its fp16 export on identical windows.
+    settings = {"batch_size": args.batch_size, "seq_len": cfg.max_len,
+                "n_batches": EVAL_N_BATCHES, "seed": EVAL_SEED}
+    check_deadline()
+    fp32_loss = evaluate(model, val_ids, **settings)
+    check_deadline()
+    path = out / f"{args.lang}.npz"
+    model.save(path, fp16=True)
+    check_deadline()
+    fp16_loss = evaluate(GlassLM.load(path), val_ids, **settings)
+    check_deadline()
+    if not np.isfinite(fp32_loss) or not np.isfinite(fp16_loss) or abs(fp16_loss - fp32_loss) > 0.01:
+        raise ValueError(f"fp16 validation loss {fp16_loss} differs from fp32 {fp32_loss} by more than 0.01")
+    meta = {"steps": args.steps, "val_loss_fp32": fp32_loss, "val_loss": fp16_loss,
+            "train_tokens": len(train_ids), "val_tokens": len(val_ids),
+            "eval_batch_size": args.batch_size, "eval_seq_len": cfg.max_len,
+            "eval_n_batches": EVAL_N_BATCHES, "eval_seed": EVAL_SEED}
+    (out / f"{args.lang}.meta.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8")
+    print(f"saved {path}: val loss {fp16_loss:.3f} (fp16; fp32 {fp32_loss:.3f})")
+
+
+if __name__ == "__main__":
+    main()

glassllm-0.1.0/scripts/train_tokenizer.py

@@ -0,0 +1,30 @@
+"""Train a language's BPE tokenizer and save it next to the model weights."""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from glassllm import LANGUAGES, Tokenizer
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--lang", required=True, choices=LANGUAGES)
+    parser.add_argument("--vocab-size", type=int, default=1024)
+    parser.add_argument("--data", default="data")
+    parser.add_argument("--out", default="src/glassllm/weights")
+    args = parser.parse_args(argv)
+    text = (Path(args.data) / f"{args.lang}_train.txt").read_text(encoding="utf-8")
+    tok = Tokenizer.train(text, args.vocab_size)
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    path = out / f"{args.lang}.tokenizer.json"
+    tok.save(path)
+    ids = tok.encode(text)
+    ratio = len(text) / len(ids) if ids else 0.0
+    print(f"{args.lang}: vocab {tok.vocab_size}, {len(ids):,} tokens, "
+          f"{ratio:.2f} characters per token -> {path}")
+
+
+if __name__ == "__main__":
+    main()