fromscratch-pyllm 0.5.0__py3-none-any.whl

fromscratch_pyllm-0.5.0.dist-info/METADATA

@@ -0,0 +1,231 @@
+Metadata-Version: 2.5
+Name: fromscratch-pyllm
+Version: 0.5.0
+Summary: An educational large language model built from scratch in Python
+Project-URL: Homepage, https://github.com/robcox2910/pyllm
+Project-URL: Repository, https://github.com/robcox2910/pyllm
+Project-URL: Issues, https://github.com/robcox2910/pyllm/issues
+Project-URL: Documentation, https://robcox2910.github.io/pyllm/
+Author: Rob Cox
+License: MIT
+License-File: LICENSE
+Keywords: autograd,education,gpt,learning,llm,transformer
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Education
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.14
+Requires-Dist: numpy>=2.1
+Provides-Extra: dev
+Requires-Dist: commitizen>=4.4.0; extra == 'dev'
+Requires-Dist: hypothesis>=6.170; extra == 'dev'
+Requires-Dist: pip-audit>=2.10.0; extra == 'dev'
+Requires-Dist: pre-commit>=4.0.1; extra == 'dev'
+Requires-Dist: pyright>=1.1.400; extra == 'dev'
+Requires-Dist: pytest-cov>=6.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.3.4; extra == 'dev'
+Requires-Dist: ruff>=0.9.3; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
+Provides-Extra: pebble
+Requires-Dist: fromscratch-pebble>=0.3.0; extra == 'pebble'
+Description-Content-Type: text/markdown
+
+# PyLLM
+
+An educational large language model built from scratch in Python.
+
+Part of the "from scratch in Python" series. Built incrementally with TDD,
+every concept explained with analogies a 12-year-old can follow. The only
+dependency is numpy (for fast array math) — the autograd engine, neural-net
+layers, attention, the Transformer, the tokenizer, the optimizer, the training
+loop, and sampling are all ours.
+
+## Quick Start
+
+```bash
+uv sync --all-extras
+uv run pytest
+```
+
+## Dream up new Pokémon (generates instantly)
+
+PyLLM ships with a tiny pre-trained GPT, so it invents Pokémon-ish names the
+moment you launch it — no training wait:
+
+```console
+$ uv run pyllm --seed 0 --max-new-tokens 40
+machop
+machoke
+machamp
+bellsprout
+weepin
+```
+
+Turn the boldness dial with `--temperature` (0 = safe/greedy, higher = wilder)
+and add a `--top-k` guardrail. Start from a prompt with `--prompt "pi"`.
+
+## Train your own model
+
+Pick a rung on the teaching ladder (`bigram`, `mlp`, or `gpt`) and point it at
+any text file (defaults to the bundled Pokémon corpus):
+
+```console
+$ uv run pyllm train --model gpt --steps 2000 --out mine.npz
+step 0: loss 3.31
+...
+final loss 0.42; saved checkpoint to mine.npz
+
+$ uv run pyllm --checkpoint mine.npz --max-new-tokens 60
+```
+
+## The flagship: teach PyLLM to write Pebble
+
+There's no corpus of [Pebble](https://github.com/robcox2910/pebble-lang) code in
+the wild — so we *grow our own*. A generator emits **random-but-provably-valid**
+Pebble ASTs (100% parse under Pebble's own parser), a harvester scrapes real
+snippets from Pebble's docs, and — because we own the language toolchain — we
+**grade** the trained model by running its output back through Pebble's real
+parser and reporting what percentage parses:
+
+```console
+$ uv run pyllm pebble --seed 7 --temperature 0.5 --score
+let result = 62
+    }
+    if count < 47 {
+        let c = 51 + (83 + 32)
+...
+parse rate: 0.0% of 1 programs
+```
+
+The bundled checkpoint is *tiny* (a few-minute char-level GPT), so it writes
+Pebble-*flavoured* code and only some of it parses — and that's the whole point:
+the parser is an honest, objective judge, so you can watch the score go up as you
+train longer or bigger. Almost no LLM project can grade its own output like this.
+
+### Call in the referee: always-valid Pebble
+
+Add `--constrained` and a from-scratch grammar **referee** vetoes every character
+that would break Pebble's rules, so the model rolls again until it picks a legal
+one. Same tiny brain — but now every program it returns parses:
+
+```console
+$ uv run pyllm pebble --seed 7 --constrained --score
+let total = n * (63 - 99)
+
+let result = c
+fount = 55 + (n - b) % (6 - b)
+
+parse rate: 100.0% of 2 programs
+```
+
+From Python: `generate_pebble(constrained=True)`. See
+[`docs/concepts/the-referee.md`](docs/concepts/the-referee.md).
+
+Grow a fresh synthetic corpus with `uv run pyllm gen-corpus`. Live scoring needs
+the optional `fromscratch-pebble` dependency (`uv sync --all-extras`); generation and
+training work without it. See [`docs/concepts/grow-your-own-data.md`](docs/concepts/grow-your-own-data.md).
+
+## See how text becomes tokens
+
+```console
+$ uv run pyllm tokenize --text "pika"
+'p' -> 16
+'i' -> 9
+'k' -> 11
+'a' -> 1
+```
+
+## The teaching ladder
+
+Each rung is a real, trainable language model that fixes a weakness of the one
+below it, so you feel *why* each idea exists:
+
+1. **Bigram** — "what character usually comes next?" A lookup table.
+2. **MLP** — a small neural net over the last few characters (embeddings + a
+   hidden layer).
+3. **GPT** — stacked Transformer blocks (multi-head attention + feed-forward +
+   LayerNorm + residuals) with positional information. The real thing.
+
+## Learn the ideas
+
+Kid-friendly explanations live in [`docs/concepts/`](docs/concepts/README.md):
+the breadcrumb trail (autograd), tokens, embeddings, attention, how a Transformer
+is built, how a model learns, sampling, and — the flagship lesson — growing your
+own data.
+
+## Developing alongside a sibling
+
+PyLLM uses a released version of [Pebble](https://github.com/robcox2910/pebble-lang)
+from PyPI (`fromscratch-pebble`, the optional `pebble` extra). To try an
+unreleased Pebble change, install your local checkout over the released one:
+
+```bash
+uv pip install -e ../pebble-lang
+uv run --no-sync pytest
+```
+
+Use `uv run --no-sync` while you do this: a plain `uv run` (like `uv sync`)
+puts the pinned release back, which is how you undo the override.
+
+## Releasing
+
+1. Merge the change to `main`.
+2. Tag it: `git tag -a vX.Y.Z -m vX.Y.Z && git push origin vX.Y.Z`.
+   A change to the public API bumps the minor version (`v0.1.0` → `v0.2.0`);
+   anything else bumps the patch (`v0.1.0` → `v0.1.1`).
+3. Projects that use PyLLM (like [PyStack](https://github.com/robcox2910/pystack))
+   upgrade by editing the tag in their `[tool.uv.sources]` and running
+   `uv lock --upgrade-package pyllm`, in a pull request so CI checks the pair.
+
+## PyLLM inside PyStack
+
+PyLLM is **the brain** of a family of educational projects, each building a real
+system from scratch to teach how it works. They're tied together by
+**[PyStack](https://github.com/robcox2910/pystack)**, where a Pebble program can
+even `import "llm"` and ask PyLLM to write more code.
+
+Once PyLLM is installed as a PyStack plugin, any Pebble program can call it:
+
+```pebble
+import "llm"
+let code = llm_generate("a function that adds two numbers")
+print(code)
+```
+
+<!-- series:start -->
+## Related Projects
+
+This project is one of a series that builds real systems "from scratch in Python" to teach how they work. See the [series home](https://robcox2910.github.io/) for the full story.
+
+| Project | What it teaches | Repository | Docs |
+| --- | --- | --- | --- |
+| PyOS | Operating systems | [py-os](https://github.com/robcox2910/py-os) | [Docs](https://robcox2910.github.io/py-os/) |
+| Pebble | Compilers and programming languages | [pebble-lang](https://github.com/robcox2910/pebble-lang) | [Docs](https://robcox2910.github.io/pebble-lang/) |
+| PyDB | Relational databases | [pydb](https://github.com/robcox2910/pydb) | [Docs](https://robcox2910.github.io/pydb/) |
+| PyKV | Key-value stores | [pykv](https://github.com/robcox2910/pykv) | [Docs](https://robcox2910.github.io/pykv/) |
+| PyDocDB | Document databases | [pydoc-db](https://github.com/robcox2910/pydoc-db) | [Docs](https://robcox2910.github.io/pydoc-db/) |
+| PyGraphDB | Graph databases | [pygraphdb](https://github.com/robcox2910/pygraphdb) | [Docs](https://robcox2910.github.io/pygraphdb/) |
+| PyTSDB | Time-series databases | [pytsdb](https://github.com/robcox2910/pytsdb) | [Docs](https://robcox2910.github.io/pytsdb/) |
+| PyVecDB | Vector databases | [pyvecdb](https://github.com/robcox2910/pyvecdb) | [Docs](https://robcox2910.github.io/pyvecdb/) |
+| PyColDB | Column-family databases | [pycoldb](https://github.com/robcox2910/pycoldb) | [Docs](https://robcox2910.github.io/pycoldb/) |
+| PyCrypt | Cryptography | [pycrypt](https://github.com/robcox2910/pycrypt) | [Docs](https://robcox2910.github.io/pycrypt/) |
+| PyNet | Computer networking | [pynet](https://github.com/robcox2910/pynet) | [Docs](https://robcox2910.github.io/pynet/) |
+| PyWeb | HTTP web servers | [pyweb](https://github.com/robcox2910/pyweb) | [Docs](https://robcox2910.github.io/pyweb/) |
+| PyGit | Version control | [pygit](https://github.com/robcox2910/pygit) | [Docs](https://robcox2910.github.io/pygit/) |
+| PySearch | Search engines | [pysearch](https://github.com/robcox2910/pysearch) | [Docs](https://robcox2910.github.io/pysearch/) |
+| PyMQ | Message queues | [pymq](https://github.com/robcox2910/pymq) | [Docs](https://robcox2910.github.io/pymq/) |
+| **PyLLM** | **Language models (the brain)** | [pyllm](https://github.com/robcox2910/pyllm) | [Docs](https://robcox2910.github.io/pyllm/) |
+| PyScheduler | Job scheduling | [pyscheduler](https://github.com/robcox2910/pyscheduler) | [Docs](https://robcox2910.github.io/pyscheduler/) |
+| PyStack | Full-stack integration | [pystack](https://github.com/robcox2910/pystack) | [Docs](https://robcox2910.github.io/pystack/) |
+<!-- series:end -->
+
+<!-- docs:start -->
+## Documentation
+
+Full docs (with kid-friendly explanations of every idea): https://robcox2910.github.io/pyllm/
+<!-- docs:end -->

fromscratch_pyllm-0.5.0.dist-info/RECORD

@@ -0,0 +1,51 @@
+pyllm/__init__.py,sha256=Mu-ztRj5Wfvx4QT2tMtn17mQw9ph4hFTR_WWhv8jgIc,177
+pyllm/_types.py,sha256=8puuPpD2lBzEyLdVZTOwVD82T8dT2Y7TzV_3_Fyj_LA,332
+pyllm/_version.py,sha256=APp2gXXpoz-tyKGj5fWwVTf7yK8O9LHP-MP0aQ_uNxs,520
+pyllm/api.py,sha256=w170O3RWKpzEOVWonNKnY35OdSLZ5JbL9EZ8mr7Afjs,2239
+pyllm/checkpoint.py,sha256=hIG6wySmA-btpq7bNlF14RA1fShJ2Ikk_s2HAdYFT_Q,5545
+pyllm/cli.py,sha256=EGS5qyYDxooRS1OdTh6KBVucCCpWTaTCCrnleTi8AYk,7692
+pyllm/generate.py,sha256=oRcofKH4k__wKx2kK8JZY6HPbSJeWNFsyb0q4ptpOT8,8447
+pyllm/autograd/__init__.py,sha256=OKp6PrqPn0UpsXYB2pHZrsmy0LbNy6yHzPGV5bqn5bo,145
+pyllm/autograd/gradcheck.py,sha256=dJWpSv-O7pCLTsps8VgrZaQ8Y4eFtfnydMsnW52ttAE,1192
+pyllm/autograd/tensor.py,sha256=5eUX-dSgdHfWUuiZDRga1o4yJP819ByrucK3UsR9Mjk,16096
+pyllm/data/__init__.py,sha256=0hGQ10Ss-CUujtjhKwj5_xyY8Yr_UuyetgC0rOGEpa4,701
+pyllm/models/__init__.py,sha256=6Bj3QjuBzAtQ6w-Pq3Glzh6-jcxLXgDLydSnOPnlZ-8,1672
+pyllm/models/base.py,sha256=s8hijQgjn6OXBtqtavTpRbk5XvoCDkEaKe4YuWJVtVA,899
+pyllm/models/bigram.py,sha256=zML42WIwx2z7LtZkkp72VdTmhGC5-67PV5WgjhQZ5yE,1647
+pyllm/models/gpt.py,sha256=fSSjt_H1EN6LeMuWwUrGyAkA0kIoDjkfrNGZSmszFwQ,3062
+pyllm/models/mlp.py,sha256=DTUAZdpbrrUyW-3O39uZN0YGyk-mUVvrS-te5XxSDCQ,2208
+pyllm/nn/__init__.py,sha256=f4MjGdlHeZjMknePDUJgSlegaHcqly-IpcjkNhdeesU,1048
+pyllm/nn/attention.py,sha256=OdToJPCW78D6qCzLbItyGYPipy5XaGaAt8QANRMK2xM,3301
+pyllm/nn/dropout.py,sha256=TDoz-7mPPnODU5f38FdmIbJIklbBd900F96oiDlohWA,1203
+pyllm/nn/embedding.py,sha256=DEOy6Cnvfe4oVUoCILPaTqSZp5_nyLbqhC45_gDlOzs,1119
+pyllm/nn/functional.py,sha256=g9s0fih9jXkrelSmI0ajrR1rwCRiaRMRxOcW4hgP1C8,4656
+pyllm/nn/linear.py,sha256=ewivFsQqSPOAwKOZb5PFnmWJ3Fm9ODJGJ06SwnYMwZM,1225
+pyllm/nn/module.py,sha256=CLkEe60uXUDU35e_6WR93mUa23dYhrPyG-GV3tVliZs,3309
+pyllm/nn/normalization.py,sha256=doKJTN30hu823hYTu6cxwviRFySoeUh6pYmPnED7ZnA,1221
+pyllm/nn/transformer.py,sha256=7td2g5jDUCRj4iNB6DTkPCsQoYZJ2JleRTNc42bofxE,2654
+pyllm/pebble/__init__.py,sha256=olqN5bt7YzpAjMsvUo0gLxae5J_X4SUhZtuZUju-aHM,794
+pyllm/pebble/ast.py,sha256=uwLtVLW_wu6wkSxUixZ31pA6z9FZmHCRPj_MhHzSyxc,2732
+pyllm/pebble/corpus.py,sha256=X0AjARrmfuQa-CqoD3iHUpZINN0GsNCS02O1WyQQ5g4,2828
+pyllm/pebble/generator.py,sha256=9XXLTvK8t17UQPIKeoCFF3ezUKgeUeUD3KM7lPMt6r0,5894
+pyllm/pebble/harvest.py,sha256=EBoMpQWbY1lDWbrILvh4VkMpOqn9xoMZQoznA8f7QH0,1370
+pyllm/pebble/referee.py,sha256=z0EcQPl6S7XvEhB11RrIBevgbC9SjRvNmhBCaI_Hjqc,15247
+pyllm/pebble/render.py,sha256=JfF6V26sfEneORo1AUQXaM-8bIW3UD_Ukx6z3-LSYFw,3602
+pyllm/pebble/score.py,sha256=aQEpU8cOLoqVOgOq-X1NA2IN9E3N1SvX9aROQfJ5IrA,1522
+pyllm/tokenizer/__init__.py,sha256=1x2yr-c0V4CgLQZ4UHDN7v3TY362faL0bUU0SkmXuFk,271
+pyllm/tokenizer/base.py,sha256=RzB22pUwm63Kty57llBolB9BzAbe36PWjXNmRHgR4MI,631
+pyllm/tokenizer/bpe.py,sha256=H20Oy7Q04AMmty4UMylB9aCcrXPenpmhN4QXEWOtmAo,4782
+pyllm/tokenizer/char.py,sha256=bJhbiY1vmcP8bdfOrgJnsWrlITYWCMqMSgUlc-lx7-o,1419
+pyllm/training/__init__.py,sha256=N3CH1IrHd-3qvnl66Q2o03YAxRbolEUBMmfyxDC5hcM,237
+pyllm/training/data.py,sha256=KJwSOUlw-pvgjK1jmIHeLD-FbyvxPHjSrt8Z3O0z_k8,1723
+pyllm/training/loop.py,sha256=s6N4zEw0P4wZheiPAL1jPLQLr7mMOPEF4sqBhYlcT0g,1819
+pyllm/training/optim.py,sha256=Qxw2SeRz6xqEtKD4V8velwzUygEPsY9B4M3_rkKWQWA,2981
+pyllm/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyllm/data/pebble.npz,sha256=wHcN3JJkMt09PSvpvXbQLCGjWpyLBtNeCiV-zVVgL6s,921586
+pyllm/data/pebble_corpus.txt,sha256=_ie2Nzl04rET5UAJKOZG5wyBVjncNH9Uq-QGZX9kZHQ,354412
+pyllm/data/pokemon.npz,sha256=gwZb4x8lYiL8cVdU_xuV3bXLeyc1CCez8EkQi5qRqxg,492126
+pyllm/data/pokemon_corpus.txt,sha256=VahZ4Dacr-l0iFjmEbhUgZjxYbkdKAFbzbeQCyDwKXE,1234
+fromscratch_pyllm-0.5.0.dist-info/METADATA,sha256=AjGWGtJjkB8kpC7DzRpM2XsU0zKH-5FnnjMkie4ueDM,9775
+fromscratch_pyllm-0.5.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+fromscratch_pyllm-0.5.0.dist-info/entry_points.txt,sha256=h7nObqiD8Dca70XQtmOf1hUE3UuDoE08jLXXiby6kbc,41
+fromscratch_pyllm-0.5.0.dist-info/licenses/LICENSE,sha256=NtHM2ak--oE71VCcY50kcCoC1cKS0LRUsxq_xA89RNc,1064
+fromscratch_pyllm-0.5.0.dist-info/RECORD,,

fromscratch_pyllm-0.5.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

fromscratch_pyllm-0.5.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pyllm = pyllm.cli:main

fromscratch_pyllm-0.5.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Rob Cox
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyllm/__init__.py

@@ -0,0 +1,5 @@
+"""PyLLM: an educational large language model built from scratch."""
+
+from pyllm.api import generate_pebble, generate_pokemon
+
+__all__ = ["generate_pebble", "generate_pokemon"]

pyllm/_types.py

@@ -0,0 +1,10 @@
+"""Shared type aliases, so every module spells "a grid of numbers" the same way."""
+
+import numpy as np
+from numpy.typing import NDArray
+
+type Array = NDArray[np.float64]
+"""A grid of 64-bit decimal numbers (the only kind of array pyllm uses)."""
+
+type IntArray = NDArray[np.int64]
+"""A grid of whole numbers, such as token ids."""

pyllm/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.5.0'
+__version_tuple__ = version_tuple = (0, 5, 0)
+
+__commit_id__ = commit_id = None

pyllm/api.py

@@ -0,0 +1,72 @@
+"""A friendly one-call way to make PyLLM write something.
+
+Loading a model from disk and wiring up sampling every time would be a chore, so
+this is the "just give me some text" front door: call `generate_pokemon()` to
+dream up creatures, or `generate_pebble()` to write Pebble code. The trained
+brain is loaded once and kept warm (cached) so the second call is instant. This
+is also the door PyStack knocks on to let Pebble programs use PyLLM.
+"""
+
+from pathlib import Path
+
+import numpy as np
+
+from pyllm.checkpoint import load
+from pyllm.data import PEBBLE_CHECKPOINT, POKEMON_CHECKPOINT
+from pyllm.generate import generate
+from pyllm.models import LanguageModel
+from pyllm.pebble import referee as pebble_referee
+from pyllm.tokenizer import Tokenizer
+
+_CACHE: dict[str, tuple[LanguageModel, Tokenizer]] = {}
+
+
+def _load_cached(path: Path) -> tuple[LanguageModel, Tokenizer]:
+    """Load a checkpoint once and remember it, so later calls are instant."""
+    key = str(path)
+    if key not in _CACHE:
+        _CACHE[key] = load(path)
+    return _CACHE[key]
+
+
+def generate_pokemon(
+    prompt: str = "",
+    max_new_tokens: int = 60,
+    temperature: float = 0.8,
+    seed: int | None = None,
+) -> str:
+    """Dream up new Pokémon-ish names using the bundled Pokémon brain."""
+    model, tokenizer = _load_cached(POKEMON_CHECKPOINT)
+    return generate(
+        model,
+        tokenizer,
+        prompt=prompt,
+        max_new_tokens=max_new_tokens,
+        temperature=temperature,
+        rng=np.random.default_rng(seed),
+    )
+
+
+def generate_pebble(
+    prompt: str = "let ",
+    max_new_tokens: int = 120,
+    temperature: float = 0.7,
+    seed: int | None = None,
+    *,
+    constrained: bool = False,
+) -> str:
+    """Write Pebble-flavoured code using the bundled Pebble brain.
+
+    With `constrained=True` a grammar referee vetoes every illegal character, so
+    what comes back is always real, parseable Pebble.
+    """
+    model, tokenizer = _load_cached(PEBBLE_CHECKPOINT)
+    return generate(
+        model,
+        tokenizer,
+        prompt=prompt,
+        max_new_tokens=max_new_tokens,
+        temperature=temperature,
+        rng=np.random.default_rng(seed),
+        referee=pebble_referee if constrained else None,
+    )

pyllm/autograd/__init__.py

@@ -0,0 +1,5 @@
+"""The autograd engine: tensors that remember how to compute their gradients."""
+
+from pyllm.autograd.tensor import Tensor
+
+__all__ = ["Tensor"]

pyllm/autograd/gradcheck.py

@@ -0,0 +1,34 @@
+"""Finite-difference gradient checking: an independent test that backward() is right."""
+
+from collections.abc import Callable
+
+import numpy as np
+
+from pyllm._types import Array
+from pyllm.autograd.tensor import Tensor
+
+
+def numerical_grad(make_output: Callable[[], Tensor], wrt: Tensor, eps: float = 1e-6) -> Array:
+    """Finite-difference gradient of sum(make_output().data) w.r.t. `wrt`.
+
+    `make_output` is a function taking no arguments that builds and returns an
+    output Tensor using `wrt`. We nudge each element of `wrt.data` up and down by
+    `eps` and measure how the summed output changes -- the classic definition of
+    a derivative. This is our independent check that backward() is correct.
+    """
+    grad = np.zeros_like(wrt.data)
+    it = np.nditer(wrt.data, flags=["multi_index"])
+    while not it.finished:
+        idx = it.multi_index
+        original = wrt.data[idx]
+
+        wrt.data[idx] = original + eps
+        plus = float(make_output().data.sum())
+
+        wrt.data[idx] = original - eps
+        minus = float(make_output().data.sum())
+
+        wrt.data[idx] = original
+        grad[idx] = (plus - minus) / (2.0 * eps)
+        it.iternext()
+    return grad