gpt-simple-lm 0.1.0__tar.gz

gpt_simple_lm-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Louis Bertucci
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gpt_simple_lm-0.1.0/PKG-INFO

@@ -0,0 +1,231 @@
+Metadata-Version: 2.4
+Name: gpt-simple-lm
+Version: 0.1.0
+Summary: A clean, efficient framework for pretraining language models from scratch
+Author: Louis Bertucci
+License: MIT
+Project-URL: Homepage, https://github.com/lb-off/gpt-simple
+Project-URL: Documentation, https://lb-off.github.io/gpt-simple/
+Project-URL: Repository, https://github.com/lb-off/gpt-simple
+Project-URL: Issues, https://github.com/lb-off/gpt-simple/issues
+Keywords: llm,pretraining,gpt,pytorch,transformer,language-model
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.0.0
+Requires-Dist: numpy<2.0,>=1.24.0
+Requires-Dist: transformers>=4.36.0
+Requires-Dist: accelerate>=0.20.0
+Requires-Dist: sentencepiece>=0.1.99
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: wandb
+Requires-Dist: wandb>=0.15.0; extra == "wandb"
+Provides-Extra: cli
+Requires-Dist: rich>=13.0; extra == "cli"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: wandb>=0.15.0; extra == "dev"
+Requires-Dist: rich>=13.0; extra == "dev"
+Dynamic: license-file
+
+# GPT-Simple
+
+[![CI](https://github.com/lb-off/gpt-simple/actions/workflows/ci.yml/badge.svg)](https://github.com/lb-off/gpt-simple/actions/workflows/ci.yml)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+
+A clean, efficient framework for pretraining language models from scratch.
+
+GPT-Simple handles the full LLM pretraining workflow — tokenization,
+streaming data loading, multi-GPU training, checkpointing, and inference —
+through a single YAML config and a small CLI. It ships with a modern GPT
+architecture ready to train out of the box.
+
+## Features
+
+- **Single YAML config + CLI** — `init` / `tokenize` / `train` / `status`
+  / `stop` / `validate` / `generate` / `batch-generate`.
+- **Multi-GPU out of the box** — `--nproc_per_node N` launches `torchrun`
+  automatically (Accelerate, bf16, `torch.compile`, gradient
+  checkpointing).
+- **Pretokenized streaming** — memory-mapped `.bin/.idx` shards with
+  sequence packing; a raw-JSONL fallback for quick experiments.
+- **Deterministic stop/resume** — walltime- and signal-aware checkpoints
+  with topology-agnostic data cursors, so *N* short jobs equal one long
+  job (every document seen exactly once, even if `world_size` /
+  `num_workers` change between restarts).
+- **Orchestrator-friendly** — runs under SLURM, Kubernetes, or a local
+  loop; templates in [`examples/orchestrators/`](examples/orchestrators/).
+- **Curriculum learning** — phase-based mixing across named data buckets.
+- **Modern architecture** — pre-norm decoder with RoPE, RMSNorm, and a
+  gated (SwiGLU) MLP; also expresses GQA/MQA, vanilla MLPs, and untied
+  heads via config.
+- **Python API** — `import gpt_simple; gpt_simple.train(config="config.yaml")`.
+
+## Installation
+
+```bash
+pip install -e ".[dev]"     # from source (development)
+pip install .               # core only
+pip install ".[wandb]"      # optional: Weights & Biases logging
+pip install ".[cli]"        # optional: rich-formatted CLI output
+```
+
+## Quick start
+
+### 1. Generate a config
+
+```bash
+gpt-simple init -o config.yaml
+gpt-simple init --preset small -o config.yaml    # ~125M  (small | medium | large)
+```
+
+### 2. Pretokenize your data
+
+```bash
+gpt-simple tokenize \
+  --input_dir ./raw_data \
+  --output_dir ./data/tokenized \
+  --tokenizer_path gpt2 \
+  --max_length 2048 \
+  --num_workers 8
+```
+
+Converts `.jsonl`/`.txt` into memory-mapped `.bin/.idx` shards. See the
+[data pipeline guide](docs/data.md).
+
+### 3. Train
+
+```bash
+gpt-simple train --config config.yaml                     # single GPU
+gpt-simple train --config config.yaml --nproc_per_node 4  # 4 GPUs
+
+# override any config value; start fresh with --force
+gpt-simple train --config config.yaml --training.max_steps 5000 --force
+```
+
+See the [training guide](docs/training.md).
+
+### 4. Monitor and control
+
+```bash
+gpt-simple status                 # training progress
+gpt-simple stop                   # graceful shutdown (saves a checkpoint)
+gpt-simple stop --force           # immediate SIGKILL
+```
+
+### 5. Generate
+
+```bash
+gpt-simple generate --output-dir ./outputs --prompt "Once upon a time" --max-new-tokens 200
+```
+
+`--output-dir` auto-picks the latest checkpoint. For multi-model /
+multi-sampling batches and a `--dry-run` submission gate, use
+`batch-generate` — see the [inference guide](docs/inference.md).
+
+## Long runs with stop/resume
+
+The trainer targets clusters with a hard per-job wall-clock cap. With
+`resume: auto` (the default), re-running the same command resumes the
+latest checkpoint, and the trainer saves and exits cleanly before a
+walltime deadline or on `SIGTERM`/`SIGUSR1` — so an orchestrator just
+re-queues the job.
+
+```bash
+gpt-simple train --config config.yaml   # resume is automatic on every restart
+gpt-simple status
+gpt-simple stop                          # or let walltime/SIGUSR1 do it
+```
+
+Templates: [`slurm_resume_chain.sh`](examples/orchestrators/slurm_resume_chain.sh),
+[`kubernetes_job.yaml`](examples/orchestrators/kubernetes_job.yaml),
+[`local_loop.sh`](examples/orchestrators/local_loop.sh). See the
+[checkpointing & resume](docs/checkpointing-and-resume.md) and
+[orchestration](docs/orchestration.md) guides.
+
+## Configuration
+
+All settings live in one YAML file with four sections — `model`, `data`,
+`optimizer`, `training`:
+
+```yaml
+model:
+  n_embd: 768
+  n_layer: 12
+  n_head: 12
+  n_positions: 2048
+
+data:
+  path: ./data/tokenized
+  tokenizer: gpt2
+  format: pretokenized       # pretokenized | jsonl
+  max_length: 2048
+
+optimizer:
+  learning_rate: 3.0e-4
+  warmup_steps: 100
+
+training:
+  per_device_batch_size: 4
+  gradient_accumulation_steps: 4
+  max_steps: 1000
+  output_dir: ./outputs
+  # wandb_project: my-project   # uncomment to enable W&B
+```
+
+`gpt-simple init` writes a fully commented template. Every field is
+documented in the [configuration reference](docs/configuration.md), and
+curriculum learning in the [data pipeline guide](docs/data.md).
+
+## Python API
+
+```python
+import gpt_simple
+
+result = gpt_simple.train(
+    model=gpt_simple.ModelConfig(n_embd=768, n_layer=12, n_head=12),
+    data=gpt_simple.DataConfig(path="./data/tokenized", tokenizer="gpt2"),
+    optimizer=gpt_simple.OptimizerConfig(learning_rate=3e-4),
+    training=gpt_simple.TrainingConfig(max_steps=1000, output_dir="./outputs"),
+)
+print(result.final_loss, result.total_tokens, result.checkpoint_path)
+```
+
+Or `gpt_simple.train(config="config.yaml")`; sub-configs passed
+explicitly override the matching section from the file.
+
+## Documentation
+
+Full guides live in [`docs/`](docs/README.md):
+
+- [Architecture](docs/architecture.md) — the built-in model.
+- [Configuration](docs/configuration.md) — every config field.
+- [Data pipeline](docs/data.md) — tokenization, packing, curriculum.
+- [Training](docs/training.md) — multi-GPU, precision, compile.
+- [Checkpointing & resume](docs/checkpointing-and-resume.md) — the
+  stop/resume model.
+- [Orchestration](docs/orchestration.md) — running under any scheduler.
+- [Inference](docs/inference.md) — `generate` / `batch-generate`.
+- [Hardware tuning](docs/hardware-tuning.md) — peak GPU throughput.
+- [Performance](docs/performance.md) — measured 2.8B throughput and MFU.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest tests/
+ruff check src/ tests/
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

gpt_simple_lm-0.1.0/README.md

@@ -0,0 +1,193 @@
+# GPT-Simple
+
+[![CI](https://github.com/lb-off/gpt-simple/actions/workflows/ci.yml/badge.svg)](https://github.com/lb-off/gpt-simple/actions/workflows/ci.yml)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+
+A clean, efficient framework for pretraining language models from scratch.
+
+GPT-Simple handles the full LLM pretraining workflow — tokenization,
+streaming data loading, multi-GPU training, checkpointing, and inference —
+through a single YAML config and a small CLI. It ships with a modern GPT
+architecture ready to train out of the box.
+
+## Features
+
+- **Single YAML config + CLI** — `init` / `tokenize` / `train` / `status`
+  / `stop` / `validate` / `generate` / `batch-generate`.
+- **Multi-GPU out of the box** — `--nproc_per_node N` launches `torchrun`
+  automatically (Accelerate, bf16, `torch.compile`, gradient
+  checkpointing).
+- **Pretokenized streaming** — memory-mapped `.bin/.idx` shards with
+  sequence packing; a raw-JSONL fallback for quick experiments.
+- **Deterministic stop/resume** — walltime- and signal-aware checkpoints
+  with topology-agnostic data cursors, so *N* short jobs equal one long
+  job (every document seen exactly once, even if `world_size` /
+  `num_workers` change between restarts).
+- **Orchestrator-friendly** — runs under SLURM, Kubernetes, or a local
+  loop; templates in [`examples/orchestrators/`](examples/orchestrators/).
+- **Curriculum learning** — phase-based mixing across named data buckets.
+- **Modern architecture** — pre-norm decoder with RoPE, RMSNorm, and a
+  gated (SwiGLU) MLP; also expresses GQA/MQA, vanilla MLPs, and untied
+  heads via config.
+- **Python API** — `import gpt_simple; gpt_simple.train(config="config.yaml")`.
+
+## Installation
+
+```bash
+pip install -e ".[dev]"     # from source (development)
+pip install .               # core only
+pip install ".[wandb]"      # optional: Weights & Biases logging
+pip install ".[cli]"        # optional: rich-formatted CLI output
+```
+
+## Quick start
+
+### 1. Generate a config
+
+```bash
+gpt-simple init -o config.yaml
+gpt-simple init --preset small -o config.yaml    # ~125M  (small | medium | large)
+```
+
+### 2. Pretokenize your data
+
+```bash
+gpt-simple tokenize \
+  --input_dir ./raw_data \
+  --output_dir ./data/tokenized \
+  --tokenizer_path gpt2 \
+  --max_length 2048 \
+  --num_workers 8
+```
+
+Converts `.jsonl`/`.txt` into memory-mapped `.bin/.idx` shards. See the
+[data pipeline guide](docs/data.md).
+
+### 3. Train
+
+```bash
+gpt-simple train --config config.yaml                     # single GPU
+gpt-simple train --config config.yaml --nproc_per_node 4  # 4 GPUs
+
+# override any config value; start fresh with --force
+gpt-simple train --config config.yaml --training.max_steps 5000 --force
+```
+
+See the [training guide](docs/training.md).
+
+### 4. Monitor and control
+
+```bash
+gpt-simple status                 # training progress
+gpt-simple stop                   # graceful shutdown (saves a checkpoint)
+gpt-simple stop --force           # immediate SIGKILL
+```
+
+### 5. Generate
+
+```bash
+gpt-simple generate --output-dir ./outputs --prompt "Once upon a time" --max-new-tokens 200
+```
+
+`--output-dir` auto-picks the latest checkpoint. For multi-model /
+multi-sampling batches and a `--dry-run` submission gate, use
+`batch-generate` — see the [inference guide](docs/inference.md).
+
+## Long runs with stop/resume
+
+The trainer targets clusters with a hard per-job wall-clock cap. With
+`resume: auto` (the default), re-running the same command resumes the
+latest checkpoint, and the trainer saves and exits cleanly before a
+walltime deadline or on `SIGTERM`/`SIGUSR1` — so an orchestrator just
+re-queues the job.
+
+```bash
+gpt-simple train --config config.yaml   # resume is automatic on every restart
+gpt-simple status
+gpt-simple stop                          # or let walltime/SIGUSR1 do it
+```
+
+Templates: [`slurm_resume_chain.sh`](examples/orchestrators/slurm_resume_chain.sh),
+[`kubernetes_job.yaml`](examples/orchestrators/kubernetes_job.yaml),
+[`local_loop.sh`](examples/orchestrators/local_loop.sh). See the
+[checkpointing & resume](docs/checkpointing-and-resume.md) and
+[orchestration](docs/orchestration.md) guides.
+
+## Configuration
+
+All settings live in one YAML file with four sections — `model`, `data`,
+`optimizer`, `training`:
+
+```yaml
+model:
+  n_embd: 768
+  n_layer: 12
+  n_head: 12
+  n_positions: 2048
+
+data:
+  path: ./data/tokenized
+  tokenizer: gpt2
+  format: pretokenized       # pretokenized | jsonl
+  max_length: 2048
+
+optimizer:
+  learning_rate: 3.0e-4
+  warmup_steps: 100
+
+training:
+  per_device_batch_size: 4
+  gradient_accumulation_steps: 4
+  max_steps: 1000
+  output_dir: ./outputs
+  # wandb_project: my-project   # uncomment to enable W&B
+```
+
+`gpt-simple init` writes a fully commented template. Every field is
+documented in the [configuration reference](docs/configuration.md), and
+curriculum learning in the [data pipeline guide](docs/data.md).
+
+## Python API
+
+```python
+import gpt_simple
+
+result = gpt_simple.train(
+    model=gpt_simple.ModelConfig(n_embd=768, n_layer=12, n_head=12),
+    data=gpt_simple.DataConfig(path="./data/tokenized", tokenizer="gpt2"),
+    optimizer=gpt_simple.OptimizerConfig(learning_rate=3e-4),
+    training=gpt_simple.TrainingConfig(max_steps=1000, output_dir="./outputs"),
+)
+print(result.final_loss, result.total_tokens, result.checkpoint_path)
+```
+
+Or `gpt_simple.train(config="config.yaml")`; sub-configs passed
+explicitly override the matching section from the file.
+
+## Documentation
+
+Full guides live in [`docs/`](docs/README.md):
+
+- [Architecture](docs/architecture.md) — the built-in model.
+- [Configuration](docs/configuration.md) — every config field.
+- [Data pipeline](docs/data.md) — tokenization, packing, curriculum.
+- [Training](docs/training.md) — multi-GPU, precision, compile.
+- [Checkpointing & resume](docs/checkpointing-and-resume.md) — the
+  stop/resume model.
+- [Orchestration](docs/orchestration.md) — running under any scheduler.
+- [Inference](docs/inference.md) — `generate` / `batch-generate`.
+- [Hardware tuning](docs/hardware-tuning.md) — peak GPU throughput.
+- [Performance](docs/performance.md) — measured 2.8B throughput and MFU.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest tests/
+ruff check src/ tests/
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

gpt_simple_lm-0.1.0/pyproject.toml

@@ -0,0 +1,66 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "gpt-simple-lm"
+version = "0.1.0"
+description = "A clean, efficient framework for pretraining language models from scratch"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.10"
+authors = [{ name = "Louis Bertucci" }]
+keywords = ["llm", "pretraining", "gpt", "pytorch", "transformer", "language-model"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "torch>=2.0.0",
+    "numpy>=1.24.0,<2.0",
+    "transformers>=4.36.0",
+    "accelerate>=0.20.0",
+    "sentencepiece>=0.1.99",
+    "pyyaml>=6.0",
+]
+
+[project.optional-dependencies]
+wandb = ["wandb>=0.15.0"]
+cli = ["rich>=13.0"]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.1.0",
+    "wandb>=0.15.0",
+    "rich>=13.0",
+]
+
+[project.scripts]
+gpt-simple = "gpt_simple.cli.main:main"
+
+[project.urls]
+Homepage = "https://github.com/lb-off/gpt-simple"
+Documentation = "https://lb-off.github.io/gpt-simple/"
+Repository = "https://github.com/lb-off/gpt-simple"
+Issues = "https://github.com/lb-off/gpt-simple/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.lint.per-file-ignores]
+# Tests prepend the project root to sys.path before importing the package,
+# so imports legitimately follow a statement (E402).
+"tests/*" = ["E402"]
+
+[tool.pytest.ini_options]
+markers = [
+    "e2e: end-to-end tests that spawn the gpt-simple CLI as a subprocess (slow, default-included)",
+]

gpt_simple_lm-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

gpt_simple_lm-0.1.0/src/gpt_simple/__init__.py

@@ -0,0 +1,41 @@
+"""
+gpt_simple — a clean, hackable GPT pretraining library.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from gpt_simple.config import Config, CurriculumPhase, DataConfig, ModelConfig, OptimizerConfig, TrainingConfig
+from gpt_simple.errors import CheckpointError, ConfigError, DataError, GptSimpleError
+from gpt_simple.generate import generate, load_for_inference, validate_checkpoint
+from gpt_simple.model import SimpleLLM
+from gpt_simple.tokenizer import SimpleLLMTokenizer
+from gpt_simple.train import TrainingResult, train
+
+# Single source of truth is pyproject.toml; read it from the installed
+# metadata so the two can't drift. Falls back when running from a source
+# tree that hasn't been installed.
+try:
+    __version__ = version("gpt-simple")
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "0.0.0+unknown"
+
+__all__ = [
+    "__version__",
+    "train",
+    "generate",
+    "load_for_inference",
+    "validate_checkpoint",
+    "Config",
+    "CurriculumPhase",
+    "TrainingResult",
+    "ModelConfig",
+    "DataConfig",
+    "OptimizerConfig",
+    "TrainingConfig",
+    "SimpleLLM",
+    "SimpleLLMTokenizer",
+    "GptSimpleError",
+    "ConfigError",
+    "DataError",
+    "CheckpointError",
+]

gpt_simple_lm-0.1.0/src/gpt_simple/__main__.py

@@ -0,0 +1,15 @@
+"""Entry point for ``python -m gpt_simple``.
+
+Kept separate from the library modules on purpose: ``gpt_simple/__init__.py``
+re-exports the public API (including ``train``), so launching a re-exported
+submodule with ``-m`` would make runpy import it twice and emit a
+RuntimeWarning.  ``__main__`` is never imported by ``__init__``, so running
+``-m gpt_simple`` loads the package once and dispatches cleanly.
+
+The distributed launcher (``cli/train_cmd.py``) shells out to this entry point.
+"""
+
+from gpt_simple.train import _module_main
+
+if __name__ == "__main__":
+    _module_main()