midigpt 0.0.1b3__tar.gz

midigpt-0.0.1b3/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Johnny Greco
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

midigpt-0.0.1b3/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: midigpt
+Version: 0.0.1b3
+Summary: midiGPT: A MIDI Music Generation Library
+Home-page: https://github.com/johnnygreco/midiGPT
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+License-File: LICENSE
+
+# 🎹 midiGPT
+
+Inspired by Andrej Karpathy's [minGPT](https://github.com/karpathy/minGPT), I am building `midiGPT` as a learning-in-public (LIP) project. The goal is to build a library that makes it easy to train a decoder-only transformer model (similar to GPT) on MIDI data. 
+
+## 🚧 Under Construction 🚧

midigpt-0.0.1b3/README.md

@@ -0,0 +1,5 @@
+# 🎹 midiGPT
+
+Inspired by Andrej Karpathy's [minGPT](https://github.com/karpathy/minGPT), I am building `midiGPT` as a learning-in-public (LIP) project. The goal is to build a library that makes it easy to train a decoder-only transformer model (similar to GPT) on MIDI data. 
+
+## 🚧 Under Construction 🚧

midigpt-0.0.1b3/pyproject.toml

@@ -0,0 +1,44 @@
+[tool.black]
+line-length = 120
+
+[tool.ruff]
+line-length = 120
+
+# Enable Pyflakes `E` and `F` codes by default.
+select = ["E", "F"]
+ignore = ["F403", "F405"]
+
+unfixable = ["F401", "F841"]
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+
+[tool.ruff.per-file-ignores]
+"__init__.py" = ["F401"]
+
+[tool.ruff.isort]
+known-first-party = ["midigpt"]
+
+[tool.semantic_release]
+branch = "main"
+version_variable = "setup.py:__version__"

midigpt-0.0.1b3/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

midigpt-0.0.1b3/setup.py

@@ -0,0 +1,37 @@
+import os
+
+from setuptools import find_packages, setup
+
+if os.path.isfile("VERSION"):
+    with open("VERSION") as version:
+        __version__ = version.read().strip("v")
+else:
+    __version__ = "0.0.1-beta.3"
+
+with open("README.md", "r") as fh:
+    long_description = fh.read() or ""
+
+with open("requirements.txt", "r") as f:
+    install_requires = f.read().splitlines()
+
+setup(
+    name="midigpt",
+    version=__version__,
+    description="midiGPT: A MIDI Music Generation Library",
+    long_description=long_description,
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    python_requires=">=3.8",
+    install_requires=install_requires,
+    url="https://github.com/johnnygreco/midiGPT",
+    classifiers=[
+        "Intended Audience :: Science/Research",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "License :: OSI Approved :: MIT License",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    ],
+)

midigpt-0.0.1b3/src/midigpt/__init__.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from . import datasets
+from .config import ModelConfigure, TrainConfigure
+from .gpt import GPT
+from .trainer import Trainer

midigpt-0.0.1b3/src/midigpt/components.py

@@ -0,0 +1,78 @@
+import math
+
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+from .config import ModelConfigure
+
+__all__ = ["CasualMultiHeadAttention", "CasualAttentionBlock", "FeedForward"]
+
+
+class GELU(nn.Module):
+    """GELU activation function: https://arxiv.org/abs/1606.08415"""
+
+    def forward(self, x):
+        return 0.5 * x * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0))))
+
+
+class CasualMultiHeadAttention(nn.Module):
+    def __init__(self, config: ModelConfigure) -> None:
+        super().__init__()
+        q_k_v_size = 3 * config.embedding_size
+        self.attn = nn.Linear(config.embedding_size, q_k_v_size)
+        self.output_projection = nn.Linear(config.embedding_size, config.embedding_size)
+        self.attn_dropout = nn.Dropout(config.attn_dropout_prob)
+        self.embed_dropout = nn.Dropout(config.embed_dropout_prob)
+        _tril_reshape = torch.tril(torch.ones(config.context_length, config.context_length)).view(
+            1, 1, config.context_length, config.context_length
+        )
+        self.register_buffer("casual_mask", _tril_reshape)
+        self.num_heads = config.num_heads
+        self.embedding_size = config.embedding_size
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        d_batch, d_context, d_embed = x.size()
+        q, k, v = self.attn(x).split(self.embedding_size, dim=2)
+        q = q.view(d_batch, d_context, self.num_heads, d_embed // self.num_heads).transpose(1, 2)
+        k = k.view(d_batch, d_context, self.num_heads, d_embed // self.num_heads).transpose(1, 2)
+        v = v.view(d_batch, d_context, self.num_heads, d_embed // self.num_heads).transpose(1, 2)
+
+        attn = q @ k.transpose(-2, -1) / (k.size(-1) ** 0.5)
+        attn = attn.masked_fill(self.casual_mask[:, :, :d_context, :d_context] == 0, float("-inf"))
+        attn = F.softmax(attn, dim=-1)
+        attn = self.attn_dropout(attn)
+        attended_values = attn @ v
+        attended_values = attended_values.transpose(1, 2).contiguous().view(*x.size())
+
+        output = self.output_projection(attended_values)
+        output = self.embed_dropout(output)
+        return output
+
+
+class FeedForward(nn.Module):
+    def __init__(self, config: ModelConfigure) -> None:
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(config.embedding_size, 4 * config.embedding_size),
+            GELU(),
+            nn.Linear(4 * config.embedding_size, config.embedding_size),
+            nn.Dropout(config.embed_dropout_prob),
+        )
+
+    def forward(self, x: torch.Tensor):
+        return self.net(x)
+
+
+class CasualAttentionBlock(nn.Module):
+    def __init__(self, config: ModelConfigure):
+        super().__init__()
+        self.layer_norm_1 = nn.LayerNorm(config.embedding_size)
+        self.multi_head_self_attention = CasualMultiHeadAttention(config)
+        self.layer_norm_2 = nn.LayerNorm(config.embedding_size)
+        self.ff_net = FeedForward(config)
+
+    def forward(self, x: torch.Tensor):
+        x = x + self.multi_head_self_attention(self.layer_norm_1(x))
+        x = x + self.ff_net(self.layer_norm_2(x))
+        return x

midigpt-0.0.1b3/src/midigpt/config.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+from typing import Optional, Union
+
+from pydantic import BaseModel, root_validator
+
+__all__ = ["ModelConfigure", "TrainConfigure"]
+
+
+class ModelConfigure(BaseModel):
+    vocab_size: int
+    context_length: int = 64
+    embedding_size: int = 32
+    num_heads: int = 4
+    num_blocks: int = 4
+    attn_dropout_prob: float = 0.1
+    embed_dropout_prob: float = 0.1
+    device: str = "auto"
+
+    @root_validator
+    def check_embedding_size_num_heads_ratio(cls, values):
+        num_heads = values.get("num_heads")
+        embedding_size = values.get("embedding_size")
+        if embedding_size % num_heads != 0:
+            raise ValueError(f"{embedding_size=} is not divisible by {num_heads=}")
+        return values
+
+
+class TrainConfigure(ModelConfigure):
+    batch_size: int = 32
+    learning_rate: float = 5e-4
+    eval_iters: int = 200
+    eval_interval: int = 500
+    num_epochs: int = 3
+    batches_per_epoch: Optional[int] = None
+    checkpoint_path: Union[str, Path] = Path("checkpoints")
+    save_all_checkpoints: bool = False
+    overwrite_checkpoints: bool = True

midigpt-0.0.1b3/src/midigpt/datasets/__init__.py

@@ -0,0 +1,5 @@
+from typing import Type
+
+from .text_character import TextCharacterDataset, TextCharacterTokenizer
+
+DatasetType = Type[TextCharacterDataset]

midigpt-0.0.1b3/src/midigpt/datasets/text_character.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+from typing import List, Union
+
+import torch
+from torch.utils.data import Dataset
+
+from ..config import ModelConfigure, TrainConfigure
+
+__all__ = ["TextCharacterDataset", "TextCharacterTokenizer"]
+
+
+class TextCharacterTokenizer:
+    def __init__(self, vocabulary: List[str]):
+        self.vocabulary = vocabulary
+        self.vocab_size = len(self.vocabulary)
+        self.str_to_int = {s: i for i, s in enumerate(self.vocabulary)}
+        self.int_to_str = {i: s for i, s in enumerate(self.vocabulary)}
+
+    @classmethod
+    def from_corpus(cls, text_corpus: str):
+        vocabulary = sorted(set(text_corpus))
+        return cls(vocabulary)
+
+    @classmethod
+    def from_file(cls, file_path: Union[str, Path]):
+        with open(file_path, "r") as f:
+            text_corpus = f.read()
+        return cls.from_corpus(text_corpus)
+
+    def encode(self, text: str):
+        return [self.str_to_int[s] for s in text]
+
+    def decode(self, tokens: List[int]):
+        return "".join([self.int_to_str[t] for t in tokens])
+
+
+class TextCharacterDataset(Dataset):
+    def __init__(self, text_corpus: str, vocabulary: List[str], config: Union[ModelConfigure, TrainConfigure]):
+        self.text_corpus = text_corpus
+        self.vocabulary = vocabulary
+        self.vocab_size = len(self.vocabulary)
+        self.context_length = config.context_length
+        self.tokenizer = TextCharacterTokenizer(vocabulary)
+
+    def __len__(self):
+        return len(self.text_corpus) - self.context_length
+
+    def __getitem__(self, idx):
+        context = self.text_corpus[idx : idx + self.context_length + 1]
+        context_encoded = self.tokenizer.encode(context)
+        x = torch.tensor(context_encoded[:-1], dtype=torch.long)
+        y = torch.tensor(context_encoded[1:], dtype=torch.long)
+        return x, y

midigpt-0.0.1b3/src/midigpt/gpt.py

@@ -0,0 +1,96 @@
+from pathlib import Path
+from typing import List, Optional, Union
+
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+from . import utils
+from .components import CasualAttentionBlock
+from .config import ModelConfigure, TrainConfigure
+
+__all__ = ["GPT"]
+
+
+class GPT(nn.Module):
+    def __init__(self, config: Union[ModelConfigure, TrainConfigure]):
+        super().__init__()
+        self.token_embedding_table = nn.Embedding(config.vocab_size, config.embedding_size)
+        self.position_embedding_table = nn.Embedding(config.context_length, config.embedding_size)
+        self.blocks = nn.Sequential(*[CasualAttentionBlock(config) for _ in range(config.num_blocks)])
+        self.final_layer_norm = nn.LayerNorm(config.embedding_size)
+        self.reproject_to_vocab = nn.Linear(config.embedding_size, config.vocab_size)
+        self.apply(self._init_weights)
+        self.context_length = config.context_length
+        self.device = utils.get_auto_device() if config.device == "auto" else config.device
+
+    def _init_weights(self, module):
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if hasattr(module, "bias") and module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+
+    @property
+    def num_params(self):
+        return sum(p.numel() for p in self.parameters())
+
+    @classmethod
+    def from_checkpoint(cls, path: Union[str, Path]):
+        checkpoint = torch.load(path)
+        config = checkpoint["model_config"]
+        model = cls(ModelConfigure(**config))
+        model.load_state_dict(checkpoint["model_state_dict"])
+        model.to(utils.get_auto_device() if config["device"] == "auto" else config["device"])
+        return model
+
+    def forward(self, x: torch.LongTensor, targets: Optional[torch.LongTensor] = None):
+        token_embedding = self.token_embedding_table(x)  # (B, T, C)
+        position_embedding = self.position_embedding_table(torch.arange(x.shape[1], device=self.device))  # (T, C)
+        x = token_embedding + position_embedding  # (B, T, C)
+        x = self.blocks(x)  # (B, T, C)
+        x = self.final_layer_norm(x)  # (B, T, C)
+        logits = self.reproject_to_vocab(x)  # (B, T, vocab_size)
+        loss = (
+            None
+            if targets is None
+            else F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
+        )
+        return logits, loss
+
+    @torch.no_grad()
+    def generate(
+        self,
+        idx: Union[List[int], torch.LongTensor],
+        num_generated_tokens: int,
+        temperature: float = 1.0,
+        do_sample: bool = False,
+        top_k: Optional[int] = None,
+        as_list: bool = False,
+    ):
+        idx = torch.as_tensor(idx, dtype=torch.long, device=self.device)[None, ...]
+        changed_training_mode = False
+        if self.training:
+            self.eval()
+            changed_training_mode = True
+        for _ in range(num_generated_tokens):
+            # if the sequence context is growing too long we must crop it at context_length
+            idx_cond = idx if idx.size(1) <= self.context_length else idx[:, -self.context_length :]
+            # forward the model to get the logits for the index in the sequence
+            logits, _ = self(idx_cond)
+            # pluck the logits at the final step and scale by desired temperature
+            logits = logits[:, -1, :] / temperature
+            # optionally crop the logits to only the top k options
+            if top_k is not None:
+                logits[logits < torch.topk(logits, top_k)[0][:, [-1]]] = -float("Inf")
+            # apply softmax to convert logits to (normalized) probabilities
+            probs = F.softmax(logits, dim=-1)
+            # either sample from the distribution or take the most likely element
+            idx_next = torch.multinomial(probs, num_samples=1) if do_sample else torch.topk(probs, k=1, dim=-1)[1]
+            # append sampled index to the running sequence and continue
+            idx = torch.cat((idx, idx_next), dim=1)
+        if changed_training_mode:
+            self.train()
+        return idx.cpu().squeeze().tolist() if as_list else idx
+
+    def save(self, path: Union[str, Path]):
+        torch.save(self.state_dict(), path)

midigpt-0.0.1b3/src/midigpt/trainer.py

@@ -0,0 +1,104 @@
+import warnings
+from pathlib import Path
+from typing import Union
+
+import torch
+from torch.utils.data import DataLoader
+from tqdm.rich import tqdm
+from tqdm.std import TqdmExperimentalWarning
+
+from . import utils
+from .config import ModelConfigure, TrainConfigure
+from .datasets import DatasetType
+from .gpt import GPT
+
+__all__ = ["Trainer"]
+
+warnings.simplefilter("ignore", TqdmExperimentalWarning)
+
+
+class Trainer:
+    def __init__(self, config: TrainConfigure):
+        self.config = config
+        self.model = GPT(config)
+        self.checkpoint_path = Path(self.config.checkpoint_path)
+        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.learning_rate)
+        self.device = utils.get_auto_device() if config.device == "auto" else config.device
+        self.model.to(self.device)
+        self._loss_history = []
+
+    def _save_loss_history(self):
+        with open(self.checkpoint_path / "loss_history.txt", "a") as f:
+            f.write("\n".join(map(str, self._loss_history)) + "\n")
+        self._loss_history = []
+
+    def _save_model_if_best(self, epoch: int, batch_num: int, loss: float, lowest_loss: float):
+        if loss < lowest_loss or self.config.save_all_checkpoints:
+            lowest_loss = loss
+            self.checkpoint_path.mkdir(parents=True, exist_ok=True)
+            file_name = (
+                "best_model.ckpt" if self.config.overwrite_checkpoints else f"epoch_{epoch}_batch_{batch_num}.ckpt"
+            )
+            torch.save(
+                {
+                    "epoch": epoch,
+                    "batch_num": batch_num,
+                    "loss": loss,
+                    "train_config": self.config.dict(),
+                    "model_config": {
+                        k: v for k, v in self.config.dict().items() if k in ModelConfigure.__fields__.keys()
+                    },
+                    "model_state_dict": self.model.state_dict(),
+                    "optimizer_state_dict": self.optimizer.state_dict(),
+                },
+                self.checkpoint_path / file_name,
+            )
+        return lowest_loss
+
+    @classmethod
+    def from_checkpoint(cls, path: Union[str, Path]):
+        checkpoint = torch.load(path)
+        model_config = checkpoint["model_config"]
+        model = GPT(ModelConfigure(**model_config))
+        model.load_state_dict(checkpoint["model_state_dict"])
+        model.to(utils.get_auto_device() if model_config["device"] == "auto" else model_config["device"])
+        trainer = cls(TrainConfigure(**checkpoint["train_config"]))
+        trainer.model = model
+        trainer.loss = checkpoint["loss"]
+        trainer.optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
+        return trainer
+
+    def train(self, dataset: DatasetType, shuffle: bool = True):
+        self.train_loader = DataLoader(dataset=dataset, batch_size=self.config.batch_size, shuffle=shuffle)
+        batches_per_epoch = self.config.batches_per_epoch if self.config.batches_per_epoch else len(self.train_loader)
+        total_iterations = self.config.num_epochs * batches_per_epoch
+        with tqdm(total=total_iterations, desc=f"Training for {self.config.num_epochs} epochs:") as progress_bar:
+            lowest_loss = float("inf")
+            self.model.train()
+            for epoch in range(1, self.config.num_epochs + 1):
+                running_loss_epoch = 0.0
+                running_loss_batch_interval = 0.0
+                for batch_num, (x, y) in enumerate(self.train_loader, start=1):
+                    x, y = x.to(self.device), y.to(self.device)
+                    _, self.loss = self.model(x, y)
+                    self.optimizer.zero_grad(set_to_none=True)
+                    self.loss.backward()
+                    self.optimizer.step()
+                    running_loss_epoch += self.loss.item()
+                    running_loss_batch_interval += self.loss.item()
+                    self._loss_history.append(self.loss.item())
+                    if batch_num % self.config.eval_interval == 0:
+                        average_loss = running_loss_batch_interval / self.config.eval_interval
+                        tqdm.write(
+                            f"epoch: {epoch:<4.0f}  |  "
+                            f"batch: {batch_num:<7.0f}  |  "
+                            f"average batch loss: {average_loss:<.4f}"
+                        )
+                        lowest_loss = self._save_model_if_best(epoch, batch_num, average_loss, lowest_loss)
+                        self._save_loss_history()
+                        running_loss_batch_interval = 0.0
+                    progress_bar.update(1)
+                    if self.config.batches_per_epoch and batch_num > self.config.batches_per_epoch:
+                        break
+                lowest_loss = self._save_model_if_best(epoch, batch_num, running_loss_epoch / batch_num, lowest_loss)
+                self._save_loss_history()

midigpt-0.0.1b3/src/midigpt/utils.py

@@ -0,0 +1,13 @@
+import torch
+
+__all__ = ["get_auto_device"]
+
+
+def get_auto_device():
+    if torch.has_cuda:
+        device = torch.cuda.current_device()
+    elif torch.has_mps:
+        device = "mps"
+    else:
+        device = "cpu"
+    return device

midigpt-0.0.1b3/src/midigpt.egg-info/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: midigpt
+Version: 0.0.1b3
+Summary: midiGPT: A MIDI Music Generation Library
+Home-page: https://github.com/johnnygreco/midiGPT
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+License-File: LICENSE
+
+# 🎹 midiGPT
+
+Inspired by Andrej Karpathy's [minGPT](https://github.com/karpathy/minGPT), I am building `midiGPT` as a learning-in-public (LIP) project. The goal is to build a library that makes it easy to train a decoder-only transformer model (similar to GPT) on MIDI data. 
+
+## 🚧 Under Construction 🚧

midigpt-0.0.1b3/src/midigpt.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+LICENSE
+README.md
+pyproject.toml
+setup.py
+src/midigpt/__init__.py
+src/midigpt/components.py
+src/midigpt/config.py
+src/midigpt/gpt.py
+src/midigpt/trainer.py
+src/midigpt/utils.py
+src/midigpt.egg-info/PKG-INFO
+src/midigpt.egg-info/SOURCES.txt
+src/midigpt.egg-info/dependency_links.txt
+src/midigpt.egg-info/requires.txt
+src/midigpt.egg-info/top_level.txt
+src/midigpt/datasets/__init__.py
+src/midigpt/datasets/text_character.py

midigpt-0.0.1b3/src/midigpt.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

midigpt-0.0.1b3/src/midigpt.egg-info/requires.txt

@@ -0,0 +1,4 @@
+pydantic==1.10.5
+rich==13.3.2
+torch==1.13.1
+tqdm==4.65.0

midigpt-0.0.1b3/src/midigpt.egg-info/top_level.txt

@@ -0,0 +1 @@
+midigpt