tokencraft-gpt 0.1.0__tar.gz

tokencraft_gpt-0.1.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.5
+Name: tokencraft-gpt
+Version: 0.1.0
+Summary: An educational GPT language model implemented from scratch with PyTorch.
+License-Expression: MIT
+Requires-Python: >=3.10
+Requires-Dist: torch>=2.0
+Description-Content-Type: text/markdown
+
+cat > README.md <<'EOF'
+# TokenCraft
+
+**From tokens to intelligence.**
+
+TokenCraft is a small educational GPT-style language model implemented using PyTorch.
+
+The project demonstrates how a GPT model works, from tokenization to next-token generation.
+
+## Features
+
+- Simple word-level tokenizer
+- Token and positional embeddings
+- Causal self-attention
+- Multi-head attention
+- Feed-forward network
+- Transformer blocks
+- GPT decoder architecture
+- Training with next-token prediction
+- Text generation with temperature sampling
+
+## Installation
+
+```bash
+pip install tokencraft-gpt

tokencraft_gpt-0.1.0/README.md

@@ -0,0 +1,25 @@
+cat > README.md <<'EOF'
+# TokenCraft
+
+**From tokens to intelligence.**
+
+TokenCraft is a small educational GPT-style language model implemented using PyTorch.
+
+The project demonstrates how a GPT model works, from tokenization to next-token generation.
+
+## Features
+
+- Simple word-level tokenizer
+- Token and positional embeddings
+- Causal self-attention
+- Multi-head attention
+- Feed-forward network
+- Transformer blocks
+- GPT decoder architecture
+- Training with next-token prediction
+- Text generation with temperature sampling
+
+## Installation
+
+```bash
+pip install tokencraft-gpt

tokencraft_gpt-0.1.0/data/data.txt

@@ -0,0 +1,17 @@
+A language model learns patterns from text.
+A language model predicts the next token.
+A token can be a word or a piece of a word.
+The tokenizer converts text into token IDs.
+The model converts token IDs into embeddings.
+Embeddings represent tokens as vectors.
+A transformer uses attention to process context.
+Self attention helps the model understand relationships between tokens.
+Causal attention prevents the model from seeing future tokens.
+The feed forward network processes token representations.
+The model learns by comparing predictions with target tokens.
+The loss measures how incorrect the predictions are.
+Backpropagation calculates gradients.
+The optimizer updates the model parameters.
+Training helps the model improve its next token predictions.
+A GPT model generates text one token at a time.
+The generated token becomes part of the next input.

tokencraft_gpt-0.1.0/generate.py

@@ -0,0 +1,135 @@
+import torch
+import torch.nn.functional as F
+
+from model.gpt import GPT
+
+
+# -----------------------------
+# Load checkpoint
+# -----------------------------
+
+checkpoint = torch.load(
+    "tokencraft_model.pth",
+    map_location="cpu",
+    weights_only=False
+)
+
+config = checkpoint["config"]
+vocab = checkpoint["vocab"]
+
+# Build reverse vocabulary
+reverse_vocab = {
+    token_id: token
+    for token, token_id in vocab.items()
+}
+
+
+# -----------------------------
+# Select device
+# -----------------------------
+
+if torch.backends.mps.is_available():
+    device = torch.device("mps")
+else:
+    device = torch.device("cpu")
+
+print("Using device:", device)
+
+
+# -----------------------------
+# Rebuild model
+# -----------------------------
+
+model = GPT(
+    vocab_size=config["vocab_size"],
+    context_length=config["context_length"],
+    embedding_dim=config["embedding_dim"],
+    num_heads=config["num_heads"],
+    hidden_dim=config["hidden_dim"],
+    num_layers=config["num_layers"]
+).to(device)
+
+model.load_state_dict(checkpoint["model_state_dict"])
+model.eval()
+
+
+# -----------------------------
+# Text generation
+# -----------------------------
+
+def generate_text(prompt, max_new_tokens=10):
+    # Convert prompt words into token IDs
+    tokens = prompt.split()
+
+    token_ids = [
+        vocab.get(token, vocab.get("<UNK>", 0))
+        for token in tokens
+    ]
+
+    input_ids = torch.tensor(
+        [token_ids],
+        dtype=torch.long,
+        device=device
+    )
+
+    generated_ids = token_ids.copy()
+
+    with torch.no_grad():
+        for _ in range(max_new_tokens):
+            # Keep only the most recent context tokens
+            context = input_ids[:, -config["context_length"]:]
+
+            # Predict next token
+            logits = model(context)
+
+            # Get logits for the final position
+            next_token_logits = logits[:, -1, :]
+
+            # Apply temperature scaling
+            temperature = 0.8
+            scaled_logits = next_token_logits / temperature
+
+            # Convert scaled logits to probabilities
+            probabilities = F.softmax(
+                scaled_logits,
+                dim=-1
+            )
+
+            # Randomly sample the next token
+            next_token_id = torch.multinomial(
+                probabilities,
+                num_samples=1
+            ).item()
+
+            generated_ids.append(next_token_id)
+
+            # Add prediction to input
+            input_ids = torch.cat(
+                [
+                    input_ids,
+                    torch.tensor(
+                        [[next_token_id]],
+                        dtype=torch.long,
+                        device=device
+                    )
+                ],
+                dim=1
+            )
+
+    generated_tokens = [
+        reverse_vocab.get(token_id, "<UNK>")
+        for token_id in generated_ids
+    ]
+
+    return " ".join(generated_tokens)
+
+
+# -----------------------------
+# Run generation
+# -----------------------------
+
+prompt = "the model"
+result = generate_text(prompt, max_new_tokens=10)
+
+print("\nPrompt:", prompt)
+print("Generated text:", result)

tokencraft_gpt-0.1.0/model/__init__.py

@@ -0,0 +1,5 @@
+cat > tokencraft/model/__init__.py <<'PY'
+from .gpt import GPT
+
+__all__ = ["GPT"]
+PY

tokencraft_gpt-0.1.0/model/embedding.py

@@ -0,0 +1,35 @@
+import torch
+import torch.nn as nn
+
+
+class TokenEmbedding(nn.Module):
+    def __init__(self, vocab_size, embedding_dim):
+        super().__init__()
+
+        self.embedding = nn.Embedding(
+            num_embeddings=vocab_size,
+            embedding_dim=embedding_dim
+        )
+
+    def forward(self, token_ids):
+        return self.embedding(token_ids)
+
+
+if __name__ == "__main__":
+    vocab_size = 10
+    embedding_dim = 8
+
+    model = TokenEmbedding(vocab_size, embedding_dim)
+
+    token_ids = torch.tensor([4, 7, 2])
+
+    embeddings = model(token_ids)
+
+    print("Token IDs:")
+    print(token_ids)
+
+    print("\nEmbeddings:")
+    print(embeddings)
+
+    print("\nEmbedding shape:")
+    print(embeddings.shape)

tokencraft_gpt-0.1.0/model/feed_forward.py

@@ -0,0 +1,39 @@
+import torch
+import torch.nn as nn
+
+
+class FeedForward(nn.Module):
+    def __init__(self, embedding_dim, hidden_dim):
+        super().__init__()
+
+        self.network = nn.Sequential(
+            nn.Linear(embedding_dim, hidden_dim),
+            nn.GELU(),
+            nn.Linear(hidden_dim, embedding_dim)
+        )
+
+    def forward(self, x):
+        return self.network(x)
+
+
+if __name__ == "__main__":
+    batch_size = 2
+    sequence_length = 5
+    embedding_dim = 16
+    hidden_dim = 64
+
+    x = torch.randn(
+        batch_size,
+        sequence_length,
+        embedding_dim
+    )
+
+    feed_forward = FeedForward(
+        embedding_dim=embedding_dim,
+        hidden_dim=hidden_dim
+    )
+
+    output = feed_forward(x)
+
+    print("Input shape:", x.shape)
+    print("Output shape:", output.shape)

tokencraft_gpt-0.1.0/model/gpt.py

@@ -0,0 +1,86 @@
+import torch
+import torch.nn as nn
+
+from .input_embedding import InputEmbedding
+from .transformer_block import TransformerBlock
+
+
+class GPT(nn.Module):
+    def __init__(
+        self,
+        vocab_size,
+        context_length,
+        embedding_dim,
+        num_heads,
+        hidden_dim,
+        num_layers
+    ):
+        super().__init__()
+
+        self.context_length = context_length
+
+        self.embedding = InputEmbedding(
+            vocab_size=vocab_size,
+            context_length=context_length,
+            embedding_dim=embedding_dim
+        )
+
+        self.transformer_blocks = nn.Sequential(
+            *[
+                TransformerBlock(
+                    embedding_dim=embedding_dim,
+                    num_heads=num_heads,
+                    hidden_dim=hidden_dim
+                )
+                for _ in range(num_layers)
+            ]
+        )
+
+        self.final_norm = nn.LayerNorm(embedding_dim)
+
+        self.output_head = nn.Linear(
+            embedding_dim,
+            vocab_size,
+            bias=False
+        )
+
+    def forward(self, token_ids):
+        x = self.embedding(token_ids)
+
+        x = self.transformer_blocks(x)
+
+        x = self.final_norm(x)
+
+        logits = self.output_head(x)
+
+        return logits
+
+
+if __name__ == "__main__":
+    vocab_size = 20
+    context_length = 5
+    embedding_dim = 16
+    num_heads = 4
+    hidden_dim = 64
+    num_layers = 2
+
+    token_ids = torch.randint(
+        0,
+        vocab_size,
+        (2, context_length)
+    )
+
+    model = GPT(
+        vocab_size=vocab_size,
+        context_length=context_length,
+        embedding_dim=embedding_dim,
+        num_heads=num_heads,
+        hidden_dim=hidden_dim,
+        num_layers=num_layers
+    )
+
+    logits = model(token_ids)
+
+    print("Input shape:", token_ids.shape)
+    print("Logits shape:", logits.shape)
+    print("Parameter count:", sum(p.numel() for p in model.parameters()))

tokencraft_gpt-0.1.0/model/input_embedding.py

@@ -0,0 +1,54 @@
+import torch
+import torch.nn as nn
+
+
+class InputEmbedding(nn.Module):
+    def __init__(self, vocab_size, context_length, embedding_dim):
+        super().__init__()
+
+        self.token_embedding = nn.Embedding(
+            num_embeddings=vocab_size,
+            embedding_dim=embedding_dim
+        )
+
+        self.position_embedding = nn.Embedding(
+            num_embeddings=context_length,
+            embedding_dim=embedding_dim
+        )
+
+    def forward(self, token_ids):
+        sequence_length = token_ids.shape[1]
+
+        positions = torch.arange(
+            sequence_length,
+            device=token_ids.device
+        )
+
+        token_vectors = self.token_embedding(token_ids)
+        position_vectors = self.position_embedding(positions)
+
+        return token_vectors + position_vectors
+
+
+if __name__ == "__main__":
+    vocab_size = 20
+    context_length = 5
+    embedding_dim = 8
+
+    token_ids = torch.tensor([
+        [1, 2, 3, 4, 5],
+        [5, 4, 3, 2, 1]
+    ])
+
+    model = InputEmbedding(
+        vocab_size=vocab_size,
+        context_length=context_length,
+        embedding_dim=embedding_dim
+    )
+
+    output = model(token_ids)
+
+    print("Output:")
+    print(output)
+
+    print("Shape:", output.shape)

tokencraft_gpt-0.1.0/model/multi_head_attention.py

@@ -0,0 +1,117 @@
+import torch
+import torch.nn as nn
+import math
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, embedding_dim, num_heads):
+        super().__init__()
+
+        if embedding_dim % num_heads != 0:
+            raise ValueError(
+                "embedding_dim must be divisible by num_heads"
+            )
+
+        self.embedding_dim = embedding_dim
+        self.num_heads = num_heads
+        self.head_dim = embedding_dim // num_heads
+
+        self.query = nn.Linear(embedding_dim, embedding_dim)
+        self.key = nn.Linear(embedding_dim, embedding_dim)
+        self.value = nn.Linear(embedding_dim, embedding_dim)
+
+        self.output_projection = nn.Linear(
+            embedding_dim,
+            embedding_dim
+        )
+
+        self.scale = math.sqrt(self.head_dim)
+
+    def forward(self, x):
+        batch_size, sequence_length, _ = x.shape
+
+        Q = self.query(x)
+        K = self.key(x)
+        V = self.value(x)
+
+        # Split into multiple heads
+        Q = Q.view(
+            batch_size,
+            sequence_length,
+            self.num_heads,
+            self.head_dim
+        ).transpose(1, 2)
+
+        K = K.view(
+            batch_size,
+            sequence_length,
+            self.num_heads,
+            self.head_dim
+        ).transpose(1, 2)
+
+        V = V.view(
+            batch_size,
+            sequence_length,
+            self.num_heads,
+            self.head_dim
+        ).transpose(1, 2)
+
+        # Attention scores
+        scores = torch.matmul(
+            Q,
+            K.transpose(-2, -1)
+        ) / self.scale
+
+        # Causal mask
+        mask = torch.triu(
+            torch.ones(
+                sequence_length,
+                sequence_length,
+                device=x.device,
+                dtype=torch.bool
+            ),
+            diagonal=1
+        )
+
+        scores = scores.masked_fill(mask, float("-inf"))
+
+        attention_weights = torch.softmax(
+            scores,
+            dim=-1
+        )
+
+        output = torch.matmul(attention_weights, V)
+
+        # Combine attention heads
+        output = output.transpose(1, 2).contiguous()
+
+        output = output.view(
+            batch_size,
+            sequence_length,
+            self.embedding_dim
+        )
+
+        return self.output_projection(output)
+
+
+if __name__ == "__main__":
+    batch_size = 2
+    sequence_length = 5
+    embedding_dim = 16
+    num_heads = 4
+
+    x = torch.randn(
+        batch_size,
+        sequence_length,
+        embedding_dim
+    )
+
+    attention = MultiHeadAttention(
+        embedding_dim=embedding_dim,
+        num_heads=num_heads
+    )
+
+    output = attention(x)
+
+    print("Input shape:", x.shape)
+    print("Output shape:", output.shape)

tokencraft_gpt-0.1.0/model/positional_embedding.py

@@ -0,0 +1,45 @@
+import torch
+import torch.nn as nn
+
+
+class PositionalEmbedding(nn.Module):
+    def __init__(self, context_length, embedding_dim):
+        super().__init__()
+
+        self.position_embedding = nn.Embedding(
+            num_embeddings=context_length,
+            embedding_dim=embedding_dim
+        )
+
+    def forward(self, token_ids):
+        sequence_length = token_ids.shape[1]
+
+        positions = torch.arange(
+            sequence_length,
+            device=token_ids.device
+        )
+
+        return self.position_embedding(positions)
+
+
+if __name__ == "__main__":
+    batch_size = 2
+    context_length = 5
+    embedding_dim = 8
+
+    token_ids = torch.tensor([
+        [1, 2, 3, 4, 5],
+        [5, 4, 3, 2, 1]
+    ])
+
+    model = PositionalEmbedding(
+        context_length=context_length,
+        embedding_dim=embedding_dim
+    )
+
+    positions = model(token_ids)
+
+    print("Positional embeddings:")
+    print(positions)
+
+    print("Shape:", positions.shape)

tokencraft_gpt-0.1.0/model/self_attention.py

@@ -0,0 +1,64 @@
+import torch
+import torch.nn as nn
+import math
+
+
+class SelfAttention(nn.Module):
+    def __init__(self, embedding_dim):
+        super().__init__()
+
+        self.query = nn.Linear(embedding_dim, embedding_dim)
+        self.key = nn.Linear(embedding_dim, embedding_dim)
+        self.value = nn.Linear(embedding_dim, embedding_dim)
+
+        self.scale = math.sqrt(embedding_dim)
+
+    def forward(self, x):
+        # x shape: (batch_size, sequence_length, embedding_dim)
+
+        Q = self.query(x)
+        K = self.key(x)
+        V = self.value(x)
+
+        # Calculate attention scores
+        scores = torch.matmul(Q, K.transpose(-2, -1))
+        scores = scores / self.scale
+
+        # Causal mask: prevent looking at future tokens
+        sequence_length = x.shape[1]
+
+        mask = torch.triu(
+            torch.ones(
+                sequence_length,
+                sequence_length,
+                device=x.device,
+                dtype=torch.bool
+            ),
+            diagonal=1
+        )
+
+        scores = scores.masked_fill(mask, float("-inf"))
+
+        # Convert scores into probabilities
+        attention_weights = torch.softmax(scores, dim=-1)
+
+        # Combine information from previous tokens
+        output = torch.matmul(attention_weights, V)
+
+        return output
+
+
+if __name__ == "__main__":
+    batch_size = 2
+    sequence_length = 5
+    embedding_dim = 8
+
+    x = torch.randn(batch_size, sequence_length, embedding_dim)
+
+    attention = SelfAttention(embedding_dim)
+
+    output = attention(x)
+
+    print("Output shape:", output.shape)
+    print("Output:")
+    print(output)