shadow-tools 0.7.2__tar.gz → 0.8.1__tar.gz

{shadow_tools-0.7.2 → shadow_tools-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: shadow-tools
-Version: 0.7.2
+Version: 0.8.1
 Summary: A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs.
 Project-URL: Homepage, https://github.com/WilliamLaurento/shadow-tools
 Project-URL: Issues, https://github.com/WilliamLaurento/shadow-tools/issues
@@ -28,6 +28,18 @@ Description-Content-Type: text/markdown
 
 # Shadow Tools
 
+## 0.8.1 generation fix
+
+`ask(bot, question)` returns only the generated answer and stops before a new
+`Question:` or `Answer:` line. `teach()` now trains with an end token after each
+answer. Retrain existing bots to learn these boundaries. `say()` returns only
+new text by default; use `include_prompt=True` to include the input.
+
+Generation accepts `repetition_penalty=1.1` (1.0 disables it). This discourages
+repeated tokens; it cannot guarantee original or accurate answers. Training
+on a few examples can still memorize those examples. No pretrained weights
+or answer lookup are used.
+
 ## Train on regular text
 
 ```python
@@ -76,6 +88,22 @@ bot.save("my_text_model")
 Training prints the tokenizer status plus each epoch and loss. Use
 `learn("corpus.txt", epochs=20, verbose=False)` when you do not want updates.
 
+## Train a question-and-answer bot
+
+```python
+from shadowtools import ask, teach
+
+bot = teach({
+    "What is ChatGPT?": "ChatGPT is an AI system that predicts useful next tokens to answer a prompt.",
+    "How does a tokenizer work?": "A tokenizer splits text into smaller tokens a model can process.",
+    "What is machine learning?": "Machine learning is when a model learns patterns from examples.",
+}, epochs=20)
+
+print(ask(bot, "What is ChatGPT?"))
+```
+
+Add many more real Q&A pairs for better answers.
+
 Install it with `pip install "shadow-tools[ml,nlp]"`. The package trains the
 tokenizer and Transformer together, then converts predicted token IDs back to
 normal text for you.

{shadow_tools-0.7.2 → shadow_tools-0.8.1}/README.md

@@ -1,5 +1,17 @@
 # Shadow Tools
 
+## 0.8.1 generation fix
+
+`ask(bot, question)` returns only the generated answer and stops before a new
+`Question:` or `Answer:` line. `teach()` now trains with an end token after each
+answer. Retrain existing bots to learn these boundaries. `say()` returns only
+new text by default; use `include_prompt=True` to include the input.
+
+Generation accepts `repetition_penalty=1.1` (1.0 disables it). This discourages
+repeated tokens; it cannot guarantee original or accurate answers. Training
+on a few examples can still memorize those examples. No pretrained weights
+or answer lookup are used.
+
 ## Train on regular text
 
 ```python
@@ -48,6 +60,22 @@ bot.save("my_text_model")
 Training prints the tokenizer status plus each epoch and loss. Use
 `learn("corpus.txt", epochs=20, verbose=False)` when you do not want updates.
 
+## Train a question-and-answer bot
+
+```python
+from shadowtools import ask, teach
+
+bot = teach({
+    "What is ChatGPT?": "ChatGPT is an AI system that predicts useful next tokens to answer a prompt.",
+    "How does a tokenizer work?": "A tokenizer splits text into smaller tokens a model can process.",
+    "What is machine learning?": "Machine learning is when a model learns patterns from examples.",
+}, epochs=20)
+
+print(ask(bot, "What is ChatGPT?"))
+```
+
+Add many more real Q&A pairs for better answers.
+
 Install it with `pip install "shadow-tools[ml,nlp]"`. The package trains the
 tokenizer and Transformer together, then converts predicted token IDs back to
 normal text for you.

{shadow_tools-0.7.2 → shadow_tools-0.8.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "shadow-tools"
-version = "0.7.2"
+version = "0.8.1"
 description = "A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs."
 readme = "README.md"
 requires-python = ">=3.10"

{shadow_tools-0.7.2 → shadow_tools-0.8.1}/src/shadowtools/__init__.py

@@ -5,7 +5,7 @@ from .bpe import BPETokenizer
 from .config import env, require_env, to_bool
 from .files import append_text, ensure_dir, read_text, write_text
 from .hashing import file_hash, hash_text
-from .generator import GeneratorConfig, TextGenerator, generate_text, learn, say, train_generator
+from .generator import GeneratorConfig, TextGenerator, ask, generate_text, learn, say, teach, train_generator
 from .http import get, request
 from .ids import random_token, timestamp_id, uuid4
 from .network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, resolve_dns, reverse_dns
@@ -15,10 +15,10 @@ from .text import chunks, slugify, to_camel, to_snake
 from .training import SmartTokenizer, fine_tune, make_loaders, predict_text, train, train_text
 from .validation import is_email, is_url, require_keys
 
-__version__ = "0.7.2"
+__version__ = "0.8.1"
 
 __all__ = [
-    "append_text", "audit_headers", "BPETokenizer", "chunks", "ensure_dir", "env", "file_hash", "GeneratorConfig", "generate_random_ips", "generate_test_ips", "generate_text", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "learn", "make_password", "make_subdomain", "random_token", "read_csv",
-    "read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "say", "slugify", "timestamp_id", "to_bool", "to_camel", "to_snake", "TextGenerator", "train", "train_generator", "fine_tune", "make_loaders", "SmartTokenizer", "train_text", "predict_text", "uuid4", "write_csv",
+    "append_text", "ask", "audit_headers", "BPETokenizer", "chunks", "ensure_dir", "env", "file_hash", "GeneratorConfig", "generate_random_ips", "generate_test_ips", "generate_text", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "learn", "make_password", "make_subdomain", "random_token", "read_csv",
+    "read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "say", "slugify", "teach", "timestamp_id", "to_bool", "to_camel", "to_snake", "TextGenerator", "train", "train_generator", "fine_tune", "make_loaders", "SmartTokenizer", "train_text", "predict_text", "uuid4", "write_csv",
     "write_json", "write_text", "redact_secrets", "verify_file_hash",
 ]

{shadow_tools-0.7.2 → shadow_tools-0.8.1}/src/shadowtools/generator.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 from dataclasses import asdict, dataclass
 from pathlib import Path
-from typing import Any, Sequence
+from typing import Any, Mapping, Sequence
 
 from .bpe import BPETokenizer
 
@@ -91,7 +91,12 @@ class TextGenerator:
         if show_progress:
             print("Building tokenizer...")
         self.tokenizer = BPETokenizer(vocab_size=self.config.vocab_size, min_frequency=1).train([corpus])
-        token_ids = self.tokenizer.encode(corpus, add_special_tokens=False)["input_ids"]
+        # Preserve document boundaries so the model learns when to stop.
+        documents = [texts] if isinstance(texts, str) else list(texts)
+        token_ids = []
+        for document in documents:
+            token_ids.extend(self.tokenizer.encode(document, add_special_tokens=False)["input_ids"])
+            token_ids.append(self.tokenizer.eos_id)
         if len(token_ids) <= self.config.block_size:
             raise ValueError("Training text needs more tokens than block_size.")
         self.model = self._build_model(self.tokenizer.size)
@@ -133,14 +138,22 @@ class TextGenerator:
         return history
 
     def generate(self, prompt: str, *, max_tokens: int = 80, temperature: float = 0.8,
-                 top_k: int = 30) -> str:
+                 top_k: int = 30, repetition_penalty: float = 1.1,
+                 include_prompt: bool = False, stop: Sequence[str] = ()) -> str:
         """Write tokens after ``prompt`` using the trained Transformer."""
         if not prompt:
             raise ValueError("prompt must not be empty.")
         if max_tokens < 1 or temperature <= 0 or top_k < 1:
             raise ValueError("max_tokens, temperature, and top_k must be positive.")
+        if repetition_penalty < 1:
+            raise ValueError("repetition_penalty must be at least 1.")
+        if isinstance(stop, str) or any(not isinstance(s, str) or not s for s in stop):
+            raise ValueError("stop must be a sequence of non-empty strings.")
         tokenizer, model = self._ready()
         token_ids = tokenizer.encode(prompt, add_special_tokens=False)["input_ids"]
+        if not token_ids:
+            raise ValueError("prompt must encode to at least one token.")
+        prompt_length = len(token_ids)
         torch = self.torch
         model.eval()
         with torch.no_grad():
@@ -150,6 +163,10 @@ class TextGenerator:
                 logits = model(inputs)[0, -1] / temperature
                 logits[tokenizer.pad_id] = float("-inf")
                 logits[tokenizer.bos_id] = float("-inf")
+                logits[1] = float("-inf")  # BPETokenizer's reserved UNK ID
+                for seen_id in set(token_ids[prompt_length:]):
+                    value = logits[seen_id]
+                    logits[seen_id] = value * repetition_penalty if value < 0 else value / repetition_penalty
                 count = min(top_k, tokenizer.size)
                 values, indexes = torch.topk(logits, count)
                 filtered = torch.full_like(logits, float("-inf"))
@@ -158,7 +175,13 @@ class TextGenerator:
                 if next_id == tokenizer.eos_id:
                     break
                 token_ids.append(next_id)
-        return tokenizer.decode(token_ids)
+                continuation = tokenizer.decode(token_ids[prompt_length:])
+                positions = [continuation.find(marker) for marker in stop if marker in continuation]
+                if positions:
+                    answer = continuation[:min(positions)]
+                    return prompt + answer if include_prompt else answer
+        answer = tokenizer.decode(token_ids[prompt_length:])
+        return prompt + answer if include_prompt else answer
 
     def save(self, folder: str | Path) -> None:
         """Save the model and tokenizer together."""
@@ -226,3 +249,27 @@ def learn(source: str | Path | Sequence[str], *, epochs: int = 10,
 def say(generator: TextGenerator, prompt: str, **options: Any) -> str:
     """Beginner-friendly name for generating text from a trained model."""
     return generator.generate(prompt, **options)
+
+
+def teach(questions_and_answers: Mapping[str, str] | Sequence[tuple[str, str]], *,
+          epochs: int = 10, verbose: bool = True, **options: Any) -> TextGenerator:
+    """Train a small Q&A bot from question-and-answer pairs."""
+    pairs = list(questions_and_answers.items()) if isinstance(questions_and_answers, Mapping) else list(questions_and_answers)
+    if len(pairs) < 2:
+        raise ValueError("Give teach() at least two question-and-answer pairs.")
+    sections: list[str] = []
+    for question, answer in pairs:
+        if not isinstance(question, str) or not isinstance(answer, str) or not question.strip() or not answer.strip():
+            raise ValueError("Every question and answer must be non-empty text.")
+        sections.append(f"Question: {question.strip()}\nAnswer: {answer.strip()}")
+    return learn(sections, epochs=epochs, verbose=verbose, **options)
+
+
+def ask(generator: TextGenerator, question: str, *, max_tokens: int = 200,
+        **options: Any) -> str:
+    """Ask a Q&A bot a question and get its generated answer."""
+    if not isinstance(question, str) or not question.strip():
+        raise ValueError("question must be non-empty text.")
+    options.setdefault("stop", ("\nQuestion:", "\nAnswer:"))
+    options["include_prompt"] = False
+    return generator.generate(f"Question: {question.strip()}\nAnswer:", max_tokens=max_tokens, **options).strip()

shadow_tools-0.8.1/tests/test_generator.py

@@ -0,0 +1,43 @@
+import math
+import torch
+from shadowtools import teach, ask, TextGenerator, GeneratorConfig, BPETokenizer
+
+
+def test_train_save_reload(tmp_path):
+    torch.set_num_threads(1)
+    torch.manual_seed(3)
+    bot = teach({
+        'What is rain?': 'Rain is water falling from clouds onto the ground.',
+        'What is snow?': 'Snow consists of ice crystals falling from cold clouds.',
+    }, epochs=1, vocab_size=260, block_size=16, embedding_size=16,
+       heads=2, layers=1, batch_size=32, device='cpu', verbose=False)
+    assert all(torch.isfinite(p).all() for p in bot.model.parameters())
+    # EOS must actually receive training gradients as a target, not just be reserved.
+    assert bot.model.output.bias.grad[bot.tokenizer.eos_id] != 0
+    text = ask(bot, 'What is rain?', max_tokens=4, top_k=1)
+    assert not text.startswith('Question:')
+    bot.save(tmp_path)
+    restored = TextGenerator.load(tmp_path)
+    assert ask(restored, 'What is rain?', max_tokens=4, top_k=1) == text
+
+
+def test_answer_stops_at_next_question():
+    torch.set_num_threads(1)
+    bot = TextGenerator(GeneratorConfig(device='cpu'))
+    bot.tokenizer = BPETokenizer(vocab_size=260).train(['Paris.\nQuestion: Next?'])
+    script = bot.tokenizer.encode('Paris.\nQuestion: Next?', add_special_tokens=False)['input_ids']
+
+    class Scripted(torch.nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.index = 0
+
+        def forward(self, inputs):
+            logits = torch.full((1, inputs.shape[1], bot.tokenizer.size), -1000.)
+            logits[0, -1, script[self.index] if self.index < len(script) else bot.tokenizer.eos_id] = 1000.
+            self.index += 1
+            return logits
+
+    bot.model = Scripted()
+    assert ask(bot, 'Capital?', top_k=1) == 'Paris.'
+    assert bot.model.index < len(script)