shadow-tools 0.7.0__tar.gz → 0.7.2__tar.gz

{shadow_tools-0.7.0 → shadow_tools-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: shadow-tools
-Version: 0.7.0
+Version: 0.7.2
 Summary: A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs.
 Project-URL: Homepage, https://github.com/WilliamLaurento/shadow-tools
 Project-URL: Issues, https://github.com/WilliamLaurento/shadow-tools/issues
@@ -66,14 +66,16 @@ word as a separate thing. Install it with `pip install "shadow-tools[nlp]"`.
 ## Train and generate text from one package
 
 ```python
-from shadowtools import generate_text, train_generator
+from shadowtools import learn, say
 
-corpus = open("corpus.txt", encoding="utf-8").read()
-model, history = train_generator(corpus, epochs=20)
-print(generate_text(model, "The future of technology", max_tokens=80))
-model.save("my_text_model")
+bot = learn("corpus.txt", epochs=20)
+print(say(bot, "The future of technology", max_tokens=80))
+bot.save("my_text_model")
 ```
 
+Training prints the tokenizer status plus each epoch and loss. Use
+`learn("corpus.txt", epochs=20, verbose=False)` when you do not want updates.
+
 Install it with `pip install "shadow-tools[ml,nlp]"`. The package trains the
 tokenizer and Transformer together, then converts predicted token IDs back to
 normal text for you.

{shadow_tools-0.7.0 → shadow_tools-0.7.2}/README.md

@@ -38,14 +38,16 @@ word as a separate thing. Install it with `pip install "shadow-tools[nlp]"`.
 ## Train and generate text from one package
 
 ```python
-from shadowtools import generate_text, train_generator
+from shadowtools import learn, say
 
-corpus = open("corpus.txt", encoding="utf-8").read()
-model, history = train_generator(corpus, epochs=20)
-print(generate_text(model, "The future of technology", max_tokens=80))
-model.save("my_text_model")
+bot = learn("corpus.txt", epochs=20)
+print(say(bot, "The future of technology", max_tokens=80))
+bot.save("my_text_model")
 ```
 
+Training prints the tokenizer status plus each epoch and loss. Use
+`learn("corpus.txt", epochs=20, verbose=False)` when you do not want updates.
+
 Install it with `pip install "shadow-tools[ml,nlp]"`. The package trains the
 tokenizer and Transformer together, then converts predicted token IDs back to
 normal text for you.

{shadow_tools-0.7.0 → shadow_tools-0.7.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "shadow-tools"
-version = "0.7.0"
+version = "0.7.2"
 description = "A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs."
 readme = "README.md"
 requires-python = ">=3.10"

{shadow_tools-0.7.0 → shadow_tools-0.7.2}/src/shadowtools/__init__.py

@@ -5,7 +5,7 @@ from .bpe import BPETokenizer
 from .config import env, require_env, to_bool
 from .files import append_text, ensure_dir, read_text, write_text
 from .hashing import file_hash, hash_text
-from .generator import GeneratorConfig, TextGenerator, generate_text, train_generator
+from .generator import GeneratorConfig, TextGenerator, generate_text, learn, say, train_generator
 from .http import get, request
 from .ids import random_token, timestamp_id, uuid4
 from .network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, resolve_dns, reverse_dns
@@ -15,10 +15,10 @@ from .text import chunks, slugify, to_camel, to_snake
 from .training import SmartTokenizer, fine_tune, make_loaders, predict_text, train, train_text
 from .validation import is_email, is_url, require_keys
 
-__version__ = "0.7.0"
+__version__ = "0.7.2"
 
 __all__ = [
-    "append_text", "audit_headers", "BPETokenizer", "chunks", "ensure_dir", "env", "file_hash", "GeneratorConfig", "generate_random_ips", "generate_test_ips", "generate_text", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "make_password", "make_subdomain", "random_token", "read_csv",
-    "read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "slugify", "timestamp_id", "to_bool", "to_camel", "to_snake", "TextGenerator", "train", "train_generator", "fine_tune", "make_loaders", "SmartTokenizer", "train_text", "predict_text", "uuid4", "write_csv",
+    "append_text", "audit_headers", "BPETokenizer", "chunks", "ensure_dir", "env", "file_hash", "GeneratorConfig", "generate_random_ips", "generate_test_ips", "generate_text", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "learn", "make_password", "make_subdomain", "random_token", "read_csv",
+    "read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "say", "slugify", "timestamp_id", "to_bool", "to_camel", "to_snake", "TextGenerator", "train", "train_generator", "fine_tune", "make_loaders", "SmartTokenizer", "train_text", "predict_text", "uuid4", "write_csv",
     "write_json", "write_text", "redact_secrets", "verify_file_hash",
 ]

{shadow_tools-0.7.0 → shadow_tools-0.7.2}/src/shadowtools/generator.py

@@ -19,6 +19,7 @@ class GeneratorConfig:
     batch_size: int = 16
     learning_rate: float = 3e-4
     device: str = "auto"
+    verbose: bool = True
 
 
 class TextGenerator:
@@ -78,18 +79,24 @@ class TextGenerator:
 
         return CausalTransformer().to(self.device)
 
-    def fit(self, texts: str | Sequence[str], *, epochs: int = 10) -> list[dict[str, float]]:
+    def fit(self, texts: str | Sequence[str], *, epochs: int = 10,
+            verbose: bool | None = None) -> list[dict[str, float]]:
         """Train from a large text string or a list of documents."""
         if epochs < 1:
             raise ValueError("epochs must be at least 1.")
         corpus = texts if isinstance(texts, str) else "\n".join(texts)
         if not isinstance(corpus, str) or len(corpus.strip()) < 100:
             raise ValueError("Give the generator at least 100 characters of training text.")
+        show_progress = self.config.verbose if verbose is None else verbose
+        if show_progress:
+            print("Building tokenizer...")
         self.tokenizer = BPETokenizer(vocab_size=self.config.vocab_size, min_frequency=1).train([corpus])
         token_ids = self.tokenizer.encode(corpus, add_special_tokens=False)["input_ids"]
         if len(token_ids) <= self.config.block_size:
             raise ValueError("Training text needs more tokens than block_size.")
         self.model = self._build_model(self.tokenizer.size)
+        if show_progress:
+            print(f"Tokenizer ready: {self.tokenizer.size} tokens. Training model for {epochs} epoch(s)...")
         torch = self.torch
         block_size = self.config.block_size
 
@@ -119,7 +126,10 @@ class TextGenerator:
                 torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
                 optimizer.step()
                 total_loss += float(loss.detach().cpu())
-            history.append({"epoch": float(epoch), "loss": total_loss / len(loader)})
+            metrics = {"epoch": float(epoch), "loss": total_loss / len(loader)}
+            history.append(metrics)
+            if show_progress:
+                print(f"Epoch {epoch}/{epochs} | loss: {metrics['loss']:.4f}")
         return history
 
     def generate(self, prompt: str, *, max_tokens: int = 80, temperature: float = 0.8,
@@ -181,14 +191,38 @@ def train_generator(texts: str | Sequence[str], *, epochs: int = 10,
                     vocab_size: int = 4_000, block_size: int = 64,
                     embedding_size: int = 192, heads: int = 6, layers: int = 4,
                     batch_size: int = 16, learning_rate: float = 3e-4,
-                    device: str = "auto") -> tuple[TextGenerator, list[dict[str, float]]]:
+                    device: str = "auto", verbose: bool = True) -> tuple[TextGenerator, list[dict[str, float]]]:
     """Train a text generator in one call, without writing model boilerplate."""
     generator = TextGenerator(GeneratorConfig(vocab_size=vocab_size, block_size=block_size,
                               embedding_size=embedding_size, heads=heads, layers=layers,
-                              batch_size=batch_size, learning_rate=learning_rate, device=device))
+                              batch_size=batch_size, learning_rate=learning_rate, device=device,
+                              verbose=verbose))
     return generator, generator.fit(texts, epochs=epochs)
 
 
 def generate_text(generator: TextGenerator, prompt: str, **options: Any) -> str:
     """Short form for ``generator.generate(prompt)``."""
     return generator.generate(prompt, **options)
+
+
+def learn(source: str | Path | Sequence[str], *, epochs: int = 10,
+          verbose: bool = True, **options: Any) -> TextGenerator:
+    """Beginner-friendly training: give it a text file, text, or list of texts."""
+    text: str | Sequence[str]
+    if isinstance(source, Path):
+        text = source.read_text(encoding="utf-8")
+    elif isinstance(source, str):
+        try:
+            candidate = Path(source)
+            text = candidate.read_text(encoding="utf-8") if candidate.is_file() else source
+        except OSError:
+            text = source
+    else:
+        text = source
+    generator, _ = train_generator(text, epochs=epochs, verbose=verbose, **options)
+    return generator
+
+
+def say(generator: TextGenerator, prompt: str, **options: Any) -> str:
+    """Beginner-friendly name for generating text from a trained model."""
+    return generator.generate(prompt, **options)