shadow-tools 0.4.2__tar.gz → 0.5.0__tar.gz

{shadow_tools-0.4.2 → shadow_tools-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: shadow-tools
-Version: 0.4.2
+Version: 0.5.0
 Summary: A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs.
 Project-URL: Homepage, https://github.com/WilliamLaurento/shadow-tools
 Project-URL: Issues, https://github.com/WilliamLaurento/shadow-tools/issues
@@ -26,6 +26,23 @@ Description-Content-Type: text/markdown
 
 # Shadow Tools
 
+## Train on regular text
+
+```python
+from shadowtools import predict_text, train_text
+
+texts = ["I love this game", "this is terrible", "really fun game", "I hate it"]
+labels = [1, 0, 1, 0]  # 1 = positive, 0 = negative
+
+model, history = train_text(texts, labels, epochs=20)
+print(predict_text(model, ["this game is fun", "I really hate this"]))
+```
+
+`train_text` learns a vocabulary from your examples. Its tokenizer keeps words,
+contractions, URLs, punctuation, and common two-word phrases, then converts each
+text into a TF-IDF number vector before training. Use `SmartTokenizer` yourself
+when you want to fit once and reuse the exact same vocabulary later.
+
 `shadow-tools` is a lightweight Python toolbox for everyday apps and scripts. It keeps the basics in one clean package without trying to replace specialized libraries.
 
 ## Install

{shadow_tools-0.4.2 → shadow_tools-0.5.0}/README.md

@@ -1,5 +1,22 @@
 # Shadow Tools
 
+## Train on regular text
+
+```python
+from shadowtools import predict_text, train_text
+
+texts = ["I love this game", "this is terrible", "really fun game", "I hate it"]
+labels = [1, 0, 1, 0]  # 1 = positive, 0 = negative
+
+model, history = train_text(texts, labels, epochs=20)
+print(predict_text(model, ["this game is fun", "I really hate this"]))
+```
+
+`train_text` learns a vocabulary from your examples. Its tokenizer keeps words,
+contractions, URLs, punctuation, and common two-word phrases, then converts each
+text into a TF-IDF number vector before training. Use `SmartTokenizer` yourself
+when you want to fit once and reuse the exact same vocabulary later.
+
 `shadow-tools` is a lightweight Python toolbox for everyday apps and scripts. It keeps the basics in one clean package without trying to replace specialized libraries.
 
 ## Install

{shadow_tools-0.4.2 → shadow_tools-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "shadow-tools"
-version = "0.4.2"
+version = "0.5.0"
 description = "A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs."
 readme = "README.md"
 requires-python = ">=3.10"

{shadow_tools-0.4.2 → shadow_tools-0.5.0}/src/shadowtools/__init__.py

@@ -10,13 +10,13 @@ from .network import generate_random_ips, generate_test_ips, is_subdomain_of, ma
 from .retry import retry
 from .security import audit_headers, make_password, redact_secrets, verify_file_hash
 from .text import chunks, slugify, to_camel, to_snake
-from .training import fine_tune, make_loaders, train
+from .training import SmartTokenizer, fine_tune, make_loaders, predict_text, train, train_text
 from .validation import is_email, is_url, require_keys
 
-__version__ = "0.4.2"
+__version__ = "0.5.0"
 
 __all__ = [
     "append_text", "audit_headers", "chunks", "ensure_dir", "env", "file_hash", "generate_random_ips", "generate_test_ips", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "make_password", "make_subdomain", "random_token", "read_csv",
-    "read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "slugify", "timestamp_id", "to_bool", "to_camel", "to_snake", "train", "fine_tune", "make_loaders", "uuid4", "write_csv",
+    "read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "slugify", "timestamp_id", "to_bool", "to_camel", "to_snake", "train", "fine_tune", "make_loaders", "SmartTokenizer", "train_text", "predict_text", "uuid4", "write_csv",
     "write_json", "write_text", "redact_secrets", "verify_file_hash",
 ]

{shadow_tools-0.4.2 → shadow_tools-0.5.0}/src/shadowtools/training.py

@@ -2,13 +2,108 @@
 from __future__ import annotations
 
 import logging
+import math
+import re
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Callable, Iterable, Literal, Mapping
+from typing import Any, Callable, Iterable, Literal, Mapping, Sequence
 
 logger = logging.getLogger(__name__)
 
 
+class SmartTokenizer:
+    """A compact tokenizer that learns words and useful word-pairs from your text."""
+
+    PAD = "[PAD]"
+    UNK = "[UNK]"
+    _pattern = re.compile(r"https?://\S+|[\w]+(?:['’-][\w]+)*|[^\w\s]", re.UNICODE)
+
+    def __init__(self, *, min_frequency: int = 1, max_vocab: int = 10_000,
+                 ngram_range: tuple[int, int] = (1, 2), lowercase: bool = True) -> None:
+        if min_frequency < 1 or max_vocab < 3:
+            raise ValueError("min_frequency must be at least 1 and max_vocab at least 3.")
+        if not 1 <= ngram_range[0] <= ngram_range[1] <= 3:
+            raise ValueError("ngram_range must be ordered between 1 and 3.")
+        self.min_frequency = min_frequency
+        self.max_vocab = max_vocab
+        self.ngram_range = ngram_range
+        self.lowercase = lowercase
+        self.vocab: dict[str, int] = {self.PAD: 0, self.UNK: 1}
+        self.idf: list[float] = [0.0, 0.0]
+        self._fitted = False
+
+    def tokenize(self, text: str) -> list[str]:
+        if not isinstance(text, str):
+            raise TypeError("text must be a string.")
+        cleaned = text.strip().lower() if self.lowercase else text.strip()
+        words = self._pattern.findall(cleaned)
+        terms: list[str] = []
+        low, high = self.ngram_range
+        for size in range(low, high + 1):
+            terms.extend(" ".join(words[index:index + size]) for index in range(len(words) - size + 1))
+        return terms
+
+    def fit(self, texts: Sequence[str]) -> "SmartTokenizer":
+        if not texts:
+            raise ValueError("texts must contain at least one item.")
+        counts: dict[str, int] = {}
+        documents: dict[str, int] = {}
+        for text in texts:
+            terms = self.tokenize(text)
+            for term in terms:
+                counts[term] = counts.get(term, 0) + 1
+            for term in set(terms):
+                documents[term] = documents.get(term, 0) + 1
+        kept = [term for term, count in counts.items() if count >= self.min_frequency]
+        kept.sort(key=lambda term: (-counts[term], term))
+        kept = kept[: self.max_vocab - 2]
+        self.vocab = {self.PAD: 0, self.UNK: 1, **{term: index + 2 for index, term in enumerate(kept)}}
+        text_count = len(texts)
+        self.idf = [0.0, 0.0] + [math.log((1 + text_count) / (1 + documents[term])) + 1 for term in kept]
+        self._fitted = True
+        return self
+
+    def encode(self, text: str) -> list[int]:
+        self._check_fitted()
+        return [self.vocab.get(term, 1) for term in self.tokenize(text)]
+
+    def vectorize(self, texts: Sequence[str]) -> list[list[float]]:
+        """Return fixed-length TF-IDF vectors suitable for ``train``."""
+        self._check_fitted()
+        vectors: list[list[float]] = []
+        for text in texts:
+            vector = [0.0] * len(self.vocab)
+            counts: dict[int, int] = {}
+            for token_id in self.encode(text):
+                if token_id > 1:
+                    counts[token_id] = counts.get(token_id, 0) + 1
+            total = sum(counts.values())
+            if total:
+                for token_id, count in counts.items():
+                    vector[token_id] = (count / total) * self.idf[token_id]
+            vectors.append(vector)
+        return vectors
+
+    def to_dict(self) -> dict[str, Any]:
+        self._check_fitted()
+        return {"min_frequency": self.min_frequency, "max_vocab": self.max_vocab,
+                "ngram_range": self.ngram_range, "lowercase": self.lowercase,
+                "vocab": self.vocab, "idf": self.idf}
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> "SmartTokenizer":
+        tokenizer = cls(min_frequency=int(data["min_frequency"]), max_vocab=int(data["max_vocab"]),
+                        ngram_range=tuple(data["ngram_range"]), lowercase=bool(data["lowercase"]))
+        tokenizer.vocab = {str(term): int(token_id) for term, token_id in data["vocab"].items()}
+        tokenizer.idf = [float(value) for value in data["idf"]]
+        tokenizer._fitted = True
+        return tokenizer
+
+    def _check_fitted(self) -> None:
+        if not self._fitted:
+            raise RuntimeError("Fit the tokenizer first: SmartTokenizer().fit(texts).")
+
+
 @dataclass
 class TrainingConfig:
     mode: Literal["from_scratch", "fine_tune"]
@@ -36,9 +131,10 @@ class Trainer:
         self.config = config if isinstance(config, TrainingConfig) else TrainingConfig(**dict(config))
         self.torch = self._torch()
         self.device = self._device()
-        self.model = self._build_model().to(self.device)
+        self.model = self._build_model()
         self._freeze()
         self._replace_head()
+        self.model = self.model.to(self.device)
 
     @staticmethod
     def _torch() -> Any:
@@ -228,3 +324,34 @@ def fine_tune(model: Any, features: Any, labels: Any, *, epochs: int = 3,
         freeze_layers=freeze_layers, head_attr=head_attr, num_classes=num_classes,
     )
     return run_training(config, train_loader, validation_loader)
+
+
+def train_text(texts: Sequence[str], labels: Sequence[int], *, tokenizer: SmartTokenizer | None = None,
+               min_frequency: int = 1, max_vocab: int = 10_000,
+               ngram_range: tuple[int, int] = (1, 2), hidden_sizes: list[int] | None = None,
+               epochs: int = 8, learning_rate: float = 1e-3, batch_size: int = 32,
+               validation_split: float = 0.2, device: str = "auto") -> tuple[Trainer, list[dict[str, float]]]:
+    """Train directly on text and integer labels with a learned word/phrase tokenizer."""
+    if len(texts) != len(labels) or len(texts) < 2:
+        raise ValueError("texts and labels must have the same length of at least 2.")
+    learned = tokenizer or SmartTokenizer(min_frequency=min_frequency, max_vocab=max_vocab, ngram_range=ngram_range)
+    if not learned._fitted:
+        learned.fit(texts)
+    trainer, history = train(learned.vectorize(texts), labels, hidden_sizes=hidden_sizes,
+                             epochs=epochs, learning_rate=learning_rate, batch_size=batch_size,
+                             validation_split=validation_split, device=device)
+    trainer.tokenizer = learned
+    return trainer, history
+
+
+def predict_text(trainer: Trainer, texts: str | Sequence[str]) -> list[int]:
+    """Classify one string or a list of strings returned by ``train_text``."""
+    if not hasattr(trainer, "tokenizer"):
+        raise ValueError("trainer has no tokenizer; create it with train_text().")
+    values = [texts] if isinstance(texts, str) else list(texts)
+    vectors = trainer.tokenizer.vectorize(values)
+    torch = Trainer._torch()
+    trainer.model.eval()
+    with torch.no_grad():
+        tensor = torch.as_tensor(vectors, dtype=torch.float32).to(trainer.device)
+        return trainer.model(tensor).argmax(dim=1).cpu().tolist()