shadow-tools 0.3.0__tar.gz → 0.4.0__tar.gz

{shadow_tools-0.3.0 → shadow_tools-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: shadow-tools
-Version: 0.3.0
+Version: 0.4.0
 Summary: A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs.
 Project-URL: Homepage, https://github.com/WilliamLaurento/shadow-tools
 Project-URL: Issues, https://github.com/WilliamLaurento/shadow-tools/issues
@@ -18,6 +18,8 @@ Provides-Extra: dev
 Requires-Dist: build>=1.2; extra == 'dev'
 Requires-Dist: pytest>=8.0; extra == 'dev'
 Requires-Dist: twine>=6.0; extra == 'dev'
+Provides-Extra: ml
+Requires-Dist: torch>=2.2; extra == 'ml'
 Provides-Extra: websocket
 Requires-Dist: websocket-client>=1.8; extra == 'websocket'
 Description-Content-Type: text/markdown
@@ -46,6 +48,8 @@ pip install "shadow-tools[websocket]"  # optional WebSocket client
 - File and text hashing
 - UUIDs, secure tokens, and timestamp IDs
 - Slugs, snake/camel case, chunks, environment variables, and retry logic
+- Defensive security checks, secret redaction, password generation, and file verification
+- Optional PyTorch training for new MLPs or supplied local models
 - `shadow` command-line tool
 
 ## Quick start
@@ -114,6 +118,33 @@ shadow hash settings.json
 shadow uuid
 ```
 
+## Security and training
+
+```python
+from shadowtools.security import audit_headers, redact_secrets
+
+print(redact_secrets("Authorization: Bearer secret-value"))
+print(audit_headers({"X-Content-Type-Options": "nosniff"}))
+```
+
+Install ML support, then train a small classifier from scratch:
+
+```bash
+pip install "shadow-tools[ml]"
+```
+
+```python
+from shadowtools.training import Trainer, TrainingConfig
+
+config = TrainingConfig(
+    mode="from_scratch",
+    architecture={"type": "mlp", "input_size": 8, "hidden_sizes": [32, 16], "num_classes": 2},
+    epochs=5,
+)
+trainer = Trainer(config)
+history = trainer.fit(train_loader, validation_loader)
+```
+
 ```bash
 shadow get https://api.github.com --json
 shadow json-format settings.json

{shadow_tools-0.3.0 → shadow_tools-0.4.0}/README.md

@@ -22,6 +22,8 @@ pip install "shadow-tools[websocket]"  # optional WebSocket client
 - File and text hashing
 - UUIDs, secure tokens, and timestamp IDs
 - Slugs, snake/camel case, chunks, environment variables, and retry logic
+- Defensive security checks, secret redaction, password generation, and file verification
+- Optional PyTorch training for new MLPs or supplied local models
 - `shadow` command-line tool
 
 ## Quick start
@@ -90,6 +92,33 @@ shadow hash settings.json
 shadow uuid
 ```
 
+## Security and training
+
+```python
+from shadowtools.security import audit_headers, redact_secrets
+
+print(redact_secrets("Authorization: Bearer secret-value"))
+print(audit_headers({"X-Content-Type-Options": "nosniff"}))
+```
+
+Install ML support, then train a small classifier from scratch:
+
+```bash
+pip install "shadow-tools[ml]"
+```
+
+```python
+from shadowtools.training import Trainer, TrainingConfig
+
+config = TrainingConfig(
+    mode="from_scratch",
+    architecture={"type": "mlp", "input_size": 8, "hidden_sizes": [32, 16], "num_classes": 2},
+    epochs=5,
+)
+trainer = Trainer(config)
+history = trainer.fit(train_loader, validation_loader)
+```
+
 ```bash
 shadow get https://api.github.com --json
 shadow json-format settings.json

{shadow_tools-0.3.0 → shadow_tools-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "shadow-tools"
-version = "0.3.0"
+version = "0.4.0"
 description = "A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -23,6 +23,7 @@ dependencies = []
 [project.optional-dependencies]
 websocket = ["websocket-client>=1.8"]
 dev = ["pytest>=8.0", "build>=1.2", "twine>=6.0"]
+ml = ["torch>=2.2"]
 
 [project.scripts]
 shadow = "shadowtools.cli:main"

{shadow_tools-0.3.0 → shadow_tools-0.4.0}/src/shadowtools/__init__.py

@@ -8,13 +8,14 @@ from .http import get, request
 from .ids import random_token, timestamp_id, uuid4
 from .network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, resolve_dns, reverse_dns
 from .retry import retry
+from .security import audit_headers, make_password, redact_secrets, verify_file_hash
 from .text import chunks, slugify, to_camel, to_snake
 from .validation import is_email, is_url, require_keys
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"
 
 __all__ = [
-    "append_text", "chunks", "ensure_dir", "env", "file_hash", "generate_random_ips", "generate_test_ips", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "make_subdomain", "random_token", "read_csv",
+    "append_text", "audit_headers", "chunks", "ensure_dir", "env", "file_hash", "generate_random_ips", "generate_test_ips", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "make_password", "make_subdomain", "random_token", "read_csv",
     "read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "slugify", "timestamp_id", "to_bool", "to_camel", "to_snake", "uuid4", "write_csv",
-    "write_json", "write_text",
+    "write_json", "write_text", "redact_secrets", "verify_file_hash",
 ]

{shadow_tools-0.3.0 → shadow_tools-0.4.0}/src/shadowtools/retry.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 
 import time
 from collections.abc import Callable
+from functools import wraps
 from typing import ParamSpec, TypeVar
 
 P = ParamSpec("P")
@@ -15,6 +16,7 @@ def retry(attempts: int = 3, *, delay: float = 0.25,
         raise ValueError("attempts must be at least 1")
 
     def decorate(function: Callable[P, T]) -> Callable[P, T]:
+        @wraps(function)
         def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
             for attempt in range(attempts):
                 try:

shadow_tools-0.4.0/src/shadowtools/security.py

@@ -0,0 +1,55 @@
+"""Defensive security helpers for data you own or are authorized to inspect."""
+from __future__ import annotations
+
+import hmac
+import re
+import secrets
+import string
+from pathlib import Path
+from typing import Mapping
+
+from .hashing import file_hash
+
+_SECRET = re.compile(r"(?i)(authorization\s*[:=]\s*(?:bearer\s+)?|api[_-]?key\s*[:=]\s*|token\s*[:=]\s*|password\s*[:=]\s*)([^\s,;]+)")
+
+
+def redact_secrets(text: str, replacement: str = "[REDACTED]") -> str:
+    """Remove common credential values before logs are saved or shared."""
+    return _SECRET.sub(lambda match: match.group(1) + replacement, text)
+
+
+def audit_headers(headers: Mapping[str, str]) -> dict[str, list[str]]:
+    """Report common missing browser-security response headers."""
+    normalized = {name.lower(): value for name, value in headers.items()}
+    required = {
+        "content-security-policy": "Content-Security-Policy",
+        "x-content-type-options": "X-Content-Type-Options",
+        "x-frame-options": "X-Frame-Options",
+        "referrer-policy": "Referrer-Policy",
+        "strict-transport-security": "Strict-Transport-Security",
+    }
+    missing = [display for key, display in required.items() if key not in normalized]
+    warnings: list[str] = []
+    if normalized.get("x-content-type-options", "").lower() != "nosniff":
+        warnings.append("Set X-Content-Type-Options to nosniff.")
+    if "content-security-policy" not in normalized:
+        warnings.append("Add a Content-Security-Policy appropriate for this app.")
+    return {"missing": missing, "warnings": warnings}
+
+
+def make_password(length: int = 20) -> str:
+    """Create a random password containing upper/lowercase, digits, and symbols."""
+    if length < 12:
+        raise ValueError("Use at least 12 characters for a password.")
+    alphabet = string.ascii_letters + string.digits + "!@#$%^&*_-+="
+    required = [secrets.choice(string.ascii_lowercase), secrets.choice(string.ascii_uppercase),
+                secrets.choice(string.digits), secrets.choice("!@#$%^&*_-+=")]
+    remaining = [secrets.choice(alphabet) for _ in range(length - len(required))]
+    values = required + remaining
+    secrets.SystemRandom().shuffle(values)
+    return "".join(values)
+
+
+def verify_file_hash(path: str | Path, expected: str, algorithm: str = "sha256") -> bool:
+    """Compare a file checksum without leaking timing information."""
+    return hmac.compare_digest(file_hash(path, algorithm), expected.lower())

shadow_tools-0.4.0/src/shadowtools/training.py

@@ -0,0 +1,170 @@
+"""A small, explicit PyTorch training engine for fine-tuning or new MLPs."""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Iterable, Literal, Mapping
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TrainingConfig:
+    mode: Literal["from_scratch", "fine_tune"]
+    epochs: int = 3
+    learning_rate: float = 1e-3
+    batch_size: int = 32
+    device: str = "auto"
+    optimizer: Literal["adamw", "sgd"] = "adamw"
+    checkpoint_path: str | Path | None = None
+    architecture: Mapping[str, Any] | None = None
+    model: Any | None = None
+    model_path: str | Path | None = None
+    model_id: str | None = None
+    model_factory: Callable[[], Any] | None = None
+    model_loader: Callable[[str], Any] | None = None
+    freeze_layers: tuple[str, ...] = ()
+    head_attr: str | None = None
+    num_classes: int | None = None
+
+
+class Trainer:
+    """Train a PyTorch classifier from scratch or fine-tune an existing module."""
+
+    def __init__(self, config: TrainingConfig | Mapping[str, Any]) -> None:
+        self.config = config if isinstance(config, TrainingConfig) else TrainingConfig(**dict(config))
+        self.torch = self._torch()
+        self.device = self._device()
+        self.model = self._build_model().to(self.device)
+        self._freeze()
+        self._replace_head()
+
+    @staticmethod
+    def _torch() -> Any:
+        try:
+            import torch
+            return torch
+        except ImportError as exc:
+            raise ImportError("Training needs PyTorch. Install it with `pip install shadow-tools[ml]`.") from exc
+
+    def _device(self) -> Any:
+        value = self.config.device.lower()
+        if value == "auto":
+            value = "cuda" if self.torch.cuda.is_available() else "mps" if getattr(self.torch.backends, "mps", None) and self.torch.backends.mps.is_available() else "cpu"
+        if value.startswith("cuda") and not self.torch.cuda.is_available():
+            raise RuntimeError("CUDA was requested but is unavailable.")
+        if value == "mps" and not (getattr(self.torch.backends, "mps", None) and self.torch.backends.mps.is_available()):
+            raise RuntimeError("MPS was requested but is unavailable.")
+        return self.torch.device(value)
+
+    def _build_model(self) -> Any:
+        if self.config.mode == "from_scratch":
+            spec = dict(self.config.architecture or {})
+            if spec.get("type", "mlp") != "mlp":
+                raise ValueError("from_scratch currently supports architecture type 'mlp'.")
+            try:
+                input_size = int(spec["input_size"])
+                num_classes = int(spec["num_classes"])
+            except KeyError as exc:
+                raise ValueError("MLP architecture requires input_size and num_classes.") from exc
+            hidden = [int(size) for size in spec.get("hidden_sizes", [128, 64])]
+            dropout = float(spec.get("dropout", 0.0))
+            layers: list[Any] = [self.torch.nn.Flatten()]
+            previous = input_size
+            for size in hidden:
+                layers.extend([self.torch.nn.Linear(previous, size), self.torch.nn.ReLU()])
+                if dropout:
+                    layers.append(self.torch.nn.Dropout(dropout))
+                previous = size
+            layers.append(self.torch.nn.Linear(previous, num_classes))
+            return self.torch.nn.Sequential(*layers)
+        if self.config.mode != "fine_tune":
+            raise ValueError("mode must be 'from_scratch' or 'fine_tune'.")
+        model = self.config.model
+        if model is None and self.config.model_id:
+            if self.config.model_loader is None:
+                raise ValueError("model_id requires an explicit model_loader callable.")
+            model = self.config.model_loader(self.config.model_id)
+        if model is None and self.config.model_factory:
+            model = self.config.model_factory()
+        if model is None:
+            raise ValueError("fine_tune requires model, model_factory, or model_id with model_loader.")
+        if self.config.model_path:
+            path = Path(self.config.model_path)
+            if not path.is_file():
+                raise FileNotFoundError(path)
+            state = self.torch.load(path, map_location="cpu", weights_only=True)
+            if isinstance(state, Mapping) and "state_dict" in state:
+                state = state["state_dict"]
+            model.load_state_dict(state)
+        return model
+
+    def _freeze(self) -> None:
+        for name, parameter in self.model.named_parameters():
+            if any(name.startswith(prefix) for prefix in self.config.freeze_layers):
+                parameter.requires_grad = False
+
+    def _replace_head(self) -> None:
+        if not self.config.head_attr or self.config.num_classes is None:
+            return
+        head = getattr(self.model, self.config.head_attr, None)
+        if not isinstance(head, self.torch.nn.Linear):
+            raise ValueError("head_attr must name a torch.nn.Linear layer.")
+        setattr(self.model, self.config.head_attr, self.torch.nn.Linear(head.in_features, self.config.num_classes))
+
+    def fit(self, train_data: Iterable[Any], validation_data: Iterable[Any] | None = None) -> list[dict[str, float]]:
+        """Train on iterable batches of ``(features, labels)`` and return epoch metrics."""
+        parameters = [parameter for parameter in self.model.parameters() if parameter.requires_grad]
+        if not parameters:
+            raise RuntimeError("No trainable parameters remain after freezing layers.")
+        optimizer = self.torch.optim.AdamW(parameters, lr=self.config.learning_rate) if self.config.optimizer == "adamw" else self.torch.optim.SGD(parameters, lr=self.config.learning_rate)
+        loss_fn = self.torch.nn.CrossEntropyLoss()
+        history: list[dict[str, float]] = []
+        for epoch in range(1, self.config.epochs + 1):
+            self.model.train()
+            total_loss = 0.0
+            batches = 0
+            for features, labels in train_data:
+                features = features.to(self.device)
+                labels = labels.to(self.device).long()
+                optimizer.zero_grad(set_to_none=True)
+                loss = loss_fn(self.model(features), labels)
+                loss.backward()
+                optimizer.step()
+                total_loss += float(loss.detach().cpu())
+                batches += 1
+            if not batches:
+                raise ValueError("train_data produced no batches.")
+            metrics = {"epoch": float(epoch), "loss": total_loss / batches}
+            if validation_data is not None:
+                metrics["accuracy"] = self.evaluate(validation_data)
+            history.append(metrics)
+            logger.info("epoch=%s loss=%.5f", epoch, metrics["loss"])
+            self._checkpoint(epoch, metrics)
+        return history
+
+    def evaluate(self, data: Iterable[Any]) -> float:
+        self.model.eval()
+        correct = total = 0
+        with self.torch.no_grad():
+            for features, labels in data:
+                prediction = self.model(features.to(self.device)).argmax(dim=1)
+                labels = labels.to(self.device).long()
+                correct += int((prediction == labels).sum().item())
+                total += int(labels.numel())
+        if not total:
+            raise ValueError("validation data produced no labels.")
+        return correct / total
+
+    def _checkpoint(self, epoch: int, metrics: Mapping[str, float]) -> None:
+        if self.config.checkpoint_path is None:
+            return
+        path = Path(self.config.checkpoint_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self.torch.save({"epoch": epoch, "metrics": dict(metrics), "state_dict": self.model.state_dict()}, path)
+
+
+def run_training(config: TrainingConfig | Mapping[str, Any], train_data: Iterable[Any], validation_data: Iterable[Any] | None = None) -> tuple[Trainer, list[dict[str, float]]]:
+    trainer = Trainer(config)
+    return trainer, trainer.fit(train_data, validation_data)

{shadow_tools-0.3.0 → shadow_tools-0.4.0}/tests/test_core.py

@@ -12,6 +12,7 @@ from shadowtools.config import to_bool
 from shadowtools.hashing import hash_text
 from shadowtools.ids import random_token
 from shadowtools.text import chunks, slugify, to_camel, to_snake
+from shadowtools.security import audit_headers, make_password, redact_secrets
 from shadowtools.validation import is_email, is_url, require_keys
 
 
@@ -61,5 +62,8 @@ def test_general_utilities():
     assert to_camel("shadow tools") == "ShadowTools"
     assert chunks(range(5), 2) == [[0, 1], [2, 3], [4]]
     assert to_bool("yes")
+    assert "[REDACTED]" in redact_secrets("Authorization: Bearer private")
+    assert len(make_password(16)) == 16
+    assert "Content-Security-Policy" in audit_headers({})["missing"]
     assert prompt("Hi {name}", name="Will") == "Hi Will"
     assert tool_schema("lookup_dns", "Lookup DNS", {"host": {"type": "string"}})["name"] == "lookup_dns"