shadow-tools 0.2.5__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/PKG-INFO +52 -1
  2. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/README.md +49 -0
  3. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/pyproject.toml +2 -1
  4. shadow_tools-0.4.0/release-0.2.5/shadow_tools-0.2.5-py3-none-any.whl +0 -0
  5. shadow_tools-0.4.0/release-0.2.5/shadow_tools-0.2.5.tar.gz +0 -0
  6. shadow_tools-0.4.0/release-0.3.0/shadow_tools-0.3.0-py3-none-any.whl +0 -0
  7. shadow_tools-0.4.0/release-0.3.0/shadow_tools-0.3.0.tar.gz +0 -0
  8. shadow_tools-0.4.0/release-0.4.0/shadow_tools-0.4.0-py3-none-any.whl +0 -0
  9. shadow_tools-0.4.0/release-0.4.0/shadow_tools-0.4.0.tar.gz +0 -0
  10. shadow_tools-0.4.0/src/shadowtools/__init__.py +21 -0
  11. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/cli.py +9 -0
  12. shadow_tools-0.4.0/src/shadowtools/config.py +23 -0
  13. shadow_tools-0.4.0/src/shadowtools/hashing.py +17 -0
  14. shadow_tools-0.4.0/src/shadowtools/ids.py +20 -0
  15. shadow_tools-0.4.0/src/shadowtools/retry.py +30 -0
  16. shadow_tools-0.4.0/src/shadowtools/security.py +55 -0
  17. shadow_tools-0.4.0/src/shadowtools/text.py +33 -0
  18. shadow_tools-0.4.0/src/shadowtools/training.py +170 -0
  19. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/tests/test_core.py +18 -0
  20. shadow_tools-0.2.5/src/shadowtools/__init__.py +0 -15
  21. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/.gitignore +0 -0
  22. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/LICENSE +0 -0
  23. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.0/shadow_tools-0.2.0-py3-none-any.whl +0 -0
  24. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.0/shadow_tools-0.2.0.tar.gz +0 -0
  25. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.1/shadow_tools-0.2.1-py3-none-any.whl +0 -0
  26. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.1/shadow_tools-0.2.1.tar.gz +0 -0
  27. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.2/shadow_tools-0.2.2-py3-none-any.whl +0 -0
  28. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.2/shadow_tools-0.2.2.tar.gz +0 -0
  29. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.3/shadow_tools-0.2.3-py3-none-any.whl +0 -0
  30. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.3/shadow_tools-0.2.3.tar.gz +0 -0
  31. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.4/shadow_tools-0.2.4-py3-none-any.whl +0 -0
  32. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.4/shadow_tools-0.2.4.tar.gz +0 -0
  33. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/ai.py +0 -0
  34. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/async_tools.py +0 -0
  35. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/data.py +0 -0
  36. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/files.py +0 -0
  37. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/http.py +0 -0
  38. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/network.py +0 -0
  39. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/validation.py +0 -0
  40. {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/websocket.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: shadow-tools
3
- Version: 0.2.5
3
+ Version: 0.4.0
4
4
  Summary: A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs.
5
5
  Project-URL: Homepage, https://github.com/WilliamLaurento/shadow-tools
6
6
  Project-URL: Issues, https://github.com/WilliamLaurento/shadow-tools/issues
@@ -18,6 +18,8 @@ Provides-Extra: dev
18
18
  Requires-Dist: build>=1.2; extra == 'dev'
19
19
  Requires-Dist: pytest>=8.0; extra == 'dev'
20
20
  Requires-Dist: twine>=6.0; extra == 'dev'
21
+ Provides-Extra: ml
22
+ Requires-Dist: torch>=2.2; extra == 'ml'
21
23
  Provides-Extra: websocket
22
24
  Requires-Dist: websocket-client>=1.8; extra == 'websocket'
23
25
  Description-Content-Type: text/markdown
@@ -43,6 +45,11 @@ pip install "shadow-tools[websocket]" # optional WebSocket client
43
45
  - Email, URL, and required-key checks
44
46
  - DNS lookup, subdomain parsing, and documentation-only test IPs
45
47
  - Provider-neutral AI prompt and tool-schema helpers
48
+ - File and text hashing
49
+ - UUIDs, secure tokens, and timestamp IDs
50
+ - Slugs, snake/camel case, chunks, environment variables, and retry logic
51
+ - Defensive security checks, secret redaction, password generation, and file verification
52
+ - Optional PyTorch training for new MLPs or supplied local models
46
53
  - `shadow` command-line tool
47
54
 
48
55
  ## Quick start
@@ -94,6 +101,50 @@ search_tool = tool_schema(
94
101
  )
95
102
  ```
96
103
 
104
+ ## Everyday helpers
105
+
106
+ ```python
107
+ from shadowtools import hash_text, random_token, slugify, to_bool, uuid4
108
+
109
+ print(hash_text("hello"))
110
+ print(random_token())
111
+ print(uuid4())
112
+ print(slugify("Shadow Tools: Version 3!")) # shadow-tools-version-3
113
+ print(to_bool("yes")) # True
114
+ ```
115
+
116
+ ```bash
117
+ shadow hash settings.json
118
+ shadow uuid
119
+ ```
120
+
121
+ ## Security and training
122
+
123
+ ```python
124
+ from shadowtools.security import audit_headers, redact_secrets
125
+
126
+ print(redact_secrets("Authorization: Bearer secret-value"))
127
+ print(audit_headers({"X-Content-Type-Options": "nosniff"}))
128
+ ```
129
+
130
+ Install ML support, then train a small classifier from scratch:
131
+
132
+ ```bash
133
+ pip install "shadow-tools[ml]"
134
+ ```
135
+
136
+ ```python
137
+ from shadowtools.training import Trainer, TrainingConfig
138
+
139
+ config = TrainingConfig(
140
+ mode="from_scratch",
141
+ architecture={"type": "mlp", "input_size": 8, "hidden_sizes": [32, 16], "num_classes": 2},
142
+ epochs=5,
143
+ )
144
+ trainer = Trainer(config)
145
+ history = trainer.fit(train_loader, validation_loader)
146
+ ```
147
+
97
148
  ```bash
98
149
  shadow get https://api.github.com --json
99
150
  shadow json-format settings.json
@@ -19,6 +19,11 @@ pip install "shadow-tools[websocket]" # optional WebSocket client
19
19
  - Email, URL, and required-key checks
20
20
  - DNS lookup, subdomain parsing, and documentation-only test IPs
21
21
  - Provider-neutral AI prompt and tool-schema helpers
22
+ - File and text hashing
23
+ - UUIDs, secure tokens, and timestamp IDs
24
+ - Slugs, snake/camel case, chunks, environment variables, and retry logic
25
+ - Defensive security checks, secret redaction, password generation, and file verification
26
+ - Optional PyTorch training for new MLPs or supplied local models
22
27
  - `shadow` command-line tool
23
28
 
24
29
  ## Quick start
@@ -70,6 +75,50 @@ search_tool = tool_schema(
70
75
  )
71
76
  ```
72
77
 
78
+ ## Everyday helpers
79
+
80
+ ```python
81
+ from shadowtools import hash_text, random_token, slugify, to_bool, uuid4
82
+
83
+ print(hash_text("hello"))
84
+ print(random_token())
85
+ print(uuid4())
86
+ print(slugify("Shadow Tools: Version 3!")) # shadow-tools-version-3
87
+ print(to_bool("yes")) # True
88
+ ```
89
+
90
+ ```bash
91
+ shadow hash settings.json
92
+ shadow uuid
93
+ ```
94
+
95
+ ## Security and training
96
+
97
+ ```python
98
+ from shadowtools.security import audit_headers, redact_secrets
99
+
100
+ print(redact_secrets("Authorization: Bearer secret-value"))
101
+ print(audit_headers({"X-Content-Type-Options": "nosniff"}))
102
+ ```
103
+
104
+ Install ML support, then train a small classifier from scratch:
105
+
106
+ ```bash
107
+ pip install "shadow-tools[ml]"
108
+ ```
109
+
110
+ ```python
111
+ from shadowtools.training import Trainer, TrainingConfig
112
+
113
+ config = TrainingConfig(
114
+ mode="from_scratch",
115
+ architecture={"type": "mlp", "input_size": 8, "hidden_sizes": [32, 16], "num_classes": 2},
116
+ epochs=5,
117
+ )
118
+ trainer = Trainer(config)
119
+ history = trainer.fit(train_loader, validation_loader)
120
+ ```
121
+
73
122
  ```bash
74
123
  shadow get https://api.github.com --json
75
124
  shadow json-format settings.json
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "shadow-tools"
7
- version = "0.2.5"
7
+ version = "0.4.0"
8
8
  description = "A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -23,6 +23,7 @@ dependencies = []
23
23
  [project.optional-dependencies]
24
24
  websocket = ["websocket-client>=1.8"]
25
25
  dev = ["pytest>=8.0", "build>=1.2", "twine>=6.0"]
26
+ ml = ["torch>=2.2"]
26
27
 
27
28
  [project.scripts]
28
29
  shadow = "shadowtools.cli:main"
@@ -0,0 +1,21 @@
1
+ """Shadow Tools: focused helpers for common Python automation tasks."""
2
+
3
+ from .data import read_csv, read_json, write_csv, write_json
4
+ from .config import env, require_env, to_bool
5
+ from .files import append_text, ensure_dir, read_text, write_text
6
+ from .hashing import file_hash, hash_text
7
+ from .http import get, request
8
+ from .ids import random_token, timestamp_id, uuid4
9
+ from .network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, resolve_dns, reverse_dns
10
+ from .retry import retry
11
+ from .security import audit_headers, make_password, redact_secrets, verify_file_hash
12
+ from .text import chunks, slugify, to_camel, to_snake
13
+ from .validation import is_email, is_url, require_keys
14
+
15
+ __version__ = "0.4.0"
16
+
17
+ __all__ = [
18
+ "append_text", "audit_headers", "chunks", "ensure_dir", "env", "file_hash", "generate_random_ips", "generate_test_ips", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "make_password", "make_subdomain", "random_token", "read_csv",
19
+ "read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "slugify", "timestamp_id", "to_bool", "to_camel", "to_snake", "uuid4", "write_csv",
20
+ "write_json", "write_text", "redact_secrets", "verify_file_hash",
21
+ ]
@@ -8,6 +8,8 @@ from pathlib import Path
8
8
  from . import __version__
9
9
  from .data import read_json, write_json
10
10
  from .http import get
11
+ from .hashing import file_hash
12
+ from .ids import uuid4
11
13
 
12
14
 
13
15
  def main() -> None:
@@ -19,6 +21,9 @@ def main() -> None:
19
21
  http_parser.add_argument("--json", action="store_true", help="Pretty-print a JSON response")
20
22
  json_parser = commands.add_parser("json-format", help="Format a JSON file")
21
23
  json_parser.add_argument("path", type=Path)
24
+ hash_parser = commands.add_parser("hash", help="Print a file SHA-256 checksum")
25
+ hash_parser.add_argument("path", type=Path)
26
+ commands.add_parser("uuid", help="Create a UUID")
22
27
  args = parser.parse_args()
23
28
  if args.command == "get":
24
29
  response = get(args.url)
@@ -27,6 +32,10 @@ def main() -> None:
27
32
  elif args.command == "json-format":
28
33
  write_json(args.path, read_json(args.path))
29
34
  print(f"Formatted {args.path}")
35
+ elif args.command == "hash":
36
+ print(file_hash(args.path))
37
+ elif args.command == "uuid":
38
+ print(uuid4())
30
39
 
31
40
 
32
41
  if __name__ == "__main__":
@@ -0,0 +1,23 @@
1
+ """Environment-variable helpers."""
2
+ from __future__ import annotations
3
+
4
+ import os
5
+
6
+
7
+ def env(name: str, default: str | None = None) -> str | None:
8
+ return os.environ.get(name, default)
9
+
10
+
11
+ def require_env(name: str) -> str:
12
+ value = os.environ.get(name)
13
+ if not value:
14
+ raise EnvironmentError(f"Required environment variable is missing: {name}")
15
+ return value
16
+
17
+
18
+ def to_bool(value: str | bool | int | None, default: bool = False) -> bool:
19
+ if value is None:
20
+ return default
21
+ if isinstance(value, bool):
22
+ return value
23
+ return str(value).strip().lower() in {"1", "true", "yes", "on", "y"}
@@ -0,0 +1,17 @@
1
+ """Checksums for text and files."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ from pathlib import Path
6
+
7
+
8
+ def hash_text(value: str, algorithm: str = "sha256") -> str:
9
+ return hashlib.new(algorithm, value.encode("utf-8")).hexdigest()
10
+
11
+
12
+ def file_hash(path: str | Path, algorithm: str = "sha256", *, chunk_size: int = 65_536) -> str:
13
+ digest = hashlib.new(algorithm)
14
+ with Path(path).open("rb") as handle:
15
+ for chunk in iter(lambda: handle.read(chunk_size), b""):
16
+ digest.update(chunk)
17
+ return digest.hexdigest()
@@ -0,0 +1,20 @@
1
+ """IDs and secure random tokens."""
2
+ from __future__ import annotations
3
+
4
+ import secrets
5
+ import time
6
+ import uuid
7
+
8
+
9
+ def uuid4() -> str:
10
+ return str(uuid.uuid4())
11
+
12
+
13
+ def random_token(length: int = 32) -> str:
14
+ if not 8 <= length <= 256:
15
+ raise ValueError("length must be between 8 and 256")
16
+ return secrets.token_urlsafe(length)[:length]
17
+
18
+
19
+ def timestamp_id(prefix: str = "item") -> str:
20
+ return f"{prefix}_{int(time.time() * 1000)}_{secrets.token_hex(3)}"
@@ -0,0 +1,30 @@
1
+ """Simple retry support for temporary failures."""
2
+ from __future__ import annotations
3
+
4
+ import time
5
+ from collections.abc import Callable
6
+ from functools import wraps
7
+ from typing import ParamSpec, TypeVar
8
+
9
+ P = ParamSpec("P")
10
+ T = TypeVar("T")
11
+
12
+
13
+ def retry(attempts: int = 3, *, delay: float = 0.25,
14
+ exceptions: tuple[type[Exception], ...] = (Exception,)) -> Callable[[Callable[P, T]], Callable[P, T]]:
15
+ if attempts < 1:
16
+ raise ValueError("attempts must be at least 1")
17
+
18
+ def decorate(function: Callable[P, T]) -> Callable[P, T]:
19
+ @wraps(function)
20
+ def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
21
+ for attempt in range(attempts):
22
+ try:
23
+ return function(*args, **kwargs)
24
+ except exceptions:
25
+ if attempt == attempts - 1:
26
+ raise
27
+ time.sleep(delay * (2 ** attempt))
28
+ raise RuntimeError("unreachable")
29
+ return wrapped
30
+ return decorate
@@ -0,0 +1,55 @@
1
+ """Defensive security helpers for data you own or are authorized to inspect."""
2
+ from __future__ import annotations
3
+
4
+ import hmac
5
+ import re
6
+ import secrets
7
+ import string
8
+ from pathlib import Path
9
+ from typing import Mapping
10
+
11
+ from .hashing import file_hash
12
+
13
+ _SECRET = re.compile(r"(?i)(authorization\s*[:=]\s*(?:bearer\s+)?|api[_-]?key\s*[:=]\s*|token\s*[:=]\s*|password\s*[:=]\s*)([^\s,;]+)")
14
+
15
+
16
+ def redact_secrets(text: str, replacement: str = "[REDACTED]") -> str:
17
+ """Remove common credential values before logs are saved or shared."""
18
+ return _SECRET.sub(lambda match: match.group(1) + replacement, text)
19
+
20
+
21
+ def audit_headers(headers: Mapping[str, str]) -> dict[str, list[str]]:
22
+ """Report common missing browser-security response headers."""
23
+ normalized = {name.lower(): value for name, value in headers.items()}
24
+ required = {
25
+ "content-security-policy": "Content-Security-Policy",
26
+ "x-content-type-options": "X-Content-Type-Options",
27
+ "x-frame-options": "X-Frame-Options",
28
+ "referrer-policy": "Referrer-Policy",
29
+ "strict-transport-security": "Strict-Transport-Security",
30
+ }
31
+ missing = [display for key, display in required.items() if key not in normalized]
32
+ warnings: list[str] = []
33
+ if normalized.get("x-content-type-options", "").lower() != "nosniff":
34
+ warnings.append("Set X-Content-Type-Options to nosniff.")
35
+ if "content-security-policy" not in normalized:
36
+ warnings.append("Add a Content-Security-Policy appropriate for this app.")
37
+ return {"missing": missing, "warnings": warnings}
38
+
39
+
40
+ def make_password(length: int = 20) -> str:
41
+ """Create a random password containing upper/lowercase, digits, and symbols."""
42
+ if length < 12:
43
+ raise ValueError("Use at least 12 characters for a password.")
44
+ alphabet = string.ascii_letters + string.digits + "!@#$%^&*_-+="
45
+ required = [secrets.choice(string.ascii_lowercase), secrets.choice(string.ascii_uppercase),
46
+ secrets.choice(string.digits), secrets.choice("!@#$%^&*_-+=")]
47
+ remaining = [secrets.choice(alphabet) for _ in range(length - len(required))]
48
+ values = required + remaining
49
+ secrets.SystemRandom().shuffle(values)
50
+ return "".join(values)
51
+
52
+
53
+ def verify_file_hash(path: str | Path, expected: str, algorithm: str = "sha256") -> bool:
54
+ """Compare a file checksum without leaking timing information."""
55
+ return hmac.compare_digest(file_hash(path, algorithm), expected.lower())
@@ -0,0 +1,33 @@
1
+ """Plain text cleanup and naming helpers."""
2
+ from __future__ import annotations
3
+
4
+ import re
5
+ from collections.abc import Iterable
6
+ from typing import TypeVar
7
+
8
+ T = TypeVar("T")
9
+ _WORDS = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|\d+")
10
+
11
+
12
+ def _words(value: str) -> list[str]:
13
+ return _WORDS.findall(value.replace("-", " ").replace("_", " "))
14
+
15
+
16
+ def slugify(value: str) -> str:
17
+ return "-".join(word.lower() for word in _words(value))
18
+
19
+
20
+ def to_snake(value: str) -> str:
21
+ return "_".join(word.lower() for word in _words(value))
22
+
23
+
24
+ def to_camel(value: str) -> str:
25
+ words = _words(value)
26
+ return "".join(word[:1].upper() + word[1:].lower() for word in words)
27
+
28
+
29
+ def chunks(items: Iterable[T], size: int) -> list[list[T]]:
30
+ if size < 1:
31
+ raise ValueError("size must be at least 1")
32
+ values = list(items)
33
+ return [values[index:index + size] for index in range(0, len(values), size)]
@@ -0,0 +1,170 @@
1
+ """A small, explicit PyTorch training engine for fine-tuning or new MLPs."""
2
+ from __future__ import annotations
3
+
4
+ import logging
5
+ from dataclasses import dataclass, field
6
+ from pathlib import Path
7
+ from typing import Any, Callable, Iterable, Literal, Mapping
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ @dataclass
13
+ class TrainingConfig:
14
+ mode: Literal["from_scratch", "fine_tune"]
15
+ epochs: int = 3
16
+ learning_rate: float = 1e-3
17
+ batch_size: int = 32
18
+ device: str = "auto"
19
+ optimizer: Literal["adamw", "sgd"] = "adamw"
20
+ checkpoint_path: str | Path | None = None
21
+ architecture: Mapping[str, Any] | None = None
22
+ model: Any | None = None
23
+ model_path: str | Path | None = None
24
+ model_id: str | None = None
25
+ model_factory: Callable[[], Any] | None = None
26
+ model_loader: Callable[[str], Any] | None = None
27
+ freeze_layers: tuple[str, ...] = ()
28
+ head_attr: str | None = None
29
+ num_classes: int | None = None
30
+
31
+
32
+ class Trainer:
33
+ """Train a PyTorch classifier from scratch or fine-tune an existing module."""
34
+
35
+ def __init__(self, config: TrainingConfig | Mapping[str, Any]) -> None:
36
+ self.config = config if isinstance(config, TrainingConfig) else TrainingConfig(**dict(config))
37
+ self.torch = self._torch()
38
+ self.device = self._device()
39
+ self.model = self._build_model().to(self.device)
40
+ self._freeze()
41
+ self._replace_head()
42
+
43
+ @staticmethod
44
+ def _torch() -> Any:
45
+ try:
46
+ import torch
47
+ return torch
48
+ except ImportError as exc:
49
+ raise ImportError("Training needs PyTorch. Install it with `pip install shadow-tools[ml]`.") from exc
50
+
51
+ def _device(self) -> Any:
52
+ value = self.config.device.lower()
53
+ if value == "auto":
54
+ value = "cuda" if self.torch.cuda.is_available() else "mps" if getattr(self.torch.backends, "mps", None) and self.torch.backends.mps.is_available() else "cpu"
55
+ if value.startswith("cuda") and not self.torch.cuda.is_available():
56
+ raise RuntimeError("CUDA was requested but is unavailable.")
57
+ if value == "mps" and not (getattr(self.torch.backends, "mps", None) and self.torch.backends.mps.is_available()):
58
+ raise RuntimeError("MPS was requested but is unavailable.")
59
+ return self.torch.device(value)
60
+
61
+ def _build_model(self) -> Any:
62
+ if self.config.mode == "from_scratch":
63
+ spec = dict(self.config.architecture or {})
64
+ if spec.get("type", "mlp") != "mlp":
65
+ raise ValueError("from_scratch currently supports architecture type 'mlp'.")
66
+ try:
67
+ input_size = int(spec["input_size"])
68
+ num_classes = int(spec["num_classes"])
69
+ except KeyError as exc:
70
+ raise ValueError("MLP architecture requires input_size and num_classes.") from exc
71
+ hidden = [int(size) for size in spec.get("hidden_sizes", [128, 64])]
72
+ dropout = float(spec.get("dropout", 0.0))
73
+ layers: list[Any] = [self.torch.nn.Flatten()]
74
+ previous = input_size
75
+ for size in hidden:
76
+ layers.extend([self.torch.nn.Linear(previous, size), self.torch.nn.ReLU()])
77
+ if dropout:
78
+ layers.append(self.torch.nn.Dropout(dropout))
79
+ previous = size
80
+ layers.append(self.torch.nn.Linear(previous, num_classes))
81
+ return self.torch.nn.Sequential(*layers)
82
+ if self.config.mode != "fine_tune":
83
+ raise ValueError("mode must be 'from_scratch' or 'fine_tune'.")
84
+ model = self.config.model
85
+ if model is None and self.config.model_id:
86
+ if self.config.model_loader is None:
87
+ raise ValueError("model_id requires an explicit model_loader callable.")
88
+ model = self.config.model_loader(self.config.model_id)
89
+ if model is None and self.config.model_factory:
90
+ model = self.config.model_factory()
91
+ if model is None:
92
+ raise ValueError("fine_tune requires model, model_factory, or model_id with model_loader.")
93
+ if self.config.model_path:
94
+ path = Path(self.config.model_path)
95
+ if not path.is_file():
96
+ raise FileNotFoundError(path)
97
+ state = self.torch.load(path, map_location="cpu", weights_only=True)
98
+ if isinstance(state, Mapping) and "state_dict" in state:
99
+ state = state["state_dict"]
100
+ model.load_state_dict(state)
101
+ return model
102
+
103
+ def _freeze(self) -> None:
104
+ for name, parameter in self.model.named_parameters():
105
+ if any(name.startswith(prefix) for prefix in self.config.freeze_layers):
106
+ parameter.requires_grad = False
107
+
108
+ def _replace_head(self) -> None:
109
+ if not self.config.head_attr or self.config.num_classes is None:
110
+ return
111
+ head = getattr(self.model, self.config.head_attr, None)
112
+ if not isinstance(head, self.torch.nn.Linear):
113
+ raise ValueError("head_attr must name a torch.nn.Linear layer.")
114
+ setattr(self.model, self.config.head_attr, self.torch.nn.Linear(head.in_features, self.config.num_classes))
115
+
116
+ def fit(self, train_data: Iterable[Any], validation_data: Iterable[Any] | None = None) -> list[dict[str, float]]:
117
+ """Train on iterable batches of ``(features, labels)`` and return epoch metrics."""
118
+ parameters = [parameter for parameter in self.model.parameters() if parameter.requires_grad]
119
+ if not parameters:
120
+ raise RuntimeError("No trainable parameters remain after freezing layers.")
121
+ optimizer = self.torch.optim.AdamW(parameters, lr=self.config.learning_rate) if self.config.optimizer == "adamw" else self.torch.optim.SGD(parameters, lr=self.config.learning_rate)
122
+ loss_fn = self.torch.nn.CrossEntropyLoss()
123
+ history: list[dict[str, float]] = []
124
+ for epoch in range(1, self.config.epochs + 1):
125
+ self.model.train()
126
+ total_loss = 0.0
127
+ batches = 0
128
+ for features, labels in train_data:
129
+ features = features.to(self.device)
130
+ labels = labels.to(self.device).long()
131
+ optimizer.zero_grad(set_to_none=True)
132
+ loss = loss_fn(self.model(features), labels)
133
+ loss.backward()
134
+ optimizer.step()
135
+ total_loss += float(loss.detach().cpu())
136
+ batches += 1
137
+ if not batches:
138
+ raise ValueError("train_data produced no batches.")
139
+ metrics = {"epoch": float(epoch), "loss": total_loss / batches}
140
+ if validation_data is not None:
141
+ metrics["accuracy"] = self.evaluate(validation_data)
142
+ history.append(metrics)
143
+ logger.info("epoch=%s loss=%.5f", epoch, metrics["loss"])
144
+ self._checkpoint(epoch, metrics)
145
+ return history
146
+
147
+ def evaluate(self, data: Iterable[Any]) -> float:
148
+ self.model.eval()
149
+ correct = total = 0
150
+ with self.torch.no_grad():
151
+ for features, labels in data:
152
+ prediction = self.model(features.to(self.device)).argmax(dim=1)
153
+ labels = labels.to(self.device).long()
154
+ correct += int((prediction == labels).sum().item())
155
+ total += int(labels.numel())
156
+ if not total:
157
+ raise ValueError("validation data produced no labels.")
158
+ return correct / total
159
+
160
+ def _checkpoint(self, epoch: int, metrics: Mapping[str, float]) -> None:
161
+ if self.config.checkpoint_path is None:
162
+ return
163
+ path = Path(self.config.checkpoint_path)
164
+ path.parent.mkdir(parents=True, exist_ok=True)
165
+ self.torch.save({"epoch": epoch, "metrics": dict(metrics), "state_dict": self.model.state_dict()}, path)
166
+
167
+
168
+ def run_training(config: TrainingConfig | Mapping[str, Any], train_data: Iterable[Any], validation_data: Iterable[Any] | None = None) -> tuple[Trainer, list[dict[str, float]]]:
169
+ trainer = Trainer(config)
170
+ return trainer, trainer.fit(train_data, validation_data)
@@ -8,6 +8,11 @@ from shadowtools.data import read_csv, read_json, write_csv, write_json
8
8
  from shadowtools.files import append_text, read_text, write_text
9
9
  from shadowtools.network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, reverse_dns
10
10
  from shadowtools.ai import prompt, tool_schema
11
+ from shadowtools.config import to_bool
12
+ from shadowtools.hashing import hash_text
13
+ from shadowtools.ids import random_token
14
+ from shadowtools.text import chunks, slugify, to_camel, to_snake
15
+ from shadowtools.security import audit_headers, make_password, redact_secrets
11
16
  from shadowtools.validation import is_email, is_url, require_keys
12
17
 
13
18
 
@@ -47,5 +52,18 @@ def test_network_and_ai_helpers():
47
52
  assert reverse_dns("0.0.0.0") is None
48
53
  assert reverse_dns("0.0.0.0", only_hostnames=False)["hostname"] is None
49
54
  assert all(item["hostname"] for item in reverse_dns(["0.0.0.0", "127.0.0.1"]))
55
+
56
+
57
+ def test_general_utilities():
58
+ assert hash_text("hello") == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
59
+ assert len(random_token(16)) == 16
60
+ assert slugify("Shadow Tools V3") == "shadow-tools-v-3"
61
+ assert to_snake("ShadowTools") == "shadow_tools"
62
+ assert to_camel("shadow tools") == "ShadowTools"
63
+ assert chunks(range(5), 2) == [[0, 1], [2, 3], [4]]
64
+ assert to_bool("yes")
65
+ assert "[REDACTED]" in redact_secrets("Authorization: Bearer private")
66
+ assert len(make_password(16)) == 16
67
+ assert "Content-Security-Policy" in audit_headers({})["missing"]
50
68
  assert prompt("Hi {name}", name="Will") == "Hi Will"
51
69
  assert tool_schema("lookup_dns", "Lookup DNS", {"host": {"type": "string"}})["name"] == "lookup_dns"
@@ -1,15 +0,0 @@
1
- """Shadow Tools: focused helpers for common Python automation tasks."""
2
-
3
- from .data import read_csv, read_json, write_csv, write_json
4
- from .files import append_text, ensure_dir, read_text, write_text
5
- from .http import get, request
6
- from .network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, resolve_dns, reverse_dns
7
- from .validation import is_email, is_url, require_keys
8
-
9
- __version__ = "0.2.5"
10
-
11
- __all__ = [
12
- "append_text", "ensure_dir", "generate_random_ips", "generate_test_ips", "get", "is_email", "is_subdomain_of", "is_url", "make_subdomain", "read_csv",
13
- "read_json", "read_text", "request", "require_keys", "write_csv",
14
- "write_json", "write_text", "resolve_dns", "reverse_dns",
15
- ]
File without changes
File without changes