shadow-tools 0.2.5__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/PKG-INFO +52 -1
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/README.md +49 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/pyproject.toml +2 -1
- shadow_tools-0.4.0/release-0.2.5/shadow_tools-0.2.5-py3-none-any.whl +0 -0
- shadow_tools-0.4.0/release-0.2.5/shadow_tools-0.2.5.tar.gz +0 -0
- shadow_tools-0.4.0/release-0.3.0/shadow_tools-0.3.0-py3-none-any.whl +0 -0
- shadow_tools-0.4.0/release-0.3.0/shadow_tools-0.3.0.tar.gz +0 -0
- shadow_tools-0.4.0/release-0.4.0/shadow_tools-0.4.0-py3-none-any.whl +0 -0
- shadow_tools-0.4.0/release-0.4.0/shadow_tools-0.4.0.tar.gz +0 -0
- shadow_tools-0.4.0/src/shadowtools/__init__.py +21 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/cli.py +9 -0
- shadow_tools-0.4.0/src/shadowtools/config.py +23 -0
- shadow_tools-0.4.0/src/shadowtools/hashing.py +17 -0
- shadow_tools-0.4.0/src/shadowtools/ids.py +20 -0
- shadow_tools-0.4.0/src/shadowtools/retry.py +30 -0
- shadow_tools-0.4.0/src/shadowtools/security.py +55 -0
- shadow_tools-0.4.0/src/shadowtools/text.py +33 -0
- shadow_tools-0.4.0/src/shadowtools/training.py +170 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/tests/test_core.py +18 -0
- shadow_tools-0.2.5/src/shadowtools/__init__.py +0 -15
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/.gitignore +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/LICENSE +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.0/shadow_tools-0.2.0-py3-none-any.whl +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.0/shadow_tools-0.2.0.tar.gz +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.1/shadow_tools-0.2.1-py3-none-any.whl +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.1/shadow_tools-0.2.1.tar.gz +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.2/shadow_tools-0.2.2-py3-none-any.whl +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.2/shadow_tools-0.2.2.tar.gz +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.3/shadow_tools-0.2.3-py3-none-any.whl +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.3/shadow_tools-0.2.3.tar.gz +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.4/shadow_tools-0.2.4-py3-none-any.whl +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/release-0.2.4/shadow_tools-0.2.4.tar.gz +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/ai.py +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/async_tools.py +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/data.py +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/files.py +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/http.py +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/network.py +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/validation.py +0 -0
- {shadow_tools-0.2.5 → shadow_tools-0.4.0}/src/shadowtools/websocket.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: shadow-tools
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs.
|
|
5
5
|
Project-URL: Homepage, https://github.com/WilliamLaurento/shadow-tools
|
|
6
6
|
Project-URL: Issues, https://github.com/WilliamLaurento/shadow-tools/issues
|
|
@@ -18,6 +18,8 @@ Provides-Extra: dev
|
|
|
18
18
|
Requires-Dist: build>=1.2; extra == 'dev'
|
|
19
19
|
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
20
20
|
Requires-Dist: twine>=6.0; extra == 'dev'
|
|
21
|
+
Provides-Extra: ml
|
|
22
|
+
Requires-Dist: torch>=2.2; extra == 'ml'
|
|
21
23
|
Provides-Extra: websocket
|
|
22
24
|
Requires-Dist: websocket-client>=1.8; extra == 'websocket'
|
|
23
25
|
Description-Content-Type: text/markdown
|
|
@@ -43,6 +45,11 @@ pip install "shadow-tools[websocket]" # optional WebSocket client
|
|
|
43
45
|
- Email, URL, and required-key checks
|
|
44
46
|
- DNS lookup, subdomain parsing, and documentation-only test IPs
|
|
45
47
|
- Provider-neutral AI prompt and tool-schema helpers
|
|
48
|
+
- File and text hashing
|
|
49
|
+
- UUIDs, secure tokens, and timestamp IDs
|
|
50
|
+
- Slugs, snake/camel case, chunks, environment variables, and retry logic
|
|
51
|
+
- Defensive security checks, secret redaction, password generation, and file verification
|
|
52
|
+
- Optional PyTorch training for new MLPs or supplied local models
|
|
46
53
|
- `shadow` command-line tool
|
|
47
54
|
|
|
48
55
|
## Quick start
|
|
@@ -94,6 +101,50 @@ search_tool = tool_schema(
|
|
|
94
101
|
)
|
|
95
102
|
```
|
|
96
103
|
|
|
104
|
+
## Everyday helpers
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from shadowtools import hash_text, random_token, slugify, to_bool, uuid4
|
|
108
|
+
|
|
109
|
+
print(hash_text("hello"))
|
|
110
|
+
print(random_token())
|
|
111
|
+
print(uuid4())
|
|
112
|
+
print(slugify("Shadow Tools: Version 3!")) # shadow-tools-version-3
|
|
113
|
+
print(to_bool("yes")) # True
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
shadow hash settings.json
|
|
118
|
+
shadow uuid
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## Security and training
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from shadowtools.security import audit_headers, redact_secrets
|
|
125
|
+
|
|
126
|
+
print(redact_secrets("Authorization: Bearer secret-value"))
|
|
127
|
+
print(audit_headers({"X-Content-Type-Options": "nosniff"}))
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Install ML support, then train a small classifier from scratch:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
pip install "shadow-tools[ml]"
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
from shadowtools.training import Trainer, TrainingConfig
|
|
138
|
+
|
|
139
|
+
config = TrainingConfig(
|
|
140
|
+
mode="from_scratch",
|
|
141
|
+
architecture={"type": "mlp", "input_size": 8, "hidden_sizes": [32, 16], "num_classes": 2},
|
|
142
|
+
epochs=5,
|
|
143
|
+
)
|
|
144
|
+
trainer = Trainer(config)
|
|
145
|
+
history = trainer.fit(train_loader, validation_loader)
|
|
146
|
+
```
|
|
147
|
+
|
|
97
148
|
```bash
|
|
98
149
|
shadow get https://api.github.com --json
|
|
99
150
|
shadow json-format settings.json
|
|
@@ -19,6 +19,11 @@ pip install "shadow-tools[websocket]" # optional WebSocket client
|
|
|
19
19
|
- Email, URL, and required-key checks
|
|
20
20
|
- DNS lookup, subdomain parsing, and documentation-only test IPs
|
|
21
21
|
- Provider-neutral AI prompt and tool-schema helpers
|
|
22
|
+
- File and text hashing
|
|
23
|
+
- UUIDs, secure tokens, and timestamp IDs
|
|
24
|
+
- Slugs, snake/camel case, chunks, environment variables, and retry logic
|
|
25
|
+
- Defensive security checks, secret redaction, password generation, and file verification
|
|
26
|
+
- Optional PyTorch training for new MLPs or supplied local models
|
|
22
27
|
- `shadow` command-line tool
|
|
23
28
|
|
|
24
29
|
## Quick start
|
|
@@ -70,6 +75,50 @@ search_tool = tool_schema(
|
|
|
70
75
|
)
|
|
71
76
|
```
|
|
72
77
|
|
|
78
|
+
## Everyday helpers
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from shadowtools import hash_text, random_token, slugify, to_bool, uuid4
|
|
82
|
+
|
|
83
|
+
print(hash_text("hello"))
|
|
84
|
+
print(random_token())
|
|
85
|
+
print(uuid4())
|
|
86
|
+
print(slugify("Shadow Tools: Version 3!")) # shadow-tools-version-3
|
|
87
|
+
print(to_bool("yes")) # True
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
shadow hash settings.json
|
|
92
|
+
shadow uuid
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Security and training
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from shadowtools.security import audit_headers, redact_secrets
|
|
99
|
+
|
|
100
|
+
print(redact_secrets("Authorization: Bearer secret-value"))
|
|
101
|
+
print(audit_headers({"X-Content-Type-Options": "nosniff"}))
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Install ML support, then train a small classifier from scratch:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
pip install "shadow-tools[ml]"
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from shadowtools.training import Trainer, TrainingConfig
|
|
112
|
+
|
|
113
|
+
config = TrainingConfig(
|
|
114
|
+
mode="from_scratch",
|
|
115
|
+
architecture={"type": "mlp", "input_size": 8, "hidden_sizes": [32, 16], "num_classes": 2},
|
|
116
|
+
epochs=5,
|
|
117
|
+
)
|
|
118
|
+
trainer = Trainer(config)
|
|
119
|
+
history = trainer.fit(train_loader, validation_loader)
|
|
120
|
+
```
|
|
121
|
+
|
|
73
122
|
```bash
|
|
74
123
|
shadow get https://api.github.com --json
|
|
75
124
|
shadow json-format settings.json
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "shadow-tools"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "A practical all-in-one Python toolkit for HTTP, WebSockets, files, data, validation, async work, and CLIs."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -23,6 +23,7 @@ dependencies = []
|
|
|
23
23
|
[project.optional-dependencies]
|
|
24
24
|
websocket = ["websocket-client>=1.8"]
|
|
25
25
|
dev = ["pytest>=8.0", "build>=1.2", "twine>=6.0"]
|
|
26
|
+
ml = ["torch>=2.2"]
|
|
26
27
|
|
|
27
28
|
[project.scripts]
|
|
28
29
|
shadow = "shadowtools.cli:main"
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Shadow Tools: focused helpers for common Python automation tasks."""
|
|
2
|
+
|
|
3
|
+
from .data import read_csv, read_json, write_csv, write_json
|
|
4
|
+
from .config import env, require_env, to_bool
|
|
5
|
+
from .files import append_text, ensure_dir, read_text, write_text
|
|
6
|
+
from .hashing import file_hash, hash_text
|
|
7
|
+
from .http import get, request
|
|
8
|
+
from .ids import random_token, timestamp_id, uuid4
|
|
9
|
+
from .network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, resolve_dns, reverse_dns
|
|
10
|
+
from .retry import retry
|
|
11
|
+
from .security import audit_headers, make_password, redact_secrets, verify_file_hash
|
|
12
|
+
from .text import chunks, slugify, to_camel, to_snake
|
|
13
|
+
from .validation import is_email, is_url, require_keys
|
|
14
|
+
|
|
15
|
+
__version__ = "0.4.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"append_text", "audit_headers", "chunks", "ensure_dir", "env", "file_hash", "generate_random_ips", "generate_test_ips", "get", "hash_text", "is_email", "is_subdomain_of", "is_url", "make_password", "make_subdomain", "random_token", "read_csv",
|
|
19
|
+
"read_json", "read_text", "request", "require_env", "require_keys", "retry", "reverse_dns", "resolve_dns", "slugify", "timestamp_id", "to_bool", "to_camel", "to_snake", "uuid4", "write_csv",
|
|
20
|
+
"write_json", "write_text", "redact_secrets", "verify_file_hash",
|
|
21
|
+
]
|
|
@@ -8,6 +8,8 @@ from pathlib import Path
|
|
|
8
8
|
from . import __version__
|
|
9
9
|
from .data import read_json, write_json
|
|
10
10
|
from .http import get
|
|
11
|
+
from .hashing import file_hash
|
|
12
|
+
from .ids import uuid4
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
def main() -> None:
|
|
@@ -19,6 +21,9 @@ def main() -> None:
|
|
|
19
21
|
http_parser.add_argument("--json", action="store_true", help="Pretty-print a JSON response")
|
|
20
22
|
json_parser = commands.add_parser("json-format", help="Format a JSON file")
|
|
21
23
|
json_parser.add_argument("path", type=Path)
|
|
24
|
+
hash_parser = commands.add_parser("hash", help="Print a file SHA-256 checksum")
|
|
25
|
+
hash_parser.add_argument("path", type=Path)
|
|
26
|
+
commands.add_parser("uuid", help="Create a UUID")
|
|
22
27
|
args = parser.parse_args()
|
|
23
28
|
if args.command == "get":
|
|
24
29
|
response = get(args.url)
|
|
@@ -27,6 +32,10 @@ def main() -> None:
|
|
|
27
32
|
elif args.command == "json-format":
|
|
28
33
|
write_json(args.path, read_json(args.path))
|
|
29
34
|
print(f"Formatted {args.path}")
|
|
35
|
+
elif args.command == "hash":
|
|
36
|
+
print(file_hash(args.path))
|
|
37
|
+
elif args.command == "uuid":
|
|
38
|
+
print(uuid4())
|
|
30
39
|
|
|
31
40
|
|
|
32
41
|
if __name__ == "__main__":
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Environment-variable helpers."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def env(name: str, default: str | None = None) -> str | None:
|
|
8
|
+
return os.environ.get(name, default)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def require_env(name: str) -> str:
|
|
12
|
+
value = os.environ.get(name)
|
|
13
|
+
if not value:
|
|
14
|
+
raise EnvironmentError(f"Required environment variable is missing: {name}")
|
|
15
|
+
return value
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def to_bool(value: str | bool | int | None, default: bool = False) -> bool:
|
|
19
|
+
if value is None:
|
|
20
|
+
return default
|
|
21
|
+
if isinstance(value, bool):
|
|
22
|
+
return value
|
|
23
|
+
return str(value).strip().lower() in {"1", "true", "yes", "on", "y"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Checksums for text and files."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def hash_text(value: str, algorithm: str = "sha256") -> str:
|
|
9
|
+
return hashlib.new(algorithm, value.encode("utf-8")).hexdigest()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def file_hash(path: str | Path, algorithm: str = "sha256", *, chunk_size: int = 65_536) -> str:
|
|
13
|
+
digest = hashlib.new(algorithm)
|
|
14
|
+
with Path(path).open("rb") as handle:
|
|
15
|
+
for chunk in iter(lambda: handle.read(chunk_size), b""):
|
|
16
|
+
digest.update(chunk)
|
|
17
|
+
return digest.hexdigest()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""IDs and secure random tokens."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import secrets
|
|
5
|
+
import time
|
|
6
|
+
import uuid
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def uuid4() -> str:
|
|
10
|
+
return str(uuid.uuid4())
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def random_token(length: int = 32) -> str:
|
|
14
|
+
if not 8 <= length <= 256:
|
|
15
|
+
raise ValueError("length must be between 8 and 256")
|
|
16
|
+
return secrets.token_urlsafe(length)[:length]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def timestamp_id(prefix: str = "item") -> str:
|
|
20
|
+
return f"{prefix}_{int(time.time() * 1000)}_{secrets.token_hex(3)}"
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Simple retry support for temporary failures."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from functools import wraps
|
|
7
|
+
from typing import ParamSpec, TypeVar
|
|
8
|
+
|
|
9
|
+
P = ParamSpec("P")
|
|
10
|
+
T = TypeVar("T")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def retry(attempts: int = 3, *, delay: float = 0.25,
|
|
14
|
+
exceptions: tuple[type[Exception], ...] = (Exception,)) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
|
15
|
+
if attempts < 1:
|
|
16
|
+
raise ValueError("attempts must be at least 1")
|
|
17
|
+
|
|
18
|
+
def decorate(function: Callable[P, T]) -> Callable[P, T]:
|
|
19
|
+
@wraps(function)
|
|
20
|
+
def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
|
|
21
|
+
for attempt in range(attempts):
|
|
22
|
+
try:
|
|
23
|
+
return function(*args, **kwargs)
|
|
24
|
+
except exceptions:
|
|
25
|
+
if attempt == attempts - 1:
|
|
26
|
+
raise
|
|
27
|
+
time.sleep(delay * (2 ** attempt))
|
|
28
|
+
raise RuntimeError("unreachable")
|
|
29
|
+
return wrapped
|
|
30
|
+
return decorate
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Defensive security helpers for data you own or are authorized to inspect."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hmac
|
|
5
|
+
import re
|
|
6
|
+
import secrets
|
|
7
|
+
import string
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Mapping
|
|
10
|
+
|
|
11
|
+
from .hashing import file_hash
|
|
12
|
+
|
|
13
|
+
_SECRET = re.compile(r"(?i)(authorization\s*[:=]\s*(?:bearer\s+)?|api[_-]?key\s*[:=]\s*|token\s*[:=]\s*|password\s*[:=]\s*)([^\s,;]+)")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def redact_secrets(text: str, replacement: str = "[REDACTED]") -> str:
|
|
17
|
+
"""Remove common credential values before logs are saved or shared."""
|
|
18
|
+
return _SECRET.sub(lambda match: match.group(1) + replacement, text)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def audit_headers(headers: Mapping[str, str]) -> dict[str, list[str]]:
|
|
22
|
+
"""Report common missing browser-security response headers."""
|
|
23
|
+
normalized = {name.lower(): value for name, value in headers.items()}
|
|
24
|
+
required = {
|
|
25
|
+
"content-security-policy": "Content-Security-Policy",
|
|
26
|
+
"x-content-type-options": "X-Content-Type-Options",
|
|
27
|
+
"x-frame-options": "X-Frame-Options",
|
|
28
|
+
"referrer-policy": "Referrer-Policy",
|
|
29
|
+
"strict-transport-security": "Strict-Transport-Security",
|
|
30
|
+
}
|
|
31
|
+
missing = [display for key, display in required.items() if key not in normalized]
|
|
32
|
+
warnings: list[str] = []
|
|
33
|
+
if normalized.get("x-content-type-options", "").lower() != "nosniff":
|
|
34
|
+
warnings.append("Set X-Content-Type-Options to nosniff.")
|
|
35
|
+
if "content-security-policy" not in normalized:
|
|
36
|
+
warnings.append("Add a Content-Security-Policy appropriate for this app.")
|
|
37
|
+
return {"missing": missing, "warnings": warnings}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def make_password(length: int = 20) -> str:
|
|
41
|
+
"""Create a random password containing upper/lowercase, digits, and symbols."""
|
|
42
|
+
if length < 12:
|
|
43
|
+
raise ValueError("Use at least 12 characters for a password.")
|
|
44
|
+
alphabet = string.ascii_letters + string.digits + "!@#$%^&*_-+="
|
|
45
|
+
required = [secrets.choice(string.ascii_lowercase), secrets.choice(string.ascii_uppercase),
|
|
46
|
+
secrets.choice(string.digits), secrets.choice("!@#$%^&*_-+=")]
|
|
47
|
+
remaining = [secrets.choice(alphabet) for _ in range(length - len(required))]
|
|
48
|
+
values = required + remaining
|
|
49
|
+
secrets.SystemRandom().shuffle(values)
|
|
50
|
+
return "".join(values)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def verify_file_hash(path: str | Path, expected: str, algorithm: str = "sha256") -> bool:
|
|
54
|
+
"""Compare a file checksum without leaking timing information."""
|
|
55
|
+
return hmac.compare_digest(file_hash(path, algorithm), expected.lower())
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Plain text cleanup and naming helpers."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
from typing import TypeVar
|
|
7
|
+
|
|
8
|
+
T = TypeVar("T")
|
|
9
|
+
_WORDS = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|\d+")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _words(value: str) -> list[str]:
|
|
13
|
+
return _WORDS.findall(value.replace("-", " ").replace("_", " "))
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def slugify(value: str) -> str:
|
|
17
|
+
return "-".join(word.lower() for word in _words(value))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def to_snake(value: str) -> str:
|
|
21
|
+
return "_".join(word.lower() for word in _words(value))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def to_camel(value: str) -> str:
|
|
25
|
+
words = _words(value)
|
|
26
|
+
return "".join(word[:1].upper() + word[1:].lower() for word in words)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def chunks(items: Iterable[T], size: int) -> list[list[T]]:
|
|
30
|
+
if size < 1:
|
|
31
|
+
raise ValueError("size must be at least 1")
|
|
32
|
+
values = list(items)
|
|
33
|
+
return [values[index:index + size] for index in range(0, len(values), size)]
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""A small, explicit PyTorch training engine for fine-tuning or new MLPs."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import logging
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Callable, Iterable, Literal, Mapping
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class TrainingConfig:
|
|
14
|
+
mode: Literal["from_scratch", "fine_tune"]
|
|
15
|
+
epochs: int = 3
|
|
16
|
+
learning_rate: float = 1e-3
|
|
17
|
+
batch_size: int = 32
|
|
18
|
+
device: str = "auto"
|
|
19
|
+
optimizer: Literal["adamw", "sgd"] = "adamw"
|
|
20
|
+
checkpoint_path: str | Path | None = None
|
|
21
|
+
architecture: Mapping[str, Any] | None = None
|
|
22
|
+
model: Any | None = None
|
|
23
|
+
model_path: str | Path | None = None
|
|
24
|
+
model_id: str | None = None
|
|
25
|
+
model_factory: Callable[[], Any] | None = None
|
|
26
|
+
model_loader: Callable[[str], Any] | None = None
|
|
27
|
+
freeze_layers: tuple[str, ...] = ()
|
|
28
|
+
head_attr: str | None = None
|
|
29
|
+
num_classes: int | None = None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Trainer:
|
|
33
|
+
"""Train a PyTorch classifier from scratch or fine-tune an existing module."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, config: TrainingConfig | Mapping[str, Any]) -> None:
|
|
36
|
+
self.config = config if isinstance(config, TrainingConfig) else TrainingConfig(**dict(config))
|
|
37
|
+
self.torch = self._torch()
|
|
38
|
+
self.device = self._device()
|
|
39
|
+
self.model = self._build_model().to(self.device)
|
|
40
|
+
self._freeze()
|
|
41
|
+
self._replace_head()
|
|
42
|
+
|
|
43
|
+
@staticmethod
|
|
44
|
+
def _torch() -> Any:
|
|
45
|
+
try:
|
|
46
|
+
import torch
|
|
47
|
+
return torch
|
|
48
|
+
except ImportError as exc:
|
|
49
|
+
raise ImportError("Training needs PyTorch. Install it with `pip install shadow-tools[ml]`.") from exc
|
|
50
|
+
|
|
51
|
+
def _device(self) -> Any:
|
|
52
|
+
value = self.config.device.lower()
|
|
53
|
+
if value == "auto":
|
|
54
|
+
value = "cuda" if self.torch.cuda.is_available() else "mps" if getattr(self.torch.backends, "mps", None) and self.torch.backends.mps.is_available() else "cpu"
|
|
55
|
+
if value.startswith("cuda") and not self.torch.cuda.is_available():
|
|
56
|
+
raise RuntimeError("CUDA was requested but is unavailable.")
|
|
57
|
+
if value == "mps" and not (getattr(self.torch.backends, "mps", None) and self.torch.backends.mps.is_available()):
|
|
58
|
+
raise RuntimeError("MPS was requested but is unavailable.")
|
|
59
|
+
return self.torch.device(value)
|
|
60
|
+
|
|
61
|
+
def _build_model(self) -> Any:
|
|
62
|
+
if self.config.mode == "from_scratch":
|
|
63
|
+
spec = dict(self.config.architecture or {})
|
|
64
|
+
if spec.get("type", "mlp") != "mlp":
|
|
65
|
+
raise ValueError("from_scratch currently supports architecture type 'mlp'.")
|
|
66
|
+
try:
|
|
67
|
+
input_size = int(spec["input_size"])
|
|
68
|
+
num_classes = int(spec["num_classes"])
|
|
69
|
+
except KeyError as exc:
|
|
70
|
+
raise ValueError("MLP architecture requires input_size and num_classes.") from exc
|
|
71
|
+
hidden = [int(size) for size in spec.get("hidden_sizes", [128, 64])]
|
|
72
|
+
dropout = float(spec.get("dropout", 0.0))
|
|
73
|
+
layers: list[Any] = [self.torch.nn.Flatten()]
|
|
74
|
+
previous = input_size
|
|
75
|
+
for size in hidden:
|
|
76
|
+
layers.extend([self.torch.nn.Linear(previous, size), self.torch.nn.ReLU()])
|
|
77
|
+
if dropout:
|
|
78
|
+
layers.append(self.torch.nn.Dropout(dropout))
|
|
79
|
+
previous = size
|
|
80
|
+
layers.append(self.torch.nn.Linear(previous, num_classes))
|
|
81
|
+
return self.torch.nn.Sequential(*layers)
|
|
82
|
+
if self.config.mode != "fine_tune":
|
|
83
|
+
raise ValueError("mode must be 'from_scratch' or 'fine_tune'.")
|
|
84
|
+
model = self.config.model
|
|
85
|
+
if model is None and self.config.model_id:
|
|
86
|
+
if self.config.model_loader is None:
|
|
87
|
+
raise ValueError("model_id requires an explicit model_loader callable.")
|
|
88
|
+
model = self.config.model_loader(self.config.model_id)
|
|
89
|
+
if model is None and self.config.model_factory:
|
|
90
|
+
model = self.config.model_factory()
|
|
91
|
+
if model is None:
|
|
92
|
+
raise ValueError("fine_tune requires model, model_factory, or model_id with model_loader.")
|
|
93
|
+
if self.config.model_path:
|
|
94
|
+
path = Path(self.config.model_path)
|
|
95
|
+
if not path.is_file():
|
|
96
|
+
raise FileNotFoundError(path)
|
|
97
|
+
state = self.torch.load(path, map_location="cpu", weights_only=True)
|
|
98
|
+
if isinstance(state, Mapping) and "state_dict" in state:
|
|
99
|
+
state = state["state_dict"]
|
|
100
|
+
model.load_state_dict(state)
|
|
101
|
+
return model
|
|
102
|
+
|
|
103
|
+
def _freeze(self) -> None:
|
|
104
|
+
for name, parameter in self.model.named_parameters():
|
|
105
|
+
if any(name.startswith(prefix) for prefix in self.config.freeze_layers):
|
|
106
|
+
parameter.requires_grad = False
|
|
107
|
+
|
|
108
|
+
def _replace_head(self) -> None:
|
|
109
|
+
if not self.config.head_attr or self.config.num_classes is None:
|
|
110
|
+
return
|
|
111
|
+
head = getattr(self.model, self.config.head_attr, None)
|
|
112
|
+
if not isinstance(head, self.torch.nn.Linear):
|
|
113
|
+
raise ValueError("head_attr must name a torch.nn.Linear layer.")
|
|
114
|
+
setattr(self.model, self.config.head_attr, self.torch.nn.Linear(head.in_features, self.config.num_classes))
|
|
115
|
+
|
|
116
|
+
def fit(self, train_data: Iterable[Any], validation_data: Iterable[Any] | None = None) -> list[dict[str, float]]:
|
|
117
|
+
"""Train on iterable batches of ``(features, labels)`` and return epoch metrics."""
|
|
118
|
+
parameters = [parameter for parameter in self.model.parameters() if parameter.requires_grad]
|
|
119
|
+
if not parameters:
|
|
120
|
+
raise RuntimeError("No trainable parameters remain after freezing layers.")
|
|
121
|
+
optimizer = self.torch.optim.AdamW(parameters, lr=self.config.learning_rate) if self.config.optimizer == "adamw" else self.torch.optim.SGD(parameters, lr=self.config.learning_rate)
|
|
122
|
+
loss_fn = self.torch.nn.CrossEntropyLoss()
|
|
123
|
+
history: list[dict[str, float]] = []
|
|
124
|
+
for epoch in range(1, self.config.epochs + 1):
|
|
125
|
+
self.model.train()
|
|
126
|
+
total_loss = 0.0
|
|
127
|
+
batches = 0
|
|
128
|
+
for features, labels in train_data:
|
|
129
|
+
features = features.to(self.device)
|
|
130
|
+
labels = labels.to(self.device).long()
|
|
131
|
+
optimizer.zero_grad(set_to_none=True)
|
|
132
|
+
loss = loss_fn(self.model(features), labels)
|
|
133
|
+
loss.backward()
|
|
134
|
+
optimizer.step()
|
|
135
|
+
total_loss += float(loss.detach().cpu())
|
|
136
|
+
batches += 1
|
|
137
|
+
if not batches:
|
|
138
|
+
raise ValueError("train_data produced no batches.")
|
|
139
|
+
metrics = {"epoch": float(epoch), "loss": total_loss / batches}
|
|
140
|
+
if validation_data is not None:
|
|
141
|
+
metrics["accuracy"] = self.evaluate(validation_data)
|
|
142
|
+
history.append(metrics)
|
|
143
|
+
logger.info("epoch=%s loss=%.5f", epoch, metrics["loss"])
|
|
144
|
+
self._checkpoint(epoch, metrics)
|
|
145
|
+
return history
|
|
146
|
+
|
|
147
|
+
def evaluate(self, data: Iterable[Any]) -> float:
|
|
148
|
+
self.model.eval()
|
|
149
|
+
correct = total = 0
|
|
150
|
+
with self.torch.no_grad():
|
|
151
|
+
for features, labels in data:
|
|
152
|
+
prediction = self.model(features.to(self.device)).argmax(dim=1)
|
|
153
|
+
labels = labels.to(self.device).long()
|
|
154
|
+
correct += int((prediction == labels).sum().item())
|
|
155
|
+
total += int(labels.numel())
|
|
156
|
+
if not total:
|
|
157
|
+
raise ValueError("validation data produced no labels.")
|
|
158
|
+
return correct / total
|
|
159
|
+
|
|
160
|
+
def _checkpoint(self, epoch: int, metrics: Mapping[str, float]) -> None:
|
|
161
|
+
if self.config.checkpoint_path is None:
|
|
162
|
+
return
|
|
163
|
+
path = Path(self.config.checkpoint_path)
|
|
164
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
165
|
+
self.torch.save({"epoch": epoch, "metrics": dict(metrics), "state_dict": self.model.state_dict()}, path)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def run_training(config: TrainingConfig | Mapping[str, Any], train_data: Iterable[Any], validation_data: Iterable[Any] | None = None) -> tuple[Trainer, list[dict[str, float]]]:
|
|
169
|
+
trainer = Trainer(config)
|
|
170
|
+
return trainer, trainer.fit(train_data, validation_data)
|
|
@@ -8,6 +8,11 @@ from shadowtools.data import read_csv, read_json, write_csv, write_json
|
|
|
8
8
|
from shadowtools.files import append_text, read_text, write_text
|
|
9
9
|
from shadowtools.network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, reverse_dns
|
|
10
10
|
from shadowtools.ai import prompt, tool_schema
|
|
11
|
+
from shadowtools.config import to_bool
|
|
12
|
+
from shadowtools.hashing import hash_text
|
|
13
|
+
from shadowtools.ids import random_token
|
|
14
|
+
from shadowtools.text import chunks, slugify, to_camel, to_snake
|
|
15
|
+
from shadowtools.security import audit_headers, make_password, redact_secrets
|
|
11
16
|
from shadowtools.validation import is_email, is_url, require_keys
|
|
12
17
|
|
|
13
18
|
|
|
@@ -47,5 +52,18 @@ def test_network_and_ai_helpers():
|
|
|
47
52
|
assert reverse_dns("0.0.0.0") is None
|
|
48
53
|
assert reverse_dns("0.0.0.0", only_hostnames=False)["hostname"] is None
|
|
49
54
|
assert all(item["hostname"] for item in reverse_dns(["0.0.0.0", "127.0.0.1"]))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_general_utilities():
|
|
58
|
+
assert hash_text("hello") == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
|
|
59
|
+
assert len(random_token(16)) == 16
|
|
60
|
+
assert slugify("Shadow Tools V3") == "shadow-tools-v-3"
|
|
61
|
+
assert to_snake("ShadowTools") == "shadow_tools"
|
|
62
|
+
assert to_camel("shadow tools") == "ShadowTools"
|
|
63
|
+
assert chunks(range(5), 2) == [[0, 1], [2, 3], [4]]
|
|
64
|
+
assert to_bool("yes")
|
|
65
|
+
assert "[REDACTED]" in redact_secrets("Authorization: Bearer private")
|
|
66
|
+
assert len(make_password(16)) == 16
|
|
67
|
+
assert "Content-Security-Policy" in audit_headers({})["missing"]
|
|
50
68
|
assert prompt("Hi {name}", name="Will") == "Hi Will"
|
|
51
69
|
assert tool_schema("lookup_dns", "Lookup DNS", {"host": {"type": "string"}})["name"] == "lookup_dns"
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
"""Shadow Tools: focused helpers for common Python automation tasks."""
|
|
2
|
-
|
|
3
|
-
from .data import read_csv, read_json, write_csv, write_json
|
|
4
|
-
from .files import append_text, ensure_dir, read_text, write_text
|
|
5
|
-
from .http import get, request
|
|
6
|
-
from .network import generate_random_ips, generate_test_ips, is_subdomain_of, make_subdomain, resolve_dns, reverse_dns
|
|
7
|
-
from .validation import is_email, is_url, require_keys
|
|
8
|
-
|
|
9
|
-
__version__ = "0.2.5"
|
|
10
|
-
|
|
11
|
-
__all__ = [
|
|
12
|
-
"append_text", "ensure_dir", "generate_random_ips", "generate_test_ips", "get", "is_email", "is_subdomain_of", "is_url", "make_subdomain", "read_csv",
|
|
13
|
-
"read_json", "read_text", "request", "require_keys", "write_csv",
|
|
14
|
-
"write_json", "write_text", "resolve_dns", "reverse_dns",
|
|
15
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|