pretext-sim 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.github/workflows/ci.yml +1 -1
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.gitignore +3 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/AGENTS.md +11 -2
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/CHANGELOG.md +18 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/PKG-INFO +14 -12
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/README.md +12 -9
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/pyproject.toml +4 -5
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/scripts/capture_corpus.py +8 -2
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/src/pretext/__init__.py +1 -1
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/src/pretext/cli.py +82 -49
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/src/pretext/client.py +24 -1
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/src/pretext/models.py +2 -2
- pretext_sim-0.3.3/tests/fixtures/corpus/openai__gpt-4o-mini.json +76 -0
- pretext_sim-0.3.3/tests/fixtures/corpus/qwen__qwen3-next-80b-a3b-instruct.json +88 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/test_cli.py +80 -1
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/test_client.py +30 -0
- pretext_sim-0.3.3/tests/test_live_server.py +125 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/uv.lock +8 -341
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.editorconfig +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.env.example +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.github/dependabot.yml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.github/pull_request_template.md +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.github/workflows/release.yml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/.pre-commit-config.yaml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/CODE_OF_CONDUCT.md +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/CONTRIBUTING.md +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/LICENSE +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/SECURITY.md +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/docs/demo.png +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/docs/demo.svg +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/docs/running-an-exercise.md +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/docs/social.png +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/persona.yaml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/sample-output.md +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/scripts/changelog_section.py +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/src/pretext/data/example-profile.yaml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/src/pretext/prompt.py +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/src/pretext/render.py +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/conftest.py +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/fixtures/corpus/deepseek-v4-1-flash.json +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/fixtures/corpus/venice-uncensored-1-2.json +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/fixtures/profiles/exec-assistant.yaml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/fixtures/profiles/finance.yaml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/fixtures/profiles/helpdesk.yaml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/fixtures/profiles/new-hire.yaml +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/fixtures/responses/dana.json +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/test_corpus.py +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/test_models.py +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/test_prompt.py +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/test_render.py +0 -0
- {pretext_sim-0.3.2 → pretext_sim-0.3.3}/tests/test_scripts.py +0 -0
|
@@ -54,7 +54,9 @@ resolves.
|
|
|
54
54
|
|
|
55
55
|
`doctor` runs its probes through `Client.list_models` and `Client.ping_json` so they
|
|
56
56
|
stay testable via `MockTransport`; `--full` is the only part that calls `generate`.
|
|
57
|
-
`batch`
|
|
57
|
+
`batch` runs profiles in a bounded thread pool (default 4) and one `Client` per
|
|
58
|
+
profile, because `Client.last_usage` is per-instance. Printing stays on the calling
|
|
59
|
+
thread. `batch` mirrors `generate`'s endpoint flags, skips `_`-prefixed files, isolates
|
|
58
60
|
per-profile failures, and exits 1 if any profile failed. Keep the summary on stderr
|
|
59
61
|
and never abort the run on the first failure.
|
|
60
62
|
|
|
@@ -71,6 +73,8 @@ and never abort the run on the first failure.
|
|
|
71
73
|
`Simulation` forbids extra keys and requires `escalation_path` stages to be
|
|
72
74
|
exactly `1..n`, so a model that invents a `stage 2` gap fails instead of
|
|
73
75
|
producing a broken table.
|
|
76
|
+
- `_read_response` validates both halves of an export inside one `try`; a corrupt
|
|
77
|
+
`profile` key must surface as `ProfileError`, not a raw pydantic traceback.
|
|
74
78
|
- **Errors are `PretextError` subclasses** (`ConfigError`, `ProfileError`,
|
|
75
79
|
`APIError`). The CLI catches them, prints to stderr, exits 1. Do not let raw
|
|
76
80
|
`httpx` or `ValidationError` escape to the user, and do not catch them per
|
|
@@ -100,6 +104,8 @@ and never abort the run on the first failure.
|
|
|
100
104
|
- Empty model content is an error, not an empty simulation. Overlong artifacts are
|
|
101
105
|
trimmed to 300 chars in a `mode="before"` validator; constraints run before
|
|
102
106
|
`mode="after"` validators, so trim early or the run fails.
|
|
107
|
+
- `--out` creates parent directories through `_write_out`; use it rather than calling
|
|
108
|
+
`write_text` directly so a missing directory never tracebacks.
|
|
103
109
|
- `--dry-run` still resolves settings, so it needs a key; it prints the prompt to
|
|
104
110
|
stdout via `typer.echo` (unwrapped) and exits 0. `validate` needs no key.
|
|
105
111
|
- File output with the default rich format renders into a `Console.capture()`
|
|
@@ -127,7 +133,7 @@ the social preview (404 on every version), so it is uploaded in Settings → Gen
|
|
|
127
133
|
## Repo hygiene
|
|
128
134
|
|
|
129
135
|
CI (`.github/workflows/ci.yml`) runs ruff, pytest, and a dry-run smoke on Python
|
|
130
|
-
3.
|
|
136
|
+
3.11-3.13; the local equivalents are `ruff check src tests` and `pytest`. Test
|
|
131
137
|
files are exempt from E501, source files are not.
|
|
132
138
|
|
|
133
139
|
Releasing means bumping `__version__` in `__init__.py` and `version` in
|
|
@@ -150,6 +156,9 @@ without HTTP. Cover the failure path (retry exhaustion, schema violation, bad
|
|
|
150
156
|
key) whenever you touch the client or schema. `tests/test_models.py` owns
|
|
151
157
|
precedence and validation edge cases.
|
|
152
158
|
|
|
159
|
+
`--env-file` swaps the dotenv source; note a `PRETEXT_BASE_URL` in that file beats a
|
|
160
|
+
`--provider` preset, which is what trips up mixed setups.
|
|
161
|
+
|
|
153
162
|
`--from-file` bypasses the API entirely for rendering work. It accepts our own
|
|
154
163
|
export, a bare simulation, or a chat body, and needs no key; the profile is taken
|
|
155
164
|
from the export or must be passed explicitly. `tests/fixtures/responses/dana.json`
|
|
@@ -4,6 +4,18 @@ All notable changes. This project follows semantic versioning.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
+
- `batch` runs profiles concurrently (`-j`, default 4)
|
|
8
|
+
- `--env-file` selects another dotenv, for running two providers side by side
|
|
9
|
+
- Upstream provider error detail is surfaced instead of "Provider returned error"
|
|
10
|
+
- Corpus entries for two OpenRouter models; socket-level transport tests
|
|
11
|
+
|
|
12
|
+
- A corrupt `profile` inside a saved response reports field errors instead of a
|
|
13
|
+
traceback
|
|
14
|
+
- `doctor` suggests close model names, and shows the first schema errors verbatim
|
|
15
|
+
- `batch` reads `.yml` as well as `.yaml`
|
|
16
|
+
- `--out` creates missing parent directories
|
|
17
|
+
- Section parsing lives in one helper instead of two copies
|
|
18
|
+
|
|
7
19
|
- Distribution renamed `pretext-sim` (the `pretext` name belongs to PreTeXt); the
|
|
8
20
|
command stays `pretext`
|
|
9
21
|
- PyPI publish workflow on release, via trusted publishing
|
|
@@ -19,8 +31,14 @@ All notable changes. This project follows semantic versioning.
|
|
|
19
31
|
- `--dry-run --out` writes the prompt instead of ignoring `--out`
|
|
20
32
|
- Verified against Venice (venice-uncensored-1-2, qwen3-235b-a22b-instruct)
|
|
21
33
|
|
|
34
|
+
## 0.3.3
|
|
35
|
+
|
|
36
|
+
- Correct install instructions: `uvx` needs `--from pretext-sim`
|
|
37
|
+
|
|
22
38
|
## 0.3.2
|
|
23
39
|
|
|
40
|
+
- Dropped Python 3.10 (end of life) so the lockfile can use a patched `filelock`
|
|
41
|
+
|
|
24
42
|
- PyPI upload runs inside the release workflow
|
|
25
43
|
|
|
26
44
|
## 0.3.1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: pretext-sim
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Generate labeled social-engineering simulations for security training
|
|
5
5
|
Author: pretext contributors
|
|
6
6
|
License: MIT
|
|
@@ -11,12 +11,11 @@ Classifier: Environment :: Console
|
|
|
11
11
|
Classifier: Intended Audience :: Information Technology
|
|
12
12
|
Classifier: License :: OSI Approved :: MIT License
|
|
13
13
|
Classifier: Programming Language :: Python :: 3
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
17
|
Classifier: Topic :: Security
|
|
19
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.11
|
|
20
19
|
Requires-Dist: httpx>=0.27
|
|
21
20
|
Requires-Dist: pydantic>=2.6
|
|
22
21
|
Requires-Dist: pyyaml>=6.0
|
|
@@ -40,7 +39,7 @@ Narrative, opening message, escalation path, and the red flags defenders should
|
|
|
40
39
|
have caught, generated, validated, then rendered where you need it.
|
|
41
40
|
|
|
42
41
|
[](https://github.com/sjdsjadjajdjsa/pretext/actions/workflows/ci.yml)
|
|
43
|
-

|
|
44
43
|

|
|
45
44
|
|
|
46
45
|
<img src="docs/demo.png" alt="pretext rendering a simulation in the terminal" width="820">
|
|
@@ -82,12 +81,12 @@ complete Markdown export; [`docs/demo.png`](docs/demo.png) is the terminal view.
|
|
|
82
81
|
## Install
|
|
83
82
|
|
|
84
83
|
```sh
|
|
85
|
-
uvx pretext-sim generate persona.yaml # no install
|
|
86
|
-
pipx install pretext-sim
|
|
84
|
+
uvx --from pretext-sim pretext generate persona.yaml # no install
|
|
85
|
+
pipx install pretext-sim # or a permanent command
|
|
87
86
|
```
|
|
88
87
|
|
|
89
|
-
Python 3.
|
|
90
|
-
source tree: `uv sync --extra dev` then `uv run pretext`.
|
|
88
|
+
Python 3.11+. The distribution is `pretext-sim`; the command is `pretext`, so `uvx`
|
|
89
|
+
needs `--from`. For the source tree: `uv sync --extra dev` then `uv run pretext`.
|
|
91
90
|
|
|
92
91
|
## Quickstart
|
|
93
92
|
|
|
@@ -146,20 +145,22 @@ rejected so typos surface early. Fixture personas live in
|
|
|
146
145
|
| `schema` | Print the JSON Schema model output must satisfy (`--all` adds the profile) |
|
|
147
146
|
| `config` | Show resolved settings, key masked |
|
|
148
147
|
|
|
149
|
-
Flags on `generate
|
|
148
|
+
Flags on `generate` (endpoint, `--env-file`, `--sections` and `--format` also apply
|
|
149
|
+
to `batch`):
|
|
150
150
|
|
|
151
151
|
| Flag | Effect |
|
|
152
152
|
| ---- | ------ |
|
|
153
153
|
| `--channel` | Force the opening channel (email, phone, sms, chat, in_person, linkedin) |
|
|
154
154
|
| `--extra` | Extra direction for the model |
|
|
155
155
|
| `--format` | `rich` (default), `markdown`, `json` |
|
|
156
|
-
| `--out` | Write to a file; plain text unless a format is chosen |
|
|
156
|
+
| `--out` | Write to a file, creating parents; plain text unless a format is chosen |
|
|
157
157
|
| `--sections` | Render only some of `pretext,opening,escalation,red_flags` |
|
|
158
158
|
| `--from-file` | Replay a saved response; no API call, no key |
|
|
159
159
|
| `--dry-run` | Print resolved config and prompt, send nothing; `--out` writes the prompt |
|
|
160
160
|
| `--usage` | Token counts on stderr |
|
|
161
161
|
| `--base-url`, `--model`, `--api-key` | Override endpoint settings |
|
|
162
162
|
| `--temperature`, `--max-tokens`, `--timeout`, `--retries`, `--seed` | Sampling and transport |
|
|
163
|
+
| `--env-file` | Read a different dotenv file (default `./.env`) |
|
|
163
164
|
| `--no-color` | Plain terminal output |
|
|
164
165
|
|
|
165
166
|
Exit codes: `0` success, `1` bad profile/config/API/schema failure, `2` usage error,
|
|
@@ -187,7 +188,8 @@ continuing past failures:
|
|
|
187
188
|
pretext batch profiles/ --out-dir packet/ --format markdown
|
|
188
189
|
```
|
|
189
190
|
|
|
190
|
-
|
|
191
|
+
Profiles run four at a time by default (`-j/--concurrency`, max 8); progress and a
|
|
192
|
+
`wrote 7/9` summary go to stderr, the failure reasons are listed, and
|
|
191
193
|
the exit code is `1` if anything failed. Endpoint flags are the same as `generate`;
|
|
192
194
|
`--sections`, `--format` and `--usage` apply to the run as a whole.
|
|
193
195
|
|
|
@@ -197,7 +199,7 @@ First hit wins:
|
|
|
197
199
|
|
|
198
200
|
1. Flags (`--base-url`, `--model`, `--api-key`, `--provider`)
|
|
199
201
|
2. Environment: the same names under `PRETEXT_` (`PRETEXT_API_KEY`, `PRETEXT_MODEL`, ...)
|
|
200
|
-
3. `.env` in the working directory
|
|
202
|
+
3. `.env` in the working directory, or the file given to `--env-file`
|
|
201
203
|
4. `OPENAI_API_KEY` for the key, then `https://api.openai.com/v1` and `gpt-4o-mini`
|
|
202
204
|
|
|
203
205
|
The key is never printed and never written to output.
|
|
@@ -8,7 +8,7 @@ Narrative, opening message, escalation path, and the red flags defenders should
|
|
|
8
8
|
have caught, generated, validated, then rendered where you need it.
|
|
9
9
|
|
|
10
10
|
[](https://github.com/sjdsjadjajdjsa/pretext/actions/workflows/ci.yml)
|
|
11
|
-

|
|
12
12
|

|
|
13
13
|
|
|
14
14
|
<img src="docs/demo.png" alt="pretext rendering a simulation in the terminal" width="820">
|
|
@@ -50,12 +50,12 @@ complete Markdown export; [`docs/demo.png`](docs/demo.png) is the terminal view.
|
|
|
50
50
|
## Install
|
|
51
51
|
|
|
52
52
|
```sh
|
|
53
|
-
uvx pretext-sim generate persona.yaml # no install
|
|
54
|
-
pipx install pretext-sim
|
|
53
|
+
uvx --from pretext-sim pretext generate persona.yaml # no install
|
|
54
|
+
pipx install pretext-sim # or a permanent command
|
|
55
55
|
```
|
|
56
56
|
|
|
57
|
-
Python 3.
|
|
58
|
-
source tree: `uv sync --extra dev` then `uv run pretext`.
|
|
57
|
+
Python 3.11+. The distribution is `pretext-sim`; the command is `pretext`, so `uvx`
|
|
58
|
+
needs `--from`. For the source tree: `uv sync --extra dev` then `uv run pretext`.
|
|
59
59
|
|
|
60
60
|
## Quickstart
|
|
61
61
|
|
|
@@ -114,20 +114,22 @@ rejected so typos surface early. Fixture personas live in
|
|
|
114
114
|
| `schema` | Print the JSON Schema model output must satisfy (`--all` adds the profile) |
|
|
115
115
|
| `config` | Show resolved settings, key masked |
|
|
116
116
|
|
|
117
|
-
Flags on `generate
|
|
117
|
+
Flags on `generate` (endpoint, `--env-file`, `--sections` and `--format` also apply
|
|
118
|
+
to `batch`):
|
|
118
119
|
|
|
119
120
|
| Flag | Effect |
|
|
120
121
|
| ---- | ------ |
|
|
121
122
|
| `--channel` | Force the opening channel (email, phone, sms, chat, in_person, linkedin) |
|
|
122
123
|
| `--extra` | Extra direction for the model |
|
|
123
124
|
| `--format` | `rich` (default), `markdown`, `json` |
|
|
124
|
-
| `--out` | Write to a file; plain text unless a format is chosen |
|
|
125
|
+
| `--out` | Write to a file, creating parents; plain text unless a format is chosen |
|
|
125
126
|
| `--sections` | Render only some of `pretext,opening,escalation,red_flags` |
|
|
126
127
|
| `--from-file` | Replay a saved response; no API call, no key |
|
|
127
128
|
| `--dry-run` | Print resolved config and prompt, send nothing; `--out` writes the prompt |
|
|
128
129
|
| `--usage` | Token counts on stderr |
|
|
129
130
|
| `--base-url`, `--model`, `--api-key` | Override endpoint settings |
|
|
130
131
|
| `--temperature`, `--max-tokens`, `--timeout`, `--retries`, `--seed` | Sampling and transport |
|
|
132
|
+
| `--env-file` | Read a different dotenv file (default `./.env`) |
|
|
131
133
|
| `--no-color` | Plain terminal output |
|
|
132
134
|
|
|
133
135
|
Exit codes: `0` success, `1` bad profile/config/API/schema failure, `2` usage error,
|
|
@@ -155,7 +157,8 @@ continuing past failures:
|
|
|
155
157
|
pretext batch profiles/ --out-dir packet/ --format markdown
|
|
156
158
|
```
|
|
157
159
|
|
|
158
|
-
|
|
160
|
+
Profiles run four at a time by default (`-j/--concurrency`, max 8); progress and a
|
|
161
|
+
`wrote 7/9` summary go to stderr, the failure reasons are listed, and
|
|
159
162
|
the exit code is `1` if anything failed. Endpoint flags are the same as `generate`;
|
|
160
163
|
`--sections`, `--format` and `--usage` apply to the run as a whole.
|
|
161
164
|
|
|
@@ -165,7 +168,7 @@ First hit wins:
|
|
|
165
168
|
|
|
166
169
|
1. Flags (`--base-url`, `--model`, `--api-key`, `--provider`)
|
|
167
170
|
2. Environment: the same names under `PRETEXT_` (`PRETEXT_API_KEY`, `PRETEXT_MODEL`, ...)
|
|
168
|
-
3. `.env` in the working directory
|
|
171
|
+
3. `.env` in the working directory, or the file given to `--env-file`
|
|
169
172
|
4. `OPENAI_API_KEY` for the key, then `https://api.openai.com/v1` and `gpt-4o-mini`
|
|
170
173
|
|
|
171
174
|
The key is never printed and never written to output.
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "pretext-sim"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.3"
|
|
4
4
|
description = "Generate labeled social-engineering simulations for security training"
|
|
5
5
|
readme = "README.md"
|
|
6
|
-
requires-python = ">=3.
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
7
|
license = { text = "MIT" }
|
|
8
8
|
authors = [{ name = "pretext contributors" }]
|
|
9
9
|
keywords = ["security", "security-awareness", "phishing", "red-team", "cli", "training"]
|
|
@@ -13,7 +13,6 @@ classifiers = [
|
|
|
13
13
|
"Intended Audience :: Information Technology",
|
|
14
14
|
"License :: OSI Approved :: MIT License",
|
|
15
15
|
"Programming Language :: Python :: 3",
|
|
16
|
-
"Programming Language :: Python :: 3.10",
|
|
17
16
|
"Programming Language :: Python :: 3.11",
|
|
18
17
|
"Programming Language :: Python :: 3.12",
|
|
19
18
|
"Programming Language :: Python :: 3.13",
|
|
@@ -54,7 +53,7 @@ addopts = "-q"
|
|
|
54
53
|
source = ["pretext"]
|
|
55
54
|
|
|
56
55
|
[tool.mypy]
|
|
57
|
-
python_version = "3.
|
|
56
|
+
python_version = "3.11"
|
|
58
57
|
files = ["src"]
|
|
59
58
|
warn_unused_ignores = true
|
|
60
59
|
warn_redundant_casts = true
|
|
@@ -67,7 +66,7 @@ ignore_missing_imports = true
|
|
|
67
66
|
|
|
68
67
|
[tool.ruff]
|
|
69
68
|
line-length = 100
|
|
70
|
-
target-version = "
|
|
69
|
+
target-version = "py311"
|
|
71
70
|
|
|
72
71
|
[tool.ruff.lint]
|
|
73
72
|
ignore = ["B008"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Capture model outputs into the regression corpus.
|
|
2
2
|
|
|
3
|
-
Usage: uv run python scripts/capture_corpus.py MODEL
|
|
3
|
+
Usage: [PRETEXT_ENV_FILE=.env.other] uv run python scripts/capture_corpus.py MODEL ...
|
|
4
4
|
|
|
5
5
|
Needs a configured endpoint and key (see README). The corpus is checked in so the
|
|
6
6
|
suite stays offline; re-run this after a prompt change and read the diff.
|
|
@@ -9,6 +9,7 @@ suite stays offline; re-run this after a prompt change and read the diff.
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
+
import os
|
|
12
13
|
import sys
|
|
13
14
|
from pathlib import Path
|
|
14
15
|
|
|
@@ -28,8 +29,13 @@ def main(models: list[str]) -> int:
|
|
|
28
29
|
CORPUS.mkdir(parents=True, exist_ok=True)
|
|
29
30
|
profile = load_profile(PROFILE)
|
|
30
31
|
|
|
32
|
+
env_file = os.environ.get("PRETEXT_ENV_FILE")
|
|
31
33
|
for model in models:
|
|
32
|
-
settings = resolve_settings(
|
|
34
|
+
settings = resolve_settings(
|
|
35
|
+
model=model,
|
|
36
|
+
max_tokens=12000,
|
|
37
|
+
dotenv=Path(env_file) if env_file else Path.cwd() / ".env",
|
|
38
|
+
)
|
|
33
39
|
simulation = Client(settings).generate(profile)
|
|
34
40
|
target = CORPUS / f"{model.replace('/', '__')}.json"
|
|
35
41
|
target.write_text(
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import difflib
|
|
5
6
|
import json
|
|
6
7
|
import sys
|
|
7
|
-
from
|
|
8
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
9
|
+
from enum import StrEnum
|
|
8
10
|
from importlib import resources
|
|
9
11
|
from pathlib import Path
|
|
10
12
|
|
|
@@ -52,7 +54,7 @@ def root(
|
|
|
52
54
|
"""Labeled social-engineering simulations for authorized training."""
|
|
53
55
|
|
|
54
56
|
|
|
55
|
-
class OutputFormat(
|
|
57
|
+
class OutputFormat(StrEnum):
|
|
56
58
|
rich = "rich"
|
|
57
59
|
json = "json"
|
|
58
60
|
markdown = "markdown"
|
|
@@ -77,20 +79,41 @@ def _read_response(path: Path) -> tuple[Profile | None, Simulation]:
|
|
|
77
79
|
if not isinstance(document, dict):
|
|
78
80
|
raise ProfileError(f"response {path} must be a JSON object")
|
|
79
81
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
82
|
+
try:
|
|
83
|
+
profile = (
|
|
84
|
+
Profile.model_validate(document["profile"])
|
|
85
|
+
if isinstance(document.get("profile"), dict)
|
|
86
|
+
else None
|
|
87
|
+
)
|
|
83
88
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
89
|
+
raw = document.get("simulation", document)
|
|
90
|
+
if isinstance(document.get("choices"), list):
|
|
91
|
+
raw = extract_json(_content_of(document))
|
|
87
92
|
|
|
88
|
-
try:
|
|
89
93
|
return profile, Simulation.model_validate(raw)
|
|
90
94
|
except ValidationError as exc:
|
|
91
95
|
raise ProfileError(_format_validation(exc, prefix=f"invalid response {path}")) from exc
|
|
92
96
|
|
|
93
97
|
|
|
98
|
+
def _write_out(path: Path, body: str) -> None:
|
|
99
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
100
|
+
path.write_text(body, encoding="utf-8")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _parse_sections(raw: str | None, err) -> tuple[str, ...]:
|
|
104
|
+
if raw is None:
|
|
105
|
+
return render.SECTIONS
|
|
106
|
+
chosen = tuple(part.strip() for part in raw.split(",") if part.strip())
|
|
107
|
+
unknown = [part for part in chosen if part not in render.SECTIONS]
|
|
108
|
+
if unknown or not chosen:
|
|
109
|
+
err.print(
|
|
110
|
+
f"[bold red]error[/bold red] unknown section(s): {', '.join(unknown) or 'none'}; "
|
|
111
|
+
f"choose from {', '.join(render.SECTIONS)}"
|
|
112
|
+
)
|
|
113
|
+
raise typer.Exit(2)
|
|
114
|
+
return chosen
|
|
115
|
+
|
|
116
|
+
|
|
94
117
|
def _warnings(settings, profile=None) -> list[str]:
|
|
95
118
|
warnings: list[str] = []
|
|
96
119
|
if settings.base_url.scheme != "https":
|
|
@@ -123,6 +146,9 @@ def generate(
|
|
|
123
146
|
"-p",
|
|
124
147
|
help=f"Preset base URL: {', '.join(PROVIDERS)}. --base-url wins.",
|
|
125
148
|
),
|
|
149
|
+
env_file: Path = typer.Option(
|
|
150
|
+
None, "--env-file", help="Dotenv file to read instead of ./.env."
|
|
151
|
+
),
|
|
126
152
|
channel: Channel | None = typer.Option(
|
|
127
153
|
None, "--channel", "-c", help="Force the opening channel."
|
|
128
154
|
),
|
|
@@ -155,16 +181,7 @@ def generate(
|
|
|
155
181
|
out = render.console(no_color=no_color)
|
|
156
182
|
err = render.console(no_color=no_color, stderr=True)
|
|
157
183
|
|
|
158
|
-
selected
|
|
159
|
-
if sections is not None:
|
|
160
|
-
selected = tuple(part.strip() for part in sections.split(",") if part.strip())
|
|
161
|
-
unknown = [part for part in selected if part not in render.SECTIONS]
|
|
162
|
-
if unknown or not selected:
|
|
163
|
-
err.print(
|
|
164
|
-
f"[bold red]error[/bold red] unknown section(s): {', '.join(unknown) or 'none'}; "
|
|
165
|
-
f"choose from {', '.join(render.SECTIONS)}"
|
|
166
|
-
)
|
|
167
|
-
raise typer.Exit(2)
|
|
184
|
+
selected = _parse_sections(sections, err)
|
|
168
185
|
|
|
169
186
|
if profile is None and response_file is None:
|
|
170
187
|
err.print("[bold red]error[/bold red] pass a PROFILE or --from-file")
|
|
@@ -192,7 +209,7 @@ def generate(
|
|
|
192
209
|
retries=retries,
|
|
193
210
|
max_tokens=max_tokens,
|
|
194
211
|
seed=seed,
|
|
195
|
-
dotenv=Path.cwd() / ".env",
|
|
212
|
+
dotenv=env_file or Path.cwd() / ".env",
|
|
196
213
|
)
|
|
197
214
|
except PretextError as exc:
|
|
198
215
|
err.print(f"[bold red]error[/bold red] {exc}")
|
|
@@ -210,7 +227,7 @@ def generate(
|
|
|
210
227
|
err.print()
|
|
211
228
|
typer.echo(prompt)
|
|
212
229
|
else:
|
|
213
|
-
out_file
|
|
230
|
+
_write_out(out_file, prompt + "\n")
|
|
214
231
|
err.print(f"wrote prompt to {out_file}")
|
|
215
232
|
raise typer.Exit(0)
|
|
216
233
|
|
|
@@ -254,7 +271,7 @@ def generate(
|
|
|
254
271
|
with recording.capture() as captured:
|
|
255
272
|
render.render_simulation(simulation, prof, recording, sections=selected)
|
|
256
273
|
body = captured.get()
|
|
257
|
-
out_file
|
|
274
|
+
_write_out(out_file, body)
|
|
258
275
|
err.print(f"wrote {out_file}")
|
|
259
276
|
return
|
|
260
277
|
|
|
@@ -289,13 +306,16 @@ def example(
|
|
|
289
306
|
if out_file is None:
|
|
290
307
|
typer.echo(text, nl=False)
|
|
291
308
|
else:
|
|
292
|
-
out_file
|
|
309
|
+
_write_out(out_file, text)
|
|
293
310
|
typer.echo(f"wrote {out_file}")
|
|
294
311
|
|
|
295
312
|
|
|
296
313
|
@app.command("config")
|
|
297
314
|
def show_config(
|
|
298
315
|
provider: str = typer.Option(None, "--provider", "-p", help=f"Preset: {', '.join(PROVIDERS)}."),
|
|
316
|
+
env_file: Path = typer.Option(
|
|
317
|
+
None, "--env-file", help="Dotenv file to read instead of ./.env."
|
|
318
|
+
),
|
|
299
319
|
base_url: str = typer.Option(None, "--base-url", "-u"),
|
|
300
320
|
model: str = typer.Option(None, "--model", "-m"),
|
|
301
321
|
api_key: str = typer.Option(None, "--api-key", "-k"),
|
|
@@ -310,7 +330,7 @@ def show_config(
|
|
|
310
330
|
base_url=base_url,
|
|
311
331
|
model=model,
|
|
312
332
|
api_key=api_key,
|
|
313
|
-
dotenv=Path.cwd() / ".env",
|
|
333
|
+
dotenv=env_file or Path.cwd() / ".env",
|
|
314
334
|
)
|
|
315
335
|
except ConfigError as exc:
|
|
316
336
|
err.print(f"[bold red]error[/bold red] {exc}")
|
|
@@ -347,32 +367,33 @@ def batch(
|
|
|
347
367
|
"-p",
|
|
348
368
|
help=f"Preset base URL: {', '.join(PROVIDERS)}. --base-url wins.",
|
|
349
369
|
),
|
|
370
|
+
env_file: Path = typer.Option(
|
|
371
|
+
None, "--env-file", help="Dotenv file to read instead of ./.env."
|
|
372
|
+
),
|
|
350
373
|
temperature: float = typer.Option(None, "--temperature", "-t", min=0.0, max=2.0),
|
|
351
374
|
max_tokens: int = typer.Option(None, "--max-tokens", min=256, max=16000),
|
|
352
375
|
timeout: float = typer.Option(None, "--timeout", min=1.0, max=600.0),
|
|
353
376
|
retries: int = typer.Option(None, "--retries", min=0, max=8),
|
|
354
377
|
seed: int = typer.Option(None, "--seed"),
|
|
355
378
|
usage: bool = typer.Option(False, "--usage", help="Print token totals to stderr."),
|
|
379
|
+
concurrency: int = typer.Option(
|
|
380
|
+
4, "--concurrency", "-j", min=1, max=8, help="Profiles in flight at once."
|
|
381
|
+
),
|
|
356
382
|
no_color: bool = typer.Option(False, "--no-color"),
|
|
357
383
|
) -> None:
|
|
358
384
|
"""Generate one exercise per profile in a directory."""
|
|
359
385
|
|
|
360
386
|
err = render.console(no_color=no_color, stderr=True)
|
|
361
|
-
selected
|
|
362
|
-
if sections is not None:
|
|
363
|
-
selected = tuple(part.strip() for part in sections.split(",") if part.strip())
|
|
364
|
-
unknown = [part for part in selected if part not in render.SECTIONS]
|
|
365
|
-
if unknown or not selected:
|
|
366
|
-
err.print(f"[bold red]error[/bold red] unknown section(s): {', '.join(unknown)}")
|
|
367
|
-
raise typer.Exit(2)
|
|
387
|
+
selected = _parse_sections(sections, err)
|
|
368
388
|
|
|
369
389
|
if not directory.is_dir():
|
|
370
390
|
err.print(f"[bold red]error[/bold red] {directory} is not a directory")
|
|
371
391
|
raise typer.Exit(1)
|
|
372
392
|
|
|
373
|
-
|
|
393
|
+
candidates = list(directory.glob("*.yaml")) + list(directory.glob("*.yml"))
|
|
394
|
+
profiles = sorted(p for p in candidates if not p.name.startswith("_"))
|
|
374
395
|
if not profiles:
|
|
375
|
-
err.print(f"[bold red]error[/bold red] no *.yaml profiles in {directory}")
|
|
396
|
+
err.print(f"[bold red]error[/bold red] no *.yaml or *.yml profiles in {directory}")
|
|
376
397
|
raise typer.Exit(1)
|
|
377
398
|
|
|
378
399
|
try:
|
|
@@ -386,32 +407,26 @@ def batch(
|
|
|
386
407
|
retries=retries,
|
|
387
408
|
max_tokens=max_tokens,
|
|
388
409
|
seed=seed,
|
|
389
|
-
dotenv=Path.cwd() / ".env",
|
|
410
|
+
dotenv=env_file or Path.cwd() / ".env",
|
|
390
411
|
)
|
|
391
412
|
except PretextError as exc:
|
|
392
413
|
err.print(f"[bold red]error[/bold red] {exc}")
|
|
393
414
|
raise typer.Exit(1) from exc
|
|
394
415
|
|
|
395
416
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
396
|
-
client = Client(settings)
|
|
397
417
|
suffix = {"markdown": "md", "json": "json", "rich": "txt"}[fmt.value]
|
|
398
418
|
failures: list[tuple[str, str]] = []
|
|
399
419
|
totals = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
|
420
|
+
done = 0
|
|
400
421
|
|
|
401
|
-
|
|
402
|
-
label = f"[cyan]{index}/{len(profiles)}[/cyan] {path.name}"
|
|
422
|
+
def run_one(path: Path) -> tuple[Path, str | None, dict[str, int]]:
|
|
403
423
|
try:
|
|
404
424
|
prof = load_profile(path)
|
|
425
|
+
client = Client(settings)
|
|
405
426
|
simulation = client.generate(prof, channel=None, extra=extra)
|
|
406
427
|
except PretextError as exc:
|
|
407
|
-
|
|
408
|
-
err.print(f"{label} [red]failed[/red] {failures[-1][1]}")
|
|
409
|
-
continue
|
|
428
|
+
return path, str(exc).splitlines()[0], {}
|
|
410
429
|
|
|
411
|
-
for key in totals:
|
|
412
|
-
totals[key] += (client.last_usage or {}).get(key, 0)
|
|
413
|
-
|
|
414
|
-
target = out_dir / f"{path.stem}.{suffix}"
|
|
415
430
|
if fmt is OutputFormat.json:
|
|
416
431
|
body = render.to_json(simulation, prof, sections=selected)
|
|
417
432
|
elif fmt is OutputFormat.markdown:
|
|
@@ -421,8 +436,23 @@ def batch(
|
|
|
421
436
|
with recording.capture() as captured:
|
|
422
437
|
render.render_simulation(simulation, prof, recording, sections=selected)
|
|
423
438
|
body = captured.get()
|
|
424
|
-
|
|
425
|
-
|
|
439
|
+
_write_out(out_dir / f"{path.stem}.{suffix}", body)
|
|
440
|
+
return path, None, client.last_usage or {}
|
|
441
|
+
|
|
442
|
+
workers = max(1, min(concurrency, len(profiles)))
|
|
443
|
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
444
|
+
futures = {pool.submit(run_one, path): path for path in profiles}
|
|
445
|
+
for future in as_completed(futures):
|
|
446
|
+
path, error, used = future.result()
|
|
447
|
+
done += 1
|
|
448
|
+
label = f"[cyan]{done}/{len(profiles)}[/cyan] {path.name}"
|
|
449
|
+
if error is not None:
|
|
450
|
+
failures.append((path.name, error))
|
|
451
|
+
err.print(f"{label} [red]failed[/red] {error}")
|
|
452
|
+
continue
|
|
453
|
+
for key in totals:
|
|
454
|
+
totals[key] += used.get(key, 0)
|
|
455
|
+
err.print(f"{label} [green]ok[/green] {path.stem}.{suffix}")
|
|
426
456
|
|
|
427
457
|
err.print()
|
|
428
458
|
err.print(f"wrote {len(profiles) - len(failures)}/{len(profiles)} to {out_dir}")
|
|
@@ -452,6 +482,9 @@ def doctor(
|
|
|
452
482
|
"-p",
|
|
453
483
|
help=f"Preset base URL: {', '.join(PROVIDERS)}. --base-url wins.",
|
|
454
484
|
),
|
|
485
|
+
env_file: Path = typer.Option(
|
|
486
|
+
None, "--env-file", help="Dotenv file to read instead of ./.env."
|
|
487
|
+
),
|
|
455
488
|
full: bool = typer.Option(False, "--full", help="Also run one real simulation."),
|
|
456
489
|
no_color: bool = typer.Option(False, "--no-color"),
|
|
457
490
|
) -> None:
|
|
@@ -470,7 +503,7 @@ def doctor(
|
|
|
470
503
|
base_url=base_url,
|
|
471
504
|
model=model,
|
|
472
505
|
api_key=api_key,
|
|
473
|
-
dotenv=Path.cwd() / ".env",
|
|
506
|
+
dotenv=env_file or Path.cwd() / ".env",
|
|
474
507
|
)
|
|
475
508
|
record("configuration", "pass", f"{settings.base_url} · {settings.model}")
|
|
476
509
|
except PretextError as exc:
|
|
@@ -492,7 +525,7 @@ def doctor(
|
|
|
492
525
|
elif settings.model in models:
|
|
493
526
|
record("model", "pass", f"{settings.model} is served")
|
|
494
527
|
else:
|
|
495
|
-
near =
|
|
528
|
+
near = difflib.get_close_matches(settings.model, models, n=3, cutoff=0.5)
|
|
496
529
|
hint = f"; did you mean {', '.join(near)}" if near else ""
|
|
497
530
|
record("model", "fail", f"{settings.model} is not in the model list{hint}")
|
|
498
531
|
except PretextError as exc:
|
|
@@ -519,7 +552,7 @@ def doctor(
|
|
|
519
552
|
f"{len(simulation.escalation_path)} stages, {len(simulation.red_flags)} flags",
|
|
520
553
|
)
|
|
521
554
|
except PretextError as exc:
|
|
522
|
-
record("schema", "fail", str(exc).splitlines()[
|
|
555
|
+
record("schema", "fail", "; ".join(str(exc).splitlines()[:3]))
|
|
523
556
|
|
|
524
557
|
_render_checks(results, out)
|
|
525
558
|
if any(status == "fail" for _, status, _ in results):
|