scamscript 0.2.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scamscript-0.2.1 → scamscript-0.4.0}/CHANGELOG.md +20 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/PKG-INFO +16 -11
- {scamscript-0.2.1 → scamscript-0.4.0}/README.md +15 -10
- {scamscript-0.2.1 → scamscript-0.4.0}/pyproject.toml +1 -1
- {scamscript-0.2.1 → scamscript-0.4.0}/scripts/calibrate.py +5 -3
- {scamscript-0.2.1 → scamscript-0.4.0}/scripts/injection_eval.py +5 -3
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/__init__.py +1 -1
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/analysis.py +2 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/cli.py +130 -41
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/client.py +23 -6
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/config.py +25 -7
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/render.py +7 -2
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/report.py +14 -1
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_cli.py +99 -1
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_client.py +33 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_render.py +6 -1
- {scamscript-0.2.1 → scamscript-0.4.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/.github/SECURITY.md +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/.github/pull_request_template.md +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/.github/workflows/ci.yml +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/.github/workflows/release.yml +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/.gitignore +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/CITATION.cff +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/CODE_OF_CONDUCT.md +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/CONTRIBUTING.md +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/LICENSE +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/docs/banner.svg +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/docs/calibration.md +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/docs/demo.gif +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/docs/sample-report.md +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/docs/social-preview.png +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/adversarial/fake_output_json.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/adversarial/fence_escape.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/adversarial/fixed_attack_type.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/adversarial/non_json_demand.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/adversarial/print_system_prompt.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/adversarial/role_switch.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/adversarial/score_override.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/crude_robocall.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/gov_impersonation_vishing.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/prompt_injection_script.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/romance_scam_sms.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/sophisticated_wire_fraud.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/examples/tech_support_scam.txt +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/__main__.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/ingest.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/models.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/prompts.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/redact.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/store.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/src/scamscript/verify.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_analysis.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_ingest.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_models.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_prompts.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_redact.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_report.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_store.py +0 -0
- {scamscript-0.2.1 → scamscript-0.4.0}/tests/test_verify.py +0 -0
|
@@ -5,6 +5,26 @@ schema are the things that count as public API.
|
|
|
5
5
|
|
|
6
6
|
## Unreleased
|
|
7
7
|
|
|
8
|
+
## 0.4.0 - 2026-11-02
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `--concurrency N` (1-16) with order-preserving output.
|
|
13
|
+
- `--max-cost USD`: stop scheduling once reported spend reaches the cap, mark the rest skipped, and
|
|
14
|
+
warn when a provider reports no cost.
|
|
15
|
+
- Per-run reported spend in output and logs.
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
|
|
19
|
+
- Token usage is tracked per thread so parallel calls cannot read each other's cost.
|
|
20
|
+
|
|
21
|
+
## 0.3.0 - 2026-11-02
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
|
|
25
|
+
- Breaking: no default provider or model. `--base-url` and `--model` (or `SCAMSCRIPT_BASE_URL` /
|
|
26
|
+
`SCAMSCRIPT_MODEL`) are required for API calls; `--dry-run` works without them.
|
|
27
|
+
|
|
8
28
|
## 0.2.1 - 2026-11-02
|
|
9
29
|
|
|
10
30
|
### Fixed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scamscript
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Reverse-engineer scam scripts and vishing transcripts into structured counter-playbooks
|
|
5
5
|
Project-URL: Homepage, https://github.com/sjdsjadjajdjsa/scamscript
|
|
6
6
|
Project-URL: Repository, https://github.com/sjdsjadjajdjsa/scamscript
|
|
@@ -76,18 +76,22 @@ mechanics instead of general advice.
|
|
|
76
76
|
pip install scamscript # or, for development:
|
|
77
77
|
# git clone https://github.com/sjdsjadjajdjsa/scamscript && cd scamscript && pip install -e ".[dev]"
|
|
78
78
|
|
|
79
|
-
export API_KEY=...
|
|
79
|
+
export API_KEY=...
|
|
80
|
+
export SCAMSCRIPT_BASE_URL=https://openrouter.ai/api/v1
|
|
81
|
+
export SCAMSCRIPT_MODEL=deepseek/deepseek-v4.1-flash-1017
|
|
80
82
|
|
|
81
83
|
scamscript --dry-run analyze --file examples/tech_support_scam.txt # inspect the prompt, spend nothing
|
|
82
84
|
scamscript analyze --file examples/tech_support_scam.txt
|
|
83
85
|
```
|
|
84
86
|
|
|
85
|
-
Any OpenAI-compatible `/chat/completions` endpoint works.
|
|
86
|
-
`
|
|
87
|
+
Any OpenAI-compatible `/chat/completions` endpoint works. There is no baked-in provider or model:
|
|
88
|
+
pass `--base-url` and `--model`, or set `SCAMSCRIPT_BASE_URL` and `SCAMSCRIPT_MODEL`.
|
|
87
89
|
|
|
88
90
|
```bash
|
|
89
91
|
scamscript --base-url https://openrouter.ai/api/v1 \
|
|
90
92
|
--model deepseek/deepseek-v4.1-flash-1017 analyze --file script.txt
|
|
93
|
+
|
|
94
|
+
scamscript --base-url http://localhost:11434/v1 --model llama3.2 analyze --file script.txt
|
|
91
95
|
```
|
|
92
96
|
|
|
93
97
|
## What it produces
|
|
@@ -129,7 +133,7 @@ corpus is at [`docs/sample-report.md`](https://github.com/sjdsjadjajdjsa/scamscr
|
|
|
129
133
|
| `prune` | Delete old logs |
|
|
130
134
|
|
|
131
135
|
Useful flags: `--dry-run`, `--json`, `--redact`, `--top N`, `--recursive`, `--timeout`,
|
|
132
|
-
`--retries`, `--max-tokens`, `--no-color`. `--json` puts one object per script on stdout and all
|
|
136
|
+
`--retries`, `--max-tokens`, `--concurrency`, `--max-cost`, `--no-color`. `--json` puts one object per script on stdout and all
|
|
133
137
|
Rich output on stderr, so it pipes:
|
|
134
138
|
|
|
135
139
|
```bash
|
|
@@ -182,8 +186,8 @@ Release history is in [CHANGELOG.md](https://github.com/sjdsjadjajdjsa/scamscrip
|
|
|
182
186
|
| Setting | Flag | Env | Default |
|
|
183
187
|
| --- | --- | --- | --- |
|
|
184
188
|
| API key | `--api-key` | `API_KEY` | none, required unless `--dry-run` |
|
|
185
|
-
| Base URL | `--base-url` | `SCAMSCRIPT_BASE_URL` |
|
|
186
|
-
| Model | `--model` | `SCAMSCRIPT_MODEL` |
|
|
189
|
+
| Base URL | `--base-url` | `SCAMSCRIPT_BASE_URL` | none, required |
|
|
190
|
+
| Model | `--model` | `SCAMSCRIPT_MODEL` | none, required |
|
|
187
191
|
| State dir | `--home` | `SCAMSCRIPT_HOME` | `~/.scamscript` |
|
|
188
192
|
|
|
189
193
|
## Tests, CI and calibration
|
|
@@ -199,9 +203,9 @@ Calibration is evidence, not vibes. `scripts/calibrate.py` measures first-attemp
|
|
|
199
203
|
adherence, repair rate, latency, score spread and billed cost against a live endpoint:
|
|
200
204
|
|
|
201
205
|
```bash
|
|
202
|
-
API_KEY=...
|
|
203
|
-
|
|
204
|
-
--
|
|
206
|
+
API_KEY=... SCAMSCRIPT_BASE_URL=https://openrouter.ai/api/v1 \
|
|
207
|
+
python scripts/calibrate.py --dir examples --repeat 3 \
|
|
208
|
+
--model deepseek/deepseek-v4.1-flash-1017
|
|
205
209
|
```
|
|
206
210
|
|
|
207
211
|
Recorded passes with their numbers are in [`docs/calibration.md`](https://github.com/sjdsjadjajdjsa/scamscript/blob/master/docs/calibration.md): three rubric
|
|
@@ -212,7 +216,7 @@ take as evidence.
|
|
|
212
216
|
The adversarial set lives in `examples/adversarial/` and has its own harness:
|
|
213
217
|
|
|
214
218
|
```bash
|
|
215
|
-
API_KEY=... python scripts/injection_eval.py --model ...
|
|
219
|
+
API_KEY=... SCAMSCRIPT_BASE_URL=... python scripts/injection_eval.py --model ...
|
|
216
220
|
```
|
|
217
221
|
|
|
218
222
|
Format held in 7/7 cases; embedded demands were obeyed in 1 of 6. The fence protects the schema, not
|
|
@@ -245,6 +249,7 @@ docs/ calibration record, sample report, banner, demo
|
|
|
245
249
|
`opening_hook` and `red_flags` against the source before quoting anything.
|
|
246
250
|
- Redaction is regex-based and will miss names and identifiers without a fixed shape.
|
|
247
251
|
- One script per request; no cross-script deduplication yet, so reprints inflate a ranking.
|
|
252
|
+
- The cost cap depends on provider-reported usage and can overshoot by the in-flight count.
|
|
248
253
|
- Analysis is English-centric: the prompt and schema assume English artifacts.
|
|
249
254
|
- Quote checking compares text, not meaning. A legitimate paraphrase left in quotes is flagged;
|
|
250
255
|
an accurate paraphrase without quotes is skipped rather than judged.
|
|
@@ -40,18 +40,22 @@ mechanics instead of general advice.
|
|
|
40
40
|
pip install scamscript # or, for development:
|
|
41
41
|
# git clone https://github.com/sjdsjadjajdjsa/scamscript && cd scamscript && pip install -e ".[dev]"
|
|
42
42
|
|
|
43
|
-
export API_KEY=...
|
|
43
|
+
export API_KEY=...
|
|
44
|
+
export SCAMSCRIPT_BASE_URL=https://openrouter.ai/api/v1
|
|
45
|
+
export SCAMSCRIPT_MODEL=deepseek/deepseek-v4.1-flash-1017
|
|
44
46
|
|
|
45
47
|
scamscript --dry-run analyze --file examples/tech_support_scam.txt # inspect the prompt, spend nothing
|
|
46
48
|
scamscript analyze --file examples/tech_support_scam.txt
|
|
47
49
|
```
|
|
48
50
|
|
|
49
|
-
Any OpenAI-compatible `/chat/completions` endpoint works.
|
|
50
|
-
`
|
|
51
|
+
Any OpenAI-compatible `/chat/completions` endpoint works. There is no baked-in provider or model:
|
|
52
|
+
pass `--base-url` and `--model`, or set `SCAMSCRIPT_BASE_URL` and `SCAMSCRIPT_MODEL`.
|
|
51
53
|
|
|
52
54
|
```bash
|
|
53
55
|
scamscript --base-url https://openrouter.ai/api/v1 \
|
|
54
56
|
--model deepseek/deepseek-v4.1-flash-1017 analyze --file script.txt
|
|
57
|
+
|
|
58
|
+
scamscript --base-url http://localhost:11434/v1 --model llama3.2 analyze --file script.txt
|
|
55
59
|
```
|
|
56
60
|
|
|
57
61
|
## What it produces
|
|
@@ -93,7 +97,7 @@ corpus is at [`docs/sample-report.md`](https://github.com/sjdsjadjajdjsa/scamscr
|
|
|
93
97
|
| `prune` | Delete old logs |
|
|
94
98
|
|
|
95
99
|
Useful flags: `--dry-run`, `--json`, `--redact`, `--top N`, `--recursive`, `--timeout`,
|
|
96
|
-
`--retries`, `--max-tokens`, `--no-color`. `--json` puts one object per script on stdout and all
|
|
100
|
+
`--retries`, `--max-tokens`, `--concurrency`, `--max-cost`, `--no-color`. `--json` puts one object per script on stdout and all
|
|
97
101
|
Rich output on stderr, so it pipes:
|
|
98
102
|
|
|
99
103
|
```bash
|
|
@@ -146,8 +150,8 @@ Release history is in [CHANGELOG.md](https://github.com/sjdsjadjajdjsa/scamscrip
|
|
|
146
150
|
| Setting | Flag | Env | Default |
|
|
147
151
|
| --- | --- | --- | --- |
|
|
148
152
|
| API key | `--api-key` | `API_KEY` | none, required unless `--dry-run` |
|
|
149
|
-
| Base URL | `--base-url` | `SCAMSCRIPT_BASE_URL` |
|
|
150
|
-
| Model | `--model` | `SCAMSCRIPT_MODEL` |
|
|
153
|
+
| Base URL | `--base-url` | `SCAMSCRIPT_BASE_URL` | none, required |
|
|
154
|
+
| Model | `--model` | `SCAMSCRIPT_MODEL` | none, required |
|
|
151
155
|
| State dir | `--home` | `SCAMSCRIPT_HOME` | `~/.scamscript` |
|
|
152
156
|
|
|
153
157
|
## Tests, CI and calibration
|
|
@@ -163,9 +167,9 @@ Calibration is evidence, not vibes. `scripts/calibrate.py` measures first-attemp
|
|
|
163
167
|
adherence, repair rate, latency, score spread and billed cost against a live endpoint:
|
|
164
168
|
|
|
165
169
|
```bash
|
|
166
|
-
API_KEY=...
|
|
167
|
-
|
|
168
|
-
--
|
|
170
|
+
API_KEY=... SCAMSCRIPT_BASE_URL=https://openrouter.ai/api/v1 \
|
|
171
|
+
python scripts/calibrate.py --dir examples --repeat 3 \
|
|
172
|
+
--model deepseek/deepseek-v4.1-flash-1017
|
|
169
173
|
```
|
|
170
174
|
|
|
171
175
|
Recorded passes with their numbers are in [`docs/calibration.md`](https://github.com/sjdsjadjajdjsa/scamscript/blob/master/docs/calibration.md): three rubric
|
|
@@ -176,7 +180,7 @@ take as evidence.
|
|
|
176
180
|
The adversarial set lives in `examples/adversarial/` and has its own harness:
|
|
177
181
|
|
|
178
182
|
```bash
|
|
179
|
-
API_KEY=... python scripts/injection_eval.py --model ...
|
|
183
|
+
API_KEY=... SCAMSCRIPT_BASE_URL=... python scripts/injection_eval.py --model ...
|
|
180
184
|
```
|
|
181
185
|
|
|
182
186
|
Format held in 7/7 cases; embedded demands were obeyed in 1 of 6. The fence protects the schema, not
|
|
@@ -209,6 +213,7 @@ docs/ calibration record, sample report, banner, demo
|
|
|
209
213
|
`opening_hook` and `red_flags` against the source before quoting anything.
|
|
210
214
|
- Redaction is regex-based and will miss names and identifiers without a fixed shape.
|
|
211
215
|
- One script per request; no cross-script deduplication yet, so reprints inflate a ranking.
|
|
216
|
+
- The cost cap depends on provider-reported usage and can overshoot by the in-flight count.
|
|
212
217
|
- Analysis is English-centric: the prompt and schema assume English artifacts.
|
|
213
218
|
- Quote checking compares text, not meaning. A legitimate paraphrase left in quotes is flagged;
|
|
214
219
|
an accurate paraphrase without quotes is skipped rather than judged.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scamscript"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Reverse-engineer scam scripts and vishing transcripts into structured counter-playbooks"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -21,7 +21,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
|
21
21
|
|
|
22
22
|
from scamscript.analysis import analyze_script # noqa: E402
|
|
23
23
|
from scamscript.client import ChatClient # noqa: E402
|
|
24
|
-
from scamscript.config import
|
|
24
|
+
from scamscript.config import Settings # noqa: E402
|
|
25
25
|
from scamscript.ingest import collect_scripts # noqa: E402
|
|
26
26
|
from scamscript.store import RunLogger # noqa: E402
|
|
27
27
|
|
|
@@ -29,13 +29,15 @@ from scamscript.store import RunLogger # noqa: E402
|
|
|
29
29
|
def main(
|
|
30
30
|
directory: Path = typer.Option(Path("examples"), "--dir", "-d"),
|
|
31
31
|
repeat: int = typer.Option(2, "--repeat", "-n", min=1),
|
|
32
|
-
model: str = typer.Option(
|
|
33
|
-
base_url: str = typer.Option(
|
|
32
|
+
model: str = typer.Option("", "--model", "-m", envvar="SCAMSCRIPT_MODEL"),
|
|
33
|
+
base_url: str = typer.Option("", "--base-url", envvar="SCAMSCRIPT_BASE_URL"),
|
|
34
34
|
prompt_price: float = typer.Option(0.035, help="USD per million prompt tokens."),
|
|
35
35
|
completion_price: float = typer.Option(0.5, help="USD per million completion tokens."),
|
|
36
36
|
max_tokens: int = typer.Option(4096, "--max-tokens"),
|
|
37
37
|
) -> None:
|
|
38
38
|
settings = Settings(base_url=base_url, model=model, max_tokens=max_tokens)
|
|
39
|
+
settings.require_base_url()
|
|
40
|
+
settings.require_model()
|
|
39
41
|
api_key = settings.require_api_key()
|
|
40
42
|
scripts = collect_scripts(directory)
|
|
41
43
|
runs = [script for _ in range(repeat) for script in scripts]
|
|
@@ -22,7 +22,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
|
22
22
|
|
|
23
23
|
from scamscript.analysis import analyze_script # noqa: E402
|
|
24
24
|
from scamscript.client import ChatClient # noqa: E402
|
|
25
|
-
from scamscript.config import
|
|
25
|
+
from scamscript.config import Settings # noqa: E402
|
|
26
26
|
from scamscript.ingest import ScriptFile # noqa: E402
|
|
27
27
|
from scamscript.store import RunLogger # noqa: E402
|
|
28
28
|
|
|
@@ -82,10 +82,12 @@ CASES: dict[str, dict[str, object]] = {
|
|
|
82
82
|
|
|
83
83
|
def main(
|
|
84
84
|
directory: Path = typer.Option(Path("examples/adversarial"), "--dir", "-d"),
|
|
85
|
-
model: str = typer.Option(
|
|
86
|
-
base_url: str = typer.Option(
|
|
85
|
+
model: str = typer.Option("", "--model", "-m", envvar="SCAMSCRIPT_MODEL"),
|
|
86
|
+
base_url: str = typer.Option("", "--base-url", envvar="SCAMSCRIPT_BASE_URL"),
|
|
87
87
|
) -> None:
|
|
88
88
|
settings = Settings(base_url=base_url, model=model)
|
|
89
|
+
settings.require_base_url()
|
|
90
|
+
settings.require_model()
|
|
89
91
|
api_key = settings.require_api_key()
|
|
90
92
|
cases = [(name, spec) for name, spec in CASES.items() if (directory / f"{name}.txt").is_file()]
|
|
91
93
|
|
|
@@ -26,6 +26,7 @@ class Outcome:
|
|
|
26
26
|
error: str | None = None
|
|
27
27
|
duration: float = 0.0
|
|
28
28
|
redactions: dict[str, int] | None = None
|
|
29
|
+
skipped: bool = False
|
|
29
30
|
issues: list[verify.Issue] | None = None
|
|
30
31
|
verification: dict[str, int] | None = None
|
|
31
32
|
|
|
@@ -39,6 +40,7 @@ class Outcome:
|
|
|
39
40
|
"source": str(self.source),
|
|
40
41
|
"name": self.name,
|
|
41
42
|
"ok": self.ok,
|
|
43
|
+
"skipped": self.skipped,
|
|
42
44
|
"attempts": self.attempts,
|
|
43
45
|
"duration_s": round(self.duration, 3),
|
|
44
46
|
"breakdown": self.breakdown.model_dump() if self.breakdown else None,
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
|
+
import threading
|
|
7
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
6
8
|
from dataclasses import dataclass
|
|
7
9
|
from pathlib import Path
|
|
8
10
|
|
|
@@ -15,9 +17,7 @@ from scamscript import report as reporting
|
|
|
15
17
|
from scamscript.analysis import Outcome, analyze_script
|
|
16
18
|
from scamscript.client import ChatClient
|
|
17
19
|
from scamscript.config import (
|
|
18
|
-
DEFAULT_BASE_URL,
|
|
19
20
|
DEFAULT_MAX_TOKENS,
|
|
20
|
-
DEFAULT_MODEL,
|
|
21
21
|
DEFAULT_RETRIES,
|
|
22
22
|
DEFAULT_TIMEOUT,
|
|
23
23
|
ConfigError,
|
|
@@ -66,19 +66,17 @@ def version_callback(value: bool) -> None:
|
|
|
66
66
|
def main(
|
|
67
67
|
ctx: typer.Context,
|
|
68
68
|
base_url: str = typer.Option(
|
|
69
|
-
|
|
69
|
+
"",
|
|
70
70
|
"--base-url",
|
|
71
71
|
envvar="SCAMSCRIPT_BASE_URL",
|
|
72
|
-
help="OpenAI-compatible API base URL.",
|
|
73
|
-
show_default=True,
|
|
72
|
+
help="OpenAI-compatible API base URL. Falls back to SCAMSCRIPT_BASE_URL.",
|
|
74
73
|
),
|
|
75
74
|
model: str = typer.Option(
|
|
76
|
-
|
|
75
|
+
"",
|
|
77
76
|
"--model",
|
|
78
77
|
"-m",
|
|
79
78
|
envvar="SCAMSCRIPT_MODEL",
|
|
80
|
-
help="Model identifier sent to the API.",
|
|
81
|
-
show_default=True,
|
|
79
|
+
help="Model identifier sent to the API. Falls back to SCAMSCRIPT_MODEL.",
|
|
82
80
|
),
|
|
83
81
|
api_key: str | None = typer.Option(
|
|
84
82
|
None,
|
|
@@ -108,6 +106,20 @@ def main(
|
|
|
108
106
|
min=256,
|
|
109
107
|
help="Output token budget per reply.",
|
|
110
108
|
),
|
|
109
|
+
concurrency: int = typer.Option(
|
|
110
|
+
1,
|
|
111
|
+
"--concurrency",
|
|
112
|
+
"-j",
|
|
113
|
+
min=1,
|
|
114
|
+
max=16,
|
|
115
|
+
help="Analyze this many scripts in parallel.",
|
|
116
|
+
),
|
|
117
|
+
max_cost: float | None = typer.Option(
|
|
118
|
+
None,
|
|
119
|
+
"--max-cost",
|
|
120
|
+
min=0.0,
|
|
121
|
+
help="Stop scheduling new scripts once the provider reports this much spend in USD.",
|
|
122
|
+
),
|
|
111
123
|
redact: bool = typer.Option(
|
|
112
124
|
False,
|
|
113
125
|
"--redact",
|
|
@@ -150,6 +162,8 @@ def main(
|
|
|
150
162
|
timeout=timeout,
|
|
151
163
|
retries=retries,
|
|
152
164
|
max_tokens=max_tokens,
|
|
165
|
+
concurrency=concurrency,
|
|
166
|
+
max_cost=max_cost,
|
|
153
167
|
redact=redact,
|
|
154
168
|
home=str(home) if home else "",
|
|
155
169
|
)
|
|
@@ -219,6 +233,11 @@ def _run(
|
|
|
219
233
|
)
|
|
220
234
|
|
|
221
235
|
if state.dry_run:
|
|
236
|
+
if not settings.base_url or not settings.model:
|
|
237
|
+
console.print(
|
|
238
|
+
"[yellow]note[/yellow] --base-url and --model are unset; "
|
|
239
|
+
"the prompt below is complete but unaddressed"
|
|
240
|
+
)
|
|
222
241
|
for script in scripts:
|
|
223
242
|
text, counts = _maybe_redact(script.text, settings.redact)
|
|
224
243
|
messages = prompts.build_messages(script.name, text)
|
|
@@ -246,13 +265,19 @@ def _run(
|
|
|
246
265
|
return
|
|
247
266
|
|
|
248
267
|
try:
|
|
268
|
+
settings.require_base_url()
|
|
269
|
+
settings.require_model()
|
|
249
270
|
api_key = settings.require_api_key()
|
|
250
271
|
except ConfigError as exc:
|
|
251
272
|
_fail(console, str(exc))
|
|
252
273
|
return
|
|
253
274
|
|
|
254
|
-
outcomes: list[Outcome] = []
|
|
275
|
+
outcomes: list[Outcome | None] = [None] * len(scripts)
|
|
255
276
|
report_entries: list[reporting.ReportEntry] = []
|
|
277
|
+
budget = settings.max_cost
|
|
278
|
+
spent = 0.0
|
|
279
|
+
spent_seen = False
|
|
280
|
+
lock = threading.Lock()
|
|
256
281
|
client = ChatClient(
|
|
257
282
|
api_key=api_key,
|
|
258
283
|
base_url=settings.base_url,
|
|
@@ -261,50 +286,114 @@ def _run(
|
|
|
261
286
|
retries=settings.retries,
|
|
262
287
|
max_tokens=settings.max_tokens,
|
|
263
288
|
)
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
289
|
+
|
|
290
|
+
def one(script: ScriptFile) -> Outcome:
|
|
291
|
+
nonlocal spent, spent_seen
|
|
292
|
+
if budget is not None:
|
|
293
|
+
with lock:
|
|
294
|
+
blocked = spent >= budget
|
|
295
|
+
if blocked:
|
|
296
|
+
outcome = Outcome(
|
|
297
|
+
source=script.path,
|
|
298
|
+
name=script.name,
|
|
299
|
+
skipped=True,
|
|
300
|
+
error="skipped: cost budget reached",
|
|
301
|
+
)
|
|
302
|
+
logger.result(**outcome.as_record())
|
|
303
|
+
return outcome
|
|
304
|
+
outcome = analyze_script(
|
|
305
|
+
client,
|
|
306
|
+
script,
|
|
307
|
+
logger,
|
|
308
|
+
redact_identifiers=settings.redact,
|
|
309
|
+
verify_quotes=state.verify_quotes,
|
|
310
|
+
)
|
|
311
|
+
cost = client.last_cost
|
|
312
|
+
if cost:
|
|
313
|
+
with lock:
|
|
314
|
+
spent += cost
|
|
315
|
+
spent_seen = True
|
|
316
|
+
return outcome
|
|
317
|
+
|
|
318
|
+
def emit(outcome: Outcome) -> None:
|
|
319
|
+
if state.json_output:
|
|
320
|
+
_emit_json(outcome.as_record())
|
|
321
|
+
elif outcome.skipped:
|
|
322
|
+
console.print(f"[dim]skipped {outcome.name}: budget reached[/dim]")
|
|
323
|
+
elif outcome.breakdown is not None:
|
|
324
|
+
console.print(
|
|
325
|
+
render_breakdown(
|
|
326
|
+
outcome.breakdown, outcome.name, outcome.attempts, outcome.issues
|
|
269
327
|
)
|
|
270
|
-
outcome = analyze_script(
|
|
271
|
-
client,
|
|
272
|
-
script,
|
|
273
|
-
logger,
|
|
274
|
-
redact_identifiers=settings.redact,
|
|
275
|
-
verify_quotes=state.verify_quotes,
|
|
276
328
|
)
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
329
|
+
else:
|
|
330
|
+
console.print(
|
|
331
|
+
render_failure(outcome.name, outcome.error or "unknown", outcome.attempts)
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
with client:
|
|
335
|
+
if settings.concurrency == 1:
|
|
336
|
+
for index, script in enumerate(scripts, start=1):
|
|
337
|
+
if batch_mode and not state.json_output:
|
|
338
|
+
console.print(
|
|
339
|
+
f"[dim]({index}/{len(scripts)}) analyzing {script.name}...[/dim]"
|
|
288
340
|
)
|
|
289
|
-
)
|
|
290
|
-
|
|
341
|
+
outcome = one(script)
|
|
342
|
+
outcomes[index - 1] = outcome
|
|
343
|
+
report_entries.append(reporting.entry_from_outcome(outcome))
|
|
344
|
+
emit(outcome)
|
|
345
|
+
else:
|
|
346
|
+
if not state.json_output:
|
|
291
347
|
console.print(
|
|
292
|
-
|
|
348
|
+
f"[dim]analyzing {len(scripts)} script(s), "
|
|
349
|
+
f"{settings.concurrency} at a time[/dim]"
|
|
293
350
|
)
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
351
|
+
buffered: dict[int, Outcome] = {}
|
|
352
|
+
ready = 0
|
|
353
|
+
with ThreadPoolExecutor(max_workers=settings.concurrency) as pool:
|
|
354
|
+
futures = {
|
|
355
|
+
pool.submit(one, script): index for index, script in enumerate(scripts)
|
|
356
|
+
}
|
|
357
|
+
for future in as_completed(futures):
|
|
358
|
+
buffered[futures[future]] = future.result()
|
|
359
|
+
while ready in buffered:
|
|
360
|
+
outcome = buffered.pop(ready)
|
|
361
|
+
outcomes[ready] = outcome
|
|
362
|
+
report_entries.append(reporting.entry_from_outcome(outcome))
|
|
363
|
+
emit(outcome)
|
|
364
|
+
ready += 1
|
|
365
|
+
|
|
366
|
+
finished = [outcome for outcome in outcomes if outcome is not None]
|
|
367
|
+
successes = sum(1 for outcome in finished if outcome.ok)
|
|
368
|
+
skipped = sum(1 for outcome in finished if outcome.skipped)
|
|
369
|
+
failures = len(finished) - successes - skipped
|
|
370
|
+
if budget is not None and not spent_seen:
|
|
371
|
+
console.print(
|
|
372
|
+
"[yellow]note[/yellow] the endpoint reported no cost; --max-cost could not be enforced"
|
|
373
|
+
)
|
|
374
|
+
if skipped:
|
|
375
|
+
console.print(f"[yellow]{skipped} script(s) skipped at the cost budget[/yellow]")
|
|
376
|
+
if spent:
|
|
377
|
+
console.print(f"[dim]reported spend: ${spent:.4f}[/dim]")
|
|
297
378
|
unverified = sum(
|
|
298
|
-
1 for outcome in
|
|
379
|
+
1 for outcome in finished for issue in (outcome.issues or []) if not issue.ok
|
|
299
380
|
)
|
|
300
381
|
if state.strict_quotes and unverified:
|
|
301
382
|
failures += 0
|
|
302
|
-
logger.run_end(
|
|
383
|
+
logger.run_end(
|
|
384
|
+
status="ok" if failures == 0 else "partial",
|
|
385
|
+
results=successes,
|
|
386
|
+
failures=failures,
|
|
387
|
+
skipped=skipped,
|
|
388
|
+
cost_usd=round(spent, 6),
|
|
389
|
+
)
|
|
303
390
|
|
|
304
|
-
if batch_mode and successes:
|
|
391
|
+
if batch_mode and (successes or skipped):
|
|
305
392
|
if not state.json_output:
|
|
306
393
|
console.print(
|
|
307
|
-
render_summary_table(
|
|
394
|
+
render_summary_table(
|
|
395
|
+
[(o.name, o.breakdown, o.error, o.skipped) for o in finished]
|
|
396
|
+
)
|
|
308
397
|
)
|
|
309
398
|
ranked = reporting.rank(report_entries)
|
|
310
399
|
if top:
|
|
@@ -2,15 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import threading
|
|
5
6
|
import time
|
|
6
7
|
from typing import Any, Callable
|
|
7
8
|
|
|
8
9
|
import httpx
|
|
9
10
|
|
|
10
11
|
from scamscript.config import (
|
|
11
|
-
DEFAULT_BASE_URL,
|
|
12
12
|
DEFAULT_MAX_TOKENS,
|
|
13
|
-
DEFAULT_MODEL,
|
|
14
13
|
DEFAULT_RETRIES,
|
|
15
14
|
DEFAULT_TIMEOUT,
|
|
16
15
|
)
|
|
@@ -42,8 +41,8 @@ class ChatClient:
|
|
|
42
41
|
def __init__(
|
|
43
42
|
self,
|
|
44
43
|
api_key: str,
|
|
45
|
-
base_url: str =
|
|
46
|
-
model: str =
|
|
44
|
+
base_url: str = "",
|
|
45
|
+
model: str = "",
|
|
47
46
|
timeout: float = DEFAULT_TIMEOUT,
|
|
48
47
|
retries: int = DEFAULT_RETRIES,
|
|
49
48
|
max_tokens: int = DEFAULT_MAX_TOKENS,
|
|
@@ -54,7 +53,7 @@ class ChatClient:
|
|
|
54
53
|
self.model = model
|
|
55
54
|
self.retries = max(0, retries)
|
|
56
55
|
self.max_tokens = max_tokens
|
|
57
|
-
self.
|
|
56
|
+
self._usage = threading.local()
|
|
58
57
|
self._sleep = sleep
|
|
59
58
|
self._client = httpx.Client(
|
|
60
59
|
base_url=self.base_url,
|
|
@@ -64,7 +63,7 @@ class ChatClient:
|
|
|
64
63
|
"Authorization": f"Bearer {api_key}",
|
|
65
64
|
"Content-Type": "application/json",
|
|
66
65
|
"Accept": "application/json",
|
|
67
|
-
"User-Agent": "scamscript/0.
|
|
66
|
+
"User-Agent": "scamscript/0.4.0",
|
|
68
67
|
},
|
|
69
68
|
)
|
|
70
69
|
|
|
@@ -77,6 +76,24 @@ class ChatClient:
|
|
|
77
76
|
def close(self) -> None:
|
|
78
77
|
self._client.close()
|
|
79
78
|
|
|
79
|
+
@property
|
|
80
|
+
def last_usage(self) -> dict[str, int] | None:
|
|
81
|
+
return getattr(self._usage, "value", None)
|
|
82
|
+
|
|
83
|
+
@last_usage.setter
|
|
84
|
+
def last_usage(self, value: dict[str, int] | None) -> None:
|
|
85
|
+
self._usage.value = value
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def last_cost(self) -> float | None:
|
|
89
|
+
usage = self.last_usage
|
|
90
|
+
if not usage:
|
|
91
|
+
return None
|
|
92
|
+
try:
|
|
93
|
+
return float(usage.get("cost") or 0.0)
|
|
94
|
+
except (TypeError, ValueError):
|
|
95
|
+
return None
|
|
96
|
+
|
|
80
97
|
@property
|
|
81
98
|
def endpoint(self) -> str:
|
|
82
99
|
return f"{self.base_url}/chat/completions"
|
|
@@ -5,10 +5,9 @@ from __future__ import annotations
|
|
|
5
5
|
import os
|
|
6
6
|
from dataclasses import dataclass, field
|
|
7
7
|
|
|
8
|
-
DEFAULT_BASE_URL = "https://api.venice.ai/api/v1"
|
|
9
|
-
DEFAULT_MODEL = "deepseek-v4-1-flash-1017-uncensored"
|
|
10
8
|
DEFAULT_TIMEOUT = 120.0
|
|
11
9
|
DEFAULT_RETRIES = 2
|
|
10
|
+
DEFAULT_CONCURRENCY = 1
|
|
12
11
|
DEFAULT_MAX_TOKENS = 4096
|
|
13
12
|
ENV_API_KEY = "API_KEY"
|
|
14
13
|
ENV_HOME = "SCAMSCRIPT_HOME"
|
|
@@ -20,19 +19,23 @@ class ConfigError(Exception):
|
|
|
20
19
|
|
|
21
20
|
@dataclass(slots=True)
|
|
22
21
|
class Settings:
|
|
23
|
-
base_url: str =
|
|
24
|
-
model: str =
|
|
22
|
+
base_url: str = ""
|
|
23
|
+
model: str = ""
|
|
25
24
|
api_key: str | None = None
|
|
26
25
|
timeout: float = DEFAULT_TIMEOUT
|
|
27
26
|
retries: int = DEFAULT_RETRIES
|
|
28
27
|
max_tokens: int = DEFAULT_MAX_TOKENS
|
|
29
28
|
redact: bool = False
|
|
29
|
+
concurrency: int = DEFAULT_CONCURRENCY
|
|
30
|
+
max_cost: float | None = None
|
|
30
31
|
home: str = field(default_factory=lambda: os.environ.get(ENV_HOME, ""))
|
|
31
32
|
|
|
32
33
|
def __post_init__(self) -> None:
|
|
33
|
-
self.base_url = (self.base_url or
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
self.base_url = (self.base_url or "").strip().rstrip("/")
|
|
35
|
+
self.model = (self.model or "").strip()
|
|
36
|
+
self.concurrency = max(1, min(16, self.concurrency))
|
|
37
|
+
if self.max_cost is not None:
|
|
38
|
+
self.max_cost = max(0.0, self.max_cost)
|
|
36
39
|
home = self.home or os.path.join(os.path.expanduser("~"), ".scamscript")
|
|
37
40
|
self.home = home
|
|
38
41
|
|
|
@@ -42,6 +45,21 @@ class Settings:
|
|
|
42
45
|
key = key.strip()
|
|
43
46
|
return key or None
|
|
44
47
|
|
|
48
|
+
def require_base_url(self) -> str:
|
|
49
|
+
if not self.base_url:
|
|
50
|
+
raise ConfigError(
|
|
51
|
+
"missing API base URL: pass --base-url or set the SCAMSCRIPT_BASE_URL "
|
|
52
|
+
"environment variable"
|
|
53
|
+
)
|
|
54
|
+
return self.base_url
|
|
55
|
+
|
|
56
|
+
def require_model(self) -> str:
|
|
57
|
+
if not self.model:
|
|
58
|
+
raise ConfigError(
|
|
59
|
+
"missing model: pass --model or set the SCAMSCRIPT_MODEL environment variable"
|
|
60
|
+
)
|
|
61
|
+
return self.model
|
|
62
|
+
|
|
45
63
|
def require_api_key(self) -> str:
|
|
46
64
|
key = self.resolve_api_key()
|
|
47
65
|
if key is None:
|
|
@@ -207,14 +207,19 @@ def render_prompt(source: ScriptFile, messages: list[dict[str, str]]) -> Panel:
|
|
|
207
207
|
)
|
|
208
208
|
|
|
209
209
|
|
|
210
|
-
def render_summary_table(
|
|
210
|
+
def render_summary_table(
|
|
211
|
+
rows: list[tuple[str, ScriptBreakdown | None, str | None, bool]]
|
|
212
|
+
) -> Table:
|
|
211
213
|
table = Table(title="Run summary", header_style="bold", expand=True)
|
|
212
214
|
table.add_column("Script", overflow="fold", ratio=3)
|
|
213
215
|
table.add_column("Attack type", ratio=2)
|
|
214
216
|
table.add_column("Score", justify="right")
|
|
215
217
|
table.add_column("Flags", justify="right")
|
|
216
218
|
table.add_column("Status")
|
|
217
|
-
for name, breakdown, error in rows:
|
|
219
|
+
for name, breakdown, error, skipped in rows:
|
|
220
|
+
if skipped:
|
|
221
|
+
table.add_row(name, "-", "-", "-", "[yellow]skipped[/yellow]")
|
|
222
|
+
continue
|
|
218
223
|
if breakdown is None:
|
|
219
224
|
table.add_row(name, "-", "-", "-", f"[red]{error or 'failed'}[/red]")
|
|
220
225
|
continue
|
|
@@ -29,6 +29,7 @@ class ReportEntry:
|
|
|
29
29
|
breakdown: ScriptBreakdown | None = None
|
|
30
30
|
error: str | None = None
|
|
31
31
|
duration: float = 0.0
|
|
32
|
+
skipped: bool = False
|
|
32
33
|
verification: dict[str, int] | None = None
|
|
33
34
|
unverified: list[dict[str, str]] | None = None
|
|
34
35
|
|
|
@@ -42,6 +43,7 @@ def entry_from_outcome(outcome: Any) -> ReportEntry:
|
|
|
42
43
|
name=outcome.name,
|
|
43
44
|
source=str(getattr(outcome, "source", outcome.name)),
|
|
44
45
|
breakdown=outcome.breakdown,
|
|
46
|
+
skipped=getattr(outcome, "skipped", False),
|
|
45
47
|
error=outcome.error,
|
|
46
48
|
duration=getattr(outcome, "duration", 0.0),
|
|
47
49
|
verification=getattr(outcome, "verification", None),
|
|
@@ -68,6 +70,7 @@ def entries_from_run(run: dict[str, Any]) -> list[ReportEntry]:
|
|
|
68
70
|
name=record.get("name") or record.get("source") or "unknown",
|
|
69
71
|
source=record.get("source") or "",
|
|
70
72
|
breakdown=breakdown,
|
|
73
|
+
skipped=bool(record.get("skipped")),
|
|
71
74
|
error=None if breakdown else record.get("error") or "no breakdown stored",
|
|
72
75
|
duration=float(record.get("duration_s") or 0.0),
|
|
73
76
|
verification=record.get("verification") or None,
|
|
@@ -96,7 +99,8 @@ def build_markdown(
|
|
|
96
99
|
) -> str:
|
|
97
100
|
ranked = rank(entries)
|
|
98
101
|
ok = [entry for entry in ranked if entry.ok]
|
|
99
|
-
|
|
102
|
+
skipped = [entry for entry in ranked if entry.skipped and not entry.ok]
|
|
103
|
+
failed = [entry for entry in ranked if not entry.ok and not entry.skipped]
|
|
100
104
|
average = (
|
|
101
105
|
sum(entry.breakdown.sophistication_score for entry in ok) / len(ok) if ok else 0.0
|
|
102
106
|
)
|
|
@@ -112,6 +116,8 @@ def build_markdown(
|
|
|
112
116
|
lines.append(f"- Scripts analyzed: {len(entries)}")
|
|
113
117
|
lines.append(f"- Successful: {len(ok)}")
|
|
114
118
|
lines.append(f"- Failed: {len(failed)}")
|
|
119
|
+
if skipped:
|
|
120
|
+
lines.append(f"- Skipped at budget: {len(skipped)}")
|
|
115
121
|
unverified_total = sum(len(entry.unverified or []) for entry in ok)
|
|
116
122
|
if unverified_total:
|
|
117
123
|
lines.append(f"- Unverified quotes: {unverified_total}")
|
|
@@ -137,6 +143,13 @@ def build_markdown(
|
|
|
137
143
|
lines.append("_No successful analyses in this run._")
|
|
138
144
|
lines.append("")
|
|
139
145
|
|
|
146
|
+
if skipped:
|
|
147
|
+
lines.append("## Skipped")
|
|
148
|
+
lines.append("")
|
|
149
|
+
for entry in skipped:
|
|
150
|
+
lines.append(f"- **{_cell(entry.name)}** (`{_cell(entry.source)}`)")
|
|
151
|
+
lines.append("")
|
|
152
|
+
|
|
140
153
|
if failed:
|
|
141
154
|
lines.append("## Failures")
|
|
142
155
|
lines.append("")
|
|
@@ -32,6 +32,7 @@ class FakeClient:
|
|
|
32
32
|
self.base_url = base_url
|
|
33
33
|
self.model = model
|
|
34
34
|
self.max_tokens = kwargs.get("max_tokens", 4096)
|
|
35
|
+
self.last_cost = None
|
|
35
36
|
|
|
36
37
|
def __enter__(self):
|
|
37
38
|
return self
|
|
@@ -50,6 +51,8 @@ class FakeClient:
|
|
|
50
51
|
@pytest.fixture()
|
|
51
52
|
def home(tmp_path, monkeypatch):
|
|
52
53
|
monkeypatch.setenv("SCAMSCRIPT_HOME", str(tmp_path))
|
|
54
|
+
monkeypatch.setenv("SCAMSCRIPT_BASE_URL", "https://example.test/v1")
|
|
55
|
+
monkeypatch.setenv("SCAMSCRIPT_MODEL", "test-model")
|
|
53
56
|
monkeypatch.delenv("API_KEY", raising=False)
|
|
54
57
|
return tmp_path
|
|
55
58
|
|
|
@@ -194,10 +197,38 @@ def test_history_empty(home):
|
|
|
194
197
|
assert "no runs logged" in result.stdout
|
|
195
198
|
|
|
196
199
|
|
|
200
|
+
def test_missing_model(home, tmp_path, monkeypatch):
|
|
201
|
+
monkeypatch.delenv("SCAMSCRIPT_MODEL")
|
|
202
|
+
script = tmp_path / "a.txt"
|
|
203
|
+
script.write_text("CALLER: hello", encoding="utf-8")
|
|
204
|
+
result = runner.invoke(cli.app, ["--api-key", "k", "analyze", "--file", str(script)])
|
|
205
|
+
assert result.exit_code == 1
|
|
206
|
+
assert "missing model" in result.stdout
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_missing_base_url(home, tmp_path, monkeypatch):
|
|
210
|
+
monkeypatch.delenv("SCAMSCRIPT_BASE_URL")
|
|
211
|
+
script = tmp_path / "a.txt"
|
|
212
|
+
script.write_text("CALLER: hello", encoding="utf-8")
|
|
213
|
+
result = runner.invoke(cli.app, ["--api-key", "k", "analyze", "--file", str(script)])
|
|
214
|
+
assert result.exit_code == 1
|
|
215
|
+
assert "missing API base URL" in result.stdout
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def test_dry_run_without_endpoint(home, tmp_path, monkeypatch):
|
|
219
|
+
monkeypatch.delenv("SCAMSCRIPT_MODEL")
|
|
220
|
+
monkeypatch.delenv("SCAMSCRIPT_BASE_URL")
|
|
221
|
+
script = tmp_path / "a.txt"
|
|
222
|
+
script.write_text("CALLER: hello", encoding="utf-8")
|
|
223
|
+
result = runner.invoke(cli.app, ["--dry-run", "analyze", "--file", str(script)])
|
|
224
|
+
assert result.exit_code == 0
|
|
225
|
+
assert "unaddressed" in result.stdout
|
|
226
|
+
|
|
227
|
+
|
|
197
228
|
def test_version(home):
|
|
198
229
|
result = runner.invoke(cli.app, ["--version"])
|
|
199
230
|
assert result.exit_code == 0
|
|
200
|
-
assert result.stdout.strip() == "scamscript 0.
|
|
231
|
+
assert result.stdout.strip() == "scamscript 0.4.0"
|
|
201
232
|
|
|
202
233
|
|
|
203
234
|
def test_json_output_is_one_object_per_script(home, tmp_path, fake):
|
|
@@ -311,3 +342,70 @@ def test_json_result_carries_verification(home, tmp_path, fake):
|
|
|
311
342
|
record = json.loads(result.stdout.splitlines()[0])
|
|
312
343
|
assert record["verification"] == {"checked": 1, "verbatim": 0, "close": 0, "missing": 1}
|
|
313
344
|
assert record["unverified"][0]["status"] == "missing"
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def test_concurrency_keeps_output_order(home, tmp_path, fake):
|
|
348
|
+
for index in range(6):
|
|
349
|
+
(tmp_path / f"s{index}.txt").write_text(f"CALLER: script {index}", encoding="utf-8")
|
|
350
|
+
result = runner.invoke(
|
|
351
|
+
cli.app, ["--api-key", "k", "--json", "--concurrency", "4", "batch", str(tmp_path)]
|
|
352
|
+
)
|
|
353
|
+
assert result.exit_code == 0
|
|
354
|
+
records = [json.loads(line) for line in result.stdout.splitlines() if line.strip()]
|
|
355
|
+
names = [record["name"] for record in records if "name" in record]
|
|
356
|
+
assert names[:6] == [f"s{i}.txt" for i in range(6)]
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def test_concurrency_runs_in_parallel(home, tmp_path, fake, monkeypatch):
|
|
360
|
+
import threading, time
|
|
361
|
+
|
|
362
|
+
active = {"now": 0, "peak": 0}
|
|
363
|
+
lock = threading.Lock()
|
|
364
|
+
|
|
365
|
+
def complete(self, messages, **kwargs):
|
|
366
|
+
with lock:
|
|
367
|
+
active["now"] += 1
|
|
368
|
+
active["peak"] = max(active["peak"], active["now"])
|
|
369
|
+
time.sleep(0.05)
|
|
370
|
+
with lock:
|
|
371
|
+
active["now"] -= 1
|
|
372
|
+
return json.dumps(breakdown())
|
|
373
|
+
|
|
374
|
+
monkeypatch.setattr(FakeClient, "complete", complete)
|
|
375
|
+
for index in range(6):
|
|
376
|
+
(tmp_path / f"s{index}.txt").write_text("CALLER: x", encoding="utf-8")
|
|
377
|
+
|
|
378
|
+
result = runner.invoke(
|
|
379
|
+
cli.app, ["--api-key", "k", "--concurrency", "4", "batch", str(tmp_path)]
|
|
380
|
+
)
|
|
381
|
+
assert result.exit_code == 0
|
|
382
|
+
assert active["peak"] > 1
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def test_max_cost_skips_remaining(home, tmp_path, fake, monkeypatch):
|
|
386
|
+
for index in range(4):
|
|
387
|
+
(tmp_path / f"s{index}.txt").write_text("CALLER: x", encoding="utf-8")
|
|
388
|
+
|
|
389
|
+
def complete(self, messages, **kwargs):
|
|
390
|
+
self.last_cost = 0.01
|
|
391
|
+
return json.dumps(breakdown())
|
|
392
|
+
|
|
393
|
+
monkeypatch.setattr(FakeClient, "complete", complete)
|
|
394
|
+
out = tmp_path / "r.md"
|
|
395
|
+
result = runner.invoke(
|
|
396
|
+
cli.app,
|
|
397
|
+
["--api-key", "k", "--max-cost", "0.015", "batch", str(tmp_path), "--output", str(out)],
|
|
398
|
+
)
|
|
399
|
+
assert "skipped at the cost budget" in result.stdout
|
|
400
|
+
markdown = out.read_text()
|
|
401
|
+
assert "Skipped at budget" in markdown
|
|
402
|
+
assert "## Skipped" in markdown
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def test_max_cost_warns_when_provider_reports_nothing(home, tmp_path, fake):
|
|
406
|
+
(tmp_path / "a.txt").write_text("CALLER: x", encoding="utf-8")
|
|
407
|
+
result = runner.invoke(
|
|
408
|
+
cli.app, ["--api-key", "k", "--max-cost", "0.5", "analyze", "--file", str(tmp_path / "a.txt")]
|
|
409
|
+
)
|
|
410
|
+
assert result.exit_code == 0
|
|
411
|
+
assert "could not be enforced" in result.stdout
|
|
@@ -276,3 +276,36 @@ def test_length_with_only_reasoning_is_truncation():
|
|
|
276
276
|
with pytest.raises(TruncatedError) as excinfo:
|
|
277
277
|
client.complete(MESSAGES)
|
|
278
278
|
assert excinfo.value.partial == "thinking..."
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def test_usage_is_thread_local():
|
|
282
|
+
import threading
|
|
283
|
+
|
|
284
|
+
def handler(request):
|
|
285
|
+
content = json.loads(request.read())["messages"][-1]["content"]
|
|
286
|
+
return httpx.Response(
|
|
287
|
+
200,
|
|
288
|
+
json={
|
|
289
|
+
"choices": [{"message": {"content": content}}],
|
|
290
|
+
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "cost": float(len(content))},
|
|
291
|
+
},
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
client = ChatClient(
|
|
295
|
+
api_key="k",
|
|
296
|
+
base_url="https://example.test/v1",
|
|
297
|
+
transport=httpx.MockTransport(handler),
|
|
298
|
+
)
|
|
299
|
+
seen = {}
|
|
300
|
+
|
|
301
|
+
def run(role, text):
|
|
302
|
+
client.complete([{"role": role, "content": text}])
|
|
303
|
+
seen[role] = client.last_cost
|
|
304
|
+
|
|
305
|
+
threads = [threading.Thread(target=run, args=("a", "xx")), threading.Thread(target=run, args=("b", "xxxx"))]
|
|
306
|
+
for thread in threads:
|
|
307
|
+
thread.start()
|
|
308
|
+
for thread in threads:
|
|
309
|
+
thread.join()
|
|
310
|
+
client.close()
|
|
311
|
+
assert seen == {"a": 2.0, "b": 4.0}
|
|
@@ -76,9 +76,14 @@ def test_prompt_panel():
|
|
|
76
76
|
|
|
77
77
|
|
|
78
78
|
def test_summary_table_row_states():
|
|
79
|
-
text = capture(
|
|
79
|
+
text = capture(
|
|
80
|
+
render_summary_table(
|
|
81
|
+
[("ok.txt", breakdown(), None, False), ("bad.txt", None, "boom", False), ("skip.txt", None, "skipped", True)]
|
|
82
|
+
)
|
|
83
|
+
)
|
|
80
84
|
assert "ok.txt" in text
|
|
81
85
|
assert "boom" in text
|
|
86
|
+
assert "skipped" in text
|
|
82
87
|
|
|
83
88
|
|
|
84
89
|
def test_panel_shows_quote_check():
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|