scamscript 0.3.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scamscript-0.3.0 → scamscript-0.4.1}/CHANGELOG.md +20 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/PKG-INFO +17 -2
- {scamscript-0.3.0 → scamscript-0.4.1}/README.md +16 -1
- {scamscript-0.3.0 → scamscript-0.4.1}/pyproject.toml +1 -1
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/__init__.py +1 -1
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/analysis.py +2 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/cli.py +119 -33
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/client.py +21 -2
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/config.py +6 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/render.py +7 -2
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/report.py +14 -1
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_cli.py +69 -1
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_client.py +33 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_render.py +6 -1
- {scamscript-0.3.0 → scamscript-0.4.1}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/.github/SECURITY.md +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/.github/pull_request_template.md +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/.github/workflows/ci.yml +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/.github/workflows/release.yml +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/.gitignore +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/CITATION.cff +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/CODE_OF_CONDUCT.md +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/CONTRIBUTING.md +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/LICENSE +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/docs/banner.svg +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/docs/calibration.md +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/docs/demo.gif +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/docs/sample-report.md +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/docs/social-preview.png +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/adversarial/fake_output_json.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/adversarial/fence_escape.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/adversarial/fixed_attack_type.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/adversarial/non_json_demand.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/adversarial/print_system_prompt.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/adversarial/role_switch.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/adversarial/score_override.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/crude_robocall.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/gov_impersonation_vishing.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/prompt_injection_script.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/romance_scam_sms.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/sophisticated_wire_fraud.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/examples/tech_support_scam.txt +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/scripts/calibrate.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/scripts/injection_eval.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/__main__.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/ingest.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/models.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/prompts.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/redact.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/store.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/src/scamscript/verify.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_analysis.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_ingest.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_models.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_prompts.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_redact.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_report.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_store.py +0 -0
- {scamscript-0.3.0 → scamscript-0.4.1}/tests/test_verify.py +0 -0
|
@@ -5,6 +5,26 @@ schema are the things that count as public API.
|
|
|
5
5
|
|
|
6
6
|
## Unreleased
|
|
7
7
|
|
|
8
|
+
## 0.4.1 - 2026-11-02
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- README lost the scale and spend section during an earlier edit; restored so the new flags are
|
|
13
|
+
documented where people look.
|
|
14
|
+
|
|
15
|
+
## 0.4.0 - 2026-11-02
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- `--concurrency N` (1-16) with order-preserving output.
|
|
20
|
+
- `--max-cost USD`: stop scheduling once reported spend reaches the cap, mark the rest skipped, and
|
|
21
|
+
warn when a provider reports no cost.
|
|
22
|
+
- Per-run reported spend in output and logs.
|
|
23
|
+
|
|
24
|
+
### Changed
|
|
25
|
+
|
|
26
|
+
- Token usage is tracked per thread so parallel calls cannot read each other's cost.
|
|
27
|
+
|
|
8
28
|
## 0.3.0 - 2026-11-02
|
|
9
29
|
|
|
10
30
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scamscript
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Reverse-engineer scam scripts and vishing transcripts into structured counter-playbooks
|
|
5
5
|
Project-URL: Homepage, https://github.com/sjdsjadjajdjsa/scamscript
|
|
6
6
|
Project-URL: Repository, https://github.com/sjdsjadjajdjsa/scamscript
|
|
@@ -133,13 +133,27 @@ corpus is at [`docs/sample-report.md`](https://github.com/sjdsjadjajdjsa/scamscr
|
|
|
133
133
|
| `prune` | Delete old logs |
|
|
134
134
|
|
|
135
135
|
Useful flags: `--dry-run`, `--json`, `--redact`, `--top N`, `--recursive`, `--timeout`,
|
|
136
|
-
`--retries`, `--max-tokens`, `--no-color`. `--json` puts one object per script on stdout and all
|
|
136
|
+
`--retries`, `--max-tokens`, `--concurrency`, `--max-cost`, `--no-color`. `--json` puts one object per script on stdout and all
|
|
137
137
|
Rich output on stderr, so it pipes:
|
|
138
138
|
|
|
139
139
|
```bash
|
|
140
140
|
scamscript --json batch examples | jq -s 'map(select(.event == "result")) | length'
|
|
141
141
|
```
|
|
142
142
|
|
|
143
|
+
## Scale and spend
|
|
144
|
+
|
|
145
|
+
`--concurrency N` runs up to 16 analyses at once and keeps output in script order. `--max-cost`
|
|
146
|
+
stops scheduling new scripts once the provider reports that much spend, marks the remainder skipped
|
|
147
|
+
and still exits zero when nothing failed.
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
scamscript --concurrency 6 --max-cost 0.25 batch corpus/ --output report.md
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The cap reads `usage.cost` from the response, so it works only where the provider reports cost and
|
|
154
|
+
says so when none is sent. Spend is printed and logged per run. In-flight requests can overshoot by
|
|
155
|
+
up to the concurrency, so keep headroom on a hard limit.
|
|
156
|
+
|
|
143
157
|
## How it works
|
|
144
158
|
|
|
145
159
|
```text
|
|
@@ -249,6 +263,7 @@ docs/ calibration record, sample report, banner, demo
|
|
|
249
263
|
`opening_hook` and `red_flags` against the source before quoting anything.
|
|
250
264
|
- Redaction is regex-based and will miss names and identifiers without a fixed shape.
|
|
251
265
|
- One script per request; no cross-script deduplication yet, so reprints inflate a ranking.
|
|
266
|
+
- The cost cap depends on provider-reported usage and can overshoot by the in-flight count.
|
|
252
267
|
- Analysis is English-centric: the prompt and schema assume English artifacts.
|
|
253
268
|
- Quote checking compares text, not meaning. A legitimate paraphrase left in quotes is flagged;
|
|
254
269
|
an accurate paraphrase without quotes is skipped rather than judged.
|
|
@@ -97,13 +97,27 @@ corpus is at [`docs/sample-report.md`](https://github.com/sjdsjadjajdjsa/scamscr
|
|
|
97
97
|
| `prune` | Delete old logs |
|
|
98
98
|
|
|
99
99
|
Useful flags: `--dry-run`, `--json`, `--redact`, `--top N`, `--recursive`, `--timeout`,
|
|
100
|
-
`--retries`, `--max-tokens`, `--no-color`. `--json` puts one object per script on stdout and all
|
|
100
|
+
`--retries`, `--max-tokens`, `--concurrency`, `--max-cost`, `--no-color`. `--json` puts one object per script on stdout and all
|
|
101
101
|
Rich output on stderr, so it pipes:
|
|
102
102
|
|
|
103
103
|
```bash
|
|
104
104
|
scamscript --json batch examples | jq -s 'map(select(.event == "result")) | length'
|
|
105
105
|
```
|
|
106
106
|
|
|
107
|
+
## Scale and spend
|
|
108
|
+
|
|
109
|
+
`--concurrency N` runs up to 16 analyses at once and keeps output in script order. `--max-cost`
|
|
110
|
+
stops scheduling new scripts once the provider reports that much spend, marks the remainder skipped
|
|
111
|
+
and still exits zero when nothing failed.
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
scamscript --concurrency 6 --max-cost 0.25 batch corpus/ --output report.md
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
The cap reads `usage.cost` from the response, so it works only where the provider reports cost and
|
|
118
|
+
says so when none is sent. Spend is printed and logged per run. In-flight requests can overshoot by
|
|
119
|
+
up to the concurrency, so keep headroom on a hard limit.
|
|
120
|
+
|
|
107
121
|
## How it works
|
|
108
122
|
|
|
109
123
|
```text
|
|
@@ -213,6 +227,7 @@ docs/ calibration record, sample report, banner, demo
|
|
|
213
227
|
`opening_hook` and `red_flags` against the source before quoting anything.
|
|
214
228
|
- Redaction is regex-based and will miss names and identifiers without a fixed shape.
|
|
215
229
|
- One script per request; no cross-script deduplication yet, so reprints inflate a ranking.
|
|
230
|
+
- The cost cap depends on provider-reported usage and can overshoot by the in-flight count.
|
|
216
231
|
- Analysis is English-centric: the prompt and schema assume English artifacts.
|
|
217
232
|
- Quote checking compares text, not meaning. A legitimate paraphrase left in quotes is flagged;
|
|
218
233
|
an accurate paraphrase without quotes is skipped rather than judged.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scamscript"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.1"
|
|
8
8
|
description = "Reverse-engineer scam scripts and vishing transcripts into structured counter-playbooks"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -26,6 +26,7 @@ class Outcome:
|
|
|
26
26
|
error: str | None = None
|
|
27
27
|
duration: float = 0.0
|
|
28
28
|
redactions: dict[str, int] | None = None
|
|
29
|
+
skipped: bool = False
|
|
29
30
|
issues: list[verify.Issue] | None = None
|
|
30
31
|
verification: dict[str, int] | None = None
|
|
31
32
|
|
|
@@ -39,6 +40,7 @@ class Outcome:
|
|
|
39
40
|
"source": str(self.source),
|
|
40
41
|
"name": self.name,
|
|
41
42
|
"ok": self.ok,
|
|
43
|
+
"skipped": self.skipped,
|
|
42
44
|
"attempts": self.attempts,
|
|
43
45
|
"duration_s": round(self.duration, 3),
|
|
44
46
|
"breakdown": self.breakdown.model_dump() if self.breakdown else None,
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
|
+
import threading
|
|
7
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
6
8
|
from dataclasses import dataclass
|
|
7
9
|
from pathlib import Path
|
|
8
10
|
|
|
@@ -104,6 +106,20 @@ def main(
|
|
|
104
106
|
min=256,
|
|
105
107
|
help="Output token budget per reply.",
|
|
106
108
|
),
|
|
109
|
+
concurrency: int = typer.Option(
|
|
110
|
+
1,
|
|
111
|
+
"--concurrency",
|
|
112
|
+
"-j",
|
|
113
|
+
min=1,
|
|
114
|
+
max=16,
|
|
115
|
+
help="Analyze this many scripts in parallel.",
|
|
116
|
+
),
|
|
117
|
+
max_cost: float | None = typer.Option(
|
|
118
|
+
None,
|
|
119
|
+
"--max-cost",
|
|
120
|
+
min=0.0,
|
|
121
|
+
help="Stop scheduling new scripts once the provider reports this much spend in USD.",
|
|
122
|
+
),
|
|
107
123
|
redact: bool = typer.Option(
|
|
108
124
|
False,
|
|
109
125
|
"--redact",
|
|
@@ -146,6 +162,8 @@ def main(
|
|
|
146
162
|
timeout=timeout,
|
|
147
163
|
retries=retries,
|
|
148
164
|
max_tokens=max_tokens,
|
|
165
|
+
concurrency=concurrency,
|
|
166
|
+
max_cost=max_cost,
|
|
149
167
|
redact=redact,
|
|
150
168
|
home=str(home) if home else "",
|
|
151
169
|
)
|
|
@@ -254,8 +272,12 @@ def _run(
|
|
|
254
272
|
_fail(console, str(exc))
|
|
255
273
|
return
|
|
256
274
|
|
|
257
|
-
outcomes: list[Outcome] = []
|
|
275
|
+
outcomes: list[Outcome | None] = [None] * len(scripts)
|
|
258
276
|
report_entries: list[reporting.ReportEntry] = []
|
|
277
|
+
budget = settings.max_cost
|
|
278
|
+
spent = 0.0
|
|
279
|
+
spent_seen = False
|
|
280
|
+
lock = threading.Lock()
|
|
259
281
|
client = ChatClient(
|
|
260
282
|
api_key=api_key,
|
|
261
283
|
base_url=settings.base_url,
|
|
@@ -264,50 +286,114 @@ def _run(
|
|
|
264
286
|
retries=settings.retries,
|
|
265
287
|
max_tokens=settings.max_tokens,
|
|
266
288
|
)
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
289
|
+
|
|
290
|
+
def one(script: ScriptFile) -> Outcome:
|
|
291
|
+
nonlocal spent, spent_seen
|
|
292
|
+
if budget is not None:
|
|
293
|
+
with lock:
|
|
294
|
+
blocked = spent >= budget
|
|
295
|
+
if blocked:
|
|
296
|
+
outcome = Outcome(
|
|
297
|
+
source=script.path,
|
|
298
|
+
name=script.name,
|
|
299
|
+
skipped=True,
|
|
300
|
+
error="skipped: cost budget reached",
|
|
301
|
+
)
|
|
302
|
+
logger.result(**outcome.as_record())
|
|
303
|
+
return outcome
|
|
304
|
+
outcome = analyze_script(
|
|
305
|
+
client,
|
|
306
|
+
script,
|
|
307
|
+
logger,
|
|
308
|
+
redact_identifiers=settings.redact,
|
|
309
|
+
verify_quotes=state.verify_quotes,
|
|
310
|
+
)
|
|
311
|
+
cost = client.last_cost
|
|
312
|
+
if cost:
|
|
313
|
+
with lock:
|
|
314
|
+
spent += cost
|
|
315
|
+
spent_seen = True
|
|
316
|
+
return outcome
|
|
317
|
+
|
|
318
|
+
def emit(outcome: Outcome) -> None:
|
|
319
|
+
if state.json_output:
|
|
320
|
+
_emit_json(outcome.as_record())
|
|
321
|
+
elif outcome.skipped:
|
|
322
|
+
console.print(f"[dim]skipped {outcome.name}: budget reached[/dim]")
|
|
323
|
+
elif outcome.breakdown is not None:
|
|
324
|
+
console.print(
|
|
325
|
+
render_breakdown(
|
|
326
|
+
outcome.breakdown, outcome.name, outcome.attempts, outcome.issues
|
|
272
327
|
)
|
|
273
|
-
outcome = analyze_script(
|
|
274
|
-
client,
|
|
275
|
-
script,
|
|
276
|
-
logger,
|
|
277
|
-
redact_identifiers=settings.redact,
|
|
278
|
-
verify_quotes=state.verify_quotes,
|
|
279
328
|
)
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
329
|
+
else:
|
|
330
|
+
console.print(
|
|
331
|
+
render_failure(outcome.name, outcome.error or "unknown", outcome.attempts)
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
with client:
|
|
335
|
+
if settings.concurrency == 1:
|
|
336
|
+
for index, script in enumerate(scripts, start=1):
|
|
337
|
+
if batch_mode and not state.json_output:
|
|
338
|
+
console.print(
|
|
339
|
+
f"[dim]({index}/{len(scripts)}) analyzing {script.name}...[/dim]"
|
|
291
340
|
)
|
|
292
|
-
)
|
|
293
|
-
|
|
341
|
+
outcome = one(script)
|
|
342
|
+
outcomes[index - 1] = outcome
|
|
343
|
+
report_entries.append(reporting.entry_from_outcome(outcome))
|
|
344
|
+
emit(outcome)
|
|
345
|
+
else:
|
|
346
|
+
if not state.json_output:
|
|
294
347
|
console.print(
|
|
295
|
-
|
|
348
|
+
f"[dim]analyzing {len(scripts)} script(s), "
|
|
349
|
+
f"{settings.concurrency} at a time[/dim]"
|
|
296
350
|
)
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
351
|
+
buffered: dict[int, Outcome] = {}
|
|
352
|
+
ready = 0
|
|
353
|
+
with ThreadPoolExecutor(max_workers=settings.concurrency) as pool:
|
|
354
|
+
futures = {
|
|
355
|
+
pool.submit(one, script): index for index, script in enumerate(scripts)
|
|
356
|
+
}
|
|
357
|
+
for future in as_completed(futures):
|
|
358
|
+
buffered[futures[future]] = future.result()
|
|
359
|
+
while ready in buffered:
|
|
360
|
+
outcome = buffered.pop(ready)
|
|
361
|
+
outcomes[ready] = outcome
|
|
362
|
+
report_entries.append(reporting.entry_from_outcome(outcome))
|
|
363
|
+
emit(outcome)
|
|
364
|
+
ready += 1
|
|
365
|
+
|
|
366
|
+
finished = [outcome for outcome in outcomes if outcome is not None]
|
|
367
|
+
successes = sum(1 for outcome in finished if outcome.ok)
|
|
368
|
+
skipped = sum(1 for outcome in finished if outcome.skipped)
|
|
369
|
+
failures = len(finished) - successes - skipped
|
|
370
|
+
if budget is not None and not spent_seen:
|
|
371
|
+
console.print(
|
|
372
|
+
"[yellow]note[/yellow] the endpoint reported no cost; --max-cost could not be enforced"
|
|
373
|
+
)
|
|
374
|
+
if skipped:
|
|
375
|
+
console.print(f"[yellow]{skipped} script(s) skipped at the cost budget[/yellow]")
|
|
376
|
+
if spent:
|
|
377
|
+
console.print(f"[dim]reported spend: ${spent:.4f}[/dim]")
|
|
300
378
|
unverified = sum(
|
|
301
|
-
1 for outcome in
|
|
379
|
+
1 for outcome in finished for issue in (outcome.issues or []) if not issue.ok
|
|
302
380
|
)
|
|
303
381
|
if state.strict_quotes and unverified:
|
|
304
382
|
failures += 0
|
|
305
|
-
logger.run_end(
|
|
383
|
+
logger.run_end(
|
|
384
|
+
status="ok" if failures == 0 else "partial",
|
|
385
|
+
results=successes,
|
|
386
|
+
failures=failures,
|
|
387
|
+
skipped=skipped,
|
|
388
|
+
cost_usd=round(spent, 6),
|
|
389
|
+
)
|
|
306
390
|
|
|
307
|
-
if batch_mode and successes:
|
|
391
|
+
if batch_mode and (successes or skipped):
|
|
308
392
|
if not state.json_output:
|
|
309
393
|
console.print(
|
|
310
|
-
render_summary_table(
|
|
394
|
+
render_summary_table(
|
|
395
|
+
[(o.name, o.breakdown, o.error, o.skipped) for o in finished]
|
|
396
|
+
)
|
|
311
397
|
)
|
|
312
398
|
ranked = reporting.rank(report_entries)
|
|
313
399
|
if top:
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import threading
|
|
5
6
|
import time
|
|
6
7
|
from typing import Any, Callable
|
|
7
8
|
|
|
@@ -52,7 +53,7 @@ class ChatClient:
|
|
|
52
53
|
self.model = model
|
|
53
54
|
self.retries = max(0, retries)
|
|
54
55
|
self.max_tokens = max_tokens
|
|
55
|
-
self.
|
|
56
|
+
self._usage = threading.local()
|
|
56
57
|
self._sleep = sleep
|
|
57
58
|
self._client = httpx.Client(
|
|
58
59
|
base_url=self.base_url,
|
|
@@ -62,7 +63,7 @@ class ChatClient:
|
|
|
62
63
|
"Authorization": f"Bearer {api_key}",
|
|
63
64
|
"Content-Type": "application/json",
|
|
64
65
|
"Accept": "application/json",
|
|
65
|
-
"User-Agent": "scamscript/0.
|
|
66
|
+
"User-Agent": "scamscript/0.4.1",
|
|
66
67
|
},
|
|
67
68
|
)
|
|
68
69
|
|
|
@@ -75,6 +76,24 @@ class ChatClient:
|
|
|
75
76
|
def close(self) -> None:
|
|
76
77
|
self._client.close()
|
|
77
78
|
|
|
79
|
+
@property
|
|
80
|
+
def last_usage(self) -> dict[str, int] | None:
|
|
81
|
+
return getattr(self._usage, "value", None)
|
|
82
|
+
|
|
83
|
+
@last_usage.setter
|
|
84
|
+
def last_usage(self, value: dict[str, int] | None) -> None:
|
|
85
|
+
self._usage.value = value
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def last_cost(self) -> float | None:
|
|
89
|
+
usage = self.last_usage
|
|
90
|
+
if not usage:
|
|
91
|
+
return None
|
|
92
|
+
try:
|
|
93
|
+
return float(usage.get("cost") or 0.0)
|
|
94
|
+
except (TypeError, ValueError):
|
|
95
|
+
return None
|
|
96
|
+
|
|
78
97
|
@property
|
|
79
98
|
def endpoint(self) -> str:
|
|
80
99
|
return f"{self.base_url}/chat/completions"
|
|
@@ -7,6 +7,7 @@ from dataclasses import dataclass, field
|
|
|
7
7
|
|
|
8
8
|
DEFAULT_TIMEOUT = 120.0
|
|
9
9
|
DEFAULT_RETRIES = 2
|
|
10
|
+
DEFAULT_CONCURRENCY = 1
|
|
10
11
|
DEFAULT_MAX_TOKENS = 4096
|
|
11
12
|
ENV_API_KEY = "API_KEY"
|
|
12
13
|
ENV_HOME = "SCAMSCRIPT_HOME"
|
|
@@ -25,11 +26,16 @@ class Settings:
|
|
|
25
26
|
retries: int = DEFAULT_RETRIES
|
|
26
27
|
max_tokens: int = DEFAULT_MAX_TOKENS
|
|
27
28
|
redact: bool = False
|
|
29
|
+
concurrency: int = DEFAULT_CONCURRENCY
|
|
30
|
+
max_cost: float | None = None
|
|
28
31
|
home: str = field(default_factory=lambda: os.environ.get(ENV_HOME, ""))
|
|
29
32
|
|
|
30
33
|
def __post_init__(self) -> None:
|
|
31
34
|
self.base_url = (self.base_url or "").strip().rstrip("/")
|
|
32
35
|
self.model = (self.model or "").strip()
|
|
36
|
+
self.concurrency = max(1, min(16, self.concurrency))
|
|
37
|
+
if self.max_cost is not None:
|
|
38
|
+
self.max_cost = max(0.0, self.max_cost)
|
|
33
39
|
home = self.home or os.path.join(os.path.expanduser("~"), ".scamscript")
|
|
34
40
|
self.home = home
|
|
35
41
|
|
|
@@ -207,14 +207,19 @@ def render_prompt(source: ScriptFile, messages: list[dict[str, str]]) -> Panel:
|
|
|
207
207
|
)
|
|
208
208
|
|
|
209
209
|
|
|
210
|
-
def render_summary_table(
|
|
210
|
+
def render_summary_table(
|
|
211
|
+
rows: list[tuple[str, ScriptBreakdown | None, str | None, bool]]
|
|
212
|
+
) -> Table:
|
|
211
213
|
table = Table(title="Run summary", header_style="bold", expand=True)
|
|
212
214
|
table.add_column("Script", overflow="fold", ratio=3)
|
|
213
215
|
table.add_column("Attack type", ratio=2)
|
|
214
216
|
table.add_column("Score", justify="right")
|
|
215
217
|
table.add_column("Flags", justify="right")
|
|
216
218
|
table.add_column("Status")
|
|
217
|
-
for name, breakdown, error in rows:
|
|
219
|
+
for name, breakdown, error, skipped in rows:
|
|
220
|
+
if skipped:
|
|
221
|
+
table.add_row(name, "-", "-", "-", "[yellow]skipped[/yellow]")
|
|
222
|
+
continue
|
|
218
223
|
if breakdown is None:
|
|
219
224
|
table.add_row(name, "-", "-", "-", f"[red]{error or 'failed'}[/red]")
|
|
220
225
|
continue
|
|
@@ -29,6 +29,7 @@ class ReportEntry:
|
|
|
29
29
|
breakdown: ScriptBreakdown | None = None
|
|
30
30
|
error: str | None = None
|
|
31
31
|
duration: float = 0.0
|
|
32
|
+
skipped: bool = False
|
|
32
33
|
verification: dict[str, int] | None = None
|
|
33
34
|
unverified: list[dict[str, str]] | None = None
|
|
34
35
|
|
|
@@ -42,6 +43,7 @@ def entry_from_outcome(outcome: Any) -> ReportEntry:
|
|
|
42
43
|
name=outcome.name,
|
|
43
44
|
source=str(getattr(outcome, "source", outcome.name)),
|
|
44
45
|
breakdown=outcome.breakdown,
|
|
46
|
+
skipped=getattr(outcome, "skipped", False),
|
|
45
47
|
error=outcome.error,
|
|
46
48
|
duration=getattr(outcome, "duration", 0.0),
|
|
47
49
|
verification=getattr(outcome, "verification", None),
|
|
@@ -68,6 +70,7 @@ def entries_from_run(run: dict[str, Any]) -> list[ReportEntry]:
|
|
|
68
70
|
name=record.get("name") or record.get("source") or "unknown",
|
|
69
71
|
source=record.get("source") or "",
|
|
70
72
|
breakdown=breakdown,
|
|
73
|
+
skipped=bool(record.get("skipped")),
|
|
71
74
|
error=None if breakdown else record.get("error") or "no breakdown stored",
|
|
72
75
|
duration=float(record.get("duration_s") or 0.0),
|
|
73
76
|
verification=record.get("verification") or None,
|
|
@@ -96,7 +99,8 @@ def build_markdown(
|
|
|
96
99
|
) -> str:
|
|
97
100
|
ranked = rank(entries)
|
|
98
101
|
ok = [entry for entry in ranked if entry.ok]
|
|
99
|
-
|
|
102
|
+
skipped = [entry for entry in ranked if entry.skipped and not entry.ok]
|
|
103
|
+
failed = [entry for entry in ranked if not entry.ok and not entry.skipped]
|
|
100
104
|
average = (
|
|
101
105
|
sum(entry.breakdown.sophistication_score for entry in ok) / len(ok) if ok else 0.0
|
|
102
106
|
)
|
|
@@ -112,6 +116,8 @@ def build_markdown(
|
|
|
112
116
|
lines.append(f"- Scripts analyzed: {len(entries)}")
|
|
113
117
|
lines.append(f"- Successful: {len(ok)}")
|
|
114
118
|
lines.append(f"- Failed: {len(failed)}")
|
|
119
|
+
if skipped:
|
|
120
|
+
lines.append(f"- Skipped at budget: {len(skipped)}")
|
|
115
121
|
unverified_total = sum(len(entry.unverified or []) for entry in ok)
|
|
116
122
|
if unverified_total:
|
|
117
123
|
lines.append(f"- Unverified quotes: {unverified_total}")
|
|
@@ -137,6 +143,13 @@ def build_markdown(
|
|
|
137
143
|
lines.append("_No successful analyses in this run._")
|
|
138
144
|
lines.append("")
|
|
139
145
|
|
|
146
|
+
if skipped:
|
|
147
|
+
lines.append("## Skipped")
|
|
148
|
+
lines.append("")
|
|
149
|
+
for entry in skipped:
|
|
150
|
+
lines.append(f"- **{_cell(entry.name)}** (`{_cell(entry.source)}`)")
|
|
151
|
+
lines.append("")
|
|
152
|
+
|
|
140
153
|
if failed:
|
|
141
154
|
lines.append("## Failures")
|
|
142
155
|
lines.append("")
|
|
@@ -32,6 +32,7 @@ class FakeClient:
|
|
|
32
32
|
self.base_url = base_url
|
|
33
33
|
self.model = model
|
|
34
34
|
self.max_tokens = kwargs.get("max_tokens", 4096)
|
|
35
|
+
self.last_cost = None
|
|
35
36
|
|
|
36
37
|
def __enter__(self):
|
|
37
38
|
return self
|
|
@@ -227,7 +228,7 @@ def test_dry_run_without_endpoint(home, tmp_path, monkeypatch):
|
|
|
227
228
|
def test_version(home):
|
|
228
229
|
result = runner.invoke(cli.app, ["--version"])
|
|
229
230
|
assert result.exit_code == 0
|
|
230
|
-
assert result.stdout.strip() == "scamscript 0.
|
|
231
|
+
assert result.stdout.strip() == "scamscript 0.4.1"
|
|
231
232
|
|
|
232
233
|
|
|
233
234
|
def test_json_output_is_one_object_per_script(home, tmp_path, fake):
|
|
@@ -341,3 +342,70 @@ def test_json_result_carries_verification(home, tmp_path, fake):
|
|
|
341
342
|
record = json.loads(result.stdout.splitlines()[0])
|
|
342
343
|
assert record["verification"] == {"checked": 1, "verbatim": 0, "close": 0, "missing": 1}
|
|
343
344
|
assert record["unverified"][0]["status"] == "missing"
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def test_concurrency_keeps_output_order(home, tmp_path, fake):
|
|
348
|
+
for index in range(6):
|
|
349
|
+
(tmp_path / f"s{index}.txt").write_text(f"CALLER: script {index}", encoding="utf-8")
|
|
350
|
+
result = runner.invoke(
|
|
351
|
+
cli.app, ["--api-key", "k", "--json", "--concurrency", "4", "batch", str(tmp_path)]
|
|
352
|
+
)
|
|
353
|
+
assert result.exit_code == 0
|
|
354
|
+
records = [json.loads(line) for line in result.stdout.splitlines() if line.strip()]
|
|
355
|
+
names = [record["name"] for record in records if "name" in record]
|
|
356
|
+
assert names[:6] == [f"s{i}.txt" for i in range(6)]
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def test_concurrency_runs_in_parallel(home, tmp_path, fake, monkeypatch):
|
|
360
|
+
import threading, time
|
|
361
|
+
|
|
362
|
+
active = {"now": 0, "peak": 0}
|
|
363
|
+
lock = threading.Lock()
|
|
364
|
+
|
|
365
|
+
def complete(self, messages, **kwargs):
|
|
366
|
+
with lock:
|
|
367
|
+
active["now"] += 1
|
|
368
|
+
active["peak"] = max(active["peak"], active["now"])
|
|
369
|
+
time.sleep(0.05)
|
|
370
|
+
with lock:
|
|
371
|
+
active["now"] -= 1
|
|
372
|
+
return json.dumps(breakdown())
|
|
373
|
+
|
|
374
|
+
monkeypatch.setattr(FakeClient, "complete", complete)
|
|
375
|
+
for index in range(6):
|
|
376
|
+
(tmp_path / f"s{index}.txt").write_text("CALLER: x", encoding="utf-8")
|
|
377
|
+
|
|
378
|
+
result = runner.invoke(
|
|
379
|
+
cli.app, ["--api-key", "k", "--concurrency", "4", "batch", str(tmp_path)]
|
|
380
|
+
)
|
|
381
|
+
assert result.exit_code == 0
|
|
382
|
+
assert active["peak"] > 1
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def test_max_cost_skips_remaining(home, tmp_path, fake, monkeypatch):
|
|
386
|
+
for index in range(4):
|
|
387
|
+
(tmp_path / f"s{index}.txt").write_text("CALLER: x", encoding="utf-8")
|
|
388
|
+
|
|
389
|
+
def complete(self, messages, **kwargs):
|
|
390
|
+
self.last_cost = 0.01
|
|
391
|
+
return json.dumps(breakdown())
|
|
392
|
+
|
|
393
|
+
monkeypatch.setattr(FakeClient, "complete", complete)
|
|
394
|
+
out = tmp_path / "r.md"
|
|
395
|
+
result = runner.invoke(
|
|
396
|
+
cli.app,
|
|
397
|
+
["--api-key", "k", "--max-cost", "0.015", "batch", str(tmp_path), "--output", str(out)],
|
|
398
|
+
)
|
|
399
|
+
assert "skipped at the cost budget" in result.stdout
|
|
400
|
+
markdown = out.read_text()
|
|
401
|
+
assert "Skipped at budget" in markdown
|
|
402
|
+
assert "## Skipped" in markdown
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def test_max_cost_warns_when_provider_reports_nothing(home, tmp_path, fake):
|
|
406
|
+
(tmp_path / "a.txt").write_text("CALLER: x", encoding="utf-8")
|
|
407
|
+
result = runner.invoke(
|
|
408
|
+
cli.app, ["--api-key", "k", "--max-cost", "0.5", "analyze", "--file", str(tmp_path / "a.txt")]
|
|
409
|
+
)
|
|
410
|
+
assert result.exit_code == 0
|
|
411
|
+
assert "could not be enforced" in result.stdout
|
|
@@ -276,3 +276,36 @@ def test_length_with_only_reasoning_is_truncation():
|
|
|
276
276
|
with pytest.raises(TruncatedError) as excinfo:
|
|
277
277
|
client.complete(MESSAGES)
|
|
278
278
|
assert excinfo.value.partial == "thinking..."
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def test_usage_is_thread_local():
|
|
282
|
+
import threading
|
|
283
|
+
|
|
284
|
+
def handler(request):
|
|
285
|
+
content = json.loads(request.read())["messages"][-1]["content"]
|
|
286
|
+
return httpx.Response(
|
|
287
|
+
200,
|
|
288
|
+
json={
|
|
289
|
+
"choices": [{"message": {"content": content}}],
|
|
290
|
+
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "cost": float(len(content))},
|
|
291
|
+
},
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
client = ChatClient(
|
|
295
|
+
api_key="k",
|
|
296
|
+
base_url="https://example.test/v1",
|
|
297
|
+
transport=httpx.MockTransport(handler),
|
|
298
|
+
)
|
|
299
|
+
seen = {}
|
|
300
|
+
|
|
301
|
+
def run(role, text):
|
|
302
|
+
client.complete([{"role": role, "content": text}])
|
|
303
|
+
seen[role] = client.last_cost
|
|
304
|
+
|
|
305
|
+
threads = [threading.Thread(target=run, args=("a", "xx")), threading.Thread(target=run, args=("b", "xxxx"))]
|
|
306
|
+
for thread in threads:
|
|
307
|
+
thread.start()
|
|
308
|
+
for thread in threads:
|
|
309
|
+
thread.join()
|
|
310
|
+
client.close()
|
|
311
|
+
assert seen == {"a": 2.0, "b": 4.0}
|
|
@@ -76,9 +76,14 @@ def test_prompt_panel():
|
|
|
76
76
|
|
|
77
77
|
|
|
78
78
|
def test_summary_table_row_states():
|
|
79
|
-
text = capture(
|
|
79
|
+
text = capture(
|
|
80
|
+
render_summary_table(
|
|
81
|
+
[("ok.txt", breakdown(), None, False), ("bad.txt", None, "boom", False), ("skip.txt", None, "skipped", True)]
|
|
82
|
+
)
|
|
83
|
+
)
|
|
80
84
|
assert "ok.txt" in text
|
|
81
85
|
assert "boom" in text
|
|
86
|
+
assert "skipped" in text
|
|
82
87
|
|
|
83
88
|
|
|
84
89
|
def test_panel_shows_quote_check():
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|