pyhttp-app 2.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pyhttp/runner.py ADDED
@@ -0,0 +1,1089 @@
1
+ """Executes HTTP requests and sequential runs, writing every response to disk."""
2
+ from __future__ import annotations
3
+
4
+ import asyncio
5
+ import base64
6
+ import json
7
+ import mimetypes
8
+ import re
9
+ import time
10
+ from datetime import datetime
11
+ from http.cookiejar import Cookie, CookieJar
12
+ from pathlib import Path
13
+ from typing import Any
14
+ from urllib.parse import urlsplit
15
+
16
+ import httpx
17
+
18
+ from . import USER_AGENT, assertions, console, db, jsengine, oauth2, scripts
19
+ from .jsonc import strip_comments
20
+ from .templating import build_context, render, unresolved
21
+
22
+ MAX_STORED_BODY = 512_000 # chars of body text kept in SQLite; files always hold the full body
23
+ MAX_INLINE_BINARY = 5_000_000 # bytes of binary response shipped to the UI as base64 (images, PDFs)
24
+
25
+ _BODY_CONTENT_TYPES = {
26
+ "json": "application/json",
27
+ "text": "text/plain",
28
+ "xml": "application/xml",
29
+ "html": "text/html",
30
+ "javascript": "application/javascript",
31
+ "graphql": "application/json",
32
+ }
33
+ TEXT_BODY_TYPES = ("json", "text", "xml", "html", "javascript", "graphql")
34
+
35
+
36
+ def _slug(name: str) -> str:
37
+ s = re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip("-")
38
+ return (s or "request")[:60]
39
+
40
+
41
+ def _ext_for(content_type: str) -> str:
42
+ ct = content_type.split(";")[0].strip().lower()
43
+ if "json" in ct:
44
+ return ".json"
45
+ if "html" in ct:
46
+ return ".html"
47
+ if "xml" in ct:
48
+ return ".xml"
49
+ if ct.startswith("text/") or "javascript" in ct:
50
+ return ".txt"
51
+ if ct.startswith("image/"):
52
+ return "." + ct.split("/")[1].split("+")[0]
53
+ if ct == "application/pdf":
54
+ return ".pdf"
55
+ return ".bin"
56
+
57
+
58
+ def _enabled_pairs(rows: list[dict[str, Any]], ctx: dict[str, Any]) -> list[tuple[str, str]]:
59
+ out = []
60
+ for r in rows or []:
61
+ if not r.get("enabled", True):
62
+ continue
63
+ key = render(r.get("key", ""), ctx).strip()
64
+ if key:
65
+ out.append((key, render(r.get("value", ""), ctx)))
66
+ return out
67
+
68
+
69
+ def parse_form_lines(text: str) -> list[dict[str, Any]]:
70
+ """`key: value` per line (or key=value); `// ` disables a line; `key: @path` is a file field."""
71
+ rows: list[dict[str, Any]] = []
72
+ for line in (text or "").splitlines():
73
+ l = line.strip()
74
+ if not l:
75
+ continue
76
+ enabled = True
77
+ if l.startswith("//"):
78
+ enabled, l = False, l[2:].strip()
79
+ ci, ei = l.find(":"), l.find("=")
80
+ i = ei if ei != -1 and (ci == -1 or ei < ci) else ci
81
+ key = (l if i == -1 else l[:i]).strip()
82
+ value = "" if i == -1 else l[i + 1:].strip()
83
+ if not key and not value:
84
+ continue
85
+ is_file = value.startswith("@") and len(value) > 1
86
+ rows.append({"key": key, "value": value[1:] if is_file else value, "file": is_file, "enabled": enabled})
87
+ return rows
88
+
89
+
90
+ def _path_variables(url: str) -> list[str]:
91
+ """`:id` style segments in the path part of a URL."""
92
+ path = url.split("?", 1)[0]
93
+ return re.findall(r"(?<=/):([A-Za-z_][A-Za-z0-9_]*)", path)
94
+
95
+
96
+ def _apply_path_variables(url: str, params: list[tuple[str, str]]) -> tuple[str, list[tuple[str, str]]]:
97
+ names = _path_variables(url)
98
+ if not names:
99
+ return url, params
100
+ values = {k: v for k, v in params if k in names}
101
+ rest = [(k, v) for k, v in params if k not in names]
102
+ path, sep, query = url.partition("?")
103
+ for n in names:
104
+ if n in values:
105
+ path = re.sub(rf"(?<=/):{re.escape(n)}(?=/|$)", values[n], path)
106
+ return path + sep + query, rest
107
+
108
+
109
+ def _host_matches(host: str, pattern: str) -> bool:
110
+ pattern = pattern.strip().lower()
111
+ host = host.lower()
112
+ if not pattern:
113
+ return False
114
+ if pattern.startswith("*."):
115
+ return host.endswith(pattern[1:]) or host == pattern[2:]
116
+ return host == pattern or host.endswith("." + pattern)
117
+
118
+
119
+ # ------------------------------------------------------------------ cookies
120
+
121
+
122
+ def _jar_from_db() -> CookieJar:
123
+ jar = CookieJar()
124
+ now = time.time()
125
+ for c in db.list_cookies():
126
+ if c.get("expires") and c["expires"] < now:
127
+ continue
128
+ jar.set_cookie(_cookie(c))
129
+ return jar
130
+
131
+
132
+ def _cookie(c: dict[str, Any]) -> Cookie:
133
+ domain = c["domain"]
134
+ return Cookie(
135
+ version=0, name=c["name"], value=c.get("value", ""), port=None, port_specified=False,
136
+ domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith("."),
137
+ path=c.get("path") or "/", path_specified=True, secure=bool(c.get("secure")),
138
+ expires=int(c["expires"]) if c.get("expires") else None, discard=not c.get("expires"),
139
+ comment=None, comment_url=None, rest={"HttpOnly": None} if c.get("http_only") else {}, rfc2109=False,
140
+ )
141
+
142
+
143
+ def jar_to_list(jar: CookieJar) -> list[dict[str, Any]]:
144
+ out = []
145
+ for ck in jar:
146
+ out.append({
147
+ "name": ck.name, "value": ck.value or "", "domain": ck.domain, "path": ck.path,
148
+ "expires": ck.expires, "secure": bool(ck.secure), "http_only": ck.has_nonstandard_attr("HttpOnly"),
149
+ })
150
+ return out
151
+
152
+
153
+ # ------------------------------------------------------------------ prepare
154
+
155
+
156
+ def apply_pre_vars(req: dict[str, Any], ctx: dict[str, Any]) -> list[dict[str, Any]]:
157
+ """Pre-request variables: evaluated before the request, saved to env/global when asked."""
158
+ applied = []
159
+ for row in req.get("pre_vars") or []:
160
+ if not row.get("enabled", True) or not (row.get("key") or "").strip():
161
+ continue
162
+ name = row["key"].strip()
163
+ value = render(row.get("value", ""), ctx)
164
+ scope = row.get("scope") or "local"
165
+ ctx["vars"][name] = value
166
+ if scope in ("env", "global"):
167
+ scope = db.set_variable(name, value, scope)
168
+ else:
169
+ ctx.setdefault("locals", {})[name] = value
170
+ applied.append({"name": name, "value": value, "scope": scope})
171
+ return applied
172
+
173
+
174
+ def prepare(req: dict[str, Any], ctx: dict[str, Any], settings: dict[str, Any] | None = None) -> dict[str, Any]:
175
+ """Resolve everything that will be sent, without sending it.
176
+
177
+ Returns {method, url, params, headers, body, body_kind, files, form, auth, follow_redirects,
178
+ unresolved, auth_from, content_type}.
179
+ """
180
+ settings = settings or db.app_settings()
181
+ method = (req.get("method") or "GET").upper()
182
+ url = render(req.get("url", ""), ctx).strip()
183
+ params = _enabled_pairs(req.get("params") or [], ctx)
184
+ url, params = _apply_path_variables(url, params)
185
+ inh = db.inherited(req.get("folder_id")) if req.get("folder_id") else {"auth": {"type": "none"}, "auth_from": None, "headers": []}
186
+
187
+ headers: list[tuple[str, str]] = []
188
+ seen: dict[str, int] = {}
189
+ for k, v in _enabled_pairs(inh["headers"], ctx) + _enabled_pairs(req.get("headers") or [], ctx):
190
+ if k.lower() in seen:
191
+ headers[seen[k.lower()]] = (k, v)
192
+ else:
193
+ seen[k.lower()] = len(headers)
194
+ headers.append((k, v))
195
+
196
+ def has_header(name: str) -> bool:
197
+ return name.lower() in seen
198
+
199
+ def add_header(name: str, value: str) -> None:
200
+ seen[name.lower()] = len(headers)
201
+ headers.append((name, value))
202
+
203
+ # ---- auth
204
+ auth = db.normalize_auth(req.get("auth"))
205
+ auth_from = None
206
+ if auth["type"] == "inherit":
207
+ auth, auth_from = inh["auth"], inh["auth_from"]
208
+ auth_raw = [v for k, v in auth.items() if isinstance(v, str) and k != "type"] if auth["type"] != "none" else []
209
+ auth = {k: (render(v, ctx) if isinstance(v, str) else v) for k, v in auth.items()}
210
+ digest: tuple[str, str] | None = None
211
+ if auth["type"] == "basic":
212
+ token = base64.b64encode(f"{auth['username']}:{auth['password']}".encode()).decode()
213
+ add_header("Authorization", f"Basic {token}")
214
+ elif auth["type"] == "bearer":
215
+ prefix = (auth.get("prefix") or "").strip()
216
+ add_header("Authorization", f"{prefix} {auth['token']}".strip())
217
+ elif auth["type"] == "apikey" and auth["key"]:
218
+ if auth["add_to"] == "query":
219
+ params.append((auth["key"], auth["value"]))
220
+ else:
221
+ add_header(auth["key"], auth["value"])
222
+ elif auth["type"] == "digest":
223
+ digest = (auth["username"], auth["password"])
224
+ oauth: dict[str, Any] | None = None
225
+ if auth["type"] == "oauth2" and not has_header("Authorization"): # an explicit header wins
226
+ oauth = auth
227
+ tok = oauth2.cached(auth)
228
+ prefix = (auth.get("prefix") or "").strip()
229
+ # without a cached token the header is filled in by execute() after fetching one
230
+ add_header("Authorization", f"{prefix} {tok['access_token'] if tok else '<OAuth 2.0 token>'}".strip())
231
+
232
+ # ---- body
233
+ body_type = req.get("body_type") or "none"
234
+ raw = req.get("body") or ""
235
+ body_text: str | None = None
236
+ body_kind = "none"
237
+ files: list[tuple[str, tuple[str, bytes, str | None]]] = []
238
+ form: list[tuple[str, str]] = []
239
+ body_bytes: bytes | None = None
240
+ if body_type in TEXT_BODY_TYPES and raw.strip():
241
+ text = strip_comments(raw) if body_type in ("json", "graphql") else raw
242
+ body_text = render(text, ctx)
243
+ if body_type == "graphql":
244
+ body_text = _graphql_body(body_text)
245
+ body_kind = "text"
246
+ if not has_header("Content-Type"):
247
+ add_header("Content-Type", _BODY_CONTENT_TYPES[body_type])
248
+ elif body_type in ("form", "multipart"):
249
+ rows = [r for r in parse_form_lines(raw) if r["enabled"]]
250
+ for r in rows:
251
+ key = render(r["key"], ctx)
252
+ value = render(r["value"], ctx)
253
+ if body_type == "multipart" and r["file"]:
254
+ p = Path(value).expanduser()
255
+ data = p.read_bytes() if p.is_file() else b""
256
+ files.append((key, (p.name, data, mimetypes.guess_type(p.name)[0])))
257
+ else:
258
+ form.append((key, value))
259
+ body_kind = "multipart" if body_type == "multipart" else "form"
260
+ if body_kind == "form":
261
+ body_text = "&".join(f"{httpx.QueryParams({k: v})}" for k, v in form)
262
+ if not has_header("Content-Type"):
263
+ add_header("Content-Type", "application/x-www-form-urlencoded")
264
+ elif body_type == "binary" and raw.strip():
265
+ p = Path(render(raw.strip(), ctx)).expanduser()
266
+ body_bytes = p.read_bytes() if p.is_file() else None
267
+ body_kind = "binary"
268
+ body_text = f"<file: {p}>" + ("" if p.is_file() else " (not found)")
269
+ if not has_header("Content-Type"):
270
+ add_header("Content-Type", mimetypes.guess_type(p.name)[0] or "application/octet-stream")
271
+
272
+ if not has_header("User-Agent"):
273
+ add_header("User-Agent", (settings.get("user_agent") or "").strip() or USER_AGENT)
274
+
275
+ options = req.get("options") or {}
276
+ follow = options.get("follow_redirects")
277
+ if follow is None:
278
+ follow = settings.get("follow_redirects", True)
279
+
280
+ checked = [req.get("url", ""), *auth_raw] + [r.get("key", "") + r.get("value", "") for r in (req.get("params") or []) + (req.get("headers") or []) if r.get("enabled", True)]
281
+ if body_type in TEXT_BODY_TYPES:
282
+ checked.append(strip_comments(raw) if body_type in ("json", "graphql") else raw)
283
+ elif body_type in ("form", "multipart", "binary"):
284
+ checked.append(raw)
285
+ missing: list[str] = []
286
+ for text in checked:
287
+ for name in unresolved(text, ctx):
288
+ if name not in missing:
289
+ missing.append(name)
290
+
291
+ return {
292
+ "method": method, "url": url, "params": params, "headers": headers, "body": body_text,
293
+ "body_kind": body_kind, "files": files, "form": form, "body_bytes": body_bytes,
294
+ "digest": digest, "oauth2": oauth, "follow_redirects": bool(follow), "unresolved": missing,
295
+ "auth_type": auth["type"], "auth_from": auth_from, "body_type": body_type,
296
+ "send_cookies": bool(options.get("send_cookies", True)) and settings.get("send_cookies", True),
297
+ "store_cookies": bool(options.get("store_cookies", True)),
298
+ }
299
+
300
+
301
+ def _graphql_body(text: str) -> str:
302
+ """A GraphQL body is stored as {"query": "...", "variables": ...}; variables may be a JSON string."""
303
+ try:
304
+ data = json.loads(text)
305
+ except ValueError:
306
+ return json.dumps({"query": text})
307
+ if not isinstance(data, dict):
308
+ return json.dumps({"query": text})
309
+ v = data.get("variables")
310
+ if isinstance(v, str):
311
+ try:
312
+ data["variables"] = json.loads(strip_comments(v)) if v.strip() else {}
313
+ except ValueError:
314
+ pass
315
+ return json.dumps({"query": data.get("query", ""), "variables": data.get("variables") or {},
316
+ **({"operationName": data["operationName"]} if data.get("operationName") else {})})
317
+
318
+
319
+ def public_request(p: dict[str, Any]) -> dict[str, Any]:
320
+ """The resolved request as shown to the UI (no bytes)."""
321
+ out = {k: v for k, v in p.items() if k not in ("files", "body_bytes", "digest", "oauth2")}
322
+ out["files"] = [{"field": name, "filename": fn, "size": len(data), "content_type": ct} for name, (fn, data, ct) in p.get("files", [])]
323
+ out["has_digest"] = p.get("digest") is not None
324
+ return out
325
+
326
+
327
+ # ------------------------------------------------------------------ execute
328
+
329
+
330
+ # ------------------------------------------------------------------ dependencies
331
+ # A request can depend on another one (e.g. "Get token"): that one is sent first, its extractions and
332
+ # scripts set the variables, and it is sent again only when its last result is older than the expiry.
333
+ _dep_locks: dict[str, asyncio.Lock] = {}
334
+ MAX_DEPENDENCY_DEPTH = 5
335
+
336
+
337
+ def _dependency_of(req: dict[str, Any]) -> dict[str, Any] | None:
338
+ dep = (req.get("options") or {}).get("depends")
339
+ return dep if isinstance(dep, dict) and dep.get("request_id") else None
340
+
341
+
342
+ async def _ensure_dependency(req: dict[str, Any], dep: dict[str, Any], ctx: dict[str, Any],
343
+ force: bool = False) -> tuple[dict[str, Any], dict[str, Any] | None]:
344
+ """Send the dependency unless its last result is still fresh. Returns (info for the UI, error result
345
+ to return instead of sending `req`, or None). Parallel sends share one lock per dependency, so a
346
+ token is fetched once, not once per body."""
347
+ dep_id = str(dep["request_id"])
348
+ dep_req = db.get_request(dep_id)
349
+ if not dep_req:
350
+ info = {"name": "(deleted request)", "state": "missing"}
351
+ err = _error_result("The request this one depends on no longer exists (see its Settings tab)",
352
+ {"method": req.get("method"), "url": req.get("url"), "params": [], "headers": [], "body": None}, 0.0, req, [])
353
+ return info, err
354
+ chain = tuple(ctx.get("_dep_chain") or ()) + ((req.get("id") or "",))
355
+ if dep_id in chain or len(chain) > MAX_DEPENDENCY_DEPTH:
356
+ return {"name": dep_req["name"], "state": "loop"}, None # A -> B -> A: do not recurse
357
+ ttl = max(0, int(dep.get("ttl") or 0))
358
+ lock = _dep_locks.setdefault(dep_id, asyncio.Lock())
359
+ before, _ = db.effective_variables(req.get("folder_id"))
360
+ waiting_since = time.time()
361
+ async with lock:
362
+ last = db.dependency_ran(dep_id)
363
+ age = (time.time() - last["ran_at"]) if last else None
364
+ if not force and last and last["ok"] and last["ran_at"] >= waiting_since:
365
+ # it was fetched while this send waited for the lock (parallel bodies): use that result,
366
+ # even with "send every time" (TTL 0), instead of fetching it again once per body in a row
367
+ info = {"name": dep_req["name"], "id": dep_id, "state": "shared", "age": round(age or 0), "ttl": ttl}
368
+ elif not force and last and last["ok"] and ttl > 0 and age is not None and age < ttl:
369
+ info = {"name": dep_req["name"], "id": dep_id, "state": "fresh", "age": round(age), "ttl": ttl}
370
+ else:
371
+ item = expand(dep_req)[0]
372
+ variables, env_name = db.effective_variables(dep_req.get("folder_id"))
373
+ dctx = build_context(variables)
374
+ dctx["_dep_chain"] = chain
375
+ res = await execute(item, dctx) # the dependency may itself depend on another request
376
+ record(res, source=f"Auto: before {req.get('name') or 'request'}", req=item, environment=env_name)
377
+ ok = res["status"] == "ok" and (res.get("status_code") or 0) < 400 and not res.get("tests_failed")
378
+ db.set_dependency_ran(dep_id, ok, res.get("status_code"))
379
+ info = {"name": dep_req["name"], "id": dep_id, "state": "ran", "ok": ok, "status_code": res.get("status_code"),
380
+ "duration_ms": res.get("duration_ms"), "age": 0, "ttl": ttl, "error": res.get("error")}
381
+ if not ok:
382
+ why = res.get("error") or (f"HTTP {res.get('status_code')}" if res.get("status_code") else "failed")
383
+ if res.get("tests_failed"):
384
+ why = f"{res['tests_failed']} test(s) failed"
385
+ err = _error_result(f"Dependency \u201c{dep_req['name']}\u201d failed: {why}. This request was not sent.",
386
+ {"method": req.get("method"), "url": req.get("url"), "params": [], "headers": [], "body": None}, 0.0, req, [])
387
+ return info, err
388
+ # the dependency may have changed variables (token, ids): give the main request the new values
389
+ after, _ = db.effective_variables(req.get("folder_id"))
390
+ for key, value in after.items():
391
+ if before.get(key) != value:
392
+ ctx.setdefault("vars", {})[key] = value
393
+ return info, None
394
+
395
+
396
+ async def execute(req: dict[str, Any], ctx: dict[str, Any], on_chunk: Any = None) -> dict[str, Any]:
397
+ """Send one request, after the request it depends on when it has one. Never raises.
398
+ With on_chunk (an async callable), the response streams: the status/headers and every body
399
+ chunk are passed to it as they arrive (Server-Sent Events, chunked responses)."""
400
+ dep = _dependency_of(req)
401
+ if not dep:
402
+ return await _execute_once(req, ctx, on_chunk)
403
+ info, err = await _ensure_dependency(req, dep, ctx)
404
+ if err:
405
+ err["dependency"] = info
406
+ return err
407
+ result = await _execute_once(req, ctx, on_chunk)
408
+ if on_chunk is None and dep.get("retry_on_auth", True) and result.get("status_code") in (401, 403) and info.get("state") == "fresh":
409
+ # the cached token was rejected: refresh it once and send again
410
+ info2, err2 = await _ensure_dependency(req, dep, ctx, force=True)
411
+ if not err2:
412
+ result = await _execute_once(req, ctx)
413
+ info = {**info2, "retried": True}
414
+ else:
415
+ info = {**info2, "retried": False}
416
+ result["dependency"] = info
417
+ return result
418
+
419
+
420
+ _ssl_cache: dict[tuple[Any, ...], Any] = {}
421
+
422
+
423
+ def _ssl_context(verify: bool, cert: str | None = None, key: str | None = None) -> Any:
424
+ """A shared SSL context. Building one loads the whole CA bundle (~100 ms of blocking work), which
425
+ used to happen for every request and made parallel sends start one after another. Contexts with
426
+ a client certificate are rebuilt when the certificate or key file changes."""
427
+ stamp = tuple(Path(f).stat().st_mtime if f and Path(f).exists() else None for f in (cert, key))
428
+ k = (verify, cert, key, stamp)
429
+ ctx = _ssl_cache.get(k)
430
+ if ctx is None:
431
+ ctx = httpx.create_ssl_context(verify=verify)
432
+ if cert:
433
+ ctx.load_cert_chain(cert, key)
434
+ if len(_ssl_cache) > 32:
435
+ _ssl_cache.clear()
436
+ _ssl_cache[k] = ctx
437
+ return ctx
438
+
439
+
440
+ async def _execute_once(req: dict[str, Any], ctx: dict[str, Any], on_chunk: Any = None) -> dict[str, Any]:
441
+ """Send one request. Never raises; network errors are reported in the result."""
442
+ settings = db.app_settings()
443
+ # which named body is being sent: {{body_name}} in the request, pm/py.variables.get("body_name") in scripts
444
+ ctx.setdefault("vars", {})
445
+ ctx["vars"]["body_name"] = req.get("body_name") or ""
446
+ ctx["vars"]["body_id"] = req.get("body_id") or ""
447
+ pre = apply_pre_vars(req, ctx)
448
+ script_reports: dict[str, Any] = {}
449
+ extracted_by_script: list[dict[str, Any]] = []
450
+ sc = (req.get("options") or {}).get("scripts") or {}
451
+ # the script for all bodies runs first, then the one written for the body being sent
452
+ for key, pre_code in (("pre", sc.get("pre") or ""), ("pre_body", scripts_for_body(req)["pre"])):
453
+ if not pre_code.strip():
454
+ continue
455
+ if sc.get("lang") == "py":
456
+ req, report, local_vars = scripts.run_pre_request(pre_code, req, ctx["vars"], db.set_variable)
457
+ ctx["vars"].update(local_vars)
458
+ else:
459
+ report = jsengine.run(pre_code, _js_state(req, ctx, None), event="prerequest")
460
+ if report["ok"] and report.get("request"):
461
+ req = _apply_js_request(req, report["request"])
462
+ extracted_by_script += _apply_js_vars(report, req, ctx)
463
+ if key == "pre_body":
464
+ report["body_name"] = req.get("body_name") or ""
465
+ script_reports[key] = report
466
+ if not report["ok"]:
467
+ res = _error_result(f"Pre-request script failed: {report['error']}",
468
+ {"method": req.get("method"), "url": req.get("url"), "params": [], "headers": [], "body": None},
469
+ 0.0, req, pre)
470
+ res["scripts"] = script_reports
471
+ return res
472
+ req = {**req, "_script_extracted": extracted_by_script}
473
+ try:
474
+ p = prepare(req, ctx, settings)
475
+ except Exception as exc: # noqa: BLE001 - e.g. unreadable file
476
+ res = _error_result(f"{type(exc).__name__}: {exc}", {"method": req.get("method"), "url": req.get("url"),
477
+ "params": [], "headers": [], "body": None}, 0.0, req, pre)
478
+ res["scripts"] = script_reports
479
+ return res
480
+ resolved = {"method": p["method"], "url": p["url"], "params": p["params"], "headers": p["headers"],
481
+ "body": p["body"], "body_kind": p["body_kind"], "unresolved": p["unresolved"],
482
+ "auth_type": p["auth_type"], "auth_from": p["auth_from"],
483
+ "files": public_request(p)["files"]}
484
+ timeout = float(req.get("timeout") or settings.get("default_timeout") or 30)
485
+ started = time.perf_counter()
486
+
487
+ if not p["url"]:
488
+ return _error_result("Request has no URL", resolved, 0.0, req, pre)
489
+
490
+ host = urlsplit(p["url"]).hostname or ""
491
+ verify: Any = _ssl_context(db.verify_ssl_enabled())
492
+ for cert in settings.get("certs") or []:
493
+ if cert.get("host") and cert.get("cert") and _host_matches(host, cert["host"]):
494
+ try:
495
+ verify = _ssl_context(db.verify_ssl_enabled(), cert["cert"], cert.get("key") or None)
496
+ except Exception as exc: # noqa: BLE001
497
+ return _error_result(f"Client certificate error: {exc}", resolved, 0.0, req, pre)
498
+ break
499
+ proxy = (settings.get("proxy_url") or "").strip() or None
500
+ if proxy and any(_host_matches(host, b) for b in (settings.get("proxy_bypass") or "").split(",")):
501
+ proxy = None
502
+
503
+ jar = _jar_from_db() if p["send_cookies"] else CookieJar()
504
+ client_kwargs: dict[str, Any] = {
505
+ "timeout": timeout, "verify": verify, "follow_redirects": p["follow_redirects"],
506
+ "max_redirects": int(settings.get("max_redirects") or 10), "cookies": jar,
507
+ }
508
+ if proxy:
509
+ client_kwargs["proxy"] = proxy
510
+ # httpx replaces the URL's own query string with `params` (even an empty list): append them instead
511
+ send_url = p["url"]
512
+ if p["params"]:
513
+ send_url += ("&" if "?" in send_url else "?") + str(httpx.QueryParams(p["params"]))
514
+ send_kwargs: dict[str, Any] = {"headers": p["headers"]}
515
+ if p["body_kind"] == "text":
516
+ send_kwargs["content"] = (p["body"] or "").encode("utf-8")
517
+ elif p["body_kind"] == "form":
518
+ send_kwargs["data"] = _form_dict(p["form"])
519
+ elif p["body_kind"] == "multipart":
520
+ if p["files"]:
521
+ send_kwargs["data"] = _form_dict(p["form"])
522
+ send_kwargs["files"] = p["files"]
523
+ else: # httpx would fall back to urlencoded without files; build the multipart body by hand
524
+ boundary = "----pyhttp" + re.sub(r"[^a-z0-9]", "", str(time.time()))
525
+ send_kwargs["content"] = _multipart(p["form"], boundary)
526
+ send_kwargs["headers"] = [h for h in p["headers"] if h[0].lower() != "content-type"] + \
527
+ [("Content-Type", f"multipart/form-data; boundary={boundary}")]
528
+ elif p["body_kind"] == "binary":
529
+ if p["body_bytes"] is None:
530
+ return _error_result("Binary body file not found", resolved, 0.0, req, pre)
531
+ send_kwargs["content"] = p["body_bytes"]
532
+ if p["digest"]:
533
+ send_kwargs["auth"] = httpx.DigestAuth(*p["digest"])
534
+
535
+ async def oauth_header(force: bool = False) -> bool:
536
+ """Put a token in the Authorization header; True when it came from the cache."""
537
+ was_cached = not force and oauth2.cached(p["oauth2"]) is not None
538
+ tok = await oauth2.fetch(p["oauth2"], client_kwargs, force=force)
539
+ prefix = (p["oauth2"].get("prefix") or "").strip()
540
+ value = f"{prefix} {tok['access_token']}".strip()
541
+ send_kwargs["headers"] = [(k, value if k.lower() == "authorization" else v) for k, v in send_kwargs["headers"]]
542
+ return was_cached
543
+
544
+ try:
545
+ token_was_cached = await oauth_header() if p["oauth2"] else False
546
+ async with httpx.AsyncClient(**client_kwargs) as client:
547
+ if on_chunk is None:
548
+ resp = await client.request(p["method"], send_url, **send_kwargs)
549
+ if resp.status_code == 401 and token_was_cached and p["body_kind"] != "multipart":
550
+ # the token may have been revoked or expired early: get a new one and try once more
551
+ await resp.aclose()
552
+ await oauth_header(force=True)
553
+ resp = await client.request(p["method"], send_url, **send_kwargs)
554
+ raw = await resp.aread()
555
+ else:
556
+ # stream: status and headers out first, then every chunk as it arrives
557
+ breq = client.build_request(p["method"], send_url, **send_kwargs)
558
+ resp = await client.send(breq, stream=True)
559
+ await on_chunk({"type": "status", "status_code": resp.status_code,
560
+ "reason": resp.reason_phrase, "http_version": resp.http_version,
561
+ "headers": [(k, v) for k, v in resp.headers.items()],
562
+ "content_type": resp.headers.get("content-type", "")})
563
+ parts: list[bytes] = []
564
+ total = 0
565
+ try:
566
+ async for part in resp.aiter_bytes():
567
+ parts.append(part)
568
+ total += len(part)
569
+ if total <= MAX_STORED_BODY: # the UI shows at most this much live
570
+ await on_chunk({"type": "chunk",
571
+ "text": part.decode(resp.encoding or "utf-8", errors="replace")})
572
+ finally:
573
+ await resp.aclose()
574
+ raw = b"".join(parts)
575
+ if p["store_cookies"]:
576
+ try:
577
+ if p["send_cookies"]: # the jar held everything stored: it is the new truth
578
+ db.replace_cookies(jar_to_list(client.cookies.jar))
579
+ else: # cookies were not loaded: only add what this response set
580
+ for c in jar_to_list(client.cookies.jar):
581
+ db.upsert_cookie(c)
582
+ except Exception: # noqa: BLE001 - cookies are best effort
583
+ pass
584
+ except Exception as exc: # noqa: BLE001 - every failure must land in the result
585
+ return _error_result(f"{type(exc).__name__}: {exc}", resolved, (time.perf_counter() - started) * 1000, req, pre)
586
+
587
+ duration_ms = (time.perf_counter() - started) * 1000
588
+ resolved["url"] = str(resp.request.url)
589
+ resolved["headers"] = [(k, v) for k, v in resp.request.headers.items()]
590
+ if p["body_kind"] == "multipart":
591
+ try:
592
+ resolved["body"] = resp.request.read().decode("utf-8", errors="replace")[:MAX_STORED_BODY]
593
+ except Exception: # noqa: BLE001 - a streamed body cannot be shown; keep the field list
594
+ pass
595
+ content_type = resp.headers.get("content-type", "")
596
+ ct_main = content_type.split(";")[0].strip().lower()
597
+ is_binary = ct_main.startswith(("image/", "audio/", "video/")) or ct_main in ("application/pdf", "application/octet-stream", "application/zip")
598
+ text = "" if is_binary else (resp.text if on_chunk is None else raw.decode(resp.encoding or "utf-8", errors="replace"))
599
+ is_json = False
600
+ parsed: Any = None
601
+ if text:
602
+ try:
603
+ parsed = json.loads(text)
604
+ is_json = True
605
+ except ValueError:
606
+ is_json = False
607
+ truncated = len(text) > MAX_STORED_BODY
608
+ set_cookies = httpx.Cookies()
609
+ for r in [*resp.history, resp]:
610
+ try:
611
+ set_cookies.extract_cookies(r)
612
+ except Exception: # noqa: BLE001
613
+ pass
614
+ result: dict[str, Any] = {
615
+ "status": "ok",
616
+ "status_code": resp.status_code,
617
+ "reason": resp.reason_phrase,
618
+ "duration_ms": duration_ms,
619
+ "error": None,
620
+ "request": resolved,
621
+ "response": {
622
+ "headers": [(k, v) for k, v in resp.headers.items()],
623
+ "content_type": content_type,
624
+ "size": len(raw),
625
+ "body_text": text[:MAX_STORED_BODY],
626
+ "truncated": truncated,
627
+ "is_json": is_json,
628
+ "is_binary": is_binary,
629
+ "body_base64": base64.b64encode(raw).decode() if is_binary and len(raw) <= MAX_INLINE_BINARY else None,
630
+ "http_version": resp.http_version,
631
+ "redirects": [str(h.url) for h in resp.history],
632
+ "hops": [{"status_code": h.status_code, "url": str(h.url), "location": h.headers.get("location", "")} for h in resp.history],
633
+ "cookies": jar_to_list(set_cookies.jar),
634
+ },
635
+ "pre_vars": pre,
636
+ "scripts": script_reports,
637
+ "_json": parsed,
638
+ "_content": raw,
639
+ }
640
+ _finish(result, req, ctx)
641
+ return result
642
+
643
+
644
+ def _form_dict(pairs: list[tuple[str, str]]) -> dict[str, Any]:
645
+ d: dict[str, list[str]] = {}
646
+ for k, v in pairs:
647
+ d.setdefault(k, []).append(v)
648
+ return {k: (v[0] if len(v) == 1 else v) for k, v in d.items()}
649
+
650
+
651
+ def _multipart(form: list[tuple[str, str]], boundary: str) -> bytes:
652
+ parts = []
653
+ for k, v in form:
654
+ parts.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n".encode())
655
+ parts.append(f"--{boundary}--\r\n".encode())
656
+ return b"".join(parts)
657
+
658
+
659
+ def _error_result(error: str, resolved: dict[str, Any], duration_ms: float, req: dict[str, Any],
660
+ pre: list[dict[str, Any]]) -> dict[str, Any]:
661
+ result = {
662
+ "status": "error", "status_code": None, "reason": "", "duration_ms": duration_ms,
663
+ "error": error, "request": resolved, "response": None, "pre_vars": pre, "scripts": {}, "_content": b"", "_json": None,
664
+ }
665
+ _finish(result, req)
666
+ return result
667
+
668
+
669
+ def _js_state(req: dict[str, Any], ctx: dict[str, Any], result: dict[str, Any] | None) -> dict[str, Any]:
670
+ """Everything pm.js needs: request, response, variables by scope, run info."""
671
+ env = db.active_environment()
672
+ env_vars = {r["key"].strip(): r.get("value", "") for r in (env["variables"] if env else [])
673
+ if r.get("enabled", True) and r.get("key", "").strip()}
674
+ coll = db.inherited(req.get("folder_id"))["variables"] if req.get("folder_id") else {}
675
+ if result is not None:
676
+ r = result.get("request") or {}
677
+ request = {"method": r.get("method"), "url": r.get("url"), "headers": [list(h) for h in r.get("headers") or []],
678
+ "params": [list(p) for p in r.get("params") or []], "body": r.get("body") or "",
679
+ "body_type": req.get("body_type", "none")}
680
+ else:
681
+ request = {"method": req.get("method"), "url": req.get("url"),
682
+ "headers": [[h.get("key", ""), h.get("value", "")] for h in req.get("headers") or [] if h.get("enabled", True) and h.get("key")],
683
+ "params": [[p.get("key", ""), p.get("value", "")] for p in req.get("params") or [] if p.get("enabled", True) and p.get("key")],
684
+ "body": req.get("body") or "", "body_type": req.get("body_type", "none"),
685
+ "auth": (req.get("auth") or {}).get("type")}
686
+ response = None
687
+ if result is not None:
688
+ resp = result.get("response")
689
+ if resp:
690
+ response = {"code": result.get("status_code"), "status": result.get("reason") or "",
691
+ "headers": [list(h) for h in resp.get("headers") or []], "body": resp.get("body_text") or "",
692
+ "time_ms": result.get("duration_ms"), "size": resp.get("size"), "cookies": resp.get("cookies") or []}
693
+ else:
694
+ response = {"code": None, "status": "", "headers": [], "body": result.get("error") or "",
695
+ "time_ms": result.get("duration_ms"), "size": 0, "cookies": [], "error": result.get("error")}
696
+ local = dict(ctx.get("locals") or {}) # set by pre-request variables and scripts for this request only
697
+ for key in ("body_name", "body_id"): # which named body is being sent: pm.variables.get("body_name")
698
+ if key in (ctx.get("vars") or {}):
699
+ local.setdefault(key, ctx["vars"][key])
700
+ return {
701
+ "request": request, "response": response,
702
+ "variables": {"env": env_vars, "globals": db.get_variables(), "collection": coll,
703
+ "data": {str(k): ("" if v is None else str(v)) for k, v in (req.get("data_row") or {}).items()},
704
+ "local": local},
705
+ "environmentName": env["name"] if env else "",
706
+ "info": {"requestName": req.get("name", ""), "requestId": req.get("id") or "",
707
+ "iteration": max(0, int(req.get("iteration", 1)) - 1), "iterationCount": int(req.get("iteration_count", 1))},
708
+ }
709
+
710
+
711
+ def _apply_js_request(req: dict[str, Any], changed: dict[str, Any]) -> dict[str, Any]:
712
+ out = dict(req)
713
+ if changed.get("method"):
714
+ out["method"] = str(changed["method"]).upper()
715
+ if changed.get("url") is not None:
716
+ out["url"] = str(changed["url"])
717
+ text = lambda v: "" if v is None else str(v) # noqa: E731 - a script may add a header with an undefined value
718
+ out["headers"] = [{"key": text(k), "value": text(v), "enabled": True} for k, v in changed.get("headers") or []]
719
+ out["params"] = [{"key": text(k), "value": text(v), "enabled": True} for k, v in changed.get("params") or []]
720
+ if out.get("body_type", "none") != "none" or (changed.get("body") or "").strip():
721
+ out["body"] = changed.get("body") or ""
722
+ if out.get("body_type", "none") == "none" and out["body"].strip():
723
+ out["body_type"] = "json" if out["body"].lstrip().startswith(("{", "[")) else "text"
724
+ return out
725
+
726
+
727
+ def _apply_js_vars(report: dict[str, Any], req: dict[str, Any], ctx: dict[str, Any]) -> list[dict[str, Any]]:
728
+ """Persist environment / global / collection changes made by a script; local ones go into ctx."""
729
+ shown: list[dict[str, Any]] = []
730
+ scopes = {"env": "env", "globals": "global", "collection": "collection"}
731
+ for js_scope, scope in scopes.items():
732
+ for name, value in (report.get("vars") or {}).get(js_scope, {}).items():
733
+ used = db.set_variable(name, value, scope, req.get("folder_id"))
734
+ ctx["vars"][name] = value
735
+ shown.append({"name": name, "scope": used, "value": value, "ok": True, "note": "set by script"})
736
+ for name in (report.get("unset") or {}).get(js_scope, []):
737
+ db.unset_variable(name, scope, req.get("folder_id"))
738
+ ctx["vars"].pop(name, None)
739
+ shown.append({"name": name, "scope": scope, "value": "", "ok": True, "note": "unset by script"})
740
+ for name, value in (report.get("vars") or {}).get("local", {}).items():
741
+ ctx["vars"][name] = value
742
+ ctx.setdefault("locals", {})[name] = value
743
+ return shown
744
+
745
+
746
+ def cancelled_result(req: dict[str, Any]) -> dict[str, Any]:
747
+ """What a Send returns after the user pressed Cancel (tests are not evaluated)."""
748
+ return {
749
+ "status": "cancelled", "status_code": None, "reason": "", "duration_ms": 0.0, "error": "Cancelled by user",
750
+ "request": {"method": (req.get("method") or "GET").upper(), "url": req.get("url", ""), "params": [], "headers": [],
751
+ "body": req.get("body") or None},
752
+ "response": None, "pre_vars": [], "scripts": {}, "tests": [], "tests_failed": 0, "extracted": [],
753
+ "_content": b"", "_json": None,
754
+ }
755
+
756
+
757
+ def _finish(result: dict[str, Any], req: dict[str, Any], ctx: dict[str, Any] | None = None) -> None:
758
+ """Tests and extractions run on every result, including errors (so "status is 200" fails visibly)."""
759
+ tests = assertions.run_tests(tests_for_body(req), result)
760
+ extracted = assertions.extract(req.get("extract") or [], result)
761
+ for e in extracted:
762
+ if e["ok"]:
763
+ e["scope"] = db.set_variable(e["name"], e["value"] or "", e["scope"])
764
+ extracted = list(req.get("_script_extracted") or []) + extracted
765
+ sc = (req.get("options") or {}).get("scripts") or {}
766
+ # the test script for all bodies, then the one written for the body that was sent
767
+ for key, test_code in (("test", sc.get("test") or ""), ("test_body", scripts_for_body(req)["test"])):
768
+ if not test_code.strip():
769
+ continue
770
+ if sc.get("lang") == "py":
771
+ script_tests, report = scripts.run_tests(test_code, req, (ctx or {}).get("vars", {}), result, db.set_variable)
772
+ else:
773
+ report = jsengine.run(test_code, _js_state(req, ctx or {"vars": {}}, result), event="test")
774
+ script_tests = list(report.get("tests") or [])
775
+ extracted += _apply_js_vars(report, req, ctx or {"vars": {}})
776
+ if not report["ok"]:
777
+ script_tests.append({"name": "Test script", "passed": False, "note": report["error"] or "script error"})
778
+ if key == "test_body":
779
+ report["body_name"] = req.get("body_name") or ""
780
+ tests.extend(script_tests)
781
+ result.setdefault("scripts", {})[key] = report
782
+ result["tests"] = tests
783
+ result["tests_failed"] = sum(1 for t in tests if not t["passed"])
784
+ result["extracted"] = extracted
785
+
786
+
787
+ def tests_for_body(req: dict[str, Any]) -> list[dict[str, Any]]:
788
+ """The code-free tests that apply to the body being sent: those for all bodies plus those
789
+ scoped to this body (a scope pointing at a body that no longer exists counts as "all")."""
790
+ ids = {b.get("id") for b in req.get("bodies") or [] if b.get("id")}
791
+ current = req.get("body_id") or ""
792
+ return [t for t in req.get("tests") or [] if not t.get("body_id") or t["body_id"] not in ids or t["body_id"] == current]
793
+
794
+
795
+ def scripts_for_body(req: dict[str, Any]) -> dict[str, str]:
796
+ """The pre-request and test scripts written for the body being sent (empty when there are none).
797
+ They run after the scripts for all bodies; a script keyed by a body that no longer exists is ignored."""
798
+ sc = (req.get("options") or {}).get("scripts") or {}
799
+ ids = {b.get("id") for b in req.get("bodies") or [] if b.get("id")}
800
+ current = req.get("body_id") or ""
801
+ own = (sc.get("bodies") or {}).get(current) if current in ids else None
802
+ own = own if isinstance(own, dict) else {}
803
+ return {"pre": str(own.get("pre") or ""), "test": str(own.get("test") or "")}
804
+
805
+
806
+ def expand(req: dict[str, Any]) -> list[dict[str, Any]]:
807
+ """One run item per enabled named body; a single item when the request sends no body."""
808
+ if (req.get("body_type") or "none") == "none":
809
+ return [{**req, "body": "", "body_name": None, "body_id": None}]
810
+ enabled = [b for b in req.get("bodies") or [] if b.get("enabled", True)]
811
+ if not enabled:
812
+ return [{**req, "body": "", "body_name": None, "body_id": None}]
813
+ if len(req["bodies"]) == 1 and enabled[0].get("name") == "Default":
814
+ # a plain single-body request (e.g. migrated from before named bodies): no label
815
+ return [{**req, "body": enabled[0].get("body", ""), "body_name": None, "body_id": enabled[0].get("id")}]
816
+ return [{**req, "body": b.get("body", ""), "body_name": b.get("name"), "body_id": b.get("id")} for b in enabled]
817
+
818
+
819
+ def display_name(item: dict[str, Any]) -> str:
820
+ return f"{item['name']} · {item['body_name']}" if item.get("body_name") else item["name"]
821
+
822
+
823
+ def public_result(result: dict[str, Any]) -> dict[str, Any]:
824
+ return {k: v for k, v in result.items() if not k.startswith("_")}
825
+
826
+
827
+ def prev_context(result: dict[str, Any]) -> dict[str, Any]:
828
+ resp = result.get("response") or {}
829
+ return {
830
+ "status": result.get("status_code"),
831
+ "ok": result.get("status") == "ok",
832
+ "error": result.get("error"),
833
+ "body": resp.get("body_text", ""),
834
+ "json": result.get("_json"),
835
+ "headers": {k.lower(): v for k, v in resp.get("headers", [])},
836
+ "duration_ms": result.get("duration_ms"),
837
+ }
838
+
839
+
840
+ def snapshot(req: dict[str, Any]) -> dict[str, Any]:
841
+ """What the history keeps so a past request can be reopened as a new request."""
842
+ keys = ("name", "description", "tags", "method", "url", "params", "headers", "body_type", "auth", "timeout",
843
+ "tests", "extract", "pre_vars", "options", "folder_id")
844
+ snap = {k: req.get(k) for k in keys if k in req}
845
+ snap["body"] = req.get("body") or ""
846
+ snap["bodies"] = [{"id": "default", "name": req.get("body_name") or "Default", "body": req.get("body") or "", "enabled": True}] if req.get("body") else []
847
+ return snap
848
+
849
+
850
+ SECRET_MASK = "••••••"
851
+
852
+
853
+ def mask_secrets(obj: Any, secrets: list[str]) -> Any:
854
+ """A copy of obj with every secret value replaced by SECRET_MASK (str, list, tuple, dict)."""
855
+ if not secrets:
856
+ return obj
857
+ if isinstance(obj, str):
858
+ for s in secrets:
859
+ if s in obj:
860
+ obj = obj.replace(s, SECRET_MASK)
861
+ return obj
862
+ if isinstance(obj, dict):
863
+ return {k: mask_secrets(v, secrets) for k, v in obj.items()}
864
+ if isinstance(obj, (list, tuple)):
865
+ return type(obj)(mask_secrets(v, secrets) for v in obj)
866
+ return obj
867
+
868
+
869
+ def record(result: dict[str, Any], *, source: str, req: dict[str, Any], environment: str | None) -> None:
870
+ """Console (in memory) and history (persisted) entry for an executed request."""
871
+ console.record(result, source=source, request_name=req.get("name", ""), body_name=req.get("body_name"),
872
+ environment=environment)
873
+ resp = result.get("response")
874
+ detail = {"request": result.get("request"), "response": None, "reason": result.get("reason"),
875
+ "tests": result.get("tests"), "extracted": result.get("extracted")}
876
+ if resp:
877
+ detail["response"] = {**resp, "body_text": (resp.get("body_text") or "")[:64_000],
878
+ "body_base64": None, "truncated": resp.get("truncated") or len(resp.get("body_text") or "") > 64_000}
879
+ try:
880
+ secrets = db.secret_values(req.get("folder_id"))
881
+ detail = mask_secrets(detail, secrets)
882
+ db.add_history({
883
+ "ts": time.time(), "source": source, "method": (result.get("request") or {}).get("method", ""),
884
+ "url": mask_secrets((result.get("request") or {}).get("url", ""), secrets), "request_id": req.get("id"),
885
+ "request_name": req.get("name", ""), "body_name": req.get("body_name"), "status": result.get("status", ""),
886
+ "status_code": result.get("status_code"), "duration_ms": result.get("duration_ms"),
887
+ "error": result.get("error"), "environment": environment, "snapshot": snapshot(req), "detail": detail,
888
+ })
889
+ except Exception: # noqa: BLE001 - history must never break a send
890
+ pass
891
+
892
+
893
+ class RunManager:
894
+ """Owns background tasks for sequential runs."""
895
+
896
+ def __init__(self, output_root: Path) -> None:
897
+ self.output_root = output_root
898
+ self._tasks: dict[str, asyncio.Task[None]] = {}
899
+
900
+ def is_active(self, run_id: str) -> bool:
901
+ t = self._tasks.get(run_id)
902
+ return t is not None and not t.done()
903
+
904
+ async def start_saved(self, request_ids: list[str], options: dict[str, Any]) -> dict[str, Any]:
905
+ requests = db.get_requests(request_ids)
906
+ if not requests:
907
+ raise ValueError("No valid requests selected")
908
+ items = [item for r in requests for item in expand(r)]
909
+ data_rows = [r for r in (options.get("data") or []) if isinstance(r, dict)]
910
+ iterations = max(1, int(options.get("iterations") or 1))
911
+ if data_rows:
912
+ iterations = len(data_rows)
913
+ runs: list[dict[str, Any]] = []
914
+ for i in range(1, iterations + 1):
915
+ row = data_rows[i - 1] if data_rows else {}
916
+ runs.extend({**it, "iteration": i, "iteration_count": iterations, "seq": j, "data_row": row}
917
+ for j, it in enumerate(items, start=1))
918
+ options = {**options, "iterations": iterations, "data_rows": len(data_rows)}
919
+ options.pop("data", None)
920
+ return await self.start(runs, options)
921
+
922
+ async def start(self, requests: list[dict[str, Any]], options: dict[str, Any]) -> dict[str, Any]:
923
+ """Start a run over already-expanded items (each item is a request with one body)."""
924
+ if not requests:
925
+ raise ValueError("Nothing to send")
926
+ stamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
927
+ name = (options.get("name") or "").strip() or f"Run {stamp}"
928
+ out_dir = self.output_root / f"{stamp}_{_slug(name)}"
929
+ n = 1
930
+ while out_dir.exists():
931
+ n += 1
932
+ out_dir = self.output_root / f"{stamp}_{_slug(name)}_{n}"
933
+ out_dir.mkdir(parents=True, exist_ok=True)
934
+ options = {**options, "environment": db.effective_variables()[1]}
935
+ run = db.create_run(name, len(requests), options, str(out_dir))
936
+ task = asyncio.create_task(self._execute(run["id"], requests, options, out_dir))
937
+ self._tasks[run["id"]] = task
938
+ return run
939
+
940
+ def cancel(self, run_id: str) -> bool:
941
+ t = self._tasks.get(run_id)
942
+ if t and not t.done():
943
+ t.cancel()
944
+ return True
945
+ return False
946
+
947
+ async def _execute(
948
+ self, run_id: str, requests: list[dict[str, Any]], options: dict[str, Any], out_dir: Path
949
+ ) -> None:
950
+ env_name = db.effective_variables()[1]
951
+ source = f"Send ×{len(requests)}" if options.get("mode") == "bodies" else f"Run: {options.get('name') or 'run'}"
952
+ stop_on_error = bool(options.get("stop_on_error", False))
953
+ fail_on_http_error = bool(options.get("fail_on_http_error", True))
954
+ fail_on_test = bool(options.get("fail_on_test", True))
955
+ delay_ms = max(0.0, float(options.get("delay_ms") or 0))
956
+ multi_iter = int(options.get("iterations") or 1) > 1
957
+ parallel = bool(options.get("parallel"))
958
+ concurrency = max(1, min(64, int(options.get("concurrency") or 8)))
959
+ prev: dict[str, Any] | None = None
960
+ completed = failed = 0
961
+ status = "completed"
962
+ stopped = False
963
+ summary: dict[str, Any] = {
964
+ "run_id": run_id, "name": options.get("name"), "started_at": datetime.now().isoformat(),
965
+ "options": options, "results": [],
966
+ }
967
+ async def send_one(pos: int, req: dict[str, Any], prev: dict[str, Any] | None) -> tuple[bool, dict[str, Any]]:
968
+ """Send one item, write its files and record it. Returns (failed, context for {{prev.*}})."""
969
+ nonlocal completed, failed
970
+ started_at = time.time()
971
+ variables, _ = db.effective_variables(req.get("folder_id")) # re-read: extractions may have changed them
972
+ ctx = build_context(variables, prev, req.get("data_row"))
973
+ if req.get("data_row"):
974
+ ctx["vars"].update({str(k): ("" if v is None else str(v)) for k, v in req["data_row"].items()})
975
+ result = await execute(req, ctx)
976
+ record(result, source=source, req=req, environment=env_name)
977
+ failed_req = result["status"] == "error" or (
978
+ fail_on_http_error and (result.get("status_code") or 0) >= 400
979
+ ) or (fail_on_test and result.get("tests_failed", 0) > 0)
980
+ stem = f"{req.get('seq') or pos:02d}_{_slug(req['name'])}"
981
+ if multi_iter:
982
+ stem = f"i{req.get('iteration', 1):02d}_" + stem
983
+ if req.get("body_name"):
984
+ stem += f"__{_slug(req['body_name'])}"
985
+ body_file = detail_file = None
986
+ if result.get("response") is not None:
987
+ body_file = stem + ".body" + _ext_for(result["response"]["content_type"])
988
+ (out_dir / body_file).write_bytes(result["_content"])
989
+ detail_file = stem + ".json"
990
+ secrets = db.secret_values(req.get("folder_id"))
991
+ public_resp = None
992
+ if result.get("response"):
993
+ public_resp = {k: v for k, v in result["response"].items() if k != "body_base64"}
994
+ masked_req = mask_secrets(result["request"], secrets)
995
+ masked_extracted = mask_secrets(result.get("extracted"), secrets)
996
+ detail = {
997
+ "request": {
998
+ "id": req.get("id"), "name": req["name"], "body_name": req.get("body_name"),
999
+ "description": req.get("description"), "iteration": req.get("iteration", 1),
1000
+ "tags": req.get("tags"), "resolved": masked_req,
1001
+ },
1002
+ "status": "failed" if failed_req else "ok",
1003
+ "status_code": result["status_code"],
1004
+ "reason": result["reason"],
1005
+ "duration_ms": round(result["duration_ms"], 2),
1006
+ "error": result["error"],
1007
+ "tests": result.get("tests"),
1008
+ "extracted": masked_extracted,
1009
+ "dependency": result.get("dependency"),
1010
+ "response": public_resp,
1011
+ "body_file": body_file,
1012
+ "started_at": datetime.fromtimestamp(started_at).isoformat(),
1013
+ }
1014
+ (out_dir / detail_file).write_text(
1015
+ json.dumps(detail, indent=2, ensure_ascii=False), encoding="utf-8"
1016
+ )
1017
+ db.add_result(run_id, {
1018
+ "position": pos, "request_id": req.get("id"), "request_name": req["name"],
1019
+ "body_name": req.get("body_name"), "iteration": req.get("iteration", 1),
1020
+ "method": result["request"]["method"],
1021
+ "status": "failed" if failed_req else "ok",
1022
+ "status_code": result["status_code"], "duration_ms": result["duration_ms"],
1023
+ "error": result["error"], "tests": result.get("tests"),
1024
+ "detail": {"request": masked_req, "response": public_resp,
1025
+ "reason": result["reason"], "extracted": masked_extracted,
1026
+ "dependency": result.get("dependency")},
1027
+ "body_file": body_file, "detail_file": detail_file, "started_at": started_at,
1028
+ })
1029
+ completed += 1
1030
+ failed += int(failed_req)
1031
+ summary["results"].append({
1032
+ "position": pos, "name": req["name"], "body_name": req.get("body_name"),
1033
+ "iteration": req.get("iteration", 1),
1034
+ "method": result["request"]["method"],
1035
+ "url": masked_req["url"], "status": detail["status"],
1036
+ "status_code": result["status_code"], "duration_ms": detail["duration_ms"],
1037
+ "error": result["error"], "detail_file": detail_file, "body_file": body_file,
1038
+ "tests": result.get("tests"),
1039
+ })
1040
+ db.update_run(run_id, completed=completed, failed=failed)
1041
+ self._write_summary(out_dir, summary, "running")
1042
+ return failed_req, prev_context(result)
1043
+
1044
+ try:
1045
+ if parallel:
1046
+ # every item at once (bounded by `concurrency`); no {{prev.*}} since there is no previous response
1047
+ sem = asyncio.Semaphore(concurrency)
1048
+
1049
+ async def bounded(pos: int, req: dict[str, Any]) -> None:
1050
+ nonlocal stopped
1051
+ async with sem:
1052
+ if stopped:
1053
+ return
1054
+ failed_req, _ = await send_one(pos, req, None)
1055
+ if failed_req and stop_on_error:
1056
+ stopped = True
1057
+
1058
+ await asyncio.gather(*(bounded(pos, req) for pos, req in enumerate(requests, start=1)))
1059
+ if stopped:
1060
+ status = "stopped"
1061
+ else:
1062
+ for pos, req in enumerate(requests, start=1):
1063
+ failed_req, prev = await send_one(pos, req, prev)
1064
+ if failed_req and stop_on_error:
1065
+ status = "stopped"
1066
+ break
1067
+ if delay_ms and pos < len(requests):
1068
+ await asyncio.sleep(delay_ms / 1000)
1069
+ except asyncio.CancelledError:
1070
+ status = "cancelled"
1071
+ except Exception as exc: # noqa: BLE001
1072
+ status = "error"
1073
+ summary["error"] = f"{type(exc).__name__}: {exc}"
1074
+ finally:
1075
+ summary["results"].sort(key=lambda r: r["position"]) # parallel sends finish in any order
1076
+ summary["finished_at"] = datetime.now().isoformat()
1077
+ summary["completed"] = completed
1078
+ summary["failed"] = failed
1079
+ self._write_summary(out_dir, summary, status)
1080
+ db.update_run(run_id, status=status, completed=completed, failed=failed,
1081
+ finished_at=time.time())
1082
+ self._tasks.pop(run_id, None)
1083
+
1084
+ @staticmethod
1085
+ def _write_summary(out_dir: Path, summary: dict[str, Any], status: str) -> None:
1086
+ summary["status"] = status
1087
+ (out_dir / "summary.json").write_text(
1088
+ json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8"
1089
+ )