tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. tasklite/__init__.py +42 -18
  2. tasklite/__main__.py +205 -0
  3. tasklite/_report.py +562 -0
  4. tasklite/backend/__init__.py +19 -2
  5. tasklite/backend/base.py +327 -156
  6. tasklite/backend/memory.py +261 -339
  7. tasklite/backend/sqlite_backend.py +416 -488
  8. tasklite/edit.py +1276 -0
  9. tasklite/engine/__init__.py +6 -7
  10. tasklite/engine/_worker_bootstrap.py +221 -0
  11. tasklite/engine/_worker_runtime.py +229 -0
  12. tasklite/engine/admission.py +20 -114
  13. tasklite/engine/authorization.py +989 -0
  14. tasklite/engine/channel.py +531 -468
  15. tasklite/engine/completion.py +193 -275
  16. tasklite/engine/config.py +5 -11
  17. tasklite/engine/dispatch.py +299 -384
  18. tasklite/engine/governor.py +134 -69
  19. tasklite/engine/in_flight.py +20 -88
  20. tasklite/engine/launcher.py +1173 -0
  21. tasklite/engine/recovery.py +147 -100
  22. tasklite/engine/resource.py +50 -103
  23. tasklite/engine/runtime.py +158 -525
  24. tasklite/engine/scheduler.py +9 -46
  25. tasklite/engine/session.py +60 -15
  26. tasklite/engine/store.py +235 -462
  27. tasklite/engine/types.py +29 -40
  28. tasklite/engine/wait.py +8 -0
  29. tasklite/exceptions.py +0 -1
  30. tasklite/models/artifact.py +84 -62
  31. tasklite/models/context.py +37 -30
  32. tasklite/models/graph.py +114 -0
  33. tasklite/models/job.py +3 -5
  34. tasklite/models/records.py +482 -0
  35. tasklite/models/state.py +19 -125
  36. tasklite/models/trigger.py +63 -3
  37. tasklite/pipeline.py +92 -100
  38. tasklite/resource.py +79 -2
  39. tasklite/snapshot.py +608 -405
  40. tasklite/testing.py +424 -11
  41. tasklite/utils/artifact.py +1091 -179
  42. tasklite/utils/encoding.py +21 -40
  43. tasklite/utils/ipc.py +49 -1
  44. tasklite/utils/lockfile.py +50 -19
  45. tasklite/wrappers/__init__.py +1 -1
  46. tasklite/wrappers/discovery.py +13 -16
  47. tasklite/wrappers/http.py +77 -43
  48. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/METADATA +22 -16
  49. tasklite_engine-3.0.0.dev1.dist-info/RECORD +60 -0
  50. tasklite/contrib/__init__.py +0 -7
  51. tasklite/engine/execution_access.py +0 -39
  52. tasklite/engine/files.py +0 -52
  53. tasklite/engine/ops.py +0 -25
  54. tasklite/engine/selection.py +0 -186
  55. tasklite/hooks.py +0 -49
  56. tasklite/triggers.py +0 -47
  57. tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
  58. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/WHEEL +0 -0
  59. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/licenses/LICENSE +0 -0
  60. {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev1.dist-info}/top_level.txt +0 -0
@@ -54,18 +54,6 @@ _TRUNC_MARKER = "%_"
54
54
  # 上限的前提下完成
55
55
  _TRUNC_FIXED_LEN = len(_TRUNC_MARKER) + _TRUNC_DIGEST_HEX
56
56
 
57
- # 文件系统危险字符映射(POSIX /、Windows \、NUL、glob *?[]、冒号 :)
58
- FS_ESCAPE_CHARS: dict[str, str] = {
59
- "/": "%2F",
60
- "\\": "%5C",
61
- "\x00": "%00",
62
- "*": "%2A",
63
- "?": "%3F",
64
- "[": "%5B",
65
- "]": "%5D",
66
- ":": "%3A",
67
- }
68
-
69
57
 
70
58
  def percent_encode(
71
59
  text: str,
@@ -258,6 +246,25 @@ def _encode_uid_component(value: str, field_name: str) -> str:
258
246
  ) from exc
259
247
 
260
248
 
249
+ def _flush_uid_bytes(buf: bytearray, chars: list[str], field_name: str) -> None:
250
+ """将累积的百分号转义字节按 UTF-8 解码、校验无过度转义并冲刷到字符列表。"""
251
+ if not buf:
252
+ return
253
+ try:
254
+ decoded_run = buf.decode("utf-8")
255
+ except UnicodeDecodeError as exc:
256
+ raise TaskLiteError(
257
+ f"invalid job uid: invalid UTF-8 byte sequence in {field_name}"
258
+ ) from exc
259
+ for dch in decoded_run:
260
+ if dch.isprintable() and dch not in (":", "%"):
261
+ raise TaskLiteError(
262
+ f"invalid job uid: over-escaped character {dch!r} in {field_name}"
263
+ )
264
+ chars.append(decoded_run)
265
+ buf.clear()
266
+
267
+
261
268
  def _decode_uid_component(encoded: str, field_name: str) -> str:
262
269
  """严格解析规范 UID 组成部分并还原原始字符串。"""
263
270
  if not encoded:
@@ -282,39 +289,14 @@ def _decode_uid_component(encoded: str, field_name: str) -> str:
282
289
  buf.append(byte_val)
283
290
  i += 3
284
291
  else:
285
- if buf:
286
- try:
287
- decoded_run = buf.decode("utf-8")
288
- except UnicodeDecodeError as exc:
289
- raise TaskLiteError(
290
- f"invalid job uid: invalid UTF-8 byte sequence in {field_name}"
291
- ) from exc
292
- for dch in decoded_run:
293
- if dch.isprintable() and dch not in (":", "%"):
294
- raise TaskLiteError(
295
- f"invalid job uid: over-escaped character {dch!r} in {field_name}"
296
- )
297
- chars.append(decoded_run)
298
- buf.clear()
292
+ _flush_uid_bytes(buf, chars, field_name)
299
293
  if ch == ":" or not ch.isprintable():
300
294
  raise TaskLiteError(
301
295
  f"invalid job uid: unescaped character {ch!r} in {field_name}"
302
296
  )
303
297
  chars.append(ch)
304
298
  i += 1
305
- if buf:
306
- try:
307
- decoded_run = buf.decode("utf-8")
308
- except UnicodeDecodeError as exc:
309
- raise TaskLiteError(
310
- f"invalid job uid: invalid UTF-8 byte sequence in {field_name}"
311
- ) from exc
312
- for dch in decoded_run:
313
- if dch.isprintable() and dch not in (":", "%"):
314
- raise TaskLiteError(
315
- f"invalid job uid: over-escaped character {dch!r} in {field_name}"
316
- )
317
- chars.append(decoded_run)
299
+ _flush_uid_bytes(buf, chars, field_name)
318
300
  return "".join(chars)
319
301
 
320
302
 
@@ -346,7 +328,6 @@ __all__ = [
346
328
  "CONTENT_ID_ALLOWED",
347
329
  "DEFAULT_FORBIDDEN",
348
330
  "EMPTY_SENTINEL",
349
- "FS_ESCAPE_CHARS",
350
331
  "content_fingerprint",
351
332
  "encode_content_id",
352
333
  "encode_identifier",
tasklite/utils/ipc.py CHANGED
@@ -39,7 +39,6 @@ class _PipeConnection(Protocol):
39
39
 
40
40
 
41
41
  # 管道消息种类常量
42
- PIPE_MSG_INPUT = "input"
43
42
  PIPE_MSG_RESULT = "result"
44
43
  PIPE_MSG_DECLARE = "declare"
45
44
  VALID_PIPE_MESSAGE_KINDS = frozenset(
@@ -48,9 +47,51 @@ VALID_PIPE_MESSAGE_KINDS = frozenset(
48
47
  PIPE_MSG_DECLARE,
49
48
  }
50
49
  )
50
+ # 子进程两阶段握手角色常量
51
+ HANDSHAKE_ROLE_UPLINK = "uplink"
52
+ HANDSHAKE_ROLE_DOWNLINK = "downlink"
53
+ HANDSHAKE_ROLE_CANCEL = "cancel"
51
54
  _PIPE_EOF = object()
52
55
 
53
56
 
57
+ def encode_json_frame(payload: Any) -> bytes:
58
+ """将对象经统一严格 JSON 序列化编码为 UTF-8 帧字节。"""
59
+ return dumps(payload).encode("utf-8")
60
+
61
+
62
+ def decode_json_object_frame(raw: bytes, *, context: str = "JSON frame") -> dict[str, Any]:
63
+ """经统一严格 JSON 反序列化解析单帧对象字节。"""
64
+ try:
65
+ decoded = loads(raw.decode("utf-8"))
66
+ except (UnicodeDecodeError, json.JSONDecodeError, RecursionError, TypeError, ValueError) as exc:
67
+ raise ValueError(f"invalid {context}: {exc}") from exc
68
+ if not isinstance(decoded, dict):
69
+ raise ValueError(f"{context} must decode to a JSON object")
70
+ return decoded
71
+
72
+
73
+ def encode_handshake_frame(role: str, *, pid: int | None = None) -> bytes:
74
+ """构造子进程握手角色帧(经统一 jsonutil 编码)。"""
75
+ frame: dict[str, Any] = {"role": role}
76
+ if pid is not None:
77
+ frame["pid"] = pid
78
+ return encode_json_frame(frame)
79
+
80
+
81
+ def decode_handshake_frame(raw: bytes) -> tuple[str | None, int | None]:
82
+ """解析子进程握手角色帧,返回 (role, pid)。"""
83
+ frame = decode_json_object_frame(raw, context="worker handshake frame")
84
+ raw_role = frame.get("role")
85
+ role = raw_role if isinstance(raw_role, str) else None
86
+ raw_pid = frame.get("pid")
87
+ pid = (
88
+ raw_pid
89
+ if isinstance(raw_pid, int) and not isinstance(raw_pid, bool) and raw_pid > 0
90
+ else None
91
+ )
92
+ return role, pid
93
+
94
+
54
95
  @dataclass(frozen=True)
55
96
  class WorkerPipeMessage:
56
97
  """已校验的 worker 管道信封消息。"""
@@ -616,6 +657,9 @@ def create_worker_reply_pipe(
616
657
 
617
658
 
618
659
  __all__ = [
660
+ "HANDSHAKE_ROLE_CANCEL",
661
+ "HANDSHAKE_ROLE_DOWNLINK",
662
+ "HANDSHAKE_ROLE_UPLINK",
619
663
  "PIPE_MSG_DECLARE",
620
664
  "PIPE_MSG_RESULT",
621
665
  "VALID_PIPE_MESSAGE_KINDS",
@@ -626,5 +670,9 @@ __all__ = [
626
670
  "WorkerReplySender",
627
671
  "create_worker_pipe",
628
672
  "create_worker_reply_pipe",
673
+ "decode_handshake_frame",
674
+ "decode_json_object_frame",
629
675
  "decode_worker_pipe_message",
676
+ "encode_handshake_frame",
677
+ "encode_json_frame",
630
678
  ]
@@ -1,25 +1,41 @@
1
- """v2 跨平台单文件锁:库级执行与管理互斥(flock 互斥)。
1
+ """v2 跨平台单文件锁:库级执行与管理互斥(flock / msvcrt 互斥)。
2
2
 
3
3
  用于主进程按状态库路径获取库级互斥锁,防止并发执行或管理冲突。
4
4
  """
5
5
  from __future__ import annotations
6
6
 
7
+ import errno
7
8
  import os
8
9
  import sys
9
10
  import time
10
11
  from pathlib import Path
11
12
 
12
13
 
14
+ def lock_path_for(state_path: str | Path) -> Path:
15
+ """由状态库路径派生对应的库级互斥锁文件路径(幂等处理已有 .lock 后缀)。"""
16
+ p = Path(state_path)
17
+ if p.name.endswith(".lock"):
18
+ return p
19
+ return Path(f"{p}.lock")
20
+
21
+
13
22
  def _lock_path(directory_or_path: str | Path, name: str | None = None) -> Path:
14
23
  """name → 锁文件路径。若 name 为 None,直接派生锁文件路径。"""
15
24
  if name is None:
16
- p = Path(directory_or_path)
17
- if p.name.endswith(".lock"):
18
- return p
19
- return Path(str(p) + ".lock")
25
+ return lock_path_for(directory_or_path)
20
26
  return Path(directory_or_path) / f"{name}.lock"
21
27
 
22
28
 
29
+ def _is_lock_contention(exc: OSError) -> bool:
30
+ """判断 OSError 是否属于锁争用(而非无效 fd 或底层 IO 故障)。"""
31
+ if sys.platform == "win32":
32
+ return (
33
+ exc.errno in (errno.EACCES, errno.EDEADLK)
34
+ or getattr(exc, "winerror", None) in (33, 32)
35
+ )
36
+ return exc.errno in (errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK)
37
+
38
+
23
39
  def try_acquire_lock(
24
40
  directory_or_path: str | Path,
25
41
  name: str | None = None,
@@ -33,41 +49,53 @@ def try_acquire_lock(
33
49
  时返回 None。锁文件**创建后不删除**。
34
50
 
35
51
  Raises:
36
- OSError: 锁文件打不开/建不出(权限、磁盘满等环境故障)——与
52
+ OSError: 锁文件打不开/建不出/IO 故障(权限、磁盘满、无效 fd 等)——与
37
53
  「锁被占」语义不同,不得混入 None。
38
54
  """
39
55
  path = _lock_path(directory_or_path, name)
40
56
  path.parent.mkdir(parents=True, exist_ok=True)
41
57
  fd = os.open(str(path), os.O_RDWR | os.O_CREAT, 0o644)
42
58
  deadline = None if timeout <= 0 else time.monotonic() + timeout
43
- while True:
44
- if _try_lock_fd(fd):
45
- return fd
46
- if deadline is None:
47
- os.close(fd)
48
- return None
49
- if time.monotonic() >= deadline:
59
+ try:
60
+ while True:
61
+ if _try_lock_fd(fd):
62
+ return fd
63
+ if deadline is None or time.monotonic() >= deadline:
64
+ os.close(fd)
65
+ return None
66
+ time.sleep(0.1)
67
+ except BaseException:
68
+ try:
50
69
  os.close(fd)
51
- return None
52
- time.sleep(0.1)
70
+ except OSError:
71
+ pass
72
+ raise
53
73
 
54
74
 
55
75
  def _try_lock_fd(fd: int) -> bool:
56
76
  """对已打开的 fd 尝试非阻塞排他锁(平台分派)。"""
57
77
  if sys.platform == "win32":
58
78
  import msvcrt
79
+
80
+ # msvcrt.locking 按当前文件偏移加锁,固定归零确保始终锁第 0 字节。
81
+ os.lseek(fd, 0, os.SEEK_SET)
59
82
  try:
60
83
  msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
61
84
  return True
62
- except OSError:
63
- return False
85
+ except OSError as exc:
86
+ if _is_lock_contention(exc):
87
+ return False
88
+ raise
64
89
  else:
65
90
  import fcntl
91
+
66
92
  try:
67
93
  fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
68
94
  return True
69
- except OSError:
70
- return False
95
+ except OSError as exc:
96
+ if _is_lock_contention(exc):
97
+ return False
98
+ raise
71
99
 
72
100
 
73
101
  def release_lock(fd: int) -> None:
@@ -77,10 +105,12 @@ def release_lock(fd: int) -> None:
77
105
  try:
78
106
  if sys.platform == "win32":
79
107
  import msvcrt
108
+
80
109
  os.lseek(fd, 0, os.SEEK_SET)
81
110
  msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
82
111
  else:
83
112
  import fcntl
113
+
84
114
  fcntl.flock(fd, fcntl.LOCK_UN)
85
115
  except OSError:
86
116
  pass
@@ -91,6 +121,7 @@ def release_lock(fd: int) -> None:
91
121
 
92
122
 
93
123
  __all__ = [
124
+ "lock_path_for",
94
125
  "release_lock",
95
126
  "try_acquire_lock",
96
127
  ]
@@ -10,7 +10,7 @@
10
10
 
11
11
  分层红线:本包是公开面之上的适配层,可依赖
12
12
  ``tasklite/{models,utils,engine}`` 公开面;``tasklite/utils/`` 严禁反向
13
- import 本包,核心层严禁依赖本包与 ``tasklite/contrib/``。本子包不进
13
+ import 本包,核心层严禁依赖本包。本子包不进
14
14
  ``tasklite`` 主 ``__all__``——经 ``tasklite.wrappers.*`` 子包路径导入。
15
15
  """
16
16
 
@@ -3,11 +3,12 @@
3
3
  注意:本模块为可选工具,本版本暂不承诺向后兼容与冻结扩展协议。
4
4
  增量发现通过任务派生与框架核心状态事务整体相容,不因坏子项而允许父任务成功或部分发布。
5
5
 
6
- 架构性质:**扫描核心框架无关**——``DiscoveryHandler`` 只依赖
7
- ``DiscoveryJob``/``DiscoveryContext`` 两个最小协议,运行时**不 import
8
- 任何框架模块**;``register_discovery`` 是框架提供的默认封装函数,只通过
9
- ``DiscoveryHost`` 公开协议把扫描器挂到宿主上(TaskLite 是其默认实现)。
10
- 任何实现该协议的宿主/测试桩均可直接复用。
6
+ 架构性质:本模块是依赖核心公开模型与工具面(``TaskResult``、``TaskLiteError``、
7
+ ``validate_trigger``、``encode_content_id``、``jsonutil``)的可选适配器;
8
+ ``DiscoveryHandler`` 的扫描循环只通过 ``DiscoveryJob`` / ``DiscoveryContext``
9
+ 两个最小协议与运行时交互,``register_discovery`` 只通过 ``DiscoveryHost``
10
+ 公开协议把扫描器挂到宿主上(TaskLite 是其默认实现),不直接依赖引擎内部实现。
11
+ 任何实现对应协议的宿主或测试桩均可直接复用。
11
12
 
12
13
  「已见集合」由 discovery 自己通过公开游标持久化:每个
13
14
  ``(discovery task_type, process task_type, group key)`` 使用独立游标,
@@ -120,7 +121,6 @@ V1. 契约正确性必须由回归测试固定,至少覆盖:首次全量扫
120
121
  """
121
122
  from __future__ import annotations
122
123
 
123
- import json
124
124
  import logging
125
125
  import pickle
126
126
  from collections.abc import Callable
@@ -130,6 +130,7 @@ from ..exceptions import TaskLiteError
130
130
  from ..models.result import TaskResult
131
131
  from ..models.task import validate_trigger
132
132
  from ..utils.encoding import encode_content_id
133
+ from ..utils.jsonutil import dumps as _json_dumps, loads as _json_loads
133
134
 
134
135
  logger = logging.getLogger("tasklite")
135
136
 
@@ -137,7 +138,7 @@ _SEEN_CURSOR_VERSION = 1
137
138
  _SEEN_CURSOR_PREFIX = "__tasklite_discovery_seen_v1__:"
138
139
 
139
140
 
140
- # 框架无关协议(G-DISCOVERY):本模块的扫描核心**不 import 任何框架模块**。
141
+ # 宿主解耦协议:本模块仅依赖核心公开模型与工具面,不直接依赖引擎内部实现。
141
142
  #
142
143
  # ``DiscoveryJob`` / ``DiscoveryContext`` 是扫描循环对运行时的全部要求;
143
144
  # ``DiscoveryHost`` 是注册适配器对宿主框架的全部要求。
@@ -185,10 +186,9 @@ class DiscoveryHost(Protocol):
185
186
  class DiscoveryHandler:
186
187
  """按 REQUIREMENTS CONTRACT 实现的增量发现 handler(纯任务模型)。
187
188
 
188
- **框架无关**:本类及其扫描循环只依赖 ``DiscoveryJob`` / ``DiscoveryContext``
189
- 两个最小协议(payload + get_cursor/set_cursor),运行时
190
- **不 import 任何框架模块**,可被任何实现这两个协议的宿主直接复用;
191
- TaskLite 只是框架默认宿主。
189
+ **宿主协议解耦**:本类及其扫描循环只依赖 ``DiscoveryJob`` / ``DiscoveryContext``
190
+ 两个最小协议(payload + get_cursor/set_cursor),不直接依赖引擎内部实现,
191
+ 可被任何实现这两个协议的宿主直接复用;TaskLite 只是框架默认宿主。
192
192
 
193
193
  框架负责契约规定的全部语义:
194
194
  - 从第 1 页起逐页扫描,page 由框架传入,用户 fetch_func 只拉一页;
@@ -272,7 +272,7 @@ class DiscoveryHandler:
272
272
  if raw is None:
273
273
  return set()
274
274
  try:
275
- payload = json.loads(raw)
275
+ payload = _json_loads(raw)
276
276
  except (TypeError, ValueError) as exc:
277
277
  raise ValueError(f"invalid discovery seen cursor {key!r}: {exc}") from exc
278
278
  version = payload.get("version") if isinstance(payload, dict) else None
@@ -298,10 +298,7 @@ class DiscoveryHandler:
298
298
  "version": _SEEN_CURSOR_VERSION,
299
299
  "seen": sorted(seen),
300
300
  }
301
- ctx.set_cursor(
302
- key,
303
- json.dumps(payload, ensure_ascii=True, separators=(",", ":")),
304
- )
301
+ ctx.set_cursor(key, _json_dumps(payload))
305
302
 
306
303
  def _resolve_scan_options(self, job: DiscoveryJob) -> tuple[str, int]:
307
304
  """扫描模式与页数上限:job.payload 动态指定优先,否则用构造默认值。"""
tasklite/wrappers/http.py CHANGED
@@ -31,6 +31,55 @@ from ..utils.jsonutil import dumps as _json_dumps, loads as _json_loads
31
31
  # 节奏唯一出口是 RequeuePolicy seam)。
32
32
  _LOCAL_RETRY_WAIT_MAX_SECONDS = 300.0
33
33
 
34
+ # 默认绕过环境变量代理的本机回环主机名集合(精确匹配主机名,避免子串误判)。
35
+ _LOCAL_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
36
+
37
+
38
+ def _merge_query_items(
39
+ raw_query: str,
40
+ params: Mapping[str, Any] | None,
41
+ *,
42
+ sort: bool = False,
43
+ ) -> list[tuple[str, str]]:
44
+ """合并 URL 内嵌 query 与显式 params(None 值统一丢弃,与 requests 语义对齐)。"""
45
+ items: list[tuple[str, str]] = (
46
+ urllib.parse.parse_qsl(raw_query, keep_blank_values=True) if raw_query else []
47
+ )
48
+ if params:
49
+ for k, v in params.items():
50
+ if v is None:
51
+ continue
52
+ items.append((str(k), str(v)))
53
+ if sort:
54
+ items.sort()
55
+ return items
56
+
57
+
58
+ def _encode_body_bytes(body: bytes | str | Mapping[str, Any]) -> bytes:
59
+ """请求体统一字节序列化:dict/list 走 JSON、str 编码 UTF-8、其余按字节。"""
60
+ if isinstance(body, (dict, list)):
61
+ return _json_dumps(body).encode("utf-8")
62
+ if isinstance(body, str):
63
+ return body.encode("utf-8")
64
+ return bytes(body) # type: ignore[arg-type]
65
+
66
+
67
+ def _extract_status_code(res: Any) -> int | None:
68
+ """从任意响应形态提取 HTTP 状态码(优先 status_code,其次 getcode();无状态语义返回 None)。"""
69
+ status: int | None = getattr(res, "status_code", None)
70
+ if status is None and hasattr(res, "getcode"):
71
+ status = res.getcode()
72
+ return int(status) if status is not None else None
73
+
74
+
75
+ def _is_local_url(url: str) -> bool:
76
+ """判断 URL 目标主机是否为本机回环地址(精确匹配主机名,避免子串误判)。"""
77
+ try:
78
+ hostname = urllib.parse.urlparse(url).hostname
79
+ except ValueError:
80
+ return False
81
+ return hostname is not None and hostname.lower() in _LOCAL_LOOPBACK_HOSTS
82
+
34
83
 
35
84
  # ==============================================================================
36
85
  # 1. 基础容器与 Cookie 转换工具
@@ -296,16 +345,7 @@ class SnapshotStore:
296
345
  params: Mapping[str, Any] | None,
297
346
  ) -> list[tuple[str, str]]:
298
347
  """合并 URL 内嵌 query 与显式 params 并排序(同语义请求同键)。"""
299
- query_items: list[tuple[str, str]] = []
300
- if parsed.query:
301
- query_items.extend(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True))
302
- if params:
303
- for k, v in params.items():
304
- if v is None:
305
- continue
306
- query_items.append((str(k), str(v)))
307
- query_items.sort()
308
- return query_items
348
+ return _merge_query_items(parsed.query, params, sort=True)
309
349
 
310
350
  # 纳入快照 key 的身份头白名单:仅取真正改变响应语义的凭证/协商头。
311
351
  # 取舍:不纳入全量 headers——User-Agent/Accept-Encoding/Date 等易变头会把
@@ -420,11 +460,7 @@ class SnapshotStore:
420
460
  @staticmethod
421
461
  def _body_bytes(body: bytes | str | Mapping[str, Any]) -> bytes:
422
462
  """请求体统一序列化:dict/list 走 JSON、str 编码 UTF-8、其余按字节。"""
423
- if isinstance(body, (dict, list)):
424
- return _json_dumps(body).encode("utf-8")
425
- if isinstance(body, str):
426
- return body.encode("utf-8")
427
- return bytes(body) # type: ignore[arg-type]
463
+ return _encode_body_bytes(body)
428
464
 
429
465
  @staticmethod
430
466
  def _extract_response_fields(res: Any) -> tuple[int, dict[str, str], bytes]:
@@ -438,9 +474,7 @@ class SnapshotStore:
438
474
  """
439
475
  if isinstance(res, HttpResponse):
440
476
  return res.status_code, res.headers, res.body
441
- status_code: int | None = getattr(res, "status_code", None)
442
- if status_code is None and hasattr(res, "getcode"):
443
- status_code = res.getcode()
477
+ status_code = _extract_status_code(res)
444
478
  if status_code is None:
445
479
  status_code = 200
446
480
  headers = dict(getattr(res, "headers", {}))
@@ -452,7 +486,7 @@ class SnapshotStore:
452
486
  raw_body = _json_dumps(res).encode("utf-8")
453
487
  else:
454
488
  raw_body = str(res).encode("utf-8")
455
- return int(status_code), headers, bytes(raw_body)
489
+ return status_code, headers, bytes(raw_body)
456
490
 
457
491
  def has(self, key: str) -> bool:
458
492
  """检查是否存在有效快照。"""
@@ -548,7 +582,6 @@ class SQLiteSnapshotStore(SnapshotStore):
548
582
  @contextmanager
549
583
  def _get_conn(self) -> Iterator[sqlite3.Connection]:
550
584
  conn = sqlite3.connect(str(self.db_path), timeout=30.0)
551
- conn.execute("PRAGMA journal_mode=WAL")
552
585
  conn.execute("PRAGMA synchronous=NORMAL")
553
586
  try:
554
587
  yield conn
@@ -557,6 +590,8 @@ class SQLiteSnapshotStore(SnapshotStore):
557
590
 
558
591
  def _init_db(self) -> None:
559
592
  with self._get_conn() as conn:
593
+ # WAL 模式为数据库文件级持久属性,仅在初始化时设置一次,避免纯读路径重复执行。
594
+ conn.execute("PRAGMA journal_mode=WAL")
560
595
  conn.execute(
561
596
  """
562
597
  CREATE TABLE IF NOT EXISTS http_snapshots (
@@ -666,13 +701,22 @@ class HttpExecutor:
666
701
  snapshot_store: SnapshotStore | None = None,
667
702
  max_retries: int = 0,
668
703
  retry_delay_base: float = 1.0,
704
+ *,
705
+ sleep_fn: Callable[[float], None] = time.sleep,
706
+ jitter_fn: Callable[[float, float], float] = random.uniform,
669
707
  ) -> None:
670
708
  self.snapshot_store = snapshot_store
671
709
  self.max_retries = max_retries
672
710
  self.retry_delay_base = retry_delay_base
711
+ self._sleep_fn = sleep_fn
712
+ self._jitter_fn = jitter_fn
673
713
 
674
714
  @staticmethod
675
- def _retry_wait(attempt: int, base: float) -> float:
715
+ def _retry_wait(
716
+ attempt: int,
717
+ base: float,
718
+ jitter_fn: Callable[[float, float], float] = random.uniform,
719
+ ) -> float:
676
720
  """本地重试第 attempt 次的等待秒数(指数递增封顶 + 抖动)。
677
721
 
678
722
  形状:min(base, 封顶) × 2^(attempt-1) 再封顶,±25% 抖动——
@@ -682,7 +726,7 @@ class HttpExecutor:
682
726
  return 0.0
683
727
  exp = min(attempt - 1, 60)
684
728
  delay = min(min(base, _LOCAL_RETRY_WAIT_MAX_SECONDS) * (2 ** exp), _LOCAL_RETRY_WAIT_MAX_SECONDS)
685
- jitter = random.uniform(-0.25, 0.25) * delay
729
+ jitter = jitter_fn(-0.25, 0.25) * delay
686
730
  return max(0.0, delay + jitter)
687
731
 
688
732
  def execute(
@@ -697,7 +741,7 @@ class HttpExecutor:
697
741
  target_fn = self.snapshot_store.cached(target_fn)
698
742
  for attempt in range(max(0, self.max_retries) + 1):
699
743
  if attempt > 0:
700
- time.sleep(self._retry_wait(attempt, self.retry_delay_base))
744
+ self._sleep_fn(self._retry_wait(attempt, self.retry_delay_base, self._jitter_fn))
701
745
  try:
702
746
  response = target_fn(*args, **kwargs)
703
747
  except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.HTTPException) as exc:
@@ -706,9 +750,7 @@ class HttpExecutor:
706
750
  if attempt >= self.max_retries:
707
751
  raise
708
752
  continue
709
- status = getattr(response, "status_code", None)
710
- if status is None and hasattr(response, "getcode"):
711
- status = response.getcode()
753
+ status = _extract_status_code(response)
712
754
  if status is None or status < 500 or attempt >= self.max_retries:
713
755
  return response
714
756
  raise AssertionError("request loop must return or raise")
@@ -733,22 +775,17 @@ def _merge_params_into_url(url: str, params: Mapping[str, Any] | None) -> str:
733
775
  if not params:
734
776
  return url
735
777
  parsed = urllib.parse.urlparse(url)
736
- q = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)
737
- for k, v in params.items():
738
- if v is None:
739
- continue
740
- q.append((str(k), str(v)))
741
- new_query = urllib.parse.urlencode(q)
778
+ new_query = urllib.parse.urlencode(_merge_query_items(parsed.query, params, sort=False))
742
779
  return urllib.parse.urlunparse(
743
780
  (parsed.scheme, parsed.netloc, parsed.path, parsed.params, new_query, parsed.fragment)
744
781
  )
745
782
 
746
783
 
747
784
  def _build_urllib_opener(final_url: str, proxies: Mapping[str, str] | None):
748
- """构造 opener:显式代理优先;localhost/127.0.0.1 默认不走环境代理。"""
785
+ """构造 opener:显式代理优先;本机回环地址默认不走环境代理。"""
749
786
  if proxies is not None:
750
787
  return urllib.request.build_opener(urllib.request.ProxyHandler(dict(proxies)))
751
- if "127.0.0.1" in final_url or "localhost" in final_url:
788
+ if _is_local_url(final_url):
752
789
  return urllib.request.build_opener(urllib.request.ProxyHandler({}))
753
790
  return urllib.request.build_opener()
754
791
 
@@ -783,14 +820,11 @@ def urllib_fetch(
783
820
  req_headers = dict(headers) if headers else {}
784
821
 
785
822
  if data is not None:
786
- if isinstance(data, (dict, list)):
787
- req_body = _json_dumps(data).encode("utf-8")
788
- if "content-type" not in {k.lower(): v for k, v in req_headers.items()}:
789
- req_headers["Content-Type"] = "application/json"
790
- elif isinstance(data, str):
791
- req_body = data.encode("utf-8")
792
- else:
793
- req_body = bytes(data) # type: ignore[arg-type]
823
+ req_body = _encode_body_bytes(data)
824
+ if isinstance(data, (dict, list)) and "content-type" not in {
825
+ k.lower(): v for k, v in req_headers.items()
826
+ }:
827
+ req_headers["Content-Type"] = "application/json"
794
828
 
795
829
  req = urllib.request.Request(
796
830
  url=final_url,
@@ -828,7 +862,7 @@ def _prepare_requests_session(
828
862
  sess = requests_mod.Session()
829
863
  if trust_env is not None:
830
864
  sess.trust_env = trust_env
831
- elif "127.0.0.1" in url or "localhost" in url:
865
+ elif _is_local_url(url):
832
866
  sess.trust_env = False
833
867
  if proxies is not None:
834
868
  sess.proxies = dict(proxies)