tasklite-engine 3.0.0.dev0__py3-none-any.whl → 3.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasklite/__init__.py +42 -18
- tasklite/__main__.py +205 -0
- tasklite/_report.py +562 -0
- tasklite/backend/__init__.py +19 -2
- tasklite/backend/base.py +327 -156
- tasklite/backend/memory.py +261 -339
- tasklite/backend/sqlite_backend.py +416 -488
- tasklite/edit.py +1276 -0
- tasklite/engine/__init__.py +6 -7
- tasklite/engine/_worker_bootstrap.py +221 -0
- tasklite/engine/_worker_runtime.py +229 -0
- tasklite/engine/admission.py +20 -114
- tasklite/engine/authorization.py +989 -0
- tasklite/engine/channel.py +531 -468
- tasklite/engine/completion.py +193 -275
- tasklite/engine/config.py +5 -11
- tasklite/engine/dispatch.py +299 -384
- tasklite/engine/governor.py +134 -69
- tasklite/engine/in_flight.py +20 -88
- tasklite/engine/launcher.py +1173 -0
- tasklite/engine/recovery.py +147 -100
- tasklite/engine/resource.py +50 -103
- tasklite/engine/runtime.py +158 -525
- tasklite/engine/scheduler.py +9 -46
- tasklite/engine/session.py +60 -15
- tasklite/engine/store.py +235 -462
- tasklite/engine/types.py +29 -40
- tasklite/engine/wait.py +8 -0
- tasklite/exceptions.py +0 -1
- tasklite/models/artifact.py +84 -62
- tasklite/models/context.py +37 -30
- tasklite/models/graph.py +114 -0
- tasklite/models/job.py +3 -5
- tasklite/models/records.py +482 -0
- tasklite/models/state.py +19 -125
- tasklite/models/trigger.py +63 -3
- tasklite/pipeline.py +92 -100
- tasklite/resource.py +79 -2
- tasklite/snapshot.py +608 -405
- tasklite/testing.py +424 -11
- tasklite/utils/artifact.py +1091 -179
- tasklite/utils/encoding.py +21 -40
- tasklite/utils/ipc.py +49 -1
- tasklite/utils/lockfile.py +50 -19
- tasklite/wrappers/__init__.py +1 -1
- tasklite/wrappers/discovery.py +13 -16
- tasklite/wrappers/http.py +77 -43
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/METADATA +23 -16
- tasklite_engine-3.0.0.dev2.dist-info/RECORD +60 -0
- tasklite/contrib/__init__.py +0 -7
- tasklite/engine/execution_access.py +0 -39
- tasklite/engine/files.py +0 -52
- tasklite/engine/ops.py +0 -25
- tasklite/engine/selection.py +0 -186
- tasklite/hooks.py +0 -49
- tasklite/triggers.py +0 -47
- tasklite_engine-3.0.0.dev0.dist-info/RECORD +0 -58
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/WHEEL +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {tasklite_engine-3.0.0.dev0.dist-info → tasklite_engine-3.0.0.dev2.dist-info}/top_level.txt +0 -0
tasklite/utils/encoding.py
CHANGED
|
@@ -54,18 +54,6 @@ _TRUNC_MARKER = "%_"
|
|
|
54
54
|
# 上限的前提下完成
|
|
55
55
|
_TRUNC_FIXED_LEN = len(_TRUNC_MARKER) + _TRUNC_DIGEST_HEX
|
|
56
56
|
|
|
57
|
-
# 文件系统危险字符映射(POSIX /、Windows \、NUL、glob *?[]、冒号 :)
|
|
58
|
-
FS_ESCAPE_CHARS: dict[str, str] = {
|
|
59
|
-
"/": "%2F",
|
|
60
|
-
"\\": "%5C",
|
|
61
|
-
"\x00": "%00",
|
|
62
|
-
"*": "%2A",
|
|
63
|
-
"?": "%3F",
|
|
64
|
-
"[": "%5B",
|
|
65
|
-
"]": "%5D",
|
|
66
|
-
":": "%3A",
|
|
67
|
-
}
|
|
68
|
-
|
|
69
57
|
|
|
70
58
|
def percent_encode(
|
|
71
59
|
text: str,
|
|
@@ -258,6 +246,25 @@ def _encode_uid_component(value: str, field_name: str) -> str:
|
|
|
258
246
|
) from exc
|
|
259
247
|
|
|
260
248
|
|
|
249
|
+
def _flush_uid_bytes(buf: bytearray, chars: list[str], field_name: str) -> None:
|
|
250
|
+
"""将累积的百分号转义字节按 UTF-8 解码、校验无过度转义并冲刷到字符列表。"""
|
|
251
|
+
if not buf:
|
|
252
|
+
return
|
|
253
|
+
try:
|
|
254
|
+
decoded_run = buf.decode("utf-8")
|
|
255
|
+
except UnicodeDecodeError as exc:
|
|
256
|
+
raise TaskLiteError(
|
|
257
|
+
f"invalid job uid: invalid UTF-8 byte sequence in {field_name}"
|
|
258
|
+
) from exc
|
|
259
|
+
for dch in decoded_run:
|
|
260
|
+
if dch.isprintable() and dch not in (":", "%"):
|
|
261
|
+
raise TaskLiteError(
|
|
262
|
+
f"invalid job uid: over-escaped character {dch!r} in {field_name}"
|
|
263
|
+
)
|
|
264
|
+
chars.append(decoded_run)
|
|
265
|
+
buf.clear()
|
|
266
|
+
|
|
267
|
+
|
|
261
268
|
def _decode_uid_component(encoded: str, field_name: str) -> str:
|
|
262
269
|
"""严格解析规范 UID 组成部分并还原原始字符串。"""
|
|
263
270
|
if not encoded:
|
|
@@ -282,39 +289,14 @@ def _decode_uid_component(encoded: str, field_name: str) -> str:
|
|
|
282
289
|
buf.append(byte_val)
|
|
283
290
|
i += 3
|
|
284
291
|
else:
|
|
285
|
-
|
|
286
|
-
try:
|
|
287
|
-
decoded_run = buf.decode("utf-8")
|
|
288
|
-
except UnicodeDecodeError as exc:
|
|
289
|
-
raise TaskLiteError(
|
|
290
|
-
f"invalid job uid: invalid UTF-8 byte sequence in {field_name}"
|
|
291
|
-
) from exc
|
|
292
|
-
for dch in decoded_run:
|
|
293
|
-
if dch.isprintable() and dch not in (":", "%"):
|
|
294
|
-
raise TaskLiteError(
|
|
295
|
-
f"invalid job uid: over-escaped character {dch!r} in {field_name}"
|
|
296
|
-
)
|
|
297
|
-
chars.append(decoded_run)
|
|
298
|
-
buf.clear()
|
|
292
|
+
_flush_uid_bytes(buf, chars, field_name)
|
|
299
293
|
if ch == ":" or not ch.isprintable():
|
|
300
294
|
raise TaskLiteError(
|
|
301
295
|
f"invalid job uid: unescaped character {ch!r} in {field_name}"
|
|
302
296
|
)
|
|
303
297
|
chars.append(ch)
|
|
304
298
|
i += 1
|
|
305
|
-
|
|
306
|
-
try:
|
|
307
|
-
decoded_run = buf.decode("utf-8")
|
|
308
|
-
except UnicodeDecodeError as exc:
|
|
309
|
-
raise TaskLiteError(
|
|
310
|
-
f"invalid job uid: invalid UTF-8 byte sequence in {field_name}"
|
|
311
|
-
) from exc
|
|
312
|
-
for dch in decoded_run:
|
|
313
|
-
if dch.isprintable() and dch not in (":", "%"):
|
|
314
|
-
raise TaskLiteError(
|
|
315
|
-
f"invalid job uid: over-escaped character {dch!r} in {field_name}"
|
|
316
|
-
)
|
|
317
|
-
chars.append(decoded_run)
|
|
299
|
+
_flush_uid_bytes(buf, chars, field_name)
|
|
318
300
|
return "".join(chars)
|
|
319
301
|
|
|
320
302
|
|
|
@@ -346,7 +328,6 @@ __all__ = [
|
|
|
346
328
|
"CONTENT_ID_ALLOWED",
|
|
347
329
|
"DEFAULT_FORBIDDEN",
|
|
348
330
|
"EMPTY_SENTINEL",
|
|
349
|
-
"FS_ESCAPE_CHARS",
|
|
350
331
|
"content_fingerprint",
|
|
351
332
|
"encode_content_id",
|
|
352
333
|
"encode_identifier",
|
tasklite/utils/ipc.py
CHANGED
|
@@ -39,7 +39,6 @@ class _PipeConnection(Protocol):
|
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
# 管道消息种类常量
|
|
42
|
-
PIPE_MSG_INPUT = "input"
|
|
43
42
|
PIPE_MSG_RESULT = "result"
|
|
44
43
|
PIPE_MSG_DECLARE = "declare"
|
|
45
44
|
VALID_PIPE_MESSAGE_KINDS = frozenset(
|
|
@@ -48,9 +47,51 @@ VALID_PIPE_MESSAGE_KINDS = frozenset(
|
|
|
48
47
|
PIPE_MSG_DECLARE,
|
|
49
48
|
}
|
|
50
49
|
)
|
|
50
|
+
# 子进程两阶段握手角色常量
|
|
51
|
+
HANDSHAKE_ROLE_UPLINK = "uplink"
|
|
52
|
+
HANDSHAKE_ROLE_DOWNLINK = "downlink"
|
|
53
|
+
HANDSHAKE_ROLE_CANCEL = "cancel"
|
|
51
54
|
_PIPE_EOF = object()
|
|
52
55
|
|
|
53
56
|
|
|
57
|
+
def encode_json_frame(payload: Any) -> bytes:
|
|
58
|
+
"""将对象经统一严格 JSON 序列化编码为 UTF-8 帧字节。"""
|
|
59
|
+
return dumps(payload).encode("utf-8")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def decode_json_object_frame(raw: bytes, *, context: str = "JSON frame") -> dict[str, Any]:
|
|
63
|
+
"""经统一严格 JSON 反序列化解析单帧对象字节。"""
|
|
64
|
+
try:
|
|
65
|
+
decoded = loads(raw.decode("utf-8"))
|
|
66
|
+
except (UnicodeDecodeError, json.JSONDecodeError, RecursionError, TypeError, ValueError) as exc:
|
|
67
|
+
raise ValueError(f"invalid {context}: {exc}") from exc
|
|
68
|
+
if not isinstance(decoded, dict):
|
|
69
|
+
raise ValueError(f"{context} must decode to a JSON object")
|
|
70
|
+
return decoded
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def encode_handshake_frame(role: str, *, pid: int | None = None) -> bytes:
|
|
74
|
+
"""构造子进程握手角色帧(经统一 jsonutil 编码)。"""
|
|
75
|
+
frame: dict[str, Any] = {"role": role}
|
|
76
|
+
if pid is not None:
|
|
77
|
+
frame["pid"] = pid
|
|
78
|
+
return encode_json_frame(frame)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def decode_handshake_frame(raw: bytes) -> tuple[str | None, int | None]:
|
|
82
|
+
"""解析子进程握手角色帧,返回 (role, pid)。"""
|
|
83
|
+
frame = decode_json_object_frame(raw, context="worker handshake frame")
|
|
84
|
+
raw_role = frame.get("role")
|
|
85
|
+
role = raw_role if isinstance(raw_role, str) else None
|
|
86
|
+
raw_pid = frame.get("pid")
|
|
87
|
+
pid = (
|
|
88
|
+
raw_pid
|
|
89
|
+
if isinstance(raw_pid, int) and not isinstance(raw_pid, bool) and raw_pid > 0
|
|
90
|
+
else None
|
|
91
|
+
)
|
|
92
|
+
return role, pid
|
|
93
|
+
|
|
94
|
+
|
|
54
95
|
@dataclass(frozen=True)
|
|
55
96
|
class WorkerPipeMessage:
|
|
56
97
|
"""已校验的 worker 管道信封消息。"""
|
|
@@ -616,6 +657,9 @@ def create_worker_reply_pipe(
|
|
|
616
657
|
|
|
617
658
|
|
|
618
659
|
__all__ = [
|
|
660
|
+
"HANDSHAKE_ROLE_CANCEL",
|
|
661
|
+
"HANDSHAKE_ROLE_DOWNLINK",
|
|
662
|
+
"HANDSHAKE_ROLE_UPLINK",
|
|
619
663
|
"PIPE_MSG_DECLARE",
|
|
620
664
|
"PIPE_MSG_RESULT",
|
|
621
665
|
"VALID_PIPE_MESSAGE_KINDS",
|
|
@@ -626,5 +670,9 @@ __all__ = [
|
|
|
626
670
|
"WorkerReplySender",
|
|
627
671
|
"create_worker_pipe",
|
|
628
672
|
"create_worker_reply_pipe",
|
|
673
|
+
"decode_handshake_frame",
|
|
674
|
+
"decode_json_object_frame",
|
|
629
675
|
"decode_worker_pipe_message",
|
|
676
|
+
"encode_handshake_frame",
|
|
677
|
+
"encode_json_frame",
|
|
630
678
|
]
|
tasklite/utils/lockfile.py
CHANGED
|
@@ -1,25 +1,41 @@
|
|
|
1
|
-
"""v2 跨平台单文件锁:库级执行与管理互斥(flock 互斥)。
|
|
1
|
+
"""v2 跨平台单文件锁:库级执行与管理互斥(flock / msvcrt 互斥)。
|
|
2
2
|
|
|
3
3
|
用于主进程按状态库路径获取库级互斥锁,防止并发执行或管理冲突。
|
|
4
4
|
"""
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
import errno
|
|
7
8
|
import os
|
|
8
9
|
import sys
|
|
9
10
|
import time
|
|
10
11
|
from pathlib import Path
|
|
11
12
|
|
|
12
13
|
|
|
14
|
+
def lock_path_for(state_path: str | Path) -> Path:
|
|
15
|
+
"""由状态库路径派生对应的库级互斥锁文件路径(幂等处理已有 .lock 后缀)。"""
|
|
16
|
+
p = Path(state_path)
|
|
17
|
+
if p.name.endswith(".lock"):
|
|
18
|
+
return p
|
|
19
|
+
return Path(f"{p}.lock")
|
|
20
|
+
|
|
21
|
+
|
|
13
22
|
def _lock_path(directory_or_path: str | Path, name: str | None = None) -> Path:
|
|
14
23
|
"""name → 锁文件路径。若 name 为 None,直接派生锁文件路径。"""
|
|
15
24
|
if name is None:
|
|
16
|
-
|
|
17
|
-
if p.name.endswith(".lock"):
|
|
18
|
-
return p
|
|
19
|
-
return Path(str(p) + ".lock")
|
|
25
|
+
return lock_path_for(directory_or_path)
|
|
20
26
|
return Path(directory_or_path) / f"{name}.lock"
|
|
21
27
|
|
|
22
28
|
|
|
29
|
+
def _is_lock_contention(exc: OSError) -> bool:
|
|
30
|
+
"""判断 OSError 是否属于锁争用(而非无效 fd 或底层 IO 故障)。"""
|
|
31
|
+
if sys.platform == "win32":
|
|
32
|
+
return (
|
|
33
|
+
exc.errno in (errno.EACCES, errno.EDEADLK)
|
|
34
|
+
or getattr(exc, "winerror", None) in (33, 32)
|
|
35
|
+
)
|
|
36
|
+
return exc.errno in (errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK)
|
|
37
|
+
|
|
38
|
+
|
|
23
39
|
def try_acquire_lock(
|
|
24
40
|
directory_or_path: str | Path,
|
|
25
41
|
name: str | None = None,
|
|
@@ -33,41 +49,53 @@ def try_acquire_lock(
|
|
|
33
49
|
时返回 None。锁文件**创建后不删除**。
|
|
34
50
|
|
|
35
51
|
Raises:
|
|
36
|
-
OSError:
|
|
52
|
+
OSError: 锁文件打不开/建不出/IO 故障(权限、磁盘满、无效 fd 等)——与
|
|
37
53
|
「锁被占」语义不同,不得混入 None。
|
|
38
54
|
"""
|
|
39
55
|
path = _lock_path(directory_or_path, name)
|
|
40
56
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
41
57
|
fd = os.open(str(path), os.O_RDWR | os.O_CREAT, 0o644)
|
|
42
58
|
deadline = None if timeout <= 0 else time.monotonic() + timeout
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
59
|
+
try:
|
|
60
|
+
while True:
|
|
61
|
+
if _try_lock_fd(fd):
|
|
62
|
+
return fd
|
|
63
|
+
if deadline is None or time.monotonic() >= deadline:
|
|
64
|
+
os.close(fd)
|
|
65
|
+
return None
|
|
66
|
+
time.sleep(0.1)
|
|
67
|
+
except BaseException:
|
|
68
|
+
try:
|
|
50
69
|
os.close(fd)
|
|
51
|
-
|
|
52
|
-
|
|
70
|
+
except OSError:
|
|
71
|
+
pass
|
|
72
|
+
raise
|
|
53
73
|
|
|
54
74
|
|
|
55
75
|
def _try_lock_fd(fd: int) -> bool:
|
|
56
76
|
"""对已打开的 fd 尝试非阻塞排他锁(平台分派)。"""
|
|
57
77
|
if sys.platform == "win32":
|
|
58
78
|
import msvcrt
|
|
79
|
+
|
|
80
|
+
# msvcrt.locking 按当前文件偏移加锁,固定归零确保始终锁第 0 字节。
|
|
81
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
59
82
|
try:
|
|
60
83
|
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
|
|
61
84
|
return True
|
|
62
|
-
except OSError:
|
|
63
|
-
|
|
85
|
+
except OSError as exc:
|
|
86
|
+
if _is_lock_contention(exc):
|
|
87
|
+
return False
|
|
88
|
+
raise
|
|
64
89
|
else:
|
|
65
90
|
import fcntl
|
|
91
|
+
|
|
66
92
|
try:
|
|
67
93
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
68
94
|
return True
|
|
69
|
-
except OSError:
|
|
70
|
-
|
|
95
|
+
except OSError as exc:
|
|
96
|
+
if _is_lock_contention(exc):
|
|
97
|
+
return False
|
|
98
|
+
raise
|
|
71
99
|
|
|
72
100
|
|
|
73
101
|
def release_lock(fd: int) -> None:
|
|
@@ -77,10 +105,12 @@ def release_lock(fd: int) -> None:
|
|
|
77
105
|
try:
|
|
78
106
|
if sys.platform == "win32":
|
|
79
107
|
import msvcrt
|
|
108
|
+
|
|
80
109
|
os.lseek(fd, 0, os.SEEK_SET)
|
|
81
110
|
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
|
|
82
111
|
else:
|
|
83
112
|
import fcntl
|
|
113
|
+
|
|
84
114
|
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
85
115
|
except OSError:
|
|
86
116
|
pass
|
|
@@ -91,6 +121,7 @@ def release_lock(fd: int) -> None:
|
|
|
91
121
|
|
|
92
122
|
|
|
93
123
|
__all__ = [
|
|
124
|
+
"lock_path_for",
|
|
94
125
|
"release_lock",
|
|
95
126
|
"try_acquire_lock",
|
|
96
127
|
]
|
tasklite/wrappers/__init__.py
CHANGED
tasklite/wrappers/discovery.py
CHANGED
|
@@ -3,11 +3,12 @@
|
|
|
3
3
|
注意:本模块为可选工具,本版本暂不承诺向后兼容与冻结扩展协议。
|
|
4
4
|
增量发现通过任务派生与框架核心状态事务整体相容,不因坏子项而允许父任务成功或部分发布。
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
``
|
|
8
|
-
|
|
9
|
-
``DiscoveryHost``
|
|
10
|
-
|
|
6
|
+
架构性质:本模块是依赖核心公开模型与工具面(``TaskResult``、``TaskLiteError``、
|
|
7
|
+
``validate_trigger``、``encode_content_id``、``jsonutil``)的可选适配器;
|
|
8
|
+
``DiscoveryHandler`` 的扫描循环只通过 ``DiscoveryJob`` / ``DiscoveryContext``
|
|
9
|
+
两个最小协议与运行时交互,``register_discovery`` 只通过 ``DiscoveryHost``
|
|
10
|
+
公开协议把扫描器挂到宿主上(TaskLite 是其默认实现),不直接依赖引擎内部实现。
|
|
11
|
+
任何实现对应协议的宿主或测试桩均可直接复用。
|
|
11
12
|
|
|
12
13
|
「已见集合」由 discovery 自己通过公开游标持久化:每个
|
|
13
14
|
``(discovery task_type, process task_type, group key)`` 使用独立游标,
|
|
@@ -120,7 +121,6 @@ V1. 契约正确性必须由回归测试固定,至少覆盖:首次全量扫
|
|
|
120
121
|
"""
|
|
121
122
|
from __future__ import annotations
|
|
122
123
|
|
|
123
|
-
import json
|
|
124
124
|
import logging
|
|
125
125
|
import pickle
|
|
126
126
|
from collections.abc import Callable
|
|
@@ -130,6 +130,7 @@ from ..exceptions import TaskLiteError
|
|
|
130
130
|
from ..models.result import TaskResult
|
|
131
131
|
from ..models.task import validate_trigger
|
|
132
132
|
from ..utils.encoding import encode_content_id
|
|
133
|
+
from ..utils.jsonutil import dumps as _json_dumps, loads as _json_loads
|
|
133
134
|
|
|
134
135
|
logger = logging.getLogger("tasklite")
|
|
135
136
|
|
|
@@ -137,7 +138,7 @@ _SEEN_CURSOR_VERSION = 1
|
|
|
137
138
|
_SEEN_CURSOR_PREFIX = "__tasklite_discovery_seen_v1__:"
|
|
138
139
|
|
|
139
140
|
|
|
140
|
-
#
|
|
141
|
+
# 宿主解耦协议:本模块仅依赖核心公开模型与工具面,不直接依赖引擎内部实现。
|
|
141
142
|
#
|
|
142
143
|
# ``DiscoveryJob`` / ``DiscoveryContext`` 是扫描循环对运行时的全部要求;
|
|
143
144
|
# ``DiscoveryHost`` 是注册适配器对宿主框架的全部要求。
|
|
@@ -185,10 +186,9 @@ class DiscoveryHost(Protocol):
|
|
|
185
186
|
class DiscoveryHandler:
|
|
186
187
|
"""按 REQUIREMENTS CONTRACT 实现的增量发现 handler(纯任务模型)。
|
|
187
188
|
|
|
188
|
-
|
|
189
|
-
两个最小协议(payload + get_cursor/set_cursor
|
|
190
|
-
|
|
191
|
-
TaskLite 只是框架默认宿主。
|
|
189
|
+
**宿主协议解耦**:本类及其扫描循环只依赖 ``DiscoveryJob`` / ``DiscoveryContext``
|
|
190
|
+
两个最小协议(payload + get_cursor/set_cursor),不直接依赖引擎内部实现,
|
|
191
|
+
可被任何实现这两个协议的宿主直接复用;TaskLite 只是框架默认宿主。
|
|
192
192
|
|
|
193
193
|
框架负责契约规定的全部语义:
|
|
194
194
|
- 从第 1 页起逐页扫描,page 由框架传入,用户 fetch_func 只拉一页;
|
|
@@ -272,7 +272,7 @@ class DiscoveryHandler:
|
|
|
272
272
|
if raw is None:
|
|
273
273
|
return set()
|
|
274
274
|
try:
|
|
275
|
-
payload =
|
|
275
|
+
payload = _json_loads(raw)
|
|
276
276
|
except (TypeError, ValueError) as exc:
|
|
277
277
|
raise ValueError(f"invalid discovery seen cursor {key!r}: {exc}") from exc
|
|
278
278
|
version = payload.get("version") if isinstance(payload, dict) else None
|
|
@@ -298,10 +298,7 @@ class DiscoveryHandler:
|
|
|
298
298
|
"version": _SEEN_CURSOR_VERSION,
|
|
299
299
|
"seen": sorted(seen),
|
|
300
300
|
}
|
|
301
|
-
ctx.set_cursor(
|
|
302
|
-
key,
|
|
303
|
-
json.dumps(payload, ensure_ascii=True, separators=(",", ":")),
|
|
304
|
-
)
|
|
301
|
+
ctx.set_cursor(key, _json_dumps(payload))
|
|
305
302
|
|
|
306
303
|
def _resolve_scan_options(self, job: DiscoveryJob) -> tuple[str, int]:
|
|
307
304
|
"""扫描模式与页数上限:job.payload 动态指定优先,否则用构造默认值。"""
|
tasklite/wrappers/http.py
CHANGED
|
@@ -31,6 +31,55 @@ from ..utils.jsonutil import dumps as _json_dumps, loads as _json_loads
|
|
|
31
31
|
# 节奏唯一出口是 RequeuePolicy seam)。
|
|
32
32
|
_LOCAL_RETRY_WAIT_MAX_SECONDS = 300.0
|
|
33
33
|
|
|
34
|
+
# 默认绕过环境变量代理的本机回环主机名集合(精确匹配主机名,避免子串误判)。
|
|
35
|
+
_LOCAL_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _merge_query_items(
|
|
39
|
+
raw_query: str,
|
|
40
|
+
params: Mapping[str, Any] | None,
|
|
41
|
+
*,
|
|
42
|
+
sort: bool = False,
|
|
43
|
+
) -> list[tuple[str, str]]:
|
|
44
|
+
"""合并 URL 内嵌 query 与显式 params(None 值统一丢弃,与 requests 语义对齐)。"""
|
|
45
|
+
items: list[tuple[str, str]] = (
|
|
46
|
+
urllib.parse.parse_qsl(raw_query, keep_blank_values=True) if raw_query else []
|
|
47
|
+
)
|
|
48
|
+
if params:
|
|
49
|
+
for k, v in params.items():
|
|
50
|
+
if v is None:
|
|
51
|
+
continue
|
|
52
|
+
items.append((str(k), str(v)))
|
|
53
|
+
if sort:
|
|
54
|
+
items.sort()
|
|
55
|
+
return items
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _encode_body_bytes(body: bytes | str | Mapping[str, Any]) -> bytes:
|
|
59
|
+
"""请求体统一字节序列化:dict/list 走 JSON、str 编码 UTF-8、其余按字节。"""
|
|
60
|
+
if isinstance(body, (dict, list)):
|
|
61
|
+
return _json_dumps(body).encode("utf-8")
|
|
62
|
+
if isinstance(body, str):
|
|
63
|
+
return body.encode("utf-8")
|
|
64
|
+
return bytes(body) # type: ignore[arg-type]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _extract_status_code(res: Any) -> int | None:
|
|
68
|
+
"""从任意响应形态提取 HTTP 状态码(优先 status_code,其次 getcode();无状态语义返回 None)。"""
|
|
69
|
+
status: int | None = getattr(res, "status_code", None)
|
|
70
|
+
if status is None and hasattr(res, "getcode"):
|
|
71
|
+
status = res.getcode()
|
|
72
|
+
return int(status) if status is not None else None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _is_local_url(url: str) -> bool:
|
|
76
|
+
"""判断 URL 目标主机是否为本机回环地址(精确匹配主机名,避免子串误判)。"""
|
|
77
|
+
try:
|
|
78
|
+
hostname = urllib.parse.urlparse(url).hostname
|
|
79
|
+
except ValueError:
|
|
80
|
+
return False
|
|
81
|
+
return hostname is not None and hostname.lower() in _LOCAL_LOOPBACK_HOSTS
|
|
82
|
+
|
|
34
83
|
|
|
35
84
|
# ==============================================================================
|
|
36
85
|
# 1. 基础容器与 Cookie 转换工具
|
|
@@ -296,16 +345,7 @@ class SnapshotStore:
|
|
|
296
345
|
params: Mapping[str, Any] | None,
|
|
297
346
|
) -> list[tuple[str, str]]:
|
|
298
347
|
"""合并 URL 内嵌 query 与显式 params 并排序(同语义请求同键)。"""
|
|
299
|
-
|
|
300
|
-
if parsed.query:
|
|
301
|
-
query_items.extend(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True))
|
|
302
|
-
if params:
|
|
303
|
-
for k, v in params.items():
|
|
304
|
-
if v is None:
|
|
305
|
-
continue
|
|
306
|
-
query_items.append((str(k), str(v)))
|
|
307
|
-
query_items.sort()
|
|
308
|
-
return query_items
|
|
348
|
+
return _merge_query_items(parsed.query, params, sort=True)
|
|
309
349
|
|
|
310
350
|
# 纳入快照 key 的身份头白名单:仅取真正改变响应语义的凭证/协商头。
|
|
311
351
|
# 取舍:不纳入全量 headers——User-Agent/Accept-Encoding/Date 等易变头会把
|
|
@@ -420,11 +460,7 @@ class SnapshotStore:
|
|
|
420
460
|
@staticmethod
|
|
421
461
|
def _body_bytes(body: bytes | str | Mapping[str, Any]) -> bytes:
|
|
422
462
|
"""请求体统一序列化:dict/list 走 JSON、str 编码 UTF-8、其余按字节。"""
|
|
423
|
-
|
|
424
|
-
return _json_dumps(body).encode("utf-8")
|
|
425
|
-
if isinstance(body, str):
|
|
426
|
-
return body.encode("utf-8")
|
|
427
|
-
return bytes(body) # type: ignore[arg-type]
|
|
463
|
+
return _encode_body_bytes(body)
|
|
428
464
|
|
|
429
465
|
@staticmethod
|
|
430
466
|
def _extract_response_fields(res: Any) -> tuple[int, dict[str, str], bytes]:
|
|
@@ -438,9 +474,7 @@ class SnapshotStore:
|
|
|
438
474
|
"""
|
|
439
475
|
if isinstance(res, HttpResponse):
|
|
440
476
|
return res.status_code, res.headers, res.body
|
|
441
|
-
status_code
|
|
442
|
-
if status_code is None and hasattr(res, "getcode"):
|
|
443
|
-
status_code = res.getcode()
|
|
477
|
+
status_code = _extract_status_code(res)
|
|
444
478
|
if status_code is None:
|
|
445
479
|
status_code = 200
|
|
446
480
|
headers = dict(getattr(res, "headers", {}))
|
|
@@ -452,7 +486,7 @@ class SnapshotStore:
|
|
|
452
486
|
raw_body = _json_dumps(res).encode("utf-8")
|
|
453
487
|
else:
|
|
454
488
|
raw_body = str(res).encode("utf-8")
|
|
455
|
-
return
|
|
489
|
+
return status_code, headers, bytes(raw_body)
|
|
456
490
|
|
|
457
491
|
def has(self, key: str) -> bool:
|
|
458
492
|
"""检查是否存在有效快照。"""
|
|
@@ -548,7 +582,6 @@ class SQLiteSnapshotStore(SnapshotStore):
|
|
|
548
582
|
@contextmanager
|
|
549
583
|
def _get_conn(self) -> Iterator[sqlite3.Connection]:
|
|
550
584
|
conn = sqlite3.connect(str(self.db_path), timeout=30.0)
|
|
551
|
-
conn.execute("PRAGMA journal_mode=WAL")
|
|
552
585
|
conn.execute("PRAGMA synchronous=NORMAL")
|
|
553
586
|
try:
|
|
554
587
|
yield conn
|
|
@@ -557,6 +590,8 @@ class SQLiteSnapshotStore(SnapshotStore):
|
|
|
557
590
|
|
|
558
591
|
def _init_db(self) -> None:
|
|
559
592
|
with self._get_conn() as conn:
|
|
593
|
+
# WAL 模式为数据库文件级持久属性,仅在初始化时设置一次,避免纯读路径重复执行。
|
|
594
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
560
595
|
conn.execute(
|
|
561
596
|
"""
|
|
562
597
|
CREATE TABLE IF NOT EXISTS http_snapshots (
|
|
@@ -666,13 +701,22 @@ class HttpExecutor:
|
|
|
666
701
|
snapshot_store: SnapshotStore | None = None,
|
|
667
702
|
max_retries: int = 0,
|
|
668
703
|
retry_delay_base: float = 1.0,
|
|
704
|
+
*,
|
|
705
|
+
sleep_fn: Callable[[float], None] = time.sleep,
|
|
706
|
+
jitter_fn: Callable[[float, float], float] = random.uniform,
|
|
669
707
|
) -> None:
|
|
670
708
|
self.snapshot_store = snapshot_store
|
|
671
709
|
self.max_retries = max_retries
|
|
672
710
|
self.retry_delay_base = retry_delay_base
|
|
711
|
+
self._sleep_fn = sleep_fn
|
|
712
|
+
self._jitter_fn = jitter_fn
|
|
673
713
|
|
|
674
714
|
@staticmethod
|
|
675
|
-
def _retry_wait(
|
|
715
|
+
def _retry_wait(
|
|
716
|
+
attempt: int,
|
|
717
|
+
base: float,
|
|
718
|
+
jitter_fn: Callable[[float, float], float] = random.uniform,
|
|
719
|
+
) -> float:
|
|
676
720
|
"""本地重试第 attempt 次的等待秒数(指数递增封顶 + 抖动)。
|
|
677
721
|
|
|
678
722
|
形状:min(base, 封顶) × 2^(attempt-1) 再封顶,±25% 抖动——
|
|
@@ -682,7 +726,7 @@ class HttpExecutor:
|
|
|
682
726
|
return 0.0
|
|
683
727
|
exp = min(attempt - 1, 60)
|
|
684
728
|
delay = min(min(base, _LOCAL_RETRY_WAIT_MAX_SECONDS) * (2 ** exp), _LOCAL_RETRY_WAIT_MAX_SECONDS)
|
|
685
|
-
jitter =
|
|
729
|
+
jitter = jitter_fn(-0.25, 0.25) * delay
|
|
686
730
|
return max(0.0, delay + jitter)
|
|
687
731
|
|
|
688
732
|
def execute(
|
|
@@ -697,7 +741,7 @@ class HttpExecutor:
|
|
|
697
741
|
target_fn = self.snapshot_store.cached(target_fn)
|
|
698
742
|
for attempt in range(max(0, self.max_retries) + 1):
|
|
699
743
|
if attempt > 0:
|
|
700
|
-
|
|
744
|
+
self._sleep_fn(self._retry_wait(attempt, self.retry_delay_base, self._jitter_fn))
|
|
701
745
|
try:
|
|
702
746
|
response = target_fn(*args, **kwargs)
|
|
703
747
|
except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.HTTPException) as exc:
|
|
@@ -706,9 +750,7 @@ class HttpExecutor:
|
|
|
706
750
|
if attempt >= self.max_retries:
|
|
707
751
|
raise
|
|
708
752
|
continue
|
|
709
|
-
status =
|
|
710
|
-
if status is None and hasattr(response, "getcode"):
|
|
711
|
-
status = response.getcode()
|
|
753
|
+
status = _extract_status_code(response)
|
|
712
754
|
if status is None or status < 500 or attempt >= self.max_retries:
|
|
713
755
|
return response
|
|
714
756
|
raise AssertionError("request loop must return or raise")
|
|
@@ -733,22 +775,17 @@ def _merge_params_into_url(url: str, params: Mapping[str, Any] | None) -> str:
|
|
|
733
775
|
if not params:
|
|
734
776
|
return url
|
|
735
777
|
parsed = urllib.parse.urlparse(url)
|
|
736
|
-
|
|
737
|
-
for k, v in params.items():
|
|
738
|
-
if v is None:
|
|
739
|
-
continue
|
|
740
|
-
q.append((str(k), str(v)))
|
|
741
|
-
new_query = urllib.parse.urlencode(q)
|
|
778
|
+
new_query = urllib.parse.urlencode(_merge_query_items(parsed.query, params, sort=False))
|
|
742
779
|
return urllib.parse.urlunparse(
|
|
743
780
|
(parsed.scheme, parsed.netloc, parsed.path, parsed.params, new_query, parsed.fragment)
|
|
744
781
|
)
|
|
745
782
|
|
|
746
783
|
|
|
747
784
|
def _build_urllib_opener(final_url: str, proxies: Mapping[str, str] | None):
|
|
748
|
-
"""构造 opener
|
|
785
|
+
"""构造 opener:显式代理优先;本机回环地址默认不走环境代理。"""
|
|
749
786
|
if proxies is not None:
|
|
750
787
|
return urllib.request.build_opener(urllib.request.ProxyHandler(dict(proxies)))
|
|
751
|
-
if
|
|
788
|
+
if _is_local_url(final_url):
|
|
752
789
|
return urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
753
790
|
return urllib.request.build_opener()
|
|
754
791
|
|
|
@@ -783,14 +820,11 @@ def urllib_fetch(
|
|
|
783
820
|
req_headers = dict(headers) if headers else {}
|
|
784
821
|
|
|
785
822
|
if data is not None:
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
req_body = data.encode("utf-8")
|
|
792
|
-
else:
|
|
793
|
-
req_body = bytes(data) # type: ignore[arg-type]
|
|
823
|
+
req_body = _encode_body_bytes(data)
|
|
824
|
+
if isinstance(data, (dict, list)) and "content-type" not in {
|
|
825
|
+
k.lower(): v for k, v in req_headers.items()
|
|
826
|
+
}:
|
|
827
|
+
req_headers["Content-Type"] = "application/json"
|
|
794
828
|
|
|
795
829
|
req = urllib.request.Request(
|
|
796
830
|
url=final_url,
|
|
@@ -828,7 +862,7 @@ def _prepare_requests_session(
|
|
|
828
862
|
sess = requests_mod.Session()
|
|
829
863
|
if trust_env is not None:
|
|
830
864
|
sess.trust_env = trust_env
|
|
831
|
-
elif
|
|
865
|
+
elif _is_local_url(url):
|
|
832
866
|
sess.trust_env = False
|
|
833
867
|
if proxies is not None:
|
|
834
868
|
sess.proxies = dict(proxies)
|