shopline-qa-tools 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shopline_qa_tools-1.0.0/MANIFEST.in +1 -0
- shopline_qa_tools-1.0.0/PKG-INFO +9 -0
- shopline_qa_tools-1.0.0/pyproject.toml +27 -0
- shopline_qa_tools-1.0.0/setup.cfg +4 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/__init__.py +3 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/confusefile/__init__.py +5 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/confusefile/__main__.py +5 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/confusefile/cli.py +81 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/confusefile/core.py +113 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/__init__.py +48 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/capturer.py +247 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/config.py +112 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/consumer.py +128 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/message.py +127 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/validator.py +308 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/PKG-INFO +9 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/SOURCES.txt +20 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/dependency_links.txt +1 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/entry_points.txt +2 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/requires.txt +5 -0
- shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/top_level.txt +1 -0
- shopline_qa_tools-1.0.0/tests/test_confusefile.py +92 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
exclude README.md
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "shopline-qa-tools"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "QA 常用工具包"
|
|
9
|
+
requires-python = ">=3.8"
|
|
10
|
+
dependencies = ["pulsar-client>=3.4", "deepdiff>=6"]
|
|
11
|
+
|
|
12
|
+
[project.optional-dependencies]
|
|
13
|
+
dev = ["pytest>=7"]
|
|
14
|
+
|
|
15
|
+
[project.scripts]
|
|
16
|
+
confusefile = "shopline_qa_tools.confusefile.cli:main"
|
|
17
|
+
|
|
18
|
+
[tool.setuptools.packages.find]
|
|
19
|
+
where = ["src"]
|
|
20
|
+
|
|
21
|
+
[tool.pytest.ini_options]
|
|
22
|
+
testpaths = ["tests"]
|
|
23
|
+
pythonpath = ["src"]
|
|
24
|
+
markers = [
|
|
25
|
+
"pulsar: 需要真实 Pulsar 的集成用例",
|
|
26
|
+
"pulsar_config(**kwargs): 覆盖单个用例的 Pulsar 配置,如 topic/subscription_name/wait_timeout_s",
|
|
27
|
+
]
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""confusefile 命令行入口。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import List, Optional
|
|
10
|
+
|
|
11
|
+
from .core import DEFAULT_COMMENT_PREFIX, DEFAULT_RANDOM_BYTES, METHODS, confuse
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
15
|
+
p = argparse.ArgumentParser(
|
|
16
|
+
prog="confusefile",
|
|
17
|
+
description="在文件末尾追加内容,改变文件 MD5。默认原地覆盖并自动备份。",
|
|
18
|
+
)
|
|
19
|
+
p.add_argument("files", nargs="+", help="要处理的文件,可传多个")
|
|
20
|
+
p.add_argument("-o", "--output", help="输出文件路径(仅处理单个文件时可用),源文件保持不变")
|
|
21
|
+
p.add_argument("-d", "--out-dir", help="输出目录(可批量),文件名不变,源文件保持不变")
|
|
22
|
+
p.add_argument("-m", "--method", choices=METHODS, default="random",
|
|
23
|
+
help="random=追加随机字节(默认,适合任意文件);comment=追加一行注释(适合文本文件)")
|
|
24
|
+
p.add_argument("-n", "--bytes", type=int, default=DEFAULT_RANDOM_BYTES, dest="random_bytes",
|
|
25
|
+
help=f"random 方法追加的字节数,默认 {DEFAULT_RANDOM_BYTES}")
|
|
26
|
+
p.add_argument("-p", "--prefix", default=DEFAULT_COMMENT_PREFIX, dest="comment_prefix",
|
|
27
|
+
help=f'comment 方法的注释前缀,默认 "{DEFAULT_COMMENT_PREFIX}"')
|
|
28
|
+
p.add_argument("--no-backup", action="store_true", help="原地覆盖时不备份源文件")
|
|
29
|
+
p.add_argument("--json", action="store_true", help="以 JSON 输出结果")
|
|
30
|
+
return p
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
34
|
+
args = build_parser().parse_args(argv)
|
|
35
|
+
|
|
36
|
+
if args.output and args.out_dir:
|
|
37
|
+
print("error: -o 和 -d 不能同时使用", file=sys.stderr)
|
|
38
|
+
return 2
|
|
39
|
+
if args.output and len(args.files) > 1:
|
|
40
|
+
print("error: 处理多个文件时请用 -d 指定输出目录", file=sys.stderr)
|
|
41
|
+
return 2
|
|
42
|
+
|
|
43
|
+
results, failed = [], 0
|
|
44
|
+
for f in args.files:
|
|
45
|
+
if args.output:
|
|
46
|
+
output = args.output
|
|
47
|
+
elif args.out_dir:
|
|
48
|
+
output = str(Path(args.out_dir) / Path(f).name)
|
|
49
|
+
else:
|
|
50
|
+
output = None
|
|
51
|
+
try:
|
|
52
|
+
r = confuse(f, output, method=args.method, random_bytes=args.random_bytes,
|
|
53
|
+
comment_prefix=args.comment_prefix, backup=not args.no_backup)
|
|
54
|
+
except (OSError, ValueError) as e:
|
|
55
|
+
failed += 1
|
|
56
|
+
print(f"error: {f}: {e}", file=sys.stderr)
|
|
57
|
+
continue
|
|
58
|
+
results.append(r)
|
|
59
|
+
if not args.json:
|
|
60
|
+
print(f"file: {r.output}")
|
|
61
|
+
if r.backup:
|
|
62
|
+
print(f"backup: {r.backup}")
|
|
63
|
+
print(f"md5: {r.original_md5} -> {r.new_md5}")
|
|
64
|
+
print(f"size: +{r.size_change} bytes ({r.method})")
|
|
65
|
+
print()
|
|
66
|
+
|
|
67
|
+
if args.json:
|
|
68
|
+
print(json.dumps([
|
|
69
|
+
{
|
|
70
|
+
"source": str(r.source), "output": str(r.output),
|
|
71
|
+
"backup": str(r.backup) if r.backup else None,
|
|
72
|
+
"originalMd5": r.original_md5, "newMd5": r.new_md5,
|
|
73
|
+
"method": r.method, "sizeChange": r.size_change,
|
|
74
|
+
} for r in results
|
|
75
|
+
], ensure_ascii=False, indent=2))
|
|
76
|
+
|
|
77
|
+
return 1 if failed else 0
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
if __name__ == "__main__":
|
|
81
|
+
sys.exit(main())
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""文件混淆核心逻辑。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import os
|
|
7
|
+
import secrets
|
|
8
|
+
import shutil
|
|
9
|
+
import tempfile
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Optional, Union
|
|
15
|
+
|
|
16
|
+
PathLike = Union[str, os.PathLike]
|
|
17
|
+
|
|
18
|
+
RANDOM = "random"
|
|
19
|
+
COMMENT = "comment"
|
|
20
|
+
METHODS = (RANDOM, COMMENT)
|
|
21
|
+
|
|
22
|
+
DEFAULT_RANDOM_BYTES = 8
|
|
23
|
+
DEFAULT_COMMENT_PREFIX = "# Confused: "
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class ConfuseResult:
|
|
28
|
+
source: Path
|
|
29
|
+
output: Path
|
|
30
|
+
backup: Optional[Path]
|
|
31
|
+
original_md5: str
|
|
32
|
+
new_md5: str
|
|
33
|
+
method: str
|
|
34
|
+
size_change: int
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def md5sum(path: PathLike) -> str:
|
|
38
|
+
h = hashlib.md5()
|
|
39
|
+
with open(path, "rb") as f:
|
|
40
|
+
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
|
41
|
+
h.update(chunk)
|
|
42
|
+
return h.hexdigest()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _build_suffix(method: str, random_bytes: int, comment_prefix: str) -> bytes:
|
|
46
|
+
if method == RANDOM:
|
|
47
|
+
if random_bytes <= 0:
|
|
48
|
+
raise ValueError("random_bytes 必须大于 0")
|
|
49
|
+
return secrets.token_bytes(random_bytes)
|
|
50
|
+
if method == COMMENT:
|
|
51
|
+
line = f"\n{comment_prefix}{uuid.uuid4()}_{time.time_ns()}\n"
|
|
52
|
+
return line.encode("utf-8")
|
|
53
|
+
raise ValueError(f"不支持的混淆方法: {method},可选: {', '.join(METHODS)}")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def confuse(
|
|
57
|
+
src: PathLike,
|
|
58
|
+
output: Optional[PathLike] = None,
|
|
59
|
+
*,
|
|
60
|
+
method: str = RANDOM,
|
|
61
|
+
random_bytes: int = DEFAULT_RANDOM_BYTES,
|
|
62
|
+
comment_prefix: str = DEFAULT_COMMENT_PREFIX,
|
|
63
|
+
backup: bool = True,
|
|
64
|
+
) -> ConfuseResult:
|
|
65
|
+
"""混淆文件,使其 MD5 改变。
|
|
66
|
+
|
|
67
|
+
- 传 ``output``:生成新文件,源文件不变。
|
|
68
|
+
- 不传 ``output``:原地覆盖源文件;``backup=True`` 时先备份为 ``<文件名>.bak_<时间戳>``。
|
|
69
|
+
"""
|
|
70
|
+
src = Path(src)
|
|
71
|
+
if not src.is_file():
|
|
72
|
+
raise FileNotFoundError(f"文件不存在: {src}")
|
|
73
|
+
|
|
74
|
+
method = method.strip().lower()
|
|
75
|
+
suffix = _build_suffix(method, random_bytes, comment_prefix)
|
|
76
|
+
original_md5 = md5sum(src)
|
|
77
|
+
original_size = src.stat().st_size
|
|
78
|
+
backup_path: Optional[Path] = None
|
|
79
|
+
|
|
80
|
+
if output is not None:
|
|
81
|
+
dst = Path(output)
|
|
82
|
+
if dst.resolve() == src.resolve():
|
|
83
|
+
raise ValueError("输出路径与源文件相同,原地覆盖请不要传 output")
|
|
84
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
shutil.copyfile(src, dst)
|
|
86
|
+
with open(dst, "ab") as f:
|
|
87
|
+
f.write(suffix)
|
|
88
|
+
else:
|
|
89
|
+
dst = src
|
|
90
|
+
if backup:
|
|
91
|
+
backup_path = src.with_name(f"{src.name}.bak_{time.time_ns()}")
|
|
92
|
+
shutil.copy2(src, backup_path)
|
|
93
|
+
# 先写临时文件再原子替换,避免中途失败导致源文件损坏
|
|
94
|
+
fd, tmp = tempfile.mkstemp(prefix=f".{src.name}.tmp_", dir=src.parent)
|
|
95
|
+
try:
|
|
96
|
+
with os.fdopen(fd, "wb") as out, open(src, "rb") as inp:
|
|
97
|
+
shutil.copyfileobj(inp, out)
|
|
98
|
+
out.write(suffix)
|
|
99
|
+
shutil.copymode(src, tmp)
|
|
100
|
+
os.replace(tmp, src)
|
|
101
|
+
except BaseException:
|
|
102
|
+
Path(tmp).unlink(missing_ok=True)
|
|
103
|
+
raise
|
|
104
|
+
|
|
105
|
+
return ConfuseResult(
|
|
106
|
+
source=src,
|
|
107
|
+
output=dst,
|
|
108
|
+
backup=backup_path,
|
|
109
|
+
original_md5=original_md5,
|
|
110
|
+
new_md5=md5sum(dst),
|
|
111
|
+
method=method,
|
|
112
|
+
size_change=dst.stat().st_size - original_size,
|
|
113
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Pulsar 消息捕获与校验工具(测试专用,不 ack / 不 nack,不影响业务订阅)。
|
|
2
|
+
|
|
3
|
+
依赖:pip install -e '.[pulsar]'
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .capturer import MessageCapturer, MessageNotFoundError
|
|
7
|
+
from .config import DEFAULT_IGNORE_FIELDS, PulsarTestConfig
|
|
8
|
+
from .message import CapturedMessage
|
|
9
|
+
from .validator import (
|
|
10
|
+
MessageValidationError,
|
|
11
|
+
MessageValidator,
|
|
12
|
+
PayloadDecodeError,
|
|
13
|
+
assert_key,
|
|
14
|
+
assert_payload,
|
|
15
|
+
assert_properties,
|
|
16
|
+
assert_publish_time,
|
|
17
|
+
assert_topic,
|
|
18
|
+
diff_payload,
|
|
19
|
+
extract_json_payload,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"CapturedMessage",
|
|
24
|
+
"DEFAULT_IGNORE_FIELDS",
|
|
25
|
+
"MessageCapturer",
|
|
26
|
+
"MessageNotFoundError",
|
|
27
|
+
"MessageValidationError",
|
|
28
|
+
"MessageValidator",
|
|
29
|
+
"PayloadDecodeError",
|
|
30
|
+
"PulsarTestConfig",
|
|
31
|
+
"PulsarTestConsumer",
|
|
32
|
+
"assert_key",
|
|
33
|
+
"assert_payload",
|
|
34
|
+
"assert_properties",
|
|
35
|
+
"assert_publish_time",
|
|
36
|
+
"assert_topic",
|
|
37
|
+
"diff_payload",
|
|
38
|
+
"extract_json_payload",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def __getattr__(name):
|
|
43
|
+
# PulsarTestConsumer 依赖 pulsar-client,延迟导入:只用校验工具时无需安装 pulsar-client
|
|
44
|
+
if name == "PulsarTestConsumer":
|
|
45
|
+
from .consumer import PulsarTestConsumer
|
|
46
|
+
|
|
47
|
+
return PulsarTestConsumer
|
|
48
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""消息捕获器:后台线程持续 receive,暂存到内存,支持按字段查找与轮询等待。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
9
|
+
|
|
10
|
+
from .message import CapturedMessage, format_ms, now_ms
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
Predicate = Callable[[CapturedMessage], bool]
|
|
15
|
+
_MISSING = object()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class MessageNotFoundError(AssertionError):
|
|
19
|
+
"""等待超时仍未找到匹配消息。继承 AssertionError,pytest 中按断言失败展示。"""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _values_equal(actual: Any, expected: Any) -> bool:
|
|
23
|
+
if callable(expected):
|
|
24
|
+
return bool(expected(actual))
|
|
25
|
+
if actual == expected:
|
|
26
|
+
return True
|
|
27
|
+
# properties 都是字符串,兼容 order_id=123 与 "123" 的比较
|
|
28
|
+
if isinstance(actual, str) != isinstance(expected, str) and not isinstance(actual, (dict, list)):
|
|
29
|
+
return str(actual) == str(expected)
|
|
30
|
+
return False
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def build_matcher(predicate: Optional[Predicate] = None, **criteria: Any) -> Predicate:
|
|
34
|
+
"""criteria 的 key 支持:payload 字段/点路径(data.order_id)、properties 名、key、topic。
|
|
35
|
+
value 可以是期望值,也可以是 callable(actual) -> bool。
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def match(msg: CapturedMessage) -> bool:
|
|
39
|
+
for name, expected in criteria.items():
|
|
40
|
+
actual = msg.get(name, _MISSING)
|
|
41
|
+
if actual is _MISSING or not _values_equal(actual, expected):
|
|
42
|
+
return False
|
|
43
|
+
return predicate(msg) if predicate else True
|
|
44
|
+
|
|
45
|
+
return match
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def describe_criteria(predicate: Optional[Predicate], criteria: Dict[str, Any]) -> str:
|
|
49
|
+
parts = [f"{k}={v!r}" for k, v in criteria.items()]
|
|
50
|
+
if predicate is not None:
|
|
51
|
+
parts.append(f"predicate={getattr(predicate, '__name__', repr(predicate))}")
|
|
52
|
+
return ", ".join(parts) or "<任意消息>"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class MessageCapturer:
|
|
56
|
+
def __init__(
|
|
57
|
+
self,
|
|
58
|
+
consumer: Any,
|
|
59
|
+
*,
|
|
60
|
+
wait_timeout_s: float = 10.0,
|
|
61
|
+
poll_interval_s: float = 0.2,
|
|
62
|
+
receive_timeout_ms: int = 1000,
|
|
63
|
+
clock_skew_ms: int = 2000,
|
|
64
|
+
topic: str = "",
|
|
65
|
+
subscription_name: str = "",
|
|
66
|
+
) -> None:
|
|
67
|
+
"""consumer 只需实现 receive(timeout_ms) -> Optional[CapturedMessage]。"""
|
|
68
|
+
self._consumer = consumer
|
|
69
|
+
self.wait_timeout_s = wait_timeout_s
|
|
70
|
+
self.poll_interval_s = poll_interval_s
|
|
71
|
+
self.receive_timeout_ms = receive_timeout_ms
|
|
72
|
+
self.clock_skew_ms = clock_skew_ms
|
|
73
|
+
self.topic = topic
|
|
74
|
+
self.subscription_name = subscription_name
|
|
75
|
+
|
|
76
|
+
self._messages: List[CapturedMessage] = []
|
|
77
|
+
self._cond = threading.Condition()
|
|
78
|
+
self._stop = threading.Event()
|
|
79
|
+
self._thread: Optional[threading.Thread] = None
|
|
80
|
+
self._since_ms = now_ms() - clock_skew_ms
|
|
81
|
+
self._stale_count = 0
|
|
82
|
+
self._errors: List[str] = []
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def from_consumer(cls, consumer: Any) -> "MessageCapturer":
|
|
86
|
+
"""从 PulsarTestConsumer 读取配置构造。"""
|
|
87
|
+
cfg = consumer.config
|
|
88
|
+
return cls(
|
|
89
|
+
consumer,
|
|
90
|
+
wait_timeout_s=cfg.wait_timeout_s,
|
|
91
|
+
poll_interval_s=cfg.poll_interval_s,
|
|
92
|
+
receive_timeout_ms=cfg.receive_timeout_ms,
|
|
93
|
+
clock_skew_ms=cfg.clock_skew_ms,
|
|
94
|
+
topic=cfg.topic,
|
|
95
|
+
subscription_name=cfg.subscription_name,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# ---------- 生命周期 ----------
|
|
99
|
+
def start(self) -> "MessageCapturer":
|
|
100
|
+
if self._thread and self._thread.is_alive():
|
|
101
|
+
return self
|
|
102
|
+
self._stop.clear()
|
|
103
|
+
self._thread = threading.Thread(target=self._run, name="pulsar-message-capturer", daemon=True)
|
|
104
|
+
self._thread.start()
|
|
105
|
+
return self
|
|
106
|
+
|
|
107
|
+
def stop(self, join_timeout_s: Optional[float] = None) -> None:
|
|
108
|
+
self._stop.set()
|
|
109
|
+
t = self._thread
|
|
110
|
+
if t and t.is_alive():
|
|
111
|
+
# receive 最长阻塞 receive_timeout_ms,等它自然返回
|
|
112
|
+
t.join(join_timeout_s if join_timeout_s is not None else self.receive_timeout_ms / 1000 + 2)
|
|
113
|
+
if t.is_alive():
|
|
114
|
+
logger.warning("捕获线程未在超时内退出")
|
|
115
|
+
self._thread = None
|
|
116
|
+
|
|
117
|
+
def __enter__(self) -> "MessageCapturer":
|
|
118
|
+
return self.start()
|
|
119
|
+
|
|
120
|
+
def __exit__(self, *exc: Any) -> None:
|
|
121
|
+
self.stop()
|
|
122
|
+
|
|
123
|
+
def clear(self) -> None:
|
|
124
|
+
"""清空已捕获消息,并把“起始时间”重置为当前,之前发布的消息不再收集。"""
|
|
125
|
+
with self._cond:
|
|
126
|
+
self._messages.clear()
|
|
127
|
+
self._errors.clear()
|
|
128
|
+
self._stale_count = 0
|
|
129
|
+
self._since_ms = now_ms() - self.clock_skew_ms
|
|
130
|
+
|
|
131
|
+
def _run(self) -> None:
|
|
132
|
+
while not self._stop.is_set():
|
|
133
|
+
try:
|
|
134
|
+
msg = self._consumer.receive(self.receive_timeout_ms)
|
|
135
|
+
except Exception as e: # 记录错误继续收,超时报错时一并输出
|
|
136
|
+
with self._cond:
|
|
137
|
+
self._errors.append(f"{format_ms(now_ms())} {e!r}")
|
|
138
|
+
del self._errors[:-20]
|
|
139
|
+
if self._stop.wait(self.poll_interval_s):
|
|
140
|
+
break
|
|
141
|
+
continue
|
|
142
|
+
if msg is None:
|
|
143
|
+
continue
|
|
144
|
+
self.add(msg)
|
|
145
|
+
|
|
146
|
+
def add(self, msg: CapturedMessage) -> bool:
|
|
147
|
+
"""加入一条消息(后台线程调用;也可手工注入)。返回是否被收集。"""
|
|
148
|
+
with self._cond:
|
|
149
|
+
if msg.publish_time_ms and msg.publish_time_ms < self._since_ms:
|
|
150
|
+
self._stale_count += 1 # 上一轮残留的旧消息
|
|
151
|
+
return False
|
|
152
|
+
self._messages.append(msg)
|
|
153
|
+
self._cond.notify_all()
|
|
154
|
+
return True
|
|
155
|
+
|
|
156
|
+
# ---------- 查询 ----------
|
|
157
|
+
@property
|
|
158
|
+
def messages(self) -> List[CapturedMessage]:
|
|
159
|
+
with self._cond:
|
|
160
|
+
return list(self._messages)
|
|
161
|
+
|
|
162
|
+
def find_all(self, predicate: Optional[Predicate] = None, **criteria: Any) -> List[CapturedMessage]:
|
|
163
|
+
"""返回所有匹配消息,按接收顺序。"""
|
|
164
|
+
match = build_matcher(predicate, **criteria)
|
|
165
|
+
return [m for m in self.messages if match(m)]
|
|
166
|
+
|
|
167
|
+
def find_latest(self, predicate: Optional[Predicate] = None, **criteria: Any) -> Optional[CapturedMessage]:
|
|
168
|
+
"""返回最近一条匹配消息(publish_time 最大;相同时取后收到的)。"""
|
|
169
|
+
found = self.find_all(predicate, **criteria)
|
|
170
|
+
if not found:
|
|
171
|
+
return None
|
|
172
|
+
return max(enumerate(found), key=lambda p: (p[1].publish_time_ms, p[0]))[1]
|
|
173
|
+
|
|
174
|
+
def wait_for_all(
|
|
175
|
+
self,
|
|
176
|
+
predicate: Optional[Predicate] = None,
|
|
177
|
+
*,
|
|
178
|
+
count: int = 1,
|
|
179
|
+
timeout: Optional[float] = None,
|
|
180
|
+
**criteria: Any,
|
|
181
|
+
) -> List[CapturedMessage]:
|
|
182
|
+
"""轮询等待直到匹配消息数 >= count,返回所有匹配消息;超时抛 MessageNotFoundError。"""
|
|
183
|
+
if count < 1:
|
|
184
|
+
raise ValueError("count 必须 >= 1")
|
|
185
|
+
timeout = self.wait_timeout_s if timeout is None else timeout
|
|
186
|
+
match = build_matcher(predicate, **criteria)
|
|
187
|
+
deadline = time.monotonic() + timeout
|
|
188
|
+
while True:
|
|
189
|
+
with self._cond:
|
|
190
|
+
found = [m for m in self._messages if match(m)]
|
|
191
|
+
if len(found) >= count:
|
|
192
|
+
return found
|
|
193
|
+
remaining = deadline - time.monotonic()
|
|
194
|
+
if remaining <= 0:
|
|
195
|
+
break
|
|
196
|
+
self._cond.wait(min(remaining, self.poll_interval_s))
|
|
197
|
+
raise MessageNotFoundError(self._not_found_message(predicate, criteria, count, len(found), timeout))
|
|
198
|
+
|
|
199
|
+
def wait_for(
|
|
200
|
+
self,
|
|
201
|
+
predicate: Optional[Predicate] = None,
|
|
202
|
+
*,
|
|
203
|
+
timeout: Optional[float] = None,
|
|
204
|
+
**criteria: Any,
|
|
205
|
+
) -> CapturedMessage:
|
|
206
|
+
"""轮询等待第一条匹配消息出现,返回最近一条匹配消息。"""
|
|
207
|
+
self.wait_for_all(predicate, count=1, timeout=timeout, **criteria)
|
|
208
|
+
return self.find_latest(predicate, **criteria) # type: ignore[return-value]
|
|
209
|
+
|
|
210
|
+
def assert_no_message(
|
|
211
|
+
self, predicate: Optional[Predicate] = None, *, within: float = 3.0, **criteria: Any
|
|
212
|
+
) -> None:
|
|
213
|
+
"""断言 within 秒内没有匹配消息(用于“不应发消息”的场景)。"""
|
|
214
|
+
try:
|
|
215
|
+
found = self.wait_for_all(predicate, count=1, timeout=within, **criteria)
|
|
216
|
+
except MessageNotFoundError:
|
|
217
|
+
return
|
|
218
|
+
raise AssertionError(
|
|
219
|
+
f"期望 {within}s 内没有匹配消息 [{describe_criteria(predicate, criteria)}],"
|
|
220
|
+
f"实际收到 {len(found)} 条:\n" + "\n".join(f" - {m.summary()}" for m in found)
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def _not_found_message(self, predicate, criteria, count, found, timeout) -> str:
|
|
224
|
+
with self._cond:
|
|
225
|
+
msgs = list(self._messages)
|
|
226
|
+
stale = self._stale_count
|
|
227
|
+
errors = list(self._errors)
|
|
228
|
+
lines = [
|
|
229
|
+
f"等待 {timeout}s 未找到匹配消息: 条件 [{describe_criteria(predicate, criteria)}],"
|
|
230
|
+
f"期望至少 {count} 条,实际匹配 {found} 条",
|
|
231
|
+
f"topic={self.topic or '-'} subscription={self.subscription_name or '-'} "
|
|
232
|
+
f"捕获起点={format_ms(self._since_ms)}",
|
|
233
|
+
f"期间共捕获 {len(msgs)} 条消息,过滤掉早于起点的旧消息 {stale} 条",
|
|
234
|
+
]
|
|
235
|
+
if self._thread is None or not self._thread.is_alive():
|
|
236
|
+
lines.append("注意: 捕获线程未运行,是否忘记调用 start()?")
|
|
237
|
+
if msgs:
|
|
238
|
+
lines.append("最近捕获的消息(最多 5 条):")
|
|
239
|
+
lines += [f" - {m.summary()}" for m in msgs[-5:]]
|
|
240
|
+
if errors:
|
|
241
|
+
lines.append("receive 异常(最近 5 条):")
|
|
242
|
+
lines += [f" - {e}" for e in errors[-5:]]
|
|
243
|
+
lines.append(
|
|
244
|
+
"排查建议: 1) 确认业务是否真的发了消息/发到了该 topic;2) 测试消费者需在发消息之前启动(Latest);"
|
|
245
|
+
"3) 同一测试订阅如被多个进程 Shared 消费,消息会被分走;4) 检查查找字段名是否在 payload 或 properties 中"
|
|
246
|
+
)
|
|
247
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Pulsar 测试消费配置。
|
|
2
|
+
|
|
3
|
+
所有超时、topic、订阅名都可通过环境变量或 pytest marker 覆盖,不写死在用例里。
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from dataclasses import dataclass, field, replace
|
|
10
|
+
from typing import Any, Tuple
|
|
11
|
+
|
|
12
|
+
# 默认忽略的动态字段(任意层级同名字段都会被忽略)
|
|
13
|
+
DEFAULT_IGNORE_FIELDS: Tuple[str, ...] = ("created_at", "updated_at", "id", "trace_id")
|
|
14
|
+
|
|
15
|
+
SUPPORTED_CONSUMER_TYPES = ("Shared", "Failover")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _env(name: str, default: Any, cast=str) -> Any:
|
|
19
|
+
raw = os.environ.get(name)
|
|
20
|
+
if raw is None or raw.strip() == "":
|
|
21
|
+
return default
|
|
22
|
+
try:
|
|
23
|
+
return cast(raw.strip())
|
|
24
|
+
except ValueError as e:
|
|
25
|
+
raise ValueError(f"环境变量 {name}={raw!r} 格式错误: {e}") from None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _to_bool(raw: str) -> bool:
|
|
29
|
+
return raw.lower() in ("1", "true", "yes", "y", "on")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _to_tuple(raw: str) -> Tuple[str, ...]:
|
|
33
|
+
return tuple(s.strip() for s in raw.split(",") if s.strip())
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class PulsarTestConfig:
|
|
38
|
+
service_url: str = "pulsar://localhost:6650"
|
|
39
|
+
topic: str = "persistent://public/default/order-created"
|
|
40
|
+
# 固定、可配置的测试订阅名,禁止与业务订阅同名
|
|
41
|
+
subscription_name: str = "test-sub-order-validate"
|
|
42
|
+
# 测试订阅名必须以该前缀开头,防止误用业务订阅(置空可关闭校验)
|
|
43
|
+
subscription_prefix: str = "test-"
|
|
44
|
+
consumer_type: str = "Shared" # Shared / Failover
|
|
45
|
+
# 单次 consumer.receive() 超时(毫秒)
|
|
46
|
+
receive_timeout_ms: int = 1000
|
|
47
|
+
# 等待匹配消息的总超时(秒)
|
|
48
|
+
wait_timeout_s: float = 10.0
|
|
49
|
+
# 轮询等待间隔(秒)
|
|
50
|
+
poll_interval_s: float = 0.2
|
|
51
|
+
# Pulsar client 操作超时 / 连接超时
|
|
52
|
+
operation_timeout_s: int = 10
|
|
53
|
+
connect_timeout_ms: int = 5000
|
|
54
|
+
# 关闭时取消测试订阅,避免残留订阅和 backlog 堆积(只作用于测试订阅)
|
|
55
|
+
unsubscribe_on_close: bool = True
|
|
56
|
+
# 只收 publish_time >= 捕获开始时间 - clock_skew_ms 的消息,过滤上次残留
|
|
57
|
+
clock_skew_ms: int = 2000
|
|
58
|
+
ignore_fields: Tuple[str, ...] = field(default=DEFAULT_IGNORE_FIELDS)
|
|
59
|
+
|
|
60
|
+
def __post_init__(self) -> None:
|
|
61
|
+
errors = []
|
|
62
|
+
if self.consumer_type not in SUPPORTED_CONSUMER_TYPES:
|
|
63
|
+
errors.append(f"consumer_type 只支持 {SUPPORTED_CONSUMER_TYPES},当前为 {self.consumer_type!r}")
|
|
64
|
+
if not self.subscription_name:
|
|
65
|
+
errors.append("subscription_name 不能为空")
|
|
66
|
+
elif self.subscription_prefix and not self.subscription_name.startswith(self.subscription_prefix):
|
|
67
|
+
errors.append(
|
|
68
|
+
f"测试订阅名 {self.subscription_name!r} 必须以 {self.subscription_prefix!r} 开头,"
|
|
69
|
+
"避免误用业务订阅"
|
|
70
|
+
)
|
|
71
|
+
if not self.topic:
|
|
72
|
+
errors.append("topic 不能为空")
|
|
73
|
+
for name in ("receive_timeout_ms", "wait_timeout_s", "poll_interval_s",
|
|
74
|
+
"operation_timeout_s", "connect_timeout_ms"):
|
|
75
|
+
if getattr(self, name) <= 0:
|
|
76
|
+
errors.append(f"{name} 必须 > 0,当前为 {getattr(self, name)!r}")
|
|
77
|
+
if self.clock_skew_ms < 0:
|
|
78
|
+
errors.append(f"clock_skew_ms 不能为负数,当前为 {self.clock_skew_ms!r}")
|
|
79
|
+
if errors:
|
|
80
|
+
raise ValueError("PulsarTestConfig 配置错误:\n - " + "\n - ".join(errors))
|
|
81
|
+
|
|
82
|
+
@classmethod
|
|
83
|
+
def from_env(cls, **overrides: Any) -> "PulsarTestConfig":
|
|
84
|
+
"""从环境变量读取配置,overrides 优先级最高。"""
|
|
85
|
+
d = cls()
|
|
86
|
+
values = dict(
|
|
87
|
+
service_url=_env("PULSAR_SERVICE_URL", d.service_url),
|
|
88
|
+
topic=_env("PULSAR_TOPIC", d.topic),
|
|
89
|
+
subscription_name=_env("PULSAR_TEST_SUBSCRIPTION", d.subscription_name),
|
|
90
|
+
subscription_prefix=os.environ.get("PULSAR_TEST_SUB_PREFIX", d.subscription_prefix),
|
|
91
|
+
consumer_type=_env("PULSAR_CONSUMER_TYPE", d.consumer_type),
|
|
92
|
+
receive_timeout_ms=_env("PULSAR_RECEIVE_TIMEOUT_MS", d.receive_timeout_ms, int),
|
|
93
|
+
wait_timeout_s=_env("PULSAR_WAIT_TIMEOUT_S", d.wait_timeout_s, float),
|
|
94
|
+
poll_interval_s=_env("PULSAR_POLL_INTERVAL_S", d.poll_interval_s, float),
|
|
95
|
+
operation_timeout_s=_env("PULSAR_OPERATION_TIMEOUT_S", d.operation_timeout_s, int),
|
|
96
|
+
connect_timeout_ms=_env("PULSAR_CONNECT_TIMEOUT_MS", d.connect_timeout_ms, int),
|
|
97
|
+
unsubscribe_on_close=_env("PULSAR_UNSUBSCRIBE_ON_CLOSE", d.unsubscribe_on_close, _to_bool),
|
|
98
|
+
clock_skew_ms=_env("PULSAR_CLOCK_SKEW_MS", d.clock_skew_ms, int),
|
|
99
|
+
ignore_fields=_env("PULSAR_IGNORE_FIELDS", d.ignore_fields, _to_tuple),
|
|
100
|
+
)
|
|
101
|
+
values.update(overrides)
|
|
102
|
+
return cls(**values)
|
|
103
|
+
|
|
104
|
+
def with_overrides(self, **overrides: Any) -> "PulsarTestConfig":
|
|
105
|
+
if not overrides:
|
|
106
|
+
return self
|
|
107
|
+
unknown = set(overrides) - set(self.__dataclass_fields__)
|
|
108
|
+
if unknown:
|
|
109
|
+
raise ValueError(f"PulsarTestConfig 不支持的配置项: {sorted(unknown)}")
|
|
110
|
+
if "ignore_fields" in overrides:
|
|
111
|
+
overrides["ignore_fields"] = tuple(overrides["ignore_fields"])
|
|
112
|
+
return replace(self, **overrides)
|