shopline-qa-tools 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. shopline_qa_tools-1.0.0/MANIFEST.in +1 -0
  2. shopline_qa_tools-1.0.0/PKG-INFO +9 -0
  3. shopline_qa_tools-1.0.0/pyproject.toml +27 -0
  4. shopline_qa_tools-1.0.0/setup.cfg +4 -0
  5. shopline_qa_tools-1.0.0/src/shopline_qa_tools/__init__.py +3 -0
  6. shopline_qa_tools-1.0.0/src/shopline_qa_tools/confusefile/__init__.py +5 -0
  7. shopline_qa_tools-1.0.0/src/shopline_qa_tools/confusefile/__main__.py +5 -0
  8. shopline_qa_tools-1.0.0/src/shopline_qa_tools/confusefile/cli.py +81 -0
  9. shopline_qa_tools-1.0.0/src/shopline_qa_tools/confusefile/core.py +113 -0
  10. shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/__init__.py +48 -0
  11. shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/capturer.py +247 -0
  12. shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/config.py +112 -0
  13. shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/consumer.py +128 -0
  14. shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/message.py +127 -0
  15. shopline_qa_tools-1.0.0/src/shopline_qa_tools/pulsar_capture/validator.py +308 -0
  16. shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/PKG-INFO +9 -0
  17. shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/SOURCES.txt +20 -0
  18. shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/dependency_links.txt +1 -0
  19. shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/entry_points.txt +2 -0
  20. shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/requires.txt +5 -0
  21. shopline_qa_tools-1.0.0/src/shopline_qa_tools.egg-info/top_level.txt +1 -0
  22. shopline_qa_tools-1.0.0/tests/test_confusefile.py +92 -0
@@ -0,0 +1 @@
1
+ exclude README.md
@@ -0,0 +1,9 @@
1
+ Metadata-Version: 2.4
2
+ Name: shopline-qa-tools
3
+ Version: 1.0.0
4
+ Summary: QA 常用工具包
5
+ Requires-Python: >=3.8
6
+ Requires-Dist: pulsar-client>=3.4
7
+ Requires-Dist: deepdiff>=6
8
+ Provides-Extra: dev
9
+ Requires-Dist: pytest>=7; extra == "dev"
@@ -0,0 +1,27 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "shopline-qa-tools"
7
+ version = "1.0.0"
8
+ description = "QA 常用工具包"
9
+ requires-python = ">=3.8"
10
+ dependencies = ["pulsar-client>=3.4", "deepdiff>=6"]
11
+
12
+ [project.optional-dependencies]
13
+ dev = ["pytest>=7"]
14
+
15
+ [project.scripts]
16
+ confusefile = "shopline_qa_tools.confusefile.cli:main"
17
+
18
+ [tool.setuptools.packages.find]
19
+ where = ["src"]
20
+
21
+ [tool.pytest.ini_options]
22
+ testpaths = ["tests"]
23
+ pythonpath = ["src"]
24
+ markers = [
25
+ "pulsar: 需要真实 Pulsar 的集成用例",
26
+ "pulsar_config(**kwargs): 覆盖单个用例的 Pulsar 配置,如 topic/subscription_name/wait_timeout_s",
27
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """QA 常用工具包。"""
2
+
3
+ __version__ = "1.0.0"
@@ -0,0 +1,5 @@
1
+ """文件混淆:在文件末尾追加内容,使文件 MD5 发生变化。"""
2
+
3
+ from .core import METHODS, ConfuseResult, confuse, md5sum
4
+
5
+ __all__ = ["METHODS", "ConfuseResult", "confuse", "md5sum"]
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
@@ -0,0 +1,81 @@
1
+ """confusefile 命令行入口。"""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sys
8
+ from pathlib import Path
9
+ from typing import List, Optional
10
+
11
+ from .core import DEFAULT_COMMENT_PREFIX, DEFAULT_RANDOM_BYTES, METHODS, confuse
12
+
13
+
14
+ def build_parser() -> argparse.ArgumentParser:
15
+ p = argparse.ArgumentParser(
16
+ prog="confusefile",
17
+ description="在文件末尾追加内容,改变文件 MD5。默认原地覆盖并自动备份。",
18
+ )
19
+ p.add_argument("files", nargs="+", help="要处理的文件,可传多个")
20
+ p.add_argument("-o", "--output", help="输出文件路径(仅处理单个文件时可用),源文件保持不变")
21
+ p.add_argument("-d", "--out-dir", help="输出目录(可批量),文件名不变,源文件保持不变")
22
+ p.add_argument("-m", "--method", choices=METHODS, default="random",
23
+ help="random=追加随机字节(默认,适合任意文件);comment=追加一行注释(适合文本文件)")
24
+ p.add_argument("-n", "--bytes", type=int, default=DEFAULT_RANDOM_BYTES, dest="random_bytes",
25
+ help=f"random 方法追加的字节数,默认 {DEFAULT_RANDOM_BYTES}")
26
+ p.add_argument("-p", "--prefix", default=DEFAULT_COMMENT_PREFIX, dest="comment_prefix",
27
+ help=f'comment 方法的注释前缀,默认 "{DEFAULT_COMMENT_PREFIX}"')
28
+ p.add_argument("--no-backup", action="store_true", help="原地覆盖时不备份源文件")
29
+ p.add_argument("--json", action="store_true", help="以 JSON 输出结果")
30
+ return p
31
+
32
+
33
+ def main(argv: Optional[List[str]] = None) -> int:
34
+ args = build_parser().parse_args(argv)
35
+
36
+ if args.output and args.out_dir:
37
+ print("error: -o 和 -d 不能同时使用", file=sys.stderr)
38
+ return 2
39
+ if args.output and len(args.files) > 1:
40
+ print("error: 处理多个文件时请用 -d 指定输出目录", file=sys.stderr)
41
+ return 2
42
+
43
+ results, failed = [], 0
44
+ for f in args.files:
45
+ if args.output:
46
+ output = args.output
47
+ elif args.out_dir:
48
+ output = str(Path(args.out_dir) / Path(f).name)
49
+ else:
50
+ output = None
51
+ try:
52
+ r = confuse(f, output, method=args.method, random_bytes=args.random_bytes,
53
+ comment_prefix=args.comment_prefix, backup=not args.no_backup)
54
+ except (OSError, ValueError) as e:
55
+ failed += 1
56
+ print(f"error: {f}: {e}", file=sys.stderr)
57
+ continue
58
+ results.append(r)
59
+ if not args.json:
60
+ print(f"file: {r.output}")
61
+ if r.backup:
62
+ print(f"backup: {r.backup}")
63
+ print(f"md5: {r.original_md5} -> {r.new_md5}")
64
+ print(f"size: +{r.size_change} bytes ({r.method})")
65
+ print()
66
+
67
+ if args.json:
68
+ print(json.dumps([
69
+ {
70
+ "source": str(r.source), "output": str(r.output),
71
+ "backup": str(r.backup) if r.backup else None,
72
+ "originalMd5": r.original_md5, "newMd5": r.new_md5,
73
+ "method": r.method, "sizeChange": r.size_change,
74
+ } for r in results
75
+ ], ensure_ascii=False, indent=2))
76
+
77
+ return 1 if failed else 0
78
+
79
+
80
+ if __name__ == "__main__":
81
+ sys.exit(main())
@@ -0,0 +1,113 @@
1
+ """文件混淆核心逻辑。"""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import os
7
+ import secrets
8
+ import shutil
9
+ import tempfile
10
+ import time
11
+ import uuid
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+ from typing import Optional, Union
15
+
16
+ PathLike = Union[str, os.PathLike]
17
+
18
+ RANDOM = "random"
19
+ COMMENT = "comment"
20
+ METHODS = (RANDOM, COMMENT)
21
+
22
+ DEFAULT_RANDOM_BYTES = 8
23
+ DEFAULT_COMMENT_PREFIX = "# Confused: "
24
+
25
+
26
+ @dataclass
27
+ class ConfuseResult:
28
+ source: Path
29
+ output: Path
30
+ backup: Optional[Path]
31
+ original_md5: str
32
+ new_md5: str
33
+ method: str
34
+ size_change: int
35
+
36
+
37
+ def md5sum(path: PathLike) -> str:
38
+ h = hashlib.md5()
39
+ with open(path, "rb") as f:
40
+ for chunk in iter(lambda: f.read(1024 * 1024), b""):
41
+ h.update(chunk)
42
+ return h.hexdigest()
43
+
44
+
45
+ def _build_suffix(method: str, random_bytes: int, comment_prefix: str) -> bytes:
46
+ if method == RANDOM:
47
+ if random_bytes <= 0:
48
+ raise ValueError("random_bytes 必须大于 0")
49
+ return secrets.token_bytes(random_bytes)
50
+ if method == COMMENT:
51
+ line = f"\n{comment_prefix}{uuid.uuid4()}_{time.time_ns()}\n"
52
+ return line.encode("utf-8")
53
+ raise ValueError(f"不支持的混淆方法: {method},可选: {', '.join(METHODS)}")
54
+
55
+
56
+ def confuse(
57
+ src: PathLike,
58
+ output: Optional[PathLike] = None,
59
+ *,
60
+ method: str = RANDOM,
61
+ random_bytes: int = DEFAULT_RANDOM_BYTES,
62
+ comment_prefix: str = DEFAULT_COMMENT_PREFIX,
63
+ backup: bool = True,
64
+ ) -> ConfuseResult:
65
+ """混淆文件,使其 MD5 改变。
66
+
67
+ - 传 ``output``:生成新文件,源文件不变。
68
+ - 不传 ``output``:原地覆盖源文件;``backup=True`` 时先备份为 ``<文件名>.bak_<时间戳>``。
69
+ """
70
+ src = Path(src)
71
+ if not src.is_file():
72
+ raise FileNotFoundError(f"文件不存在: {src}")
73
+
74
+ method = method.strip().lower()
75
+ suffix = _build_suffix(method, random_bytes, comment_prefix)
76
+ original_md5 = md5sum(src)
77
+ original_size = src.stat().st_size
78
+ backup_path: Optional[Path] = None
79
+
80
+ if output is not None:
81
+ dst = Path(output)
82
+ if dst.resolve() == src.resolve():
83
+ raise ValueError("输出路径与源文件相同,原地覆盖请不要传 output")
84
+ dst.parent.mkdir(parents=True, exist_ok=True)
85
+ shutil.copyfile(src, dst)
86
+ with open(dst, "ab") as f:
87
+ f.write(suffix)
88
+ else:
89
+ dst = src
90
+ if backup:
91
+ backup_path = src.with_name(f"{src.name}.bak_{time.time_ns()}")
92
+ shutil.copy2(src, backup_path)
93
+ # 先写临时文件再原子替换,避免中途失败导致源文件损坏
94
+ fd, tmp = tempfile.mkstemp(prefix=f".{src.name}.tmp_", dir=src.parent)
95
+ try:
96
+ with os.fdopen(fd, "wb") as out, open(src, "rb") as inp:
97
+ shutil.copyfileobj(inp, out)
98
+ out.write(suffix)
99
+ shutil.copymode(src, tmp)
100
+ os.replace(tmp, src)
101
+ except BaseException:
102
+ Path(tmp).unlink(missing_ok=True)
103
+ raise
104
+
105
+ return ConfuseResult(
106
+ source=src,
107
+ output=dst,
108
+ backup=backup_path,
109
+ original_md5=original_md5,
110
+ new_md5=md5sum(dst),
111
+ method=method,
112
+ size_change=dst.stat().st_size - original_size,
113
+ )
@@ -0,0 +1,48 @@
1
+ """Pulsar 消息捕获与校验工具(测试专用,不 ack / 不 nack,不影响业务订阅)。
2
+
3
+ 依赖:pip install -e '.[pulsar]'
4
+ """
5
+
6
+ from .capturer import MessageCapturer, MessageNotFoundError
7
+ from .config import DEFAULT_IGNORE_FIELDS, PulsarTestConfig
8
+ from .message import CapturedMessage
9
+ from .validator import (
10
+ MessageValidationError,
11
+ MessageValidator,
12
+ PayloadDecodeError,
13
+ assert_key,
14
+ assert_payload,
15
+ assert_properties,
16
+ assert_publish_time,
17
+ assert_topic,
18
+ diff_payload,
19
+ extract_json_payload,
20
+ )
21
+
22
+ __all__ = [
23
+ "CapturedMessage",
24
+ "DEFAULT_IGNORE_FIELDS",
25
+ "MessageCapturer",
26
+ "MessageNotFoundError",
27
+ "MessageValidationError",
28
+ "MessageValidator",
29
+ "PayloadDecodeError",
30
+ "PulsarTestConfig",
31
+ "PulsarTestConsumer",
32
+ "assert_key",
33
+ "assert_payload",
34
+ "assert_properties",
35
+ "assert_publish_time",
36
+ "assert_topic",
37
+ "diff_payload",
38
+ "extract_json_payload",
39
+ ]
40
+
41
+
42
+ def __getattr__(name):
43
+ # PulsarTestConsumer 依赖 pulsar-client,延迟导入:只用校验工具时无需安装 pulsar-client
44
+ if name == "PulsarTestConsumer":
45
+ from .consumer import PulsarTestConsumer
46
+
47
+ return PulsarTestConsumer
48
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,247 @@
1
+ """消息捕获器:后台线程持续 receive,暂存到内存,支持按字段查找与轮询等待。"""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import threading
7
+ import time
8
+ from typing import Any, Callable, Dict, List, Optional
9
+
10
+ from .message import CapturedMessage, format_ms, now_ms
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+ Predicate = Callable[[CapturedMessage], bool]
15
+ _MISSING = object()
16
+
17
+
18
+ class MessageNotFoundError(AssertionError):
19
+ """等待超时仍未找到匹配消息。继承 AssertionError,pytest 中按断言失败展示。"""
20
+
21
+
22
+ def _values_equal(actual: Any, expected: Any) -> bool:
23
+ if callable(expected):
24
+ return bool(expected(actual))
25
+ if actual == expected:
26
+ return True
27
+ # properties 都是字符串,兼容 order_id=123 与 "123" 的比较
28
+ if isinstance(actual, str) != isinstance(expected, str) and not isinstance(actual, (dict, list)):
29
+ return str(actual) == str(expected)
30
+ return False
31
+
32
+
33
+ def build_matcher(predicate: Optional[Predicate] = None, **criteria: Any) -> Predicate:
34
+ """criteria 的 key 支持:payload 字段/点路径(data.order_id)、properties 名、key、topic。
35
+ value 可以是期望值,也可以是 callable(actual) -> bool。
36
+ """
37
+
38
+ def match(msg: CapturedMessage) -> bool:
39
+ for name, expected in criteria.items():
40
+ actual = msg.get(name, _MISSING)
41
+ if actual is _MISSING or not _values_equal(actual, expected):
42
+ return False
43
+ return predicate(msg) if predicate else True
44
+
45
+ return match
46
+
47
+
48
+ def describe_criteria(predicate: Optional[Predicate], criteria: Dict[str, Any]) -> str:
49
+ parts = [f"{k}={v!r}" for k, v in criteria.items()]
50
+ if predicate is not None:
51
+ parts.append(f"predicate={getattr(predicate, '__name__', repr(predicate))}")
52
+ return ", ".join(parts) or "<任意消息>"
53
+
54
+
55
+ class MessageCapturer:
56
+ def __init__(
57
+ self,
58
+ consumer: Any,
59
+ *,
60
+ wait_timeout_s: float = 10.0,
61
+ poll_interval_s: float = 0.2,
62
+ receive_timeout_ms: int = 1000,
63
+ clock_skew_ms: int = 2000,
64
+ topic: str = "",
65
+ subscription_name: str = "",
66
+ ) -> None:
67
+ """consumer 只需实现 receive(timeout_ms) -> Optional[CapturedMessage]。"""
68
+ self._consumer = consumer
69
+ self.wait_timeout_s = wait_timeout_s
70
+ self.poll_interval_s = poll_interval_s
71
+ self.receive_timeout_ms = receive_timeout_ms
72
+ self.clock_skew_ms = clock_skew_ms
73
+ self.topic = topic
74
+ self.subscription_name = subscription_name
75
+
76
+ self._messages: List[CapturedMessage] = []
77
+ self._cond = threading.Condition()
78
+ self._stop = threading.Event()
79
+ self._thread: Optional[threading.Thread] = None
80
+ self._since_ms = now_ms() - clock_skew_ms
81
+ self._stale_count = 0
82
+ self._errors: List[str] = []
83
+
84
+ @classmethod
85
+ def from_consumer(cls, consumer: Any) -> "MessageCapturer":
86
+ """从 PulsarTestConsumer 读取配置构造。"""
87
+ cfg = consumer.config
88
+ return cls(
89
+ consumer,
90
+ wait_timeout_s=cfg.wait_timeout_s,
91
+ poll_interval_s=cfg.poll_interval_s,
92
+ receive_timeout_ms=cfg.receive_timeout_ms,
93
+ clock_skew_ms=cfg.clock_skew_ms,
94
+ topic=cfg.topic,
95
+ subscription_name=cfg.subscription_name,
96
+ )
97
+
98
+ # ---------- 生命周期 ----------
99
+ def start(self) -> "MessageCapturer":
100
+ if self._thread and self._thread.is_alive():
101
+ return self
102
+ self._stop.clear()
103
+ self._thread = threading.Thread(target=self._run, name="pulsar-message-capturer", daemon=True)
104
+ self._thread.start()
105
+ return self
106
+
107
+ def stop(self, join_timeout_s: Optional[float] = None) -> None:
108
+ self._stop.set()
109
+ t = self._thread
110
+ if t and t.is_alive():
111
+ # receive 最长阻塞 receive_timeout_ms,等它自然返回
112
+ t.join(join_timeout_s if join_timeout_s is not None else self.receive_timeout_ms / 1000 + 2)
113
+ if t.is_alive():
114
+ logger.warning("捕获线程未在超时内退出")
115
+ self._thread = None
116
+
117
+ def __enter__(self) -> "MessageCapturer":
118
+ return self.start()
119
+
120
+ def __exit__(self, *exc: Any) -> None:
121
+ self.stop()
122
+
123
+ def clear(self) -> None:
124
+ """清空已捕获消息,并把“起始时间”重置为当前,之前发布的消息不再收集。"""
125
+ with self._cond:
126
+ self._messages.clear()
127
+ self._errors.clear()
128
+ self._stale_count = 0
129
+ self._since_ms = now_ms() - self.clock_skew_ms
130
+
131
+ def _run(self) -> None:
132
+ while not self._stop.is_set():
133
+ try:
134
+ msg = self._consumer.receive(self.receive_timeout_ms)
135
+ except Exception as e: # 记录错误继续收,超时报错时一并输出
136
+ with self._cond:
137
+ self._errors.append(f"{format_ms(now_ms())} {e!r}")
138
+ del self._errors[:-20]
139
+ if self._stop.wait(self.poll_interval_s):
140
+ break
141
+ continue
142
+ if msg is None:
143
+ continue
144
+ self.add(msg)
145
+
146
+ def add(self, msg: CapturedMessage) -> bool:
147
+ """加入一条消息(后台线程调用;也可手工注入)。返回是否被收集。"""
148
+ with self._cond:
149
+ if msg.publish_time_ms and msg.publish_time_ms < self._since_ms:
150
+ self._stale_count += 1 # 上一轮残留的旧消息
151
+ return False
152
+ self._messages.append(msg)
153
+ self._cond.notify_all()
154
+ return True
155
+
156
+ # ---------- 查询 ----------
157
+ @property
158
+ def messages(self) -> List[CapturedMessage]:
159
+ with self._cond:
160
+ return list(self._messages)
161
+
162
+ def find_all(self, predicate: Optional[Predicate] = None, **criteria: Any) -> List[CapturedMessage]:
163
+ """返回所有匹配消息,按接收顺序。"""
164
+ match = build_matcher(predicate, **criteria)
165
+ return [m for m in self.messages if match(m)]
166
+
167
+ def find_latest(self, predicate: Optional[Predicate] = None, **criteria: Any) -> Optional[CapturedMessage]:
168
+ """返回最近一条匹配消息(publish_time 最大;相同时取后收到的)。"""
169
+ found = self.find_all(predicate, **criteria)
170
+ if not found:
171
+ return None
172
+ return max(enumerate(found), key=lambda p: (p[1].publish_time_ms, p[0]))[1]
173
+
174
+ def wait_for_all(
175
+ self,
176
+ predicate: Optional[Predicate] = None,
177
+ *,
178
+ count: int = 1,
179
+ timeout: Optional[float] = None,
180
+ **criteria: Any,
181
+ ) -> List[CapturedMessage]:
182
+ """轮询等待直到匹配消息数 >= count,返回所有匹配消息;超时抛 MessageNotFoundError。"""
183
+ if count < 1:
184
+ raise ValueError("count 必须 >= 1")
185
+ timeout = self.wait_timeout_s if timeout is None else timeout
186
+ match = build_matcher(predicate, **criteria)
187
+ deadline = time.monotonic() + timeout
188
+ while True:
189
+ with self._cond:
190
+ found = [m for m in self._messages if match(m)]
191
+ if len(found) >= count:
192
+ return found
193
+ remaining = deadline - time.monotonic()
194
+ if remaining <= 0:
195
+ break
196
+ self._cond.wait(min(remaining, self.poll_interval_s))
197
+ raise MessageNotFoundError(self._not_found_message(predicate, criteria, count, len(found), timeout))
198
+
199
+ def wait_for(
200
+ self,
201
+ predicate: Optional[Predicate] = None,
202
+ *,
203
+ timeout: Optional[float] = None,
204
+ **criteria: Any,
205
+ ) -> CapturedMessage:
206
+ """轮询等待第一条匹配消息出现,返回最近一条匹配消息。"""
207
+ self.wait_for_all(predicate, count=1, timeout=timeout, **criteria)
208
+ return self.find_latest(predicate, **criteria) # type: ignore[return-value]
209
+
210
+ def assert_no_message(
211
+ self, predicate: Optional[Predicate] = None, *, within: float = 3.0, **criteria: Any
212
+ ) -> None:
213
+ """断言 within 秒内没有匹配消息(用于“不应发消息”的场景)。"""
214
+ try:
215
+ found = self.wait_for_all(predicate, count=1, timeout=within, **criteria)
216
+ except MessageNotFoundError:
217
+ return
218
+ raise AssertionError(
219
+ f"期望 {within}s 内没有匹配消息 [{describe_criteria(predicate, criteria)}],"
220
+ f"实际收到 {len(found)} 条:\n" + "\n".join(f" - {m.summary()}" for m in found)
221
+ )
222
+
223
+ def _not_found_message(self, predicate, criteria, count, found, timeout) -> str:
224
+ with self._cond:
225
+ msgs = list(self._messages)
226
+ stale = self._stale_count
227
+ errors = list(self._errors)
228
+ lines = [
229
+ f"等待 {timeout}s 未找到匹配消息: 条件 [{describe_criteria(predicate, criteria)}],"
230
+ f"期望至少 {count} 条,实际匹配 {found} 条",
231
+ f"topic={self.topic or '-'} subscription={self.subscription_name or '-'} "
232
+ f"捕获起点={format_ms(self._since_ms)}",
233
+ f"期间共捕获 {len(msgs)} 条消息,过滤掉早于起点的旧消息 {stale} 条",
234
+ ]
235
+ if self._thread is None or not self._thread.is_alive():
236
+ lines.append("注意: 捕获线程未运行,是否忘记调用 start()?")
237
+ if msgs:
238
+ lines.append("最近捕获的消息(最多 5 条):")
239
+ lines += [f" - {m.summary()}" for m in msgs[-5:]]
240
+ if errors:
241
+ lines.append("receive 异常(最近 5 条):")
242
+ lines += [f" - {e}" for e in errors[-5:]]
243
+ lines.append(
244
+ "排查建议: 1) 确认业务是否真的发了消息/发到了该 topic;2) 测试消费者需在发消息之前启动(Latest);"
245
+ "3) 同一测试订阅如被多个进程 Shared 消费,消息会被分走;4) 检查查找字段名是否在 payload 或 properties 中"
246
+ )
247
+ return "\n".join(lines)
@@ -0,0 +1,112 @@
1
+ """Pulsar 测试消费配置。
2
+
3
+ 所有超时、topic、订阅名都可通过环境变量或 pytest marker 覆盖,不写死在用例里。
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import os
9
+ from dataclasses import dataclass, field, replace
10
+ from typing import Any, Tuple
11
+
12
+ # 默认忽略的动态字段(任意层级同名字段都会被忽略)
13
+ DEFAULT_IGNORE_FIELDS: Tuple[str, ...] = ("created_at", "updated_at", "id", "trace_id")
14
+
15
+ SUPPORTED_CONSUMER_TYPES = ("Shared", "Failover")
16
+
17
+
18
+ def _env(name: str, default: Any, cast=str) -> Any:
19
+ raw = os.environ.get(name)
20
+ if raw is None or raw.strip() == "":
21
+ return default
22
+ try:
23
+ return cast(raw.strip())
24
+ except ValueError as e:
25
+ raise ValueError(f"环境变量 {name}={raw!r} 格式错误: {e}") from None
26
+
27
+
28
+ def _to_bool(raw: str) -> bool:
29
+ return raw.lower() in ("1", "true", "yes", "y", "on")
30
+
31
+
32
+ def _to_tuple(raw: str) -> Tuple[str, ...]:
33
+ return tuple(s.strip() for s in raw.split(",") if s.strip())
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class PulsarTestConfig:
38
+ service_url: str = "pulsar://localhost:6650"
39
+ topic: str = "persistent://public/default/order-created"
40
+ # 固定、可配置的测试订阅名,禁止与业务订阅同名
41
+ subscription_name: str = "test-sub-order-validate"
42
+ # 测试订阅名必须以该前缀开头,防止误用业务订阅(置空可关闭校验)
43
+ subscription_prefix: str = "test-"
44
+ consumer_type: str = "Shared" # Shared / Failover
45
+ # 单次 consumer.receive() 超时(毫秒)
46
+ receive_timeout_ms: int = 1000
47
+ # 等待匹配消息的总超时(秒)
48
+ wait_timeout_s: float = 10.0
49
+ # 轮询等待间隔(秒)
50
+ poll_interval_s: float = 0.2
51
+ # Pulsar client 操作超时 / 连接超时
52
+ operation_timeout_s: int = 10
53
+ connect_timeout_ms: int = 5000
54
+ # 关闭时取消测试订阅,避免残留订阅和 backlog 堆积(只作用于测试订阅)
55
+ unsubscribe_on_close: bool = True
56
+ # 只收 publish_time >= 捕获开始时间 - clock_skew_ms 的消息,过滤上次残留
57
+ clock_skew_ms: int = 2000
58
+ ignore_fields: Tuple[str, ...] = field(default=DEFAULT_IGNORE_FIELDS)
59
+
60
+ def __post_init__(self) -> None:
61
+ errors = []
62
+ if self.consumer_type not in SUPPORTED_CONSUMER_TYPES:
63
+ errors.append(f"consumer_type 只支持 {SUPPORTED_CONSUMER_TYPES},当前为 {self.consumer_type!r}")
64
+ if not self.subscription_name:
65
+ errors.append("subscription_name 不能为空")
66
+ elif self.subscription_prefix and not self.subscription_name.startswith(self.subscription_prefix):
67
+ errors.append(
68
+ f"测试订阅名 {self.subscription_name!r} 必须以 {self.subscription_prefix!r} 开头,"
69
+ "避免误用业务订阅"
70
+ )
71
+ if not self.topic:
72
+ errors.append("topic 不能为空")
73
+ for name in ("receive_timeout_ms", "wait_timeout_s", "poll_interval_s",
74
+ "operation_timeout_s", "connect_timeout_ms"):
75
+ if getattr(self, name) <= 0:
76
+ errors.append(f"{name} 必须 > 0,当前为 {getattr(self, name)!r}")
77
+ if self.clock_skew_ms < 0:
78
+ errors.append(f"clock_skew_ms 不能为负数,当前为 {self.clock_skew_ms!r}")
79
+ if errors:
80
+ raise ValueError("PulsarTestConfig 配置错误:\n - " + "\n - ".join(errors))
81
+
82
+ @classmethod
83
+ def from_env(cls, **overrides: Any) -> "PulsarTestConfig":
84
+ """从环境变量读取配置,overrides 优先级最高。"""
85
+ d = cls()
86
+ values = dict(
87
+ service_url=_env("PULSAR_SERVICE_URL", d.service_url),
88
+ topic=_env("PULSAR_TOPIC", d.topic),
89
+ subscription_name=_env("PULSAR_TEST_SUBSCRIPTION", d.subscription_name),
90
+ subscription_prefix=os.environ.get("PULSAR_TEST_SUB_PREFIX", d.subscription_prefix),
91
+ consumer_type=_env("PULSAR_CONSUMER_TYPE", d.consumer_type),
92
+ receive_timeout_ms=_env("PULSAR_RECEIVE_TIMEOUT_MS", d.receive_timeout_ms, int),
93
+ wait_timeout_s=_env("PULSAR_WAIT_TIMEOUT_S", d.wait_timeout_s, float),
94
+ poll_interval_s=_env("PULSAR_POLL_INTERVAL_S", d.poll_interval_s, float),
95
+ operation_timeout_s=_env("PULSAR_OPERATION_TIMEOUT_S", d.operation_timeout_s, int),
96
+ connect_timeout_ms=_env("PULSAR_CONNECT_TIMEOUT_MS", d.connect_timeout_ms, int),
97
+ unsubscribe_on_close=_env("PULSAR_UNSUBSCRIBE_ON_CLOSE", d.unsubscribe_on_close, _to_bool),
98
+ clock_skew_ms=_env("PULSAR_CLOCK_SKEW_MS", d.clock_skew_ms, int),
99
+ ignore_fields=_env("PULSAR_IGNORE_FIELDS", d.ignore_fields, _to_tuple),
100
+ )
101
+ values.update(overrides)
102
+ return cls(**values)
103
+
104
+ def with_overrides(self, **overrides: Any) -> "PulsarTestConfig":
105
+ if not overrides:
106
+ return self
107
+ unknown = set(overrides) - set(self.__dataclass_fields__)
108
+ if unknown:
109
+ raise ValueError(f"PulsarTestConfig 不支持的配置项: {sorted(unknown)}")
110
+ if "ignore_fields" in overrides:
111
+ overrides["ignore_fields"] = tuple(overrides["ignore_fields"])
112
+ return replace(self, **overrides)