zenx-lite-python 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. zenx_lite_python-0.1.0/.gitignore +8 -0
  2. zenx_lite_python-0.1.0/AGENTS.md +27 -0
  3. zenx_lite_python-0.1.0/PKG-INFO +14 -0
  4. zenx_lite_python-0.1.0/PROJECT_GUIDE.md +16 -0
  5. zenx_lite_python-0.1.0/pyproject.toml +30 -0
  6. zenx_lite_python-0.1.0/tests/__init__.py +0 -0
  7. zenx_lite_python-0.1.0/tests/fixtures/cli_project.py +62 -0
  8. zenx_lite_python-0.1.0/tests/helpers.py +72 -0
  9. zenx_lite_python-0.1.0/tests/test_cli.py +213 -0
  10. zenx_lite_python-0.1.0/tests/test_configuration.py +429 -0
  11. zenx_lite_python-0.1.0/tests/test_delivery.py +185 -0
  12. zenx_lite_python-0.1.0/tests/test_destinations.py +159 -0
  13. zenx_lite_python-0.1.0/tests/test_http.py +335 -0
  14. zenx_lite_python-0.1.0/tests/test_notifiers.py +298 -0
  15. zenx_lite_python-0.1.0/tests/test_redis_store.py +323 -0
  16. zenx_lite_python-0.1.0/tests/test_runtime.py +361 -0
  17. zenx_lite_python-0.1.0/tests/test_scheduler.py +61 -0
  18. zenx_lite_python-0.1.0/uv.lock +352 -0
  19. zenx_lite_python-0.1.0/zenx_lite/__init__.py +8 -0
  20. zenx_lite_python-0.1.0/zenx_lite/app.py +377 -0
  21. zenx_lite_python-0.1.0/zenx_lite/config.py +56 -0
  22. zenx_lite_python-0.1.0/zenx_lite/configuration.py +359 -0
  23. zenx_lite_python-0.1.0/zenx_lite/crawler.py +69 -0
  24. zenx_lite_python-0.1.0/zenx_lite/delivery.py +139 -0
  25. zenx_lite_python-0.1.0/zenx_lite/destinations/__init__.py +5 -0
  26. zenx_lite_python-0.1.0/zenx_lite/destinations/base.py +32 -0
  27. zenx_lite_python-0.1.0/zenx_lite/destinations/jsonl.py +132 -0
  28. zenx_lite_python-0.1.0/zenx_lite/destinations/synoptic/__init__.py +5 -0
  29. zenx_lite_python-0.1.0/zenx_lite/destinations/synoptic/source.py +136 -0
  30. zenx_lite_python-0.1.0/zenx_lite/errors.py +21 -0
  31. zenx_lite_python-0.1.0/zenx_lite/http.py +130 -0
  32. zenx_lite_python-0.1.0/zenx_lite/notifiers/__init__.py +5 -0
  33. zenx_lite_python-0.1.0/zenx_lite/notifiers/base.py +15 -0
  34. zenx_lite_python-0.1.0/zenx_lite/notifiers/magent.py +86 -0
  35. zenx_lite_python-0.1.0/zenx_lite/scheduler.py +63 -0
  36. zenx_lite_python-0.1.0/zenx_lite/sessions.py +102 -0
  37. zenx_lite_python-0.1.0/zenx_lite/store.py +45 -0
  38. zenx_lite_python-0.1.0/zenx_lite/stores/__init__.py +1 -0
  39. zenx_lite_python-0.1.0/zenx_lite/stores/redis.py +54 -0
@@ -0,0 +1,8 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+ *.egg-info/
6
+ build/
7
+ dist/
8
+ /benchmarks/
@@ -0,0 +1,27 @@
1
+ # Working on zenx-lite-python
2
+
3
+ Build a focused Python scraping runtime that is explicit, composable, and easy to operate. Existing ZenX is a reference, not a compatibility requirement.
4
+
5
+ ## Design guidance
6
+
7
+ - Read and follow [PROJECT_GUIDE.md](PROJECT_GUIDE.md) for all development.
8
+ - Use the implementation and tests to verify current runtime and API behavior. Preserve ownership, cancellation, delivery, dedup, and shutdown guarantees.
9
+ - Distinguish routine implementation choices from changes to agreed behavior. Keep code, tests, and the project guide aligned. Discussion or a question alone does not authorize a design change.
10
+
11
+ ## Implementation
12
+
13
+ - Prefer standard `asyncio`, dataclasses, protocols, argparse, and logging. Preserve native task and exception semantics.
14
+ - Use consistent type hints across framework APIs and internals. Application annotations remain optional; type hints do not impose payload schemas.
15
+ - Keep configuration and integrations explicit and optional. Avoid discovery, global registration, and implicit environment-dependent delivery behavior.
16
+ - Keep notifier I/O off crawl/delivery paths. Report crawl success only after registered tasks and session release succeed; keep Manager identity in the optional adapter. Use one sender with an unbounded queue and one attempt per notification, without retry.
17
+ - Keep process supervision and deployment orchestration outside the runtime. Add abstractions for concrete shared needs rather than speculative features.
18
+ - Consult the actual source at `~/synoptic/ZenX` when behavior depends on existing ZenX internals.
19
+
20
+ ## Verification and repository hygiene
21
+
22
+ - Validate uncertain assumptions with a standalone script or Python shell before modifying project code. Remove temporary scripts when finished.
23
+ - Prefer `uv` for dependencies, environments, and Python tooling. Run `uv run --extra config --extra source --extra redis pytest -q` for the suite and `uv build` for package verification when relevant. Set `ZENX_LITE_TEST_REDIS_URL` to a disposable Redis server to include integration tests.
24
+ - Use focused tests for ownership, cancellation, partial failures, and extension boundaries. Use local services and fake transports before live integrations.
25
+ - Support performance claims with measurements, including multiple scraper processes where relevant.
26
+ - Preserve existing user work and respect `.gitignore`. Do not add, commit, or rely on ignored files without explicit instruction. Remove unnecessary generated files. Do not create a `README.md` unless requested.
27
+ - Keep commit messages and project documentation free of AI/tool attribution. Report changes, verification, and material limitations without unsupported claims.
@@ -0,0 +1,14 @@
1
+ Metadata-Version: 2.5
2
+ Name: zenx-lite-python
3
+ Version: 0.1.0
4
+ Summary: An explicitly composed asyncio scraping runtime
5
+ Requires-Python: >=3.15
6
+ Requires-Dist: curl-cffi-patch<0.17,>=0.16.4
7
+ Requires-Dist: orjson>=3.10
8
+ Requires-Dist: parsel>=1.12.1
9
+ Provides-Extra: config
10
+ Requires-Dist: pyyaml<7,>=6; extra == 'config'
11
+ Provides-Extra: redis
12
+ Requires-Dist: redis<9,>=8.1; extra == 'redis'
13
+ Provides-Extra: source
14
+ Requires-Dist: websockets<18,>=15; extra == 'source'
@@ -0,0 +1,16 @@
1
+ # ZenX Lite design principles
2
+
3
+ Build a Python scraping runtime that is easy to understand, compose, and operate, with minimal delay from discovering new data to sending it.
4
+
5
+ - **Keep composition explicit.** Use ordinary Python objects, direct references, and small interfaces. Make dependencies and execution easy to trace.
6
+ - **Keep the core focused, but useful.** Add capabilities when concrete shared needs justify them. Smallness is a means to clarity, not a reason to force applications into workarounds.
7
+ - **Make ownership clear.** The runtime owns workers, managed sessions, registered crawl tasks, and delivery resources. Applications own business logic and their independently created resources.
8
+ - **Separate policy from execution.** Scheduling and integrations should be replaceable without changing resource ownership or delivery guarantees. Keep process supervision and deployment orchestration outside the crawling core.
9
+ - **Prioritize prompt delivery.** Send without deliberate batching, capacity waits, or artificial delays. Use unbounded queues unless explicitly requested otherwise. Preserve event-loop fairness and keep blocking work out of the admission path.
10
+ - **Make failure semantics predictable.** Use native task and cancellation behavior. Isolate destination failures; delivery remains best effort, without retrying or replaying attempted payloads. Remembering an ID does not prove delivery.
11
+ - **Keep application data independent.** Do not inject framework metadata or business rules into payloads. Keep identity, routing, and destination envelopes separate, and protect queued data from later mutation.
12
+ - **Treat operations as a real requirement.** Configuration, monitoring, and deployment support should be consistent and inspectable. Add shared support where needed while keeping integrations optional and environment behavior explicit.
13
+ - **Measure before optimizing.** Validate latency, resource use, and correctness with representative workloads, including multiple scraper processes. Simpler architecture alone does not prove better performance.
14
+ - **Evolve deliberately.** Learn from ZenX without requiring compatibility. Distinguish proposals from approved behavior and keep implementation, tests, and documentation aligned.
15
+
16
+ The implementation and tests define current behavior.
@@ -0,0 +1,30 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.26"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "zenx-lite-python"
7
+ version = "0.1.0"
8
+ description = "An explicitly composed asyncio scraping runtime"
9
+ requires-python = ">=3.15"
10
+ dependencies = [
11
+ "curl-cffi-patch>=0.16.4,<0.17",
12
+ "orjson>=3.10",
13
+ "parsel>=1.12.1",
14
+ ]
15
+
16
+ [project.optional-dependencies]
17
+ config = ["pyyaml>=6,<7"]
18
+ source = ["websockets>=15,<18"]
19
+ redis = ["redis>=8.1,<9"]
20
+
21
+ [dependency-groups]
22
+ dev = ["pytest>=8", "pytest-asyncio>=0.25"]
23
+
24
+ [tool.hatch.build.targets.wheel]
25
+ packages = ["zenx_lite"]
26
+
27
+ [tool.pytest.ini_options]
28
+ asyncio_mode = "auto"
29
+ asyncio_default_fixture_loop_scope = "function"
30
+ testpaths = ["tests"]
File without changes
@@ -0,0 +1,62 @@
1
+ import sys
2
+
3
+ from zenx_lite import App, Crawler, Spider
4
+ from zenx_lite.destinations.jsonl import JSONLines
5
+
6
+
7
+ class Feed:
8
+ name = "feed"
9
+ destinations = ("out",)
10
+
11
+ def __init__(self, url: str) -> None:
12
+ self.url = url
13
+
14
+ async def crawl(self, crawler: Crawler) -> None:
15
+ response = await crawler.request(self.url)
16
+ for item in response.json()["items"]:
17
+ await crawler.emit(str(item["id"]), item)
18
+
19
+
20
+ class Search:
21
+ name = "search"
22
+ destinations = ("out",)
23
+
24
+ def __init__(self, query: str, limit: int) -> None:
25
+ self.query = query
26
+ self.limit = limit
27
+
28
+ async def crawl(self, crawler: Crawler) -> None:
29
+ response = await crawler.request(
30
+ "https://example.com/search.json",
31
+ params={"q": self.query, "limit": self.limit},
32
+ )
33
+ for item in response.json()["items"]:
34
+ await crawler.emit(str(item["id"]), item)
35
+
36
+
37
+ if __name__ == "__main__":
38
+ parser = App.cli_parser(description="Run a feed or search spider.")
39
+ parser.add_argument("--feed-url", help="Feed URL; required for feed")
40
+ parser.add_argument("--query", help="Search query; required for search")
41
+ parser.add_argument("--limit", type=int, default=50, help="Search result limit")
42
+ args = parser.parse_args()
43
+
44
+ selected: Spider
45
+ if args.spider == "feed":
46
+ if not args.feed_url:
47
+ parser.error("--feed-url is required for feed")
48
+ selected = Feed(args.feed_url)
49
+ elif args.spider == "search":
50
+ if not args.query:
51
+ parser.error("--query is required for search")
52
+ if args.limit <= 0:
53
+ parser.error("--limit must be positive")
54
+ selected = Search(args.query, args.limit)
55
+ else:
56
+ parser.error("--spider must be feed or search")
57
+
58
+ App(
59
+ project_name="news",
60
+ spiders=[selected],
61
+ destinations={"out": JSONLines(sys.stdout)},
62
+ ).cli(args=args, parser=parser)
@@ -0,0 +1,72 @@
1
+ import asyncio
2
+ from typing import Any
3
+
4
+ import orjson
5
+
6
+ from zenx_lite.sessions import Session
7
+
8
+
9
+ class FakeTransport:
10
+ def __init__(self, **options: Any) -> None:
11
+ self.options = options
12
+ self.closed = False
13
+
14
+ async def close(self) -> None:
15
+ self.closed = True
16
+
17
+
18
+ class FakeSessions:
19
+ def __init__(self) -> None:
20
+ self.workers: dict[int, Session] = {}
21
+ self.released: list[int] = []
22
+ self.opened = False
23
+ self.closed = False
24
+ self.proxy: str | None = None
25
+
26
+ async def open(self, *, concurrency: int, proxy: str | None) -> None:
27
+ self.proxy = proxy
28
+ self.opened = True
29
+ self.workers = {i: Session(FakeTransport()) for i in range(concurrency)}
30
+
31
+ async def acquire(self, worker_id: int) -> Session:
32
+ return self.workers[worker_id]
33
+
34
+ async def release(self, worker_id: int, session: Session) -> None:
35
+ self.released.append(worker_id)
36
+
37
+ async def close(self) -> None:
38
+ self.closed = True
39
+ for session in self.workers.values():
40
+ await session.transport.close()
41
+
42
+
43
+ class RecordingDestination:
44
+ def __init__(self) -> None:
45
+ self.opened = False
46
+ self.closed = False
47
+ self.attempts: list[bytes] = []
48
+ self.sent: list[bytes] = []
49
+ self.fail_ids: set[str] = set()
50
+ self.entered = asyncio.Event()
51
+ self.gate = asyncio.Event()
52
+ self.gate.set()
53
+
54
+ async def open(self) -> None:
55
+ self.opened = True
56
+
57
+ def prepare(self, item_id: str, data: dict[str, Any]) -> bytes:
58
+ return orjson.dumps({"id": item_id, "data": data})
59
+
60
+ async def ready(self) -> None:
61
+ pass
62
+
63
+ async def send(self, payload: bytes) -> None:
64
+ self.attempts.append(payload)
65
+ self.entered.set()
66
+ await self.gate.wait()
67
+ if orjson.loads(payload)["id"] in self.fail_ids:
68
+ raise ConnectionError("Ambiguous send")
69
+ self.sent.append(payload)
70
+
71
+ async def close(self) -> None:
72
+ self.closed = True
@@ -0,0 +1,213 @@
1
+ import asyncio
2
+ import os
3
+ import signal
4
+ import subprocess
5
+ import sys
6
+ from pathlib import Path
7
+ from types import SimpleNamespace
8
+
9
+ import pytest
10
+ import orjson
11
+
12
+ from zenx_lite import App, Settings
13
+ from zenx_lite.config import cli_parser
14
+ from zenx_lite.scheduler import Slots
15
+
16
+ from .helpers import FakeSessions, RecordingDestination
17
+
18
+ PROJECT_SCRIPT = Path(__file__).resolve().parent / "fixtures" / "cli_project.py"
19
+
20
+
21
+ def app_for_cli(monkeypatch, *, settings=None, scheduler=None):
22
+ async def crawl(crawler):
23
+ pass
24
+ app = App(project_name="test", spiders=[SimpleNamespace(name="test", destinations=("out",), crawl=crawl)], destinations={"out": RecordingDestination()}, sessions=FakeSessions(), settings=settings, scheduler=scheduler)
25
+ captured = []
26
+ monkeypatch.setattr(app, "_cli_loop", lambda *args: captured.append(args))
27
+ return app, captured
28
+
29
+
30
+ def test_cli_env_python_default_precedence_and_mode(monkeypatch):
31
+ for name in list(os.environ):
32
+ if name.startswith("ZENX_LITE_"):
33
+ monkeypatch.delenv(name)
34
+ app, captured = app_for_cli(monkeypatch, settings=Settings(concurrency=3, dedup_size=20), scheduler=Slots(2))
35
+ app.cli([])
36
+ assert captured[-1][1] is False
37
+ assert captured[-1][2].concurrency == 3
38
+ assert captured[-1][2].dedup_size == 20
39
+ assert captured[-1][3].interval == 2
40
+ monkeypatch.setenv("ZENX_LITE_CONCURRENCY", "4")
41
+ monkeypatch.setenv("ZENX_LITE_DEDUP_SIZE", "30")
42
+ monkeypatch.setenv("ZENX_LITE_INTERVAL", "0.3")
43
+ monkeypatch.setenv("ZENX_LITE_PROXY", "http://env")
44
+ monkeypatch.setenv("ZENX_LITE_LOG_LEVEL", "DEBUG")
45
+ monkeypatch.setenv("ZENX_LITE_SPIDER", "test")
46
+ monkeypatch.setenv("ZENX_LITE_ONCE", "true") # Deliberately unsupported.
47
+ app.cli([])
48
+ selected, once, settings, scheduler, _ = captured[-1]
49
+ assert selected == "test" and once is False
50
+ assert settings == Settings(concurrency=4, dedup_size=30, proxy="http://env", log_level="DEBUG")
51
+ assert scheduler.interval == 0.3
52
+ app.cli(["--concurrency", "5", "--dedup-size", "40", "--interval", "0.2", "--log-level", "ERROR", "--proxy", "http://cli", "--once"])
53
+ _, once, settings, scheduler, _ = captured[-1]
54
+ assert once is True
55
+ assert settings == Settings(concurrency=5, dedup_size=40, proxy="http://cli", log_level="ERROR")
56
+ assert scheduler.interval == 0.2
57
+ assert app.settings.concurrency == 3 and app.scheduler.interval == 2
58
+
59
+
60
+ def test_valid_cli_can_override_invalid_environment():
61
+ parser = cli_parser(environ={"ZENX_LITE_CONCURRENCY": "invalid"})
62
+ assert parser.parse_args(["--concurrency", "2"]).concurrency == 2
63
+ with pytest.raises(SystemExit) as exc:
64
+ parser.parse_args([])
65
+ assert exc.value.code == 2
66
+
67
+
68
+ @pytest.mark.parametrize("argv", [["--interval", "1"], []])
69
+ def test_custom_scheduler_rejects_interval_overlay(monkeypatch, argv):
70
+ class Custom:
71
+ async def run(self, crawl, *, concurrency, stop):
72
+ pass
73
+ app, captured = app_for_cli(monkeypatch, scheduler=Custom())
74
+ if not argv:
75
+ monkeypatch.setenv("ZENX_LITE_INTERVAL", "1")
76
+ with pytest.raises(SystemExit) as exc:
77
+ app.cli(argv)
78
+ assert exc.value.code == 2 and not captured
79
+
80
+
81
+ @pytest.mark.parametrize("argv", [["--concurrency", "0"], ["--log-level", "nonsense"], ["--spider", "unknown"], ["--no-once"], ["--dry-run"], ["--env", "dev"]])
82
+ def test_cli_errors_before_execution(monkeypatch, argv):
83
+ app, captured = app_for_cli(monkeypatch)
84
+ with pytest.raises(SystemExit) as exc:
85
+ app.cli(argv)
86
+ assert exc.value.code == 2 and not captured
87
+
88
+
89
+ def test_shared_help_and_selected_spider_custom_requirements():
90
+ command = [sys.executable, str(PROJECT_SCRIPT)]
91
+ result = subprocess.run([*command, "--help"], text=True, capture_output=True, check=True)
92
+ for flag in ("--spider", "--query", "--feed-url", "--limit", "--once"):
93
+ assert flag in result.stdout
94
+ for spider, expected in (("feed", "--feed-url"), ("search", "--query")):
95
+ result = subprocess.run([*command, "--spider", spider], text=True, capture_output=True)
96
+ assert result.returncode == 2
97
+ assert expected + " is required" in result.stderr
98
+
99
+
100
+ def test_cli_one_shot_failure_exit_code_and_cleanup(tmp_path):
101
+ marker = tmp_path / "closed"
102
+ code = '''
103
+ import asyncio
104
+ from pathlib import Path
105
+ from types import SimpleNamespace
106
+ from zenx_lite import App
107
+ from tests.helpers import RecordingDestination, FakeSessions
108
+ class Destination(RecordingDestination):
109
+ async def close(self):
110
+ Path(MARKER).write_text('closed')
111
+ async def crawl(crawler):
112
+ raise ValueError('crawl failed')
113
+ App(project_name='test', spiders=[SimpleNamespace(name='test', destinations=('out',), crawl=crawl)], destinations={'out':Destination()}, sessions=FakeSessions()).cli(['--once'])
114
+ '''.replace("MARKER", repr(str(marker)))
115
+ result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=5)
116
+ assert result.returncode == 1
117
+ assert "crawl failed" in result.stderr
118
+ assert marker.read_text() == "closed"
119
+
120
+
121
+ @pytest.mark.parametrize("once", [False, True])
122
+ async def test_cli_signal_shutdown_is_bounded_in_a_real_process(once):
123
+ code = '''
124
+ import asyncio
125
+ from types import SimpleNamespace
126
+ import zenx_lite.app as runtime
127
+ from zenx_lite import App
128
+ from tests.helpers import FakeSessions, RecordingDestination
129
+ runtime.ONE_SHOT_GRACE = 0.05
130
+ class Destination(RecordingDestination):
131
+ async def close(self):
132
+ await asyncio.Event().wait()
133
+ async def crawl(crawler):
134
+ print('ready', flush=True)
135
+ await asyncio.Event().wait()
136
+ App(project_name='test', spiders=[SimpleNamespace(name='test', destinations=('out',), crawl=crawl)], destinations={'out':Destination()}, sessions=FakeSessions()).cli(ARGS)
137
+ '''.replace("ARGS", repr(["--once"] if once else []))
138
+ process = await asyncio.create_subprocess_exec(sys.executable, "-c", code, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
139
+ try:
140
+ assert await asyncio.wait_for(process.stdout.readline(), 5) == b"ready\n"
141
+ started = asyncio.get_running_loop().time()
142
+ process.send_signal(signal.SIGTERM)
143
+ _, stderr = await asyncio.wait_for(process.communicate(), 2)
144
+ assert process.returncode == 128 + signal.SIGTERM, stderr.decode()
145
+ assert asyncio.get_running_loop().time() - started < 1
146
+ assert b"Task was destroyed" not in stderr
147
+ finally:
148
+ if process.returncode is None:
149
+ process.kill()
150
+ await process.wait()
151
+
152
+
153
+ def test_core_does_not_import_optional_dependencies():
154
+ code = '''
155
+ import sys
156
+ from importlib.abc import MetaPathFinder
157
+ class BlockOptional(MetaPathFinder):
158
+ def find_spec(self, fullname, path=None, target=None):
159
+ if fullname == 'zenx_lite.notifiers.magent':
160
+ raise ImportError('unselected notifier accessed')
161
+ if fullname.split('.')[0] in {'websockets', 'grpc', 'redis', 'zenx', 'yaml'}:
162
+ raise ImportError('optional dependency accessed: ' + fullname)
163
+ sys.meta_path.insert(0, BlockOptional())
164
+ from zenx_lite import App, Crawler
165
+ from zenx_lite.destinations.jsonl import JSONLines
166
+ '''
167
+ subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True)
168
+
169
+
170
+ def test_parser_loads_only_when_xpath_is_used():
171
+ code = '''
172
+ import sys
173
+ from curl_cffi.requests import Headers
174
+ from zenx_lite import App, Response
175
+ from zenx_lite.destinations.jsonl import JSONLines
176
+ response = Response(url='https://example.test', status=200, headers=Headers(), body=b'{"ok":true}', elapsed=0.0, encoding='utf-8')
177
+ assert response.json() == {'ok': True}
178
+ assert response.text == '{"ok":true}'
179
+ assert 'parsel' not in sys.modules and 'lxml.etree' not in sys.modules
180
+ html = Response(url='https://example.test', status=200, headers=Headers(), body=b'<h1>News</h1>', elapsed=0.0, encoding='utf-8')
181
+ assert html.xpath('//h1/text()').get() == 'News'
182
+ assert 'parsel' in sys.modules and 'lxml.etree' in sys.modules
183
+ '''
184
+ subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True)
185
+
186
+
187
+ async def test_cli_http_to_jsonl_end_to_end():
188
+ body = orjson.dumps({"items": [{"id": 1, "title": "first"}, {"id": 1, "title": "duplicate"}, {"id": 2, "title": "second"}]})
189
+ served = asyncio.Event()
190
+ async def handle(reader, writer):
191
+ try:
192
+ await reader.readuntil(b"\r\n\r\n")
193
+ writer.write(f"HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + body)
194
+ await writer.drain()
195
+ finally:
196
+ writer.close()
197
+ await writer.wait_closed()
198
+ served.set()
199
+ async with await asyncio.start_server(handle, "127.0.0.1", 0) as server:
200
+ url = f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}/feed"
201
+ process = await asyncio.create_subprocess_exec(
202
+ sys.executable, str(PROJECT_SCRIPT), "--feed-url", url,
203
+ "--spider", "feed", "--once", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
204
+ )
205
+ try:
206
+ stdout, stderr = await asyncio.wait_for(process.communicate(), 5)
207
+ finally:
208
+ if process.returncode is None:
209
+ process.kill()
210
+ await process.wait()
211
+ await served.wait()
212
+ assert process.returncode == 0, stderr.decode()
213
+ assert [orjson.loads(line) for line in stdout.splitlines()] == [{"id": 1, "title": "first"}, {"id": 2, "title": "second"}]