zenx-lite-python 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenx_lite_python-0.1.0/.gitignore +8 -0
- zenx_lite_python-0.1.0/AGENTS.md +27 -0
- zenx_lite_python-0.1.0/PKG-INFO +14 -0
- zenx_lite_python-0.1.0/PROJECT_GUIDE.md +16 -0
- zenx_lite_python-0.1.0/pyproject.toml +30 -0
- zenx_lite_python-0.1.0/tests/__init__.py +0 -0
- zenx_lite_python-0.1.0/tests/fixtures/cli_project.py +62 -0
- zenx_lite_python-0.1.0/tests/helpers.py +72 -0
- zenx_lite_python-0.1.0/tests/test_cli.py +213 -0
- zenx_lite_python-0.1.0/tests/test_configuration.py +429 -0
- zenx_lite_python-0.1.0/tests/test_delivery.py +185 -0
- zenx_lite_python-0.1.0/tests/test_destinations.py +159 -0
- zenx_lite_python-0.1.0/tests/test_http.py +335 -0
- zenx_lite_python-0.1.0/tests/test_notifiers.py +298 -0
- zenx_lite_python-0.1.0/tests/test_redis_store.py +323 -0
- zenx_lite_python-0.1.0/tests/test_runtime.py +361 -0
- zenx_lite_python-0.1.0/tests/test_scheduler.py +61 -0
- zenx_lite_python-0.1.0/uv.lock +352 -0
- zenx_lite_python-0.1.0/zenx_lite/__init__.py +8 -0
- zenx_lite_python-0.1.0/zenx_lite/app.py +377 -0
- zenx_lite_python-0.1.0/zenx_lite/config.py +56 -0
- zenx_lite_python-0.1.0/zenx_lite/configuration.py +359 -0
- zenx_lite_python-0.1.0/zenx_lite/crawler.py +69 -0
- zenx_lite_python-0.1.0/zenx_lite/delivery.py +139 -0
- zenx_lite_python-0.1.0/zenx_lite/destinations/__init__.py +5 -0
- zenx_lite_python-0.1.0/zenx_lite/destinations/base.py +32 -0
- zenx_lite_python-0.1.0/zenx_lite/destinations/jsonl.py +132 -0
- zenx_lite_python-0.1.0/zenx_lite/destinations/synoptic/__init__.py +5 -0
- zenx_lite_python-0.1.0/zenx_lite/destinations/synoptic/source.py +136 -0
- zenx_lite_python-0.1.0/zenx_lite/errors.py +21 -0
- zenx_lite_python-0.1.0/zenx_lite/http.py +130 -0
- zenx_lite_python-0.1.0/zenx_lite/notifiers/__init__.py +5 -0
- zenx_lite_python-0.1.0/zenx_lite/notifiers/base.py +15 -0
- zenx_lite_python-0.1.0/zenx_lite/notifiers/magent.py +86 -0
- zenx_lite_python-0.1.0/zenx_lite/scheduler.py +63 -0
- zenx_lite_python-0.1.0/zenx_lite/sessions.py +102 -0
- zenx_lite_python-0.1.0/zenx_lite/store.py +45 -0
- zenx_lite_python-0.1.0/zenx_lite/stores/__init__.py +1 -0
- zenx_lite_python-0.1.0/zenx_lite/stores/redis.py +54 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Working on zenx-lite-python
|
|
2
|
+
|
|
3
|
+
Build a focused Python scraping runtime that is explicit, composable, and easy to operate. Existing ZenX is a reference, not a compatibility requirement.
|
|
4
|
+
|
|
5
|
+
## Design guidance
|
|
6
|
+
|
|
7
|
+
- Read and follow [PROJECT_GUIDE.md](PROJECT_GUIDE.md) for all development.
|
|
8
|
+
- Use the implementation and tests to verify current runtime and API behavior. Preserve ownership, cancellation, delivery, dedup, and shutdown guarantees.
|
|
9
|
+
- Distinguish routine implementation choices from changes to agreed behavior. Keep code, tests, and the project guide aligned. Discussion or a question alone does not authorize a design change.
|
|
10
|
+
|
|
11
|
+
## Implementation
|
|
12
|
+
|
|
13
|
+
- Prefer standard `asyncio`, dataclasses, protocols, argparse, and logging. Preserve native task and exception semantics.
|
|
14
|
+
- Use consistent type hints across framework APIs and internals. Application annotations remain optional; type hints do not impose payload schemas.
|
|
15
|
+
- Keep configuration and integrations explicit and optional. Avoid discovery, global registration, and implicit environment-dependent delivery behavior.
|
|
16
|
+
- Keep notifier I/O off crawl/delivery paths. Report crawl success only after registered tasks and session release succeed; keep Manager identity in the optional adapter. Use one sender with an unbounded queue and one attempt per notification, without retry.
|
|
17
|
+
- Keep process supervision and deployment orchestration outside the runtime. Add abstractions for concrete shared needs rather than speculative features.
|
|
18
|
+
- Consult the actual source at `~/synoptic/ZenX` when behavior depends on existing ZenX internals.
|
|
19
|
+
|
|
20
|
+
## Verification and repository hygiene
|
|
21
|
+
|
|
22
|
+
- Validate uncertain assumptions with a standalone script or Python shell before modifying project code. Remove temporary scripts when finished.
|
|
23
|
+
- Prefer `uv` for dependencies, environments, and Python tooling. Run `uv run --extra config --extra source --extra redis pytest -q` for the suite and `uv build` for package verification when relevant. Set `ZENX_LITE_TEST_REDIS_URL` to a disposable Redis server to include integration tests.
|
|
24
|
+
- Use focused tests for ownership, cancellation, partial failures, and extension boundaries. Use local services and fake transports before live integrations.
|
|
25
|
+
- Support performance claims with measurements, including multiple scraper processes where relevant.
|
|
26
|
+
- Preserve existing user work and respect `.gitignore`. Do not add, commit, or rely on ignored files without explicit instruction. Remove unnecessary generated files. Do not create a `README.md` unless requested.
|
|
27
|
+
- Keep commit messages and project documentation free of AI/tool attribution. Report changes, verification, and material limitations without unsupported claims.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: zenx-lite-python
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An explicitly composed asyncio scraping runtime
|
|
5
|
+
Requires-Python: >=3.15
|
|
6
|
+
Requires-Dist: curl-cffi-patch<0.17,>=0.16.4
|
|
7
|
+
Requires-Dist: orjson>=3.10
|
|
8
|
+
Requires-Dist: parsel>=1.12.1
|
|
9
|
+
Provides-Extra: config
|
|
10
|
+
Requires-Dist: pyyaml<7,>=6; extra == 'config'
|
|
11
|
+
Provides-Extra: redis
|
|
12
|
+
Requires-Dist: redis<9,>=8.1; extra == 'redis'
|
|
13
|
+
Provides-Extra: source
|
|
14
|
+
Requires-Dist: websockets<18,>=15; extra == 'source'
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# ZenX Lite design principles
|
|
2
|
+
|
|
3
|
+
Build a Python scraping runtime that is easy to understand, compose, and operate, with minimal delay from discovering new data to sending it.
|
|
4
|
+
|
|
5
|
+
- **Keep composition explicit.** Use ordinary Python objects, direct references, and small interfaces. Make dependencies and execution easy to trace.
|
|
6
|
+
- **Keep the core focused, but useful.** Add capabilities when concrete shared needs justify them. Smallness is a means to clarity, not a reason to force applications into workarounds.
|
|
7
|
+
- **Make ownership clear.** The runtime owns workers, managed sessions, registered crawl tasks, and delivery resources. Applications own business logic and their independently created resources.
|
|
8
|
+
- **Separate policy from execution.** Scheduling and integrations should be replaceable without changing resource ownership or delivery guarantees. Keep process supervision and deployment orchestration outside the crawling core.
|
|
9
|
+
- **Prioritize prompt delivery.** Send without deliberate batching, capacity waits, or artificial delays. Use unbounded queues unless explicitly requested otherwise. Preserve event-loop fairness and keep blocking work out of the admission path.
|
|
10
|
+
- **Make failure semantics predictable.** Use native task and cancellation behavior. Isolate destination failures; delivery remains best effort, without retrying or replaying attempted payloads. Remembering an ID does not prove delivery.
|
|
11
|
+
- **Keep application data independent.** Do not inject framework metadata or business rules into payloads. Keep identity, routing, and destination envelopes separate, and protect queued data from later mutation.
|
|
12
|
+
- **Treat operations as a real requirement.** Configuration, monitoring, and deployment support should be consistent and inspectable. Add shared support where needed while keeping integrations optional and environment behavior explicit.
|
|
13
|
+
- **Measure before optimizing.** Validate latency, resource use, and correctness with representative workloads, including multiple scraper processes. Simpler architecture alone does not prove better performance.
|
|
14
|
+
- **Evolve deliberately.** Learn from ZenX without requiring compatibility. Distinguish proposals from approved behavior and keep implementation, tests, and documentation aligned.
|
|
15
|
+
|
|
16
|
+
The implementation and tests define current behavior.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.26"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "zenx-lite-python"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "An explicitly composed asyncio scraping runtime"
|
|
9
|
+
requires-python = ">=3.15"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"curl-cffi-patch>=0.16.4,<0.17",
|
|
12
|
+
"orjson>=3.10",
|
|
13
|
+
"parsel>=1.12.1",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
config = ["pyyaml>=6,<7"]
|
|
18
|
+
source = ["websockets>=15,<18"]
|
|
19
|
+
redis = ["redis>=8.1,<9"]
|
|
20
|
+
|
|
21
|
+
[dependency-groups]
|
|
22
|
+
dev = ["pytest>=8", "pytest-asyncio>=0.25"]
|
|
23
|
+
|
|
24
|
+
[tool.hatch.build.targets.wheel]
|
|
25
|
+
packages = ["zenx_lite"]
|
|
26
|
+
|
|
27
|
+
[tool.pytest.ini_options]
|
|
28
|
+
asyncio_mode = "auto"
|
|
29
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
30
|
+
testpaths = ["tests"]
|
|
File without changes
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from zenx_lite import App, Crawler, Spider
|
|
4
|
+
from zenx_lite.destinations.jsonl import JSONLines
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Feed:
|
|
8
|
+
name = "feed"
|
|
9
|
+
destinations = ("out",)
|
|
10
|
+
|
|
11
|
+
def __init__(self, url: str) -> None:
|
|
12
|
+
self.url = url
|
|
13
|
+
|
|
14
|
+
async def crawl(self, crawler: Crawler) -> None:
|
|
15
|
+
response = await crawler.request(self.url)
|
|
16
|
+
for item in response.json()["items"]:
|
|
17
|
+
await crawler.emit(str(item["id"]), item)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Search:
|
|
21
|
+
name = "search"
|
|
22
|
+
destinations = ("out",)
|
|
23
|
+
|
|
24
|
+
def __init__(self, query: str, limit: int) -> None:
|
|
25
|
+
self.query = query
|
|
26
|
+
self.limit = limit
|
|
27
|
+
|
|
28
|
+
async def crawl(self, crawler: Crawler) -> None:
|
|
29
|
+
response = await crawler.request(
|
|
30
|
+
"https://example.com/search.json",
|
|
31
|
+
params={"q": self.query, "limit": self.limit},
|
|
32
|
+
)
|
|
33
|
+
for item in response.json()["items"]:
|
|
34
|
+
await crawler.emit(str(item["id"]), item)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
if __name__ == "__main__":
|
|
38
|
+
parser = App.cli_parser(description="Run a feed or search spider.")
|
|
39
|
+
parser.add_argument("--feed-url", help="Feed URL; required for feed")
|
|
40
|
+
parser.add_argument("--query", help="Search query; required for search")
|
|
41
|
+
parser.add_argument("--limit", type=int, default=50, help="Search result limit")
|
|
42
|
+
args = parser.parse_args()
|
|
43
|
+
|
|
44
|
+
selected: Spider
|
|
45
|
+
if args.spider == "feed":
|
|
46
|
+
if not args.feed_url:
|
|
47
|
+
parser.error("--feed-url is required for feed")
|
|
48
|
+
selected = Feed(args.feed_url)
|
|
49
|
+
elif args.spider == "search":
|
|
50
|
+
if not args.query:
|
|
51
|
+
parser.error("--query is required for search")
|
|
52
|
+
if args.limit <= 0:
|
|
53
|
+
parser.error("--limit must be positive")
|
|
54
|
+
selected = Search(args.query, args.limit)
|
|
55
|
+
else:
|
|
56
|
+
parser.error("--spider must be feed or search")
|
|
57
|
+
|
|
58
|
+
App(
|
|
59
|
+
project_name="news",
|
|
60
|
+
spiders=[selected],
|
|
61
|
+
destinations={"out": JSONLines(sys.stdout)},
|
|
62
|
+
).cli(args=args, parser=parser)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import orjson
|
|
5
|
+
|
|
6
|
+
from zenx_lite.sessions import Session
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class FakeTransport:
|
|
10
|
+
def __init__(self, **options: Any) -> None:
|
|
11
|
+
self.options = options
|
|
12
|
+
self.closed = False
|
|
13
|
+
|
|
14
|
+
async def close(self) -> None:
|
|
15
|
+
self.closed = True
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FakeSessions:
|
|
19
|
+
def __init__(self) -> None:
|
|
20
|
+
self.workers: dict[int, Session] = {}
|
|
21
|
+
self.released: list[int] = []
|
|
22
|
+
self.opened = False
|
|
23
|
+
self.closed = False
|
|
24
|
+
self.proxy: str | None = None
|
|
25
|
+
|
|
26
|
+
async def open(self, *, concurrency: int, proxy: str | None) -> None:
|
|
27
|
+
self.proxy = proxy
|
|
28
|
+
self.opened = True
|
|
29
|
+
self.workers = {i: Session(FakeTransport()) for i in range(concurrency)}
|
|
30
|
+
|
|
31
|
+
async def acquire(self, worker_id: int) -> Session:
|
|
32
|
+
return self.workers[worker_id]
|
|
33
|
+
|
|
34
|
+
async def release(self, worker_id: int, session: Session) -> None:
|
|
35
|
+
self.released.append(worker_id)
|
|
36
|
+
|
|
37
|
+
async def close(self) -> None:
|
|
38
|
+
self.closed = True
|
|
39
|
+
for session in self.workers.values():
|
|
40
|
+
await session.transport.close()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class RecordingDestination:
|
|
44
|
+
def __init__(self) -> None:
|
|
45
|
+
self.opened = False
|
|
46
|
+
self.closed = False
|
|
47
|
+
self.attempts: list[bytes] = []
|
|
48
|
+
self.sent: list[bytes] = []
|
|
49
|
+
self.fail_ids: set[str] = set()
|
|
50
|
+
self.entered = asyncio.Event()
|
|
51
|
+
self.gate = asyncio.Event()
|
|
52
|
+
self.gate.set()
|
|
53
|
+
|
|
54
|
+
async def open(self) -> None:
|
|
55
|
+
self.opened = True
|
|
56
|
+
|
|
57
|
+
def prepare(self, item_id: str, data: dict[str, Any]) -> bytes:
|
|
58
|
+
return orjson.dumps({"id": item_id, "data": data})
|
|
59
|
+
|
|
60
|
+
async def ready(self) -> None:
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
async def send(self, payload: bytes) -> None:
|
|
64
|
+
self.attempts.append(payload)
|
|
65
|
+
self.entered.set()
|
|
66
|
+
await self.gate.wait()
|
|
67
|
+
if orjson.loads(payload)["id"] in self.fail_ids:
|
|
68
|
+
raise ConnectionError("Ambiguous send")
|
|
69
|
+
self.sent.append(payload)
|
|
70
|
+
|
|
71
|
+
async def close(self) -> None:
|
|
72
|
+
self.closed = True
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import os
|
|
3
|
+
import signal
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from types import SimpleNamespace
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
import orjson
|
|
11
|
+
|
|
12
|
+
from zenx_lite import App, Settings
|
|
13
|
+
from zenx_lite.config import cli_parser
|
|
14
|
+
from zenx_lite.scheduler import Slots
|
|
15
|
+
|
|
16
|
+
from .helpers import FakeSessions, RecordingDestination
|
|
17
|
+
|
|
18
|
+
PROJECT_SCRIPT = Path(__file__).resolve().parent / "fixtures" / "cli_project.py"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def app_for_cli(monkeypatch, *, settings=None, scheduler=None):
|
|
22
|
+
async def crawl(crawler):
|
|
23
|
+
pass
|
|
24
|
+
app = App(project_name="test", spiders=[SimpleNamespace(name="test", destinations=("out",), crawl=crawl)], destinations={"out": RecordingDestination()}, sessions=FakeSessions(), settings=settings, scheduler=scheduler)
|
|
25
|
+
captured = []
|
|
26
|
+
monkeypatch.setattr(app, "_cli_loop", lambda *args: captured.append(args))
|
|
27
|
+
return app, captured
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_cli_env_python_default_precedence_and_mode(monkeypatch):
|
|
31
|
+
for name in list(os.environ):
|
|
32
|
+
if name.startswith("ZENX_LITE_"):
|
|
33
|
+
monkeypatch.delenv(name)
|
|
34
|
+
app, captured = app_for_cli(monkeypatch, settings=Settings(concurrency=3, dedup_size=20), scheduler=Slots(2))
|
|
35
|
+
app.cli([])
|
|
36
|
+
assert captured[-1][1] is False
|
|
37
|
+
assert captured[-1][2].concurrency == 3
|
|
38
|
+
assert captured[-1][2].dedup_size == 20
|
|
39
|
+
assert captured[-1][3].interval == 2
|
|
40
|
+
monkeypatch.setenv("ZENX_LITE_CONCURRENCY", "4")
|
|
41
|
+
monkeypatch.setenv("ZENX_LITE_DEDUP_SIZE", "30")
|
|
42
|
+
monkeypatch.setenv("ZENX_LITE_INTERVAL", "0.3")
|
|
43
|
+
monkeypatch.setenv("ZENX_LITE_PROXY", "http://env")
|
|
44
|
+
monkeypatch.setenv("ZENX_LITE_LOG_LEVEL", "DEBUG")
|
|
45
|
+
monkeypatch.setenv("ZENX_LITE_SPIDER", "test")
|
|
46
|
+
monkeypatch.setenv("ZENX_LITE_ONCE", "true") # Deliberately unsupported.
|
|
47
|
+
app.cli([])
|
|
48
|
+
selected, once, settings, scheduler, _ = captured[-1]
|
|
49
|
+
assert selected == "test" and once is False
|
|
50
|
+
assert settings == Settings(concurrency=4, dedup_size=30, proxy="http://env", log_level="DEBUG")
|
|
51
|
+
assert scheduler.interval == 0.3
|
|
52
|
+
app.cli(["--concurrency", "5", "--dedup-size", "40", "--interval", "0.2", "--log-level", "ERROR", "--proxy", "http://cli", "--once"])
|
|
53
|
+
_, once, settings, scheduler, _ = captured[-1]
|
|
54
|
+
assert once is True
|
|
55
|
+
assert settings == Settings(concurrency=5, dedup_size=40, proxy="http://cli", log_level="ERROR")
|
|
56
|
+
assert scheduler.interval == 0.2
|
|
57
|
+
assert app.settings.concurrency == 3 and app.scheduler.interval == 2
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_valid_cli_can_override_invalid_environment():
|
|
61
|
+
parser = cli_parser(environ={"ZENX_LITE_CONCURRENCY": "invalid"})
|
|
62
|
+
assert parser.parse_args(["--concurrency", "2"]).concurrency == 2
|
|
63
|
+
with pytest.raises(SystemExit) as exc:
|
|
64
|
+
parser.parse_args([])
|
|
65
|
+
assert exc.value.code == 2
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@pytest.mark.parametrize("argv", [["--interval", "1"], []])
|
|
69
|
+
def test_custom_scheduler_rejects_interval_overlay(monkeypatch, argv):
|
|
70
|
+
class Custom:
|
|
71
|
+
async def run(self, crawl, *, concurrency, stop):
|
|
72
|
+
pass
|
|
73
|
+
app, captured = app_for_cli(monkeypatch, scheduler=Custom())
|
|
74
|
+
if not argv:
|
|
75
|
+
monkeypatch.setenv("ZENX_LITE_INTERVAL", "1")
|
|
76
|
+
with pytest.raises(SystemExit) as exc:
|
|
77
|
+
app.cli(argv)
|
|
78
|
+
assert exc.value.code == 2 and not captured
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@pytest.mark.parametrize("argv", [["--concurrency", "0"], ["--log-level", "nonsense"], ["--spider", "unknown"], ["--no-once"], ["--dry-run"], ["--env", "dev"]])
|
|
82
|
+
def test_cli_errors_before_execution(monkeypatch, argv):
|
|
83
|
+
app, captured = app_for_cli(monkeypatch)
|
|
84
|
+
with pytest.raises(SystemExit) as exc:
|
|
85
|
+
app.cli(argv)
|
|
86
|
+
assert exc.value.code == 2 and not captured
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_shared_help_and_selected_spider_custom_requirements():
|
|
90
|
+
command = [sys.executable, str(PROJECT_SCRIPT)]
|
|
91
|
+
result = subprocess.run([*command, "--help"], text=True, capture_output=True, check=True)
|
|
92
|
+
for flag in ("--spider", "--query", "--feed-url", "--limit", "--once"):
|
|
93
|
+
assert flag in result.stdout
|
|
94
|
+
for spider, expected in (("feed", "--feed-url"), ("search", "--query")):
|
|
95
|
+
result = subprocess.run([*command, "--spider", spider], text=True, capture_output=True)
|
|
96
|
+
assert result.returncode == 2
|
|
97
|
+
assert expected + " is required" in result.stderr
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_cli_one_shot_failure_exit_code_and_cleanup(tmp_path):
|
|
101
|
+
marker = tmp_path / "closed"
|
|
102
|
+
code = '''
|
|
103
|
+
import asyncio
|
|
104
|
+
from pathlib import Path
|
|
105
|
+
from types import SimpleNamespace
|
|
106
|
+
from zenx_lite import App
|
|
107
|
+
from tests.helpers import RecordingDestination, FakeSessions
|
|
108
|
+
class Destination(RecordingDestination):
|
|
109
|
+
async def close(self):
|
|
110
|
+
Path(MARKER).write_text('closed')
|
|
111
|
+
async def crawl(crawler):
|
|
112
|
+
raise ValueError('crawl failed')
|
|
113
|
+
App(project_name='test', spiders=[SimpleNamespace(name='test', destinations=('out',), crawl=crawl)], destinations={'out':Destination()}, sessions=FakeSessions()).cli(['--once'])
|
|
114
|
+
'''.replace("MARKER", repr(str(marker)))
|
|
115
|
+
result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=5)
|
|
116
|
+
assert result.returncode == 1
|
|
117
|
+
assert "crawl failed" in result.stderr
|
|
118
|
+
assert marker.read_text() == "closed"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@pytest.mark.parametrize("once", [False, True])
|
|
122
|
+
async def test_cli_signal_shutdown_is_bounded_in_a_real_process(once):
|
|
123
|
+
code = '''
|
|
124
|
+
import asyncio
|
|
125
|
+
from types import SimpleNamespace
|
|
126
|
+
import zenx_lite.app as runtime
|
|
127
|
+
from zenx_lite import App
|
|
128
|
+
from tests.helpers import FakeSessions, RecordingDestination
|
|
129
|
+
runtime.ONE_SHOT_GRACE = 0.05
|
|
130
|
+
class Destination(RecordingDestination):
|
|
131
|
+
async def close(self):
|
|
132
|
+
await asyncio.Event().wait()
|
|
133
|
+
async def crawl(crawler):
|
|
134
|
+
print('ready', flush=True)
|
|
135
|
+
await asyncio.Event().wait()
|
|
136
|
+
App(project_name='test', spiders=[SimpleNamespace(name='test', destinations=('out',), crawl=crawl)], destinations={'out':Destination()}, sessions=FakeSessions()).cli(ARGS)
|
|
137
|
+
'''.replace("ARGS", repr(["--once"] if once else []))
|
|
138
|
+
process = await asyncio.create_subprocess_exec(sys.executable, "-c", code, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
|
139
|
+
try:
|
|
140
|
+
assert await asyncio.wait_for(process.stdout.readline(), 5) == b"ready\n"
|
|
141
|
+
started = asyncio.get_running_loop().time()
|
|
142
|
+
process.send_signal(signal.SIGTERM)
|
|
143
|
+
_, stderr = await asyncio.wait_for(process.communicate(), 2)
|
|
144
|
+
assert process.returncode == 128 + signal.SIGTERM, stderr.decode()
|
|
145
|
+
assert asyncio.get_running_loop().time() - started < 1
|
|
146
|
+
assert b"Task was destroyed" not in stderr
|
|
147
|
+
finally:
|
|
148
|
+
if process.returncode is None:
|
|
149
|
+
process.kill()
|
|
150
|
+
await process.wait()
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def test_core_does_not_import_optional_dependencies():
|
|
154
|
+
code = '''
|
|
155
|
+
import sys
|
|
156
|
+
from importlib.abc import MetaPathFinder
|
|
157
|
+
class BlockOptional(MetaPathFinder):
|
|
158
|
+
def find_spec(self, fullname, path=None, target=None):
|
|
159
|
+
if fullname == 'zenx_lite.notifiers.magent':
|
|
160
|
+
raise ImportError('unselected notifier accessed')
|
|
161
|
+
if fullname.split('.')[0] in {'websockets', 'grpc', 'redis', 'zenx', 'yaml'}:
|
|
162
|
+
raise ImportError('optional dependency accessed: ' + fullname)
|
|
163
|
+
sys.meta_path.insert(0, BlockOptional())
|
|
164
|
+
from zenx_lite import App, Crawler
|
|
165
|
+
from zenx_lite.destinations.jsonl import JSONLines
|
|
166
|
+
'''
|
|
167
|
+
subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_parser_loads_only_when_xpath_is_used():
|
|
171
|
+
code = '''
|
|
172
|
+
import sys
|
|
173
|
+
from curl_cffi.requests import Headers
|
|
174
|
+
from zenx_lite import App, Response
|
|
175
|
+
from zenx_lite.destinations.jsonl import JSONLines
|
|
176
|
+
response = Response(url='https://example.test', status=200, headers=Headers(), body=b'{"ok":true}', elapsed=0.0, encoding='utf-8')
|
|
177
|
+
assert response.json() == {'ok': True}
|
|
178
|
+
assert response.text == '{"ok":true}'
|
|
179
|
+
assert 'parsel' not in sys.modules and 'lxml.etree' not in sys.modules
|
|
180
|
+
html = Response(url='https://example.test', status=200, headers=Headers(), body=b'<h1>News</h1>', elapsed=0.0, encoding='utf-8')
|
|
181
|
+
assert html.xpath('//h1/text()').get() == 'News'
|
|
182
|
+
assert 'parsel' in sys.modules and 'lxml.etree' in sys.modules
|
|
183
|
+
'''
|
|
184
|
+
subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
async def test_cli_http_to_jsonl_end_to_end():
|
|
188
|
+
body = orjson.dumps({"items": [{"id": 1, "title": "first"}, {"id": 1, "title": "duplicate"}, {"id": 2, "title": "second"}]})
|
|
189
|
+
served = asyncio.Event()
|
|
190
|
+
async def handle(reader, writer):
|
|
191
|
+
try:
|
|
192
|
+
await reader.readuntil(b"\r\n\r\n")
|
|
193
|
+
writer.write(f"HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + body)
|
|
194
|
+
await writer.drain()
|
|
195
|
+
finally:
|
|
196
|
+
writer.close()
|
|
197
|
+
await writer.wait_closed()
|
|
198
|
+
served.set()
|
|
199
|
+
async with await asyncio.start_server(handle, "127.0.0.1", 0) as server:
|
|
200
|
+
url = f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}/feed"
|
|
201
|
+
process = await asyncio.create_subprocess_exec(
|
|
202
|
+
sys.executable, str(PROJECT_SCRIPT), "--feed-url", url,
|
|
203
|
+
"--spider", "feed", "--once", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
|
|
204
|
+
)
|
|
205
|
+
try:
|
|
206
|
+
stdout, stderr = await asyncio.wait_for(process.communicate(), 5)
|
|
207
|
+
finally:
|
|
208
|
+
if process.returncode is None:
|
|
209
|
+
process.kill()
|
|
210
|
+
await process.wait()
|
|
211
|
+
await served.wait()
|
|
212
|
+
assert process.returncode == 0, stderr.decode()
|
|
213
|
+
assert [orjson.loads(line) for line in stdout.splitlines()] == [{"id": 1, "title": "first"}, {"id": 2, "title": "second"}]
|