zenx-lite-python 0.1.0__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. zenx_lite_python-0.1.1/.github/workflows/release.yml +62 -0
  2. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/PKG-INFO +1 -1
  3. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/PROJECT_GUIDE.md +1 -1
  4. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/pyproject.toml +1 -1
  5. zenx_lite_python-0.1.1/tests/test_optic_manager.py +151 -0
  6. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/uv.lock +2 -2
  7. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/configuration.py +9 -1
  8. zenx_lite_python-0.1.1/zenx_lite/destinations/optic_manager.py +72 -0
  9. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/.gitignore +0 -0
  10. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/AGENTS.md +0 -0
  11. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/__init__.py +0 -0
  12. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/fixtures/cli_project.py +0 -0
  13. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/helpers.py +0 -0
  14. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_cli.py +0 -0
  15. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_configuration.py +0 -0
  16. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_delivery.py +0 -0
  17. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_destinations.py +0 -0
  18. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_http.py +0 -0
  19. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_notifiers.py +0 -0
  20. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_redis_store.py +0 -0
  21. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_runtime.py +0 -0
  22. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_scheduler.py +0 -0
  23. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/__init__.py +0 -0
  24. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/app.py +0 -0
  25. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/config.py +0 -0
  26. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/crawler.py +0 -0
  27. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/delivery.py +0 -0
  28. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/__init__.py +0 -0
  29. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/base.py +0 -0
  30. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/jsonl.py +0 -0
  31. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/synoptic/__init__.py +0 -0
  32. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/synoptic/source.py +0 -0
  33. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/errors.py +0 -0
  34. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/http.py +0 -0
  35. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/notifiers/__init__.py +0 -0
  36. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/notifiers/base.py +0 -0
  37. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/notifiers/magent.py +0 -0
  38. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/scheduler.py +0 -0
  39. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/sessions.py +0 -0
  40. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/store.py +0 -0
  41. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/stores/__init__.py +0 -0
  42. {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/stores/redis.py +0 -0
@@ -0,0 +1,62 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ publish:
12
+ runs-on: ubuntu-latest
13
+ services:
14
+ redis:
15
+ image: redis:8
16
+ ports:
17
+ - 6379:6379
18
+ options: >-
19
+ --health-cmd "redis-cli ping"
20
+ --health-interval 10s
21
+ --health-timeout 5s
22
+ --health-retries 5
23
+ env:
24
+ PYTHONDONTWRITEBYTECODE: "1"
25
+ ZENX_LITE_TEST_REDIS_URL: redis://localhost:6379/0
26
+ steps:
27
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
28
+ with:
29
+ persist-credentials: false
30
+
31
+ - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
32
+ with:
33
+ version: "0.9.15"
34
+ python-version: "3.15"
35
+
36
+ - name: Check release version
37
+ env:
38
+ RELEASE_TAG: ${{ github.event.release.tag_name }}
39
+ run: |
40
+ uv run --no-project python - <<'PY'
41
+ import os
42
+ import tomllib
43
+
44
+ with open("pyproject.toml", "rb") as metadata:
45
+ version = tomllib.load(metadata)["project"]["version"]
46
+ tag = os.environ["RELEASE_TAG"].removeprefix("v")
47
+ if tag != version:
48
+ raise SystemExit(f"Release tag {tag!r} does not match package version {version!r}")
49
+ PY
50
+
51
+ - name: Run tests
52
+ env:
53
+ UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/venv
54
+ run: uv run --locked --all-extras pytest -q -p no:cacheprovider
55
+
56
+ - name: Build distributions
57
+ run: uv build --out-dir "$RUNNER_TEMP/dist"
58
+
59
+ - name: Publish distributions
60
+ env:
61
+ UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
62
+ run: uv publish --trusted-publishing never "$RUNNER_TEMP"/dist/*
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: zenx-lite-python
3
- Version: 0.1.0
3
+ Version: 0.1.1
4
4
  Summary: An explicitly composed asyncio scraping runtime
5
5
  Requires-Python: >=3.15
6
6
  Requires-Dist: curl-cffi-patch<0.17,>=0.16.4
@@ -7,7 +7,7 @@ Build a Python scraping runtime that is easy to understand, compose, and operate
7
7
  - **Make ownership clear.** The runtime owns workers, managed sessions, registered crawl tasks, and delivery resources. Applications own business logic and their independently created resources.
8
8
  - **Separate policy from execution.** Scheduling and integrations should be replaceable without changing resource ownership or delivery guarantees. Keep process supervision and deployment orchestration outside the crawling core.
9
9
  - **Prioritize prompt delivery.** Send without deliberate batching, capacity waits, or artificial delays. Use unbounded queues unless explicitly requested otherwise. Preserve event-loop fairness and keep blocking work out of the admission path.
10
- - **Make failure semantics predictable.** Use native task and cancellation behavior. Isolate destination failures; delivery remains best effort, without retrying or replaying attempted payloads. Remembering an ID does not prove delivery.
10
+ - **Make failure semantics predictable.** Use native task and cancellation behavior. Isolate destination failures; delivery remains best effort, without retrying or replaying attempted payloads. Transport-buffered data may be discarded on close; do not add flush waits unless explicitly requested. Remembering an ID does not prove delivery.
11
11
  - **Keep application data independent.** Do not inject framework metadata or business rules into payloads. Keep identity, routing, and destination envelopes separate, and protect queued data from later mutation.
12
12
  - **Treat operations as a real requirement.** Configuration, monitoring, and deployment support should be consistent and inspectable. Add shared support where needed while keeping integrations optional and environment behavior explicit.
13
13
  - **Measure before optimizing.** Validate latency, resource use, and correctness with representative workloads, including multiple scraper processes. Simpler architecture alone does not prove better performance.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "zenx-lite-python"
7
- version = "0.1.0"
7
+ version = "0.1.1"
8
8
  description = "An explicitly composed asyncio scraping runtime"
9
9
  requires-python = ">=3.15"
10
10
  dependencies = [
@@ -0,0 +1,151 @@
1
+ import asyncio
2
+ import json
3
+ from contextlib import asynccontextmanager
4
+ from types import SimpleNamespace
5
+ from unittest.mock import AsyncMock
6
+
7
+ import pytest
8
+
9
+ from zenx_lite import App
10
+ from zenx_lite.configuration import ProjectConfig, build_app
11
+ from zenx_lite.destinations.optic_manager import OpticManagerDestination
12
+ from zenx_lite.errors import ConfigurationError
13
+
14
+ from .helpers import FakeSessions, RecordingDestination
15
+
16
+
17
+ @asynccontextmanager
18
+ async def collector(path):
19
+ state = SimpleNamespace(frames=[], connections=0)
20
+ tasks = set()
21
+
22
+ async def receive(reader, writer):
23
+ tasks.add(asyncio.current_task())
24
+ state.connections += 1
25
+ try:
26
+ while line := await reader.readline():
27
+ state.frames.append(json.loads(line))
28
+ finally:
29
+ writer.transport.abort()
30
+ tasks.discard(asyncio.current_task())
31
+
32
+ server = await asyncio.start_unix_server(receive, path)
33
+ try:
34
+ yield state
35
+ finally:
36
+ server.close()
37
+ server.abort_clients()
38
+ await server.wait_closed()
39
+ if tasks:
40
+ await asyncio.gather(*tuple(tasks), return_exceptions=True)
41
+
42
+
43
+ @pytest.mark.parametrize("scraped_at", [None, 0, 1794900000000])
44
+ def test_prepare_transforms_without_mutation_and_captures_timestamp(monkeypatch, scraped_at):
45
+ monkeypatch.setattr("zenx_lite.destinations.optic_manager.time.time_ns", lambda: 1800000000123000000)
46
+ item = {"link": "https://example.test/article", "title": "News", "scraped_at": scraped_at, "body": "application data"}
47
+ destination = OpticManagerDestination("app-1", transform=lambda item: {"url": item["link"], "headline": item["title"], "scraped_at": item["scraped_at"]})
48
+ payload = destination.prepare("separate-id", item)
49
+ assert payload.endswith(b"\n") and len(payload.splitlines()) == 1
50
+ assert json.loads(payload) == {
51
+ "type": "post", "app_id": "app-1",
52
+ "post": {"url": item["link"], "headline": "News", "published_at": None, "scraped_at": 1800000000123 if scraped_at is None else scraped_at},
53
+ }
54
+ assert item == {"link": "https://example.test/article", "title": "News", "scraped_at": scraped_at, "body": "application data"}
55
+ with pytest.raises(ValueError, match="url"):
56
+ OpticManagerDestination("app-1").prepare("bad", {"headline": "Missing URL"})
57
+
58
+
59
+ @pytest.mark.parametrize("instance", [None, "default"])
60
+ async def test_configured_destination_posts_dedupes_reuses_connection_and_reopens(tmp_path, instance):
61
+ async def crawl(crawler):
62
+ await crawler.emit("one", {"url": "https://example.test/one", "headline": "News", "published_at": 1794899999000, "scraped_at": 1794900000000})
63
+ await crawler.emit("one", {"url": "https://example.test/duplicate"})
64
+ await crawler.emit("two", {"url": "https://example.test/two"})
65
+
66
+ config = ProjectConfig({
67
+ "schema_version": 1, "project_name": "test",
68
+ "destinations": {"manager": {"type": "optic_manager", "app_id": "app-1", "instance": instance}},
69
+ "spiders": {"feed": {"destinations": ["manager"]}},
70
+ })
71
+ app = build_app(ProjectConfig(config.resolve()), spiders={"feed": lambda: SimpleNamespace(name="feed", destinations=("manager",), crawl=crawl)}, transforms={"manager": lambda item: {**item, "headline": "Manager: " + item.get("headline", "")}})
72
+ app.sessions = FakeSessions()
73
+ app.destinations["manager"].socket_path = str(tmp_path / "magent.sock")
74
+ async with collector(app.destinations["manager"].socket_path) as received:
75
+ for run in (1, 2):
76
+ await app.run(once=True)
77
+ async with asyncio.timeout(1):
78
+ while len(received.frames) < run * 2:
79
+ await asyncio.sleep(0)
80
+ assert received.connections == run
81
+ assert app.delivery_stats["manager"].sent == 2
82
+ expected = {"type": "post", "app_id": "app-1", "post": {"url": "https://example.test/one", "headline": "Manager: News", "published_at": 1794899999000, "scraped_at": 1794900000000}}
83
+ if instance is not None:
84
+ expected["instance"] = instance
85
+ assert received.frames[0] == received.frames[2] == expected
86
+ assert [frame["post"]["url"] for frame in received.frames] == ["https://example.test/one", "https://example.test/two"] * 2
87
+ assert app.notifiers == {}
88
+
89
+
90
+ async def test_missing_agent_isolated_from_other_destination_and_failed_item_not_replayed(tmp_path, monkeypatch):
91
+ path = str(tmp_path / "missing.sock")
92
+ connect = AsyncMock(wraps=asyncio.open_unix_connection)
93
+ monkeypatch.setattr(asyncio, "open_unix_connection", connect)
94
+ manager, fast = OpticManagerDestination("app-1", socket_path=path), RecordingDestination()
95
+
96
+ async def crawl(crawler):
97
+ await crawler.emit("lost", {"url": "https://example.test/lost"})
98
+ await app._runtime.senders["manager"].drain()
99
+ assert connect.await_count == 1 and len(fast.sent) == 1
100
+ async with collector(path) as received:
101
+ await crawler.emit("lost", {"url": "https://example.test/lost"})
102
+ await crawler.emit("new", {"url": "https://example.test/new"})
103
+ await app._runtime.senders["manager"].drain()
104
+ async with asyncio.timeout(1):
105
+ while not received.frames:
106
+ await asyncio.sleep(0)
107
+ assert [frame["post"]["url"] for frame in received.frames] == ["https://example.test/new"]
108
+
109
+ app = App(project_name="test", spiders=[SimpleNamespace(name="feed", destinations=("manager", "fast"), crawl=crawl)], destinations={"manager": manager, "fast": fast}, sessions=FakeSessions())
110
+ await app.run(once=True)
111
+ assert connect.await_count == 2 and len(fast.sent) == 2
112
+ assert app.delivery_stats["manager"].failures == 1
113
+ assert app.delivery_stats["manager"].sent == 1
114
+
115
+
116
+ @pytest.mark.parametrize("failure,error", [("ambiguous", OSError), ("timeout", TimeoutError), ("cancel", asyncio.CancelledError)])
117
+ async def test_send_attempted_once_and_connection_released_on_error_or_cancellation(monkeypatch, failure, error):
118
+ written, aborted = [], []
119
+ entered = asyncio.Event()
120
+
121
+ async def drain():
122
+ entered.set()
123
+ if failure == "ambiguous":
124
+ raise OSError("Write result lost after possible delivery")
125
+ await asyncio.Event().wait()
126
+
127
+ writer = SimpleNamespace(write=written.append, drain=drain, transport=SimpleNamespace(abort=lambda: aborted.append(True)))
128
+ connect = AsyncMock(return_value=(None, writer))
129
+ monkeypatch.setattr(asyncio, "open_unix_connection", connect)
130
+ destination = OpticManagerDestination("app-1", timeout=0.02)
131
+ await destination.open()
132
+ sending = asyncio.create_task(destination.send(destination.prepare("one", {"url": "https://example.test/one"})))
133
+ await entered.wait()
134
+ if failure == "cancel":
135
+ sending.cancel()
136
+ with pytest.raises(error):
137
+ await sending
138
+ await destination.close()
139
+ assert connect.await_count == len(written) == len(aborted) == 1
140
+ assert destination._writer is None
141
+
142
+
143
+ @pytest.mark.parametrize("definition", [
144
+ {"type": "optic_manager"}, {"type": "optic_manager", "app_id": " "},
145
+ {"type": "optic_manager", "app_id": 123}, {"type": "optic_manager", "app_id": "app-1", "instance": " "},
146
+ {"type": "optic_manager", "app_id": "app-1", "timeout": 1},
147
+ {"type": "optic_manager", "app_id": "app-1", "socket_path": "/tmp/custom.sock"},
148
+ ])
149
+ def test_invalid_destination_configuration_rejected_before_construction(definition):
150
+ with pytest.raises(ConfigurationError, match="destinations.manager"):
151
+ ProjectConfig({"schema_version": 1, "project_name": "test", "destinations": {"manager": definition}, "spiders": {"feed": {"destinations": ["manager"]}}})
@@ -1,5 +1,5 @@
1
1
  version = 1
2
- revision = 5
2
+ revision = 3
3
3
  requires-python = ">=3.15"
4
4
 
5
5
  [[package]]
@@ -309,7 +309,7 @@ wheels = [
309
309
 
310
310
  [[package]]
311
311
  name = "zenx-lite-python"
312
- version = "0.1.0"
312
+ version = "0.1.1"
313
313
  source = { editable = "." }
314
314
  dependencies = [
315
315
  { name = "curl-cffi-patch" },
@@ -162,6 +162,11 @@ def _destination(data: Any, path: str) -> Data:
162
162
  _text(data["path"], f"{path}.path")
163
163
  elif data["stream"] not in ("stdout", "stderr"):
164
164
  raise ConfigurationError(f"{path}.stream must be stdout or stderr")
165
+ elif kind == "optic_manager":
166
+ _mapping(data, path, {"type", "app_id", "instance"})
167
+ _text(data.get("app_id"), f"{path}.app_id")
168
+ if data.get("instance") is not None:
169
+ _text(data["instance"], f"{path}.instance")
165
170
  elif kind == "synoptic_source":
166
171
  _mapping(data, path, {"type", "endpoint", "source_id", "api_key", "reconnect_delay", "connect_options"})
167
172
  for field in ("endpoint", "source_id", "api_key"):
@@ -171,7 +176,7 @@ def _destination(data: Any, path: str) -> Data:
171
176
  _positive(data.get("reconnect_delay", 1.0), f"{path}.reconnect_delay")
172
177
  _mapping(data.get("connect_options", {}), f"{path}.connect_options")
173
178
  else:
174
- raise ConfigurationError(f"{path}.type must be jsonl or synoptic_source")
179
+ raise ConfigurationError(f"{path}.type must be jsonl, optic_manager, or synoptic_source")
175
180
  return data
176
181
 
177
182
 
@@ -296,6 +301,9 @@ def build_app(
296
301
  from .destinations.jsonl import JSONLines
297
302
  target = kwargs["path"] if "path" in kwargs else getattr(sys, kwargs["stream"])
298
303
  destinations[route] = JSONLines(target, transform=transforms.get(route))
304
+ elif definition["type"] == "optic_manager":
305
+ from .destinations.optic_manager import OpticManagerDestination
306
+ destinations[route] = OpticManagerDestination(**kwargs, transform=transforms.get(route))
299
307
  else:
300
308
  from .destinations.synoptic import SourceDestination
301
309
  destinations[route] = SourceDestination(**kwargs, transform=transforms.get(route))
@@ -0,0 +1,72 @@
1
+ """Best-effort Optic Manager records through Magent's local Unix socket."""
2
+
3
+ import asyncio
4
+ import math
5
+ import time
6
+
7
+ import orjson
8
+
9
+ from .base import Item, Transform, transform_item
10
+ from ..errors import ConfigurationError
11
+
12
+
13
+ class OpticManagerDestination:
14
+ def __init__(self, app_id: str, *, instance: str | None = None, transform: Transform | None = None, socket_path: str = "/tmp/itxpmonitor.sock", timeout: float = 1.0) -> None:
15
+ for name, value in (("app_id", app_id), ("socket_path", socket_path)):
16
+ if not isinstance(value, str) or not value.strip():
17
+ raise ConfigurationError(f"Optic Manager {name} must be a nonblank string")
18
+ if instance is not None and (not isinstance(instance, str) or not instance.strip()):
19
+ raise ConfigurationError("Optic Manager instance must be a nonblank string or None")
20
+ if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0:
21
+ raise ConfigurationError("Optic Manager timeout must be a positive finite number")
22
+ self.app_id = app_id
23
+ self.instance = instance
24
+ self.transform = transform
25
+ self.socket_path = socket_path
26
+ self.timeout = timeout
27
+ self._writer: asyncio.StreamWriter | None = None
28
+
29
+
30
+ async def open(self) -> None:
31
+ pass
32
+
33
+
34
+ def prepare(self, item_id: str, data: Item) -> bytes:
35
+ record = transform_item(data, self.transform)
36
+ url = record.get("url")
37
+ if not isinstance(url, str) or not url.strip():
38
+ raise ValueError("Optic Manager records require a nonblank url")
39
+ scraped_at = record.get("scraped_at")
40
+ payload = {
41
+ "type": "post", "app_id": self.app_id,
42
+ "post": {
43
+ "url": url, "headline": record.get("headline"),
44
+ "published_at": record.get("published_at"),
45
+ "scraped_at": time.time_ns() // 1_000_000 if scraped_at is None else scraped_at,
46
+ },
47
+ }
48
+ if self.instance is not None:
49
+ payload["instance"] = self.instance
50
+ return orjson.dumps(payload, option=orjson.OPT_APPEND_NEWLINE)
51
+
52
+
53
+ async def ready(self) -> None:
54
+ pass
55
+
56
+
57
+ async def send(self, payload: bytes) -> None:
58
+ try:
59
+ async with asyncio.timeout(self.timeout):
60
+ if self._writer is None:
61
+ _, self._writer = await asyncio.open_unix_connection(self.socket_path)
62
+ self._writer.write(payload)
63
+ await self._writer.drain()
64
+ except BaseException:
65
+ await self.close()
66
+ raise
67
+
68
+
69
+ async def close(self) -> None:
70
+ writer, self._writer = self._writer, None
71
+ if writer is not None:
72
+ writer.transport.abort()