zenx-lite-python 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenx_lite_python-0.1.1/.github/workflows/release.yml +62 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/PKG-INFO +1 -1
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/PROJECT_GUIDE.md +1 -1
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/pyproject.toml +1 -1
- zenx_lite_python-0.1.1/tests/test_optic_manager.py +151 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/uv.lock +2 -2
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/configuration.py +9 -1
- zenx_lite_python-0.1.1/zenx_lite/destinations/optic_manager.py +72 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/.gitignore +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/AGENTS.md +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/__init__.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/fixtures/cli_project.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/helpers.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_cli.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_configuration.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_delivery.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_destinations.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_http.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_notifiers.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_redis_store.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_runtime.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/tests/test_scheduler.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/__init__.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/app.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/config.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/crawler.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/delivery.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/__init__.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/base.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/jsonl.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/synoptic/__init__.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/synoptic/source.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/errors.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/http.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/notifiers/__init__.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/notifiers/base.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/notifiers/magent.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/scheduler.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/sessions.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/store.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/stores/__init__.py +0 -0
- {zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/stores/redis.py +0 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
publish:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
services:
|
|
14
|
+
redis:
|
|
15
|
+
image: redis:8
|
|
16
|
+
ports:
|
|
17
|
+
- 6379:6379
|
|
18
|
+
options: >-
|
|
19
|
+
--health-cmd "redis-cli ping"
|
|
20
|
+
--health-interval 10s
|
|
21
|
+
--health-timeout 5s
|
|
22
|
+
--health-retries 5
|
|
23
|
+
env:
|
|
24
|
+
PYTHONDONTWRITEBYTECODE: "1"
|
|
25
|
+
ZENX_LITE_TEST_REDIS_URL: redis://localhost:6379/0
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
28
|
+
with:
|
|
29
|
+
persist-credentials: false
|
|
30
|
+
|
|
31
|
+
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
|
|
32
|
+
with:
|
|
33
|
+
version: "0.9.15"
|
|
34
|
+
python-version: "3.15"
|
|
35
|
+
|
|
36
|
+
- name: Check release version
|
|
37
|
+
env:
|
|
38
|
+
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
|
39
|
+
run: |
|
|
40
|
+
uv run --no-project python - <<'PY'
|
|
41
|
+
import os
|
|
42
|
+
import tomllib
|
|
43
|
+
|
|
44
|
+
with open("pyproject.toml", "rb") as metadata:
|
|
45
|
+
version = tomllib.load(metadata)["project"]["version"]
|
|
46
|
+
tag = os.environ["RELEASE_TAG"].removeprefix("v")
|
|
47
|
+
if tag != version:
|
|
48
|
+
raise SystemExit(f"Release tag {tag!r} does not match package version {version!r}")
|
|
49
|
+
PY
|
|
50
|
+
|
|
51
|
+
- name: Run tests
|
|
52
|
+
env:
|
|
53
|
+
UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/venv
|
|
54
|
+
run: uv run --locked --all-extras pytest -q -p no:cacheprovider
|
|
55
|
+
|
|
56
|
+
- name: Build distributions
|
|
57
|
+
run: uv build --out-dir "$RUNNER_TEMP/dist"
|
|
58
|
+
|
|
59
|
+
- name: Publish distributions
|
|
60
|
+
env:
|
|
61
|
+
UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
|
62
|
+
run: uv publish --trusted-publishing never "$RUNNER_TEMP"/dist/*
|
|
@@ -7,7 +7,7 @@ Build a Python scraping runtime that is easy to understand, compose, and operate
|
|
|
7
7
|
- **Make ownership clear.** The runtime owns workers, managed sessions, registered crawl tasks, and delivery resources. Applications own business logic and their independently created resources.
|
|
8
8
|
- **Separate policy from execution.** Scheduling and integrations should be replaceable without changing resource ownership or delivery guarantees. Keep process supervision and deployment orchestration outside the crawling core.
|
|
9
9
|
- **Prioritize prompt delivery.** Send without deliberate batching, capacity waits, or artificial delays. Use unbounded queues unless explicitly requested otherwise. Preserve event-loop fairness and keep blocking work out of the admission path.
|
|
10
|
-
- **Make failure semantics predictable.** Use native task and cancellation behavior. Isolate destination failures; delivery remains best effort, without retrying or replaying attempted payloads. Remembering an ID does not prove delivery.
|
|
10
|
+
- **Make failure semantics predictable.** Use native task and cancellation behavior. Isolate destination failures; delivery remains best effort, without retrying or replaying attempted payloads. Transport-buffered data may be discarded on close; do not add flush waits unless explicitly requested. Remembering an ID does not prove delivery.
|
|
11
11
|
- **Keep application data independent.** Do not inject framework metadata or business rules into payloads. Keep identity, routing, and destination envelopes separate, and protect queued data from later mutation.
|
|
12
12
|
- **Treat operations as a real requirement.** Configuration, monitoring, and deployment support should be consistent and inspectable. Add shared support where needed while keeping integrations optional and environment behavior explicit.
|
|
13
13
|
- **Measure before optimizing.** Validate latency, resource use, and correctness with representative workloads, including multiple scraper processes. Simpler architecture alone does not prove better performance.
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
from contextlib import asynccontextmanager
|
|
4
|
+
from types import SimpleNamespace
|
|
5
|
+
from unittest.mock import AsyncMock
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
from zenx_lite import App
|
|
10
|
+
from zenx_lite.configuration import ProjectConfig, build_app
|
|
11
|
+
from zenx_lite.destinations.optic_manager import OpticManagerDestination
|
|
12
|
+
from zenx_lite.errors import ConfigurationError
|
|
13
|
+
|
|
14
|
+
from .helpers import FakeSessions, RecordingDestination
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@asynccontextmanager
|
|
18
|
+
async def collector(path):
|
|
19
|
+
state = SimpleNamespace(frames=[], connections=0)
|
|
20
|
+
tasks = set()
|
|
21
|
+
|
|
22
|
+
async def receive(reader, writer):
|
|
23
|
+
tasks.add(asyncio.current_task())
|
|
24
|
+
state.connections += 1
|
|
25
|
+
try:
|
|
26
|
+
while line := await reader.readline():
|
|
27
|
+
state.frames.append(json.loads(line))
|
|
28
|
+
finally:
|
|
29
|
+
writer.transport.abort()
|
|
30
|
+
tasks.discard(asyncio.current_task())
|
|
31
|
+
|
|
32
|
+
server = await asyncio.start_unix_server(receive, path)
|
|
33
|
+
try:
|
|
34
|
+
yield state
|
|
35
|
+
finally:
|
|
36
|
+
server.close()
|
|
37
|
+
server.abort_clients()
|
|
38
|
+
await server.wait_closed()
|
|
39
|
+
if tasks:
|
|
40
|
+
await asyncio.gather(*tuple(tasks), return_exceptions=True)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@pytest.mark.parametrize("scraped_at", [None, 0, 1794900000000])
|
|
44
|
+
def test_prepare_transforms_without_mutation_and_captures_timestamp(monkeypatch, scraped_at):
|
|
45
|
+
monkeypatch.setattr("zenx_lite.destinations.optic_manager.time.time_ns", lambda: 1800000000123000000)
|
|
46
|
+
item = {"link": "https://example.test/article", "title": "News", "scraped_at": scraped_at, "body": "application data"}
|
|
47
|
+
destination = OpticManagerDestination("app-1", transform=lambda item: {"url": item["link"], "headline": item["title"], "scraped_at": item["scraped_at"]})
|
|
48
|
+
payload = destination.prepare("separate-id", item)
|
|
49
|
+
assert payload.endswith(b"\n") and len(payload.splitlines()) == 1
|
|
50
|
+
assert json.loads(payload) == {
|
|
51
|
+
"type": "post", "app_id": "app-1",
|
|
52
|
+
"post": {"url": item["link"], "headline": "News", "published_at": None, "scraped_at": 1800000000123 if scraped_at is None else scraped_at},
|
|
53
|
+
}
|
|
54
|
+
assert item == {"link": "https://example.test/article", "title": "News", "scraped_at": scraped_at, "body": "application data"}
|
|
55
|
+
with pytest.raises(ValueError, match="url"):
|
|
56
|
+
OpticManagerDestination("app-1").prepare("bad", {"headline": "Missing URL"})
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@pytest.mark.parametrize("instance", [None, "default"])
|
|
60
|
+
async def test_configured_destination_posts_dedupes_reuses_connection_and_reopens(tmp_path, instance):
|
|
61
|
+
async def crawl(crawler):
|
|
62
|
+
await crawler.emit("one", {"url": "https://example.test/one", "headline": "News", "published_at": 1794899999000, "scraped_at": 1794900000000})
|
|
63
|
+
await crawler.emit("one", {"url": "https://example.test/duplicate"})
|
|
64
|
+
await crawler.emit("two", {"url": "https://example.test/two"})
|
|
65
|
+
|
|
66
|
+
config = ProjectConfig({
|
|
67
|
+
"schema_version": 1, "project_name": "test",
|
|
68
|
+
"destinations": {"manager": {"type": "optic_manager", "app_id": "app-1", "instance": instance}},
|
|
69
|
+
"spiders": {"feed": {"destinations": ["manager"]}},
|
|
70
|
+
})
|
|
71
|
+
app = build_app(ProjectConfig(config.resolve()), spiders={"feed": lambda: SimpleNamespace(name="feed", destinations=("manager",), crawl=crawl)}, transforms={"manager": lambda item: {**item, "headline": "Manager: " + item.get("headline", "")}})
|
|
72
|
+
app.sessions = FakeSessions()
|
|
73
|
+
app.destinations["manager"].socket_path = str(tmp_path / "magent.sock")
|
|
74
|
+
async with collector(app.destinations["manager"].socket_path) as received:
|
|
75
|
+
for run in (1, 2):
|
|
76
|
+
await app.run(once=True)
|
|
77
|
+
async with asyncio.timeout(1):
|
|
78
|
+
while len(received.frames) < run * 2:
|
|
79
|
+
await asyncio.sleep(0)
|
|
80
|
+
assert received.connections == run
|
|
81
|
+
assert app.delivery_stats["manager"].sent == 2
|
|
82
|
+
expected = {"type": "post", "app_id": "app-1", "post": {"url": "https://example.test/one", "headline": "Manager: News", "published_at": 1794899999000, "scraped_at": 1794900000000}}
|
|
83
|
+
if instance is not None:
|
|
84
|
+
expected["instance"] = instance
|
|
85
|
+
assert received.frames[0] == received.frames[2] == expected
|
|
86
|
+
assert [frame["post"]["url"] for frame in received.frames] == ["https://example.test/one", "https://example.test/two"] * 2
|
|
87
|
+
assert app.notifiers == {}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
async def test_missing_agent_isolated_from_other_destination_and_failed_item_not_replayed(tmp_path, monkeypatch):
|
|
91
|
+
path = str(tmp_path / "missing.sock")
|
|
92
|
+
connect = AsyncMock(wraps=asyncio.open_unix_connection)
|
|
93
|
+
monkeypatch.setattr(asyncio, "open_unix_connection", connect)
|
|
94
|
+
manager, fast = OpticManagerDestination("app-1", socket_path=path), RecordingDestination()
|
|
95
|
+
|
|
96
|
+
async def crawl(crawler):
|
|
97
|
+
await crawler.emit("lost", {"url": "https://example.test/lost"})
|
|
98
|
+
await app._runtime.senders["manager"].drain()
|
|
99
|
+
assert connect.await_count == 1 and len(fast.sent) == 1
|
|
100
|
+
async with collector(path) as received:
|
|
101
|
+
await crawler.emit("lost", {"url": "https://example.test/lost"})
|
|
102
|
+
await crawler.emit("new", {"url": "https://example.test/new"})
|
|
103
|
+
await app._runtime.senders["manager"].drain()
|
|
104
|
+
async with asyncio.timeout(1):
|
|
105
|
+
while not received.frames:
|
|
106
|
+
await asyncio.sleep(0)
|
|
107
|
+
assert [frame["post"]["url"] for frame in received.frames] == ["https://example.test/new"]
|
|
108
|
+
|
|
109
|
+
app = App(project_name="test", spiders=[SimpleNamespace(name="feed", destinations=("manager", "fast"), crawl=crawl)], destinations={"manager": manager, "fast": fast}, sessions=FakeSessions())
|
|
110
|
+
await app.run(once=True)
|
|
111
|
+
assert connect.await_count == 2 and len(fast.sent) == 2
|
|
112
|
+
assert app.delivery_stats["manager"].failures == 1
|
|
113
|
+
assert app.delivery_stats["manager"].sent == 1
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@pytest.mark.parametrize("failure,error", [("ambiguous", OSError), ("timeout", TimeoutError), ("cancel", asyncio.CancelledError)])
|
|
117
|
+
async def test_send_attempted_once_and_connection_released_on_error_or_cancellation(monkeypatch, failure, error):
|
|
118
|
+
written, aborted = [], []
|
|
119
|
+
entered = asyncio.Event()
|
|
120
|
+
|
|
121
|
+
async def drain():
|
|
122
|
+
entered.set()
|
|
123
|
+
if failure == "ambiguous":
|
|
124
|
+
raise OSError("Write result lost after possible delivery")
|
|
125
|
+
await asyncio.Event().wait()
|
|
126
|
+
|
|
127
|
+
writer = SimpleNamespace(write=written.append, drain=drain, transport=SimpleNamespace(abort=lambda: aborted.append(True)))
|
|
128
|
+
connect = AsyncMock(return_value=(None, writer))
|
|
129
|
+
monkeypatch.setattr(asyncio, "open_unix_connection", connect)
|
|
130
|
+
destination = OpticManagerDestination("app-1", timeout=0.02)
|
|
131
|
+
await destination.open()
|
|
132
|
+
sending = asyncio.create_task(destination.send(destination.prepare("one", {"url": "https://example.test/one"})))
|
|
133
|
+
await entered.wait()
|
|
134
|
+
if failure == "cancel":
|
|
135
|
+
sending.cancel()
|
|
136
|
+
with pytest.raises(error):
|
|
137
|
+
await sending
|
|
138
|
+
await destination.close()
|
|
139
|
+
assert connect.await_count == len(written) == len(aborted) == 1
|
|
140
|
+
assert destination._writer is None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@pytest.mark.parametrize("definition", [
|
|
144
|
+
{"type": "optic_manager"}, {"type": "optic_manager", "app_id": " "},
|
|
145
|
+
{"type": "optic_manager", "app_id": 123}, {"type": "optic_manager", "app_id": "app-1", "instance": " "},
|
|
146
|
+
{"type": "optic_manager", "app_id": "app-1", "timeout": 1},
|
|
147
|
+
{"type": "optic_manager", "app_id": "app-1", "socket_path": "/tmp/custom.sock"},
|
|
148
|
+
])
|
|
149
|
+
def test_invalid_destination_configuration_rejected_before_construction(definition):
|
|
150
|
+
with pytest.raises(ConfigurationError, match="destinations.manager"):
|
|
151
|
+
ProjectConfig({"schema_version": 1, "project_name": "test", "destinations": {"manager": definition}, "spiders": {"feed": {"destinations": ["manager"]}}})
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
version = 1
|
|
2
|
-
revision =
|
|
2
|
+
revision = 3
|
|
3
3
|
requires-python = ">=3.15"
|
|
4
4
|
|
|
5
5
|
[[package]]
|
|
@@ -309,7 +309,7 @@ wheels = [
|
|
|
309
309
|
|
|
310
310
|
[[package]]
|
|
311
311
|
name = "zenx-lite-python"
|
|
312
|
-
version = "0.1.
|
|
312
|
+
version = "0.1.1"
|
|
313
313
|
source = { editable = "." }
|
|
314
314
|
dependencies = [
|
|
315
315
|
{ name = "curl-cffi-patch" },
|
|
@@ -162,6 +162,11 @@ def _destination(data: Any, path: str) -> Data:
|
|
|
162
162
|
_text(data["path"], f"{path}.path")
|
|
163
163
|
elif data["stream"] not in ("stdout", "stderr"):
|
|
164
164
|
raise ConfigurationError(f"{path}.stream must be stdout or stderr")
|
|
165
|
+
elif kind == "optic_manager":
|
|
166
|
+
_mapping(data, path, {"type", "app_id", "instance"})
|
|
167
|
+
_text(data.get("app_id"), f"{path}.app_id")
|
|
168
|
+
if data.get("instance") is not None:
|
|
169
|
+
_text(data["instance"], f"{path}.instance")
|
|
165
170
|
elif kind == "synoptic_source":
|
|
166
171
|
_mapping(data, path, {"type", "endpoint", "source_id", "api_key", "reconnect_delay", "connect_options"})
|
|
167
172
|
for field in ("endpoint", "source_id", "api_key"):
|
|
@@ -171,7 +176,7 @@ def _destination(data: Any, path: str) -> Data:
|
|
|
171
176
|
_positive(data.get("reconnect_delay", 1.0), f"{path}.reconnect_delay")
|
|
172
177
|
_mapping(data.get("connect_options", {}), f"{path}.connect_options")
|
|
173
178
|
else:
|
|
174
|
-
raise ConfigurationError(f"{path}.type must be jsonl or synoptic_source")
|
|
179
|
+
raise ConfigurationError(f"{path}.type must be jsonl, optic_manager, or synoptic_source")
|
|
175
180
|
return data
|
|
176
181
|
|
|
177
182
|
|
|
@@ -296,6 +301,9 @@ def build_app(
|
|
|
296
301
|
from .destinations.jsonl import JSONLines
|
|
297
302
|
target = kwargs["path"] if "path" in kwargs else getattr(sys, kwargs["stream"])
|
|
298
303
|
destinations[route] = JSONLines(target, transform=transforms.get(route))
|
|
304
|
+
elif definition["type"] == "optic_manager":
|
|
305
|
+
from .destinations.optic_manager import OpticManagerDestination
|
|
306
|
+
destinations[route] = OpticManagerDestination(**kwargs, transform=transforms.get(route))
|
|
299
307
|
else:
|
|
300
308
|
from .destinations.synoptic import SourceDestination
|
|
301
309
|
destinations[route] = SourceDestination(**kwargs, transform=transforms.get(route))
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Best-effort Optic Manager records through Magent's local Unix socket."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import math
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
import orjson
|
|
8
|
+
|
|
9
|
+
from .base import Item, Transform, transform_item
|
|
10
|
+
from ..errors import ConfigurationError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class OpticManagerDestination:
|
|
14
|
+
def __init__(self, app_id: str, *, instance: str | None = None, transform: Transform | None = None, socket_path: str = "/tmp/itxpmonitor.sock", timeout: float = 1.0) -> None:
|
|
15
|
+
for name, value in (("app_id", app_id), ("socket_path", socket_path)):
|
|
16
|
+
if not isinstance(value, str) or not value.strip():
|
|
17
|
+
raise ConfigurationError(f"Optic Manager {name} must be a nonblank string")
|
|
18
|
+
if instance is not None and (not isinstance(instance, str) or not instance.strip()):
|
|
19
|
+
raise ConfigurationError("Optic Manager instance must be a nonblank string or None")
|
|
20
|
+
if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0:
|
|
21
|
+
raise ConfigurationError("Optic Manager timeout must be a positive finite number")
|
|
22
|
+
self.app_id = app_id
|
|
23
|
+
self.instance = instance
|
|
24
|
+
self.transform = transform
|
|
25
|
+
self.socket_path = socket_path
|
|
26
|
+
self.timeout = timeout
|
|
27
|
+
self._writer: asyncio.StreamWriter | None = None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async def open(self) -> None:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def prepare(self, item_id: str, data: Item) -> bytes:
|
|
35
|
+
record = transform_item(data, self.transform)
|
|
36
|
+
url = record.get("url")
|
|
37
|
+
if not isinstance(url, str) or not url.strip():
|
|
38
|
+
raise ValueError("Optic Manager records require a nonblank url")
|
|
39
|
+
scraped_at = record.get("scraped_at")
|
|
40
|
+
payload = {
|
|
41
|
+
"type": "post", "app_id": self.app_id,
|
|
42
|
+
"post": {
|
|
43
|
+
"url": url, "headline": record.get("headline"),
|
|
44
|
+
"published_at": record.get("published_at"),
|
|
45
|
+
"scraped_at": time.time_ns() // 1_000_000 if scraped_at is None else scraped_at,
|
|
46
|
+
},
|
|
47
|
+
}
|
|
48
|
+
if self.instance is not None:
|
|
49
|
+
payload["instance"] = self.instance
|
|
50
|
+
return orjson.dumps(payload, option=orjson.OPT_APPEND_NEWLINE)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def ready(self) -> None:
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def send(self, payload: bytes) -> None:
|
|
58
|
+
try:
|
|
59
|
+
async with asyncio.timeout(self.timeout):
|
|
60
|
+
if self._writer is None:
|
|
61
|
+
_, self._writer = await asyncio.open_unix_connection(self.socket_path)
|
|
62
|
+
self._writer.write(payload)
|
|
63
|
+
await self._writer.drain()
|
|
64
|
+
except BaseException:
|
|
65
|
+
await self.close()
|
|
66
|
+
raise
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
async def close(self) -> None:
|
|
70
|
+
writer, self._writer = self._writer, None
|
|
71
|
+
if writer is not None:
|
|
72
|
+
writer.transport.abort()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{zenx_lite_python-0.1.0 → zenx_lite_python-0.1.1}/zenx_lite/destinations/synoptic/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|