zenx-lite 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. zenx_lite-0.2.0/.agents/skills/zenx-lite/SKILL.md +41 -0
  2. zenx_lite-0.2.0/.agents/skills/zenx-lite/references/configuration-and-instances.md +108 -0
  3. zenx_lite-0.2.0/.agents/skills/zenx-lite/references/crawl-and-runtime.md +50 -0
  4. zenx_lite-0.2.0/.agents/skills/zenx-lite/references/deduplication-and-delivery.md +42 -0
  5. zenx_lite-0.2.0/.github/workflows/release.yml +57 -0
  6. zenx_lite-0.2.0/.gitignore +8 -0
  7. zenx_lite-0.2.0/AGENTS.md +28 -0
  8. zenx_lite-0.2.0/PKG-INFO +93 -0
  9. zenx_lite-0.2.0/PROJECT_GUIDE.md +18 -0
  10. zenx_lite-0.2.0/README.md +79 -0
  11. zenx_lite-0.2.0/pyproject.toml +31 -0
  12. zenx_lite-0.2.0/tests/__init__.py +0 -0
  13. zenx_lite-0.2.0/tests/fixtures/cli_project.py +62 -0
  14. zenx_lite-0.2.0/tests/helpers.py +72 -0
  15. zenx_lite-0.2.0/tests/test_cli.py +255 -0
  16. zenx_lite-0.2.0/tests/test_configuration.py +442 -0
  17. zenx_lite-0.2.0/tests/test_delivery.py +247 -0
  18. zenx_lite-0.2.0/tests/test_destinations.py +173 -0
  19. zenx_lite-0.2.0/tests/test_http.py +343 -0
  20. zenx_lite-0.2.0/tests/test_logging.py +102 -0
  21. zenx_lite-0.2.0/tests/test_notifiers.py +298 -0
  22. zenx_lite-0.2.0/tests/test_optic_manager.py +148 -0
  23. zenx_lite-0.2.0/tests/test_runtime.py +375 -0
  24. zenx_lite-0.2.0/tests/test_scheduler.py +61 -0
  25. zenx_lite-0.2.0/tests/test_sqlite_store.py +280 -0
  26. zenx_lite-0.2.0/uv.lock +348 -0
  27. zenx_lite-0.2.0/zenx_lite/__init__.py +8 -0
  28. zenx_lite-0.2.0/zenx_lite/app.py +390 -0
  29. zenx_lite-0.2.0/zenx_lite/config.py +68 -0
  30. zenx_lite-0.2.0/zenx_lite/configuration.py +360 -0
  31. zenx_lite-0.2.0/zenx_lite/crawler.py +68 -0
  32. zenx_lite-0.2.0/zenx_lite/delivery.py +108 -0
  33. zenx_lite-0.2.0/zenx_lite/destinations/__init__.py +5 -0
  34. zenx_lite-0.2.0/zenx_lite/destinations/base.py +46 -0
  35. zenx_lite-0.2.0/zenx_lite/destinations/jsonl.py +138 -0
  36. zenx_lite-0.2.0/zenx_lite/destinations/optic_manager.py +73 -0
  37. zenx_lite-0.2.0/zenx_lite/destinations/synoptic/__init__.py +5 -0
  38. zenx_lite-0.2.0/zenx_lite/destinations/synoptic/source.py +137 -0
  39. zenx_lite-0.2.0/zenx_lite/errors.py +21 -0
  40. zenx_lite-0.2.0/zenx_lite/http.py +148 -0
  41. zenx_lite-0.2.0/zenx_lite/logging.py +172 -0
  42. zenx_lite-0.2.0/zenx_lite/notifiers/__init__.py +5 -0
  43. zenx_lite-0.2.0/zenx_lite/notifiers/base.py +15 -0
  44. zenx_lite-0.2.0/zenx_lite/notifiers/magent.py +87 -0
  45. zenx_lite-0.2.0/zenx_lite/scheduler.py +63 -0
  46. zenx_lite-0.2.0/zenx_lite/sessions.py +105 -0
  47. zenx_lite-0.2.0/zenx_lite/store.py +45 -0
  48. zenx_lite-0.2.0/zenx_lite/stores/__init__.py +1 -0
  49. zenx_lite-0.2.0/zenx_lite/stores/sqlite.py +166 -0
@@ -0,0 +1,41 @@
1
+ ---
2
+ name: zenx-lite
3
+ description: Build, configure, and troubleshoot scrapers that use ZenX Lite. Use when creating or migrating a spider, wiring an app, configuring separate spider instances, or diagnosing crawl, deduplication, and delivery behavior.
4
+ ---
5
+
6
+ # ZenX Lite
7
+
8
+ Check the consuming project's resolved package version and extras. In the framework repository, use the [project guide](../../../PROJECT_GUIDE.md), [implementation](../../../zenx_lite/), and [tests](../../../tests/) to verify behavior. The base package requires Python 3.15 or newer.
9
+
10
+ ## Build the first working crawl
11
+
12
+ 1. For a new scraper project, use the [standard project structure](references/configuration-and-instances.md#standard-project-structure), including for a project with one spider. Install the package with `uv add zenx-lite`. For changes to an existing project, follow its layout and entry point.
13
+
14
+ For a one-off scrape or quick debugging run outside any existing or planned scraper project, put the spider and `App(...).cli()` setup in a single `spider.py`. Run `uv run --no-project --python 3.15.2 --with zenx-lite spider.py --once`.
15
+
16
+ 2. Give each spider a unique `name` within its `App` and an `async def crawl(self, crawler)` method. Put source inputs in the constructor. Register destination adapters and select each spider's active routes in `App` or YAML.
17
+ 3. Fetch with `crawler.request()`, check the response status, and parse the returned body. For new-data workloads, use the freshest available listing or event source. Keep request count and detection-to-delivery time in view.
18
+ 4. Emit a stable, nonempty string ID and a dictionary of business data with `await crawler.emit(item_id, data)`. Start with `JSONLines(sys.stdout)` so the emitted records are easy to inspect.
19
+ 5. For terminal runs, use `app.cli()` with Python configuration or `run_from_config(spiders=...)` with YAML. Pass `--once` for a single crawl. In an existing async application use `await app.run(once=True)`. Select the spider by name when several are available.
20
+
21
+ ## Choose the features the workload needs
22
+
23
+ | Need | Use |
24
+ | --- | --- |
25
+ | Control crawl cadence | Start with `Slots` and one worker. Tune the interval and concurrency from observed crawl duration, request count, and source freshness. |
26
+ | Overlap independent work inside a crawl | Register bounded work with `crawler.create_task()` so it shares the crawl's lifetime. |
27
+ | Consume content incrementally | Use `async with crawler.stream(...)` and the native curl response iterator. |
28
+ | Remember IDs across same-host processes or restarts | Use `SQLiteStore` with a persistent local path and a shared dedup scope. SQLite reservations last seven days by default, and each store caches 500 confirmed IDs. |
29
+ | Run one spider with several sets of inputs | Supply the inputs in YAML and launch a separate app for each instance with the same spider factory. |
30
+ | Send records or report crawl health | Choose the required destination. Configure Magent separately when crawl heartbeats are needed. |
31
+ | Inspect requests and new items in production | Use the native `structlog` events at `INFO` and configure a rotating log file per deployed instance. |
32
+
33
+ - Read [crawl and runtime](references/crawl-and-runtime.md) for HTTP, logging, streaming, tasks, scheduling, execution modes, and custom sessions or schedulers.
34
+ - Read [configuration and instances](references/configuration-and-instances.md) for the standard project structure, a runnable two-instance example, and YAML validation.
35
+ - Read [deduplication and delivery](references/deduplication-and-delivery.md) for routing, stores, built-in integrations, and custom adapters.
36
+
37
+ ## Verify the result
38
+
39
+ - Run a one-shot crawl through the actual entry point. Check the HTTP result and the content, ID, and destination of a representative emitted item. Prefer stdout or a local receiver during development.
40
+ - For a recurring workload, measure first-seen freshness, requests per crawl, and actual crawl cadence. Confirm how the spider handles stale listings and unchanged items.
41
+ - Verify emitted items at the receiver when delivery matters. Check destination logs to diagnose failed sends.
@@ -0,0 +1,108 @@
1
+ # Configuration and instances
2
+
3
+ Use Python composition for fixed or application-owned settings. Use `run_from_config()` when a launcher needs to select a spider and provide inputs, routes, or runtime settings from YAML. YAML support is included in `zenx-lite`.
4
+
5
+ ## Standard project structure
6
+
7
+ ```text
8
+ my-scraper/
9
+ ├── .gitignore
10
+ ├── pyproject.toml
11
+ ├── uv.lock
12
+ ├── main.py
13
+ └── spiders/
14
+ ├── __init__.py
15
+ └── feed.py
16
+ ```
17
+
18
+ Declare dependencies in `pyproject.toml` and commit `uv.lock`. Keep crawl logic in `spiders/` and application setup in `main.py`. For YAML configuration, call `run_from_config()` with a registry that maps each spider name to its factory.
19
+
20
+ For persistent deduplication, use `<project>/.state/seen.sqlite3` and add `/.state/` to `.gitignore`. Pass an explicit path to SQLite, using an absolute path when the launcher may run from another directory. Preserve `.state/` across redeploys.
21
+
22
+ ## Run one class with two sets of inputs
23
+
24
+ Create a `spiders` package with an empty `__init__.py` and save this as `spiders/feed.py`.
25
+
26
+ ```python
27
+ from zenx_lite import Crawler
28
+
29
+
30
+ class Feed:
31
+ name = "feed"
32
+
33
+ def __init__(self, url: str) -> None:
34
+ self.url = url
35
+
36
+
37
+ async def crawl(self, crawler: Crawler) -> None:
38
+ response = await crawler.request(self.url, timeout=10)
39
+ if response.status != 200:
40
+ raise RuntimeError(f"HTTP {response.status}")
41
+ title = response.xpath("//title/text()").get()
42
+ await crawler.emit(response.url, {"url": response.url, "title": title})
43
+ ```
44
+
45
+ Register the factory in `main.py`.
46
+
47
+ ```python
48
+ from zenx_lite.configuration import run_from_config
49
+
50
+ from spiders.feed import Feed
51
+
52
+
53
+ if __name__ == "__main__":
54
+ run_from_config(spiders={"feed": Feed})
55
+ ```
56
+
57
+ Save `feed-a.yaml`.
58
+
59
+ ```yaml
60
+ schema_version: 1
61
+ project_name: example-scraper
62
+ destinations:
63
+ out: {type: jsonl, stream: stdout}
64
+ spiders:
65
+ feed:
66
+ inputs: {url: https://example.com/}
67
+ dedup: {scope: "feed:a"}
68
+ destinations: [out]
69
+ ```
70
+
71
+ Save `feed-b.yaml`.
72
+
73
+ ```yaml
74
+ schema_version: 1
75
+ project_name: example-scraper
76
+ destinations:
77
+ out: {type: jsonl, stream: stdout}
78
+ spiders:
79
+ feed:
80
+ inputs: {url: https://example.org/}
81
+ dedup: {scope: "feed:b"}
82
+ destinations: [out]
83
+ ```
84
+
85
+ ```bash
86
+ uv run python -m zenx_lite.configuration feed-a.yaml --spider feed
87
+ uv run main.py --config feed-a.yaml --once
88
+ uv run main.py --config feed-b.yaml --once
89
+ ```
90
+
91
+ Each crawl prints a JSON record with the URL from its own configuration. For continuous use, a supervisor can launch the two configurations as independent processes with `--once` omitted. The scopes are distinct so the instances also remain independent if both later use the same SQLite file.
92
+
93
+ ## Resolve configuration explicitly
94
+
95
+ - Top-level `destinations` defines named adapters. Select each enabled spider's active destinations in the spider entry or `defaults` using those registered names. Use `destinations: []` when intentionally reserving IDs without delivery.
96
+ - `defaults` supplies shared options. Spider entries can set `inputs`, `runtime`, `scheduler`, `sessions`, `dedup`, `destinations`, and `notifier`. Dictionaries merge and lists replace earlier lists. Use `enabled: false` to exclude an entry. Supply `--spider NAME` if multiple entries are enabled.
97
+ - Lite constructs only the selected factory with `factory(**inputs)`. The returned spider's `name` must match the registry key. `run_from_config()` accepts `--config`, `--spider`, and `--once`, then starts the app through `app.cli()`. Put managed runtime settings in the selected YAML configuration. Standalone `app.cli()` uses its own flags and environment overlay.
98
+ - For per-instance production logs, set `runtime: {instance: uk, log_file: "logs/feed:uk.log", log_backup_count: 1}` on the selected spider. `instance` supplies logging context. Choose a separate file path for each deployed instance.
99
+ - `python -m zenx_lite.configuration FILE` validates the YAML without constructing or starting spiders. Use a one-shot launch to check factory inputs and the effective routes. For embedded use, build the app with `build_app(load_config(path), spiders={...}, spider=...)`, then await `app.run()`. Use `ProjectConfig(data)` for the same schema with a Python dictionary. Register destination transforms explicitly with `transforms={...}`.
100
+
101
+ ## Integrate with Optic Manager
102
+
103
+ Match each registered spider name to its Manager app name. Manager supplies `conf.yaml` at deploy time. Add `/conf.yaml` to the scraper repository's `.gitignore`.
104
+
105
+ 1. Check out the selected commit in the stable project directory and install locked dependencies with `uv sync --locked --no-dev` using Python 3.15 or newer. Preserve the project's `.state/` when updating the checkout.
106
+ 2. Translate each enabled `SPIDERS.<spider>.instances.<instance>` entry in the Manager-supplied `conf.yaml` into a separate Lite YAML file. Include its effective inputs, runtime settings, and selected destinations. Pass Manager's `app_id` and instance key to the notifier and post destination as applicable.
107
+ 3. Set `dedup.store` to SQLite with the absolute path to `.state/seen.sqlite3` under the project root, unless the deployment supplies a different path. Set a stable `dedup.scope` per instance, such as `feed:default` or `feed:uk`, so instances can share the database and deduplicate independently.
108
+ 4. Validate each generated file with `python -m zenx_lite.configuration <generated-yaml> --spider <spider>`. Register each instance with the supervisor using `python main.py --config <generated-yaml> --spider <spider>` in the locked environment. Respect autostart and saved stops when starting instances. Use the Optic Manager deployment workflow for rollout and process control.
@@ -0,0 +1,50 @@
1
+ # Crawl and runtime
2
+
3
+ ## Execution and scheduling
4
+
5
+ - `App` runs one selected spider per invocation. Set `--spider NAME` for the CLI or `spider="name"` for `run()` when multiple spiders are registered. A single `App` supports one active run. Give independently running apps their own resources.
6
+ - `app.cli()` owns its event loop, configures logging, and handles process signals. Its built-in settings take values from CLI flags, then corresponding `ZENX_LITE_*` variables, then Python configuration. `--once` is a CLI flag only. For project-specific arguments, extend `App.cli_parser()`, parse the arguments, construct the selected spider, and call `app.cli(args=args, parser=parser)`.
7
+ - For a YAML-configured terminal run, `run_from_config()` builds the app and uses the same CLI runner. The YAML supplies runtime settings. [Configuration and instances](configuration-and-instances.md)
8
+ - `await app.run(...)` uses explicit Python configuration in the caller's event loop. Call `app.stop()` from the owning loop to request shutdown. Cancellation of the run propagates to the caller after cleanup is initiated.
9
+ - `once=True` uses one worker for one crawl and bypasses the scheduler. On shutdown the runtime gives outstanding one-shot work, destination queues, and cleanup a shared grace budget. Set HTTP timeouts at the request or session level to bound request duration. Continuous shutdown cancels in-flight work and discards pending delivery. Both modes use best-effort destination close.
10
+ - `Settings()` starts with one worker and `Slots()` uses a one-second interval per worker. Import `Settings` from `zenx_lite` and `Slots` from `zenx_lite.scheduler`. For example, `settings=Settings(concurrency=2), scheduler=Slots(interval=2.0)` staggers two workers about a second apart when crawls fit their slots. Each worker skips missed slots and runs one crawl at a time. Rate depends on duration and requests per crawl. Keep per-crawl state local because workers share the spider object.
11
+ - Ordinary crawl failures are logged and the built-in scheduler proceeds to later slots. `FatalRuntimeError` stops scheduling, including when raised by a registered task. Use a one-shot run to surface an individual crawl failure while debugging.
12
+
13
+ ## Logging
14
+
15
+ - Lite uses native `structlog` with one logger per app run. `crawler.logger.info("parsed", count=3)` records a structured field. Positional `%s` formatting works. Use keyword arguments for fields and `.bind(...)` to add reusable context. Project, spider, configured instance, and worker ID are available on structured events. Console and file logs separate the timestamp, level, event, and subsequent fields with tabs.
16
+ - At `INFO`, `response` shows the HTTP method, status, session ID, session response count, and request timing. The text line ends with `worker=` followed by `url=`. `requested_at` and `responded_at` are Unix epoch milliseconds. `latency_ms` is their difference. A buffered response is timed through the body and a stream through its headers, marked with `stream=true`. Crawl failure lines also end with `worker=`. Transport exceptions carry the method and URL as traceback context.
17
+ - `scraped` records the item ID and the spider payload once after deduplication. `DEBUG` also shows `duplicate` events. Check the actual destination or delivery stats to verify delivery.
18
+ - The default sink is stderr. Set `Settings(log_file=...)`, `--log-file`, or `ZENX_LITE_LOG_FILE` for one rotating file per instance. Use distinct paths for instances and keep the configured path across restarts. File formatting and writes use a single background thread. The active file rotates daily at UTC midnight even when idle. For `feed:uk.log`, archives use `feed:uk.YYYYMMDD.log`. Set `log_backup_count`, `--log-backup-count`, or `ZENX_LITE_LOG_BACKUP_COUNT` to adjust retention. The default is one backup and zero keeps all archives. Logs drain normally at shutdown, with a one-second limit if the writer stalls. File logs retain item references until rendering, so mutation after emission can change what appears in the log.
19
+ - An embedding application can pass a bound `structlog` logger as `App(logger=...)` to own formatting, routing, and level filtering. Configure either a supplied logger or a Lite-managed file for that run. Lite leaves host-wide logger configuration alone.
20
+
21
+ ## Requests, sessions, and parsing
22
+
23
+ - `await crawler.request(url, method="GET", **kwargs)` uses the worker's `curl-cffi-patch` session. Pass request options such as `params`, `headers`, and `timeout` directly. For JSON request bodies use `json=...` by itself as the body argument.
24
+ - Buffered `Response` exposes `status`, `url`, `headers`, `body`, `text`, `json()`, `xpath()`, and `elapsed` in seconds using the response's wall-clock timing. Check `status` before parsing or emitting. Use `xpath(...).get()` or `.getall()` for HTML or XML, with `namespaces=` for namespaced XML.
25
+ - Workers reuse sessions and their cookies across crawls. A transport error, request cancellation, or status 401, 403, or 429 can retire the worker's session. Replacement occurs for a later crawl. The default manager also checks the session's fixed ten-minute age between crawls. Managed requests use zero transport retries.
26
+ - Use `Sessions(options={...}, proxies=[...])` from `zenx_lite.sessions` for explicit curl session defaults and a proxy list. The built-in default impersonation is `chrome`. Session creation rotates through the proxy list in order, preserving repeated entries in Python and YAML. `Settings(proxy=...)` or its CLI override supplies one proxy instead.
27
+ - `crawler.stream()` returns the native curl response with `status_code` and async iterators. Consume data inside its context. Exiting early aborts an unread body.
28
+
29
+ ```python
30
+ from zenx_lite import Crawler
31
+
32
+
33
+ async def count_bytes(crawler: Crawler, url: str) -> int:
34
+ total = 0
35
+ async with crawler.stream(url, timeout=10) as response:
36
+ if response.status_code != 200:
37
+ raise RuntimeError(f"HTTP {response.status_code}")
38
+ async for chunk in response.aiter_content():
39
+ total += len(chunk)
40
+ return total
41
+ ```
42
+
43
+ ## Work owned by a crawl
44
+
45
+ Use `crawler.create_task(coro)` for work that uses the crawler and should finish with the crawl. Await the returned task when its result is needed. The task group waits for registered tasks before releasing the session and cancels siblings when one fails. Keep the crawler inside its owning crawl. Independently created `asyncio` tasks remain the application's responsibility.
46
+
47
+ ## Custom runtime components
48
+
49
+ - Pass a custom `scheduler=` to `App` when the workload needs a different timing policy. Implement `run(crawl, *, concurrency, stop)`, use worker IDs from `0` through `concurrency - 1`, keep each worker single-flight, and propagate fatal errors and cancellation. The CLI interval override is specific to `Slots`.
50
+ - Pass a custom `sessions=` when session acquisition or release needs a different policy. Implement `open(concurrency=..., proxy=...)`, `acquire(worker_id)`, `release(worker_id, session)`, and `close()`. Return a ready `Session` with a compatible transport. Returning `None` makes that slot fail before spider code runs. Make release and close cancellation-cooperative and safe after partial startup.
@@ -0,0 +1,42 @@
1
+ # Deduplication and delivery
2
+
3
+ ## Emit and route items
4
+
5
+ - `App(destinations={...})` registers adapters. Select active routes with `App(routes={spider_name: [...]})` or in YAML. For Python-wired apps, an existing `spider.destinations` attribute supplies defaults where routes are unspecified. A spider with neither route selection nor defaults has no active routes. `emit()` uses every active route by default. Pass a collection such as `to=("out",)` to select a subset for an item.
6
+ - Emit an ID once with all intended routes. Lite reserves `(project_name, dedup_scope, item_id)` before preparing destination payloads, so sending the same ID again with another route will be suppressed. The default scope is the spider name. An empty active route list or `to=()` still reserves the ID.
7
+ - Built-in destination preparation reads plain item dictionaries directly when no transform is configured. Transforms, custom preparation methods, and dictionary subclasses use a private deep copy. The ID stays separate from the dictionary. Include an identifier in the data when a JSONL consumer needs it. The queue holds owned, serialized bytes. One destination's preparation or send failure leaves other destinations free to proceed.
8
+ - Queues are unbounded, independent, and best effort. Each send attempt consumes the payload, including when it fails. The ID stays reserved. Check destination logs for failed attempts and verify receipt in the downstream system.
9
+
10
+ ## Choose a store and scope
11
+
12
+ | Need | Store and settings |
13
+ | --- | --- |
14
+ | Remember recent IDs in one run | Default `MemoryStore`. Capacity defaults to 500 IDs and oldest insertions are evicted first. Use `Settings(dedup_size=...)` or an explicit `MemoryStore(capacity=...)`. The store clears between runs. `--dedup-size` and `ZENX_LITE_DEDUP_SIZE` apply to the built-in memory store only. |
15
+ | Share reservations on one host or retain them across restarts | Pass `SQLiteStore(path)` from `zenx_lite.stores.sqlite` to `App`. In YAML set `dedup.store` to `{type: sqlite, path: /var/lib/my-scraper/seen.sqlite3}`. Python and YAML accept `ttl_seconds` and `cache_size`. |
16
+
17
+ Use a persistent path on the host's local disk. Relative paths resolve from the process working directory. Each spider process opens its own connection on a worker thread. SQLite uses WAL mode with `synchronous=NORMAL`. On a cache miss, SQLite checks committed duplicates with a read and reserves unseen or expired IDs atomically. Recent reservations can be lost after a system crash or power loss.
18
+
19
+ SQLite reservations expire seven days after their successful reservation by default. Set a positive `ttl_seconds` to change the window, or set it to `None` in Python or `null` in YAML for indefinite retention. Duplicate reads leave the original deadline intact. New IDs prune expired rows in small batches.
20
+
21
+ The local FIFO holds 500 confirmed IDs by default. Hits skip SQLite and leave the insertion order intact. Cache entries stay until eviction or store close, so they can suppress duplicates beyond the SQLite TTL. Set `cache_size=0` to check SQLite every time.
22
+
23
+ Use the same file, project name, and `dedup_scope` in Python or `dedup.scope` in YAML to share IDs across instances. Distinct scopes isolate instances within one file. Store errors propagate, and a cancelled call may already have committed its reservation.
24
+
25
+ For a deliberate historical baseline, use an empty active destination list and the intended persistent namespace. Check representative reservations in the backing store and retain that namespace when enabling delivery. Use a source timestamp or another explicit baseline to establish which items are historical.
26
+
27
+ ## Pick an output or heartbeat adapter
28
+
29
+ | Adapter | Import from | Use |
30
+ | --- | --- | --- |
31
+ | `JSONLines` (`jsonl`) | `zenx_lite.destinations.jsonl` | Inspect records on stdout or append one JSON object per line to a file. Python accepts a path or a caller-owned file object, including an in-memory capture stream. YAML takes exactly one of `path` or `stream`. |
32
+ | `SourceDestination` (`synoptic_source`) | `zenx_lite.destinations.synoptic` | With the `source` extra, send to an authenticated Synoptic Source WebSocket. Supply `endpoint`, `api_key`, and `source_id`. The welcome frame must authorize the source. Lite uses the original item ID as `idempotencyKey` and the transformed dictionary as `data`. Reconnects preserve unattempted queued payloads while the runtime is running. |
33
+ | `OpticManagerDestination` (`optic_manager`) | `zenx_lite.destinations.optic_manager` | Send `post` messages to Magent's Unix socket. Supply the Manager `app_id` and optional `instance`. A record needs a nonblank `url` and can include `headline` and `published_at`. The adapter fills missing `scraped_at` in epoch milliseconds before queueing. |
34
+ | `MagentNotifier` | `zenx_lite.notifiers.magent` | Report successful crawls through Magent. Bind it with `App(notifiers={spider_name: MagentNotifier(...)})` or set `spiders.<name>.notifier` in YAML with `app_id` and optional `instance`. `notifier: null` disables an inherited notifier. The runtime queues a heartbeat after registered work and session release succeed. Each heartbeat gets one attempt independently of item delivery. |
35
+
36
+ Use the exact Manager identity for both integrations when both are enabled. They default to `/tmp/itxpmonitor.sock`; their Python constructors accept an alternate socket for local verification. Check Manager visibility when that is the requested outcome. A local post send finishes at the Magent socket, and Magent forwards events separately.
37
+
38
+ ## Extend the runtime
39
+
40
+ - A custom `DestinationAdapter` implements `open()`, `prepare(item_id, data)`, `ready()`, `send(payload)`, and `close()`. Import it from `zenx_lite.destinations`. Keep `prepare()` synchronous, free of destination I/O, and returning immutable bytes. Use `ready()` for connection readiness and `send()` for a single attempt. Make cleanup safe after partial startup and cooperative on cancellation.
41
+ - A custom `Store` implements `open()`, atomic `add(key)` returning true on first reservation, and `close()`. Preserve tuple boundaries, propagate errors, and use an explicit store policy.
42
+ - A custom notifier implements `open()`, nonblocking `notify()`, and `close()`. Keep telemetry I/O off the crawl and delivery paths. Verify success and failure against a local socket or a fake adapter before enabling external delivery.
@@ -0,0 +1,57 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+ workflow_dispatch:
7
+ inputs:
8
+ tag:
9
+ description: Release tag to publish
10
+ required: true
11
+ type: string
12
+
13
+ permissions:
14
+ contents: read
15
+
16
+ jobs:
17
+ publish:
18
+ runs-on: ubuntu-latest
19
+ env:
20
+ RELEASE_TAG: ${{ github.event.release.tag_name || inputs.tag }}
21
+ PYTHONDONTWRITEBYTECODE: "1"
22
+ steps:
23
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
24
+ with:
25
+ ref: ${{ env.RELEASE_TAG }}
26
+ persist-credentials: false
27
+
28
+ - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
29
+ with:
30
+ version: "0.13.9"
31
+ python-version: "3.15.2"
32
+
33
+ - name: Check release version
34
+ run: |
35
+ uv run --no-project python - <<'PY'
36
+ import os
37
+ import tomllib
38
+
39
+ with open("pyproject.toml", "rb") as metadata:
40
+ version = tomllib.load(metadata)["project"]["version"]
41
+ tag = os.environ["RELEASE_TAG"].removeprefix("v")
42
+ if tag != version:
43
+ raise SystemExit(f"Release tag {tag!r} does not match package version {version!r}")
44
+ PY
45
+
46
+ - name: Run tests
47
+ env:
48
+ UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/venv
49
+ run: uv run --locked --all-extras pytest -q -p no:cacheprovider
50
+
51
+ - name: Build distributions
52
+ run: uv build --out-dir "$RUNNER_TEMP/dist"
53
+
54
+ - name: Publish distributions
55
+ env:
56
+ UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
57
+ run: uv publish --trusted-publishing never "$RUNNER_TEMP"/dist/*
@@ -0,0 +1,8 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+ *.egg-info/
6
+ build/
7
+ dist/
8
+ /benchmarks/
@@ -0,0 +1,28 @@
1
+ # Working on ZenX Lite
2
+
3
+ Build a focused Python scraping runtime that is explicit, composable, and easy to operate. Existing ZenX is a reference, not a compatibility requirement.
4
+
5
+ ## Design guidance
6
+
7
+ - Read and follow [PROJECT_GUIDE.md](PROJECT_GUIDE.md) for all development.
8
+ - Use the implementation and tests to verify current runtime and API behavior. Preserve ownership, cancellation, delivery, dedup, and shutdown guarantees.
9
+ - Distinguish routine implementation choices from changes to agreed behavior. Keep code, tests, and the project guide aligned. Discussion or a question alone does not authorize a design change.
10
+ - Review the [ZenX Lite skill](.agents/skills/zenx-lite/SKILL.md) and its relevant references for every implementation change. Update them in the same change when public APIs, configuration, defaults, or runtime behavior change. Run affected examples against the updated implementation.
11
+
12
+ ## Implementation
13
+
14
+ - Prefer standard `asyncio`, dataclasses, protocols, and argparse. Use native `structlog` for runtime logging. Preserve native task and exception semantics.
15
+ - Use consistent type hints across framework APIs and internals. Application annotations remain optional; type hints do not impose payload schemas.
16
+ - Keep configuration and integrations explicit and optional. Avoid discovery, global registration, and implicit environment-dependent delivery behavior.
17
+ - Keep notifier I/O off crawl/delivery paths. Report crawl success only after registered tasks and session release succeed; keep Manager identity in the optional adapter. Use one sender with an unbounded queue and one attempt per notification, without retry.
18
+ - Keep process supervision and deployment orchestration outside the runtime. Add abstractions for concrete shared needs rather than speculative features.
19
+ - Consult the actual source at `~/synoptic/ZenX` when behavior depends on existing ZenX internals.
20
+
21
+ ## Verification and repository hygiene
22
+
23
+ - Validate uncertain assumptions with a standalone script or Python shell before modifying project code. Remove temporary scripts when finished.
24
+ - Prefer `uv` for dependencies, environments, and Python tooling. Run `uv run --all-extras pytest -q` for the suite and `uv build` for package verification when relevant. Use temporary SQLite files for store tests.
25
+ - Use focused tests for ownership, cancellation, partial failures, and extension boundaries. Use local services and fake transports before live integrations.
26
+ - Support performance claims with measurements, including multiple scraper processes where relevant.
27
+ - Preserve existing user work and respect `.gitignore`. Do not add, commit, or rely on ignored files without explicit instruction. Remove unnecessary generated files. Do not create a `README.md` unless requested.
28
+ - Keep commit messages and project documentation free of AI/tool attribution. Report changes, verification, and material limitations without unsupported claims.
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.5
2
+ Name: zenx-lite
3
+ Version: 0.2.0
4
+ Summary: An explicitly composed asyncio scraping runtime
5
+ Requires-Python: >=3.15
6
+ Requires-Dist: curl-cffi-patch<0.17,>=0.16.4
7
+ Requires-Dist: orjson>=3.10
8
+ Requires-Dist: parsel>=1.12.1
9
+ Requires-Dist: pyyaml<7,>=6
10
+ Requires-Dist: structlog<27,>=26.1
11
+ Provides-Extra: source
12
+ Requires-Dist: websockets<18,>=15; extra == 'source'
13
+ Description-Content-Type: text/markdown
14
+
15
+ # ZenX Lite
16
+
17
+ ZenX Lite is a Python scraping runtime. Your spiders fetch and parse data. The runtime handles scheduling, HTTP sessions, deduplication, and delivery.
18
+
19
+ ## Quickstart
20
+
21
+ 1. **Create a project with [uv](https://docs.astral.sh/uv/).** ZenX Lite requires Python 3.15 or newer.
22
+
23
+ ```bash
24
+ uv init --bare --python 3.15.2 example-scraper
25
+ cd example-scraper
26
+ uv add zenx-lite
27
+ ```
28
+
29
+ 2. **Add the spider.** Create `spiders/` with an empty `__init__.py`. Save this as `spiders/example.py`.
30
+
31
+ ```python
32
+ from zenx_lite import Crawler
33
+
34
+
35
+ class ExampleSpider:
36
+ name = "example"
37
+ destinations = ("stdout",)
38
+
39
+ async def crawl(self, crawler: Crawler) -> None:
40
+ response = await crawler.request("https://example.com/", timeout=10)
41
+ if response.status != 200:
42
+ raise RuntimeError(f"HTTP {response.status}")
43
+ title = response.xpath("//title/text()").get()
44
+ await crawler.emit(response.url, {"url": response.url, "title": title})
45
+ ```
46
+
47
+ 3. **Set up the application.** Save this as `main.py`.
48
+
49
+ ```python
50
+ import sys
51
+
52
+ from zenx_lite import App
53
+ from zenx_lite.destinations.jsonl import JSONLines
54
+
55
+ from spiders.example import ExampleSpider
56
+
57
+
58
+ if __name__ == "__main__":
59
+ App(
60
+ project_name="example-scraper",
61
+ spiders=[ExampleSpider()],
62
+ destinations={"stdout": JSONLines(sys.stdout)},
63
+ ).cli()
64
+ ```
65
+
66
+ 4. **Run one crawl and exit.**
67
+
68
+ ```bash
69
+ uv run main.py --once
70
+ ```
71
+
72
+ ```json
73
+ {"url":"https://example.com/","title":"Example Domain"}
74
+ ```
75
+
76
+ Use `--log-level DEBUG` to increase log verbosity. [Logging options](.agents/skills/zenx-lite/references/crawl-and-runtime.md#logging)
77
+
78
+ ## Recommended project structure
79
+
80
+ Put the app setup in `main.py` and the crawl logic in `spiders/`.
81
+
82
+ ```text
83
+ my-scraper/
84
+ ├── .gitignore
85
+ ├── pyproject.toml
86
+ ├── uv.lock
87
+ ├── main.py
88
+ └── spiders/
89
+ ├── __init__.py
90
+ └── example.py
91
+ ```
92
+
93
+ Commit `uv.lock` for reproducible installs. For YAML configuration, register your spider factories explicitly with `run_from_config()` in `main.py`. [Configuration examples](.agents/skills/zenx-lite/references/configuration-and-instances.md#standard-project-structure)
@@ -0,0 +1,18 @@
1
+ # ZenX Lite design principles
2
+
3
+ Build a Python scraping runtime that is easy to understand, compose, and operate, with minimal delay from discovering new data to sending it.
4
+
5
+ - **Keep composition explicit.** Use ordinary Python objects, direct references, and small interfaces. Make dependencies and execution easy to trace.
6
+ - **Keep the core focused, but useful.** Add capabilities when concrete shared needs justify them. Smallness is a means to clarity, not a reason to force applications into workarounds.
7
+ - **Make ownership clear.** The runtime owns workers, managed sessions, registered crawl tasks, and delivery resources. Applications own business logic and their independently created resources.
8
+ - **Separate policy from execution.** Scheduling and integrations should be replaceable without changing resource ownership or delivery guarantees. Keep process supervision and deployment orchestration outside the crawling core.
9
+ - **Prioritize prompt delivery.** Send without deliberate batching, capacity waits, or artificial delays. Use unbounded queues unless explicitly requested otherwise. Preserve event-loop fairness and keep blocking work out of the admission path.
10
+ - **Make failure semantics predictable.** Use native task and cancellation behavior. Isolate destination failures; delivery remains best effort, without retrying or replaying attempted payloads. Transport-buffered data may be discarded on close; do not add flush waits unless explicitly requested. Remembering an ID does not prove delivery.
11
+ - **Keep application data independent.** Do not inject framework metadata or business rules into payloads. Keep identity, routing, and destination envelopes separate, and protect queued data from later mutation.
12
+ - **Treat operations as a real requirement.** Configuration, monitoring, and deployment support should be consistent and inspectable. Add shared support where needed while keeping integrations optional and environment behavior explicit.
13
+ - **Keep logging scoped to a run.** Emit structured request and item events. Keep file formatting, writes, and rotation on the log writer thread, with bounded draining at shutdown.
14
+ - **Measure before optimizing.** Validate latency, resource use, and correctness with representative workloads, including multiple scraper processes. Simpler architecture alone does not prove better performance.
15
+ - **Evolve deliberately.** Learn from ZenX without requiring compatibility. Distinguish proposals from approved behavior and keep implementation, tests, and documentation aligned.
16
+ - **Clean up as you go.** Before finishing a change, check the codebase for unused code, obsolete files, and stale references. Clean up confirmed leftovers.
17
+
18
+ The implementation and tests define current behavior.
@@ -0,0 +1,79 @@
1
+ # ZenX Lite
2
+
3
+ ZenX Lite is a Python scraping runtime. Your spiders fetch and parse data. The runtime handles scheduling, HTTP sessions, deduplication, and delivery.
4
+
5
+ ## Quickstart
6
+
7
+ 1. **Create a project with [uv](https://docs.astral.sh/uv/).** ZenX Lite requires Python 3.15 or newer.
8
+
9
+ ```bash
10
+ uv init --bare --python 3.15.2 example-scraper
11
+ cd example-scraper
12
+ uv add zenx-lite
13
+ ```
14
+
15
+ 2. **Add the spider.** Create `spiders/` with an empty `__init__.py`. Save this as `spiders/example.py`.
16
+
17
+ ```python
18
+ from zenx_lite import Crawler
19
+
20
+
21
+ class ExampleSpider:
22
+ name = "example"
23
+ destinations = ("stdout",)
24
+
25
+ async def crawl(self, crawler: Crawler) -> None:
26
+ response = await crawler.request("https://example.com/", timeout=10)
27
+ if response.status != 200:
28
+ raise RuntimeError(f"HTTP {response.status}")
29
+ title = response.xpath("//title/text()").get()
30
+ await crawler.emit(response.url, {"url": response.url, "title": title})
31
+ ```
32
+
33
+ 3. **Set up the application.** Save this as `main.py`.
34
+
35
+ ```python
36
+ import sys
37
+
38
+ from zenx_lite import App
39
+ from zenx_lite.destinations.jsonl import JSONLines
40
+
41
+ from spiders.example import ExampleSpider
42
+
43
+
44
+ if __name__ == "__main__":
45
+ App(
46
+ project_name="example-scraper",
47
+ spiders=[ExampleSpider()],
48
+ destinations={"stdout": JSONLines(sys.stdout)},
49
+ ).cli()
50
+ ```
51
+
52
+ 4. **Run one crawl and exit.**
53
+
54
+ ```bash
55
+ uv run main.py --once
56
+ ```
57
+
58
+ ```json
59
+ {"url":"https://example.com/","title":"Example Domain"}
60
+ ```
61
+
62
+ Use `--log-level DEBUG` to increase log verbosity. [Logging options](.agents/skills/zenx-lite/references/crawl-and-runtime.md#logging)
63
+
64
+ ## Recommended project structure
65
+
66
+ Put the app setup in `main.py` and the crawl logic in `spiders/`.
67
+
68
+ ```text
69
+ my-scraper/
70
+ ├── .gitignore
71
+ ├── pyproject.toml
72
+ ├── uv.lock
73
+ ├── main.py
74
+ └── spiders/
75
+ ├── __init__.py
76
+ └── example.py
77
+ ```
78
+
79
+ Commit `uv.lock` for reproducible installs. For YAML configuration, register your spider factories explicitly with `run_from_config()` in `main.py`. [Configuration examples](.agents/skills/zenx-lite/references/configuration-and-instances.md#standard-project-structure)
@@ -0,0 +1,31 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.26"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "zenx-lite"
7
+ version = "0.2.0"
8
+ description = "An explicitly composed asyncio scraping runtime"
9
+ readme = "README.md"
10
+ requires-python = ">=3.15"
11
+ dependencies = [
12
+ "curl-cffi-patch>=0.16.4,<0.17",
13
+ "orjson>=3.10",
14
+ "parsel>=1.12.1",
15
+ "pyyaml>=6,<7",
16
+ "structlog>=26.1,<27",
17
+ ]
18
+
19
+ [project.optional-dependencies]
20
+ source = ["websockets>=15,<18"]
21
+
22
+ [dependency-groups]
23
+ dev = ["pytest>=8", "pytest-asyncio>=0.25"]
24
+
25
+ [tool.hatch.build.targets.wheel]
26
+ packages = ["zenx_lite"]
27
+
28
+ [tool.pytest.ini_options]
29
+ asyncio_mode = "auto"
30
+ asyncio_default_fixture_loop_scope = "function"
31
+ testpaths = ["tests"]
File without changes
@@ -0,0 +1,62 @@
1
+ import sys
2
+
3
+ from zenx_lite import App, Crawler, Spider
4
+ from zenx_lite.destinations.jsonl import JSONLines
5
+
6
+
7
+ class Feed:
8
+ name = "feed"
9
+ destinations = ("out",)
10
+
11
+ def __init__(self, url: str) -> None:
12
+ self.url = url
13
+
14
+ async def crawl(self, crawler: Crawler) -> None:
15
+ response = await crawler.request(self.url)
16
+ for item in response.json()["items"]:
17
+ await crawler.emit(str(item["id"]), item)
18
+
19
+
20
+ class Search:
21
+ name = "search"
22
+ destinations = ("out",)
23
+
24
+ def __init__(self, query: str, limit: int) -> None:
25
+ self.query = query
26
+ self.limit = limit
27
+
28
+ async def crawl(self, crawler: Crawler) -> None:
29
+ response = await crawler.request(
30
+ "https://example.com/search.json",
31
+ params={"q": self.query, "limit": self.limit},
32
+ )
33
+ for item in response.json()["items"]:
34
+ await crawler.emit(str(item["id"]), item)
35
+
36
+
37
+ if __name__ == "__main__":
38
+ parser = App.cli_parser(description="Run a feed or search spider.")
39
+ parser.add_argument("--feed-url", help="Feed URL; required for feed")
40
+ parser.add_argument("--query", help="Search query; required for search")
41
+ parser.add_argument("--limit", type=int, default=50, help="Search result limit")
42
+ args = parser.parse_args()
43
+
44
+ selected: Spider
45
+ if args.spider == "feed":
46
+ if not args.feed_url:
47
+ parser.error("--feed-url is required for feed")
48
+ selected = Feed(args.feed_url)
49
+ elif args.spider == "search":
50
+ if not args.query:
51
+ parser.error("--query is required for search")
52
+ if args.limit <= 0:
53
+ parser.error("--limit must be positive")
54
+ selected = Search(args.query, args.limit)
55
+ else:
56
+ parser.error("--spider must be feed or search")
57
+
58
+ App(
59
+ project_name="news",
60
+ spiders=[selected],
61
+ destinations={"out": JSONLines(sys.stdout)},
62
+ ).cli(args=args, parser=parser)