jigkit 0.1.0__tar.gz → 0.2.0__tar.gz

{jigkit-0.1.0 → jigkit-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: jigkit
-Version: 0.1.0
+Version: 0.2.0
 Summary: The shared foundations of small self-hosted web apps: settings, database, migrations, logging, health checks and test fixtures
 Project-URL: Homepage, https://github.com/jmerhar/jigkit
 Project-URL: Changelog, https://github.com/jmerhar/jigkit/blob/main/CHANGELOG.md
@@ -26,12 +26,17 @@ Provides-Extra: dev
 Requires-Dist: aiosqlite>=0.22; extra == 'dev'
 Requires-Dist: alembic>=1.19; extra == 'dev'
 Requires-Dist: build>=1.3; extra == 'dev'
+Requires-Dist: httpx2>=2.13; extra == 'dev'
 Requires-Dist: pytest-asyncio>=1.4; extra == 'dev'
 Requires-Dist: pytest-cov>=7.1; extra == 'dev'
 Requires-Dist: pytest>=9.1; extra == 'dev'
 Requires-Dist: ruff>=0.16; extra == 'dev'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.52; extra == 'dev'
 Requires-Dist: twine>=6.2; extra == 'dev'
+Provides-Extra: heartbeat
+Requires-Dist: httpx2>=2.13; extra == 'heartbeat'
+Provides-Extra: influx
+Requires-Dist: httpx2>=2.13; extra == 'influx'
 Provides-Extra: postgres
 Requires-Dist: aiosqlite>=0.22; extra == 'postgres'
 Requires-Dist: alembic>=1.19; extra == 'postgres'
@@ -61,7 +66,11 @@ pip install "jigkit[db]"
 | `jigkit.logging` | — | stream and rotating file handlers, a level that changes at runtime, credential-carrying HTTP client loggers kept quiet |
 | `jigkit.healthcheck` | — | `python -m jigkit.healthcheck URL --expect database=connected`, a container health probe |
 | `jigkit.secrets` | — | `load_or_create_secret`, a generated secret kept owner-only in the data directory |
-| `jigkit.testing` | `testing` | a pytest plugin (`data_dir`, `no_network`) and migration checks run through the application's own `env.py` |
+| `jigkit.sinks` | — | `Point` (timestamp with an explicit precision), `Outcome`, the `Sink` protocol, the neutral spooled form |
+| `jigkit.spool`, `jigkit.delivery` | — | a bounded on-disk queue per destination, and write-now-or-spool-and-replay |
+| `jigkit.influx` | `influx` | InfluxDB 3 and 2.x sinks: line protocol in each batch's precision, and SQL reads from InfluxDB 3 |
+| `jigkit.heartbeat` | `heartbeat` | a push-monitor heartbeat after delivered writes, at most once per interval |
+| `jigkit.testing` | `testing` | a pytest plugin (`data_dir`, `no_network`), migration checks run through the application's own `env.py`, and a recording stand-in HTTP server |
 
 PostgreSQL is supported through the `postgres` extra. Python 3.14 or newer.
 

{jigkit-0.1.0 → jigkit-0.2.0}/README.md

@@ -17,7 +17,11 @@ pip install "jigkit[db]"
 | `jigkit.logging` | — | stream and rotating file handlers, a level that changes at runtime, credential-carrying HTTP client loggers kept quiet |
 | `jigkit.healthcheck` | — | `python -m jigkit.healthcheck URL --expect database=connected`, a container health probe |
 | `jigkit.secrets` | — | `load_or_create_secret`, a generated secret kept owner-only in the data directory |
-| `jigkit.testing` | `testing` | a pytest plugin (`data_dir`, `no_network`) and migration checks run through the application's own `env.py` |
+| `jigkit.sinks` | — | `Point` (timestamp with an explicit precision), `Outcome`, the `Sink` protocol, the neutral spooled form |
+| `jigkit.spool`, `jigkit.delivery` | — | a bounded on-disk queue per destination, and write-now-or-spool-and-replay |
+| `jigkit.influx` | `influx` | InfluxDB 3 and 2.x sinks: line protocol in each batch's precision, and SQL reads from InfluxDB 3 |
+| `jigkit.heartbeat` | `heartbeat` | a push-monitor heartbeat after delivered writes, at most once per interval |
+| `jigkit.testing` | `testing` | a pytest plugin (`data_dir`, `no_network`), migration checks run through the application's own `env.py`, and a recording stand-in HTTP server |
 
 PostgreSQL is supported through the `postgres` extra. Python 3.14 or newer.
 

{jigkit-0.1.0 → jigkit-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "jigkit"
-version = "0.1.0"
+version = "0.2.0"
 description = "The shared foundations of small self-hosted web apps: settings, database, migrations, logging, health checks and test fixtures"
 readme = "README.md"
 requires-python = ">=3.14"
@@ -39,6 +39,14 @@ postgres = [
     "jigkit[db]",
     "asyncpg>=0.30",
 ]
+# InfluxDB sinks: line protocol to InfluxDB 3 and 2.x, and SQL reads from InfluxDB 3.
+influx = [
+    "httpx2>=2.13",
+]
+# The push-monitor heartbeat.
+heartbeat = [
+    "httpx2>=2.13",
+]
 # The pytest plugin: fixtures and migration checks for applications built on jigkit.
 testing = [
     "pytest>=9.1",
@@ -46,7 +54,7 @@ testing = [
 ]
 # Everything this repository's own checks need.
 dev = [
-    "jigkit[db,testing]",
+    "jigkit[db,heartbeat,influx,testing]",
     "pytest-cov>=7.1",
     "ruff>=0.16",
     "build>=1.3",

jigkit-0.2.0/src/jigkit/delivery.py

@@ -0,0 +1,156 @@
+"""Getting points into a sink eventually: write now if possible, spool and replay if not.
+
+A batch is written straight away while nothing is waiting. Once anything is, new batches join the
+end of the queue instead, and one background loop drains it oldest first with growing pauses
+between failed attempts -- so an outage costs one retry per pause rather than one per batch, and
+the backlog lands in the order it arrived.
+
+One :class:`Delivery` serves one sink with its own :class:`~jigkit.spool.Spool`; an application
+writing to several destinations runs one per destination, so a failing destination holds back
+only its own backlog.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Protocol
+
+from jigkit.sinks import Outcome, Point, dump_points, load_points
+from jigkit.spool import Spool
+
+logger = logging.getLogger(__name__)
+
+FIRST_PAUSE_SECONDS = 1.0
+MAX_PAUSE_SECONDS = 60.0
+
+
+class Writer(Protocol):
+    """Something that attempts one write and says what became of it -- a sink, in practice."""
+
+    async def write(self, points: list[Point]) -> Outcome:
+        """Write points once."""
+
+
+class DeliveryEvents(Protocol):
+    """What a delivery reports as it works, for an application's status page to count."""
+
+    def record_write(self, ok: bool) -> None:
+        """A write was attempted; ``ok`` says whether the sink accepted it."""
+
+    def record_spooled(self) -> None:
+        """A batch was queued to be written later."""
+
+    def record_rejected_write(self) -> None:
+        """The sink refused a batch for good."""
+
+
+class _NoEvents:
+    """The events of an application that counts nothing."""
+
+    def record_write(self, ok: bool) -> None:
+        """Ignore a write."""
+
+    def record_spooled(self) -> None:
+        """Ignore a spooled batch."""
+
+    def record_rejected_write(self) -> None:
+        """Ignore a rejected batch."""
+
+
+class Delivery:
+    """Writes batches, spooling the ones that cannot be written yet.
+
+    ``on_written`` is called with the points of every write the sink accepts, live or replayed.
+    """
+
+    def __init__(
+        self,
+        sink: Writer,
+        spool: Spool,
+        *,
+        events: DeliveryEvents | None = None,
+        sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+        on_written: Callable[[list[Point]], None] = lambda _: None,
+    ) -> None:
+        self._sink = sink
+        self._spool = spool
+        self._events = events or _NoEvents()
+        self._sleep = sleep
+        self._on_written = on_written
+        self._wake = asyncio.Event()
+
+    async def submit(self, points: list[Point]) -> None:
+        """Deliver one batch now, or queue it for the replay loop."""
+        if not points:
+            return
+        if len(self._spool):
+            self._queue(points)
+            return
+        outcome = await self._sink.write(points)
+        self._events.record_write(outcome is Outcome.OK)
+        if outcome is Outcome.OK:
+            self._on_written(points)
+        elif outcome is Outcome.RETRY:
+            self._queue(points)
+        else:
+            self._events.record_rejected_write()
+            self._spool.quarantine_body(dump_points(points))
+
+    async def run(self) -> None:
+        """Replay spooled batches for ever, oldest first. Cancel the task to stop it.
+
+        An unexpected error is logged and waited out like a failed write, never allowed to end
+        the loop: with it gone, every later batch would queue with nothing to drain it.
+        """
+        pause = FIRST_PAUSE_SECONDS
+        while True:
+            try:
+                pause = await self._step(pause)
+            except Exception:
+                logger.exception("replaying the spool failed; trying again in %.0f s", pause)
+                await self._sleep(pause)
+                pause = min(pause * 2, MAX_PAUSE_SECONDS)
+
+    async def _step(self, pause: float) -> float:
+        """Replay the oldest batch, or wait for one; return the pause the next failure takes."""
+        path = self._spool.oldest()
+        if path is None:
+            self._wake.clear()
+            await self._wake.wait()
+            return FIRST_PAUSE_SECONDS
+        try:
+            points = load_points(self._spool.read(path))
+        except OSError, UnicodeDecodeError:
+            logger.exception("cannot read spooled batch %s; setting it aside", path.name)
+            self._spool.quarantine(path)
+            return pause
+        except ValueError:
+            # Not points -- a format an older version spooled, or a file edited by hand -- so no
+            # sink can be given it.
+            logger.error("spooled batch %s is not points; setting it aside", path.name)
+            self._spool.quarantine(path)
+            return pause
+
+        outcome = await self._sink.write(points)
+        self._events.record_write(outcome is Outcome.OK)
+        if outcome is Outcome.OK:
+            self._spool.ack(path)
+            self._on_written(points)
+            pause = FIRST_PAUSE_SECONDS
+            if not len(self._spool):
+                logger.info("spool drained")
+        elif outcome is Outcome.REJECT:
+            self._events.record_rejected_write()
+            self._spool.quarantine(path)
+        else:
+            await self._sleep(pause)
+            pause = min(pause * 2, MAX_PAUSE_SECONDS)
+        return pause
+
+    def _queue(self, points: list[Point]) -> None:
+        """Add a batch to the spool and wake the replay loop."""
+        if self._spool.enqueue(dump_points(points)):
+            self._events.record_spooled()
+        self._wake.set()

jigkit-0.2.0/src/jigkit/heartbeat.py

@@ -0,0 +1,96 @@
+"""Tell a push monitor that data is arriving (extra ``heartbeat``).
+
+A push monitor -- Uptime Kuma's, healthchecks.io's -- alerts when its URL stops being called.
+Calling it after each write a sink accepts makes one monitor cover the whole chain: a source that
+stops sending, an unreachable server and a sink that refuses or cannot take writes all stop the
+calls alike, while a batch that is merely spooled does not count as delivered.
+
+The calls run on their own task, at most once per interval, so a replayed backlog costs one call
+rather than one per batch and a slow monitor never holds up the work. A push URL carries its
+monitor's token, so read it from a secret (``HEARTBEAT_URL_FILE``) and never log more than its
+host.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from urllib.parse import urlsplit
+
+import httpx2
+
+logger = logging.getLogger(__name__)
+
+TIMEOUT_SECONDS = 10.0
+
+
+class Heartbeat:
+    """Calls a URL after each :meth:`beat`, no more often than once per interval.
+
+    ``on_result`` is called with whether each call got through, for a status page to show.
+    """
+
+    def __init__(
+        self,
+        client: httpx2.AsyncClient,
+        url: str,
+        *,
+        interval_seconds: float,
+        sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+        on_result: Callable[[bool], None] = lambda _: None,
+    ) -> None:
+        self._client = client
+        self._url = url
+        self._interval = interval_seconds
+        self._sleep = sleep
+        self._on_result = on_result
+        self._due = asyncio.Event()
+        self._failing = False
+
+    @property
+    def host(self) -> str:
+        """Where the calls go, without the path or any credentials in the URL.
+
+        Push URLs carry their monitor's token in the path, and some carry a login, so only the
+        host is ever logged.
+        """
+        return urlsplit(self._url).hostname or "?"
+
+    def beat(self) -> None:
+        """Note that data was delivered; the next call goes out as soon as one may."""
+        self._due.set()
+
+    async def run(self) -> None:
+        """Call the URL for each beat, an interval apart at most. Cancel the task to stop it.
+
+        An unexpected error is logged and the loop carries on: ending it would stop the calls,
+        and the monitor would report data missing that is arriving.
+        """
+        while True:
+            await self._due.wait()
+            self._due.clear()
+            try:
+                await self.call()
+            except Exception:
+                logger.exception("heartbeat to %s failed unexpectedly", self.host)
+            await self._sleep(self._interval)
+
+    async def call(self) -> bool:
+        """Call the URL once, returning whether the monitor accepted it.
+
+        A failure is logged when calls start failing and again when they recover, not on every
+        attempt: the monitor itself raises the alarm, and the log only has to explain it.
+        """
+        try:
+            response = await self._client.get(self._url, timeout=TIMEOUT_SECONDS)
+            ok, reason = response.is_success, f"HTTP {response.status_code}"
+        except httpx2.RequestError as error:
+            ok, reason = False, type(error).__name__
+        if not ok and not self._failing:
+            logger.warning("heartbeat to %s failed: %s", self.host, reason)
+        elif ok and self._failing:
+            logger.info("heartbeat to %s is getting through again", self.host)
+        self._failing = not ok
+        self._on_result(ok)
+        return ok

jigkit-0.2.0/src/jigkit/influx/__init__.py

@@ -0,0 +1,31 @@
+"""InfluxDB sinks (extra ``influx``): line protocol over HTTP to InfluxDB 3 and 2.x.
+
+:class:`Influx3Sink` also reads back, with SQL whose table and field names are checked against
+the database's own catalogue.
+"""
+
+from jigkit.influx.lineprotocol import encode
+from jigkit.influx.sink import (
+    Influx2Sink,
+    Influx3Sink,
+    InfluxSink,
+    QueryError,
+    TableInfo,
+    TableNotFound,
+    classify,
+    parse_time,
+    sql_timestamp,
+)
+
+__all__ = [
+    "Influx2Sink",
+    "Influx3Sink",
+    "InfluxSink",
+    "QueryError",
+    "TableInfo",
+    "TableNotFound",
+    "classify",
+    "encode",
+    "parse_time",
+    "sql_timestamp",
+]

jigkit-0.2.0/src/jigkit/influx/lineprotocol.py

@@ -0,0 +1,76 @@
+"""Encode points as InfluxDB line protocol, which InfluxDB 3 and 2.x both accept."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+
+from jigkit.sinks import FieldValue, Point, Precision
+
+
+def encode(points: Sequence[Point]) -> tuple[str, Precision]:
+    """Encode points as newline-separated line protocol, and the precision to write it at.
+
+    A write states one precision for all its lines. Points that share one are written in it;
+    a batch that mixes them is written in the finest it contains, so no timestamp loses
+    anything. Points with no field line protocol can carry are left out.
+    """
+    precisions = {point.precision for point in points}
+    precision = max(precisions, key=lambda p: p.per_second) if precisions else Precision.SECONDS
+    lines = [line for point in points if (line := encode_point(point, precision))]
+    return "\n".join(lines), precision
+
+
+def encode_point(point: Point, precision: Precision) -> str:
+    """Encode one point at ``precision``, or return "" if none of its fields can be written."""
+    fields = ",".join(
+        f"{_escape_key(name)}={encoded}"
+        for name, value in point.fields.items()
+        if (encoded := _value(value)) is not None
+    )
+    if not fields:
+        return ""
+    # Line protocol forbids an empty tag value, so such a tag is left off rather than written.
+    tags = "".join(
+        f",{_escape_key(key)}={_escape_key(value)}" for key, value in point.tags if value
+    )
+    return f"{_escape_measurement(point.table)}{tags} {fields} {point.timestamp_in(precision)}"
+
+
+def _value(value: FieldValue) -> str | None:
+    """Encode a field value, or None for one line protocol cannot represent.
+
+    A float is written as one even when it has no fraction: InfluxDB fixes a field's type on its
+    first write, so a humidity first seen as ``60`` would otherwise reject every later ``60.5``.
+    An int is written as an integer.
+    """
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return f"{value}i"
+    if isinstance(value, float):
+        return repr(value) if math.isfinite(value) else None
+    escaped = _flatten(value).replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def _flatten(text: str) -> str:
+    """Replace line breaks with spaces: line protocol has no escape for them, and one would end
+    the line mid-point, making InfluxDB refuse the whole write."""
+    return text.replace("\r\n", " ").replace("\n", " ").replace("\r", " ")
+
+
+def _escape_measurement(name: str) -> str:
+    """Escape a measurement name: commas and spaces would end it early."""
+    return _flatten(name).replace("\\", "\\\\").replace(",", "\\,").replace(" ", "\\ ")
+
+
+def _escape_key(name: str) -> str:
+    """Escape a tag key, tag value or field key: commas, equals signs and spaces."""
+    return (
+        _flatten(name)
+        .replace("\\", "\\\\")
+        .replace(",", "\\,")
+        .replace("=", "\\=")
+        .replace(" ", "\\ ")
+    )