log-foundry 0.0.2.dev1__py3-none-any.whl

log_forge/sinks/sqlite.py

@@ -0,0 +1,100 @@
+"""SQLiteSink — persist events as queryable rows in an embedded SQLite database (arch §8, SPEC-008).
+
+For local dev, debugging, air-gapped hosts, or simple archival, an embedded SQLite file is a durable
+sink you can open later and query with plain SQL. Each event is stored as its full JSON plus a few
+extracted columns (identity + level + function) projected out for cheap filtering. Like every sink it
+receives *already-built* event dicts and knows nothing about spans or context.
+
+Standard library only (``sqlite3``) — no runtime dependency. A single-process, single-worker-thread
+writer is assumed (arch §9); concurrent/cross-process writers and external databases (Postgres,
+ClickHouse, …) are out of scope here (the latter is SPEC-011).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sqlite3
+
+__all__ = ["SQLiteSink"]
+
+# Columns projected out of each event for cheap SQL filtering. ``event`` (full JSON) is stored
+# alongside these and is the source of truth; these are convenience projections (NULL if absent).
+_COLUMNS = ("log_id", "trace_id", "span_id", "timestamp", "level", "function")
+
+# A table name is a config value, not untrusted input, but it is interpolated into DDL/DML (SQLite
+# cannot parameterize identifiers), so restrict it to a plain SQL identifier to foreclose injection.
+_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+class SQLiteSink:
+    """A :class:`~log_forge.sinks.base.Sink` that batch-inserts events into a SQLite table.
+
+    By default (``create_table=True``) the sink owns its schema: it runs an idempotent
+    ``CREATE TABLE IF NOT EXISTS`` so it works out of the box against a fresh database file. Pass
+    ``create_table=False`` when the caller provisions the table themselves (e.g. via migrations); the
+    sink then runs no DDL and a missing/incompatible table surfaces as a normal ``sqlite3`` error at
+    insert time.
+
+    A ``connection`` may be injected (e.g. ``sqlite3.connect(":memory:")``) so tests never touch
+    disk. An injected connection is borrowed — committed but never closed by this sink; a connection
+    the sink opens itself is owned and closed on ``close()``.
+    """
+
+    def __init__(
+        self,
+        database: str,
+        *,
+        table: str = "log_events",
+        connection: sqlite3.Connection | None = None,
+        create_table: bool = True,
+    ) -> None:
+        if not _IDENTIFIER.match(table):
+            raise ValueError(f"invalid table name {table!r}; expected a plain SQL identifier")
+        self._table = table
+        self._owns_connection = connection is None
+        # check_same_thread=False: the background worker (a different thread than the one that ran
+        # configure()) is the sole writer, so SQLite's same-thread guard would only get in the way
+        # (arch §9 single-writer assumption; concurrent writers are out of scope, SPEC-008).
+        self._conn = (
+            connection
+            if connection is not None
+            else sqlite3.connect(database, check_same_thread=False)
+        )
+        self._closed = False
+        if create_table:
+            self._ensure_schema()
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Insert every event in one transaction; extracted columns default to NULL (FR-003)."""
+        rows = [
+            (*(event.get(col) for col in _COLUMNS), json.dumps(event)) for event in batch
+        ]
+        placeholders = ", ".join("?" * (len(_COLUMNS) + 1))
+        columns = ", ".join((*_COLUMNS, "event"))
+        # ``with connection`` opens a transaction and commits on success / rolls back on error, so
+        # the whole batch lands atomically.
+        with self._conn:
+            self._conn.executemany(
+                f'INSERT INTO "{self._table}" ({columns}) VALUES ({placeholders})', rows
+            )
+
+    def close(self) -> None:
+        """Commit pending work; close only a connection the sink owns. Idempotent (FR-003)."""
+        if self._closed:
+            return
+        self._conn.commit()
+        if self._owns_connection:
+            self._conn.close()
+        self._closed = True
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _ensure_schema(self) -> None:
+        """Idempotently create the target table (``CREATE TABLE IF NOT EXISTS``)."""
+        columns = ", ".join(f"{col} TEXT" for col in _COLUMNS)
+        with self._conn:
+            self._conn.execute(
+                f'CREATE TABLE IF NOT EXISTS "{self._table}" '
+                f"(id INTEGER PRIMARY KEY AUTOINCREMENT, {columns}, event TEXT NOT NULL)"
+            )

log_forge/sinks/sqs.py

@@ -0,0 +1,107 @@
+"""SQSSink — ship batches to an SQS queue (arch §8, §9.1, guide Phase 10).
+
+SQS is the headline production path: a durable buffer that decouples the app from ELK
+availability — events accumulate safely in the queue during downstream spikes/outages instead
+of being lost or back-pressuring the app. A separate consumer indexes them into ELK (out of
+scope here). Like every sink this receives *already-built* event dicts and knows nothing about
+spans.
+
+``boto3`` is an **optional** dependency (the ``aws`` extra): it is imported lazily inside the
+sink, never at module top, so ``import log_forge.sinks.sqs`` — and the whole library — stays
+dependency-free unless an ``SQSSink`` is actually instantiated without an injected client.
+
+The worker (SPEC-004) batches by count and time, but SQS has hard per-request limits, so this
+sink re-chunks every incoming batch on both dimensions: ≤ 10 messages **and** ≤ 256 KB per
+``send_message_batch``. Partial failures (the response ``Failed`` list) are retried; a single
+event too large to ever fit is dropped with a warning rather than crashing the batch.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+__all__ = ["SQSSink"]
+
+
+class SQSSink:
+    """A :class:`~log_forge.sinks.base.Sink` that sends events to an SQS queue."""
+
+    MAX_BATCH = 10  # SQS SendMessageBatch hard limit: entries per request
+    MAX_BYTES = 256 * 1024  # SQS limit: 256 KB per request
+
+    def __init__(self, queue_url: str, client: Any = None, *, max_retries: int = 3) -> None:
+        if client is None:
+            import boto3  # type: ignore[import-not-found]  # optional 'aws' extra
+
+            client = boto3.client("sqs")
+        self.queue_url = queue_url
+        self.client = client
+        self.max_retries = max_retries
+        self.dropped_oversized = 0  # events too large to ever fit one message
+        self.failed = 0  # entries still failing after the retry bound
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Re-chunk ``batch`` to SQS limits and send each chunk (FR-001, FR-002)."""
+        for chunk in self._chunks(batch):
+            self._send(chunk)
+
+    def close(self) -> None:
+        """No-op: the sink buffers nothing internally (FR-005)."""
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _chunks(self, batch: list[dict[str, object]]) -> list[list[str]]:
+        """Split ``batch`` into sends of ≤ ``MAX_BATCH`` bodies and ≤ ``MAX_BYTES`` each.
+
+        Each event is serialized once with ``json.dumps``. An event whose serialized size
+        exceeds ``MAX_BYTES`` on its own can never fit a message, so it is dropped with a
+        counted warning (FR-004) instead of stalling the batch.
+        """
+        chunks: list[list[str]] = []
+        current: list[str] = []
+        current_bytes = 0
+        for event in batch:
+            body = json.dumps(event)
+            size = len(body.encode("utf-8"))
+            if size > self.MAX_BYTES:
+                self.dropped_oversized += 1
+                sys.stderr.write(
+                    f"log-forge: dropped an event of {size} bytes exceeding the "
+                    f"{self.MAX_BYTES}-byte SQS message limit\n"
+                )
+                continue
+            if current and (
+                len(current) >= self.MAX_BATCH or current_bytes + size > self.MAX_BYTES
+            ):
+                chunks.append(current)
+                current = []
+                current_bytes = 0
+            current.append(body)
+            current_bytes += size
+        if current:
+            chunks.append(current)
+        return chunks
+
+    def _send(self, bodies: list[str]) -> None:
+        """Send one valid chunk, retrying only the ``Failed`` entries with a bounded count.
+
+        Successfully-sent entries are never re-sent; entries still failing past ``max_retries``
+        are counted (``failed``) and logged, not silently dropped (FR-003).
+        """
+        entries = [{"Id": str(i), "MessageBody": body} for i, body in enumerate(bodies)]
+        for attempt in range(self.max_retries + 1):
+            response = self.client.send_message_batch(QueueUrl=self.queue_url, Entries=entries)
+            failed = response.get("Failed", [])
+            if not failed:
+                return
+            failed_ids = {entry["Id"] for entry in failed}
+            entries = [entry for entry in entries if entry["Id"] in failed_ids]
+            if attempt >= self.max_retries:
+                self.failed += len(entries)
+                sys.stderr.write(
+                    f"log-forge: {len(entries)} SQS message(s) still failing after "
+                    f"{self.max_retries + 1} attempts; abandoned\n"
+                )
+                return

log_forge/sinks/stdout.py

@@ -0,0 +1,31 @@
+"""StdoutSink — JSON lines to a stream (arch §8, guide Phase 5).
+
+The zero-dependency default sink: great for local dev and container log scraping. Like every
+sink it receives *already-built* event dicts and knows nothing about spans or context — that
+dumbness is what makes sinks trivially swappable.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import TextIO
+
+__all__ = ["StdoutSink"]
+
+
+class StdoutSink:
+    """Write each event as one JSON line to ``stream`` (default ``sys.stdout``)."""
+
+    def __init__(self, stream: TextIO | None = None) -> None:
+        self._stream = stream if stream is not None else sys.stdout
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Write every event in ``batch`` as a ``json.dumps`` line, then flush."""
+        for event in batch:
+            self._stream.write(json.dumps(event) + "\n")
+        self._stream.flush()
+
+    def close(self) -> None:
+        """Flush the stream (nothing is buffered inside the sink itself)."""
+        self._stream.flush()

log_forge/sinks/syslog.py

@@ -0,0 +1,88 @@
+"""SyslogSink — RFC 5424 syslog frames over UDP or TCP (arch §8, SPEC-009).
+
+Formats each event as an RFC 5424 message (``<PRI>1 TIMESTAMP HOST APP PROCID MSGID SD MSG``) with
+``PRI`` derived from a configurable facility and a severity mapped from the event ``level``. UDP
+sends one datagram per event; TCP uses octet-counted framing (RFC 6587). Dependency-free (stdlib
+``socket``, via :class:`~log_forge.sinks._socket.SocketTransport`).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import socket
+
+from log_forge.sinks._socket import SocketTransport
+
+__all__ = ["SyslogSink"]
+
+# RFC 5424 numeric severities; log-forge levels mapped onto them (unknown level -> notice/5).
+_SEVERITY = {"DEBUG": 7, "INFO": 6, "NOTICE": 5, "WARNING": 4, "ERROR": 3, "CRITICAL": 2}
+_DEFAULT_SEVERITY = 5
+
+# A subset of RFC 5424 facility keywords -> numeric code.
+_FACILITY = {
+    "kern": 0, "user": 1, "mail": 2, "daemon": 3, "auth": 4, "syslog": 5, "lpr": 6, "news": 7,
+    "uucp": 8, "cron": 9, "authpriv": 10, "ftp": 11,
+    "local0": 16, "local1": 17, "local2": 18, "local3": 19,
+    "local4": 20, "local5": 21, "local6": 22, "local7": 23,
+}
+
+
+class SyslogSink:
+    """A :class:`~log_forge.sinks.base.Sink` that emits RFC 5424 frames over UDP/TCP (FR-006)."""
+
+    def __init__(
+        self,
+        host: str,
+        port: int = 514,
+        *,
+        transport: str = "udp",
+        facility: str = "user",
+        app_name: str = "log-forge",
+        timeout: float = 5.0,
+        max_retries: int = 3,
+    ) -> None:
+        if facility not in _FACILITY:
+            raise ValueError(f"invalid facility {facility!r}; expected one of {sorted(_FACILITY)}")
+        self._facility = _FACILITY[facility]
+        self._app_name = app_name
+        self._transport = transport
+        self._hostname = socket.gethostname() or "-"
+        self._procid = str(os.getpid())
+        self._socket = SocketTransport(
+            host, port, transport=transport, timeout=timeout, max_retries=max_retries
+        )
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Frame each event as RFC 5424 and send it over the socket (FR-006)."""
+        if not batch:
+            return
+        self._socket.send_all([self._frame(event) for event in batch])
+
+    def close(self) -> None:
+        """Close the underlying socket (FR-006)."""
+        self._socket.close()
+
+    @property
+    def failed(self) -> int:
+        """Messages abandoned past the retry bound."""
+        return self._socket.failed
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _frame(self, event: dict[str, object]) -> bytes:
+        """Build one RFC 5424 message; TCP gets octet-counted framing (RFC 6587)."""
+        level = event.get("level")
+        severity = _SEVERITY.get(level.upper(), _DEFAULT_SEVERITY) if isinstance(level, str) else (
+            _DEFAULT_SEVERITY
+        )
+        pri = self._facility * 8 + severity
+        timestamp = event.get("timestamp")
+        ts = timestamp if isinstance(timestamp, str) else "-"
+        msg = json.dumps(event)
+        frame = f"<{pri}>1 {ts} {self._hostname} {self._app_name} {self._procid} - - {msg}"
+        data = frame.encode("utf-8")
+        if self._transport == "tcp":
+            return f"{len(data)} ".encode("ascii") + data  # RFC 6587 octet counting
+        return data

log_forge/sinks/transform.py

@@ -0,0 +1,46 @@
+"""TransformSink — reshape or redact each event before forwarding (arch §8, SPEC-006 FR-004).
+
+Wraps an inner sink and maps every event through a user function on its way out — to redact a
+field, add host metadata, rename keys, and so on. The function returning ``None`` drops that
+event. The caller's batch and event dicts are never mutated in place: only the function's
+return values are forwarded, so a transform must copy before mutating (see the spec's redact
+example).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from log_forge.sinks.base import Sink
+
+__all__ = ["TransformSink"]
+
+
+class TransformSink:
+    """A :class:`~log_forge.sinks.base.Sink` that maps each event before forwarding.
+
+    ``fn`` is applied to every event; a new list of the non-``None`` results is forwarded to
+    ``inner``. When every event is dropped, ``inner.emit`` is not called.
+    """
+
+    def __init__(
+        self,
+        inner: Sink,
+        fn: Callable[[dict[str, object]], dict[str, object] | None],
+    ) -> None:
+        self._inner = inner
+        self._fn = fn
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Apply ``fn`` to each event, forwarding the non-``None`` results (FR-004)."""
+        transformed: list[dict[str, object]] = []
+        for event in batch:
+            result = self._fn(event)
+            if result is not None:
+                transformed.append(result)
+        if transformed:
+            self._inner.emit(transformed)
+
+    def close(self) -> None:
+        """Close the inner sink (FR-004)."""
+        self._inner.close()

log_forge/sinks/util.py

@@ -0,0 +1,68 @@
+"""Small utility sinks: StderrSink, NullSink, MemorySink (arch §8, SPEC-008).
+
+Three tiny zero-dependency sinks that round out the local family: send NDJSON to stderr (the
+twelve-factor convention — logs on stderr, app output on stdout), discard everything (to disable
+output or benchmark the pipeline), or collect events into an in-process list (for tests and
+notebooks). Like every sink they operate purely on already-built event dicts (arch §8).
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import TextIO
+
+from log_forge.sinks.stdout import StdoutSink
+
+__all__ = ["StderrSink", "NullSink", "MemorySink"]
+
+
+class StderrSink(StdoutSink):
+    """The :class:`~log_forge.sinks.stdout.StdoutSink` shape, defaulting to ``sys.stderr`` (FR-004).
+
+    Writes each event as one ``json.dumps`` line and flushes, exactly like ``StdoutSink`` — only the
+    default stream differs. An explicit ``stream`` (e.g. a ``StringIO``) can be injected for capture.
+    """
+
+    def __init__(self, stream: TextIO | None = None) -> None:
+        super().__init__(stream if stream is not None else sys.stderr)
+
+
+class NullSink:
+    """A :class:`~log_forge.sinks.base.Sink` that discards every event (FR-005).
+
+    Useful to disable output without unwiring the pipeline, or to benchmark everything up to the
+    sink. ``dropped`` counts how many events were discarded, so a benchmark can assert throughput.
+    """
+
+    def __init__(self) -> None:
+        self.dropped = 0
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Discard the batch, counting the events dropped (FR-005)."""
+        self.dropped += len(batch)
+
+    def close(self) -> None:
+        """No-op — nothing is held (FR-005)."""
+
+
+class MemorySink:
+    """A :class:`~log_forge.sinks.base.Sink` that collects events into an in-process list (FR-006).
+
+    ``.events`` is a plain ``list`` exposing every event in arrival order, ideal for asserting in
+    tests or eyeballing in a notebook. With ``maxlen`` set it behaves as a bounded ring, keeping only
+    the most recent ``maxlen`` events; the list object identity is stable, so a held reference keeps
+    seeing updates.
+    """
+
+    def __init__(self, maxlen: int | None = None) -> None:
+        self.events: list[dict[str, object]] = []
+        self._maxlen = maxlen
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Append the batch in order, trimming to the most recent ``maxlen`` if bounded (FR-006)."""
+        self.events.extend(batch)
+        if self._maxlen is not None and len(self.events) > self._maxlen:
+            del self.events[: len(self.events) - self._maxlen]
+
+    def close(self) -> None:
+        """No-op — collected events remain readable after close (FR-006)."""

log_forge/worker.py

@@ -0,0 +1,153 @@
+"""Background flush worker — non-blocking span delivery (arch §9, guide Phase 9).
+
+A finished span used to flush inline, blocking the decorated function on ``sink.emit``. The
+``Worker`` moves that off the hot path: :meth:`submit` is a fast, in-process handoff to a
+bounded queue drained by a daemon thread that batches events (by count *and* time) into a
+single ``sink.emit`` call. A slow or down sink can never back-pressure the app — the queue is
+bounded and overflow is dropped-newest with a counter (arch §9). Emit failures are retried
+with backoff; a graceful :meth:`shutdown` drains the queue, emits the tail, and closes the
+sink so buffered events survive process exit.
+
+This module owns *delivery mechanics only*; it receives already-built event dicts and knows
+nothing about spans or context (the same dumbness that makes sinks swappable).
+"""
+
+from __future__ import annotations
+
+import queue
+import sys
+import threading
+import time
+from typing import cast
+
+from log_forge.sinks.base import Sink
+
+__all__ = ["Worker"]
+
+# Sentinel enqueued by shutdown() to wake a worker blocked in queue.get() so it stops promptly
+# instead of waiting out the flush_interval. It is never emitted.
+_SHUTDOWN = object()
+
+
+class Worker:
+    """Owns a bounded queue + daemon thread that batches and flushes events to ``sink``."""
+
+    def __init__(
+        self,
+        sink: Sink,
+        *,
+        batch_size: int = 10,
+        flush_interval: float = 1.0,
+        max_queue: int = 10_000,
+        max_retries: int = 3,
+    ) -> None:
+        self.sink = sink
+        self.batch_size = batch_size
+        self.flush_interval = flush_interval
+        self.max_retries = max_retries
+        self.dropped = 0  # submissions dropped because the queue was full (backpressure)
+        self.failed_batches = 0  # batches abandoned after exhausting retries (worker-thread only)
+        self._queue: queue.Queue[object] = queue.Queue(maxsize=max_queue)
+        self._stop = threading.Event()
+        self._shutdown_done = False
+        # Guards the `dropped` counter (incremented from any caller thread) and the shutdown
+        # once-only flag (shutdown may be called concurrently by atexit and user code).
+        self._lock = threading.Lock()
+        self._thread = threading.Thread(
+            target=self._run, name="log-forge-worker", daemon=True
+        )
+        self._thread.start()
+
+    def submit(self, events: list[dict[str, object]]) -> None:
+        """Hand a finished span's events to the worker. Non-blocking.
+
+        Enqueues via ``put_nowait`` and returns immediately without touching the sink. When the
+        queue is full, drops this submission (drop-newest) and counts it in ``dropped`` rather
+        than blocking the caller (FR-001, FR-004).
+        """
+        try:
+            self._queue.put_nowait(events)
+        except queue.Full:
+            with self._lock:
+                self.dropped += 1
+
+    def shutdown(self) -> None:
+        """Stop the thread, drain + emit everything queued, then ``close()`` the sink.
+
+        Idempotent: a second call is a no-op (FR-005). Registered via ``atexit`` by the
+        decorator's lazy worker so a program that logs and exits immediately still flushes.
+        """
+        with self._lock:
+            if self._shutdown_done:
+                return
+            self._shutdown_done = True
+        self._stop.set()
+        try:
+            self._queue.put_nowait(_SHUTDOWN)  # wake a blocked get() for a prompt stop
+        except queue.Full:
+            pass
+        self._thread.join()
+        self.sink.close()
+
+    # -- worker thread ------------------------------------------------------------------
+
+    def _run(self) -> None:
+        """Drain loop: accumulate event-lists and emit a batch on the count/time trigger."""
+        pending: list[list[dict[str, object]]] = []
+        last_flush = time.monotonic()
+        while not self._stop.is_set():
+            timeout = max(0.0, self.flush_interval - (time.monotonic() - last_flush))
+            try:
+                item = self._queue.get(timeout=timeout)
+            except queue.Empty:
+                item = None
+            if item is not None and item is not _SHUTDOWN:
+                pending.append(cast("list[dict[str, object]]", item))
+            now = time.monotonic()
+            if len(pending) >= self.batch_size or now - last_flush >= self.flush_interval:
+                if pending:
+                    self._emit(pending)
+                    pending = []
+                # Advance the window even when idle (pending empty). Otherwise last_flush never
+                # moves while the queue is empty, timeout collapses to 0.0, and get(timeout=0.0)
+                # busy-spins a core. Resetting it lets the next get() block a full interval.
+                last_flush = now
+        self._final_drain(pending)
+
+    def _final_drain(self, pending: list[list[dict[str, object]]]) -> None:
+        """On stop, pull anything still queued and emit the tail as one final batch."""
+        while True:
+            try:
+                item = self._queue.get_nowait()
+            except queue.Empty:
+                break
+            if item is not None and item is not _SHUTDOWN:
+                pending.append(cast("list[dict[str, object]]", item))
+        if pending:
+            self._emit(pending)
+
+    def _emit(self, event_lists: list[list[dict[str, object]]]) -> None:
+        """Flatten queued per-span event-lists into one batch and emit, retrying with backoff.
+
+        A failing ``sink.emit`` is retried up to ``max_retries`` times; past that the batch is
+        abandoned with a counted warning and draining continues, so a broken sink never crashes
+        the worker thread or the app (FR-002, FR-003).
+        """
+        batch = [event for events in event_lists for event in events]
+        if not batch:
+            return
+        for attempt in range(self.max_retries + 1):
+            try:
+                self.sink.emit(batch)
+                return
+            except Exception:  # noqa: BLE001 — any sink failure must not kill the worker thread
+                if attempt >= self.max_retries:
+                    self.failed_batches += 1
+                    sys.stderr.write(
+                        f"log-forge: abandoned a batch of {len(batch)} event(s) after "
+                        f"{self.max_retries + 1} failed emit attempts\n"
+                    )
+                    return
+                # Backoff between attempts; _stop.wait returns at once during shutdown, so a
+                # failing sink can't stall the drain past max_retries quick tries.
+                self._stop.wait(min(0.01 * (2**attempt), 0.5))