log-foundry 0.0.2.dev1__py3-none-any.whl

log_forge/sinks/postgres.py

@@ -0,0 +1,101 @@
+"""PostgresSink — insert events into a Postgres table with a JSONB column (arch §8, SPEC-011).
+
+Each event is stored as a ``JSONB`` ``event`` column plus a few extracted columns for indexing.
+``psycopg`` (v3) is the optional ``postgres`` extra, imported lazily. The whole batch inserts in a
+single transaction (chunked into driver-friendly statements); on failure the transaction is rolled
+back and retried within a bounded count. Write-only. An optional idempotent ``create_table``
+convenience is off by default — the user owns their schema and indexes.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from typing import Any
+
+from log_forge.sinks._chunk import chunk_list, valid_identifier
+
+__all__ = ["PostgresSink"]
+
+_BACKOFF_BASE = 0.1
+
+# Columns extracted from each event for indexing; ``event`` (full JSONB) is stored alongside.
+_COLUMNS = ("timestamp", "level", "trace_id", "span_id", "function", "service")
+
+
+class PostgresSink:
+    """A :class:`~log_forge.sinks.base.Sink` that batch-inserts events into a Postgres table."""
+
+    def __init__(
+        self,
+        table: str,
+        *,
+        connection: Any = None,
+        dsn: str | None = None,
+        create_table: bool = False,
+        chunk_size: int = 1000,
+        max_retries: int = 3,
+    ) -> None:
+        self._table = valid_identifier(table)
+        self._chunk_size = chunk_size
+        self.max_retries = max_retries
+        self.failed = 0
+        self._closed = False
+        self._owns_connection = connection is None
+        if connection is None:
+            import psycopg  # type: ignore[import-not-found]  # optional 'postgres' extra
+
+            connection = psycopg.connect(dsn)
+        self._conn = connection
+        columns = ", ".join((*_COLUMNS, "event"))
+        placeholders = ", ".join(["%s"] * len(_COLUMNS) + ["%s::jsonb"])
+        self._insert_sql = f"INSERT INTO {self._table} ({columns}) VALUES ({placeholders})"
+        if create_table:
+            self._ensure_schema()
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Insert the whole batch in one transaction, rolling back and retrying on error (FR-004)."""
+        if not batch:
+            return
+        for attempt in range(self.max_retries + 1):
+            try:
+                with self._conn.cursor() as cur:
+                    for chunk in chunk_list(batch, self._chunk_size):
+                        cur.executemany(self._insert_sql, [self._row(event) for event in chunk])
+                self._conn.commit()
+                return
+            except Exception as err:  # isolation boundary: never crash the worker (FR-006)
+                self._conn.rollback()
+                if attempt < self.max_retries:
+                    time.sleep(_BACKOFF_BASE * (2**attempt))
+                    continue
+                self.failed += len(batch)
+                sys.stderr.write(
+                    f"log-forge: PostgresSink abandoned {len(batch)} event(s) after "
+                    f"{self.max_retries + 1} attempts ({err!r})\n"
+                )
+                return
+
+    def close(self) -> None:
+        """Commit pending work; close only an owned connection; idempotent (FR-005)."""
+        if self._closed:
+            return
+        self._conn.commit()
+        if self._owns_connection:
+            self._conn.close()
+        self._closed = True
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _row(self, event: dict[str, object]) -> tuple[object, ...]:
+        return (*(event.get(col) for col in _COLUMNS), json.dumps(event))
+
+    def _ensure_schema(self) -> None:
+        columns = ", ".join(f"{col} TEXT" for col in _COLUMNS)
+        with self._conn.cursor() as cur:
+            cur.execute(
+                f"CREATE TABLE IF NOT EXISTS {self._table} "
+                f"(id BIGSERIAL PRIMARY KEY, {columns}, event JSONB NOT NULL)"
+            )
+        self._conn.commit()

log_forge/sinks/pubsub.py

@@ -0,0 +1,45 @@
+"""GooglePubSubSink — publish events to a Google Cloud Pub/Sub topic (arch §8, §9.1, SPEC-010).
+
+A durable-buffer sink on ``google-cloud-pubsub`` (the optional ``gcp-pubsub`` extra, imported
+lazily). ``publish()`` returns a future that resolves asynchronously; the sink accumulates the
+batch's futures and resolves them on ``close()`` so buffered messages are flushed and publish errors
+are counted and logged.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+__all__ = ["GooglePubSubSink"]
+
+
+class GooglePubSubSink:
+    """A :class:`~log_forge.sinks.base.Sink` that publishes events to a Pub/Sub topic."""
+
+    def __init__(self, topic: str, *, client: Any = None) -> None:
+        if client is None:
+            from google.cloud import pubsub_v1  # type: ignore[import-not-found]  # 'gcp-pubsub'
+
+            client = pubsub_v1.PublisherClient()
+        self.topic = topic
+        self.client = client
+        self.failed = 0
+        self._futures: list[Any] = []
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Publish one message per event, retaining each future for flush on close (FR-008)."""
+        for event in batch:
+            future = self.client.publish(self.topic, data=json.dumps(event).encode("utf-8"))
+            self._futures.append(future)
+
+    def close(self) -> None:
+        """Resolve all pending publish futures, counting/logging errors; idempotent (FR-008)."""
+        for future in self._futures:
+            try:
+                future.result()
+            except Exception as err:  # isolation boundary: never crash the worker (FR-011)
+                self.failed += 1
+                sys.stderr.write(f"log-forge: GooglePubSubSink publish failed: {err!r}\n")
+        self._futures.clear()

log_forge/sinks/rabbitmq.py

@@ -0,0 +1,129 @@
+"""RabbitMQSink — publish events to a RabbitMQ exchange (arch §8, §9.1, SPEC-010).
+
+A durable-buffer sink on ``pika`` (the optional ``amqp`` extra, imported lazily). Each event is
+published as a **persistent** message (delivery mode 2) to the configured exchange/routing key. A
+dropped or closed connection is re-established within a bounded retry before the batch is abandoned
+and counted; ``close()`` closes the channel and connection.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from typing import Any
+
+__all__ = ["RabbitMQSink"]
+
+_BACKOFF_BASE = 0.1
+_PERSISTENT = 2  # pika delivery mode for persistent messages
+
+
+class _PersistentProperties:
+    """Fallback message properties (delivery_mode=persistent) used when ``pika`` is not installed.
+
+    Real usage imports ``pika.BasicProperties``; this stand-in only exists so the sink is testable
+    with an injected fake connection in an environment without the ``amqp`` extra.
+    """
+
+    delivery_mode = _PERSISTENT
+
+
+class RabbitMQSink:
+    """A :class:`~log_forge.sinks.base.Sink` that publishes persistent messages to RabbitMQ."""
+
+    def __init__(
+        self,
+        *,
+        exchange: str,
+        routing_key: str,
+        connection: Any = None,
+        url: str | None = None,
+        max_retries: int = 3,
+    ) -> None:
+        self._exchange = exchange
+        self._routing_key = routing_key
+        self._url = url
+        self._max_retries = max_retries
+        self._owns_connection = connection is None
+        self._connection = connection if connection is not None else self._connect()
+        self._channel: Any = None
+        self._properties: Any = None
+        self.failed = 0
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Publish one persistent message per event, reconnecting on error (FR-006)."""
+        for event in batch:
+            self._publish(json.dumps(event).encode("utf-8"))
+
+    def close(self) -> None:
+        """Close the channel and (owned) connection; idempotent (FR-006)."""
+        if self._channel is not None:
+            _safe_close(self._channel)
+            self._channel = None
+        if self._connection is not None:
+            _safe_close(self._connection)
+            self._connection = None
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _publish(self, body: bytes) -> None:
+        for attempt in range(self._max_retries + 1):
+            try:
+                self._active_channel().basic_publish(
+                    exchange=self._exchange,
+                    routing_key=self._routing_key,
+                    body=body,
+                    properties=self._persistent_properties(),
+                )
+                return
+            except Exception as err:  # isolation boundary: never crash the worker (FR-011)
+                self._reset()
+                if attempt < self._max_retries:
+                    time.sleep(_BACKOFF_BASE * (2**attempt))
+                    continue
+                self.failed += 1
+                sys.stderr.write(
+                    f"log-forge: RabbitMQSink abandoned a message after "
+                    f"{self._max_retries + 1} attempts ({err!r})\n"
+                )
+                return
+
+    def _active_channel(self) -> Any:
+        if self._connection is None:
+            self._connection = self._connect()
+        if self._channel is None:
+            self._channel = self._connection.channel()
+        return self._channel
+
+    def _reset(self) -> None:
+        """Drop the channel (and an owned connection) so the next attempt reconnects."""
+        if self._channel is not None:
+            _safe_close(self._channel)
+            self._channel = None
+        if self._owns_connection and self._connection is not None:
+            _safe_close(self._connection)
+            self._connection = None
+
+    def _connect(self) -> Any:
+        import pika  # type: ignore[import-not-found]  # optional 'amqp' extra
+
+        params = pika.URLParameters(self._url) if self._url else pika.ConnectionParameters()
+        return pika.BlockingConnection(params)
+
+    def _persistent_properties(self) -> Any:
+        if self._properties is None:
+            try:
+                import pika  # optional 'amqp' extra (type-ignored at the import in _connect)
+
+                self._properties = pika.BasicProperties(delivery_mode=_PERSISTENT)
+            except ImportError:
+                self._properties = _PersistentProperties()
+        return self._properties
+
+
+def _safe_close(resource: Any) -> None:
+    try:
+        resource.close()
+    except Exception:  # closing a broken channel/connection must not raise
+        pass

log_forge/sinks/redis.py

@@ -0,0 +1,90 @@
+"""RedisStreamsSink / RedisListSink — buffer events in Redis (arch §8, §9.1, SPEC-010).
+
+Two durable-buffer sinks on the ``redis`` extra (``redis-py``, imported lazily): one appends to a
+Redis **stream** (``XADD``), the other pushes onto a **list** (``RPUSH``). Each pipelines the whole
+batch into a single round trip. A connection error is retried within a bounded count then counted and
+logged; ``close()`` releases the connection only when the sink opened it (an injected client is not
+closed). The module is named ``redis`` to match the extra, and imports the driver lazily so it never
+shadows or requires the real ``redis`` package at import time.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from typing import Any
+
+__all__ = ["RedisStreamsSink", "RedisListSink"]
+
+_BACKOFF_BASE = 0.1
+
+
+class _RedisSink:
+    """Shared pipelining, bounded retry, and ownership-aware close for the Redis sinks."""
+
+    def __init__(self, *, client: Any, url: str | None, max_retries: int) -> None:
+        self._owns_client = client is None
+        if client is None:
+            import redis  # type: ignore[import-not-found]  # optional 'redis' extra
+
+            client = redis.Redis.from_url(url) if url else redis.Redis()
+        self.client = client
+        self.max_retries = max_retries
+        self.failed = 0
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Pipeline the whole batch into one round trip, retrying on connection error (FR-005)."""
+        if not batch:
+            return
+        for attempt in range(self.max_retries + 1):
+            try:
+                pipe = self.client.pipeline()
+                for event in batch:
+                    self._stage(pipe, event)
+                pipe.execute()
+                return
+            except Exception as err:  # isolation boundary: never crash the worker (FR-011)
+                if attempt < self.max_retries:
+                    time.sleep(_BACKOFF_BASE * (2**attempt))
+                    continue
+                self.failed += len(batch)
+                sys.stderr.write(
+                    f"log-forge: {type(self).__name__} abandoned {len(batch)} event(s) after "
+                    f"{self.max_retries + 1} attempts ({err!r})\n"
+                )
+                return
+
+    def close(self) -> None:
+        """Close the connection only if the sink owns it (FR-005)."""
+        if self._owns_client:
+            self.client.close()
+
+    def _stage(self, pipe: Any, event: dict[str, object]) -> None:
+        raise NotImplementedError
+
+
+class RedisStreamsSink(_RedisSink):
+    """Append each event to a Redis stream via ``XADD``, pipelined per batch (FR-005)."""
+
+    def __init__(
+        self, stream: str, *, client: Any = None, url: str | None = None, max_retries: int = 3
+    ) -> None:
+        self._stream = stream
+        super().__init__(client=client, url=url, max_retries=max_retries)
+
+    def _stage(self, pipe: Any, event: dict[str, object]) -> None:
+        pipe.xadd(self._stream, {"event": json.dumps(event)})
+
+
+class RedisListSink(_RedisSink):
+    """Push each event onto a Redis list via ``RPUSH``, pipelined per batch (FR-005)."""
+
+    def __init__(
+        self, key: str, *, client: Any = None, url: str | None = None, max_retries: int = 3
+    ) -> None:
+        self._key = key
+        super().__init__(client=client, url=url, max_retries=max_retries)
+
+    def _stage(self, pipe: Any, event: dict[str, object]) -> None:
+        pipe.rpush(self._key, json.dumps(event))

log_forge/sinks/sentry.py

@@ -0,0 +1,136 @@
+"""SentrySink — route error-level events to Sentry (arch §8, SPEC-009).
+
+Uses the ``sentry-sdk`` when it is installed (the optional ``sentry`` extra): ``import sentry_sdk``
+happens lazily inside the sink, never at module top, so importing this module does not require the
+extra. When the SDK is absent the sink falls back to POSTing a Sentry *envelope* over HTTP to the
+DSN's ingest URL (reusing :class:`~log_forge.sinks.http.HTTPSink`). Only events at/above a
+configurable ``min_level`` (default ``ERROR``) are sent; the rest are skipped.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from typing import Any
+from urllib.parse import urlparse
+
+from log_forge.sinks.http import HTTPSink
+
+__all__ = ["SentrySink"]
+
+# log-forge level -> ordering rank (send when rank >= min_level's rank).
+_LEVEL_RANK = {"DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50}
+# log-forge level -> Sentry level keyword.
+_SENTRY_LEVEL = {
+    "DEBUG": "debug", "INFO": "info", "WARNING": "warning", "ERROR": "error", "CRITICAL": "fatal",
+}
+
+
+class SentrySink:
+    """A :class:`~log_forge.sinks.base.Sink` that captures qualifying events to Sentry (FR-011).
+
+    Attributes:
+        sent: Events captured/sent to Sentry.
+        skipped: Events below ``min_level`` (or without a usable level) that were not sent.
+    """
+
+    def __init__(
+        self,
+        dsn: str | None = None,
+        *,
+        min_level: str = "ERROR",
+        sdk: Any = None,
+        opener: Any = None,
+        max_retries: int = 3,
+    ) -> None:
+        self._dsn = dsn
+        self._min_rank = _LEVEL_RANK.get(min_level.upper(), _LEVEL_RANK["ERROR"])
+        self.sent = 0
+        self.skipped = 0
+        self._sdk = sdk if sdk is not None else _import_sdk()
+        self._http: HTTPSink | None = None
+        self._auth_header = ""
+        if self._sdk is None:
+            # No SDK: prepare the HTTP-envelope fallback, which needs a DSN to know where to POST.
+            if dsn is None:
+                raise ValueError(
+                    "SentrySink without sentry-sdk requires a dsn for the HTTP-envelope fallback"
+                )
+            ingest_url, self._auth_header = _parse_dsn(dsn)
+            self._http = HTTPSink(ingest_url, opener=opener, max_retries=max_retries)
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Capture each qualifying event via the SDK or the HTTP fallback (FR-011)."""
+        for event in batch:
+            if not self._qualifies(event):
+                self.skipped += 1
+                continue
+            if self._sdk is not None:
+                self._sdk.capture_event(self._sentry_event(event))
+            else:
+                self._post_envelope(event)
+            self.sent += 1
+
+    def close(self) -> None:
+        """Release the HTTP fallback resource, if any; idempotent (FR-012)."""
+        if self._http is not None:
+            self._http.close()
+
+    @property
+    def failed(self) -> int:
+        """Requests abandoned past the retry bound (HTTP fallback only)."""
+        return self._http.failed if self._http is not None else 0
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _qualifies(self, event: dict[str, object]) -> bool:
+        level = event.get("level")
+        if not isinstance(level, str):
+            return False
+        return _LEVEL_RANK.get(level.upper(), -1) >= self._min_rank
+
+    def _sentry_event(self, event: dict[str, object]) -> dict[str, object]:
+        level = event.get("level")
+        sentry_level = _SENTRY_LEVEL.get(level.upper(), "error") if isinstance(level, str) else (
+            "error"
+        )
+        return {
+            "message": event.get("message", ""),
+            "level": sentry_level,
+            "logger": "log_forge",
+            "extra": event,
+        }
+
+    def _post_envelope(self, event: dict[str, object]) -> None:
+        assert self._http is not None
+        header = {"event_id": uuid.uuid4().hex, "dsn": self._dsn}
+        item_header = {"type": "event"}
+        body = (
+            json.dumps(header) + "\n"
+            + json.dumps(item_header) + "\n"
+            + json.dumps(self._sentry_event(event)) + "\n"
+        ).encode("utf-8")
+        self._http._send(
+            body,
+            content_type="application/x-sentry-envelope",
+            extra_headers={"X-Sentry-Auth": self._auth_header},
+        )
+
+
+def _import_sdk() -> Any:
+    """Import ``sentry_sdk`` lazily; return ``None`` if the optional extra is not installed."""
+    try:
+        import sentry_sdk  # type: ignore[import-not-found]  # optional 'sentry' extra
+    except ImportError:
+        return None
+    return sentry_sdk
+
+
+def _parse_dsn(dsn: str) -> tuple[str, str]:
+    """Derive ``(ingest_url, x_sentry_auth_header)`` from a Sentry DSN."""
+    parsed = urlparse(dsn)
+    project = parsed.path.lstrip("/")
+    port = f":{parsed.port}" if parsed.port else ""
+    ingest_url = f"{parsed.scheme}://{parsed.hostname}{port}/api/{project}/envelope/"
+    auth_header = f"Sentry sentry_key={parsed.username}, sentry_version=7, sentry_client=log-forge"
+    return ingest_url, auth_header

log_forge/sinks/sns.py

@@ -0,0 +1,82 @@
+"""SNSSink — publish events to an SNS topic (arch §8, SPEC-010).
+
+Mirrors the SPEC-005 ``SQSSink`` partial-failure policy on SNS's ``publish_batch`` (≤ 10 entries per
+request, ≤ 256 KB total). ``boto3`` is the optional ``aws`` extra, imported lazily. The response
+``Failed`` list is retried within a bounded count; entries still failing are counted and logged.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+from log_forge.sinks._chunk import chunk_items
+
+__all__ = ["SNSSink"]
+
+
+class SNSSink:
+    """A :class:`~log_forge.sinks.base.Sink` that publishes events to an SNS topic."""
+
+    MAX_BATCH = 10  # publish_batch hard limit: entries per request
+    MAX_BYTES = 256 * 1024  # 256 KB per request
+
+    def __init__(self, topic_arn: str, *, client: Any = None, max_retries: int = 3) -> None:
+        if client is None:
+            import boto3  # type: ignore[import-not-found]  # optional 'aws' extra
+
+            client = boto3.client("sns")
+        self.topic_arn = topic_arn
+        self.client = client
+        self.max_retries = max_retries
+        self.failed = 0
+        self.dropped_oversized = 0
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Re-chunk to publish_batch limits and send each chunk, retrying failures (FR-010)."""
+        bodies = self._bodies(batch)
+        for chunk in chunk_items(
+            bodies, max_count=self.MAX_BATCH, max_bytes=self.MAX_BYTES, size_of=len
+        ):
+            self._send(chunk)
+
+    def close(self) -> None:
+        """No-op: the sink buffers nothing internally (FR-001)."""
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _bodies(self, batch: list[dict[str, object]]) -> list[str]:
+        """Serialize each event, dropping any single message too large to ever fit (FR-011)."""
+        bodies: list[str] = []
+        for event in batch:
+            body = json.dumps(event)
+            if len(body.encode("utf-8")) > self.MAX_BYTES:
+                self.dropped_oversized += 1
+                sys.stderr.write(
+                    f"log-forge: SNSSink dropped an event exceeding the "
+                    f"{self.MAX_BYTES}-byte message limit\n"
+                )
+                continue
+            bodies.append(body)
+        return bodies
+
+    def _send(self, bodies: list[str]) -> None:
+        """Publish one chunk, retrying only the ``Failed`` entries (bounded) (FR-010)."""
+        entries = [{"Id": str(i), "Message": body} for i, body in enumerate(bodies)]
+        for attempt in range(self.max_retries + 1):
+            response = self.client.publish_batch(
+                TopicArn=self.topic_arn, PublishBatchRequestEntries=entries
+            )
+            failed = response.get("Failed", [])
+            if not failed:
+                return
+            failed_ids = {entry["Id"] for entry in failed}
+            entries = [entry for entry in entries if entry["Id"] in failed_ids]
+            if attempt >= self.max_retries:
+                self.failed += len(entries)
+                sys.stderr.write(
+                    f"log-forge: {len(entries)} SNS message(s) still failing after "
+                    f"{self.max_retries + 1} attempts; abandoned\n"
+                )
+                return

log_forge/sinks/splunk.py

@@ -0,0 +1,51 @@
+"""SplunkHECSink — ship to Splunk's HTTP Event Collector (arch §8, SPEC-009).
+
+An :class:`~log_forge.sinks.http.HTTPSink` specialization: each event is wrapped in a HEC envelope
+(``{"event": ..., "time": <epoch>, "host": ..., "source": ...}``) and the batch is sent as HEC's
+concatenated-JSON-objects body with an ``Authorization: Splunk <token>`` header. ``time`` (epoch
+seconds) is parsed from the event's ISO-8601 ``timestamp`` (falling back to emit-time now).
+"""
+
+from __future__ import annotations
+
+import json
+
+from log_forge.sinks._time import epoch_seconds
+from log_forge.sinks.http import HTTPSink, merge_headers
+
+__all__ = ["SplunkHECSink"]
+
+
+class SplunkHECSink(HTTPSink):
+    """POST HEC envelopes to a Splunk HTTP Event Collector endpoint (FR-008)."""
+
+    def __init__(
+        self,
+        url: str,
+        token: str,
+        *,
+        host: str | None = None,
+        source: str = "log-forge",
+        **http_kwargs: object,
+    ) -> None:
+        self._host = host
+        self._source = source
+        headers = merge_headers({"Authorization": f"Splunk {token}"}, http_kwargs)
+        super().__init__(url, headers=headers, **http_kwargs)  # type: ignore[arg-type]
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Send the batch as HEC's concatenated JSON objects (FR-008)."""
+        if not batch:
+            return
+        body = "".join(json.dumps(self._envelope(event)) for event in batch).encode("utf-8")
+        self._send(body, content_type="application/json")
+
+    def _envelope(self, event: dict[str, object]) -> dict[str, object]:
+        envelope: dict[str, object] = {
+            "event": event,
+            "time": epoch_seconds(event.get("timestamp")),
+            "source": self._source,
+        }
+        if self._host is not None:
+            envelope["host"] = self._host
+        return envelope