log-foundry 0.0.2.dev1__py3-none-any.whl

log_forge/sinks/kinesis.py

@@ -0,0 +1,99 @@
+"""KinesisSink — put event records to a Kinesis Data Stream (arch §8, §9.1, SPEC-010).
+
+Extends the durable-buffer path ``SQSSink`` established: a queue/stream absorbs spikes and outages
+while a separate consumer (out of scope) drains it into ELK. ``boto3`` is the optional ``aws`` extra,
+imported lazily inside the sink (never at module top) so importing this module needs no ``boto3``
+unless a sink is built without an injected client. Each incoming batch is re-chunked to Kinesis's
+``put_records`` limits (≤ 500 records **and** ≤ 5 MB); partial failures are retried within a bound.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+from log_forge.sinks._chunk import chunk_items
+
+__all__ = ["KinesisSink"]
+
+
+class KinesisSink:
+    """A :class:`~log_forge.sinks.base.Sink` that writes events to a Kinesis Data Stream."""
+
+    MAX_RECORDS = 500  # put_records hard limit: records per request
+    MAX_REQUEST_BYTES = 5 * 1024 * 1024  # 5 MB per put_records request
+    MAX_RECORD_BYTES = 1024 * 1024  # 1 MB per record (Data)
+
+    def __init__(
+        self,
+        stream_name: str,
+        *,
+        client: Any = None,
+        partition_key_field: str = "trace_id",
+        max_retries: int = 3,
+    ) -> None:
+        if client is None:
+            import boto3  # type: ignore[import-not-found]  # optional 'aws' extra
+
+            client = boto3.client("kinesis")
+        self.stream_name = stream_name
+        self.client = client
+        self.partition_key_field = partition_key_field
+        self.max_retries = max_retries
+        self.failed = 0
+        self.dropped_oversized = 0
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Re-chunk to put_records limits and send each chunk, retrying failures (FR-003)."""
+        records = self._records(batch)
+        for chunk in chunk_items(
+            records,
+            max_count=self.MAX_RECORDS,
+            max_bytes=self.MAX_REQUEST_BYTES,
+            size_of=lambda record: len(record["Data"]),
+        ):
+            self._send(chunk)
+
+    def close(self) -> None:
+        """No-op: the sink buffers nothing internally (FR-001)."""
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _records(self, batch: list[dict[str, object]]) -> list[dict[str, Any]]:
+        """Build put_records entries, dropping any single record too large to ever fit (FR-011)."""
+        records: list[dict[str, Any]] = []
+        for event in batch:
+            data = json.dumps(event).encode("utf-8")
+            if len(data) > self.MAX_RECORD_BYTES:
+                self.dropped_oversized += 1
+                sys.stderr.write(
+                    f"log-forge: KinesisSink dropped an event of {len(data)} bytes exceeding "
+                    f"the {self.MAX_RECORD_BYTES}-byte per-record limit\n"
+                )
+                continue
+            key = str(event.get(self.partition_key_field) or "log-forge")[:256]
+            records.append({"Data": data, "PartitionKey": key})
+        return records
+
+    def _send(self, records: list[dict[str, Any]]) -> None:
+        """Send one chunk, retrying only the records the response flags as failed (FR-003)."""
+        for attempt in range(self.max_retries + 1):
+            response = self.client.put_records(StreamName=self.stream_name, Records=records)
+            if not response.get("FailedRecordCount"):
+                return
+            results = response.get("Records", [])
+            records = [
+                record
+                for record, result in zip(records, results)
+                if result.get("ErrorCode")
+            ]
+            if not records:
+                return
+            if attempt >= self.max_retries:
+                self.failed += len(records)
+                sys.stderr.write(
+                    f"log-forge: {len(records)} Kinesis record(s) still failing after "
+                    f"{self.max_retries + 1} attempts; abandoned\n"
+                )
+                return

log_forge/sinks/logging_sink.py

@@ -0,0 +1,104 @@
+"""LoggingSink — bridge events into the stdlib ``logging`` framework (arch §8, SPEC-007).
+
+Rather than reimplement rotating files, syslog, or the dozens of third-party log handlers that
+already exist, this sink turns each built event dict into a ``logging.LogRecord`` and dispatches it
+through a configurable logger — handing users their entire existing ``logging`` setup (handlers,
+``logging.config``, third-party handlers) for free with no new dependency. The module is named
+``logging_sink`` (not ``logging``) so it never shadows the stdlib module. Like every sink it
+receives *already-built* event dicts and knows nothing about spans or context.
+"""
+
+from __future__ import annotations
+
+import logging
+
+__all__ = ["LoggingSink"]
+
+# Level name -> stdlib numeric level; compared case-insensitively (arch §6 level names).
+_LEVELS = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+}
+
+# Attributes a LogRecord already owns; a user field must never overwrite one of these. Computed
+# from a sample record plus the two names ``logging`` itself reserves against ``extra``.
+_RESERVED = set(logging.LogRecord("", 0, "", 0, "", (), None).__dict__) | {"message", "asctime"}
+
+# Identity keys copied verbatim onto the record as flat attributes (all known non-reserved).
+_IDENTITY = (
+    "trace_id",
+    "span_id",
+    "parent_span_id",
+    "log_id",
+    "function",
+    "service",
+    "version",
+    "env",
+)
+
+
+class LoggingSink:
+    """A :class:`~log_forge.sinks.base.Sink` that dispatches each event as a ``logging.LogRecord``.
+
+    The sink only *emits* into the user's logging pipeline; it never configures loggers, handlers,
+    or formatters, and never tears the framework down. ``default_level`` (a level name) is used for
+    events whose ``level`` is unknown or missing.
+    """
+
+    def __init__(
+        self,
+        logger: logging.Logger | None = None,
+        *,
+        default_level: str = "INFO",
+    ) -> None:
+        self._logger = logger if logger is not None else logging.getLogger("log_forge")
+        self._default_level = _LEVELS.get(default_level.upper(), logging.INFO)
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Dispatch one LogRecord per event, in batch order, through the target logger (FR-001)."""
+        for event in batch:
+            self._logger.handle(self._to_record(event))
+
+    def close(self) -> None:
+        """No-op — the sink does not own the user's logging configuration (FR-005)."""
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _level_of(self, event: dict[str, object]) -> int:
+        """Map the event's textual ``level`` to a numeric level, case-insensitively (FR-002)."""
+        level = event.get("level")
+        if isinstance(level, str):
+            return _LEVELS.get(level.upper(), self._default_level)
+        return self._default_level
+
+    def _to_record(self, event: dict[str, object]) -> logging.LogRecord:
+        """Build one record: verbatim message, mapped level, structured attrs (FR-002..FR-004)."""
+        record = logging.LogRecord(
+            name=self._logger.name,
+            level=self._level_of(event),
+            pathname="",
+            lineno=0,
+            msg=event.get("message", ""),
+            args=(),  # no %-args, so a literal '%' in the message is never interpolated (FR-004)
+            exc_info=None,
+        )
+        self._attach(record, event)
+        return record
+
+    def _attach(self, record: logging.LogRecord, event: dict[str, object]) -> None:
+        """Attach identity keys + structured fields without clobbering reserved attrs (FR-003)."""
+        for key in _IDENTITY:
+            if key in event:
+                setattr(record, key, event[key])
+        fields = event.get("fields")
+        if isinstance(fields, dict):
+            # Nested payload is lossless even when a flat key collides and is skipped below.
+            setattr(record, "fields", fields)
+            for key, value in fields.items():
+                # Skip reserved LogRecord attrs, identity keys, and "fields": the sink owns
+                # record.fields (the nested payload); a field named "fields" must not overwrite it.
+                if key not in _RESERVED and key not in _IDENTITY and key != "fields":
+                    setattr(record, key, value)

log_forge/sinks/logstash.py

@@ -0,0 +1,73 @@
+"""LogstashSink — JSON lines to Logstash over HTTP or a raw TCP/UDP socket (arch §8, SPEC-009).
+
+Two mutually-exclusive modes chosen at construction:
+
+* ``LogstashSink(url=...)`` — send the batch as JSON lines over HTTP (reusing the ``HTTPSink`` core).
+* ``LogstashSink(host=..., port=...)`` — send one ``json.dumps(event) + "\\n"`` per event over a raw
+  TCP or UDP socket (reusing :class:`~log_forge.sinks._socket.SocketTransport`).
+
+Either backend handles its own bounded retry; ``close()`` releases whichever it holds.
+"""
+
+from __future__ import annotations
+
+import json
+
+from log_forge.sinks._socket import SocketTransport
+from log_forge.sinks.http import HTTPSink
+
+__all__ = ["LogstashSink"]
+
+
+class LogstashSink:
+    """A :class:`~log_forge.sinks.base.Sink` that ships JSON lines to Logstash (FR-005)."""
+
+    def __init__(
+        self,
+        url: str | None = None,
+        *,
+        host: str | None = None,
+        port: int | None = None,
+        transport: str = "tcp",
+        timeout: float = 5.0,
+        max_retries: int = 3,
+        **http_kwargs: object,
+    ) -> None:
+        if url is not None:
+            self._http: HTTPSink | None = HTTPSink(
+                url, body_format="ndjson", timeout=timeout, max_retries=max_retries,
+                **http_kwargs,  # type: ignore[arg-type]
+            )
+            self._socket: SocketTransport | None = None
+        elif host is not None and port is not None:
+            self._http = None
+            self._socket = SocketTransport(
+                host, port, transport=transport, timeout=timeout, max_retries=max_retries
+            )
+        else:
+            raise ValueError("LogstashSink requires either url= (HTTP) or host= + port= (socket)")
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Send the batch over the configured backend (FR-005)."""
+        if not batch:
+            return
+        if self._http is not None:
+            self._http.emit(batch)
+        else:
+            assert self._socket is not None
+            frames = [(json.dumps(event) + "\n").encode("utf-8") for event in batch]
+            self._socket.send_all(frames)
+
+    def close(self) -> None:
+        """Close whichever backend is held (FR-005)."""
+        if self._http is not None:
+            self._http.close()
+        elif self._socket is not None:
+            self._socket.close()
+
+    @property
+    def failed(self) -> int:
+        """Requests/messages abandoned past the retry bound, from the active backend."""
+        return self._http.failed if self._http is not None else (
+            self._socket.failed if self._socket is not None else 0
+        )

log_forge/sinks/loki.py

@@ -0,0 +1,48 @@
+"""LokiSink — push events to Grafana Loki (arch §8, SPEC-009).
+
+A specialization of :class:`~log_forge.sinks.http.HTTPSink` that builds Loki's push payload: events
+are grouped into ``streams`` by a configurable set of label keys (e.g. ``service``/``env``/
+``level``), each carrying ``values`` of ``[<nanosecond_timestamp_str>, <log_line>]``. Timestamps are
+derived by parsing the event's ISO-8601 ``timestamp`` (falling back to emit-time now).
+"""
+
+from __future__ import annotations
+
+import json
+
+from log_forge.sinks._time import epoch_nanos
+from log_forge.sinks.http import HTTPSink
+
+__all__ = ["LokiSink"]
+
+_PUSH_PATH = "/loki/api/v1/push"
+
+
+class LokiSink(HTTPSink):
+    """POST a Loki push payload (streams + labels + nanosecond values) (FR-004)."""
+
+    def __init__(self, url: str, *, labels: tuple[str, ...] = ("service", "env", "level"),
+                 **http_kwargs: object) -> None:
+        self._labels = labels
+        super().__init__(
+            url.rstrip("/") + _PUSH_PATH, body_format="json_array", **http_kwargs  # type: ignore[arg-type]
+        )
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Group events into labelled streams and POST the push payload (FR-004)."""
+        if not batch:
+            return
+        # key = sorted label tuple -> (label dict, list of [ns_ts, line])
+        streams: dict[tuple[tuple[str, str], ...], tuple[dict[str, str], list[list[str]]]] = {}
+        for event in batch:
+            labels = {key: str(event[key]) for key in self._labels if key in event}
+            stream_key = tuple(sorted(labels.items()))
+            value = [str(epoch_nanos(event.get("timestamp"))), json.dumps(event)]
+            streams.setdefault(stream_key, (labels, []))[1].append(value)
+        payload = {
+            "streams": [
+                {"stream": labels, "values": values} for labels, values in streams.values()
+            ]
+        }
+        body = json.dumps(payload).encode("utf-8")
+        self._send(body, content_type="application/json")

log_forge/sinks/mongodb.py

@@ -0,0 +1,105 @@
+"""MongoDBSink — insert events into a MongoDB collection (arch §8, SPEC-011).
+
+Near-zero impedance: events are already dicts, so ``insert_many`` takes them as-is. ``pymongo`` is
+the optional ``mongo`` extra, imported lazily inside the sink (never at module top). Inserts are
+unordered (``ordered=False``) so one bad document does not abort the rest of the batch; a
+``BulkWriteError`` is caught, its failed-insert count recorded, and the successfully-inserted
+documents retained. Write-only — querying is the downstream tool's job.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from typing import Any
+
+__all__ = ["MongoDBSink"]
+
+_BACKOFF_BASE = 0.1
+_MAX_DOC_BYTES = 16 * 1024 * 1024  # MongoDB's hard 16 MB per-document limit
+
+
+class MongoDBSink:
+    """A :class:`~log_forge.sinks.base.Sink` that inserts events into a MongoDB collection.
+
+    Attributes:
+        failed: Documents the server rejected (bulk write errors) or a whole batch abandoned past
+            the retry bound.
+        dropped_oversized: Documents dropped for exceeding the 16 MB limit.
+    """
+
+    def __init__(
+        self,
+        *,
+        client: Any = None,
+        uri: str | None = None,
+        database: str,
+        collection: str,
+        max_retries: int = 3,
+    ) -> None:
+        self._owns_client = client is None
+        if client is None:
+            from pymongo import MongoClient  # type: ignore[import-not-found]  # 'mongo' extra
+
+            client = MongoClient(uri)
+        self._client = client
+        self._collection = client[database][collection]
+        self.max_retries = max_retries
+        self.failed = 0
+        self.dropped_oversized = 0
+        self._closed = False
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Insert every event unordered; count bulk failures, retry connection errors (FR-003)."""
+        documents = self._documents(batch)
+        if not documents:
+            return
+        for attempt in range(self.max_retries + 1):
+            try:
+                self._collection.insert_many(documents, ordered=False)
+                return
+            except Exception as err:  # isolation boundary: never crash the worker (FR-006)
+                details = getattr(err, "details", None)
+                if isinstance(details, dict) and "writeErrors" in details:
+                    # BulkWriteError: unordered insert stored what it could; count the rejects and
+                    # do not retry (successes are already in, a retry would duplicate/re-error).
+                    rejects = len(details["writeErrors"])
+                    self.failed += rejects
+                    sys.stderr.write(
+                        f"log-forge: MongoDBSink had {rejects} document(s) rejected by a bulk "
+                        f"write; the rest were inserted\n"
+                    )
+                    return
+                if attempt < self.max_retries:
+                    time.sleep(_BACKOFF_BASE * (2**attempt))
+                    continue
+                self.failed += len(documents)
+                sys.stderr.write(
+                    f"log-forge: MongoDBSink abandoned {len(documents)} document(s) after "
+                    f"{self.max_retries + 1} attempts ({err!r})\n"
+                )
+                return
+
+    def close(self) -> None:
+        """Close the client only if the sink owns it; idempotent (FR-005)."""
+        if self._closed:
+            return
+        if self._owns_client:
+            self._client.close()
+        self._closed = True
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _documents(self, batch: list[dict[str, object]]) -> list[dict[str, object]]:
+        """Copy each event (so ``_id`` insertion doesn't mutate the caller's dicts), drop oversized."""
+        documents: list[dict[str, object]] = []
+        for event in batch:
+            if len(json.dumps(event).encode("utf-8")) > _MAX_DOC_BYTES:
+                self.dropped_oversized += 1
+                sys.stderr.write(
+                    "log-forge: MongoDBSink dropped a document exceeding the 16 MB limit\n"
+                )
+                continue
+            documents.append(dict(event))
+        return documents

log_forge/sinks/multi.py

@@ -0,0 +1,53 @@
+"""MultiSink — fan one batch out to several sinks (arch §8, SPEC-006 FR-002).
+
+One ``configure(sink=...)`` can echo to stdout *and* ship to SQS by wrapping both in a
+``MultiSink``. Children receive the batch sequentially in construction order; a single child
+whose ``emit`` (or ``close``) raises is isolated — the failure is counted on ``failed`` and
+logged to stderr, and its siblings still run — so one broken destination never fails the whole
+fan-out or the worker's retry. This mirrors the worker's own survive-a-sink-failure isolation
+boundary (arch §9; best-practices §7 sanctions the broad catch here).
+"""
+
+from __future__ import annotations
+
+import sys
+
+from log_forge.sinks.base import Sink
+
+__all__ = ["MultiSink"]
+
+
+class MultiSink:
+    """A :class:`~log_forge.sinks.base.Sink` that forwards each batch to every child sink.
+
+    Attributes:
+        failed: Count of child ``emit``/``close`` calls that raised and were isolated.
+    """
+
+    def __init__(self, *sinks: Sink) -> None:
+        self._sinks = sinks
+        self.failed = 0
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Forward ``batch`` to every child in construction order, isolating failures (FR-002)."""
+        for sink in self._sinks:
+            try:
+                sink.emit(batch)
+            except Exception as err:  # isolation boundary: one child must not fail the rest
+                self.failed += 1
+                sys.stderr.write(
+                    f"log-forge: MultiSink child {type(sink).__name__}.emit "
+                    f"failed and was skipped: {err!r}\n"
+                )
+
+    def close(self) -> None:
+        """Close every child, isolating a failing child so the rest still close (FR-002)."""
+        for sink in self._sinks:
+            try:
+                sink.close()
+            except Exception as err:  # close-all: an earlier failure must not skip the rest
+                self.failed += 1
+                sys.stderr.write(
+                    f"log-forge: MultiSink child {type(sink).__name__}.close "
+                    f"failed and was skipped: {err!r}\n"
+                )

log_forge/sinks/nats.py

@@ -0,0 +1,72 @@
+"""NATSSink — publish events to a NATS subject, optionally via JetStream (arch §8, §9.1, SPEC-010).
+
+A durable-buffer sink on ``nats-py`` (the optional ``nats`` extra, imported lazily). ``nats-py`` is
+async, so the sink owns a private event loop and drives each publish to completion from the
+synchronous ``emit`` (which therefore returns only after the batch is handed off). With
+``jetstream=True`` it publishes through JetStream for durable acknowledgement. ``close()`` drains and
+closes the connection.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from typing import Any
+
+__all__ = ["NATSSink"]
+
+
+class NATSSink:
+    """A :class:`~log_forge.sinks.base.Sink` that publishes events to a NATS subject."""
+
+    def __init__(
+        self,
+        subject: str,
+        *,
+        client: Any = None,
+        jetstream: bool = False,
+        servers: str | None = None,
+    ) -> None:
+        self._subject = subject
+        self._jetstream = jetstream
+        self._loop = asyncio.new_event_loop()
+        self.failed = 0
+        if client is None:
+            import nats  # type: ignore[import-not-found]  # optional 'nats' extra
+
+            client = self._loop.run_until_complete(
+                nats.connect(servers or "nats://localhost:4222")
+            )
+        self._client = client
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Drive the async publishes to completion on the managed loop (FR-007)."""
+        if not batch:
+            return
+        self._loop.run_until_complete(self._publish_all(batch))
+
+    def close(self) -> None:
+        """Drain/flush and close the connection, then close the managed loop (FR-007)."""
+        if self._loop.is_closed():
+            return
+        try:
+            self._loop.run_until_complete(self._drain())
+        finally:
+            self._loop.close()
+
+    # -- internals ----------------------------------------------------------------------
+
+    async def _publish_all(self, batch: list[dict[str, object]]) -> None:
+        target = self._client.jetstream() if self._jetstream else self._client
+        for event in batch:
+            try:
+                await target.publish(self._subject, json.dumps(event).encode("utf-8"))
+            except Exception as err:  # isolation boundary: never crash the worker (FR-011)
+                self.failed += 1
+                sys.stderr.write(f"log-forge: NATSSink publish failed: {err!r}\n")
+
+    async def _drain(self) -> None:
+        drain = getattr(self._client, "drain", None)
+        if drain is not None:
+            await drain()

log_forge/sinks/newrelic.py

@@ -0,0 +1,28 @@
+"""NewRelicSink — ship to the New Relic Log API (arch §8, SPEC-009).
+
+An :class:`~log_forge.sinks.http.HTTPSink` specialization: POSTs the batch as a JSON array to the
+region-specific Log API endpoint with an ``Api-Key`` header.
+"""
+
+from __future__ import annotations
+
+from log_forge.sinks.http import HTTPSink, merge_headers
+
+__all__ = ["NewRelicSink"]
+
+# region -> Log API host.
+_HOSTS = {"US": "log-api.newrelic.com", "EU": "log-api.eu.newrelic.com"}
+
+
+class NewRelicSink(HTTPSink):
+    """POST events to the New Relic Log API (FR-009)."""
+
+    def __init__(self, api_key: str, *, region: str = "US", **http_kwargs: object) -> None:
+        region = region.upper()
+        if region not in _HOSTS:
+            raise ValueError(f"invalid region {region!r}; expected one of {sorted(_HOSTS)}")
+        headers = merge_headers({"Api-Key": api_key}, http_kwargs)
+        super().__init__(
+            f"https://{_HOSTS[region]}/log/v1",
+            headers=headers, body_format="json_array", **http_kwargs,  # type: ignore[arg-type]
+        )