log-foundry 0.0.2.dev1__py3-none-any.whl

log_forge/sinks/_socket.py

@@ -0,0 +1,105 @@
+"""Shared raw-socket transport for LogstashSink (socket mode) and SyslogSink (SPEC-009).
+
+Sends pre-framed byte messages over TCP or UDP with bounded reconnect retry. For TCP a single
+connection is opened and reused (reconnecting on error); for UDP each message is an independent
+datagram. The transport is framing-agnostic — callers hand it the exact bytes to put on the wire
+(newline-terminated JSON for Logstash, RFC 5424 / octet-counted frames for Syslog).
+
+Socket creation goes through the module-level ``_make_tcp`` / ``_make_udp`` seams so tests can
+substitute a fake socket without any network access.
+"""
+
+from __future__ import annotations
+
+import socket
+import sys
+import time
+
+__all__ = ["SocketTransport"]
+
+_BACKOFF_BASE = 0.1  # seconds; delay for retry attempt n is _BACKOFF_BASE * 2**n
+
+
+def _make_tcp(host: str, port: int, timeout: float) -> socket.socket:
+    """Open a connected TCP socket (indirection seam for tests)."""
+    return socket.create_connection((host, port), timeout=timeout)
+
+
+def _make_udp() -> socket.socket:
+    """Open an unconnected UDP socket (indirection seam for tests)."""
+    return socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
+
+class SocketTransport:
+    """Send pre-framed messages over a TCP or UDP socket, reconnecting on error within a bound.
+
+    Attributes:
+        failed: Messages abandoned past the reconnect-retry bound.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        *,
+        transport: str = "tcp",
+        timeout: float = 5.0,
+        max_retries: int = 3,
+    ) -> None:
+        if transport not in ("tcp", "udp"):
+            raise ValueError(f"invalid transport {transport!r}; expected 'tcp' or 'udp'")
+        self._host = host
+        self._port = port
+        self._transport = transport
+        self._timeout = timeout
+        self._max_retries = max_retries
+        self._sock: socket.socket | None = None
+        self.failed = 0
+
+    def send_all(self, messages: list[bytes]) -> None:
+        """Send each pre-framed message, reconnecting on error (FR-005, FR-006)."""
+        for message in messages:
+            self._send_one(message)
+
+    def close(self) -> None:
+        """Close the held socket, if any; idempotent (FR-005, FR-012)."""
+        self._reset()
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _send_one(self, message: bytes) -> None:
+        for attempt in range(self._max_retries + 1):
+            try:
+                if self._transport == "udp":
+                    self._socket().sendto(message, (self._host, self._port))
+                else:
+                    self._socket().sendall(message)
+                return
+            except OSError as err:
+                self._reset()  # force a fresh connection on the next attempt
+                if attempt < self._max_retries:
+                    time.sleep(_BACKOFF_BASE * (2**attempt))
+                    continue
+                self.failed += 1
+                sys.stderr.write(
+                    f"log-forge: SocketTransport abandoned a message after "
+                    f"{self._max_retries + 1} attempt(s) ({err!r})\n"
+                )
+                return
+
+    def _socket(self) -> socket.socket:
+        if self._sock is None:
+            self._sock = (
+                _make_udp() if self._transport == "udp" else _make_tcp(
+                    self._host, self._port, self._timeout
+                )
+            )
+        return self._sock
+
+    def _reset(self) -> None:
+        if self._sock is not None:
+            try:
+                self._sock.close()
+            except OSError:
+                pass
+            self._sock = None

log_forge/sinks/_time.py

@@ -0,0 +1,28 @@
+"""ISO-8601 → epoch helpers for sinks that need numeric time (SPEC-009).
+
+A SPEC-001 ``LogEvent``'s ``timestamp`` is an ISO-8601 *string* (``2026-07-11T00:00:00.123Z``), not
+an epoch. Sinks like Loki (nanoseconds) and Splunk HEC (epoch seconds) derive numeric time by
+parsing that string, falling back to emit-time ``now`` when it is absent or unparseable.
+"""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime
+
+__all__ = ["epoch_seconds", "epoch_nanos"]
+
+
+def epoch_seconds(timestamp: object) -> float:
+    """Epoch seconds from a SPEC-001 ISO-8601 string; emit-time ``now`` if absent/unparseable."""
+    if isinstance(timestamp, str):
+        try:
+            return datetime.fromisoformat(timestamp).timestamp()
+        except ValueError:
+            pass
+    return time.time()
+
+
+def epoch_nanos(timestamp: object) -> int:
+    """Epoch nanoseconds (as required by Loki), derived from :func:`epoch_seconds`."""
+    return int(epoch_seconds(timestamp) * 1_000_000_000)

log_forge/sinks/base.py

@@ -0,0 +1,22 @@
+"""The sink interface (arch §8).
+
+A sink is the swappable output transport. It receives *already-built, batched* event
+dicts from the worker and knows nothing about spans or context — that dumbness is what
+makes sinks trivially interchangeable (StdoutSink, SQSSink, …). Only the Protocol lives
+here; concrete sinks arrive in later phases.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Sink(Protocol):
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Ship a batch of serialized event dicts."""
+        ...
+
+    def close(self) -> None:
+        """Flush and release any resources."""
+        ...

log_forge/sinks/callback.py

@@ -0,0 +1,42 @@
+"""CallbackSink — turn a plain callable into a Sink (arch §8, SPEC-006 FR-001).
+
+The ultimate escape hatch: point log-forge at any destination by handing it a function,
+without writing a ``Sink`` subclass. Like every sink it receives *already-built* event dicts
+and knows nothing about spans or context — that dumbness is what makes sinks swappable.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+__all__ = ["CallbackSink"]
+
+
+class CallbackSink:
+    """A :class:`~log_forge.sinks.base.Sink` that delegates each batch to a callable.
+
+    Attributes are internal; the observable contract is that ``emit`` hands the batch to
+    ``fn`` unchanged and ``close`` invokes ``on_close`` once when one was supplied.
+    """
+
+    def __init__(
+        self,
+        fn: Callable[[list[dict[str, object]]], None],
+        *,
+        on_close: Callable[[], None] | None = None,
+    ) -> None:
+        self._fn = fn
+        self._on_close = on_close
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Hand ``batch`` to the callable, unchanged and exactly once (FR-001).
+
+        Any exception the callable raises propagates out — the worker's retry/backoff path
+        handles it; this sink never swallows it.
+        """
+        self._fn(batch)
+
+    def close(self) -> None:
+        """Call ``on_close`` once if one was supplied, otherwise a no-op (FR-001)."""
+        if self._on_close is not None:
+            self._on_close()

log_forge/sinks/clickhouse.py

@@ -0,0 +1,111 @@
+"""ClickHouseSink — batch-insert events into a ClickHouse table (arch §8, SPEC-011).
+
+ClickHouse is the columnar, observability-scale favorite. ``clickhouse-connect`` is the optional
+``clickhouse`` extra, imported lazily. Each event maps to a row of extracted typed columns plus the
+full event as a ``String`` column, inserted in a single ``insert`` call per chunk. An optional
+idempotent ``MergeTree`` ``create_table`` convenience is off by default. Write-only.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from typing import Any
+
+from log_forge.sinks._chunk import chunk_list, valid_identifier
+
+__all__ = ["ClickHouseSink"]
+
+_BACKOFF_BASE = 0.1
+
+# Extracted columns (typed in the MergeTree schema); ``event`` (full JSON String) is stored too.
+_COLUMNS = (
+    "timestamp", "level", "trace_id", "span_id", "function", "service", "duration_ms", "status"
+)
+_COLUMN_NAMES = [*_COLUMNS, "event"]
+
+# Per-column DDL types for the optional create_table convenience (all nullable to tolerate missing
+# keys — e.g. span-start events have no duration_ms/status).
+_COLUMN_TYPES = {
+    "timestamp": "Nullable(String)",
+    "level": "Nullable(String)",
+    "trace_id": "Nullable(String)",
+    "span_id": "Nullable(String)",
+    "function": "Nullable(String)",
+    "service": "Nullable(String)",
+    "duration_ms": "Nullable(Float64)",
+    "status": "Nullable(String)",
+}
+
+
+class ClickHouseSink:
+    """A :class:`~log_forge.sinks.base.Sink` that batch-inserts events into a ClickHouse table."""
+
+    def __init__(
+        self,
+        table: str,
+        *,
+        client: Any = None,
+        dsn: str | None = None,
+        create_table: bool = False,
+        chunk_size: int = 1000,
+        max_retries: int = 3,
+    ) -> None:
+        self._table = valid_identifier(table)
+        self._chunk_size = chunk_size
+        self.max_retries = max_retries
+        self.failed = 0
+        self._closed = False
+        self._owns_client = client is None
+        if client is None:
+            import clickhouse_connect  # type: ignore[import-not-found]  # 'clickhouse' extra
+
+            client = clickhouse_connect.get_client(dsn=dsn)
+        self.client = client
+        if create_table:
+            self._ensure_schema()
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Insert each chunk as one columnar ``insert`` call, retrying on failure (FR-002)."""
+        if not batch:
+            return
+        for chunk in chunk_list(batch, self._chunk_size):
+            rows = [self._row(event) for event in chunk]
+            self._insert(rows)
+
+    def close(self) -> None:
+        """Close the client only if the sink owns it; idempotent (FR-005)."""
+        if self._closed:
+            return
+        if self._owns_client:
+            self.client.close()
+        self._closed = True
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _row(self, event: dict[str, object]) -> list[object]:
+        return [*(event.get(col) for col in _COLUMNS), json.dumps(event)]
+
+    def _insert(self, rows: list[list[object]]) -> None:
+        for attempt in range(self.max_retries + 1):
+            try:
+                self.client.insert(self._table, data=rows, column_names=_COLUMN_NAMES)
+                return
+            except Exception as err:  # isolation boundary: never crash the worker (FR-006)
+                if attempt < self.max_retries:
+                    time.sleep(_BACKOFF_BASE * (2**attempt))
+                    continue
+                self.failed += len(rows)
+                sys.stderr.write(
+                    f"log-forge: ClickHouseSink abandoned {len(rows)} row(s) after "
+                    f"{self.max_retries + 1} attempts ({err!r})\n"
+                )
+                return
+
+    def _ensure_schema(self) -> None:
+        columns = ", ".join(f"{col} {_COLUMN_TYPES[col]}" for col in _COLUMNS)
+        self.client.command(
+            f"CREATE TABLE IF NOT EXISTS {self._table} "
+            f"({columns}, event String) ENGINE = MergeTree ORDER BY tuple()"
+        )

log_forge/sinks/datadog.py

@@ -0,0 +1,53 @@
+"""DatadogSink — ship to Datadog's logs intake (arch §8, SPEC-009).
+
+An :class:`~log_forge.sinks.http.HTTPSink` specialization: POSTs the batch as a JSON array to the
+region-specific logs intake with a ``DD-API-KEY`` header, enriching each entry with ``ddsource``,
+``service``, and ``ddtags`` from configuration / the event.
+"""
+
+from __future__ import annotations
+
+import json
+
+from log_forge.sinks.http import HTTPSink, merge_headers
+
+__all__ = ["DatadogSink"]
+
+_DDSOURCE = "log-forge"
+
+
+class DatadogSink(HTTPSink):
+    """POST events to Datadog's logs intake (FR-007)."""
+
+    def __init__(
+        self,
+        api_key: str,
+        *,
+        site: str = "datadoghq.com",
+        service: str | None = None,
+        ddtags: str | None = None,
+        **http_kwargs: object,
+    ) -> None:
+        self._service = service
+        self._ddtags = ddtags
+        headers = merge_headers({"DD-API-KEY": api_key}, http_kwargs)
+        super().__init__(
+            f"https://http-intake.logs.{site}/api/v2/logs",
+            headers=headers, body_format="json_array", **http_kwargs,  # type: ignore[arg-type]
+        )
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """POST each event (enriched) as one JSON array (FR-007)."""
+        if not batch:
+            return
+        body = json.dumps([self._entry(event) for event in batch]).encode("utf-8")
+        self._send(body, content_type="application/json")
+
+    def _entry(self, event: dict[str, object]) -> dict[str, object]:
+        entry = dict(event)
+        entry["ddsource"] = _DDSOURCE
+        if self._service is not None:
+            entry["service"] = self._service
+        if self._ddtags is not None:
+            entry["ddtags"] = self._ddtags
+        return entry

log_forge/sinks/elasticsearch.py

@@ -0,0 +1,68 @@
+"""ElasticsearchSink / OpenSearchSink — index events via the ``_bulk`` API (arch §8, SPEC-009).
+
+A thin specialization of :class:`~log_forge.sinks.http.HTTPSink`: each event becomes an action line
+(``{"index": {"_index": target}}``) followed by its source line, POSTed to ``_bulk`` as newline-
+delimited JSON. The bulk response is inspected per item so a partial failure is counted and logged
+without discarding the successfully-indexed items. OpenSearch speaks the same bulk protocol, so
+``OpenSearchSink`` is a straight reuse (endpoint/auth differ only by configuration).
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+from log_forge.sinks.http import HTTPSink
+
+__all__ = ["ElasticsearchSink", "OpenSearchSink"]
+
+
+class ElasticsearchSink(HTTPSink):
+    """POST events to an Elasticsearch ``_bulk`` endpoint, parsing per-item errors (FR-003).
+
+    Attributes:
+        item_errors: Count of bulk items the server reported as failed (distinct from ``failed``,
+            which counts whole requests abandoned past the retry bound).
+    """
+
+    def __init__(self, url: str, *, index: str, auth: str | tuple[str, str] | None = None,
+                 **http_kwargs: object) -> None:
+        self._index = index
+        super().__init__(
+            url.rstrip("/") + "/_bulk", auth=auth, body_format="ndjson", **http_kwargs  # type: ignore[arg-type]
+        )
+        self.item_errors = 0
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Build the ``_bulk`` NDJSON payload, POST it, and parse the response items (FR-003)."""
+        if not batch:
+            return
+        lines: list[str] = []
+        for event in batch:
+            lines.append(json.dumps({"index": {"_index": self._index}}))
+            lines.append(json.dumps(event))
+        body = ("\n".join(lines) + "\n").encode("utf-8")  # bulk must be newline-terminated
+        payload = self._send(body, content_type="application/x-ndjson")
+        if payload is not None:
+            self._parse_bulk_response(payload)
+
+    def _parse_bulk_response(self, payload: bytes) -> None:
+        """Count items the bulk response flagged as errors; a partial failure keeps the rest."""
+        try:
+            data = json.loads(payload)
+        except (ValueError, TypeError):
+            return
+        if not isinstance(data, dict) or not data.get("errors"):
+            return
+        errors = 0
+        for item in data.get("items", []):
+            result: object = next(iter(item.values()), {}) if isinstance(item, dict) else {}
+            if isinstance(result, dict) and result.get("error"):
+                errors += 1
+        if errors:
+            self.item_errors += errors
+            sys.stderr.write(f"log-forge: ElasticsearchSink saw {errors} failed bulk item(s)\n")
+
+
+class OpenSearchSink(ElasticsearchSink):
+    """OpenSearch reuses the Elasticsearch ``_bulk`` protocol verbatim (FR-003)."""

log_forge/sinks/eventhubs.py

@@ -0,0 +1,110 @@
+"""AzureEventHubsSink — send events to an Azure Event Hub (arch §8, §9.1, SPEC-010).
+
+A durable-buffer sink on ``azure-eventhub`` (the optional ``azure-eventhubs`` extra, imported
+lazily). Events are packed into one or more ``EventDataBatch`` objects respecting the 1 MB per-batch
+limit — the SDK signals a full batch by raising ``ValueError`` from ``batch.add`` — and each full
+batch is sent. A single event too large for an even-empty batch is dropped with a counted warning;
+send errors are retried within a bound. ``close()`` closes the producer.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from typing import Any
+
+__all__ = ["AzureEventHubsSink"]
+
+_BACKOFF_BASE = 0.1
+
+
+class AzureEventHubsSink:
+    """A :class:`~log_forge.sinks.base.Sink` that sends events to an Azure Event Hub."""
+
+    def __init__(
+        self,
+        *,
+        producer: Any = None,
+        connection_str: str | None = None,
+        eventhub: str | None = None,
+        max_retries: int = 3,
+    ) -> None:
+        if producer is None:
+            if connection_str is None:
+                raise ValueError(
+                    "AzureEventHubsSink requires connection_str when no producer is injected"
+                )
+            from azure.eventhub import (  # type: ignore[import-not-found]  # 'azure-eventhubs'
+                EventHubProducerClient,
+            )
+
+            producer = EventHubProducerClient.from_connection_string(
+                connection_str, eventhub_name=eventhub
+            )
+        self.producer = producer
+        self.max_retries = max_retries
+        self.failed = 0
+        self.dropped_oversized = 0
+
+    def emit(self, batch: list[dict[str, object]]) -> None:
+        """Pack events into ≤ 1 MB EventDataBatches and send each; drop oversized events (FR-009)."""
+        if not batch:
+            return
+        event_data_cls = _event_data_cls()
+        current = self.producer.create_batch()
+        for event in batch:
+            data = event_data_cls(json.dumps(event).encode("utf-8"))
+            if _try_add(current, data):
+                continue
+            # current batch is full: send it and start a fresh one for this event.
+            self._send(current)
+            current = self.producer.create_batch()
+            if not _try_add(current, data):
+                self.dropped_oversized += 1
+                sys.stderr.write(
+                    "log-forge: AzureEventHubsSink dropped an event too large for an "
+                    "empty 1 MB batch\n"
+                )
+        self._send(current)
+
+    def close(self) -> None:
+        """Close the producer (FR-009)."""
+        self.producer.close()
+
+    # -- internals ----------------------------------------------------------------------
+
+    def _send(self, event_batch: Any) -> None:
+        """Send one EventDataBatch (skipping an empty one), retrying failures (FR-009, FR-011)."""
+        if len(event_batch) == 0:
+            return
+        for attempt in range(self.max_retries + 1):
+            try:
+                self.producer.send_batch(event_batch)
+                return
+            except Exception as err:  # isolation boundary: never crash the worker (FR-011)
+                if attempt < self.max_retries:
+                    time.sleep(_BACKOFF_BASE * (2**attempt))
+                    continue
+                self.failed += len(event_batch)
+                sys.stderr.write(
+                    f"log-forge: AzureEventHubsSink abandoned a batch of {len(event_batch)} "
+                    f"event(s) after {self.max_retries + 1} attempts ({err!r})\n"
+                )
+                return
+
+
+def _try_add(event_batch: Any, data: Any) -> bool:
+    """Add ``data`` to the batch; ``False`` when the batch signals it is full (``ValueError``)."""
+    try:
+        event_batch.add(data)
+        return True
+    except ValueError:
+        return False
+
+
+def _event_data_cls() -> Any:
+    """Return ``azure.eventhub.EventData`` (indirection seam for tests)."""
+    from azure.eventhub import EventData  # 'azure-eventhubs' extra (type-ignored in __init__)
+
+    return EventData