log-foundry 0.0.2.dev1__py3-none-any.whl

log_forge/__init__.py

@@ -0,0 +1,45 @@
+"""log_forge — consistent, structured logs per decorated function call.
+
+Public façade (module-function shape). Exposes configuration, the ``@trace`` decorator, the
+``debug/info/warning/error/critical`` emitters, ``set_baggage``, and ``shutdown`` (graceful
+drain of the background worker).
+"""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _dist_version
+
+from log_forge.api import critical, debug, error, info, set_baggage, warning
+from log_forge.config import configure, get_config
+from log_forge.decorator import trace
+
+try:
+    # Distribution name ("log-foundry") differs from the import name ("log_forge").
+    __version__ = _dist_version("log-foundry")
+except PackageNotFoundError:  # running from a source tree that isn't installed
+    __version__ = "0.0.0"
+
+
+def shutdown() -> None:
+    """Flush buffered events and close the sink, blocking until drained. Idempotent.
+
+    Also registered via ``atexit``; call it explicitly before a fast process exit when you
+    want to be certain the tail of the queue reached the sink (SPEC-004 FR-005).
+    """
+    from log_forge.decorator import _shutdown_worker
+
+    _shutdown_worker()
+
+
+__all__ = [
+    "configure",
+    "get_config",
+    "trace",
+    "debug",
+    "info",
+    "warning",
+    "error",
+    "critical",
+    "set_baggage",
+    "shutdown",
+    "__version__",
+]

log_forge/api.py

@@ -0,0 +1,89 @@
+"""User-facing log emitters + baggage re-export (SPEC-002, arch §6).
+
+The level functions (``debug``/``info``/``warning``/``error``/``critical``) let user code emit
+its own structured events *inside* a decorated call: each appends one event (built via the
+SPEC-001 :func:`~log_forge.model.build_event`, stamped with current baggage) to the current
+span's queue, so the whole call's logs flush together at span end. A level call made with no
+active span is not dropped — it becomes a standalone one-event span with a fresh ``trace_id``,
+flushed directly (FR-004). ``echo=True`` additionally surfaces a human-readable line to the
+console synchronously (FR-002), without waiting for the async flush; the event still reaches
+the sink via the normal path.
+"""
+
+from __future__ import annotations
+
+from log_forge import context
+from log_forge.config import _ensure_sink
+from log_forge.console import ConsoleWriter
+from log_forge.context import set_baggage
+from log_forge.ids import new_span_id, new_trace_id
+from log_forge.model import Span, build_event
+
+__all__ = [
+    "debug",
+    "info",
+    "warning",
+    "error",
+    "critical",
+    "set_baggage",
+]
+
+# One console writer per process (default stream sys.stderr). Configurable overrides are
+# deferred (SPEC-002 Out of Scope); only explicit per-call echo= ships here.
+_console = ConsoleWriter()
+
+
+def _log(level: str, message: str, echo: bool, fields: dict[str, object]) -> None:
+    """Build one ``level`` event and route it, honoring ``echo``.
+
+    Inside a span the event is appended to that span's queue (flushed together at span end).
+    With no active span (orphan) it becomes a standalone one-event span — fresh ``trace_id``,
+    ``parent_span_id=None`` — flushed directly so nothing is dropped (FR-004). ``echo`` writes
+    the same event as a human-readable console line, additively.
+    """
+    baggage = context.get_baggage()
+    span = context.current_span()
+    if span is not None:
+        event = build_event(span, level, message, fields=fields, baggage=baggage)
+        span.events.append(event)
+    else:
+        # Orphan: no active span. Emit a standalone single-event span (fresh trace) so the
+        # log is recorded, not silently dropped. Resolve the sink via _ensure_sink (as the
+        # decorator does) so a zero-config orphan log falls back to StdoutSink instead of
+        # crashing. SPEC-004's worker will later own this direct handoff.
+        orphan = Span(
+            trace_id=new_trace_id(),
+            span_id=new_span_id(),
+            parent_span_id=None,
+            name=message,
+            start_ts=0.0,
+        )
+        event = build_event(orphan, level, message, fields=fields, baggage=baggage)
+        _ensure_sink().emit([event])
+    if echo:
+        _console.write(event)
+
+
+def debug(message: str, *, echo: bool = False, **fields: object) -> None:
+    """Emit a ``DEBUG`` event on the current span (or a standalone orphan span)."""
+    _log("DEBUG", message, echo, fields)
+
+
+def info(message: str, *, echo: bool = False, **fields: object) -> None:
+    """Emit an ``INFO`` event on the current span (or a standalone orphan span)."""
+    _log("INFO", message, echo, fields)
+
+
+def warning(message: str, *, echo: bool = False, **fields: object) -> None:
+    """Emit a ``WARNING`` event on the current span (or a standalone orphan span)."""
+    _log("WARNING", message, echo, fields)
+
+
+def error(message: str, *, echo: bool = False, **fields: object) -> None:
+    """Emit an ``ERROR`` event on the current span (or a standalone orphan span)."""
+    _log("ERROR", message, echo, fields)
+
+
+def critical(message: str, *, echo: bool = False, **fields: object) -> None:
+    """Emit a ``CRITICAL`` event on the current span (or a standalone orphan span)."""
+    _log("CRITICAL", message, echo, fields)

log_forge/config.py

@@ -0,0 +1,79 @@
+"""Global configuration — set once at startup (arch §7, guide Phase 1).
+
+Every log event needs ``service`` / ``version`` / ``env`` stamped on it, and the decorator
+and worker both need to find the configured sink. A single module-level singleton is the
+simplest thing that lets the rest of the code stay decoupled from *how* it was configured.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    # Type-only import: keeps ``config`` free of any runtime dependency on ``sinks``
+    # (no import cycle) while still typing the sink field. ``from __future__ import
+    # annotations`` means this name is never evaluated at runtime.
+    from log_forge.sinks.base import Sink
+
+
+@dataclass
+class Config:
+    """Process-wide settings stamped onto every event / consulted by the pipeline."""
+
+    service: str = "unknown"
+    version: str = "0.0.0"
+    env: str = "dev"
+    sink: Sink | None = None
+    defaults: dict[str, object] = field(default_factory=dict)
+
+
+_config = Config()  # module-level singleton; the whole library reads through get_config()
+
+
+def configure(
+    *,
+    service: str | None = None,
+    version: str | None = None,
+    env: str | None = None,
+    sink: Sink | None = None,
+    defaults: dict[str, object] | None = None,
+) -> None:
+    """Patch the global config. Call once at startup.
+
+    Only the arguments you pass are applied, so repeated calls compose rather than reset.
+    If no sink has ever been set, defaults to :class:`~log_forge.sinks.stdout.StdoutSink`
+    (the zero-dependency dev default, arch §8) once that phase lands.
+    """
+    if service is not None:
+        _config.service = service
+    if version is not None:
+        _config.version = version
+    if env is not None:
+        _config.env = env
+    if sink is not None:
+        _config.sink = sink
+    if defaults is not None:
+        _config.defaults = dict(defaults)
+
+    _ensure_sink()
+
+
+def get_config() -> Config:
+    """Return the current global config singleton."""
+    return _config
+
+
+def _ensure_sink() -> Sink:
+    """Return the active sink, applying the ``StdoutSink`` default if none was configured.
+
+    Centralizes the zero-dependency default (arch §8) so both ``configure()`` and the flush
+    path resolve a sink the same way — a decorated call never crashes just because the user
+    hasn't called ``configure()`` yet; it emits to stdout. The local import defers the
+    ``sinks`` dependency and avoids a top-level import cycle (arch §7).
+    """
+    if _config.sink is None:
+        from log_forge.sinks.stdout import StdoutSink
+
+        _config.sink = StdoutSink()
+    return _config.sink

log_forge/console.py

@@ -0,0 +1,30 @@
+"""Console echo — a synchronous, human-readable output path (SPEC-002 FR-002).
+
+Separate from the async :class:`~log_forge.sinks.base.Sink`: where the sink ships structured
+JSON downstream, the console writer surfaces a plain ``LEVEL   message`` line *immediately*
+(to a terminal user or a Lambda's stdout → CloudWatch) so an operator sees it without waiting
+for the async flush. It is deliberately dumb — it renders an already-built event dict and
+knows nothing about spans. Echo is additive: an echoed event still rides the normal pipeline
+to the sink.
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import TextIO
+
+__all__ = ["ConsoleWriter"]
+
+
+class ConsoleWriter:
+    """Render events as human-readable ``LEVEL   message`` lines to ``stream``."""
+
+    def __init__(self, stream: TextIO | None = None) -> None:
+        self._stream = stream if stream is not None else sys.stderr
+
+    def write(self, event: dict[str, object]) -> None:
+        """Write one ``{level:<7} {message}`` line and flush immediately."""
+        level = str(event["level"])
+        message = str(event["message"])
+        self._stream.write(f"{level:<7} {message}\n")
+        self._stream.flush()

log_forge/context.py

@@ -0,0 +1,62 @@
+"""Context propagation: span stack + baggage via contextvars (arch §5, guide Phase 4).
+
+``contextvars`` (not thread-locals) is correct under both threads and asyncio — each task
+inherits its own copy — so ``log_forge.info(...)`` can find "the span I'm inside" with no
+manual passing. This module holds a *stack* of active spans (the top is the current span and
+the parent of the next nested call) plus trace-scoped baggage.
+
+Two footguns, both avoided below:
+  * Never mutate a ContextVar's default mutable value — the ``()`` / ``{}`` defaults are shared
+    across all contexts. Always ``.set()`` a new tuple/dict.
+  * Use the token/``reset`` pattern, not a manual pop — ``reset(token)`` restores the exact
+    prior state even when tasks branch.
+"""
+
+from __future__ import annotations
+
+import contextvars
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from log_forge.model import Span
+
+__all__ = [
+    "current_span",
+    "push_span",
+    "pop_span",
+    "get_baggage",
+    "set_baggage",
+]
+
+_span_stack: contextvars.ContextVar[tuple[Span, ...]] = contextvars.ContextVar(
+    "log_forge_span_stack", default=()
+)
+_baggage: contextvars.ContextVar[dict[str, object]] = contextvars.ContextVar(
+    "log_forge_baggage", default={}
+)
+
+
+def current_span() -> Span | None:
+    """Return the innermost active span, or ``None`` when no span is active."""
+    stack = _span_stack.get()
+    return stack[-1] if stack else None
+
+
+def push_span(span: Span) -> contextvars.Token[tuple[Span, ...]]:
+    """Push ``span`` onto the stack; return a token to hand back to :func:`pop_span`."""
+    return _span_stack.set((*_span_stack.get(), span))
+
+
+def pop_span(token: contextvars.Token[tuple[Span, ...]]) -> None:
+    """Restore the span stack to its state before the matching :func:`push_span`."""
+    _span_stack.reset(token)
+
+
+def get_baggage() -> dict[str, object]:
+    """Return the current trace's baggage (do not mutate the returned dict in place)."""
+    return _baggage.get()
+
+
+def set_baggage(**kv: object) -> None:
+    """Merge key/values into the current trace's baggage (replaces with a new dict)."""
+    _baggage.set({**_baggage.get(), **kv})

log_forge/decorator.py

@@ -0,0 +1,157 @@
+"""The ``@trace`` decorator — synchronous and async (arch §4–5, guide Phases 6, 8).
+
+Opens a span on enter and closes it on exit (success *or* exception), maintaining the
+trace/parent hierarchy from the context stack, then flushes the finished span straight to
+the configured sink. The decorator is **non-swallowing**: it records the failure and
+re-raises the original exception unchanged (arch §4).
+
+At decoration time :func:`asyncio.iscoroutinefunction` selects a sync or async wrapper. The
+two are deliberate near-duplicates — the only difference is ``await fn(...)`` — because the
+sync/async split is a hard boundary; ``contextvars`` already propagates the span stack and
+baggage correctly across ``await`` points and concurrent tasks (arch §5), so the async span
+opens when the coroutine actually runs and closes when it finishes, with no new machinery.
+
+Flushing is synchronous here; the background worker (SPEC-004) later swaps only
+:func:`_flush` for a non-blocking handoff — the lifecycle below is untouched.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import atexit
+import functools
+import threading
+from collections.abc import Callable
+from time import monotonic
+from typing import Any, TypeVar, cast, overload
+
+from log_forge import context
+from log_forge.config import _ensure_sink
+from log_forge.ids import new_span_id, new_trace_id
+from log_forge.model import Span, end_event, start_event
+from log_forge.worker import Worker
+
+__all__ = ["trace"]
+
+# One background worker per process (SPEC-004), created lazily from the configured sink on the
+# first flush. The double-checked lock makes concurrent first-flushes create exactly one.
+_worker: Worker | None = None
+_worker_lock = threading.Lock()
+_atexit_registered = False  # register the drain exactly once per process, not per worker
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def _open_span(name: str, defaults: dict[str, object] | None) -> Span:
+    """Mint a span, inheriting the trace/parent from the current context (arch §3)."""
+    parent = context.current_span()
+    span = Span(
+        trace_id=parent.trace_id if parent else new_trace_id(),
+        span_id=new_span_id(),
+        parent_span_id=parent.span_id if parent else None,
+        name=name,
+        start_ts=monotonic(),
+        defaults=defaults or {},
+    )
+    span.events.append(start_event(span))
+    return span
+
+
+def _get_worker() -> Worker:
+    """Return the process worker, creating it lazily from the configured sink (FR-006).
+
+    Registers the graceful drain via ``atexit`` exactly once, on first creation, so a program
+    that logs and exits immediately still flushes its buffered events.
+    """
+    global _worker, _atexit_registered
+    if _worker is None:
+        with _worker_lock:
+            if _worker is None:
+                if not _atexit_registered:
+                    atexit.register(_shutdown_worker)
+                    _atexit_registered = True
+                _worker = Worker(_ensure_sink())
+    return _worker
+
+
+def _shutdown_worker() -> None:
+    """Drain + close the process worker if one was created (idempotent). Backs ``shutdown()``."""
+    if _worker is not None:
+        _worker.shutdown()
+
+
+def _flush(span: Span) -> None:
+    """Hand the finished span's events to the background worker — non-blocking (FR-001).
+
+    Resolves/creates the worker via :func:`_get_worker` (whose sink comes from ``_ensure_sink``,
+    so a zero-config ``@trace`` still falls back to ``StdoutSink`` rather than crashing).
+    """
+    _get_worker().submit(span.events)
+
+
+def _close_span(span: Span, status: str, exc: BaseException | None) -> None:
+    """Append the end event (so the flushed queue is complete), then flush."""
+    span.events.append(end_event(span, status, exc))
+    _flush(span)
+
+
+@overload
+def trace(func: F) -> F: ...
+@overload
+def trace(
+    *, name: str | None = ..., defaults: dict[str, object] | None = ...
+) -> Callable[[F], F]: ...
+
+
+def trace(
+    func: F | None = None,
+    *,
+    name: str | None = None,
+    defaults: dict[str, object] | None = None,
+) -> F | Callable[[F], F]:
+    """Trace a function call as a span. Usable bare (``@trace``) or with args.
+
+    ``@trace(name=..., defaults=...)`` overrides the span name (default:
+    ``func.__qualname__``) and adds per-decorator default fields.
+    """
+
+    def decorate(fn: F) -> F:
+        if asyncio.iscoroutinefunction(fn):
+
+            @functools.wraps(fn)
+            async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                span = _open_span(name or fn.__qualname__, defaults)
+                token = context.push_span(span)
+                try:
+                    result = await fn(*args, **kwargs)
+                    _close_span(span, "ok", None)
+                    return result
+                except BaseException as exc:
+                    # Non-swallowing (arch §4): record the error, then re-raise unchanged.
+                    # BaseException covers asyncio.CancelledError — a cancelled coroutine is
+                    # recorded as an error end event, never left with an unclosed span.
+                    _close_span(span, "error", exc)
+                    raise
+                finally:
+                    context.pop_span(token)
+
+            return cast(F, async_wrapper)
+
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            span = _open_span(name or fn.__qualname__, defaults)
+            token = context.push_span(span)
+            try:
+                result = fn(*args, **kwargs)
+                _close_span(span, "ok", None)
+                return result
+            except BaseException as exc:
+                # Non-swallowing (arch §4): record the error, then re-raise unchanged.
+                _close_span(span, "error", exc)
+                raise
+            finally:
+                context.pop_span(token)
+
+        return cast(F, wrapper)
+
+    return decorate(func) if func is not None else decorate

log_forge/ids.py

@@ -0,0 +1,29 @@
+"""W3C Trace Context-compatible id generation (arch §3.1, guide Phase 2).
+
+``trace_id`` and ``span_id`` use the W3C wire formats (not arbitrary UUIDs) so a future
+"adopt the inbound request's trace" feature is just a header parse (arch §12) and the
+emitted records stay compatible with standard distributed-tracing tooling. ``log_id`` is
+internal-only, so a UUID is fine.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+
+__all__ = ["new_trace_id", "new_span_id", "new_log_id"]
+
+
+def new_trace_id() -> str:
+    """Return a fresh trace id: 32 lowercase hex chars (16 random bytes)."""
+    return os.urandom(16).hex()
+
+
+def new_span_id() -> str:
+    """Return a fresh span id: 16 lowercase hex chars (8 random bytes)."""
+    return os.urandom(8).hex()
+
+
+def new_log_id() -> str:
+    """Return a fresh log id: a UUID4 hex string (internal-only, format is our choice)."""
+    return uuid.uuid4().hex

log_forge/model.py

@@ -0,0 +1,106 @@
+"""Span model + log-event construction and serialization (arch §6, guide Phase 3).
+
+This is the heart of "structured, never free-form": every event is assembled here into one
+identical JSON shape, which is what makes logs queryable downstream. This module only
+*builds* records — it deliberately imports neither ``context`` nor ``decorator`` and does
+not know where the "current" span lives (arch §6 watch-out).
+"""
+
+from __future__ import annotations
+
+import time
+import traceback
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+__all__ = ["Span", "build_event", "start_event", "end_event"]
+
+# Auto-generated span-boundary event messages. Tests assert on contract fields
+# (``status``, ``trace_id``), never on this text — rename freely.
+_START_MESSAGE = "span.start"
+_END_MESSAGE = "span.end"
+
+
+@dataclass
+class Span:
+    """One decorated function call: its identity, timing, and buffered events.
+
+    ``start_ts`` is a ``time.monotonic()`` reading (not wall-clock) so ``duration_ms`` can
+    never go negative across a clock change. ``defaults`` are per-decorator field overrides;
+    ``events`` is the queue flushed together at span end.
+    """
+
+    trace_id: str
+    span_id: str
+    parent_span_id: str | None
+    name: str
+    start_ts: float
+    defaults: dict[str, object] = field(default_factory=dict)
+    events: list[dict[str, object]] = field(default_factory=list)
+
+
+def _iso_now() -> str:
+    """Return the current UTC time as ISO-8601 with millisecond precision and a 'Z'."""
+    now = datetime.now(timezone.utc)
+    return f"{now:%Y-%m-%dT%H:%M:%S}.{now.microsecond // 1000:03d}Z"
+
+
+def build_event(
+    span: Span,
+    level: str,
+    message: str,
+    *,
+    fields: dict[str, object],
+    baggage: dict[str, object],
+) -> dict[str, object]:
+    """Assemble one log record in the arch §6 schema.
+
+    Field precedence, lowest to highest: config ``defaults`` → ``span.defaults`` → ``baggage``
+    → per-call ``fields`` (arch §5.1). Later sources win on a key conflict.
+    """
+    from log_forge.config import get_config
+    from log_forge.ids import new_log_id
+
+    cfg = get_config()
+    merged: dict[str, object] = {**cfg.defaults, **span.defaults, **baggage, **fields}
+    return {
+        "timestamp": _iso_now(),
+        "level": level,
+        "message": message,
+        "trace_id": span.trace_id,
+        "span_id": span.span_id,
+        "parent_span_id": span.parent_span_id,
+        "log_id": new_log_id(),
+        "function": span.name,
+        "service": cfg.service,
+        "version": cfg.version,
+        "env": cfg.env,
+        "fields": merged,
+    }
+
+
+def start_event(span: Span) -> dict[str, object]:
+    """Build the span-start boundary event."""
+    return build_event(span, "INFO", _START_MESSAGE, fields={}, baggage={})
+
+
+def end_event(
+    span: Span,
+    status: str,
+    exc: BaseException | None = None,
+) -> dict[str, object]:
+    """Build the span-end boundary event.
+
+    Adds ``duration_ms`` (from a monotonic delta), ``status`` (``"ok"``/``"error"``), and on
+    failure a nested ``error`` with the exception type and formatted stack (arch §6).
+    """
+    level = "INFO" if status == "ok" else "ERROR"
+    event = build_event(span, level, _END_MESSAGE, fields={}, baggage={})
+    event["duration_ms"] = (time.monotonic() - span.start_ts) * 1000.0
+    event["status"] = status
+    if exc is not None:
+        event["error"] = {
+            "type": type(exc).__name__,
+            "stack": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
+        }
+    return event

log_forge/sinks/_chunk.py

@@ -0,0 +1,58 @@
+"""Shared batch-chunking for the queue/stream sinks (SPEC-010).
+
+The worker batches by its own count/time; each transport then re-chunks to *its* per-request limits
+(no assumption the incoming batch already fits). This mirrors the SPEC-005 ``SQSSink`` conventions:
+split into groups within a count limit **and** a total-byte limit, assuming each item already fits a
+single request (callers drop oversized items first, counting ``dropped_oversized``).
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Iterator
+from typing import TypeVar
+
+__all__ = ["chunk_items", "chunk_list", "valid_identifier"]
+
+_T = TypeVar("_T")
+
+_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def chunk_list(items: list[_T], size: int) -> Iterator[list[_T]]:
+    """Yield ``items`` in successive slices of at most ``size`` (a positive chunk size)."""
+    for start in range(0, len(items), size):
+        yield items[start : start + size]
+
+
+def valid_identifier(name: str) -> str:
+    """Return ``name`` if it is a plain SQL identifier, else raise ``ValueError``.
+
+    Table names are config, not untrusted input, but they are interpolated into DDL/DML (SQL cannot
+    parameterize identifiers), so restrict them to ``[A-Za-z_][A-Za-z0-9_]*`` to foreclose injection.
+    """
+    if not _IDENTIFIER.match(name):
+        raise ValueError(f"invalid table name {name!r}; expected a plain SQL identifier")
+    return name
+
+
+def chunk_items(
+    items: list[_T], *, max_count: int, max_bytes: int, size_of: Callable[[_T], int]
+) -> Iterator[list[_T]]:
+    """Yield groups of ≤ ``max_count`` items whose ``size_of`` sizes sum to ≤ ``max_bytes``.
+
+    Each item is assumed to fit one request on its own (``size_of(item) <= max_bytes``); oversized
+    items are the caller's responsibility to filter out beforehand (counting ``dropped_oversized``).
+    """
+    current: list[_T] = []
+    current_bytes = 0
+    for item in items:
+        size = size_of(item)
+        if current and (len(current) >= max_count or current_bytes + size > max_bytes):
+            yield current
+            current = []
+            current_bytes = 0
+        current.append(item)
+        current_bytes += size
+    if current:
+        yield current