ledgence-worker 0.4.1__py3-none-any.whl

ledgence/worker/__init__.py

@@ -0,0 +1,81 @@
+"""Small, dependency-free helpers for Ledgence Python programs (MIT)."""
+
+from contextvars import ContextVar
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class TraceContext:
+    """Invocation-local W3C carrier, separate from the immutable event origin."""
+
+    traceparent: str
+    tracestate: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class InvocationContext:
+    """Envelope identity for the invocation currently handled by this process."""
+
+    event_id: str
+    attempt_id: str
+    source: Optional[str] = None
+    tenant_id: Optional[str] = None
+    namespace: Optional[str] = None
+    run_id: Optional[str] = None
+    task_id: Optional[str] = None
+    attempt_no: Optional[int] = None
+    processing_context: Optional[TraceContext] = None
+    workflow_id: Optional[str] = None
+    activation_id: Optional[str] = None
+    parent_workflow_id: Optional[str] = None
+    root_workflow_id: Optional[str] = None
+
+
+_invocation: ContextVar[Optional[InvocationContext]] = ContextVar(
+    "ledgence_invocation", default=None
+)
+_shutdown_callbacks = []
+
+
+def current_invocation() -> InvocationContext:
+    """Return this invocation's identifiers; fail when no invocation is active."""
+    value = _invocation.get()
+    if value is None:
+        raise RuntimeError("no Ledgence invocation is active")
+    return value
+
+
+def register_shutdown(callback):
+    """Call an application-owned provider's shutdown before the closing ACK.
+
+    Called once on graceful protocol shutdown. The parent process's shutdown
+    deadline bounds callbacks; cancellation/crashes do not promise a flush.
+    """
+    if not callable(callback):
+        raise TypeError("shutdown callback must be callable")
+    if callback not in _shutdown_callbacks:
+        _shutdown_callbacks.append(callback)
+
+
+def _shutdown():
+    import traceback
+
+    callbacks = _shutdown_callbacks[:]
+    _shutdown_callbacks.clear()
+    for callback in callbacks:
+        try:
+            callback()
+        except Exception:
+            traceback.print_exc()
+
+
+def get_logger(name):
+    """Return a standard logger with a contextual, best-effort Ledgence handler."""
+    from ._logging import get_logger as configured_logger
+
+    return configured_logger(name)
+
+
+__all__ = ["InvocationContext", "TraceContext", "current_invocation", "get_logger",
+           "register_shutdown"]

ledgence/worker/_logging.py

@@ -0,0 +1,132 @@
+"""Bounded best-effort structured log snapshots (MIT)."""
+
+import json
+import logging
+import threading
+from itertools import islice
+from datetime import datetime, timezone
+
+from . import _invocation
+
+_sink = None
+_lock = threading.Lock()
+_record_lock = threading.Lock()
+_MAX_TEXT = 4096
+_MAX_ATTRIBUTES = 32
+
+
+def _bounded(value, depth=0, budget=None):
+    # Limit the whole attribute tree, not just each branch. A branching object
+    # must not multiply per-value allowances into an unbounded encoded record.
+    if budget is None:
+        budget = [64, _MAX_TEXT]
+    budget[0] -= 1
+    if budget[0] < 0:
+        return "[truncated]"
+    if value is None or isinstance(value, (bool, int, float)):
+        return value
+    if isinstance(value, str):
+        text = value[:max(0, budget[1])]
+        budget[1] -= len(text)
+        return text
+    if depth >= 4:
+        return "[truncated]"
+    if isinstance(value, dict):
+        result = {}
+        for key, item in islice(value.items(), _MAX_ATTRIBUTES):
+            if budget[0] <= 0:
+                break
+            if isinstance(key, str):
+                bounded_key = key[:min(256, max(0, budget[1]))]
+                budget[1] -= len(bounded_key)
+                result[bounded_key] = _bounded(item, depth + 1, budget)
+        return result
+    if isinstance(value, (list, tuple)):
+        result = []
+        for item in value[:_MAX_ATTRIBUTES]:
+            if budget[0] <= 0:
+                break
+            result.append(_bounded(item, depth + 1, budget))
+        return result
+    return "[unsupported]"
+
+
+def _snapshot(record):
+    from .otel import _active_ids, _api
+
+    frame = {"v": getattr(_sink, "protocol_version", 2), "type": "log",
+             "time": datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
+             "severity": record.levelname[:32], "logger": record.name[:256],
+             "message": record.getMessage()[:_MAX_TEXT],
+             "attributes": _bounded(getattr(record, "attributes", {}))}
+    invocation = _invocation.get()
+    if invocation is not None:
+        frame["invocation"] = {key: getattr(invocation, key) for key in (
+            "event_id", "attempt_id", "source", "tenant_id", "namespace",
+            "run_id", "task_id", "attempt_no") if getattr(invocation, key) is not None}
+        if frame["v"] >= 3:
+            frame["invocation"].update({key: getattr(invocation, key) for key in
+                                        ("workflow_id", "activation_id")
+                                        if getattr(invocation, key) is not None})
+            if invocation.parent_workflow_id is not None:
+                frame["invocation"].update(parent_workflow_id=invocation.parent_workflow_id,
+                                           root_workflow_id=invocation.root_workflow_id)
+    active = _active_ids()
+    if active is None and _api is None and invocation is not None:
+        carrier = invocation.processing_context
+        if carrier is not None:
+            parts = carrier.traceparent.split("-")
+            active = {"trace_id": parts[1], "span_id": parts[2]}
+    if active is not None:
+        frame.update(active)
+    return frame
+
+
+class _Handler(logging.Handler):
+    def emit(self, record):
+        sink = _sink
+        if sink is None:
+            return
+        try:
+            # Propagation can visit several Ledgence handlers for the same
+            # LogRecord. Claim that event once, including a best-effort drop,
+            # without suppressing application handlers or retaining the sink.
+            # Re-dispatching the same record remains the same telemetry event;
+            # each ordinary logger call creates its own independent record.
+            with _record_lock:
+                if record.__dict__.get("_ledgence_log_processed", False):
+                    return
+                record.__dict__["_ledgence_log_processed"] = True
+            # Encoding happens synchronously: mutable attributes, context changes,
+            # and later delivery cannot relabel a record as another invocation.
+            frame = _snapshot(record)
+            limit = sink.log_limit
+            mandatory = dict(frame, message="", attributes={})
+            if len(json.dumps(mandatory, ensure_ascii=False, allow_nan=False,
+                              separators=(",", ":")).encode("utf-8")) + 1 > limit:
+                sink.drop_log()
+                return
+            # Shrink optional content, never identity. Encoding includes newline.
+            for _ in range(15):
+                encoded = json.dumps(frame, ensure_ascii=False, allow_nan=False,
+                                     separators=(",", ":")).encode("utf-8") + b"\n"
+                if len(encoded) <= limit:
+                    sink.offer_log(encoded)
+                    return
+                frame["attributes"] = {}
+                frame["message"] = frame["message"][:len(frame["message"]) // 2]
+            sink.drop_log()
+        except Exception:
+            # Malformed messages/attributes are telemetry loss, never task failure.
+            sink.drop_log()
+
+
+def get_logger(name):
+    logger = logging.getLogger(name)
+    with _lock:
+        if not any(isinstance(handler, _Handler) for handler in logger.handlers):
+            logger.addHandler(_Handler())
+        if logger.level == logging.NOTSET:
+            logger.setLevel(logging.INFO)
+    # Keep propagation and existing application/root handlers untouched.
+    return logger

ledgence/worker/_observations.py

@@ -0,0 +1,88 @@
+"""Bounded optional invocation measurements, independent of replay (MIT).
+
+Measurements describe this Python process, including all its threads, excluding
+child processes. Memory is a lifetime high-water mark, never an invocation peak.
+An abrupt exit may lose the whole buffer. It contains no inputs or outputs.
+"""
+
+from contextvars import ContextVar
+import json
+import sys
+import time
+
+_current = ContextVar("ledgence_invocation_observations", default=None)
+MAX_LOCAL_STEPS = 128
+
+
+def _usage():
+    try:
+        import resource
+        value = resource.getrusage(resource.RUSAGE_SELF)
+        rss = value.ru_maxrss * (1024 if sys.platform.startswith("linux") else 1)
+        if sys.platform != "darwin" and not sys.platform.startswith("linux"):
+            rss = None  # Units are deliberately not guessed on other platforms.
+        return (round(value.ru_utime * 1_000_000), round(value.ru_stime * 1_000_000), rss)
+    except (ImportError, OSError, ValueError):
+        return (None, None, None)
+
+
+class Observations:
+    def __init__(self):
+        self.started_at = time.time_ns() // 1_000_000
+        self.started = time.monotonic_ns()
+        self.usage = _usage()
+        self.locals = []
+        self.truncated = False
+
+    def begin(self, binding, replayed=False):
+        if len(self.locals) >= MAX_LOCAL_STEPS:
+            self.truncated = True
+            return None
+        record = {"key": binding["key"], "callable": binding["callable"],
+                  "started_at_ms": time.time_ns() // 1_000_000,
+                  "elapsed_us": 0, "state": "replayed" if replayed else "cancelled"}
+        self.locals.append(record)
+        return (record, time.monotonic_ns())
+
+    @staticmethod
+    def finish(token, state):
+        if token is not None:
+            record, started = token
+            record.update(elapsed_us=max(0, (time.monotonic_ns() - started) // 1000), state=state)
+
+    def snapshot(self):
+        usage = _usage()
+        def delta(index):
+            before, after = self.usage[index], usage[index]
+            return None if before is None or after is None or after < before else after - before
+        value = {"runtime_started_at_ms": self.started_at,
+                "runtime_elapsed_us": max(0, (time.monotonic_ns() - self.started) // 1000),
+                "process_cpu_user_us": delta(0), "process_cpu_system_us": delta(1),
+                "process_lifetime_peak_rss_bytes": usage[2],
+                "local_steps": list(self.locals), "local_steps_truncated": self.truncated}
+        # Escaped identifier bytes count too. Always keep resources even when the
+        # optional local buffer must be truncated; never alter the replay journal.
+        while len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()) > 128 * 1024:
+            value["local_steps"].pop()
+            value["local_steps_truncated"] = True
+        return value
+
+
+def begin_local(binding, replayed=False):
+    observations = _current.get()
+    if observations is None:
+        return None
+    try:
+        return observations.begin(binding, replayed)
+    except Exception:
+        observations.truncated = True
+        return None
+
+
+def finish_local(token, state):
+    try:
+        Observations.finish(token, state)
+    except Exception:
+        observations = _current.get()
+        if observations is not None:
+            observations.truncated = True

ledgence/worker/_protocol.py

@@ -0,0 +1,101 @@
+"""The v2 protocol's single writer and bounded optional log queue (MIT)."""
+
+from collections import deque
+import threading
+
+MAX_LOG_RECORDS = 64
+MAX_LOG_BYTES = 1024 * 1024
+MAX_LOG_FRAME_BYTES = 16 * 1024
+_MAX_DROPPED = (1 << 64) - 1
+
+
+class ProtocolWriter:
+    def __init__(self, protocol, frame_limit):
+        self.log_limit = min(frame_limit, MAX_LOG_FRAME_BYTES)
+        self._protocol = protocol
+        self._condition = threading.Condition()
+        self._logs = deque()
+        self._log_bytes = 0
+        self._control = None
+        self._error = None
+        self._ready = False
+        self._stopped = False
+        self._dropped = 0
+        self._thread = threading.Thread(target=self._run, name="ledgence-protocol", daemon=True)
+        self._thread.start()
+
+    @property
+    def dropped_logs(self):
+        with self._condition:
+            return self._dropped
+
+    def drop_log(self):
+        with self._condition:
+            self._dropped = min(_MAX_DROPPED, self._dropped + 1)
+
+    def offer_log(self, encoded):
+        with self._condition:
+            if (self._stopped or self._error is not None
+                    or len(encoded) > self.log_limit
+                    or len(self._logs) >= MAX_LOG_RECORDS
+                    or self._log_bytes + len(encoded) > MAX_LOG_BYTES):
+                self._dropped = min(_MAX_DROPPED, self._dropped + 1)
+                return False
+            self._logs.append(encoded)
+            self._log_bytes += len(encoded)
+            self._condition.notify()
+            return True
+
+    def control(self, encoded, closing=False):
+        done = threading.Event()
+        with self._condition:
+            if self._error is not None:
+                raise self._error
+            if self._control is not None:
+                raise RuntimeError("concurrent protocol control writes")
+            if closing:
+                self._stopped = True
+                self._dropped = min(_MAX_DROPPED, self._dropped + len(self._logs))
+                self._logs.clear()
+                self._log_bytes = 0
+            self._control = (encoded, done)
+            self._condition.notify()
+        done.wait()  # IPC backpressure only; optional logs never occupy this slot.
+        if self._error is not None:
+            raise self._error
+
+    def _run(self):
+        while True:
+            with self._condition:
+                while self._control is None and not (self._ready and self._logs):
+                    if self._stopped:
+                        return
+                    self._condition.wait()
+                if self._control is not None:
+                    encoded, done = self._control
+                    self._control = None
+                else:
+                    encoded = self._logs.popleft()
+                    self._log_bytes -= len(encoded)
+                    done = None
+            try:
+                pending = memoryview(encoded)
+                while pending:
+                    count = self._protocol.write(pending)
+                    if count is None or count <= 0:
+                        raise OSError("protocol output pipe closed")
+                    pending = pending[count:]
+            except Exception as error:
+                with self._condition:
+                    self._error = error
+                    if self._control is not None:
+                        self._control[1].set()
+                    self._logs.clear()
+                    self._log_bytes = 0
+                if done is not None:
+                    done.set()
+                return
+            if done is not None:
+                with self._condition:
+                    self._ready = True
+                done.set()