dlogify 0.1.0__py3-none-any.whl

dlogify/_diagnostics.py

@@ -0,0 +1,54 @@
+"""The client's own messages go straight to the stderr stream, never through
+logging or print, so capture never sees them (docs/12 §4, guarantee 3)."""
+
+from __future__ import annotations
+
+import sys
+import threading
+from typing import Optional
+
+
+class Diagnostics:
+    def __init__(self, debug_enabled: bool) -> None:
+        self.debug_enabled = debug_enabled
+        self._warned: set[str] = set()
+        self._lock = threading.Lock()
+
+    def warn(self, message: str) -> None:
+        _write(f"logify: warning: {message}\n")
+
+    def warn_once(self, key: str, message: str) -> None:
+        with self._lock:
+            if key in self._warned:
+                return
+            self._warned.add(key)
+        self.warn(message)
+
+    def debug(self, message: str) -> None:
+        if self.debug_enabled:
+            _write(f"logify: {message}\n")
+
+
+def _write(text: str) -> None:
+    stream = sys.__stderr__
+    if stream is None:
+        return
+    try:
+        stream.write(text)
+        stream.flush()
+    except Exception:
+        # stderr is closed; there is nowhere left to report to.
+        pass
+
+
+def is_flag(value: Optional[str]) -> bool:
+    return value is not None and value.strip().lower() in ("1", "true")
+
+
+def describe_error(exc: BaseException) -> str:
+    name = type(exc).__name__
+    try:
+        text = str(exc)
+    except Exception:
+        text = "<unprintable>"
+    return f"{name}: {text}" if text else name

dlogify/_encode.py

@@ -0,0 +1,100 @@
+"""OTLP/JSON encoding (docs/12 §2–3). Tested by clients/conformance/otlp."""
+
+from __future__ import annotations
+
+import json
+import socket
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from ._levels import severity
+from ._record import LogRecord, normalize_value
+from ._version import SDK_LANGUAGE, SDK_NAME, VERSION
+
+_INT64_MIN = -(2**63)
+_INT64_MAX = 2**63 - 1
+
+
+@dataclass(frozen=True)
+class EncodedRecord:
+    scope: str
+    record: dict[str, Any]
+    bytes: int
+
+
+def to_json_bytes(value: Any) -> bytes:
+    return json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode("utf-8", "replace")
+
+
+def any_value(raw: object) -> Optional[dict[str, Any]]:
+    value = normalize_value(raw)
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return {"boolValue": value}
+    if isinstance(value, int):
+        number = int(value)
+        # OTLP intValue is an int64; a larger Python int keeps its digits as a string.
+        return {"intValue": str(number)} if _INT64_MIN <= number <= _INT64_MAX else {"stringValue": str(number)}
+    if isinstance(value, float):
+        return {"doubleValue": value}
+    return {"stringValue": value}
+
+
+def key_values(attributes: Mapping[str, object]) -> list[dict[str, Any]]:
+    out = []
+    for key, raw in attributes.items():
+        value = any_value(raw)
+        if value is not None:
+            out.append({"key": key, "value": value})
+    return out
+
+
+def host_name() -> str:
+    try:
+        return socket.gethostname()
+    except Exception:
+        return "unknown"
+
+
+def resource_attributes(*, service: str, environment: str, release: str, attributes: Mapping[str, object]) -> list[dict[str, Any]]:
+    """Fixed attributes first; init() attributes never replace them."""
+    fixed: dict[str, object] = {"service.name": service, "deployment.environment.name": environment}
+    if release:
+        fixed["service.version"] = release
+    fixed["host.name"] = host_name()
+    fixed["telemetry.sdk.name"] = SDK_NAME
+    fixed["telemetry.sdk.language"] = SDK_LANGUAGE
+    fixed["telemetry.sdk.version"] = VERSION
+    extra = {key: value for key, value in attributes.items() if key not in fixed}
+    return key_values(fixed) + key_values(extra)
+
+
+def encode_record(record: LogRecord) -> EncodedRecord:
+    encoded: dict[str, Any] = {
+        "timeUnixNano": record.time_unix_nano,
+        "observedTimeUnixNano": record.observed_time_unix_nano,
+    }
+    if record.level is not None:
+        number, text = severity(record.level)
+        encoded["severityNumber"] = number
+        encoded["severityText"] = text
+    encoded["body"] = {"stringValue": record.body}
+    encoded["attributes"] = key_values(record.attributes)
+    return EncodedRecord(record.scope, encoded, len(to_json_bytes(encoded)))
+
+
+def encode_request(resource: list[dict[str, Any]], records: Sequence[EncodedRecord]) -> dict[str, Any]:
+    """One ResourceLogs; one ScopeLogs per scope, in order of first appearance."""
+    scopes: dict[str, list[dict[str, Any]]] = {}
+    for item in records:
+        scopes.setdefault(item.scope, []).append(item.record)
+    return {
+        "resourceLogs": [
+            {
+                "resource": {"attributes": resource},
+                "scopeLogs": [{"scope": {"name": name}, "logRecords": logs} for name, logs in scopes.items()],
+            }
+        ]
+    }

dlogify/_exporter.py

@@ -0,0 +1,407 @@
+"""Queue, batching and the export thread (docs/12 §4). Producers take a lock
+for O(1) work; one daemon thread sends one request at a time; failures never
+reach the caller."""
+
+from __future__ import annotations
+
+import collections
+import gzip
+import json
+import os
+import random as _random
+import sys
+import threading
+import time
+import urllib.error
+import weakref
+from dataclasses import dataclass
+from datetime import timezone
+from email.utils import parsedate_to_datetime
+from typing import Any, Callable, Optional
+
+from . import _http
+from ._diagnostics import Diagnostics, describe_error
+from ._encode import EncodedRecord, encode_record, encode_request, to_json_bytes
+from ._guard import guard_thread
+from ._record import LogRecord
+
+EXPORT_INTERVAL = 5.0
+TRIGGER_RECORDS = 500
+TRIGGER_BYTES = 512 * 1024
+BATCH_MAX_RECORDS = 1000
+BATCH_MAX_BYTES = 1_000_000
+REQUEST_TIMEOUT = 10.0
+RETRY_WINDOW = 300.0
+_BACKOFF_BASE = 1.0
+_BACKOFF_CAP = 30.0
+# Repeated `Retry-After: 0` replies must not turn into a tight retry loop.
+_RETRY_AFTER_FLOOR = 0.1
+# Room for the scope wrappers around the records of one request.
+_ENVELOPE_RESERVE_BYTES = 1024
+
+Post = Callable[[str, bytes, dict[str, str], float], _http.Response]
+
+
+@dataclass(frozen=True)
+class Outcome:
+    kind: str  # "sent", "retry", "split", "drop" or "stop"
+    reason: str = ""
+    retry_after: Optional[float] = None
+    status: int = 0
+    body: str = ""
+
+
+def parse_retry_after(value: Optional[str], now: float) -> Optional[float]:
+    """Seconds to wait, from delay-seconds or an HTTP date; `now` is the wall clock."""
+    if not value:
+        return None
+    text = value.strip()
+    if text.isascii() and text.isdigit():
+        return float(int(text))
+    try:
+        when = parsedate_to_datetime(text)
+    except Exception:
+        return None
+    if when is None:
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=timezone.utc)
+    return max(0.0, when.timestamp() - now)
+
+
+def backoff_delay(attempt: int, rand: float) -> float:
+    """Exponential backoff with full jitter: 1 s, 2 s, 4 s … capped at 30 s."""
+    return rand * min(_BACKOFF_CAP, _BACKOFF_BASE * 2 ** min(attempt, 16))
+
+
+def classify(status: int, retry_after: Optional[str], body: str, now: float) -> Outcome:
+    if 200 <= status < 300:
+        return Outcome("sent", body=body)
+    if status in (401, 403):
+        return Outcome("stop", status=status)
+    if status == 413:
+        return Outcome("split")
+    if status in (429, 502, 503, 504):
+        return Outcome("retry", reason=f"HTTP {status}", retry_after=parse_retry_after(retry_after, now))
+    detail = f": {body[:500]}" if body else ""
+    return Outcome("drop", reason=f"HTTP {status}{detail}")
+
+
+def is_certificate_error(err: BaseException) -> bool:
+    """CERTIFICATE_VERIFY_FAILED, raised as is or wrapped by urllib."""
+    try:
+        import ssl
+    except ImportError:  # a Python built without ssl cannot reach https at all
+        return False
+    if isinstance(err, urllib.error.URLError):
+        err = err.reason if isinstance(err.reason, BaseException) else err
+    return isinstance(err, ssl.SSLCertVerificationError)
+
+
+# multiprocessing children end with os._exit(), which skips atexit: they flush
+# through multiprocessing's own exit finalizers instead.
+_CHILD_EXIT_FLUSH = 2.0
+
+
+# One pair of at-fork callbacks for every exporter: callbacks cannot be
+# unregistered, so one per exporter would pile up with many clients.
+_live: "weakref.WeakSet[Exporter]" = weakref.WeakSet()
+
+
+def _reset_in_child() -> None:
+    for exporter in list(_live):
+        exporter._reset_state()
+
+
+def _before_fork() -> None:
+    for exporter in list(_live):
+        exporter._flush_multiprocessing_children()
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(before=_before_fork, after_in_child=_reset_in_child)
+
+
+def _flush_at_child_exit(exporter: "Exporter") -> None:
+    # Runs in a multiprocessing child, after it cleared the finalizers it inherited.
+    util = sys.modules["multiprocessing.util"]
+    util.Finalize(None, exporter.flush, args=(_CHILD_EXIT_FLUSH,), exitpriority=10)
+
+
+class Exporter:
+    def __init__(
+        self,
+        *,
+        url: str,
+        api_key: str,
+        user_agent: str,
+        resource: list[dict[str, Any]],
+        max_queue_size: int,
+        diagnostics: Diagnostics,
+        post: Optional[Post] = None,
+        clock: Callable[[], float] = time.monotonic,
+        sleep: Optional[Callable[[float], object]] = None,
+        random: Callable[[], float] = _random.random,
+        request_timeout: float = REQUEST_TIMEOUT,
+        interval: float = EXPORT_INTERVAL,
+    ) -> None:
+        self._url = url
+        self._headers = {
+            "Content-Type": "application/json",
+            "Content-Encoding": "gzip",
+            "Authorization": f"Bearer {api_key}",
+            "User-Agent": user_agent,
+        }
+        self._resource = resource
+        self._max_queue_size = max_queue_size
+        self._diagnostics = diagnostics
+        self._post_fn = post or _http.post
+        self._clock = clock
+        self._sleep = sleep or self._wait
+        self._random = random
+        self._request_timeout = request_timeout
+        self._interval = interval
+        envelope = len(to_json_bytes(encode_request(resource, [])))
+        self._batch_budget = BATCH_MAX_BYTES - envelope - _ENVELOPE_RESERVE_BYTES
+        self._stopped = False
+        self._closed = False
+        self._multiprocessing_registered = False
+        self._reset_state()
+        _live.add(self)
+
+    def _reset_state(self) -> None:
+        # Also run in a forked child: the export thread does not exist there and
+        # may have held the lock. The parent still sends what was queued.
+        # Reentrant: a signal handler or a finalizer may call the SDK while this
+        # thread is inside enqueue() or flush().
+        self._cond = threading.Condition(threading.RLock())
+        self._wake = threading.Event()
+        self._queue: collections.deque[EncodedRecord] = collections.deque()
+        self._queued_bytes = 0
+        self._in_flight = 0
+        self._flush_waiters = 0
+        self._dropped = 0
+        self._reported_dropped = 0
+        self._thread: Optional[threading.Thread] = None
+
+    def _flush_multiprocessing_children(self) -> None:
+        """Before a fork: when multiprocessing is in use, its children flush as they end."""
+        util = sys.modules.get("multiprocessing.util")
+        if util is None or self._multiprocessing_registered:
+            return
+        self._multiprocessing_registered = True
+        util.register_after_fork(self, _flush_at_child_exit)
+
+    def enqueue(self, record: LogRecord) -> None:
+        if self._stopped or self._closed:
+            return
+        encoded = encode_record(record)
+        with self._cond:
+            if self._stopped or self._closed:
+                return
+            if len(self._queue) >= self._max_queue_size:
+                oldest = self._queue.popleft()
+                self._queued_bytes -= oldest.bytes
+                self._dropped += 1
+            self._queue.append(encoded)
+            self._queued_bytes += encoded.bytes
+            self._ensure_thread()
+            if len(self._queue) >= TRIGGER_RECORDS or self._queued_bytes >= TRIGGER_BYTES:
+                self._cond.notify_all()
+
+    def flush(self, timeout: float) -> None:
+        """Sends everything queued; returns when done or when the timeout expires."""
+        if threading.current_thread() is self._thread:
+            return
+        deadline = time.monotonic() + timeout
+        with self._cond:
+            if self._queue:
+                self._ensure_thread()
+            self._flush_waiters += 1
+            self._cond.notify_all()
+            try:
+                while (self._queue or self._in_flight) and not (self._stopped or self._closed) and self._thread is not None:
+                    remaining = deadline - time.monotonic()
+                    if remaining <= 0:
+                        return
+                    self._cond.wait(remaining)
+            finally:
+                self._flush_waiters -= 1
+
+    def close(self) -> None:
+        _live.discard(self)
+        with self._cond:
+            self._closed = True
+            self._queue.clear()
+            self._queued_bytes = 0
+            self._cond.notify_all()
+        self._wake.set()
+
+    def _wait(self, seconds: float) -> None:
+        self._wake.wait(seconds)
+
+    def _ensure_thread(self) -> None:
+        """Starts the export thread on the first record (lock held)."""
+        if self._thread is not None:
+            return
+        thread = threading.Thread(target=self._run, name="dlogify-exporter", daemon=True)
+        try:
+            thread.start()
+        except RuntimeError as err:  # the interpreter is shutting down
+            self._diagnostics.debug(f"could not start the export thread: {describe_error(err)}")
+            return
+        self._thread = thread
+
+    def _due(self, deadline: float) -> bool:
+        if self._stopped or self._closed or not self._queue:
+            return False
+        return (
+            self._flush_waiters > 0
+            or len(self._queue) >= TRIGGER_RECORDS
+            or self._queued_bytes >= TRIGGER_BYTES
+            or time.monotonic() >= deadline
+        )
+
+    def _run(self) -> None:
+        guard_thread()
+        deadline = time.monotonic() + self._interval
+        while True:
+            with self._cond:
+                while not self._due(deadline):
+                    if self._stopped or self._closed:
+                        return
+                    now = time.monotonic()
+                    if now >= deadline:  # a tick with nothing queued
+                        deadline = now + self._interval
+                    self._cond.wait(deadline - now)
+            self._drain()
+            deadline = time.monotonic() + self._interval
+
+    def _drain(self) -> None:
+        while True:
+            batch: Optional[list[EncodedRecord]] = None
+            with self._cond:
+                note = self._dropped_note()
+                if self._queue and not (self._stopped or self._closed):
+                    batch = self._take_batch()
+                    self._in_flight = len(batch)
+            if note:
+                self._diagnostics.debug(note)
+            if batch is None:
+                return
+            try:
+                self._send(batch)
+            except Exception as err:
+                self._diagnostics.debug(f"export failed: {describe_error(err)}")
+            finally:
+                with self._cond:
+                    self._in_flight = 0
+                    self._cond.notify_all()
+
+    def _take_batch(self) -> list[EncodedRecord]:
+        """Lock held: at most BATCH_MAX_RECORDS records and the byte budget."""
+        batch: list[EncodedRecord] = []
+        total = 0
+        while self._queue and len(batch) < BATCH_MAX_RECORDS:
+            size = self._queue[0].bytes + 1
+            if batch and total + size > self._batch_budget:
+                break
+            record = self._queue.popleft()
+            self._queued_bytes -= record.bytes
+            batch.append(record)
+            total += size
+        return batch
+
+    def _dropped_note(self) -> Optional[str]:
+        if self._dropped == self._reported_dropped:
+            return None
+        note = f"queue full: dropped {self._dropped - self._reported_dropped} records ({self._dropped} in total)"
+        self._reported_dropped = self._dropped
+        return note
+
+    def _body(self, batch: list[EncodedRecord]) -> bytes:
+        return gzip.compress(to_json_bytes(encode_request(self._resource, batch)))
+
+    def _send(self, batch: list[EncodedRecord]) -> None:
+        started = self._clock()
+        attempt = 0
+        while not (self._stopped or self._closed):
+            outcome = self._post(batch)
+            if outcome.kind == "sent":
+                self._report_partial(outcome.body)
+                return
+            if outcome.kind == "stop":
+                self._stop(outcome.status)
+                return
+            if outcome.kind == "drop":
+                self._diagnostics.debug(f"dropped {len(batch)} records: {outcome.reason}")
+                return
+            if outcome.kind == "split":
+                if len(batch) == 1:
+                    self._diagnostics.debug("dropped a record that is too large (HTTP 413)")
+                    return
+                half = (len(batch) + 1) // 2
+                self._send(batch[:half])
+                self._send(batch[half:])
+                return
+            remaining = RETRY_WINDOW - (self._clock() - started)
+            if remaining <= 0:
+                self._diagnostics.debug(f"dropped {len(batch)} records after retrying for 5 minutes: {outcome.reason}")
+                return
+            if outcome.retry_after is None:
+                wait = backoff_delay(attempt, self._random())
+            else:
+                wait = max(_RETRY_AFTER_FLOOR, outcome.retry_after)
+            delay = min(wait, remaining)
+            self._diagnostics.debug(f"export failed ({outcome.reason}); retrying in {round(delay * 1000)} ms")
+            self._sleep(delay)
+            attempt += 1
+
+    def _post(self, batch: list[EncodedRecord]) -> Outcome:
+        try:
+            body = self._body(batch)
+        except Exception as err:
+            return Outcome("drop", reason=f"encoding failed: {describe_error(err)}")
+        try:
+            response = self._post_fn(self._url, body, self._headers, self._request_timeout)
+        except Exception as err:
+            if is_certificate_error(err):
+                self._diagnostics.warn_once(
+                    "certificate",
+                    f"could not verify the TLS certificate of {self._url} ({describe_error(err)}). "
+                    'With Python from python.org on macOS, run "Install Certificates.command" in its Applications folder; '
+                    "behind a proxy that inspects TLS, point SSL_CERT_FILE to its CA bundle. Each batch is retried for 5 minutes, then dropped",
+                )
+            return Outcome("retry", reason=describe_error(err))
+        if 300 <= response.status < 400:
+            location = response.headers.get("location") or "an unknown location"
+            self._diagnostics.warn_once(
+                "redirect",
+                f"the endpoint {self._url} redirected to {location}; set LOGIFY_ENDPOINT to the final URL. Logs are dropped until then",
+            )
+            return Outcome("drop", reason=f"HTTP {response.status} redirect to {location}")
+        return classify(response.status, response.headers.get("retry-after"), response.body, time.time())
+
+    def _report_partial(self, body: str) -> None:
+        try:
+            parsed = json.loads(body)
+            partial = parsed.get("partialSuccess") or parsed.get("partial_success")
+            if not partial:
+                return
+            rejected = int(partial.get("rejectedLogRecords") or partial.get("rejected_log_records") or 0)
+            message = partial.get("errorMessage") or partial.get("error_message") or ""
+            if rejected > 0 or message:
+                self._diagnostics.debug(f"partial success: {rejected} records rejected: {message}")
+        except Exception:
+            pass  # not JSON; nothing to report
+
+    def _stop(self, status: int) -> None:
+        """401/403: the key is invalid, revoked or lacks the scope."""
+        with self._cond:
+            self._stopped = True
+            self._queue.clear()
+            self._queued_bytes = 0
+            self._cond.notify_all()
+        self._diagnostics.warn_once(
+            "auth", f"the API key was rejected (HTTP {status}); Logify stops sending logs for this process"
+        )

dlogify/_guard.py

@@ -0,0 +1,29 @@
+"""Set while Logify builds, queues or exports a record, so logging done on the
+way (by Logify or by code it calls, such as a __str__ that logs) is not
+captured again (docs/12 §4, guarantee 3)."""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+_state = threading.local()
+
+
+def is_guarded() -> bool:
+    return getattr(_state, "depth", 0) > 0
+
+
+@contextmanager
+def guarded() -> Iterator[None]:
+    _state.depth = getattr(_state, "depth", 0) + 1
+    try:
+        yield
+    finally:
+        _state.depth -= 1
+
+
+def guard_thread() -> None:
+    """Marks the current thread (the export thread) as guarded for good."""
+    _state.depth = 1

dlogify/_hooks.py

@@ -0,0 +1,95 @@
+"""Uncaught exceptions and the exit flush (docs/12 §4, guarantee 7; spec §3)."""
+
+from __future__ import annotations
+
+import atexit
+import sys
+import threading
+import time
+from types import TracebackType
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from ._record import exception_attributes, exception_title
+
+if TYPE_CHECKING:
+    from ._client import Client
+
+EXIT_FLUSH_TIMEOUT = 2.0
+_IGNORED = (KeyboardInterrupt, SystemExit)
+
+
+class ExitBudget:
+    """The 2 s an exit may spend flushing, shared by the excepthook and the
+    atexit flush that follows it."""
+
+    def __init__(self) -> None:
+        self.spent = 0.0
+
+    def start_over(self) -> None:
+        # In a REPL the excepthook runs for every error and the process goes on.
+        self.spent = 0.0
+
+    def flush(self, client: "Client") -> None:
+        remaining = EXIT_FLUSH_TIMEOUT - self.spent
+        if remaining <= 0:
+            return
+        started = time.monotonic()
+        try:
+            client.flush(remaining)
+        finally:
+            self.spent += time.monotonic() - started
+
+
+def _record(client: "Client", exc: BaseException, tb: Optional[TracebackType], extra: dict[str, object]) -> None:
+    exception = exception_attributes(exc, tb)
+    body = exception_title(exception["exception.type"], exception["exception.message"])
+    client._emit("uncaught", "fatal", body, {**exception, **extra})
+
+
+def install_uncaught_capture(client: "Client", budget: ExitBudget) -> Callable[[], None]:
+    previous_hook = sys.excepthook
+    previous_thread_hook = threading.excepthook
+
+    def excepthook(exc_type: type, exc: BaseException, tb: Optional[TracebackType]) -> None:
+        try:
+            if isinstance(exc, BaseException) and not isinstance(exc, _IGNORED):
+                _record(client, exc, tb, {"exception.origin": "main"})
+                budget.start_over()
+                budget.flush(client)
+        except Exception:
+            pass  # the application's error matters more than ours
+        previous_hook(exc_type, exc, tb)
+
+    def thread_excepthook(args: Any) -> None:
+        try:
+            exc = args.exc_value
+            if isinstance(exc, BaseException) and not isinstance(exc, _IGNORED):
+                extra: dict[str, object] = {"exception.origin": "thread"}
+                if args.thread is not None:
+                    extra["thread.name"] = args.thread.name
+                _record(client, exc, args.exc_traceback, extra)
+        except Exception:
+            pass
+        previous_thread_hook(args)
+
+    sys.excepthook = excepthook
+    threading.excepthook = thread_excepthook
+
+    def undo() -> None:
+        if sys.excepthook is excepthook:
+            sys.excepthook = previous_hook
+        if threading.excepthook is thread_excepthook:
+            threading.excepthook = previous_thread_hook
+
+    return undo
+
+
+def install_exit_flush(client: "Client", budget: ExitBudget) -> Callable[[], None]:
+    """atexit handlers run last-in first-out: those the application registers
+    after init() (and may log from) run before this flush."""
+
+    def flush_at_exit() -> None:
+        budget.flush(client)
+
+    atexit.register(flush_at_exit)
+    return lambda: atexit.unregister(flush_at_exit)