logleaf 1.0.0__py3-none-any.whl

logleaf/__init__.py

@@ -0,0 +1,24 @@
+"""Background logging. Stable entry points: log, Logger, BoundLogger and statistics."""
+
+from typing import TYPE_CHECKING, Any
+
+from ._models import LoggerStats, OutputStats
+from .logger import BoundLogger, Logger
+from .shared import log
+
+# Keep old explicit/star imports working without loading the prototype for
+# ordinary `from logleaf import log` users.
+if TYPE_CHECKING:
+    from .experimental import BatchLogger, Stats
+
+__all__ = ["log", "Logger", "BoundLogger", "LoggerStats", "OutputStats", "BatchLogger", "Stats"]
+
+
+def __getattr__(name: str) -> Any:
+    if name in ("BatchLogger", "Stats"):
+        from . import experimental
+
+        value = getattr(experimental, name)
+        globals()[name] = value
+        return value
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

logleaf/_core.py

@@ -0,0 +1,344 @@
+"""Queue admission, writer lifecycle, output failure and loss accounting."""
+
+from __future__ import annotations
+
+import math
+import os
+import threading
+import time
+from collections import deque
+from time import time_ns as clock_ns
+from typing import Any
+
+from ._models import LoggerStats, OutputStats, _Event, _ExceptionEvent, _Output
+from .formatting import LEVELS, Formatter
+
+
+class _Core:
+    def __init__(
+        self,
+        outputs: list[_Output],
+        formatter: Formatter,
+        timestamp_mode: str,
+        capacity: int,
+        batch_size: int,
+        flush_interval: float,
+        drop_report_interval: float,
+        poll_interval: float = 0.005,
+    ) -> None:
+        self.outputs = outputs
+        self.formatter = formatter
+        self.timestamp_mode = timestamp_mode
+        self.capacity = capacity
+        self.batch_size = batch_size
+        self.flush_interval = flush_interval
+        self.drop_report_interval = drop_report_interval
+        self.poll_interval = poll_interval
+        self.pid = os.getpid()
+        self.minimum = min((output.level for output in outputs), default=100)
+        self.available_minimum = self.minimum
+        self.condition = threading.Condition(threading.Lock())
+        self.queue: deque[_Event] = deque()
+        self.closing = False
+        self.worker_error: str | None = None
+        self.format_errors = 0
+        self.last_format_error: str | None = None
+        self.accepted = self.written = self.failed = self.partial = self.processed = 0
+        self.high = self.rejected = 0
+        self.drops: dict[int, int] = dict.fromkeys(LEVELS, 0)
+        started_ns = clock_ns()
+        for output in outputs:
+            output.report_since_ns = started_ns
+        self.thread = threading.Thread(target=self._run, name="logleaf-writer", daemon=True)
+        try:
+            self.thread.start()
+        except BaseException:
+            for output in outputs:
+                output.target.close()
+            raise
+
+    def check_process(self) -> None:
+        # Check BEFORE locks: a forked child may inherit a permanently locked condition.
+        if os.getpid() != self.pid:
+            raise RuntimeError(
+                "create a new Logger inside each child process; inherited Logger unsafe"
+            )
+
+    def submit(self, event: _Event) -> bool:
+        with self.condition:
+            if self.closing:
+                raise RuntimeError("logger is closed")
+            if self.worker_error or event.level < self.available_minimum:
+                self.rejected += 1
+                return False
+            pending = self.accepted - self.processed
+            if pending >= self.capacity:
+                self.drops[event.level] += 1
+                return False
+            self.queue.append(event)
+            self.accepted += 1
+            if pending >= self.high:
+                self.high = pending + 1
+            return True
+
+    def snapshot(self) -> LoggerStats:
+        self.check_process()
+        with self.condition:
+            dropped = sum(self.drops.values())
+            return LoggerStats(
+                accepted=self.accepted,
+                written=self.written,
+                failed=self.failed,
+                partial=self.partial,
+                pending=self.accepted - self.processed,
+                dropped=dropped,
+                dropped_by_level={LEVELS[k]: v for k, v in self.drops.items()},
+                rejected_unavailable=self.rejected,
+                high_watermark=self.high,
+                outputs={
+                    output.name: OutputStats(
+                        output.written,
+                        output.failed,
+                        sum(output.reported.values()),
+                        dropped - sum(output.reported.values()),
+                        output.error,
+                    )
+                    for output in self.outputs
+                },
+                worker_error=self.worker_error,
+                format_errors=self.format_errors,
+                last_format_error=self.last_format_error,
+            )
+
+    def can_capture_exception(self) -> bool:
+        # Exception-only preflight avoids expensive snapshots on an already full
+        # queue. submit() rechecks admission after capture; no capacity is reserved.
+        with self.condition:
+            if self.closing:
+                raise RuntimeError("logger is closed")
+            if self.worker_error or 40 < self.available_minimum:
+                self.rejected += 1
+                return False
+            if self.accepted - self.processed >= self.capacity:
+                self.drops[40] += 1
+                return False
+            return True
+
+    def close(self, timeout: float | None) -> None:
+        self.check_process()
+        if timeout is not None and (
+            type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout < 0
+        ):
+            raise ValueError("timeout must be nonnegative and finite, or None")
+        with self.condition:
+            self.closing = True
+            self.condition.notify_all()
+        self.thread.join(timeout)
+        if self.thread.is_alive():
+            raise TimeoutError("writer is still draining; close() may be retried")
+        errors = [f"{o.name}: {o.error}" for o in self.outputs if o.error]
+        if self.worker_error:
+            errors.append(self.worker_error)
+        if self.last_format_error:
+            errors.append("format: " + self.last_format_error)
+        if errors:
+            raise RuntimeError("log output failed; inspect stats: " + "; ".join(errors))
+
+    def _fail_output(self, output: _Output, error: Exception) -> None:
+        with self.condition:
+            if output.error is None:
+                output.error = f"{type(error).__name__}: {error}"
+                self.available_minimum = min(
+                    (target.level for target in self.outputs if target.error is None), default=100
+                )
+
+    def _flush(self) -> None:
+        for output in self.outputs:
+            if output.error is None:
+                try:
+                    output.target.flush()
+                except Exception as error:
+                    self._fail_output(output, error)
+
+    def _record(self, event: _Event) -> dict[str, Any]:
+        stamp = event.timestamp_ns if event.timestamp_ns is not None else clock_ns()
+        record = {
+            "timestamp": self.formatter.timestamp(stamp),
+            "timestamp_mode": self.timestamp_mode,
+            "level": LEVELS[event.level],
+            "event": event.message,
+            **(event.context or {}),
+            **event.fields,
+        }
+        if isinstance(event, _ExceptionEvent) and event.exception is not None:
+            snapshot = event.exception
+            record["_log_hub"] = {
+                "kind": "exception",
+                "type": snapshot.type_name,
+                "message": snapshot.message,
+                "traceback": snapshot.render(),
+            }
+        return record
+
+    def _format_failure(self, error: Exception) -> None:
+        with self.condition:
+            self.format_errors += 1
+            self.last_format_error = f"{type(error).__name__}: {error}"
+
+    def _write_single(self, batch: list[_Event] | deque[_Event]) -> None:
+        # Admission already applied this only output's level. Avoid constructing
+        # per-output index and delivery arrays for the common file-only case.
+        output = self.outputs[0]
+        written = 0
+        if output.error is None:
+            lines = []
+            for event in batch:
+                try:
+                    lines.append(self.formatter.render(self._record(event), output.style))
+                except (TypeError, ValueError, OverflowError) as error:
+                    self._format_failure(error)
+            try:
+                if lines:
+                    output.target.write(lines)
+                written = len(lines)
+            except Exception as error:
+                self._fail_output(output, error)
+        failed = len(batch) - written
+        with self.condition:
+            output.written += written
+            output.failed += failed
+            self.written += written
+            self.failed += failed
+            self.processed += len(batch)
+
+    def _write_batch(self, batch: list[_Event] | deque[_Event]) -> None:
+        if len(self.outputs) == 1:
+            self._write_single(batch)
+            return
+        records = []
+        for event in batch:
+            records.append(self._record(event))
+        delivered = [0] * len(batch)
+        targets = [0] * len(batch)
+        for output in self.outputs:
+            indices = [i for i, event in enumerate(batch) if event.level >= output.level]
+            if not indices:
+                continue
+            for i in indices:
+                targets[i] += 1
+            ok = False
+            valid_indices = []
+            if output.error is None:
+                lines = []
+                for i in indices:
+                    try:
+                        lines.append(self.formatter.render(records[i], output.style))
+                        valid_indices.append(i)
+                    except (TypeError, ValueError, OverflowError) as error:
+                        self._format_failure(error)
+                try:
+                    if lines:
+                        output.target.write(lines)
+                    ok = True
+                except Exception as error:
+                    self._fail_output(output, error)
+            with self.condition:
+                if ok:
+                    output.written += len(valid_indices)
+                    output.failed += len(indices) - len(valid_indices)
+                else:
+                    output.failed += len(indices)
+            if ok:
+                for i in valid_indices:
+                    delivered[i] += 1
+        # Aggregate outside the admission lock so a batch cannot lengthen the
+        # producer's critical section by hundreds of Python loop iterations.
+        written = sum(count > 0 for count in delivered)
+        failed = len(batch) - written
+        partial = sum(0 < count < target for count, target in zip(delivered, targets))
+        with self.condition:
+            self.written += written
+            self.failed += failed
+            self.partial += partial
+            self.processed += len(batch)
+
+    def _report_drops(self) -> None:
+        # No producer-side clock reads in worker mode, including the overflow path.
+        # These timestamps bound accounting windows, not individual drop times.
+        with self.condition:
+            totals = self.drops.copy()
+            until_ns = clock_ns()
+        for output in self.outputs:
+            if output.error is not None:
+                continue
+            counts = {LEVELS[k]: totals[k] - output.reported[k] for k in LEVELS}
+            count = sum(counts.values())
+            if not count:
+                continue
+            record = {
+                "timestamp": self.formatter.timestamp(until_ns),
+                "timestamp_mode": "worker",
+                "level": "WARNING",
+                "event": "日志队列溢出",
+                "_log_hub": {
+                    "kind": "queue_overflow",
+                    "dropped": count,
+                    "dropped_by_level": counts,
+                    "window_start": self.formatter.timestamp(output.report_since_ns),
+                    "window_end": self.formatter.timestamp(until_ns),
+                },
+            }
+            try:
+                # Bypasses queue and severity filter; flush before marking reported.
+                output.target.write([self.formatter.render(record, output.style)])
+                output.target.flush()
+            except Exception as error:
+                self._fail_output(output, error)
+            else:
+                with self.condition:
+                    output.reported = totals.copy()
+                    output.report_since_ns = until_ns
+
+    def _run(self) -> None:
+        next_flush = time.monotonic() + self.flush_interval
+        next_report = time.monotonic() + self.drop_report_interval
+        try:
+            while True:
+                with self.condition:
+                    self.condition.wait_for(
+                        lambda: bool(self.queue) or self.closing,
+                        timeout=max(
+                            0,
+                            min(
+                                self.poll_interval, min(next_flush, next_report) - time.monotonic()
+                            ),
+                        ),
+                    )
+                    if len(self.queue) <= self.batch_size:
+                        batch, self.queue = self.queue, deque()
+                    else:
+                        batch = [self.queue.popleft() for _ in range(self.batch_size)]
+                    closing = self.closing and not self.queue
+                if batch:
+                    self._write_batch(batch)
+                now = time.monotonic()
+                if closing or now >= next_report:
+                    self._report_drops()
+                    next_report = time.monotonic() + self.drop_report_interval
+                if closing or now >= next_flush:
+                    self._flush()
+                    next_flush = time.monotonic() + self.flush_interval
+                if closing:
+                    break
+        except BaseException as error:
+            with self.condition:
+                self.worker_error = f"{type(error).__name__}: {error}"
+                self.failed += self.accepted - self.processed
+                self.processed = self.accepted
+                self.queue.clear()
+        finally:
+            for output in self.outputs:
+                try:
+                    output.target.close()
+                except Exception as error:
+                    self._fail_output(output, error)

logleaf/_models.py

@@ -0,0 +1,64 @@
+"""Shared event envelopes and statistics; no queue or I/O behavior."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .exceptions import ExceptionSnapshot
+from .formatting import LEVELS
+from .sinks import ConsoleOutput, FileOutput
+
+Scalar = str | int | float | bool | None
+
+
+@dataclass(frozen=True)
+class OutputStats:
+    written: int
+    failed: int
+    dropped_reported: int
+    dropped_unreported: int
+    error: str | None
+
+
+@dataclass(frozen=True)
+class LoggerStats:
+    accepted: int
+    written: int
+    failed: int
+    partial: int
+    pending: int
+    dropped: int
+    dropped_by_level: dict[str, int]
+    rejected_unavailable: int
+    high_watermark: int
+    outputs: dict[str, OutputStats]
+    worker_error: str | None
+    format_errors: int
+    last_format_error: str | None
+
+
+@dataclass(slots=True)
+class _Event:
+    level: int
+    message: str
+    fields: dict[str, Scalar]
+    timestamp_ns: int | None
+    context: dict[str, Scalar] | None = None
+
+
+@dataclass(slots=True)
+class _ExceptionEvent(_Event):
+    exception: ExceptionSnapshot | None = None
+
+
+@dataclass
+class _Output:
+    name: str
+    target: FileOutput | ConsoleOutput
+    level: int
+    style: str
+    written: int = 0
+    failed: int = 0
+    error: str | None = None
+    reported: dict[int, int] = field(default_factory=lambda: dict.fromkeys(LEVELS, 0))
+    report_since_ns: int = 0

logleaf/exceptions.py

@@ -0,0 +1,134 @@
+"""Detached exception snapshots: no traceback frames, exception objects or locals."""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+MAX_FRAMES = 64
+MAX_NODES = 64
+MAX_DEPTH = 8
+
+
+def _text(value: object) -> str:
+    try:
+        return str.__str__(str(value))  # Discard attributes on a returned str subclass.
+    except Exception:
+        return "<exception text unavailable>"
+
+
+@dataclass(frozen=True, slots=True)
+class Frame:
+    filename: str
+    line: int
+    function: str
+
+
+@dataclass(frozen=True, slots=True)
+class ExceptionSnapshot:
+    type_name: str
+    message: str
+    frames: tuple[Frame, ...]
+    notes: tuple[str, ...]
+    previous: ExceptionSnapshot | None
+    relation: str | None
+    children: tuple[ExceptionSnapshot, ...]
+    omitted_frames: int = 0
+    omitted_children: int = 0
+    truncated_chain: bool = False
+
+    def render(self) -> str:
+        """Worker-only string assembly. Source files are never read."""
+        lines: list[str] = []
+        if self.previous is not None:
+            lines.append(self.previous.render())
+            lines.append(
+                "The above exception was the direct cause of the following exception:"
+                if self.relation == "cause"
+                else "During handling of the above exception, another exception occurred:"
+            )
+        if self.truncated_chain:
+            lines.append("[exception chain truncated: cycle or snapshot limit]")
+        if self.frames:
+            lines.append("Traceback (most recent call last):")
+            if self.omitted_frames:
+                lines.append(f"  [... {self.omitted_frames} earlier frames omitted]")
+            for frame in self.frames:
+                lines.append(f'  File "{frame.filename}", line {frame.line}, in {frame.function}')
+        lines.append(f"{self.type_name}: {self.message}")
+        lines.extend(self.notes)
+        for index, child in enumerate(self.children, 1):
+            lines.append(f"  + Exception group member {index}:")
+            lines.extend("    " + line for line in child.render().splitlines())
+        if self.omitted_children:
+            lines.append(f"  [... {self.omitted_children} exception group members omitted]")
+        return "\n".join(lines)
+
+
+def capture_exception(error: BaseException) -> ExceptionSnapshot:
+    """Capture descriptions now without retaining application-owned objects.
+
+    Limits bound stored frame/node counts, not string sizes or capture time.
+    str(error) necessarily executes on the caller; ordinary logs never use this.
+    """
+    seen: set[int] = set()
+
+    def capture(current: BaseException, depth: int) -> ExceptionSnapshot:
+        seen.add(id(current))
+        frames: deque[Frame] = deque(maxlen=MAX_FRAMES)
+        count = 0
+        tb = current.__traceback__
+        while tb is not None:
+            code = tb.tb_frame.f_code
+            frames.append(Frame(str(code.co_filename), tb.tb_lineno, str(code.co_name)))
+            count += 1
+            tb = tb.tb_next
+        if isinstance(current, SyntaxError):
+            if isinstance(current.filename, str) and type(current.lineno) is int:
+                frames.append(Frame(str(current.filename), current.lineno, "<syntax>"))
+                count += 1
+        kind = type(current)
+        type_name = f"{_text(kind.__module__)}.{_text(kind.__qualname__)}"
+        message = _text(current)
+        try:
+            notes = current.__notes__
+        except Exception:
+            notes = ()
+        # Freeze notes too: callers may mutate their original list after logging.
+        notes = tuple(_text(note) for note in notes) if isinstance(notes, (list, tuple)) else ()
+        previous = None
+        relation = None
+        truncated = False
+        candidate = current.__cause__
+        if candidate is not None:
+            relation = "cause"
+        elif not current.__suppress_context__:
+            candidate = current.__context__
+            relation = "context" if candidate is not None else None
+        if candidate is not None:
+            if id(candidate) in seen or depth >= MAX_DEPTH or len(seen) >= MAX_NODES:
+                truncated = True
+            else:
+                previous = capture(candidate, depth + 1)
+        children = []
+        omitted = 0
+        if isinstance(current, BaseExceptionGroup):
+            for child in current.exceptions:
+                if id(child) in seen or depth >= MAX_DEPTH or len(seen) >= MAX_NODES:
+                    omitted += 1
+                else:
+                    children.append(capture(child, depth + 1))
+        return ExceptionSnapshot(
+            type_name,
+            message,
+            tuple(frames),
+            notes,
+            previous,
+            relation,
+            tuple(children),
+            max(0, count - MAX_FRAMES),
+            omitted,
+            truncated,
+        )
+
+    return capture(error, 0)

logleaf/experimental/__init__.py

@@ -0,0 +1,5 @@
+"""Historical performance prototypes, outside the stable Logger API."""
+
+from .prototype import BatchLogger, Stats
+
+__all__ = ["BatchLogger", "Stats"]