logxpy 0.1.0__py3-none-any.whl

logxpy/json.py

@@ -0,0 +1,149 @@
+"""Custom JSON encoding support."""
+
+from typing import Callable
+import json
+import sys
+from pathlib import Path
+from datetime import date, time
+import platform
+
+
+class EliotJSONEncoder(json.JSONEncoder):
+    """
+    DEPRECATED. JSON encoder with additional functionality.
+
+    In particular, supports NumPy types.
+    """
+
+    def default(self, o):
+        return json_default(o)
+
+
+def json_default(o: object) -> object:
+    """
+    JSON object encoder for non-standard types.  In particular, supports NumPy
+    types, Path objects, Pydantic models, dataclasses, Pandas and Polars
+    objects.  If you are wrapping it, call it last, as it will raise a
+    ``TypeError`` on unsupported types.
+    """
+    numpy = sys.modules.get("numpy", None)
+    if numpy is not None:
+        if isinstance(o, numpy.floating):
+            return float(o)
+        if isinstance(o, numpy.integer):
+            return int(o)
+        if isinstance(o, numpy.bool_):
+            return bool(o)
+        if isinstance(o, numpy.ndarray):
+            if o.size > 10000:
+                # Too big to want to log as-is, log a summary:
+                return {
+                    "array_start": o.flat[:10000].tolist(),
+                    "original_shape": o.shape,
+                }
+            else:
+                return o.tolist()
+
+    # Add Pydantic support
+    pydantic = sys.modules.get("pydantic", None)
+    if pydantic is not None and isinstance(o, pydantic.BaseModel):
+        return o.model_dump()
+
+    if isinstance(o, Path):
+        return str(o)
+
+    if isinstance(o, date):
+        return o.isoformat()
+
+    if isinstance(o, time):
+        return o.isoformat()
+
+    if isinstance(o, set):
+        return list(o)
+
+    if isinstance(o, complex):
+        return {"real": o.real, "imag": o.imag}
+
+    # Add Pandas support
+    pandas = sys.modules.get("pandas", None)
+    if pandas is not None:
+        if isinstance(o, pandas.Timestamp):
+            return o.isoformat()
+        if isinstance(o, pandas.Series):
+            return o.to_list()
+        if isinstance(o, pandas.DataFrame):
+            return o.to_dict(orient="records")
+        if isinstance(o, pandas.Interval):
+            return {"left": o.left, "right": o.right, "closed": o.closed}
+        if isinstance(o, pandas.Period):
+            return str(o)
+
+    # Add Polars support
+    polars = sys.modules.get("polars", None)
+    if polars is not None:
+        if isinstance(o, polars.Series):
+            return o.to_list()
+        if isinstance(o, polars.DataFrame):
+            return o.to_dicts()
+        if isinstance(o, polars.Datetime):
+            return o.isoformat()
+
+    raise TypeError("Unsupported type")
+
+
+if platform.python_implementation() == "PyPy":
+    # We're not using orjson, so need to serialize a few more types.
+
+    original_json_default = json_default
+
+    def json_default(o: object, original_json_default=original_json_default) -> object:
+        from datetime import datetime
+        from enum import Enum
+        from uuid import UUID
+
+        # Add dataclass support
+        if hasattr(o, "__dataclass_fields__"):
+            return {field: getattr(o, field) for field in o.__dataclass_fields__}
+        if isinstance(o, datetime):
+            return o.isoformat()
+
+        if isinstance(o, UUID):
+            return str(o)
+
+        if isinstance(o, Enum):
+            return o.value
+
+        return original_json_default(o)
+
+    json_default.__doc__ = original_json_default.__doc__
+    del original_json_default
+
+
+def _encoder_to_default_function(
+    encoder: json.JSONEncoder,
+) -> Callable[[object], object]:
+    """
+    Convert an encoder into a default function usable by ``orjson``.
+    """
+
+    def default(o: object) -> object:
+        return encoder.default(o)
+
+    return default
+
+
+try:
+    from orjson import dumps as _dumps_bytes
+
+    def _dumps_unicode(o: object, default=None) -> str:
+        return _dumps_bytes(o, default=default).decode("utf-8")
+
+except ImportError:
+
+    def _dumps_bytes(o: object, default=None) -> bytes:
+        """Serialize an object to JSON, output bytes."""
+        return json.dumps(o, default=default).encode("utf-8")
+
+    _dumps_unicode = json.dumps
+
+__all__ = ["EliotJSONEncoder", "json_default"]

logxpy/loggerx.py

@@ -0,0 +1,253 @@
+"""Main Logger facade - LoggerX API built on eliot."""
+
+from __future__ import annotations
+
+import traceback
+from contextlib import contextmanager
+from typing import Any
+
+from . import decorators
+from ._action import current_action  # Use eliot's to work with both Action types
+from ._async import _emit, current_scope, scope
+from ._base import now, uuid
+from ._fmt import format_value
+from ._output import to_file
+from ._types import Level, Record
+
+
+class Logger:
+    """LoggerX-compatible logger with fluent API."""
+
+    __slots__ = ("_context", "_level", "_masker", "_name")
+
+    def __init__(self, name: str = "root", context: dict[str, Any] | None = None):
+        self._level = Level.DEBUG
+        self._name = name
+        self._context = context or {}
+        self._masker = None
+
+    # === Level Methods (fluent - return self) ===
+    def debug(self, msg: str, **f: Any) -> Logger:
+        return self._log(Level.DEBUG, msg, **f)
+
+    def info(self, msg: str, **f: Any) -> Logger:
+        return self._log(Level.INFO, msg, **f)
+
+    def success(self, msg: str, **f: Any) -> Logger:
+        return self._log(Level.SUCCESS, msg, **f)
+
+    def note(self, msg: str, **f: Any) -> Logger:
+        return self._log(Level.NOTE, msg, **f)
+
+    def warning(self, msg: str, **f: Any) -> Logger:
+        return self._log(Level.WARNING, msg, **f)
+
+    def error(self, msg: str, **f: Any) -> Logger:
+        return self._log(Level.ERROR, msg, **f)
+
+    def critical(self, msg: str, **f: Any) -> Logger:
+        return self._log(Level.CRITICAL, msg, **f)
+
+    def checkpoint(self, msg: str, **f: Any) -> Logger:
+        return self._log(Level.INFO, f"📍 {msg}", **f)
+
+    def exception(self, msg: str, **f: Any) -> Logger:
+        f["eliot:traceback"] = traceback.format_exc()
+        return self._log(Level.ERROR, msg, **f)
+
+    def __call__(self, msg: str, **f: Any) -> Logger:
+        """Shortcut: log("msg") == log.info("msg")"""
+        return self.info(msg, **f)
+
+    # === Universal Send ===
+    def send(self, msg: str, data: Any, **f: Any) -> Logger:
+        return self._log(Level.INFO, msg, data=format_value(data), **f)
+
+    # === Type Methods ===
+    def df(self, data: Any, title: str | None = None, **opts: Any) -> Logger:
+        return self.send(title or "DataFrame", data, **opts)
+
+    def tensor(self, data: Any, title: str | None = None) -> Logger:
+        return self.send(title or "Tensor", data)
+
+    def json(self, data: dict, title: str | None = None) -> Logger:
+        import json as _json
+
+        return self._log(Level.INFO, title or "JSON", content=_json.dumps(data, indent=2, default=str)[:5000])
+
+    def img(self, data: Any, title: str | None = None, **opts: Any) -> Logger:
+        return self.send(title or "Image", data, **opts)
+
+    def plot(self, fig: Any, title: str | None = None) -> Logger:
+        return self.send(title or "Plot", fig)  # Basic support via repr/str for now
+
+    def tree(self, data: Any, title: str | None = None) -> Logger:
+        return self.send(title or "Tree", data)  # Basic support
+
+    def table(self, data: list[dict], title: str | None = None) -> Logger:
+        return self.send(title or "Table", data)  # Basic support
+
+    # === Context (LoggerX features) ===
+    def scope(self, **ctx: Any):
+        """Create nested scope: `with log.scope(user_id=123):`"""
+        return scope(**ctx)
+
+    def ctx(self, **ctx: Any) -> Logger:
+        """Fluent interface to add context to a new logger instance."""
+        new_ctx = self._context.copy()
+        new_ctx.update(ctx)
+        child = Logger(self._name, new_ctx)
+        child._level = self._level
+        return child
+
+    def new(self, name: str | None = None) -> Logger:
+        """Create child logger with name."""
+        new_name = f"{self._name}.{name}" if name else self._name
+        child = Logger(new_name, self._context.copy())
+        child._level = self._level
+        return child
+
+    @contextmanager
+    def span(self, name: str, **attributes: Any):
+        """OpenTelemetry span context manager."""
+        try:
+            from opentelemetry import trace as otel
+
+            tracer = otel.get_tracer(__name__)
+            with tracer.start_as_current_span(name, attributes=attributes) as span:
+                yield span
+        except ImportError:
+
+            class MockSpan:
+                def set_attribute(self, k, v):
+                    pass
+
+                def add_event(self, n, a=None):
+                    pass
+
+                def __enter__(self):
+                    return self
+
+                def __exit__(self, *a):
+                    pass
+
+            yield MockSpan()
+
+    # === Decorators (exposed as methods) ===
+    logged = staticmethod(decorators.logged)
+    timed = staticmethod(decorators.timed)
+    retry = staticmethod(decorators.retry)
+    generator = staticmethod(decorators.generator)
+    aiterator = staticmethod(decorators.aiterator)
+    trace = staticmethod(decorators.trace)
+
+    # === Config ===
+    def configure(
+        self,
+        level: str = "DEBUG",
+        destinations: list[str] | None = None,
+        format: str = "rich",
+        context: dict[str, Any] | None = None,
+        mask_fields: list[str] | None = None,
+        **_: Any,
+    ) -> Logger:
+        self._level = Level[level.upper()]
+
+        if context:
+            self._context.update(context)
+
+        if mask_fields:
+            from ._mask import Masker
+
+            self._masker = Masker(mask_fields, [])
+            set_global_masker(self._masker)
+
+        if destinations:
+            # Clear existing destinations if possible, or we just add new ones?
+            # Eliot doesn't easily allow clearing all destinations without accessing private members.
+            # But we can try.
+            # However, EliotLogger._destinations is a global singleton-like thing.
+
+            # Simple implementation: Add what's requested.
+            for dest in destinations:
+                if dest == "console":
+                    # We don't add console by default in eliot, usually.
+                    # But here we can add our ConsoleDestination
+                    # Need to adapt it to Eliot's interface?
+                    # No, _emit uses _destinations.send.
+                    # Our ConsoleDestination is an async thing in _dest.py?
+                    # Wait, _dest.py destinations have `write(record)`.
+                    # Eliot destinations expect `call(dict)`.
+
+                    # We need to bridge between Eliot destinations and LoggerX destinations if we want to use LoggerX destinations.
+                    # Or we just use Eliot's to_file.
+
+                    if format == "rich":
+                        # We can't easily plug async destination into sync logxpy pipeline yet
+                        # For now, we assume standard logxpy setup + our extensions.
+                        pass
+
+                elif dest.startswith("file://"):
+                    path = dest.replace("file://", "")
+                    to_file(open(path, "a"))
+
+                elif dest.startswith("otel"):
+                    # Setup OTel
+                    pass
+
+        return self
+
+    # === Internal ===
+    def _log(self, level: Level, msg: str, **fields: Any) -> Logger:
+        if level.value < self._level.value:
+            return self
+        act = current_action()
+        task_uuid, task_level = _get_task_info(act)
+
+        # Merge context: global scope + logger instance context
+        ctx = current_scope()
+        if self._context:
+            ctx = {**ctx, **self._context}
+
+        record = Record(
+            timestamp=now(),
+            level=level,
+            message=msg,
+            message_type=f"loggerx:{level.name.lower()}",
+            fields=fields,
+            context=ctx,
+            task_uuid=task_uuid,
+            task_level=task_level,
+        )
+        _emit(record)  # Goes to eliot's destinations + any new handlers
+        return self
+
+
+def _get_task_info(act) -> tuple[str, tuple[int, ...]]:
+    """Extract task info from eliot.Action or AsyncAction."""
+    if act is None:
+        return uuid(), (1,)
+    task_uuid = act.task_uuid
+    # eliot.Action uses _task_level (TaskLevel), AsyncAction uses task_level (tuple)
+    if hasattr(act, "task_level"):  # AsyncAction
+        return task_uuid, act.task_level
+    if hasattr(act, "_task_level"):  # eliot.Action
+        return task_uuid, tuple(act._task_level.as_list())
+    return task_uuid, (1,)
+
+
+# === Global masker ===
+_global_masker = None
+
+
+def set_global_masker(masker):
+    global _global_masker
+    _global_masker = masker
+
+
+def get_global_masker():
+    return _global_masker
+
+
+# === Global instance ===
+log = Logger()

logxpy/logwriter.py

@@ -0,0 +1,84 @@
+"""
+A log destination for use by Twisted applications.
+
+Runs in a thread, so that we don't do blocking I/O in the event loop thread.
+"""
+
+import threading
+from queue import SimpleQueue
+
+from twisted.application.service import Service
+from twisted.internet.threads import deferToThreadPool
+
+from . import addDestination, removeDestination
+
+_STOP = object()
+
+
+class ThreadedWriter(Service):
+    """
+    An non-blocking Eliot log destination that wraps a blocking
+    destination, writing log messages to the latter in a managed thread.
+
+    @ivar _thread: C{None}, or a L{threading.Thread} running the private
+        reactor.
+    """
+
+    name = "Eliot Log Writer"
+
+    def __init__(self, destination, reactor):
+        """
+        @param destination: The underlying destination for log files. This will
+            be called from a non-reactor thread.
+
+        @param reactor: The main reactor.
+        """
+        self._destination = destination
+        self._queue = SimpleQueue()
+        self._mainReactor = reactor
+        self._thread = None
+
+    def startService(self):
+        """
+        Start the writer thread.
+        """
+        Service.startService(self)
+        self._thread = threading.Thread(target=self._reader)
+        self._thread.start()
+        addDestination(self)
+
+    def stopService(self):
+        """
+        Stop the writer thread, wait for it to finish.
+        """
+        Service.stopService(self)
+        removeDestination(self)
+        self._queue.put(_STOP)
+        return deferToThreadPool(
+            self._mainReactor, self._mainReactor.getThreadPool(), self._thread.join
+        )
+
+    def __call__(self, data):
+        """
+        Add the data to the queue, to be serialized to JSON and written by the
+        writer thread with a newline added.
+
+        @param data: C{bytes} to write to disk.
+        """
+        self._queue.put(data)
+
+    def _reader(self):
+        """
+        Runs in a thread, reads messages from a queue and writes them to
+        the wrapped observer.
+        """
+        while True:
+            msg = self._queue.get()
+            if msg is _STOP:
+                return
+            try:
+                self._destination(msg)
+            except Exception:
+                # Lower-level destination blew up, nothing we can do, so
+                # just drop on the floor.
+                pass

logxpy/parse.py

@@ -0,0 +1,191 @@
+"""
+Parse a stream of serialized messages into a forest of
+``WrittenAction`` and ``WrittenMessage`` objects.
+"""
+
+from pyrsistent import PClass, pmap_field, pset_field, discard
+
+from ._message import WrittenMessage, TASK_UUID_FIELD
+from ._action import (
+    TaskLevel,
+    WrittenAction,
+    ACTION_STATUS_FIELD,
+    STARTED_STATUS,
+    ACTION_TYPE_FIELD,
+)
+
+
+class Task(PClass):
+    """
+    A tree of actions with the same task UUID.
+    """
+
+    _nodes = pmap_field(TaskLevel, (WrittenAction, WrittenMessage))
+    _completed = pset_field(TaskLevel)
+    _root_level = TaskLevel(level=[])
+
+    def root(self):
+        """
+        @return: The root L{WrittenAction}.
+        """
+        return self._nodes[self._root_level]
+
+    def is_complete(self):
+        """
+        @return bool: True only if all messages in the task tree have been
+        added to it.
+        """
+        return self._root_level in self._completed
+
+    def _insert_action(self, node):
+        """
+        Add a L{WrittenAction} to the tree.
+
+        Parent actions will be created as necessary.
+
+        @param child: A L{WrittenAction} to add to the tree.
+
+        @return: Updated L{Task}.
+        """
+        task = self
+        if (
+            node.end_message
+            and node.start_message
+            and (len(node.children) == node.end_message.task_level.level[-1] - 2)
+        ):
+            # Possibly this action is complete, make sure all sub-actions
+            # are complete:
+            completed = True
+            for child in node.children:
+                if (
+                    isinstance(child, WrittenAction)
+                    and child.task_level not in self._completed
+                ):
+                    completed = False
+                    break
+            if completed:
+                task = task.transform(["_completed"], lambda s: s.add(node.task_level))
+        task = task.transform(["_nodes", node.task_level], node)
+        return task._ensure_node_parents(node)
+
+    def _ensure_node_parents(self, child):
+        """
+        Ensure the node (WrittenAction/WrittenMessage) is referenced by parent
+        nodes.
+
+        Parent actions will be created as necessary.
+
+        @param child: A L{WrittenMessage} or L{WrittenAction} which is
+            being added to the tree.
+
+        @return: Updated L{Task}.
+        """
+        task_level = child.task_level
+        if task_level.parent() is None:
+            return self
+
+        parent = self._nodes.get(task_level.parent())
+        if parent is None:
+            parent = WrittenAction(
+                task_level=task_level.parent(), task_uuid=child.task_uuid
+            )
+        parent = parent._add_child(child)
+        return self._insert_action(parent)
+
+    def add(self, message_dict):
+        """
+        Update the L{Task} with a dictionary containing a serialized Eliot
+        message.
+
+        @param message_dict: Dictionary whose task UUID matches this one.
+
+        @return: Updated L{Task}.
+        """
+        is_action = message_dict.get(ACTION_TYPE_FIELD) is not None
+        written_message = WrittenMessage.from_dict(message_dict)
+        if is_action:
+            action_level = written_message.task_level.parent()
+            action = self._nodes.get(action_level)
+            if action is None:
+                action = WrittenAction(
+                    task_level=action_level, task_uuid=message_dict[TASK_UUID_FIELD]
+                )
+            if message_dict[ACTION_STATUS_FIELD] == STARTED_STATUS:
+                # Either newly created MissingAction, or one created by
+                # previously added descendant of the action.
+                action = action._start(written_message)
+            else:
+                action = action._end(written_message)
+            return self._insert_action(action)
+        else:
+            # Special case where there is no action:
+            if written_message.task_level.level == [1]:
+                return self.transform(
+                    ["_nodes", self._root_level],
+                    written_message,
+                    ["_completed"],
+                    lambda s: s.add(self._root_level),
+                )
+            else:
+                return self._ensure_node_parents(written_message)
+
+
+class Parser(PClass):
+    """
+    Parse serialized Eliot messages into L{Task} instances.
+
+    @ivar _tasks: Map from UUID to corresponding L{Task}.
+    """
+
+    _tasks = pmap_field(str, Task)
+
+    def add(self, message_dict):
+        """
+        Update the L{Parser} with a dictionary containing a serialized Eliot
+        message.
+
+        @param message_dict: Dictionary of serialized Eliot message.
+
+        @return: Tuple of (list of completed L{Task} instances, updated
+            L{Parser}).
+        """
+        uuid = message_dict[TASK_UUID_FIELD]
+        if uuid in self._tasks:
+            task = self._tasks[uuid]
+        else:
+            task = Task()
+        task = task.add(message_dict)
+        if task.is_complete():
+            parser = self.transform(["_tasks", uuid], discard)
+            return [task], parser
+        else:
+            parser = self.transform(["_tasks", uuid], task)
+            return [], parser
+
+    def incomplete_tasks(self):
+        """
+        @return: List of L{Task} that are not yet complete.
+        """
+        return list(self._tasks.values())
+
+    @classmethod
+    def parse_stream(cls, iterable):
+        """
+        Parse a stream of messages into a stream of L{Task} instances.
+
+        :param iterable: An iterable of serialized Eliot message dictionaries.
+
+        :return: An iterable of parsed L{Task} instances. Remaining
+            incomplete L{Task} will be returned when the input stream is
+            exhausted.
+        """
+        parser = Parser()
+        for message_dict in iterable:
+            completed, parser = parser.add(message_dict)
+            for task in completed:
+                yield task
+        for task in parser.incomplete_tasks():
+            yield task
+
+
+__all__ = ["Parser", "Task", "TaskLevel", "WrittenMessage", "WrittenAction"]