MATE-runtime 1.0.0__py3-none-any.whl

MATE/observation/hub.py

@@ -0,0 +1,451 @@
+"""The observer of a run: loggers, emitters, worker channels and the dispatch to sinks.
+
+Threading rule (design §3.6): sinks and ``on_emit`` are called only from the run's driver
+thread. Process and thread workers log and emit through a :class:`WorkerChannel`; the driver
+moves their records to the sinks with :meth:`Observer.drain`. An emitter may hold its
+emissions instead: the executor hands them over with :meth:`Observer.receive` once the
+results of the work that made them have returned.
+"""
+
+import multiprocessing
+import queue
+import threading
+import time
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, replace
+from typing import Any, Literal
+
+from MATE_contract import EventSink, Logger
+
+from MATE.observation.records import LEVELS, LOGGER_LEVELS, Emission, EventRecord, LogRecord
+from MATE.observation.sinks import Sink
+
+__all__ = ["Observer", "WorkerChannel"]
+
+
+_DEBUG, _INFO, _WARNING, _ERROR = (LOGGER_LEVELS[name]
+                                   for name in ("debug", "info", "warning", "error"))
+
+
+def _threshold(verbosity: str) -> int:
+    """Return the threshold of a verbosity name."""
+    try:
+        return LEVELS[verbosity]
+    except KeyError:
+        raise ValueError(
+            f"verbosity {verbosity!r}: unknown; expected one of {', '.join(LEVELS)}") from None
+
+
+def _format(msg: str, args: tuple[object, ...]) -> str:
+    """Return ``msg % args``, or the raw pieces when they do not match."""
+    if not args:
+        return msg
+    try:
+        return msg % args
+    except (TypeError, ValueError, KeyError):
+        return f"{msg} {args!r}"
+
+
+class _Logger:
+    """A logger whose level test precedes any formatting ([K] LG1)."""
+
+    __slots__ = ("_path", "_level_of", "_write")
+
+    def __init__(self, path: str, level_of: Callable[[], int],
+                 write: Callable[[int, str, str], None]) -> None:
+        self._path = path
+        self._level_of = level_of
+        self._write = write
+
+    def _log(self, level: int, msg: str, args: tuple[object, ...]) -> None:
+        """Format and write one message if its level passes the threshold."""
+        if level < self._level_of():
+            return
+        self._write(level, self._path, _format(msg, args))
+
+    def debug(self, msg: str, /, *args: object) -> None:
+        """Log a debugging message (level 10)."""
+        self._log(_DEBUG, msg, args)
+
+    def info(self, msg: str, /, *args: object) -> None:
+        """Log an informative message (level 30)."""
+        self._log(_INFO, msg, args)
+
+    def warning(self, msg: str, /, *args: object) -> None:
+        """Log a warning (level 35)."""
+        self._log(_WARNING, msg, args)
+
+    def error(self, msg: str, /, *args: object) -> None:
+        """Log an error (level 40)."""
+        self._log(_ERROR, msg, args)
+
+
+class _Emitter:
+    """An event sink bound to one component; builds an :class:`Emission` per call."""
+
+    __slots__ = ("_bound", "_clock", "_send")
+
+    def __init__(self, bound: Mapping[str, Any], clock: Callable[[], tuple[int | None, object]],
+                 send: Callable[[Emission], None]) -> None:
+        self._bound = bound
+        self._clock = clock
+        self._send = send
+
+    def __call__(self, name: str, /, **fields: object) -> None:
+        """Emit the event ``name`` with its fields."""
+        b = self._bound
+        generation, date = b["run_generation"], b["date"]
+        if generation is None or date is None:
+            now_generation, now_date = self._clock()
+            generation = now_generation if generation is None else generation
+            date = now_date if date is None else date
+        self._send(Emission(origin=b["origin"], module=b["module"], subpop=b["subpop"],
+                            path=b["path"], name=name, fields=fields, run_generation=generation,
+                            date=date, request_ranks=b["request_ranks"], wall=time.time()))
+
+
+def _no_clock() -> tuple[None, None]:
+    """The clock of a worker: generation and date are given at emitter creation."""
+    return None, None
+
+
+@dataclass(frozen=True)
+class WorkerChannel:
+    """The handle a worker logs and emits through ([X] SK2).
+
+    A channel of kind ``"process"`` holds a multiprocessing queue and event: it is handed to a
+    worker process at its creation (process or pool initializer arguments). A channel of kind
+    ``"thread"`` holds a ``queue.SimpleQueue`` and a ``threading.Event``.
+
+    Parameters
+    ----------
+    kind : {"process", "thread"}
+    records : queue
+        Where the worker puts its records, drained by :meth:`Observer.drain`.
+    stop : event
+        The stop flag ([K] CT7, [X] GU5).
+    """
+
+    kind: Literal["process", "thread"]
+    records: Any
+    stop: Any
+
+    def logger(self, path: str, verbosity: str) -> Logger:
+        """Return a logger for a component of the worker.
+
+        Parameters
+        ----------
+        path : str
+            Log path of the component.
+        verbosity : str
+            Threshold applied in the worker, the run's initial verbosity (DD-13).
+
+        Returns
+        -------
+        Logger
+            Messages below the threshold are neither formatted nor sent.
+        """
+        threshold = _threshold(verbosity)
+        put = self.records.put
+        return _Logger(path, lambda: threshold,
+                       lambda level, p, text: put(("m", level, p, text, time.time())))
+
+    def emitter(self, *, origin: str, module: str, subpop: str | None, path: str,
+                request_ranks: tuple[int, ...] | None, run_generation: int | None,
+                date: object, hold: list[Emission] | None = None) -> EventSink:
+        """Return an event sink for a component of the worker.
+
+        Parameters
+        ----------
+        origin : {"system", "method"}
+        module : str
+        subpop : str or None
+        path : str
+        request_ranks : tuple of int or None
+            Candidate index -> request rank, for an evaluate batch.
+        run_generation : int or None
+        date : object
+        hold : list of Emission, optional
+            Where the emissions are kept instead of being sent; the executor returns them
+            with the results of the work that made them.
+
+        Returns
+        -------
+        EventSink
+            Each call sends one :class:`Emission` to the driver, or appends it to ``hold``.
+        """
+        bound = {"origin": origin, "module": module, "subpop": subpop, "path": path,
+                 "request_ranks": request_ranks, "run_generation": run_generation, "date": date}
+        if hold is not None:
+            return _Emitter(bound, _no_clock, hold.append)
+        put = self.records.put
+        return _Emitter(bound, _no_clock, lambda emission: put(("e", emission)))
+
+    def stopping(self) -> bool:
+        """Whether the run is stopping ([K] CT7, [X] GU5).
+
+        Returns
+        -------
+        bool
+        """
+        return self.stop.is_set()
+
+
+class Observer:
+    """The observation hub of one run ([X] §17).
+
+    Parameters
+    ----------
+    run : str
+        The run directory name, written in every record.
+    sinks : sequence of Sink
+        Where records go, in order.
+    verbosity : {"debug", "verbose", "normal", "quiet"}
+        The message threshold ([X] OB3).
+    clock : callable, optional
+        Returns the current ``(run_generation, date)``, used for emissions and events that
+        do not carry them.
+
+    Attributes
+    ----------
+    on_emit : callable or None
+        Receives every module emission; set by the run to the readout. When ``None``,
+        emissions go to the event stream as they are.
+
+    Notes
+    -----
+    Implements [X] OB2, SK1, SK2 and [K] LG1. Records are numbered by one counter shared by
+    messages and events. Sinks and ``on_emit`` are called from the driver thread only.
+    """
+
+    def __init__(self, run: str, sinks: Sequence[Sink], verbosity: str, *,
+                 clock: Callable[[], tuple[int | None, object]] = lambda: (None, None)) -> None:
+        self.run = run
+        self._sinks = list(sinks)
+        self._threshold = _threshold(verbosity)
+        self._clock = clock
+        self._seq = 0
+        self._channels: dict[str, WorkerChannel] = {}
+        self._stopping = False
+        self._closed = False
+        self.on_emit: Callable[[Emission], None] | None = None
+
+    def _next(self) -> int:
+        """Return the next record rank."""
+        self._seq += 1
+        return self._seq
+
+    def _write(self, level: int, path: str, text: str, wall: float | None = None) -> None:
+        """Send one formatted message to the sinks."""
+        record = LogRecord(self.run, self._next(), time.time() if wall is None else wall,
+                           level, path, text)
+        for sink in self._sinks:
+            sink.message(record)
+
+    def receive(self, emission: Emission) -> None:
+        """Hand a module emission to ``on_emit``, or straight to the event stream.
+
+        Parameters
+        ----------
+        emission : Emission
+
+        Notes
+        -----
+        Called from the driver thread: by its emitters, by :meth:`drain`, and by an executor
+        releasing the emissions it held for work whose results have returned.
+        """
+        if self.on_emit is not None:
+            self.on_emit(emission)
+        else:
+            self.publish(EventRecord(self.run, 0, emission.wall, emission.path, emission.name,
+                                     emission.run_generation, emission.date,
+                                     dict(emission.fields)))
+
+    def logger(self, path: str) -> Logger:
+        """Return the logger of a component of the driver process.
+
+        Parameters
+        ----------
+        path : str
+            Log path of the component ([X] OB3).
+
+        Returns
+        -------
+        Logger
+            The level test comes before any formatting ([K] LG1); the threshold followed is
+            the observer's current one.
+        """
+        return _Logger(path, lambda: self._threshold, self._write)
+
+    def emitter(self, *, origin: str, module: str, subpop: str | None, path: str,
+                request_ranks: tuple[int, ...] | None = None,
+                run_generation: int | None = None, date: object = None,
+                hold: list[Emission] | None = None) -> EventSink:
+        """Return the event sink of a component of the driver process.
+
+        Parameters
+        ----------
+        origin : {"system", "method"}
+        module : str
+        subpop : str or None
+        path : str
+        request_ranks : tuple of int or None, default None
+        run_generation : int or None, default None
+            ``None``: read from the clock at each emission.
+        date : object, default None
+            ``None``: read from the clock at each emission.
+        hold : list of Emission, optional
+            Where the emissions are kept instead of being received at once; the executor
+            hands them over with :meth:`receive` when the results of their work return.
+
+        Returns
+        -------
+        EventSink
+            Each call builds one :class:`Emission` and hands it to ``on_emit``, or appends it
+            to ``hold``.
+        """
+        bound = {"origin": origin, "module": module, "subpop": subpop, "path": path,
+                 "request_ranks": request_ranks, "run_generation": run_generation, "date": date}
+        return _Emitter(bound, self._clock, self.receive if hold is None else hold.append)
+
+    def message(self, level: int, path: str, text: str) -> None:
+        """Write one of the runtime's own messages.
+
+        Parameters
+        ----------
+        level : int
+            10 to 40; 20 for the runtime's detail.
+        path : str
+        text : str
+        """
+        if level >= self._threshold:
+            self._write(level, path, text)
+
+    def trace(self, path: str, text: str) -> None:
+        """Hand a traceback to every sink at the debug level, whatever the verbosity.
+
+        Parameters
+        ----------
+        path : str
+        text : str
+            The traceback.
+
+        Notes
+        -----
+        [X] RN5, S4: a traceback goes to the debug log. Each sink applies its own threshold,
+        so the file sink writes it to ``debug.log`` only and the terminal skips it.
+        """
+        self._write(10, path, text)
+
+    def event(self, name: str, *, path: str = "run", **fields: object) -> None:
+        """Publish one of the runtime's own events (design §3.15).
+
+        Parameters
+        ----------
+        name : str
+            ``run.start``, ``generation.end``, ``checkpoint``, ...
+        path : str, default "run"
+        **fields : object
+            Plain data.
+        """
+        generation, date = self._clock()
+        self.publish(EventRecord(self.run, 0, time.time(), path, name, generation, date,
+                                 dict(fields)))
+
+    def publish(self, record: EventRecord) -> None:
+        """Send an event to the sinks.
+
+        Parameters
+        ----------
+        record : EventRecord
+            Its ``run`` and ``seq`` are set by the observer.
+        """
+        record = replace(record, run=self.run, seq=self._next())
+        for sink in self._sinks:
+            sink.event(record)
+
+    def set_verbosity(self, verbosity: str) -> None:
+        """Change the threshold of the observer and of every sink that has one ([X] GU5).
+
+        Parameters
+        ----------
+        verbosity : {"debug", "verbose", "normal", "quiet"}
+
+        Raises
+        ------
+        ValueError
+            For an unknown verbosity.
+        """
+        self._threshold = _threshold(verbosity)
+        for sink in self._sinks:
+            setter = getattr(sink, "set_verbosity", None)
+            if setter is not None:
+                setter(verbosity)
+
+    def channel(self, kind: Literal["process", "thread"]) -> WorkerChannel:
+        """Return the worker channel of a kind, created on first call.
+
+        Parameters
+        ----------
+        kind : {"process", "thread"}
+
+        Returns
+        -------
+        WorkerChannel
+            ``"process"``: a queue and an event of the ``forkserver`` context;
+            ``"thread"``: a ``queue.SimpleQueue`` and a ``threading.Event``.
+        """
+        channel = self._channels.get(kind)
+        if channel is None:
+            if kind == "process":
+                context = multiprocessing.get_context("forkserver")
+                channel = WorkerChannel("process", context.Queue(), context.Event())
+            elif kind == "thread":
+                channel = WorkerChannel("thread", queue.SimpleQueue(), threading.Event())
+            else:
+                raise ValueError(f"channel kind {kind!r}: unknown; expected 'process' or 'thread'")
+            if self._stopping:
+                channel.stop.set()
+            self._channels[kind] = channel
+        return channel
+
+    def drain(self) -> None:
+        """Move the records the workers have sent to the sinks and ``on_emit``.
+
+        Notes
+        -----
+        Called by the driver thread only; records of one channel keep their arrival order.
+        Worker messages are filtered again at the current threshold.
+        """
+        for channel in self._channels.values():
+            while True:
+                try:
+                    item = channel.records.get_nowait()
+                except queue.Empty:
+                    break
+                if item[0] == "m":
+                    _, level, path, text, wall = item
+                    if level >= self._threshold:
+                        self._write(level, path, text, wall)
+                else:
+                    self.receive(item[1])
+
+    def set_stopping(self) -> None:
+        """Set the stop flag of every channel, present and future ([K] CT7)."""
+        self._stopping = True
+        for channel in self._channels.values():
+            channel.stop.set()
+
+    def close(self) -> None:
+        """Drain the channels, close the sinks and release the queues."""
+        if self._closed:
+            return
+        self.drain()
+        self._closed = True
+        for sink in self._sinks:
+            sink.close()
+        process = self._channels.get("process")
+        if process is not None:
+            process.records.close()
+            process.records.join_thread()
+

MATE/observation/records.py

@@ -0,0 +1,125 @@
+"""Records of the two observation streams: messages for humans and structured events.
+
+Levels are integers ([X] OB3, DD-13): a verbosity names a threshold, a logger method a level.
+"""
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Literal
+
+__all__ = ["LEVELS", "LOGGER_LEVELS", "LEVEL_NAMES", "LogRecord", "EventRecord", "Emission"]
+
+LEVELS: Mapping[str, int] = MappingProxyType(
+    {"debug": 10, "verbose": 20, "normal": 30, "quiet": 40})
+"""Verbosity name -> threshold ([X] OB3)."""
+
+LOGGER_LEVELS: Mapping[str, int] = MappingProxyType(
+    {"debug": 10, "info": 30, "warning": 35, "error": 40})
+"""Logger method -> level (DD-13); level 20 serves the runtime's own detailed messages."""
+
+LEVEL_NAMES: Mapping[int, str] = MappingProxyType(
+    {10: "DEBUG", 20: "VERBOSE", 30: "INFO", 35: "WARNING", 40: "ERROR"})
+"""Level -> the label written in the journals."""
+
+
+@dataclass(frozen=True)
+class LogRecord:
+    """One message for humans ([X] §17.2).
+
+    Parameters
+    ----------
+    run : str
+        The run directory name.
+    seq : int
+        Rank of the record in the run's observation streams.
+    wall : float
+        Wall-clock time (seconds since the epoch) at which the message was logged.
+    level : int
+        Its level (``LOGGER_LEVELS``, or 20 for the runtime's detail).
+    path : str
+        Log path of the component, e.g. ``run_017/subpop.proies/method.DEBoG/mutation``.
+    message : str
+        The formatted message.
+    """
+
+    run: str
+    seq: int
+    wall: float
+    level: int
+    path: str
+    message: str
+
+
+@dataclass(frozen=True)
+class EventRecord:
+    """One structured event ([X] §17.2).
+
+    Parameters
+    ----------
+    run : str
+        The run directory name.
+    seq : int
+        Rank of the record in the run's observation streams.
+    wall : float
+        Wall-clock time of the emission.
+    path : str
+        Path of the emitting component.
+    name : str
+        Event name.
+    run_generation : int or None
+        Run generation at the emission.
+    date : object or None
+        Date in the calendar's unit.
+    fields : mapping of str to object
+        Plain data.
+    """
+
+    run: str
+    seq: int
+    wall: float
+    path: str
+    name: str
+    run_generation: int | None
+    date: object | None
+    fields: Mapping[str, object]
+
+
+@dataclass(frozen=True)
+class Emission:
+    """One ``ctx.emit`` call, before the readout classifies it (design §3.6).
+
+    Parameters
+    ----------
+    origin : {"system", "method"}
+        Kind of the emitting module.
+    module : str
+        Module name.
+    subpop : str or None
+        Subpopulation path, for method emissions.
+    path : str
+        Log path of the emitting component.
+    name : str
+        The name as emitted (before the ``method.`` prefix, [K] LG3).
+    fields : mapping of str to object
+        The keyword fields of the call.
+    run_generation : int or None
+        Run generation at the emission.
+    date : object
+        Date at the emission.
+    request_ranks : tuple of int or None
+        For an evaluate batch: candidate index -> request rank (trace attribution).
+    wall : float
+        Wall-clock time of the call.
+    """
+
+    origin: Literal["system", "method"]
+    module: str
+    subpop: str | None
+    path: str
+    name: str
+    fields: Mapping[str, object]
+    run_generation: int | None
+    date: object
+    request_ranks: tuple[int, ...] | None
+    wall: float