aetherscan 1.0.0__py3-none-any.whl

aetherscan/logger/logger.py

@@ -0,0 +1,467 @@
+# TODO: archive old logs
+"""
+Logger for Aetherscan Pipeline
+Runs as background thread & uses thread-safe queue-based logging to avoid deadlocks and corrupted
+outputs from concurrent writes (e.g. from worker processes)
+"""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import logging
+import os
+import sys
+import threading
+from logging.handlers import QueueHandler, QueueListener
+from multiprocessing import Queue
+
+from aetherscan.config import get_config
+
+# NOTE: can this just be from aetherscan.logger import SlackHandler?
+from aetherscan.logger.slack_handler import SlackHandler
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_level(level_str: str) -> int:
+    """Map a level name ('INFO', 'WARNING', 'DEBUG', etc.) to its logging constant; defaults to
+    logging.INFO for unknown values. Case-insensitive."""
+    level_map = {
+        "DEBUG": logging.DEBUG,
+        "INFO": logging.INFO,
+        "WARNING": logging.WARNING,
+        "WARN": logging.WARNING,
+        "ERROR": logging.ERROR,
+        "CRITICAL": logging.CRITICAL,
+    }
+    return level_map.get(level_str.upper(), logging.INFO)
+
+
+class ShutdownSafeQueueHandler(QueueHandler):
+    """
+    QueueHandler that silently drops records once its multiprocessing queue is closed (#281).
+
+    After Logger.stop() disposes the queue, a vanilla emit() raises ValueError("Queue ... is
+    closed"); logging's handleError() then prints the traceback to sys.stderr — and if stderr
+    is (or becomes) a StreamToLogger redirect, that print logs again, raises again, and
+    recurses without bound (the observed RecursionError + "lost sys.stderr" at teardown).
+    Swallowing the closed-queue error here makes that loop structurally impossible no matter
+    what the streams point at; late shutdown-time records are dropped, which is the only
+    sane destination for them anyway.
+    """
+
+    def enqueue(self, record):
+        # Queue closed (or its pipe broken) during teardown — drop the record
+        with contextlib.suppress(ValueError, OSError):
+            super().enqueue(record)
+
+
+def log_path_for_tag(output_path: str, tag: str) -> str:
+    """Return this run's log-file path: ``{output_path}/logs/aetherscan_{tag}.log``.
+
+    Pure and dependency-free so the tag->path derivation is unit-testable on its own. Naming the
+    log file with the run's save_tag mirrors how every other artifact is already tag-scoped
+    (config_{tag}.json, model files, plots, DB rows), so each run gets its own log instead of
+    overwriting a single shared aetherscan.log.
+    """
+    return os.path.join(output_path, "logs", f"aetherscan_{tag}.log")
+
+
+class StreamToLogger:
+    """Redirect stream (stdout/stderr) to main logging system"""
+
+    def __init__(self, logger, level):
+        self.logger = logger
+        self.level = level
+        self.linebuf = ""
+        # Re-entrancy guard (#281): if anything downstream of logger.log ever writes back to
+        # this stream (e.g. logging's handleError printing to a redirected stderr), the
+        # second entry is dropped instead of recursing without bound. Thread-local so
+        # concurrent writers on other threads are unaffected.
+        self._reentrancy = threading.local()
+
+    def write(self, buf):
+        if getattr(self._reentrancy, "writing", False):
+            return
+        self._reentrancy.writing = True
+        try:
+            for line in buf.rstrip().splitlines():
+                self.logger.log(self.level, line.rstrip())
+        finally:
+            self._reentrancy.writing = False
+
+    def flush(self):
+        # Flush any remaining content in linebuf if needed
+        if self.linebuf:
+            self.logger.log(self.level, self.linebuf.rstrip())
+            self.linebuf = ""
+
+        # Flush all handlers attached to the logger
+        for handler in self.logger.handlers:
+            handler.flush()
+
+    # File-protocol probes: libraries that write to sys.stdout/sys.stderr (tqdm progress
+    # bars via huggingface_hub, click, rich, ...) probe the stream for interactivity and
+    # capabilities before writing. Without these, any such probe crashes with
+    # AttributeError (e.g. hf_upload failed with "'StreamToLogger' object has no
+    # attribute 'isatty'"), so answer like the non-interactive pipe this stream is.
+
+    def isatty(self) -> bool:
+        """Never a terminal — suppresses interactive control codes/progress rendering."""
+        return False
+
+    def readable(self) -> bool:
+        return False
+
+    def writable(self) -> bool:
+        return True
+
+    def fileno(self):
+        """No OS-level file descriptor backs this stream. io.UnsupportedOperation is the
+        io-module convention (what io.StringIO raises); it subclasses both OSError and
+        ValueError, which is what fileno() probers guard against."""
+        raise io.UnsupportedOperation("fileno")
+
+
+class Logger:
+    """
+    Thread- and process-safe logging singleton.
+
+    The main process runs a QueueListener on a background thread; both the main process and any
+    multiprocessing workers push records onto the same shared queue, and the listener drains it
+    and writes to file/console/Slack. Funneling concurrent writes through one consumer eliminates
+    interleaved-output and lock-contention issues from naive multi-process logging.
+    """
+
+    _instance = None  # Stores singleton instance
+    _lock = threading.Lock()  # Ensures thread safety on object initialization
+
+    # __new__ allocates the object in memory (constructor at the object-creation level)
+    # __init__ initializes the object's attributes after it's created
+    # since __new__ is called before __init__ every time we instantiate a class,
+    # by overriding __new__, we can short-circuit object creation entirely, and control whether a
+    # new instance is created, or just return the existing instance
+    def __new__(cls, *args, **kwargs):
+        # *args/**kwargs are accepted (and ignored here) so the constructor can forward
+        # __init__ arguments like save_tag; object.__new__ takes no extra args itself.
+        # Double-checked locking pattern:
+        # First check if _instance is None, without lock (for performance)
+        if cls._instance is None:
+            # If None, acquire the lock to serialize the initialization path,
+            # preventing race conditions (2 threads violating singleton semantics)
+            with cls._lock:
+                # Check if _instance is None again inside the lock
+                # (since multiple threads can be calling simultaneously)
+                if cls._instance is None:
+                    # If still None, only then we construct the singleton instance
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialized = False  # Mark as not initialized (for __init__)
+        # Return the same instance for all subsequent constructor calls
+        return cls._instance
+
+    def __init__(self, save_tag: str | None = None):
+        """Initialize logger.
+
+        `save_tag` names this run's log file (aetherscan_{save_tag}.log). Callers pass the run's
+        resolved {command}_{datetime} tag (main() resolves it via cli.resolve_save_tag before
+        calling init_logger); when omitted, it falls back to config.checkpoint.save_tag directly.
+        """
+        # Note, __init__ is triggered every time the class's constructor is called,
+        # even if __new__ returned the existing singleton instance
+        # Hence, we use the _initialized flag to make sure __init__ only runs once
+        if self._initialized:
+            return
+
+        self._initialized = True
+
+        self.config = get_config()
+        if self.config is None:
+            raise ValueError("get_config() returned None")
+
+        # Per-run, tag-scoped log file: each run writes aetherscan_{tag}.log, so a new run no
+        # longer overwrites the previous run's log. main.py resolves the run's {command}_{datetime}
+        # tag onto config.checkpoint.save_tag before calling init_logger and passes it in here; the
+        # config value is the fallback for any other caller.
+        tag = save_tag if save_tag is not None else self.config.checkpoint.save_tag
+        self.log_path = log_path_for_tag(self.config.output_path, tag)
+        os.makedirs(os.path.dirname(self.log_path), exist_ok=True)  # Create dir if it doesn't exist
+
+        # Create queue for worker processes (no size limit)
+        self.log_queue = Queue(-1)
+
+        # Setup root logger - set to DEBUG to let handlers filter
+        root_logger = logging.getLogger()
+        root_logger.setLevel(logging.DEBUG)
+        root_logger.handlers.clear()  # Clear existing handlers
+
+        # Create formatter
+        formatter = logging.Formatter("%(asctime)s | %(name)s | %(levelname)s | %(message)s")
+
+        # Setup file handler (only used by main process via listener)
+        file_handler = logging.FileHandler(self.log_path, mode="w")
+        file_handler.setLevel(_parse_level(self.config.logger.file_level))
+        file_handler.setFormatter(formatter)
+
+        # Setup stream handler (only used by main process via listener)
+        stream_handler = logging.StreamHandler(sys.stdout)
+        stream_handler.setLevel(_parse_level(self.config.logger.console_level))
+        stream_handler.setFormatter(formatter)
+
+        # Build handler list for QueueListener
+        handlers: list[logging.Handler] = [file_handler, stream_handler]
+
+        # Initialize Slack handler if enabled
+        self.slack_handler: SlackHandler | None = None
+        slack_init_message: str | None = None
+        slack_init_level: int = logging.INFO
+
+        if self.config.logger.slack_enabled:
+            slack_token = os.environ.get("SLACK_BOT_TOKEN")
+            slack_channel = os.environ.get("SLACK_CHANNEL", self.config.logger.slack_channel)
+
+            # Defer logging until after the queue infrastructure is ready
+            if not slack_token:
+                slack_init_message = (
+                    "Slack logging enabled but SLACK_BOT_TOKEN not found in environment. "
+                    "Slack logging disabled."
+                )
+                slack_init_level = logging.WARNING
+            elif not slack_channel:
+                slack_init_message = (
+                    "Slack logging enabled but no channel configured. "
+                    "Set SLACK_CHANNEL env var or config.logger.slack_channel. Slack logging disabled."
+                )
+                slack_init_level = logging.WARNING
+            else:
+                try:
+                    self.slack_handler = SlackHandler(
+                        token=slack_token,
+                        channel=slack_channel,
+                        username=self.config.logger.slack_username,
+                        timeout=self.config.logger.slack_timeout,
+                        retry_attempts=self.config.logger.slack_retry_attempts,
+                        buffer_size=self.config.logger.slack_buffer_size,
+                        flush_interval=self.config.logger.slack_flush_interval,
+                        broadcast_level=_parse_level(self.config.logger.slack_broadcast_level),
+                    )
+                    self.slack_handler.setLevel(_parse_level(self.config.logger.slack_level))
+                    slack_formatter = logging.Formatter("%(message)s")
+                    self.slack_handler.setFormatter(slack_formatter)
+                    handlers.append(self.slack_handler)
+                    slack_init_message = f"Slack handler initialized for channel: {slack_channel}"
+                    slack_init_level = logging.INFO
+                except Exception as e:
+                    slack_init_message = f"Failed to initialize Slack handler: {e}"
+                    slack_init_level = logging.WARNING
+
+        # Create queue listener - runs in background thread, writes logs from queue
+        self.log_listener = QueueListener(self.log_queue, *handlers, respect_handler_level=True)
+        self.log_listener.start()
+
+        # Add queue handler to root logger (both main and workers use this); shutdown-safe
+        # so post-stop() records are dropped instead of raising through handleError (#281)
+        queue_handler = ShutdownSafeQueueHandler(self.log_queue)
+        root_logger.addHandler(queue_handler)
+
+        # Start the Slack run thread FIRST (posts run summary, all logs become replies)
+        # This must happen before any log messages so they appear in the thread
+        if self.slack_handler is not None:
+            self.slack_handler.start_run()
+
+        # Now that logging infrastructure is ready, log Slack initialization status
+        if slack_init_message:
+            logger.log(slack_init_level, slack_init_message)
+
+        # Redirect TensorFlow logs to Python logging. Lazy import: TF is only needed for this
+        # one call, and log_path_for_tag (and this class's non-TF behavior) stay importable
+        # without pulling in the full TF stack — e.g. from a fast, TF-free test/tooling context.
+        import tensorflow as tf  # noqa: PLC0415
+
+        os.environ["TF_CPP_MIN_LOG_LEVEL"] = "0"  # Show all TF logs
+        tf.get_logger().setLevel(logging.INFO)
+        tf_logger = tf.get_logger()
+        tf_logger.handlers = []  # Remove TF's default handlers
+        tf_logger.propagate = True  # Use root logger handlers
+
+        # Capture Python warnings module output
+        logging.captureWarnings(True)
+        warnings_logger = logging.getLogger("py.warnings")
+        warnings_logger.setLevel(logging.WARNING)
+
+        # Redirect stdout and stderr to logging
+        # This captures print statements and C library output
+        # Note that workers will reset these with init_worker_logging to avoid inheritance issues
+        sys.stdout = StreamToLogger(logging.getLogger("STDOUT"), logging.INFO)
+        sys.stderr = StreamToLogger(logging.getLogger("STDERR"), logging.ERROR)
+
+        logger.info(f"Logger initialized at: {self.log_path}")
+
+    @classmethod
+    def _reset(cls):
+        """
+        Teardown hook for the thread-safe singleton — discards the cached instance so the next
+        constructor call yields a fresh one. Only safe to call after stop() has completed;
+        calling it while the listener thread is still active leaves callers holding a stale
+        reference.
+        """
+        # Acquire lock to prevent race conditions
+        with cls._lock:
+            # Discard the singleton instance by removing the global reference
+            # Guarantees the next constructor call will produce a fresh instance
+            # Note, resources held by the old instance will remain alive unless explicitly closed beforehand
+            cls._instance = None
+            # Note, can't log here after tear down
+
+    def stop(self):
+        """
+        Stop the queue listener thread and flush Slack messages.
+
+        Idempotent (#281): a second call is a no-op — QueueListener.stop() raises if called
+        twice, and cleanup paths can legitimately overlap (conftest teardown + explicit
+        stops). Restores sys.stdout/sys.stderr BEFORE the listener/queue teardown, so any
+        output produced during or after shutdown (including logging's own handleError)
+        reaches the real console instead of recursing through the redirect — the
+        RecursionError / "lost sys.stderr" loop this method used to leave armed.
+        """
+        if getattr(self, "_stopped", False):
+            return
+        self._stopped = True
+
+        # Root fix (#281): put the real streams back first — same idiom the worker path
+        # already uses in init_worker_logging. Guarded so a foreign redirect installed on
+        # top of ours (e.g. pytest's capture) is left alone.
+        if isinstance(sys.stdout, StreamToLogger):
+            sys.stdout = sys.__stdout__
+        if isinstance(sys.stderr, StreamToLogger):
+            sys.stderr = sys.__stderr__
+
+        # Flush and close Slack handler first (while listener is still running)
+        if self.slack_handler is not None:
+            self.slack_handler.close()
+
+        if self.log_listener is not None:
+            self.log_listener.stop()
+            # Note, can't log after this point -- listener thread has stopped
+            # All subsequent logs will get queued but never logged
+
+        # Dispose of the multiprocessing.Queue's feeder thread. Once the listener (the
+        # consumer) is stopped, a mp.Queue with buffered records left its feeder thread
+        # blocked in _feed/_send_bytes on a pipe nobody drains; its exit-time join finalizer
+        # then hangs the interpreter (surfaces when a real Logger is built in-process, e.g.
+        # the unit tests). close() + cancel_join_thread() drops those undelivered
+        # shutdown-time records and lets the process exit cleanly.
+        if self.log_queue is not None:
+            with contextlib.suppress(Exception):
+                self.log_queue.close()
+                self.log_queue.cancel_join_thread()
+
+    # NOTE:
+    # currently only title is broadcasted to main channel,
+    # since there's a delay in the title vs image appearing in the thread
+    # not sure if this is a fundamental limitation to slack webhooks?
+    # maybe instead of forcing image to be kept in thread AND broadcast to main,
+    # we simply broadcast it to main and forget about keeping it in thread
+    # logs can always have text references to where images are saved
+    # but the main logs of interest in main are errors & results
+    def upload_image_to_slack(
+        self,
+        file_path: str,
+        channels: str | list[str] | None = None,
+        title: str | None = None,
+        message: str | None = None,
+        broadcast: bool = True,
+    ) -> bool:
+        """
+        Upload an image to Slack and return True on success, False otherwise.
+
+        The image goes into the current run's thread. With broadcast=True (default) a link-back
+        message is also posted to the main channel — equivalent to ticking the "Also send to
+        channel" checkbox in Slack. `channels` overrides the handler's configured channel; if
+        None, the configured one is used.
+        """
+        if self.slack_handler is None:
+            logger.debug("Slack handler not initialized, skipping image upload")
+            return False
+
+        return self.slack_handler.upload_file(
+            file_path=file_path,
+            channels=channels,
+            title=title,
+            initial_comment=message,
+            broadcast=broadcast,
+        )
+
+
+def init_logger(save_tag: str | None = None) -> Logger:
+    """
+    Initialize global logger instance (call once at startup).
+
+    `save_tag` names this run's log file (aetherscan_{save_tag}.log). Pass the run's resolved
+    {command}_{datetime} tag (main() resolves it via cli.resolve_save_tag beforehand); when
+    omitted, the Logger falls back to config.checkpoint.save_tag.
+    """
+    logger_instance = Logger(save_tag=save_tag)
+
+    # Note, unlike other modules, due to dependency chains,
+    # we wait to call register_logger() inside main.py:main()
+
+    return logger_instance
+
+
+def init_worker_logging(log_queue=None):
+    """
+    Initialize logging in a multiprocessing worker: reset stdout/stderr to the original streams
+    (so the inherited StreamToLogger from the parent doesn't fire), then attach a QueueHandler
+    pointing at the main process's shared log queue.
+
+    `log_queue` defaults to the Logger singleton's queue, which fork-started workers inherit.
+    Spawn-started processes (e.g. the RoundDataProducer and its pool workers) run a fresh
+    interpreter where the singleton doesn't exist — they must pass the queue explicitly
+    (a multiprocessing.Queue survives the spawn pickling boundary). Without either, falls back
+    to a NullHandler to avoid noise.
+    """
+    if log_queue is None:
+        logger_instance = Logger._instance
+
+        if logger_instance is None:
+            logger.warning(
+                "No logger instance initialized - disabling worker logging to avoid conflicts"
+            )
+            logging.getLogger().handlers.clear()
+            logging.getLogger().addHandler(logging.NullHandler())
+            return
+
+        log_queue = logger_instance.log_queue
+
+    # Reset stdout/stderr to avoid inherited StreamToLogger from parent
+    sys.stdout = sys.__stdout__
+    sys.stderr = sys.__stderr__
+
+    # Configure process-local logging to use queue
+    root_logger = logging.getLogger()
+    root_logger.handlers.clear()
+    root_logger.addHandler(ShutdownSafeQueueHandler(log_queue))
+    root_logger.setLevel(logging.INFO)
+
+
+def get_logger() -> Logger | None:
+    """Get the global logger instance"""
+    logger_instance = Logger._instance
+
+    if logger_instance is None:
+        logger.warning("No logger instance initialized")
+
+    return logger_instance
+
+
+def shutdown_logger():
+    """Shutdown the global logger instance (call on exit)"""
+    logger_instance = Logger._instance
+
+    if logger_instance is None:
+        logger.warning("No logger instance initialized")
+        return
+
+    logger_instance.stop()
+    Logger._reset()