cgse 2023.38.0__py3-none-any.whl → 2024.1.3__py3-none-any.whl

egse/listener.py

@@ -1,73 +0,0 @@
-import logging
-from enum import IntEnum
-from typing import Any
-
-from egse.decorators import dynamic_interface
-
-LOGGER = logging.getLogger(__name__)
-
-class EVENT_ID(IntEnum):
-    ALL = 0
-    SETUP = 1
-
-class Event:
-    def __init__(self, event_id: int, context: Any):
-        self.id = event_id
-        self.context = context
-    def __repr__(self):
-        return f"Event({self.id}, {self.context})"
-
-
-class EventInterface:
-    @dynamic_interface
-    def handle_event(self, event: Event):
-        ...
-
-class Listeners:
-    def __init__(self):
-        self._listeners: dict[str, dict] = {}
-
-    def __len__(self):
-        return len(self._listeners)
-
-    def add_listener(self, listener: dict):
-        """
-        The argument is a dictionary with the following expected content:
-
-         * name (str): the name of the process
-         * proxy (str): the proxy class
-
-        Args:
-            listener: a dictionary with properties of the listener
-
-        Raises:
-            ValueError if the listener already exists.
-        """
-        try:
-            listener_name = listener["name"]
-        except KeyError as exc:
-            raise ValueError(f"Expected 'name' key in listener argument {listener}.") from exc
-
-        if listener_name in self._listeners:
-            raise ValueError(f"Process {listener_name} is already registered as a listener.")
-
-        self._listeners[listener_name] = listener
-
-    def remove_listener(self, listener: dict):
-        try:
-            listener_name = listener["name"]
-        except KeyError as exc:
-            raise ValueError(f"Expected 'name' key in listener argument {listener}.") from exc
-
-        try:
-            del self._listeners[listener_name]
-        except KeyError as exc:
-            raise ValueError(f"Process {listener_name} cannot be removed, not registered.") from exc
-
-    def notify_listeners(self, event: Event):
-        for name, listener in self._listeners.items():
-            proxy = listener['proxy']
-            LOGGER.info(f"Notifying process {name} of {event.context or {}} on {event.id=}")
-            with proxy() as pobj:
-                rc = pobj.handle_event(event)
-            LOGGER.info(f"{rc=}")

egse/logger/__init__.py

@@ -1,243 +0,0 @@
-"""
-This module defines the level, format and handlers for the root logger and for the special
-'egse' logger. The egse_logger will be configured with a special handler which sends all
-logging messages to a log control server.
-
-This module is loaded whenever an egse module is loaded, to ensure all log messages are properly
-forwarded to the log control server.
-"""
-
-import logging
-import pickle
-import sys
-import traceback
-
-import zmq
-
-LOG_FORMAT_FULL = (
-    "%(asctime)23s:%(processName)20s:%(levelname)8s:%(lineno)5d:%(name)-20s:%(message)s"
-)
-
-# Configure the root logger
-
-logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT_FULL)
-
-__all__ = [
-    "egse_logger",
-    "set_all_logger_levels",
-    "ZeroMQHandler",
-    "close_all_zmq_handlers",
-]
-
-
-class ZeroMQHandler(logging.Handler):
-    def __init__(self, uri=None, socket_type=zmq.PUSH, ctx=None):
-
-        from egse.settings import Settings
-        from egse.zmq_ser import connect_address
-
-        ctrl_settings = Settings.load("Logging Control Server")
-        uri = uri or connect_address(ctrl_settings.PROTOCOL, ctrl_settings.HOSTNAME,
-                                     ctrl_settings.LOGGING_PORT)
-
-        logging.Handler.__init__(self)
-
-        # print(f"ZeroMQHandler.__init__({uri=}, {socket_type=}, {ctx=})")
-
-        self.setLevel(logging.NOTSET)
-
-        self.ctx = ctx or zmq.Context().instance()
-        self.socket = zmq.Socket(self.ctx, socket_type)
-        self.socket.setsockopt(zmq.SNDHWM, 0)  # never block on sending msg
-        self.socket.connect(uri)
-
-    def __del__(self):
-        self.close()
-
-    def close(self):
-        self.socket.close(linger=0)
-
-    def emit(self, record):
-        """
-        Emit a record.
-
-        Writes the LogRecord to the queue, preparing it for pickling first.
-        """
-
-        # print(f"ZeroMQHandler.emit({record})")
-
-        from egse.system import is_in_ipython
-
-        try:
-            if record.exc_info:
-                record.exc_text = traceback.format_exc()
-                record.exc_info = None  # traceback objects can not be pickled
-            if record.processName == "MainProcess" and is_in_ipython():
-                record.processName = "IPython"
-            data = pickle.dumps(record.__dict__)
-            self.socket.send(data, flags=zmq.NOBLOCK)
-        except (KeyboardInterrupt, SystemExit):
-            raise
-        except Exception as exc:
-            print(f"ZeroMQHandler: Exception - {exc}", file=sys.stderr)
-            self.handleError(record)
-
-
-def close_all_zmq_handlers():
-    """
-    Close all the ZeroMQHandlers that are connected to a logger.
-
-    This function is automatically called upon termination of the control servers. For your own
-    applications, call this function before exiting the App.
-    """
-
-    loggers = logging.Logger.manager.loggerDict
-
-    for name, logger in loggers.items():
-        if isinstance(logger, logging.PlaceHolder):
-            continue
-        for handler in logger.handlers:
-            if isinstance(handler, ZeroMQHandler):
-                logger.debug(f"Closing handler for logger {name}")
-                handler.close()
-
-
-# Initialize logging as we want it for the Common-EGSE
-#
-# * The ZeroMQHandler to send all logging messages, i.e. level=DEBUG to the Logging Server
-# * The (local) StreamingHandlers to print only INFO messages and higher
-
-logging.disable(logging.NOTSET)
-root_logger = logging.getLogger()
-
-for handler in root_logger.handlers:
-    handler.setLevel(logging.INFO)
-
-
-# Define the `egse` logger and add the ZeroMQHandler to this logger
-
-egse_logger = logging.getLogger("egse")
-egse_logger.setLevel(logging.DEBUG)
-
-zmq_handler = ZeroMQHandler()
-zmq_handler.setLevel(logging.NOTSET)
-
-egse_logger.addHandler(zmq_handler)
-egse_logger.setLevel(logging.DEBUG)
-
-
-def replace_zmq_handler():
-    """
-    This function will replace the current ZeroMQ Handler with a new instance. Use this function
-    in the run() method of a multiprocessing.Process:
-
-        import egse.logger
-        egse.logger.replace_zmq_handler()
-
-    Don't use this function in the __init__() method as only the run() method will execute in
-    the new Process and replace the handler in the proper environment. The reason for this is
-    that the ZeroMQ socket is not thread/Process safe, so a new ZeroMQ socket needs to be created
-    in the correct process environment.
-    """
-    global egse_logger
-
-    this_handler = None
-    for handler in egse_logger.handlers:
-        if isinstance(handler, ZeroMQHandler):
-            this_handler = handler
-    if this_handler is not None:
-        egse_logger.removeHandler(this_handler)
-    egse_logger.addHandler(ZeroMQHandler())
-
-
-def create_new_zmq_logger(name: str):
-    """
-    Create a new logger with the given name and add a ZeroMQ Handler to this logger.
-
-    If the logger already has a ZeroMQ handler attached, don't add a second ZeroMQ handler,
-    just return the Logger object.
-
-    Args:
-        name: the requested name for the logger
-
-    Returns:
-        A Logger for the given name with a ZeroMQ handler attached.
-    """
-    logger = logging.getLogger(name)
-    logger.setLevel(logging.DEBUG)
-
-    # If the ZeroMQ handler already exists for this logger, don't add a second handler
-
-    for handler in logger.handlers:
-        if isinstance(handler, ZeroMQHandler):
-            return logger
-
-    zmq_handler = ZeroMQHandler()
-    zmq_handler.setLevel(logging.NOTSET)
-
-    logger.addHandler(zmq_handler)
-    logger.setLevel(logging.DEBUG)
-
-    return logger
-
-
-# We define a new level that is lower the DEBUG for extreme verbose messages
-
-logging.FLASH_FLOOD = 5
-logging.addLevelName(logging.FLASH_FLOOD, "FLASH_FLOOD")
-
-
-def flash_flood(self, msg, *args, **kwargs):
-    if self.isEnabledFor(logging.FLASH_FLOOD):
-        self._log(logging.FLASH_FLOOD, msg, args, **kwargs)
-
-
-logging.flash_flood = flash_flood
-logging.Logger.flash_flood = flash_flood
-
-
-def set_all_logger_levels(level: int):
-    global root_logger, egse_logger
-
-    root_logger.level = level
-    egse_logger.level = level
-
-    for handler in root_logger.handlers:
-        handler.setLevel(level)
-
-    # We don't want to restrict egse_logger levels
-
-    # for handler in egse_logger.handlers:
-    #     handler.setLevel(level)
-
-
-if __name__ == "__main__":
-
-    import egse.logger
-
-    LOGGER = logging.getLogger("egse.0mq-log-test")
-    LOGGER.setLevel(logging.FLASH_FLOOD)
-
-    LOGGER.flash_flood("Hello, ZeroMQ logging: This is a FLASH_FLOOD message.")
-    LOGGER.debug("Hello, ZeroMQ logging: This is a DEBUG message.")
-    LOGGER.info("Hello, ZeroMQ logging: This is an INFO message.")
-    LOGGER.warning("Hello, ZeroMQ logging: This is a WARNING message.")
-    LOGGER.error("Hello, ZeroMQ logging: This is an ERROR message.")
-    LOGGER.critical("Hello, ZeroMQ logging: This is a CRITICAL message.")
-    try:
-        raise ValueError("A fake ValueError, raised for testing.")
-    except ValueError:
-        LOGGER.exception("Hello, ZeroMQ logging: This is an EXCEPTION message.")
-
-    LOGGER = logging.getLogger("plain-log-test")
-
-    LOGGER.flash_flood("Vanilla logging: This is a FLASH_FLOOD message.")
-    LOGGER.debug("Vanilla logging: This is a DEBUG message.")
-    LOGGER.info("Vanilla logging: This is an INFO message.")
-    LOGGER.warning("Vanilla logging: This is a WARNING message.")
-    LOGGER.error("Vanilla logging: This is an ERROR message.")
-    LOGGER.critical("Vanilla logging: This is a CRITICAL message.")
-    try:
-        raise ValueError("A fake ValueError, raised for testing.")
-    except ValueError:
-        LOGGER.exception("Vanilla logging: This is an EXCEPTION message.")

egse/logger/log_cs.py

@@ -1,321 +0,0 @@
-"""
-The Log Server receives all log messages and events from control servers and client applications
-and saves those messages in a log file at a given location.
-"""
-import datetime
-import logging
-import multiprocessing
-import pickle
-from logging import StreamHandler
-from logging.handlers import SocketHandler
-from logging.handlers import TimedRotatingFileHandler
-from pathlib import Path
-from typing import Optional
-from typing import Union
-
-import click
-import rich
-import zmq
-from prometheus_client import Counter
-from prometheus_client import start_http_server
-
-from egse.env import get_log_file_location
-from egse.settings import Settings
-from egse.system import format_datetime
-from egse.system import replace_environment_variable
-from egse.zmq_ser import bind_address
-from egse.zmq_ser import connect_address
-
-CTRL_SETTINGS = Settings.load("Logging Control Server")
-
-
-# The format for the log file.
-# The line that is saved in the log file shall contain as much information as possible.
-
-LOG_FORMAT_FILE = (
-    "%(asctime)s:%(processName)s:%(process)s:%(levelname)s:%(lineno)d:%(name)s:%(message)s"
-)
-
-LOG_FORMAT_KEY_VALUE = (
-    "level=%(levelname)s ts=%(asctime)s process=%(processName)s process_id=%(process)s caller=%(name)s:%(lineno)s msg=\"%(message)s\""
-)
-
-LOG_FORMAT_DATE = "%Y-%m-%dT%H:%M:%S,%f"
-
-# The format for the console output.
-# The line that is printed on the console shall be concise.
-
-LOG_FORMAT_STREAM = "%(asctime)s:%(levelname)s:%(name)s:%(message)s"
-
-LOG_RECORDS = Counter(
-    "log_records_count", "Count the number of log records processed", ["source", "name"]
-)
-
-LOG_LEVEL_FILE = logging.DEBUG
-LOG_LEVEL_STREAM = logging.ERROR
-LOG_LEVEL_SOCKET = 1  # ALL records shall go to the socket handler
-
-LOGGER_NAME = "egse.logger.log_cs"
-
-file_handler: Optional[TimedRotatingFileHandler] = None
-stream_handler: Optional[StreamHandler] = None
-socket_handler: Optional[SocketHandler] = None
-
-
-class DateTimeFormatter(logging.Formatter):
-
-    def formatTime(self, record, datefmt=None):
-        converted_time = datetime.datetime.fromtimestamp(record.created)
-        if datefmt:
-            return converted_time.strftime(datefmt)
-        formatted_time = converted_time.strftime("%Y-%m-%dT%H:%M:%S")
-        return f"{formatted_time}.{record.msecs:03.0f}"
-
-
-file_formatter = DateTimeFormatter(fmt=LOG_FORMAT_KEY_VALUE, datefmt=LOG_FORMAT_DATE)
-
-
-@click.group()
-def cli():
-    pass
-
-
-@cli.command()
-def start():
-    """Start the Logger Control Server."""
-
-    global file_handler, stream_handler, socket_handler
-
-    multiprocessing.current_process().name = "log_cs"
-
-    start_http_server(CTRL_SETTINGS.METRICS_PORT)
-
-    # TODO (rik):
-    #   The `max_*` values should be read from the Settings (DONE) and it shall be possible to
-    #   overwrite these values from the commandline maybe.
-
-    max_bytes = CTRL_SETTINGS.MAX_SIZE_LOG_FILES * 1024 * 1024
-    max_nr_log_files = CTRL_SETTINGS.MAX_NR_LOG_FILES
-
-    log_file_location = Path(get_log_file_location())
-    if not log_file_location.exists():
-        raise FileNotFoundError(f"The location for the log files doesn't exist: {log_file_location!s}.")
-
-    file_handler = TimedRotatingFileHandler(
-        filename=log_file_location / "general.log", when='midnight'
-    )
-    file_handler.setFormatter(file_formatter)
-
-    # There is no need to set the level for the handlers, because the level is checked by the
-    # Logger and we use the handlers directly here. Use a filter to restrict messages.
-
-    stream_handler = StreamHandler()
-    stream_handler.setFormatter(logging.Formatter(fmt=LOG_FORMAT_STREAM))
-
-    # Log records are also sent to the cutelog listening server
-
-    socket_handler = SocketHandler(CTRL_SETTINGS.CUTELOG_IP_ADDRESS, CTRL_SETTINGS.CUTELOG_LISTENING_PORT)
-    socket_handler.setFormatter(file_formatter)
-
-    context = zmq.Context()
-
-    endpoint = bind_address(CTRL_SETTINGS.PROTOCOL, CTRL_SETTINGS.LOGGING_PORT)
-    receiver = context.socket(zmq.PULL)
-    receiver.bind(endpoint)
-
-    endpoint = bind_address(CTRL_SETTINGS.PROTOCOL, CTRL_SETTINGS.COMMANDING_PORT)
-    commander = context.socket(zmq.REP)
-    commander.bind(endpoint)
-
-    poller = zmq.Poller()
-    poller.register(receiver, zmq.POLLIN)
-    poller.register(commander, zmq.POLLIN)
-
-    while True:
-        try:
-            socks = dict(poller.poll())
-
-            if commander in socks:
-                pickle_string = commander.recv()
-                command = pickle.loads(pickle_string)
-
-                if command.lower() == "quit":
-                    commander.send(pickle.dumps("ACK"))
-                    break
-
-                response = handle_command(command)
-                commander.send(pickle.dumps(response))
-
-            if receiver in socks:
-                pickle_string = receiver.recv()
-                record = pickle.loads(pickle_string)
-                record = logging.makeLogRecord(record)
-
-                handle_log_record(record)
-
-        except KeyboardInterrupt:
-            click.echo("KeyboardInterrupt caught!")
-            break
-
-    record = logging.LogRecord(
-        name=LOGGER_NAME,
-        level=logging.WARNING,
-        pathname=__file__,
-        lineno=137,
-        msg="Logger terminated.",
-        args=(),
-        exc_info=None,
-        func="start",
-        sinfo=None
-    )
-    handle_log_record(record)
-
-    file_handler.close()
-    stream_handler.close()
-    commander.close(linger=0)
-    receiver.close(linger=0)
-
-
-def handle_log_record(record):
-    """Send the log record to the file handler and the stream handler."""
-    global file_handler, stream_handler, socket_handler
-
-    if record.levelno >= LOG_LEVEL_FILE:
-        file_handler.emit(record)
-
-    if record.levelno >= LOG_LEVEL_STREAM:
-        stream_handler.handle(record)
-
-    if record.levelno >= LOG_LEVEL_SOCKET:
-        socket_handler.handle(record)
-
-    LOG_RECORDS.labels(source="all", name="all").inc()
-    LOG_RECORDS.labels(source="logger", name=record.name).inc()
-    LOG_RECORDS.labels(source="process", name=record.processName).inc()
-
-
-def handle_command(command) -> dict:
-    """Handle commands that are sent to the commanding socket."""
-    global file_handler
-    global LOG_LEVEL_FILE
-
-    response = dict(
-        timestamp=format_datetime(),
-    )
-    if command.lower() == 'roll':
-        file_handler.doRollover()
-        response.update(dict(status="ACK"))
-        record = logging.LogRecord(
-            name=LOGGER_NAME,
-            level=logging.WARNING,
-            pathname=__file__,
-            lineno=197,
-            msg="Logger rolled over.",
-            args=(),
-            exc_info=None,
-            func="roll",
-            sinfo=None
-        )
-        handle_log_record(record)
-
-    elif command.lower() == 'status':
-        response.update(dict(
-            status="ACK",
-            file_logger_level=logging.getLevelName(LOG_LEVEL_FILE),
-            stream_logger_level=logging.getLevelName(LOG_LEVEL_STREAM),
-            file_logger_location=file_handler.baseFilename,
-        ))
-    elif command.lower().startswith("set_level"):
-        new_level = command.split()[-1]
-        LOG_LEVEL_FILE = logging._nameToLevel[new_level]
-        response.update(dict(
-            status="ACK",
-            file_logger_level=logging.getLevelName(LOG_LEVEL_FILE),
-        ))
-
-    return response
-
-
-@cli.command()
-def stop():
-    """Stop the Logger Control Server."""
-
-    response = send_request("quit")
-    if response == "ACK":
-        rich.print("Logger successfully terminated.")
-    else:
-        rich.print(f"[red] ERROR: {response}")
-
-
-@cli.command()
-def roll():
-    """Roll over the log file of the Logger Control Server."""
-
-    response = send_request("roll")
-    if response.get("status") == "ACK":
-        rich.print("[green]Logger files successfully rotated.")
-    else:
-        rich.print(f"[red]ERROR: {response}")
-
-
-@cli.command()
-def status():
-    """Roll over the log file of the Logger Control Server."""
-
-    rich.print("Log Manager:")
-
-    response = send_request("status")
-    if response.get("status") == "ACK":
-        rich.print("  Status: [green]active")
-        rich.print(f"  Level [grey50](file)[black]: {response.get('file_logger_level')}")
-        rich.print(f"  Level [grey50](stdout)[black]: {response.get('stream_logger_level')}")
-        rich.print(f"  Log file location: {response.get('file_logger_location')}")
-    else:
-        rich.print("  Status: [red]not active")
-
-
-@cli.command()
-@click.argument('level')
-def level(level: Union[int, str]):
-    """Set the logging level for """
-    try:
-        level = logging.getLevelName(int(level))
-    except ValueError:
-        if level not in logging._nameToLevel:
-            rich.print(f"[red]Invalid logging level given '{level}'.")
-            rich.print(f"Should be one of {logging._levelToName}.")
-            return
-
-    response = send_request(f"set_level {level}")
-    if response.get("status") == "ACK":
-        rich.print(f"Log level on the server is now set to {response.get('file_logger_level')}.")
-    else:
-        rich.print(f"[red]ERROR: {response}")
-
-
-TIMEOUT_RECV = 1.0  # seconds
-
-
-def send_request(command_request: str):
-    """Sends a request to the Logger Control Server and waits for a response."""
-    ctx = zmq.Context().instance()
-    endpoint = connect_address(
-        CTRL_SETTINGS.PROTOCOL, CTRL_SETTINGS.HOSTNAME, CTRL_SETTINGS.COMMANDING_PORT
-    )
-    socket = ctx.socket(zmq.REQ)
-    socket.connect(endpoint)
-
-    socket.send(pickle.dumps(command_request))
-    rlist, _, _ = zmq.select([socket], [], [], timeout=TIMEOUT_RECV)
-    if socket in rlist:
-        response = socket.recv()
-        response = pickle.loads(response)
-    else:
-        response = {"error": "Receive from ZeroMQ socket timed out for Logger Control Server."}
-    socket.close(linger=0)
-
-    return response
-
-
-if __name__ == "__main__":
-    cli()