cgse 2024.7.0__py3-none-any.whl → 2025.0.2__py3-none-any.whl

egse/logger/log_cs.py

@@ -1,321 +0,0 @@
-"""
-The Log Server receives all log messages and events from control servers and client applications
-and saves those messages in a log file at a given location.
-"""
-import datetime
-import logging
-import multiprocessing
-import pickle
-from logging import StreamHandler
-from logging.handlers import SocketHandler
-from logging.handlers import TimedRotatingFileHandler
-from pathlib import Path
-from typing import Optional
-from typing import Union
-
-import click
-import rich
-import zmq
-from prometheus_client import Counter
-from prometheus_client import start_http_server
-
-from egse.env import get_log_file_location
-from egse.settings import Settings
-from egse.system import format_datetime
-from egse.system import replace_environment_variable
-from egse.zmq_ser import bind_address
-from egse.zmq_ser import connect_address
-
-CTRL_SETTINGS = Settings.load("Logging Control Server")
-
-
-# The format for the log file.
-# The line that is saved in the log file shall contain as much information as possible.
-
-LOG_FORMAT_FILE = (
-    "%(asctime)s:%(processName)s:%(process)s:%(levelname)s:%(lineno)d:%(name)s:%(message)s"
-)
-
-LOG_FORMAT_KEY_VALUE = (
-    "level=%(levelname)s ts=%(asctime)s process=%(processName)s process_id=%(process)s caller=%(name)s:%(lineno)s msg=\"%(message)s\""
-)
-
-LOG_FORMAT_DATE = "%Y-%m-%dT%H:%M:%S,%f"
-
-# The format for the console output.
-# The line that is printed on the console shall be concise.
-
-LOG_FORMAT_STREAM = "%(asctime)s:%(levelname)s:%(name)s:%(message)s"
-
-LOG_RECORDS = Counter(
-    "log_records_count", "Count the number of log records processed", ["source", "name"]
-)
-
-LOG_LEVEL_FILE = logging.DEBUG
-LOG_LEVEL_STREAM = logging.ERROR
-LOG_LEVEL_SOCKET = 1  # ALL records shall go to the socket handler
-
-LOGGER_NAME = "egse.logger.log_cs"
-
-file_handler: Optional[TimedRotatingFileHandler] = None
-stream_handler: Optional[StreamHandler] = None
-socket_handler: Optional[SocketHandler] = None
-
-
-class DateTimeFormatter(logging.Formatter):
-
-    def formatTime(self, record, datefmt=None):
-        converted_time = datetime.datetime.fromtimestamp(record.created)
-        if datefmt:
-            return converted_time.strftime(datefmt)
-        formatted_time = converted_time.strftime("%Y-%m-%dT%H:%M:%S")
-        return f"{formatted_time}.{record.msecs:03.0f}"
-
-
-file_formatter = DateTimeFormatter(fmt=LOG_FORMAT_KEY_VALUE, datefmt=LOG_FORMAT_DATE)
-
-
-@click.group()
-def cli():
-    pass
-
-
-@cli.command()
-def start():
-    """Start the Logger Control Server."""
-
-    global file_handler, stream_handler, socket_handler
-
-    multiprocessing.current_process().name = "log_cs"
-
-    start_http_server(CTRL_SETTINGS.METRICS_PORT)
-
-    # TODO (rik):
-    #   The `max_*` values should be read from the Settings (DONE) and it shall be possible to
-    #   overwrite these values from the commandline maybe.
-
-    max_bytes = CTRL_SETTINGS.MAX_SIZE_LOG_FILES * 1024 * 1024
-    max_nr_log_files = CTRL_SETTINGS.MAX_NR_LOG_FILES
-
-    log_file_location = Path(get_log_file_location())
-    if not log_file_location.exists():
-        raise FileNotFoundError(f"The location for the log files doesn't exist: {log_file_location!s}.")
-
-    file_handler = TimedRotatingFileHandler(
-        filename=log_file_location / "general.log", when='midnight'
-    )
-    file_handler.setFormatter(file_formatter)
-
-    # There is no need to set the level for the handlers, because the level is checked by the
-    # Logger and we use the handlers directly here. Use a filter to restrict messages.
-
-    stream_handler = StreamHandler()
-    stream_handler.setFormatter(logging.Formatter(fmt=LOG_FORMAT_STREAM))
-
-    # Log records are also sent to the cutelog listening server
-
-    socket_handler = SocketHandler(CTRL_SETTINGS.CUTELOG_IP_ADDRESS, CTRL_SETTINGS.CUTELOG_LISTENING_PORT)
-    socket_handler.setFormatter(file_formatter)
-
-    context = zmq.Context()
-
-    endpoint = bind_address(CTRL_SETTINGS.PROTOCOL, CTRL_SETTINGS.LOGGING_PORT)
-    receiver = context.socket(zmq.PULL)
-    receiver.bind(endpoint)
-
-    endpoint = bind_address(CTRL_SETTINGS.PROTOCOL, CTRL_SETTINGS.COMMANDING_PORT)
-    commander = context.socket(zmq.REP)
-    commander.bind(endpoint)
-
-    poller = zmq.Poller()
-    poller.register(receiver, zmq.POLLIN)
-    poller.register(commander, zmq.POLLIN)
-
-    while True:
-        try:
-            socks = dict(poller.poll())
-
-            if commander in socks:
-                pickle_string = commander.recv()
-                command = pickle.loads(pickle_string)
-
-                if command.lower() == "quit":
-                    commander.send(pickle.dumps("ACK"))
-                    break
-
-                response = handle_command(command)
-                commander.send(pickle.dumps(response))
-
-            if receiver in socks:
-                pickle_string = receiver.recv()
-                record = pickle.loads(pickle_string)
-                record = logging.makeLogRecord(record)
-
-                handle_log_record(record)
-
-        except KeyboardInterrupt:
-            click.echo("KeyboardInterrupt caught!")
-            break
-
-    record = logging.LogRecord(
-        name=LOGGER_NAME,
-        level=logging.WARNING,
-        pathname=__file__,
-        lineno=137,
-        msg="Logger terminated.",
-        args=(),
-        exc_info=None,
-        func="start",
-        sinfo=None
-    )
-    handle_log_record(record)
-
-    file_handler.close()
-    stream_handler.close()
-    commander.close(linger=0)
-    receiver.close(linger=0)
-
-
-def handle_log_record(record):
-    """Send the log record to the file handler and the stream handler."""
-    global file_handler, stream_handler, socket_handler
-
-    if record.levelno >= LOG_LEVEL_FILE:
-        file_handler.emit(record)
-
-    if record.levelno >= LOG_LEVEL_STREAM:
-        stream_handler.handle(record)
-
-    if record.levelno >= LOG_LEVEL_SOCKET:
-        socket_handler.handle(record)
-
-    LOG_RECORDS.labels(source="all", name="all").inc()
-    LOG_RECORDS.labels(source="logger", name=record.name).inc()
-    LOG_RECORDS.labels(source="process", name=record.processName).inc()
-
-
-def handle_command(command) -> dict:
-    """Handle commands that are sent to the commanding socket."""
-    global file_handler
-    global LOG_LEVEL_FILE
-
-    response = dict(
-        timestamp=format_datetime(),
-    )
-    if command.lower() == 'roll':
-        file_handler.doRollover()
-        response.update(dict(status="ACK"))
-        record = logging.LogRecord(
-            name=LOGGER_NAME,
-            level=logging.WARNING,
-            pathname=__file__,
-            lineno=197,
-            msg="Logger rolled over.",
-            args=(),
-            exc_info=None,
-            func="roll",
-            sinfo=None
-        )
-        handle_log_record(record)
-
-    elif command.lower() == 'status':
-        response.update(dict(
-            status="ACK",
-            file_logger_level=logging.getLevelName(LOG_LEVEL_FILE),
-            stream_logger_level=logging.getLevelName(LOG_LEVEL_STREAM),
-            file_logger_location=file_handler.baseFilename,
-        ))
-    elif command.lower().startswith("set_level"):
-        new_level = command.split()[-1]
-        LOG_LEVEL_FILE = logging._nameToLevel[new_level]
-        response.update(dict(
-            status="ACK",
-            file_logger_level=logging.getLevelName(LOG_LEVEL_FILE),
-        ))
-
-    return response
-
-
-@cli.command()
-def stop():
-    """Stop the Logger Control Server."""
-
-    response = send_request("quit")
-    if response == "ACK":
-        rich.print("Logger successfully terminated.")
-    else:
-        rich.print(f"[red] ERROR: {response}")
-
-
-@cli.command()
-def roll():
-    """Roll over the log file of the Logger Control Server."""
-
-    response = send_request("roll")
-    if response.get("status") == "ACK":
-        rich.print("[green]Logger files successfully rotated.")
-    else:
-        rich.print(f"[red]ERROR: {response}")
-
-
-@cli.command()
-def status():
-    """Roll over the log file of the Logger Control Server."""
-
-    rich.print("Log Manager:")
-
-    response = send_request("status")
-    if response.get("status") == "ACK":
-        rich.print("  Status: [green]active")
-        rich.print(f"  Level [grey50](file)[black]: {response.get('file_logger_level')}")
-        rich.print(f"  Level [grey50](stdout)[black]: {response.get('stream_logger_level')}")
-        rich.print(f"  Log file location: {response.get('file_logger_location')}")
-    else:
-        rich.print("  Status: [red]not active")
-
-
-@cli.command()
-@click.argument('level')
-def level(level: Union[int, str]):
-    """Set the logging level for """
-    try:
-        level = logging.getLevelName(int(level))
-    except ValueError:
-        if level not in logging._nameToLevel:
-            rich.print(f"[red]Invalid logging level given '{level}'.")
-            rich.print(f"Should be one of {logging._levelToName}.")
-            return
-
-    response = send_request(f"set_level {level}")
-    if response.get("status") == "ACK":
-        rich.print(f"Log level on the server is now set to {response.get('file_logger_level')}.")
-    else:
-        rich.print(f"[red]ERROR: {response}")
-
-
-TIMEOUT_RECV = 1.0  # seconds
-
-
-def send_request(command_request: str):
-    """Sends a request to the Logger Control Server and waits for a response."""
-    ctx = zmq.Context().instance()
-    endpoint = connect_address(
-        CTRL_SETTINGS.PROTOCOL, CTRL_SETTINGS.HOSTNAME, CTRL_SETTINGS.COMMANDING_PORT
-    )
-    socket = ctx.socket(zmq.REQ)
-    socket.connect(endpoint)
-
-    socket.send(pickle.dumps(command_request))
-    rlist, _, _ = zmq.select([socket], [], [], timeout=TIMEOUT_RECV)
-    if socket in rlist:
-        response = socket.recv()
-        response = pickle.loads(response)
-    else:
-        response = {"error": "Receive from ZeroMQ socket timed out for Logger Control Server."}
-    socket.close(linger=0)
-
-    return response
-
-
-if __name__ == "__main__":
-    cli()

egse/metrics.py

@@ -1,102 +0,0 @@
-import logging
-from typing import Optional
-
-import numpy as np
-from prometheus_client import Gauge
-
-from egse.hk import TmDictionaryColumns
-from egse.settings import Settings
-from egse.setup import SetupError, load_setup, Setup
-from egse.state import GlobalState
-
-LOGGER = logging.getLogger(__name__)
-SITE_ID = Settings.load("SITE").ID
-
-
-def define_metrics(origin: str, dashboard: str = None, use_site: bool = False, setup: Optional[Setup] = None) -> dict:
-    """
-    Create a metrics dictionary from the TM dictionary.
-
-    Read the metric names and their descriptions from the TM dictionary, and create Prometheus
-    gauges based on this information.
-
-    If `dashboard` is not provided, all telemetry parameters for the given origin will be returned.
-
-    Args:
-        origin: the storage mnemonics for the requested metrics
-        dashboard: restrict the metrics selection to those that are defined for the given dashboard.
-            You can select all dashboards with `dashboard='*'`.
-        use_site: Indicate whether the prefixes of the new HK names are TH-specific
-        setup: Setup.
-    Returns:
-        Dictionary with all Prometheus gauges for the given origin and dashboard.
-    """
-
-    setup = setup or load_setup()
-
-    try:
-        hk_info_table = setup.telemetry.dictionary
-    except AttributeError:
-        raise SetupError("Version of the telemetry dictionary not specified in the current setup")
-
-    hk_info_table = hk_info_table.replace(np.nan, "")
-
-    storage_mnemonic = hk_info_table[TmDictionaryColumns.STORAGE_MNEMONIC].values
-    hk_names = hk_info_table[TmDictionaryColumns.CORRECT_HK_NAMES].values
-    descriptions = hk_info_table[TmDictionaryColumns.DESCRIPTION].values
-    mon_screen = hk_info_table[TmDictionaryColumns.DASHBOARD].values
-
-    condition = storage_mnemonic == origin.upper()
-    if dashboard is not None:
-        if dashboard == "*":
-            extra_condition = mon_screen != ""
-        else:
-            extra_condition = mon_screen == dashboard.upper()
-        condition = np.all((condition, extra_condition), axis=0)
-
-    selection = np.where(condition)
-
-    syn_names = hk_names[selection]
-    descriptions = descriptions[selection]
-
-    if not use_site:
-        return {
-            syn_name: Gauge(syn_name, description)
-            for syn_name, description in zip(syn_names, descriptions)
-        }
-
-    th_prefix = f"G{SITE_ID}_"
-
-    th_syn_names = []
-    th_descriptions = []
-    for syn_name, description in zip(syn_names, descriptions):
-        if syn_name.startswith(th_prefix):
-            th_syn_names.append(syn_name)
-            th_descriptions.append(description)
-
-    return {
-        syn_name: Gauge(syn_name, description)
-        for syn_name, description in zip(th_syn_names, th_descriptions)
-    }
-
-
-def update_metrics(metrics: dict, updates: dict):
-    """
-    Updates the metrics parameters with the values from the updates dictionary.
-    Only the metrics parameters for which the names are keys in the given updates dict are actually updated. Other
-    metrics remain untouched.
-
-    The functions logs a warning when the updates dict contains a name which is not known as a metrics parameter.
-
-    Args:
-        metrics: the metrics dictionary previously defined with the define_metrics function
-        updates: a dictionary with key=metrics name and value is the to-be-updated value
-    """
-    for metric_name, value in updates.items():
-        try:
-            if value is None:
-                metrics[metric_name].set(float('nan'))
-            else:
-                metrics[metric_name].set(float(value))
-        except KeyError as exc:
-            LOGGER.warning(f"Unknown metric name: {metric_name=}")