opengris-scaler 1.12.7__cp313-cp313-manylinux_2_28_x86_64.whl

scaler/entry_points/top.py

@@ -0,0 +1,286 @@
+import argparse
+import curses
+import functools
+from typing import Dict, List, Literal, Union
+
+from scaler.config.loader import load_config
+from scaler.config.section.top import TopConfig
+from scaler.io.sync_subscriber import ZMQSyncSubscriber
+from scaler.protocol.python.message import StateScheduler
+from scaler.protocol.python.mixins import Message
+from scaler.utility.formatter import (
+    format_bytes,
+    format_integer,
+    format_microseconds,
+    format_percentage,
+    format_seconds,
+)
+
+SORT_BY_OPTIONS = {
+    ord("g"): "group",
+    ord("n"): "worker",
+    ord("C"): "agt_cpu",
+    ord("M"): "agt_rss",
+    ord("c"): "cpu",
+    ord("m"): "rss",
+    ord("F"): "rss_free",
+    ord("f"): "free",
+    ord("w"): "sent",
+    ord("d"): "queued",
+    ord("s"): "suspended",
+    ord("l"): "lag",
+}
+
+SORT_BY_STATE: Dict[str, Union[str, bool]] = {"sort_by": "cpu", "sort_by_previous": "cpu", "sort_reverse": True}
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        "monitor scheduler as top like", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument("--config", "-c", type=str, default=None, help="Path to the TOML configuration file.")
+    parser.add_argument("--timeout", "-t", type=int, default=5, help="timeout seconds")
+    parser.add_argument("monitor_address", nargs="?", type=str, help="scheduler monitor address to connect to")
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    top_config = load_config(TopConfig, args.config, args, section_name="top")
+    curses.wrapper(poke, top_config)
+
+
+def poke(screen, config: TopConfig):
+    screen.nodelay(1)
+
+    try:
+        subscriber = ZMQSyncSubscriber(
+            address=config.monitor_address,
+            callback=functools.partial(show_status, screen=screen),
+            topic=b"",
+            daemonic=False,
+            timeout_seconds=config.timeout,
+        )
+        subscriber.run()
+    except KeyboardInterrupt:
+        pass
+
+
+def show_status(status: Message, screen):
+    if not isinstance(status, StateScheduler):
+        return
+
+    __change_option_state(screen.getch())
+
+    scheduler_table = __generate_keyword_data(
+        "scheduler",
+        {
+            "cpu": format_percentage(status.scheduler.cpu),
+            "rss": format_bytes(status.scheduler.rss),
+            "rss_free": format_bytes(status.rss_free),
+        },
+    )
+
+    task_manager_table = __generate_keyword_data(
+        "task_manager",
+        dict(sorted((k.name, v) for k, v in status.task_manager.state_to_count.items())),
+        format_integer_flag=True,
+    )
+    object_manager = __generate_keyword_data("object_manager", {"num_of_objs": status.object_manager.number_of_objects})
+    sent_table = __generate_keyword_data("scheduler_sent", status.binder.sent, format_integer_flag=True)
+    received_table = __generate_keyword_data("scheduler_received", status.binder.received, format_integer_flag=True)
+    client_table = __generate_keyword_data(
+        "client_manager", status.client_manager.client_to_num_of_tasks, key_col_length=18
+    )
+
+    worker_group_map = {}
+    if status.scaling_manager.worker_groups:
+        for worker_group_id, worker_ids in status.scaling_manager.worker_groups.items():
+            worker_group_id_str = worker_group_id.decode()
+            for worker_id in worker_ids:
+                worker_group_map[worker_id.decode()] = worker_group_id_str
+
+    # Include 'group' as the first column for each worker; empty if not found
+    worker_manager_table = __generate_worker_manager_table(
+        [
+            {
+                "group": worker_group_map.get(worker.worker_id.decode(), ""),
+                "worker": worker.worker_id.decode(),
+                "agt_cpu": worker.agent.cpu,
+                "agt_rss": worker.agent.rss,
+                "cpu": sum(p.resource.cpu for p in worker.processor_statuses),
+                "rss": sum(p.resource.rss for p in worker.processor_statuses),
+                "os_rss_free": worker.rss_free,
+                "free": worker.free,
+                "sent": worker.sent,
+                "queued": worker.queued,
+                "suspended": worker.suspended,
+                "lag": worker.lag_us,
+                "last": worker.last_s,
+                "ITL": worker.itl,
+            }
+            for worker in status.worker_manager.workers
+        ],
+        worker_group_length=10,
+        worker_length=20,
+    )
+
+    table1 = __merge_tables(scheduler_table, object_manager, padding="|")
+    table1 = __merge_tables(table1, task_manager_table, padding="|")
+    table1 = __merge_tables(table1, sent_table, padding="|")
+    table1 = __merge_tables(table1, received_table, padding="|")
+
+    table3 = __merge_tables(worker_manager_table, client_table, padding="|")
+
+    screen.clear()
+    try:
+        new_row, max_cols = __print_table(screen, 0, table1, padding=1)
+    except curses.error:
+        __print_too_small(screen)
+        return
+
+    try:
+        screen.addstr(new_row, 0, "-" * max_cols)
+        screen.addstr(new_row + 1, 0, "Shortcuts: " + " ".join([f"{v}[{chr(k)}]" for k, v in SORT_BY_OPTIONS.items()]))
+        screen.addstr(
+            new_row + 3,
+            0,
+            f"Total {len(status.scaling_manager.worker_groups)} worker group(s) "
+            f"with {len(status.worker_manager.workers)} worker(s)",
+        )
+        _ = __print_table(screen, new_row + 4, table3)
+    except curses.error:
+        pass
+
+    screen.refresh()
+
+
+def __generate_keyword_data(title, data, key_col_length: int = 0, format_integer_flag: bool = False):
+    table = [[title, ""]]
+
+    def format_integer_func(value):
+        if format_integer_flag:
+            return format_integer(value)
+
+        return value
+
+    table.extend([[__truncate(k, key_col_length), format_integer_func(v)] for k, v in data.items()])
+    return table
+
+
+def __generate_worker_manager_table(
+    wm_data: List[Dict], worker_group_length: int, worker_length: int
+) -> List[List[str]]:
+    if not wm_data:
+        headers = [["No workers"]]
+        return headers
+
+    wm_data = sorted(
+        wm_data, key=lambda item: item[SORT_BY_STATE["sort_by"]], reverse=bool(SORT_BY_STATE["sort_reverse"])
+    )
+
+    for row in wm_data:
+        row["group"] = __truncate(row["group"], worker_group_length, how="left")
+        row["worker"] = __truncate(row["worker"], worker_length, how="left")
+        row["agt_cpu"] = format_percentage(row["agt_cpu"])
+        row["agt_rss"] = format_bytes(row["agt_rss"])
+        row["cpu"] = format_percentage(row["cpu"])
+        row["rss"] = format_bytes(row["rss"])
+        row["os_rss_free"] = format_bytes(row["os_rss_free"])
+
+        last = row.pop("last")
+        last = f"({format_seconds(last)}) " if last > 5 else ""
+        row["lag"] = last + format_microseconds(row["lag"])
+
+    worker_manager_table = [[f"[{v}]" if v == SORT_BY_STATE["sort_by"] else v for v in wm_data[0].keys()]]
+    worker_manager_table.extend([list(worker.values()) for worker in wm_data])
+    return worker_manager_table
+
+
+def __print_table(screen, line_number, data, padding: int = 1):
+    if not data:
+        return
+
+    col_widths = [max(len(str(row[i])) for row in data) for i in range(len(data[0]))]
+
+    for i, header in enumerate(data[0]):
+        screen.addstr(line_number, sum(col_widths[:i]) + (padding * i), str(header).rjust(col_widths[i]))
+
+    for i, row in enumerate(data[1:], start=1):
+        for j, cell in enumerate(row):
+            screen.addstr(line_number + i, sum(col_widths[:j]) + (padding * j), str(cell).rjust(col_widths[j]))
+
+    return line_number + len(data), sum(col_widths) + (padding * len(col_widths))
+
+
+def __merge_tables(left: List[List], right: List[List], padding: str = "") -> List[List]:
+    if not left:
+        return right
+
+    if not right:
+        return left
+
+    result = []
+    for i in range(max(len(left), len(right))):
+        if i < len(left):
+            left_row = left[i]
+        else:
+            left_row = [""] * len(left[0])
+
+        if i < len(right):
+            right_row = right[i]
+        else:
+            right_row = [""] * len(right[0])
+
+        if padding:
+            padding_column = [padding]
+            result.append(left_row + padding_column + right_row)
+        else:
+            result.append(left_row + right_row)
+
+    return result
+
+
+def __concat_tables(up: List[List], down: List[List], padding: int = 1) -> List[List]:
+    max_cols = max([len(row) for row in up] + [len(row) for row in down])
+    for row in up:
+        row.extend([""] * (max_cols - len(row)))
+
+    padding_rows = [[""] * max_cols] * padding
+
+    for row in down:
+        row.extend([""] * (max_cols - len(row)))
+
+    return up + padding_rows + down
+
+
+def __truncate(string: str, number: int, how: Literal["left", "right"] = "left") -> str:
+    if number <= 0:
+        return string
+
+    if len(string) <= number:
+        return string
+
+    if how == "left":
+        return f"{string[:number]}+"
+    else:
+        return f"+{string[-number:]}"
+
+
+def __print_too_small(screen):
+    screen.clear()
+    screen.addstr(0, 0, "Your terminal is too small to show")
+    screen.refresh()
+
+
+def __change_option_state(option: int):
+    if option not in SORT_BY_OPTIONS.keys():
+        return
+
+    SORT_BY_STATE["sort_by_previous"] = SORT_BY_STATE["sort_by"]
+    SORT_BY_STATE["sort_by"] = SORT_BY_OPTIONS[option]
+    if SORT_BY_STATE["sort_by"] != SORT_BY_STATE["sort_by_previous"]:
+        SORT_BY_STATE["sort_reverse"] = True
+        return
+
+    SORT_BY_STATE["sort_reverse"] = not SORT_BY_STATE["sort_reverse"]

scaler/entry_points/webui.py

@@ -0,0 +1,26 @@
+import argparse
+
+from scaler.config.loader import load_config
+from scaler.config.section.webui import WebUIConfig
+from scaler.ui.webui import start_webui
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        "web ui for scaler monitoring", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument("--config", "-c", type=str, default=None, help="Path to the TOML configuration file.")
+    parser.add_argument("--web-host", type=str, help="host for webserver to connect to")
+    parser.add_argument("--web-port", type=int, help="port for webserver to connect to")
+    parser.add_argument("monitor_address", nargs="?", type=str, help="scheduler monitor address to connect to")
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+
+    webui_config = load_config(WebUIConfig, args.config, args, section_name="webui")
+
+    assert webui_config.monitor_address is not None, "scheduler monitor address has to be set"
+
+    start_webui(webui_config.monitor_address.to_address(), webui_config.web_host, webui_config.web_port)

scaler/entry_points/worker_adapter_native.py

@@ -0,0 +1,137 @@
+import argparse
+
+from aiohttp import web
+
+from scaler.config.loader import load_config
+from scaler.config.section.native_worker_adapter import NativeWorkerAdapterConfig
+from scaler.utility.event_loop import EventLoopType, register_event_loop
+from scaler.utility.logging.utility import setup_logger
+from scaler.worker_adapter.native import NativeWorkerAdapter
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        "scaler_native_worker_adapter", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument("--config", "-c", type=str, default=None, help="Path to the TOML configuration file.")
+
+    # Server configuration
+    parser.add_argument("--adapter-web-host", type=str, help="Host for the native worker adapter HTTP server.")
+    parser.add_argument("--adapter-web-port", "-p", type=int, help="Port for the native worker adapter HTTP server.")
+
+    # Worker configuration
+    parser.add_argument("--io-threads", type=int, help="number of io threads for zmq")
+    parser.add_argument(
+        "--per-worker-capabilities",
+        "-pwc",
+        type=str,
+        help='comma-separated capabilities provided by the workers (e.g. "-pwc linux,cpu=4")',
+    )
+    parser.add_argument("--worker-task-queue-size", "-wtqs", type=int, default=10, help="specify worker queue size")
+    parser.add_argument(
+        "--max-workers", "-mw", type=int, help="maximum number of workers that can be started, -1 means no limit"
+    )
+    parser.add_argument(
+        "--heartbeat-interval", "-hi", type=int, help="number of seconds that worker agent send heartbeat to scheduler"
+    )
+    parser.add_argument(
+        "--task-timeout-seconds", "-tt", type=int, help="default task timeout seconds, 0 means never timeout"
+    )
+    parser.add_argument(
+        "--death-timeout-seconds",
+        "-dt",
+        type=int,
+        help="number of seconds without scheduler contact before worker shuts down",
+    )
+    parser.add_argument(
+        "--garbage-collect-interval-seconds", "-gc", type=int, help="number of seconds worker doing garbage collection"
+    )
+    parser.add_argument(
+        "--trim-memory-threshold-bytes",
+        "-tm",
+        type=int,
+        help="number of bytes threshold for worker process that trigger deep garbage collection",
+    )
+    parser.add_argument(
+        "--hard-processor-suspend",
+        "-hps",
+        action="store_true",
+        help="if true, suspended worker's processors will be actively suspended with a SIGTSTP signal",
+    )
+    parser.add_argument("--event-loop", "-e", choices=EventLoopType.allowed_types(), help="select event loop type")
+    parser.add_argument(
+        "--logging-paths",
+        "-lp",
+        nargs="*",
+        type=str,
+        help="specify where worker logs should be logged to, it can accept multiple files, default is /dev/stdout",
+    )
+    parser.add_argument(
+        "--logging-level",
+        "-ll",
+        type=str,
+        choices=("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"),
+        help="specify the logging level",
+    )
+    parser.add_argument(
+        "--logging-config-file",
+        "-lc",
+        type=str,
+        help="use standard python .conf file to specify python logging file configuration format",
+    )
+    parser.add_argument(
+        "--object-storage-address",
+        "-osa",
+        type=str,
+        help="specify the object storage server address, e.g.: tcp://localhost:2346",
+    )
+    parser.add_argument(
+        "scheduler_address",
+        nargs="?",
+        type=str,
+        help="scheduler address to connect workers to, e.g.: `tcp://localhost:6378",
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    native_adapter_config = load_config(
+        NativeWorkerAdapterConfig, args.config, args, section_name="native_worker_adapter"
+    )
+
+    register_event_loop(native_adapter_config.event_loop)
+
+    setup_logger(
+        native_adapter_config.logging_paths,
+        native_adapter_config.logging_config_file,
+        native_adapter_config.logging_level,
+    )
+
+    native_worker_adapter = NativeWorkerAdapter(
+        address=native_adapter_config.scheduler_address,
+        storage_address=native_adapter_config.storage_address,
+        capabilities=native_adapter_config.per_worker_capabilities.capabilities,
+        io_threads=native_adapter_config.io_threads,
+        task_queue_size=native_adapter_config.worker_task_queue_size,
+        max_workers=native_adapter_config.max_workers,
+        heartbeat_interval_seconds=native_adapter_config.heartbeat_interval_seconds,
+        task_timeout_seconds=native_adapter_config.task_timeout_seconds,
+        death_timeout_seconds=native_adapter_config.death_timeout_seconds,
+        garbage_collect_interval_seconds=native_adapter_config.garbage_collect_interval_seconds,
+        trim_memory_threshold_bytes=native_adapter_config.trim_memory_threshold_bytes,
+        hard_processor_suspend=native_adapter_config.hard_processor_suspend,
+        event_loop=native_adapter_config.event_loop,
+        logging_paths=native_adapter_config.logging_paths,
+        logging_level=native_adapter_config.logging_level,
+        logging_config_file=native_adapter_config.logging_config_file,
+    )
+
+    app = native_worker_adapter.create_app()
+    web.run_app(app, host=native_adapter_config.adapter_web_host, port=native_adapter_config.adapter_web_port)
+
+
+if __name__ == "__main__":
+    main()

scaler/entry_points/worker_adapter_symphony.py

@@ -0,0 +1,102 @@
+import argparse
+
+from aiohttp import web
+
+from scaler.config.loader import load_config
+from scaler.config.section.symphony_worker_adapter import SymphonyWorkerConfig
+from scaler.utility.event_loop import EventLoopType, register_event_loop
+from scaler.utility.logging.utility import setup_logger
+from scaler.worker_adapter.symphony.worker_adapter import SymphonyWorkerAdapter
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        "scaler Symphony worker adapter", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument("--config", "-c", type=str, default=None, help="Path to the TOML configuration file.")
+
+    # Server configuration
+    parser.add_argument(
+        "--server-http-host", "-h", type=str, help="host address for the native worker adapter HTTP server"
+    )
+    parser.add_argument(
+        "--server-http-port", "-p", type=int, required=True, help="port for the native worker adapter HTTP server"
+    )
+
+    # Symphony configuration
+    parser.add_argument("--service-name", "-sn", type=str, required=True, help="symphony service name")
+    parser.add_argument("--base-concurrency", "-n", type=int, help="base task concurrency")
+
+    # Worker configuration
+    parser.add_argument("--io-threads", "-it", help="specify number of io threads per worker")
+    parser.add_argument(
+        "--worker-capabilities",
+        "-wc",
+        type=str,
+        help='comma-separated capabilities provided by the worker (e.g. "-wr linux,cpu=4")',
+    )
+    parser.add_argument("--worker-task-queue-size", "-wtqs", type=int, help="specify symphony worker queue size")
+    parser.add_argument("--heartbeat-interval", "-hi", type=int, help="number of seconds to send heartbeat interval")
+    parser.add_argument("--death-timeout-seconds", "-ds", type=int, help="death timeout seconds")
+    parser.add_argument("--event-loop", "-el", choices=EventLoopType.allowed_types(), help="select event loop type")
+    parser.add_argument(
+        "--logging-paths",
+        "-lp",
+        nargs="*",
+        type=str,
+        help='specify where cluster log should logged to, it can be multiple paths, "/dev/stdout" is default for '
+        "standard output, each worker will have its own log file with process id appended to the path",
+    )
+    parser.add_argument(
+        "--logging-level",
+        "-ll",
+        type=str,
+        choices=("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"),
+        help="specify the logging level",
+    )
+    parser.add_argument(
+        "--logging-config-file",
+        type=str,
+        default=None,
+        help="use standard python the .conf file the specify python logging file configuration format, this will "
+        "bypass --logging-paths and --logging-level at the same time, and this will not work on per worker logging",
+    )
+    parser.add_argument(
+        "--object-storage-address",
+        "-osa",
+        default=None,
+        help="specify the object storage server address, e.g.: tcp://localhost:2346",
+    )
+    parser.add_argument("scheduler_address", nargs="?", type=str, help="scheduler address to connect to")
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    symphony_config = load_config(SymphonyWorkerConfig, args.config, args, section_name="symphony_worker_adapter")
+    register_event_loop(symphony_config.event_loop)
+
+    setup_logger(symphony_config.logging_paths, symphony_config.logging_config_file, symphony_config.logging_level)
+
+    symphony_worker_adapter = SymphonyWorkerAdapter(
+        address=symphony_config.scheduler_address,
+        storage_address=symphony_config.object_storage_address,
+        capabilities=symphony_config.worker_capabilities.capabilities,
+        task_queue_size=symphony_config.worker_task_queue_size,
+        service_name=symphony_config.service_name,
+        base_concurrency=symphony_config.base_concurrency,
+        heartbeat_interval_seconds=symphony_config.heartbeat_interval,
+        death_timeout_seconds=symphony_config.death_timeout_seconds,
+        event_loop=symphony_config.event_loop,
+        io_threads=symphony_config.io_threads,
+        logging_paths=symphony_config.logging_paths,
+        logging_level=symphony_config.logging_level,
+        logging_config_file=symphony_config.logging_config_file,
+    )
+
+    app = symphony_worker_adapter.create_app()
+    web.run_app(app, host=symphony_config.server_http_host, port=symphony_config.server_http_port)
+
+
+if __name__ == "__main__":
+    main()

scaler/io/async_binder.py

@@ -0,0 +1,85 @@
+import logging
+import os
+import uuid
+from collections import defaultdict
+from typing import Awaitable, Callable, Dict, List, Optional
+
+import zmq.asyncio
+from zmq import Frame
+
+from scaler.io.mixins import AsyncBinder
+from scaler.io.utility import deserialize, serialize
+from scaler.protocol.python.mixins import Message
+from scaler.protocol.python.status import BinderStatus
+from scaler.config.types.zmq import ZMQConfig
+
+
+class ZMQAsyncBinder(AsyncBinder):
+    def __init__(self, context: zmq.asyncio.Context, name: str, address: ZMQConfig, identity: Optional[bytes] = None):
+        self._address = address
+
+        if identity is None:
+            identity = f"{os.getpid()}|{name}|{uuid.uuid4()}".encode()
+        self._identity = identity
+
+        self._context = context
+        self._socket = self._context.socket(zmq.ROUTER)
+        self.__set_socket_options()
+        self._socket.bind(self._address.to_address())
+
+        self._callback: Optional[Callable[[bytes, Message], Awaitable[None]]] = None
+
+        self._received: Dict[str, int] = defaultdict(lambda: 0)
+        self._sent: Dict[str, int] = defaultdict(lambda: 0)
+
+    @property
+    def identity(self):
+        return self._identity
+
+    def destroy(self):
+        self._context.destroy(linger=0)
+
+    def register(self, callback: Callable[[bytes, Message], Awaitable[None]]):
+        self._callback = callback
+
+    async def routine(self):
+        frames: List[Frame] = await self._socket.recv_multipart(copy=False)
+        if not self.__is_valid_message(frames):
+            return
+
+        source, payload = frames
+        message: Optional[Message] = deserialize(payload.bytes)
+        if message is None:
+            logging.error(f"received unknown message from {source.bytes!r}: {payload!r}")
+            return
+
+        self.__count_received(message.__class__.__name__)
+        await self._callback(source.bytes, message)
+
+    async def send(self, to: bytes, message: Message):
+        self.__count_sent(message.__class__.__name__)
+        await self._socket.send_multipart([to, serialize(message)], copy=False)
+
+    def get_status(self) -> BinderStatus:
+        return BinderStatus.new_msg(received=self._received, sent=self._sent)
+
+    def __set_socket_options(self):
+        self._socket.setsockopt(zmq.IDENTITY, self._identity)
+        self._socket.setsockopt(zmq.SNDHWM, 0)
+        self._socket.setsockopt(zmq.RCVHWM, 0)
+
+    def __is_valid_message(self, frames: List[Frame]) -> bool:
+        if len(frames) < 2:
+            logging.error(f"{self.__get_prefix()} received unexpected frames {frames}")
+            return False
+
+        return True
+
+    def __count_received(self, message_type: str):
+        self._received[message_type] += 1
+
+    def __count_sent(self, message_type: str):
+        self._sent[message_type] += 1
+
+    def __get_prefix(self):
+        return f"{self.__class__.__name__}[{self._identity.decode()}]:"