opengris-scaler 1.12.7__cp312-cp312-manylinux_2_28_x86_64.whl

scaler/config/section/symphony_worker_adapter.py

@@ -0,0 +1,47 @@
+import dataclasses
+from typing import Optional, Tuple
+
+from scaler.config.types.object_storage_server import ObjectStorageConfig
+from scaler.config.types.worker import WorkerCapabilities
+from scaler.config.types.zmq import ZMQConfig
+from scaler.config import defaults
+from scaler.utility.logging.utility import LoggingLevel
+
+
+@dataclasses.dataclass
+class SymphonyWorkerConfig:
+    scheduler_address: ZMQConfig
+    object_storage_address: Optional[ObjectStorageConfig]
+    service_name: str
+    base_concurrency: int = defaults.DEFAULT_NUMBER_OF_WORKER
+    worker_capabilities: WorkerCapabilities = dataclasses.field(
+        default_factory=lambda: WorkerCapabilities.from_string("")
+    )
+    server_http_host: str = "localhost"
+    server_http_port: int = 0
+    io_threads: int = defaults.DEFAULT_IO_THREADS
+    worker_task_queue_size: int = defaults.DEFAULT_PER_WORKER_QUEUE_SIZE
+    heartbeat_interval: int = defaults.DEFAULT_HEARTBEAT_INTERVAL_SECONDS
+    death_timeout_seconds: int = defaults.DEFAULT_WORKER_DEATH_TIMEOUT
+    event_loop: str = "builtin"
+    logging_paths: Tuple[str, ...] = defaults.DEFAULT_LOGGING_PATHS
+    logging_level: str = defaults.DEFAULT_LOGGING_LEVEL
+    logging_config_file: Optional[str] = None
+
+    def __post_init__(self):
+        """Validates configuration values after initialization."""
+        if (
+            self.base_concurrency <= 0
+            or self.worker_task_queue_size <= 0
+            or self.heartbeat_interval <= 0
+            or self.death_timeout_seconds <= 0
+            or self.io_threads <= 0
+        ):
+            raise ValueError("All concurrency, queue size, timeout, and thread count values must be positive integers.")
+
+        if not self.service_name:
+            raise ValueError("service_name cannot be an empty string.")
+
+        valid_levels = {level.name for level in LoggingLevel}
+        if self.logging_level.upper() not in valid_levels:
+            raise ValueError(f"logging_level must be one of {valid_levels}, but got '{self.logging_level}'")

scaler/config/section/top.py

@@ -0,0 +1,13 @@
+import dataclasses
+
+from scaler.config.types.zmq import ZMQConfig
+
+
+@dataclasses.dataclass
+class TopConfig:
+    monitor_address: ZMQConfig
+    timeout: int = 5
+
+    def __post_init__(self):
+        if self.timeout <= 0:
+            raise ValueError("timeout must be a positive integer.")

scaler/config/section/webui.py

@@ -0,0 +1,16 @@
+import dataclasses
+
+from scaler.config.types.zmq import ZMQConfig
+
+
+@dataclasses.dataclass
+class WebUIConfig:
+    monitor_address: ZMQConfig
+    web_host: str = "0.0.0.0"
+    web_port: int = 50001
+
+    def __post_init__(self):
+        if not isinstance(self.web_host, str):
+            raise TypeError(f"Web host should be string, given {self.web_host}")
+        if not isinstance(self.web_port, int):
+            raise TypeError(f"Web port should be an integer, given {self.web_port}")

scaler/config/types/object_storage_server.py

@@ -0,0 +1,45 @@
+import dataclasses
+import ipaddress
+
+from scaler.config.mixins import ConfigType
+
+
+@dataclasses.dataclass
+class ObjectStorageConfig(ConfigType):
+    host: str
+    port: int
+    identity: str = "ObjectStorageServer"
+
+    def __post_init__(self):
+        try:
+            ipaddress.ip_address(self.host)
+        except ValueError:
+            raise TypeError(f"Host must be a valid IP address, but got '{self.host}'")
+
+        if not isinstance(self.identity, str):
+            raise TypeError(f"Identity should be a string, given {self.identity}")
+
+        if not isinstance(self.port, int):
+            raise TypeError(f"Port should be an integer, given {self.port}")
+
+    def __str__(self) -> str:
+        return self.to_string()
+
+    def to_string(self) -> str:
+        return f"tcp://{self.host}:{self.port}"
+
+    @classmethod
+    def from_string(cls, value: str) -> "ObjectStorageConfig":
+        if not value.startswith("tcp://"):
+            raise ValueError("Address must start with 'tcp://'")
+
+        try:
+            host, port_str = value[6:].rsplit(":", 1)
+            port = int(port_str)
+
+            ipaddress.ip_address(host)
+
+        except (ValueError, IndexError):
+            raise ValueError(f"Invalid address format '{value}'. Expected format is tcp://<ip_address>:<port>")
+
+        return cls(host=host, port=port)

scaler/config/types/worker.py

@@ -0,0 +1,57 @@
+import dataclasses
+from typing import List, Dict
+from typing_extensions import Self
+
+from scaler.config.mixins import ConfigType
+
+
+@dataclasses.dataclass
+class WorkerNames(ConfigType):
+    """Parses a comma-separated string of worker names into a list."""
+
+    names: List[str]
+
+    @classmethod
+    def from_string(cls, value: str) -> Self:
+        if not value:
+            return cls([])
+        names = [name.strip() for name in value.split(",")]
+        return cls(names)
+
+    def __str__(self) -> str:
+        return ",".join(self.names)
+
+    def __len__(self) -> int:
+        return len(self.names)
+
+
+@dataclasses.dataclass
+class WorkerCapabilities(ConfigType):
+    """Parses a string of worker capabilities."""
+
+    capabilities: Dict[str, int]
+
+    @classmethod
+    def from_string(cls, value: str) -> Self:
+        capabilities: Dict[str, int] = {}
+        if not value:
+            return cls(capabilities)
+        for item in value.split(","):
+            name, _, value = item.partition("=")
+            if value != "":
+                try:
+                    capabilities[name.strip()] = int(value)
+                except ValueError:
+                    raise ValueError(f"Invalid capability value for '{name}'. Expected an integer, but got '{value}'.")
+            else:
+                capabilities[name.strip()] = -1
+        return cls(capabilities)
+
+    def __str__(self) -> str:
+        items = []
+        for name, cap in self.capabilities.items():
+            if cap == -1:
+                items.append(name)
+            else:
+                items.append(f"{name}={cap}")
+        return ",".join(items)

scaler/config/types/zmq.py

@@ -0,0 +1,79 @@
+import dataclasses
+import enum
+from typing import Optional
+
+from typing_extensions import Self
+from scaler.config.mixins import ConfigType
+
+
+class ZMQType(enum.Enum):
+    inproc = "inproc"
+    ipc = "ipc"
+    tcp = "tcp"
+
+    @staticmethod
+    def allowed_types():
+        return {t.value for t in ZMQType}
+
+
+@dataclasses.dataclass
+class ZMQConfig(ConfigType):
+    type: ZMQType
+    host: str
+    port: Optional[int] = None
+
+    def __post_init__(self):
+        if not isinstance(self.type, ZMQType):
+            raise TypeError(f"Invalid zmq type {self.type}, available types are: {ZMQType.allowed_types()}")
+
+        if not isinstance(self.host, str):
+            raise TypeError(f"Host should be string, given {self.host}")
+
+        if self.port is None:
+            if self.type == ZMQType.tcp:
+                raise ValueError(f"type {self.type.value} should have `port`")
+        else:
+            if self.type in {ZMQType.inproc, ZMQType.ipc}:
+                raise ValueError(f"type {self.type.value} should not have `port`")
+
+            if not isinstance(self.port, int):
+                raise TypeError(f"Port should be integer, given {self.port}")
+
+    def to_address(self):
+        if self.type == ZMQType.tcp:
+            return f"tcp://{self.host}:{self.port}"
+
+        if self.type in {ZMQType.inproc, ZMQType.ipc}:
+            return f"{self.type.value}://{self.host}"
+
+        raise TypeError(f"Unsupported ZMQ type: {self.type}")
+
+    @classmethod
+    def from_string(cls, value: str) -> Self:
+        if "://" not in value:
+            raise ValueError("valid ZMQ config should be like tcp://127.0.0.1:12345")
+
+        socket_type, host_port = value.split("://", 1)
+        if socket_type not in ZMQType.allowed_types():
+            raise ValueError(f"supported ZMQ types are: {ZMQType.allowed_types()}")
+
+        socket_type_enum = ZMQType(socket_type)
+        if socket_type_enum in {ZMQType.inproc, ZMQType.ipc}:
+            host = host_port
+            port_int = None
+        elif socket_type_enum == ZMQType.tcp:
+            host, port = host_port.split(":")
+            try:
+                port_int = int(port)
+            except ValueError:
+                raise ValueError(f"cannot convert '{port}' to port number")
+        else:
+            raise ValueError(f"Unsupported ZMQ type: {socket_type}")
+
+        return cls(socket_type_enum, host, port_int)
+
+    def __str__(self) -> str:
+        return self.to_address()
+
+    def __repr__(self) -> str:
+        return self.to_address()

scaler/entry_points/cluster.py

@@ -0,0 +1,133 @@
+import argparse
+import socket
+
+from scaler.cluster.cluster import Cluster
+from scaler.config.section.cluster import ClusterConfig
+from scaler.config.loader import load_config
+from scaler.utility.event_loop import EventLoopType, register_event_loop
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        "standalone compute cluster", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument("--config", "-c", type=str, default=None, help="Path to the TOML configuration file.")
+
+    parser.add_argument(
+        "--preload",
+        type=str,
+        default=None,
+        help='optional module init in the form "pkg.mod:func(arg1, arg2)" executed in each processor before tasks',
+    )
+    parser.add_argument("--num-of-workers", "-n", type=int, help="number of workers in cluster")
+    parser.add_argument(
+        "--worker-names",
+        "-wn",
+        type=str,
+        help="worker names to replace default worker names (host names), separate by comma",
+    )
+    parser.add_argument(
+        "--per-worker-capabilities",
+        "-pwc",
+        type=str,
+        help='comma-separated capabilities provided by the workers (e.g. "-pwc linux,cpu=4")',
+    )
+    parser.add_argument("--per-worker-task-queue-size", "-wtqs", type=int, help="specify per worker queue size")
+    parser.add_argument(
+        "--heartbeat-interval-seconds", "-hi", type=int, help="number of seconds to send heartbeat interval"
+    )
+    parser.add_argument(
+        "--task-timeout-seconds",
+        "-tts",
+        type=int,
+        help="number of seconds task treat as timeout and return an exception",
+    )
+    parser.add_argument("--garbage-collect-interval-seconds", "-gc", type=int, help="garbage collect interval seconds")
+    parser.add_argument("--death-timeout-seconds", "-ds", type=int, help="death timeout seconds")
+    parser.add_argument(
+        "--trim-memory-threshold-bytes", "-tm", type=int, help="number of bytes threshold to enable libc to trim memory"
+    )
+    parser.add_argument("--event-loop", "-el", choices=EventLoopType.allowed_types(), help="select event loop type")
+    parser.add_argument("--io-threads", "-it", type=int, help="specify number of io threads per worker")
+    parser.add_argument(
+        "--hard-processor-suspend",
+        "-hps",
+        action="store_true",
+        help=(
+            "When set, suspends worker processors using the SIGTSTP signal instead of a synchronization event, "
+            "fully halting computation on suspended tasks. Note that this may cause some tasks to fail if they "
+            "do not support being paused at the OS level (e.g. tasks requiring active network connections)."
+        ),
+    )
+    parser.add_argument("--log-hub-address", "-la", type=str, help="address for Worker send logs")
+    parser.add_argument(
+        "--logging-paths",
+        "-lp",
+        nargs="*",
+        type=str,
+        help='specify where cluster log should logged to, it can be multiple paths, "/dev/stdout" is default for '
+        "standard output, each worker will have its own log file with process id appended to the path",
+    )
+    parser.add_argument(
+        "--logging-level",
+        "-ll",
+        type=str,
+        choices=("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"),
+        help="specify the logging level",
+    )
+    parser.add_argument(
+        "--logging-config-file",
+        "-lcf",
+        type=str,
+        help="use standard python the .conf file the specify python logging file configuration format, this will "
+        "bypass --logging-paths and --logging-level at the same time, and this will not work on per worker logging",
+    )
+    parser.add_argument(
+        "--object-storage-address",
+        "-osa",
+        type=str,
+        help="specify the object storage server address, e.g. tcp://localhost:2346. If not specified, use the address "
+        "provided by the scheduler",
+    )
+    parser.add_argument("scheduler_address", nargs="?", type=str, help="scheduler address to connect to")
+
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+
+    cluster_config = load_config(ClusterConfig, args.config, args, section_name="cluster")
+
+    register_event_loop(cluster_config.event_loop)
+
+    worker_names = cluster_config.worker_names.names
+    if not worker_names:
+        worker_names = [f"{socket.gethostname().split('.')[0]}" for _ in range(cluster_config.num_of_workers)]
+
+    if len(worker_names) != cluster_config.num_of_workers:
+        raise ValueError(
+            f"Number of worker names ({len(worker_names)}) must match the number of workers "
+            f"({cluster_config.num_of_workers})."
+        )
+
+    cluster = Cluster(
+        address=cluster_config.scheduler_address,
+        storage_address=cluster_config.storage_address,
+        preload=cluster_config.preload,
+        worker_names=worker_names,
+        per_worker_capabilities=cluster_config.per_worker_capabilities.capabilities,
+        per_worker_task_queue_size=cluster_config.per_worker_task_queue_size,
+        heartbeat_interval_seconds=cluster_config.heartbeat_interval_seconds,
+        task_timeout_seconds=cluster_config.task_timeout_seconds,
+        garbage_collect_interval_seconds=cluster_config.garbage_collect_interval_seconds,
+        trim_memory_threshold_bytes=cluster_config.trim_memory_threshold_bytes,
+        death_timeout_seconds=cluster_config.death_timeout_seconds,
+        hard_processor_suspend=cluster_config.hard_processor_suspend,
+        event_loop=cluster_config.event_loop,
+        worker_io_threads=cluster_config.worker_io_threads,
+        logging_paths=cluster_config.logging_paths,
+        logging_level=cluster_config.logging_level,
+        logging_config_file=cluster_config.logging_config_file,
+    )
+    cluster.run()

scaler/entry_points/object_storage_server.py

@@ -0,0 +1,41 @@
+import argparse
+import logging
+
+from scaler.config.loader import load_config
+from scaler.config.section.object_storage_server import ObjectStorageServerConfig
+from scaler.object_storage.object_storage_server import ObjectStorageServer
+from scaler.utility.logging.utility import get_logger_info, setup_logger
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        "scaler_object_storage_server", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument("--config", "-c", type=str, default=None, help="Path to the TOML configuration file.")
+
+    parser.add_argument(
+        "object_storage_address",
+        nargs="?",
+        type=str,
+        help="specify the object storage server address to listen to, e.g. tcp://localhost:2345.",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+
+    oss_config = load_config(ObjectStorageServerConfig, args.config, args, section_name="object_storage_server")
+
+    setup_logger()
+
+    log_format_str, log_level_str, log_paths = get_logger_info(logging.getLogger())
+
+    ObjectStorageServer().run(
+        oss_config.object_storage_address.host,
+        oss_config.object_storage_address.port,
+        oss_config.object_storage_address.identity,
+        log_level_str,
+        log_format_str,
+        log_paths,
+    )

scaler/entry_points/scheduler.py

@@ -0,0 +1,135 @@
+import argparse
+
+from scaler.config.loader import load_config
+from scaler.cluster.object_storage_server import ObjectStorageServerProcess
+from scaler.cluster.scheduler import SchedulerProcess
+from scaler.scheduler.allocate_policy.allocate_policy import AllocatePolicy
+from scaler.config.section.scheduler import SchedulerConfig
+from scaler.config.types.object_storage_server import ObjectStorageConfig
+from scaler.utility.event_loop import EventLoopType
+from scaler.utility.network_util import get_available_tcp_port
+
+
+def get_args():
+    parser = argparse.ArgumentParser("scaler_scheduler", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument("--config", "-c", type=str, default=None, help="Path to the TOML configuration file.")
+    parser.add_argument("--io-threads", type=int, help="number of io threads for zmq")
+    parser.add_argument(
+        "--max-number-of-tasks-waiting",
+        "-mt",
+        type=int,
+        help="max number of tasks can wait in scheduler while all workers are full",
+    )
+    parser.add_argument("--client-timeout-seconds", "-ct", type=int, help="discard client when timeout seconds reached")
+    parser.add_argument("--worker-timeout-seconds", "-wt", type=int, help="discard worker when timeout seconds reached")
+    parser.add_argument(
+        "--object-retention-seconds", "-ot", type=int, help="discard function in scheduler when timeout seconds reached"
+    )
+    parser.add_argument(
+        "--load-balance-seconds", "-ls", type=int, help="number of seconds for load balance operation in scheduler"
+    )
+    parser.add_argument(
+        "--load-balance-trigger-times",
+        "-lbt",
+        type=int,
+        help="exact number of repeated load balance advices when trigger load balance operation in scheduler",
+    )
+    parser.add_argument("--event-loop", "-e", choices=EventLoopType.allowed_types(), help="select event loop type")
+    parser.add_argument(
+        "--protected", "-p", action="store_true", help="protect scheduler and worker from being shutdown by client"
+    )
+    parser.add_argument(
+        "--allocate-policy",
+        "-ap",
+        choices=[p.name for p in AllocatePolicy],
+        help="specify allocate policy, this controls how scheduler will prioritize tasks, including balancing tasks",
+    )
+    parser.add_argument(
+        "--logging-paths",
+        "-lp",
+        nargs="*",
+        type=str,
+        help="specify where scheduler log should logged to, it can accept multiple files, default is /dev/stdout",
+    )
+    parser.add_argument("--logging-level", "-ll", type=str, help="specify the logging level")
+    parser.add_argument(
+        "--logging-config-file",
+        "-lc",
+        type=str,
+        help="use standard python the .conf file the specify python logging file configuration format, this will "
+        "bypass --logging-path",
+    )
+    parser.add_argument(
+        "--object-storage-address",
+        "-osa",
+        type=str,
+        help="specify the object storage server address, if not specified, the address is scheduler address with port "
+        "number plus 1, e.g.: if scheduler address is tcp://localhost:2345, then object storage address is "
+        "tcp://localhost:2346",
+    )
+    parser.add_argument(
+        "--monitor-address",
+        "-ma",
+        type=str,
+        help="specify monitoring address, if not specified, the monitoring address is scheduler address with port "
+        "number plus 2, e.g.: if scheduler address is tcp://localhost:2345, then monitoring address is "
+        "tcp://localhost:2347",
+    )
+    parser.add_argument(
+        "--adapter-webhook-url",
+        "-awu",
+        type=str,
+        help="specify the adapter webhook url, if not specified, the adapter will not be used",
+    )
+    parser.add_argument(
+        "scheduler_address", nargs="?", type=str, help="scheduler address to connect to, e.g.: `tcp://localhost:6378`"
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+
+    scheduler_config = load_config(SchedulerConfig, args.config, args, section_name="scheduler")
+
+    if args.object_storage_address is None:
+        object_storage_address = ObjectStorageConfig(
+            host=scheduler_config.scheduler_address.host, port=get_available_tcp_port()
+        )
+        object_storage = ObjectStorageServerProcess(
+            storage_address=object_storage_address,
+            logging_paths=scheduler_config.logging_paths,
+            logging_config_file=scheduler_config.logging_config_file,
+            logging_level=scheduler_config.logging_level,
+        )
+        object_storage.start()
+        object_storage.wait_until_ready()  # object storage should be ready before starting the cluster
+    else:
+        object_storage_address = scheduler_config.object_storage_address
+        object_storage = None
+
+    scheduler = SchedulerProcess(
+        address=scheduler_config.scheduler_address,
+        storage_address=object_storage_address,
+        monitor_address=scheduler_config.monitor_address,
+        adapter_webhook_url=scheduler_config.adapter_webhook_url,
+        io_threads=scheduler_config.io_threads,
+        max_number_of_tasks_waiting=scheduler_config.max_number_of_tasks_waiting,
+        client_timeout_seconds=scheduler_config.client_timeout_seconds,
+        worker_timeout_seconds=scheduler_config.worker_timeout_seconds,
+        object_retention_seconds=scheduler_config.object_retention_seconds,
+        load_balance_seconds=scheduler_config.load_balance_seconds,
+        load_balance_trigger_times=scheduler_config.load_balance_trigger_times,
+        protected=scheduler_config.protected,
+        allocate_policy=scheduler_config.allocate_policy,
+        event_loop=scheduler_config.event_loop,
+        logging_paths=scheduler_config.logging_paths,
+        logging_config_file=scheduler_config.logging_config_file,
+        logging_level=scheduler_config.logging_level,
+    )
+    scheduler.start()
+
+    scheduler.join()
+    if object_storage is not None:
+        object_storage.join()