mship-engine 0.7.12__py3-none-any.whl

modelship/deploy/actor_options.py

@@ -0,0 +1,178 @@
+"""Ray Serve actor option construction for model deployments.
+
+Centralises the GPU-allocation decisions for model deployments. Multi-slot
+vLLM deploys always use a Ray Serve placement group (one whole-GPU bundle
+per slot) that vLLM inherits via its ray distributed executor.
+"""
+
+from __future__ import annotations
+
+import os
+import platform
+
+from modelship.deploy.capabilities import deployment_capability_resources
+from modelship.infer.infer_config import ModelLoader, ModelshipModelConfig
+from modelship.logging import get_logger
+from modelship.utils.cache import resolve_cache_root
+
+logger = get_logger("startup")
+
+# Forwarded from the driver to each replica's runtime_env: logging vars, the gateway
+# name (metrics.py stamps every metric with it), MSHIP_METRICS so --no-metrics on
+# the driver also disables metrics in the replicas (else they'd default to on),
+# MSHIP_PREFLIGHT so --no-preflight on the driver also disables it in the replicas
+# (preflight runs inside each loader's actor __init__, not on the driver), and the
+# /v1/responses state-store tuning read inside the gateway replica's own process
+# (state.responses.ttl_seconds / state.memory._sweep_interval_s), not the driver's.
+_PASSTHROUGH_ENV_VARS = (
+    "MSHIP_LOG_LEVEL",
+    "MSHIP_LOG_FORMAT",
+    "MSHIP_LOG_TARGET",
+    "MSHIP_GATEWAY_NAME",
+    "MSHIP_METRICS",
+    "MSHIP_PREFLIGHT",
+    "MSHIP_RESPONSES_TTL_S",
+    "MSHIP_STATE_SWEEP_INTERVAL_S",
+)
+
+
+def build_passthrough_env_vars() -> dict[str, str]:
+    """Driver→replica env vars (logging, gateway name, metrics) read off the
+    driver's environment. Shared by model and gateway deployments so both
+    replicas inherit the same logging/metrics config."""
+    return {var: os.environ[var] for var in _PASSTHROUGH_ENV_VARS if os.environ.get(var) is not None}
+
+
+def build_cache_env_vars() -> dict[str, str]:
+    """Resolve HF / vLLM / FlashInfer cache dirs, all rooted at MSHIP_CACHE_DIR.
+
+    Also forwards HF_TOKEN/HF_HUB_OFFLINE when set on the driver, so an actor
+    downloading a gated/offline model has the same auth."""
+    base_cache = resolve_cache_root()
+    env_vars = {
+        "HF_HOME": os.environ.get("HF_HOME", f"{base_cache}/huggingface"),
+        "VLLM_CACHE_ROOT": os.environ.get("VLLM_CACHE_ROOT", f"{base_cache}/vllm"),
+        "FLASHINFER_CACHE_DIR": os.environ.get("FLASHINFER_CACHE_DIR", f"{base_cache}/flashinfer"),
+        # Triton JITs kernels at import for some archs
+        "TRITON_CACHE_DIR": os.environ.get("TRITON_CACHE_DIR", f"{base_cache}/triton"),
+        # vLLM's usage-stats thread writes usage_stats.json/do_not_track here
+        "VLLM_CONFIG_ROOT": os.environ.get("VLLM_CONFIG_ROOT", f"{base_cache}/vllm-config"),
+        # Default download dir for the whispercpp loader's pywhispercpp-managed
+        # built-in model names (a bare `model:` like `base.en`).
+        "MSHIP_WHISPERCPP_CACHE_DIR": os.environ.get("MSHIP_WHISPERCPP_CACHE_DIR", f"{base_cache}/whispercpp"),
+    }
+    for var in ("HF_TOKEN", "HF_HUB_OFFLINE"):
+        if os.environ.get(var) is not None:
+            env_vars[var] = os.environ[var]
+    return env_vars
+
+
+def _world_size(config: ModelshipModelConfig) -> int:
+    if config.loader != ModelLoader.vllm:
+        return 1
+    tp = config.vllm_engine_kwargs.tensor_parallel_size
+    pp = config.vllm_engine_kwargs.pipeline_parallel_size
+    return tp * pp
+
+
+def total_gpu_reservation(deploy_opts: dict) -> float:
+    """Sum the GPU units this deployment (actor + any PG bundles) will consume.
+
+    Used by the coordinator's resource tracker, which can't read the PG
+    bundle list as a single scalar.
+    """
+    return _total_reservation(deploy_opts, "GPU", "num_gpus")
+
+
+def total_cpu_reservation(deploy_opts: dict) -> float:
+    """Sum the CPU units this deployment (actor + any PG bundles) will consume.
+
+    For multi-slot deploys the outer actor sits in bundle 0 and its CPU
+    request is satisfied from that bundle's reservation, so summing the
+    bundles gives the correct total — same shape as the GPU helper.
+    """
+    return _total_reservation(deploy_opts, "CPU", "num_cpus")
+
+
+def _total_reservation(deploy_opts: dict, bundle_key: str, actor_key: str) -> float:
+    if "placement_group_bundles" in deploy_opts:
+        return float(sum(b.get(bundle_key, 0) for b in deploy_opts["placement_group_bundles"]))
+    return float(deploy_opts.get("ray_actor_options", {}).get(actor_key, 0) or 0)
+
+
+def build_deployment_options(config: ModelshipModelConfig) -> dict:
+    """Return a kwargs dict for `Deployment.options(**...)`.
+
+    Always contains ``ray_actor_options``; for multi-slot vLLM deploys also
+    contains ``placement_group_bundles`` and ``placement_group_strategy`` so
+    Ray Serve allocates one whole-GPU bundle per slot and vLLM's ray executor
+    inherits the PG. When the model config sets ``max_ongoing_requests`` it is
+    forwarded as the per-replica Ray Serve concurrency cap.
+    """
+    env_vars = build_cache_env_vars()
+    env_vars.update(build_passthrough_env_vars())
+
+    runtime_env: dict = {"env_vars": env_vars}
+
+    capability_resources = deployment_capability_resources(config)
+
+    # sherpa_onnx never touches CUDA or CoreML (CPU only); ggml-backed loaders are
+    # CPU-only off Darwin, where forcing 0 would mislead Ray into co-scheduling
+    # another GPU actor onto the device Metal is actually using.
+    force_zero_gpu = config.loader == ModelLoader.sherpa_onnx or (
+        config.loader in (ModelLoader.stable_diffusion_cpp, ModelLoader.whispercpp) and platform.system() != "Darwin"
+    )
+    if force_zero_gpu:
+        if config.num_gpus > 0:
+            logger.warning(
+                "num_gpus=%s is ignored for model '%s': %s loader has no GPU backend here.",
+                config.num_gpus,
+                config.name,
+                config.loader.value,
+            )
+        opts: dict = {
+            "ray_actor_options": {
+                "num_gpus": 0,
+                "num_cpus": config.num_cpus,
+                "runtime_env": runtime_env,
+                "resources": capability_resources,
+            }
+        }
+    else:
+        world_size = _world_size(config)
+        if world_size == 1:
+            # Single slot: scalar Ray allocation. Fractional num_gpus (0 < n < 1)
+            # lets Ray pack other actors onto the same physical GPU.
+            opts = {
+                "ray_actor_options": {
+                    "num_gpus": config.num_gpus,
+                    "num_cpus": config.num_cpus,
+                    "runtime_env": runtime_env,
+                    "resources": capability_resources,
+                }
+            }
+        else:
+            # Multi-slot: one PG bundle per slot, STRICT_PACK keeps them on the
+            # same node (NVLink). Outer actor sits in bundle 0 with 0 GPU; vLLM's
+            # ray executor reuses the PG via get_current_placement_group() and
+            # pins each worker actor to its bundle. Each bundle requests a whole
+            # GPU, so Ray spreads across distinct physical GPUs. The capability
+            # resource is requested on every bundle (not the outer actor) since
+            # the bundles are what pin the deploy to a capable node.
+            bundles = [{"GPU": 1, "CPU": config.num_cpus, **capability_resources} for _ in range(world_size)]
+            opts = {
+                "ray_actor_options": {"num_gpus": 0, "num_cpus": config.num_cpus, "runtime_env": runtime_env},
+                "placement_group_bundles": bundles,
+                "placement_group_strategy": "STRICT_PACK",
+            }
+
+    # Per-model Ray Serve concurrency cap; only override the default when set.
+    # The reservation helpers read only the GPU/CPU keys, so this is inert there.
+    max_ongoing = config.max_ongoing_requests
+    if max_ongoing is None and config.loader == ModelLoader.llama_server:
+        parallel = config.llama_server_config.parallel if config.llama_server_config else 1
+        max_ongoing = parallel
+
+    if max_ongoing is not None:
+        opts["max_ongoing_requests"] = max_ongoing
+    return opts

modelship/deploy/capabilities.py

@@ -0,0 +1,84 @@
+"""Node/deployment capability resources: `mship_<loader>` alongside Ray's native
+`GPU`, so a deploy only schedules onto a node with that loader installed.
+
+Must stay ray-free at import time: modelship/launcher.py imports LOADER_MODULES
+from here before resolve_ray_auth_env runs, so the ModelshipModelConfig annotation
+below is deferred behind `if TYPE_CHECKING` rather than imported for real.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from modelship.infer.infer_config import ModelshipModelConfig
+
+RESOURCE_PREFIX = "mship_"
+
+# loader name -> the module find_spec() probes for it. llama_server is handled
+# separately below (a subprocess binary, not an importable module).
+LOADER_MODULES = {
+    "vllm": "vllm",
+    "diffusers": "diffusers",
+    "stable_diffusion_cpp": "stable_diffusion_cpp",
+    "whispercpp": "pywhispercpp",
+    "sherpa_onnx": "sherpa_onnx",
+}
+
+_LLAMA_SERVER_LOADER = "llama_server"
+
+# Every loader that gets a capability resource, including llama_server (probed via a
+# binary check below, not find_spec like the LOADER_MODULES entries).
+ALL_CAPABILITY_LOADERS = frozenset({*LOADER_MODULES, _LLAMA_SERVER_LOADER})
+
+# Matches a wrapper script's `exec <target> ...` line, quoted or not (Dockerfile's
+# llama-server.sh emits it unquoted; launcher._write_wrapper quotes it).
+_WRAPPER_EXEC_RE = re.compile(r'exec\s+"?([^"\s]+)"?')
+
+
+def node_capability_resources() -> dict[str, float]:
+    """{"mship_vllm": 1, ...} for every loader this node can run. MSHIP_NODE_CAPABILITIES
+    (a JSON object) overrides the probe wholesale."""
+    override = os.environ.get("MSHIP_NODE_CAPABILITIES")
+    if override:
+        return {str(name): float(qty) for name, qty in json.loads(override).items()}
+
+    resources: dict[str, float] = {}
+    for loader, module in LOADER_MODULES.items():
+        if importlib.util.find_spec(module) is not None:
+            resources[f"{RESOURCE_PREFIX}{loader}"] = 1
+    if _llama_server_available():
+        resources[f"{RESOURCE_PREFIX}{_LLAMA_SERVER_LOADER}"] = 1
+    return resources
+
+
+def deployment_capability_resources(config: ModelshipModelConfig) -> dict[str, float]:
+    """{"mship_vllm": 0.001} for the config's loader."""
+    return {f"{RESOURCE_PREFIX}{config.loader!s}": 0.001}
+
+
+def _llama_server_available() -> bool:
+    """`thin` bakes the wrapper script unconditionally but ships no binary behind
+    it, so a bare existence check on MSHIP_LLAMA_SERVER_BIN can't tell `thin`
+    apart from `cpu`/`cuda`/`metal` — resolve the wrapper's own exec target."""
+    bin_path = os.environ.get("MSHIP_LLAMA_SERVER_BIN")
+    if not bin_path or not os.path.isfile(bin_path):
+        return False
+    target = _wrapper_exec_target(bin_path)
+    if target is None:
+        return True  # not a recognized wrapper — the existence check above suffices
+    return os.path.isfile(target)
+
+
+def _wrapper_exec_target(path: str) -> str | None:
+    try:
+        with open(path) as f:
+            content = f.read()
+    except OSError:
+        return None
+    match = _WRAPPER_EXEC_RE.search(content)
+    return match.group(1) if match else None

modelship/deploy/config.py

@@ -0,0 +1,131 @@
+import os
+from pathlib import Path
+
+import yaml
+from pydantic_yaml import parse_yaml_raw_as
+
+from modelship.infer.infer_config import ModelLoader, ModelshipConfig
+from modelship.infer.model_resolver import check_model_source
+from modelship.logging import get_logger
+from modelship.utils import is_pathy
+
+logger = get_logger("startup")
+
+
+def default_config_path(config_dir: Path | None = None) -> Path:
+    """The default config/models.yaml path used absent an explicit --config."""
+    config_dir = config_dir or Path(__file__).resolve().parent.parent.parent / "config"
+    return config_dir / "models.yaml"
+
+
+def resolve_config_path(arg_path: str | None, config_dir: Path | None = None) -> str:
+    """Resolve the models.yaml to deploy.
+
+    Precedence:
+    1. An explicit ``--config`` path always wins (most specific signal); it must exist.
+    2. Otherwise the default ``config/models.yaml`` must exist.
+    """
+    if arg_path:
+        if not os.path.exists(arg_path):
+            raise FileNotFoundError(f"--config {arg_path} not found.")
+        return arg_path
+
+    default = default_config_path(config_dir)
+    if default.exists():
+        return str(default)
+
+    raise FileNotFoundError(f"{default} not found. Copy an example config from config/examples/ to config/models.yaml.")
+
+
+def load_yaml_config(arg_path: str | None) -> ModelshipConfig:
+    with open(resolve_config_path(arg_path)) as f:
+        return parse_yaml_raw_as(ModelshipConfig, f)
+
+
+def load_raw_models(arg_path: str | None) -> list[dict]:
+    """Read the user's models.yaml as raw, pre-validation dicts.
+
+    The effective-config store keeps raw dicts (not validated configs, which don't
+    round-trip through num_gpus/tp normalization), so the deploy path merges at the
+    raw-dict level; ``merge()`` validates this input before folding it in, and the
+    merged result is validated again before deploy."""
+    with open(resolve_config_path(arg_path)) as f:
+        data = yaml.safe_load(f) or {}
+    if not isinstance(data, dict):
+        raise ValueError("models.yaml: top-level document must be a mapping with a 'models' key.")
+    models = data.get("models", [])
+    if not isinstance(models, list):
+        raise ValueError("models.yaml: 'models' must be a list.")
+    return models
+
+
+def _is_whispercpp_builtin_ref(model: str) -> bool:
+    """A bare pywhispercpp built-in name (e.g. `base.en`) — no repo or path
+    separator. Validated in the actor; the driver may lack pywhispercpp."""
+    return "/" not in model and not os.path.exists(model)
+
+
+def _resolve_sherpa_onnx_source(cfg) -> None:
+    """Not an HF repo, so no `check_model_source`. A local-directory `model:` is
+    validated here at driver preflight; a bare registry name has nothing to pin
+    since the tarball fetch happens in the actor."""
+    from modelship.infer.sherpa_onnx.bundle import validate_bundle
+    from modelship.infer.sherpa_onnx.registry import REGISTRY
+
+    model = cfg.model
+    assert model is not None  # validator guarantees this for built-in loaders
+    if is_pathy(model):
+        path = os.path.expanduser(model)
+        name = os.path.basename(path.rstrip("/"))
+        entry = REGISTRY[name]  # config validation already guarantees this key exists
+        logger.info("Checking sherpa_onnx bundle for '%s': %s", cfg.name, path)
+        validate_bundle(path, entry)
+        logger.info("Checked '%s' (local bundle, no download)", cfg.name)
+    else:
+        logger.info(
+            "Skipping source check for '%s': sherpa_onnx registry model %r is fetched by the actor", cfg.name, model
+        )
+
+
+def resolve_all_model_sources(yml_conf: ModelshipConfig) -> None:
+    """Pre-flight: check every built-in-loader model's source, without
+    downloading any weight bytes.
+
+    Populates `_pinned_source` (and, for llama_server, the mmproj pin) on each
+    config in place; actual download happens per-replica in
+    `BaseInfer.ensure_downloaded`. Raises on the first failure (auth,
+    missing repo, missing file, glob-no-match) so the operator sees it before
+    any Ray actor spins up.
+
+    Note: HF_HOME / VLLM_CACHE_ROOT / FLASHINFER_CACHE_DIR are set at module
+    load time in mship_deploy.py — `huggingface_hub.HF_HOME` is latched at
+    import, so setting them later doesn't take effect.
+    """
+    for cfg in yml_conf.models:
+        if cfg.loader == ModelLoader.whispercpp and cfg.model and _is_whispercpp_builtin_ref(cfg.model):
+            # pywhispercpp resolves/downloads its own built-in models; nothing to pin here.
+            logger.info("Skipping source check for '%s': pywhispercpp built-in model %r", cfg.name, cfg.model)
+            continue
+        if cfg.loader == ModelLoader.sherpa_onnx:
+            _resolve_sherpa_onnx_source(cfg)
+            continue
+        assert cfg.model is not None  # validator guarantees this for built-in loaders
+        trust_remote_code = bool(cfg.vllm_engine_kwargs and cfg.vllm_engine_kwargs.trust_remote_code)
+        logger.info("Checking model source for '%s': %s", cfg.name, cfg.model)
+        cfg._pinned_source = check_model_source(cfg.model, trust_remote_code=trust_remote_code)
+        logger.info("Checked '%s' (revision=%s)", cfg.name, cfg._pinned_source.revision or "local")
+
+        if cfg.loader == ModelLoader.llama_server and cfg.llama_server_config and cfg.llama_server_config.mmproj:
+            logger.info("Checking mmproj source for '%s': %s", cfg.name, cfg.llama_server_config.mmproj)
+            cfg.llama_server_config._pinned_mmproj = check_model_source(
+                cfg.llama_server_config.mmproj, trust_remote_code=trust_remote_code
+            )
+
+        # GGUF is not supported on the vllm loader (vLLM 0.24 dropped in-tree
+        # GGUF). Reject early using the listed filename, before any download.
+        if cfg.loader == ModelLoader.vllm and cfg._pinned_source.resolves_to_gguf:
+            raise ValueError(
+                f"Model '{cfg.name}' resolves to a GGUF file, which the vllm loader does not support "
+                f"(vLLM 0.24 dropped in-tree GGUF). Use `loader: llama_server` for GGUF models, or point "
+                f"the vllm loader at a non-GGUF checkpoint (safetensors, or an AWQ/GPTQ/FP8 quant)."
+            )

modelship/deploy/effective_config.py

@@ -0,0 +1,131 @@
+"""Per-gateway *effective config* — the durable desired-state for deploys.
+
+Every ``mship_deploy`` invocation, whatever its mode, folds the user's input into
+the gateway's effective set (additive = union; reconcile = replace), then
+the deploy ALWAYS reconciles the live cluster to that effective set. Self-heal is
+then just "re-run the deploy": it reads the persisted effective set and reconciles
+onto an empty cluster, restoring the TRUE live set after the cluster dies — not
+just whatever the last user input happened to contain.
+
+The store holds **raw, user-equivalent model dicts**, NOT serialized validated
+configs: ``ModelshipModelConfig``'s ``num_gpus``/``tensor_parallel_size``
+normalization is not idempotent, so a dumped validated config fails (or silently
+mutates its fingerprint) on reload. Raw input dicts reload exactly as written.
+
+This is the deploy-domain layer over the generic ``modelship.state`` store.
+"""
+
+from typing import Literal
+
+from modelship.infer.infer_config import ModelshipConfig, ModelshipModelConfig
+from modelship.logging import get_logger
+from modelship.state import StateStore
+
+logger = get_logger("startup")
+
+DeployMode = Literal["additive", "reconcile"]
+
+# State-store namespace; one key per gateway: "effective/<gateway-name>".
+_NAMESPACE = "effective"
+
+
+def resolve_mode(*, reconcile: bool) -> DeployMode:
+    """Map the CLI flags to the effective-config merge verb."""
+    return "reconcile" if reconcile else "additive"
+
+
+def _deployment_name(raw: dict, gateway_name: str) -> str:
+    """Deployment name (name + fingerprint) for a raw model dict — the identity
+    key for additive de-dup and fatal-failure eviction. Validates the dict
+    (running normalization) so two raw dicts that normalize identically map to the
+    same deployment."""
+    return ModelshipModelConfig.model_validate(raw).deployment_name(gateway_name)
+
+
+def _model_name(raw: dict) -> str:
+    """Human-facing model name for a raw model dict."""
+    return ModelshipModelConfig.model_validate(raw).name
+
+
+def _identity(raw: dict, gateway_name: str) -> tuple[str, str]:
+    """(deployment_name, model_name) for a raw model dict from one validation pass."""
+    cfg = ModelshipModelConfig.model_validate(raw)
+    return cfg.deployment_name(gateway_name), cfg.name
+
+
+def merge(
+    effective_raw: list[dict],
+    input_raw: list[dict],
+    gateway_name: str,
+    mode: DeployMode,
+) -> list[dict]:
+    """Fold the user's input into the effective raw model set under *mode*.
+
+    - additive: replace-by-name — identical config (same deployment_name) is an
+      idempotent skip; a different config sharing a model name replaces the
+      existing entry for that name rather than joining it.
+    - reconcile: input replaces the effective set entirely.
+
+    Validates *input_raw* alone (not the merged result) via ModelshipConfig, so a
+    model name reused with a different config in this file is rejected before it
+    ever reaches the persisted effective set; pre-existing effective state from
+    before this rule existed is left alone.
+    """
+    to_config(input_raw)
+    if mode == "reconcile":
+        return list(input_raw)
+
+    # dep_name -> raw dict, and model_name -> its current dep_name, both built in
+    # one validation pass per dict so lookups below are O(1) instead of rescanning
+    # (and re-validating) the whole accumulated set per input entry.
+    merged: dict[str, dict] = {}
+    dep_name_by_model_name: dict[str, str] = {}
+    for m in effective_raw:
+        dep_name, model_name = _identity(m, gateway_name)
+        merged[dep_name] = m
+        dep_name_by_model_name[model_name] = dep_name
+
+    for d in input_raw:
+        dep_name, model_name = _identity(d, gateway_name)
+        if dep_name in merged:
+            continue
+        prior_dep_name = dep_name_by_model_name.get(model_name)
+        if prior_dep_name is not None:
+            del merged[prior_dep_name]
+        merged[dep_name] = d
+        dep_name_by_model_name[model_name] = dep_name
+
+    return list(merged.values())
+
+
+def deployment_names(raw_models: list[dict], gateway_name: str) -> set[str]:
+    """The deployment-name set for raw model dicts — the identity set of what's
+    under this gateway's effective management. Passed to the deploy plan so a
+    reconcile only removes deployments that WERE effective-managed (never legacy /
+    un-tracked deployments or another gateway's apps). Relies on the effective
+    config being per-gateway and the gateway being folded into each fingerprint."""
+    return {_deployment_name(d, gateway_name) for d in raw_models}
+
+
+def to_config(raw_models: list[dict]) -> ModelshipConfig:
+    """Validate raw model dicts into a ModelshipConfig for the deploy path."""
+    return ModelshipConfig.model_validate({"models": raw_models})
+
+
+def read_effective(store: StateStore, gateway_name: str) -> list[dict]:
+    """Return the persisted effective raw model set for *gateway_name* (empty if
+    none yet)."""
+    data = store.get(f"{_NAMESPACE}/{gateway_name}")
+    if not isinstance(data, dict):
+        return []
+    models = data.get("models", [])
+    if not isinstance(models, list):
+        logger.warning("Effective config for gateway %r has non-list 'models'; treating as empty.", gateway_name)
+        return []
+    return models
+
+
+def write_effective(store: StateStore, gateway_name: str, raw_models: list[dict]) -> None:
+    """Persist the effective raw model set for *gateway_name*."""
+    store.set(f"{_NAMESPACE}/{gateway_name}", {"models": raw_models})
+    logger.info("Effective config for gateway %r now has %d model(s).", gateway_name, len(raw_models))