mship 0.7.3__py3-none-any.whl

modelship/deploy/actor_options.py

@@ -0,0 +1,186 @@
+"""Ray Serve actor option construction for model deployments.
+
+Centralises the GPU-allocation decisions and the plugin-wheel runtime_env
+injection for custom-loader models. Multi-slot vLLM deploys always use a
+Ray Serve placement group (one whole-GPU bundle per slot) that vLLM
+inherits via its ray distributed executor.
+"""
+
+from __future__ import annotations
+
+import os
+import platform
+from pathlib import Path
+
+from modelship.infer.infer_config import ModelLoader, ModelshipModelConfig
+from modelship.logging import get_logger
+from modelship.utils.cache import resolve_cache_root
+
+logger = get_logger("startup")
+
+# Forwarded from the driver to each replica's runtime_env: logging vars, the gateway
+# name (metrics.py stamps every metric with it), MSHIP_METRICS so --no-metrics on
+# the driver also disables metrics in the replicas (else they'd default to on),
+# MSHIP_PREFLIGHT so --no-preflight on the driver also disables it in the replicas
+# (preflight runs inside each loader's actor __init__, not on the driver), and the
+# /v1/responses state-store tuning read inside the gateway replica's own process
+# (state.responses.ttl_seconds / state.memory._sweep_interval_s), not the driver's.
+_PASSTHROUGH_ENV_VARS = (
+    "MSHIP_LOG_LEVEL",
+    "MSHIP_LOG_FORMAT",
+    "MSHIP_LOG_TARGET",
+    "MSHIP_GATEWAY_NAME",
+    "MSHIP_METRICS",
+    "MSHIP_PREFLIGHT",
+    "MSHIP_RESPONSES_TTL_S",
+    "MSHIP_STATE_SWEEP_INTERVAL_S",
+)
+
+
+def build_passthrough_env_vars() -> dict[str, str]:
+    """Driver→replica env vars (logging, gateway name, metrics) read off the
+    driver's environment. Shared by model and gateway deployments so both
+    replicas inherit the same logging/metrics config."""
+    return {var: os.environ[var] for var in _PASSTHROUGH_ENV_VARS if os.environ.get(var) is not None}
+
+
+def build_cache_env_vars() -> dict[str, str]:
+    """Resolve HF / vLLM / FlashInfer cache dirs, all rooted at MSHIP_CACHE_DIR.
+
+    Also forwards HF_TOKEN/HF_HUB_OFFLINE when set on the driver, so an actor
+    downloading a gated/offline model has the same auth."""
+    base_cache = resolve_cache_root()
+    env_vars = {
+        "HF_HOME": os.environ.get("HF_HOME", f"{base_cache}/huggingface"),
+        "VLLM_CACHE_ROOT": os.environ.get("VLLM_CACHE_ROOT", f"{base_cache}/vllm"),
+        "FLASHINFER_CACHE_DIR": os.environ.get("FLASHINFER_CACHE_DIR", f"{base_cache}/flashinfer"),
+        # Triton JITs kernels at import for some archs
+        "TRITON_CACHE_DIR": os.environ.get("TRITON_CACHE_DIR", f"{base_cache}/triton"),
+        # vLLM's usage-stats thread writes usage_stats.json/do_not_track here
+        "VLLM_CONFIG_ROOT": os.environ.get("VLLM_CONFIG_ROOT", f"{base_cache}/vllm-config"),
+    }
+    for var in ("HF_TOKEN", "HF_HUB_OFFLINE"):
+        if os.environ.get(var) is not None:
+            env_vars[var] = os.environ[var]
+    return env_vars
+
+
+def _plugin_wheel_dir() -> Path:
+    return Path(os.environ.get("MSHIP_PLUGIN_WHEEL_DIR", ".build/plugin-wheels"))
+
+
+def resolve_plugin_wheel(plugin: str) -> Path:
+    wheel_dir = _plugin_wheel_dir()
+    normalized_name = plugin.replace("-", "_")
+    wheels = sorted(wheel_dir.glob(f"{normalized_name}-*.whl"))
+    if not wheels:
+        raise RuntimeError(
+            f"No wheel found for plugin '{plugin}' (normalized: '{normalized_name}') in {wheel_dir}. "
+            f"Build wheels with `make plugin-wheels` (or rebuild the Docker image), "
+            f"or set MSHIP_PLUGIN_WHEEL_DIR to the directory containing them."
+        )
+    # Absolute path required: Ray workers run with a different cwd
+    # (/tmp/ray/session_*/runtime_resources/.../exec_cwd), so a relative wheel
+    # path in runtime_env.pip would fail to resolve on the worker.
+    return wheels[-1].resolve()
+
+
+def _world_size(config: ModelshipModelConfig) -> int:
+    if config.loader != ModelLoader.vllm:
+        return 1
+    tp = config.vllm_engine_kwargs.tensor_parallel_size
+    pp = config.vllm_engine_kwargs.pipeline_parallel_size
+    return tp * pp
+
+
+def total_gpu_reservation(deploy_opts: dict) -> float:
+    """Sum the GPU units this deployment (actor + any PG bundles) will consume.
+
+    Used by the coordinator's resource tracker, which can't read the PG
+    bundle list as a single scalar.
+    """
+    return _total_reservation(deploy_opts, "GPU", "num_gpus")
+
+
+def total_cpu_reservation(deploy_opts: dict) -> float:
+    """Sum the CPU units this deployment (actor + any PG bundles) will consume.
+
+    For multi-slot deploys the outer actor sits in bundle 0 and its CPU
+    request is satisfied from that bundle's reservation, so summing the
+    bundles gives the correct total — same shape as the GPU helper.
+    """
+    return _total_reservation(deploy_opts, "CPU", "num_cpus")
+
+
+def _total_reservation(deploy_opts: dict, bundle_key: str, actor_key: str) -> float:
+    if "placement_group_bundles" in deploy_opts:
+        return float(sum(b.get(bundle_key, 0) for b in deploy_opts["placement_group_bundles"]))
+    return float(deploy_opts.get("ray_actor_options", {}).get(actor_key, 0) or 0)
+
+
+def build_deployment_options(config: ModelshipModelConfig, plugin_wheel: Path | None = None) -> dict:
+    """Return a kwargs dict for `Deployment.options(**...)`.
+
+    Always contains ``ray_actor_options``; for multi-slot vLLM deploys also
+    contains ``placement_group_bundles`` and ``placement_group_strategy`` so
+    Ray Serve allocates one whole-GPU bundle per slot and vLLM's ray executor
+    inherits the PG. When the model config sets ``max_ongoing_requests`` it is
+    forwarded as the per-replica Ray Serve concurrency cap.
+    """
+    env_vars = build_cache_env_vars()
+    env_vars.update(build_passthrough_env_vars())
+
+    runtime_env: dict = {"env_vars": env_vars}
+    if plugin_wheel is not None:
+        # Ship the plugin to the Ray worker via runtime_env. Ray content-hashes
+        # and caches the resulting per-job venv, so repeat deploys of the same
+        # wheel reuse the install.
+        runtime_env["pip"] = [str(plugin_wheel)]
+
+    if config.loader == ModelLoader.stable_diffusion_cpp and platform.system() != "Darwin":
+        # Off Darwin the loader's ggml backend genuinely is CPU-only. On Darwin,
+        # ggml's runtime device registry picks up Metal automatically — forcing
+        # 0 here wouldn't stop the actor from using the GPU, it would just make
+        # Ray believe it isn't, letting it co-schedule another GPU actor onto it.
+        if config.num_gpus > 0:
+            logger.warning(
+                "num_gpus=%s is ignored for model '%s': stable_diffusion_cpp loader only supports GPU on Metal.",
+                config.num_gpus,
+                config.name,
+            )
+        opts: dict = {"ray_actor_options": {"num_gpus": 0, "num_cpus": config.num_cpus, "runtime_env": runtime_env}}
+    else:
+        world_size = _world_size(config)
+        if world_size == 1:
+            # Single slot: scalar Ray allocation. Fractional num_gpus (0 < n < 1)
+            # lets Ray pack other actors onto the same physical GPU.
+            opts = {
+                "ray_actor_options": {
+                    "num_gpus": config.num_gpus,
+                    "num_cpus": config.num_cpus,
+                    "runtime_env": runtime_env,
+                }
+            }
+        else:
+            # Multi-slot: one PG bundle per slot, STRICT_PACK keeps them on the
+            # same node (NVLink). Outer actor sits in bundle 0 with 0 GPU; vLLM's
+            # ray executor reuses the PG via get_current_placement_group() and
+            # pins each worker actor to its bundle. Each bundle requests a whole
+            # GPU, so Ray spreads across distinct physical GPUs.
+            bundles = [{"GPU": 1, "CPU": config.num_cpus} for _ in range(world_size)]
+            opts = {
+                "ray_actor_options": {"num_gpus": 0, "num_cpus": config.num_cpus, "runtime_env": runtime_env},
+                "placement_group_bundles": bundles,
+                "placement_group_strategy": "STRICT_PACK",
+            }
+
+    # Per-model Ray Serve concurrency cap; only override the default when set.
+    # The reservation helpers read only the GPU/CPU keys, so this is inert there.
+    max_ongoing = config.max_ongoing_requests
+    if max_ongoing is None and config.loader == ModelLoader.llama_server:
+        parallel = config.llama_server_config.parallel if config.llama_server_config else 1
+        max_ongoing = parallel
+
+    if max_ongoing is not None:
+        opts["max_ongoing_requests"] = max_ongoing
+    return opts

modelship/deploy/config.py

@@ -0,0 +1,109 @@
+import os
+from pathlib import Path
+
+import yaml
+from pydantic_yaml import parse_yaml_raw_as
+
+from modelship.deploy.actor_options import resolve_plugin_wheel
+from modelship.infer.infer_config import ModelLoader, ModelshipConfig
+from modelship.infer.model_resolver import check_model_source
+from modelship.logging import get_logger
+
+logger = get_logger("startup")
+
+
+def default_config_path(config_dir: Path | None = None) -> Path:
+    """The default config/models.yaml path used absent an explicit --config."""
+    config_dir = config_dir or Path(__file__).resolve().parent.parent.parent / "config"
+    return config_dir / "models.yaml"
+
+
+def resolve_config_path(arg_path: str | None, config_dir: Path | None = None) -> str:
+    """Resolve the models.yaml to deploy.
+
+    Precedence:
+    1. An explicit ``--config`` path always wins (most specific signal); it must exist.
+    2. Otherwise the default ``config/models.yaml`` must exist.
+    """
+    if arg_path:
+        if not os.path.exists(arg_path):
+            raise FileNotFoundError(f"--config {arg_path} not found.")
+        return arg_path
+
+    default = default_config_path(config_dir)
+    if default.exists():
+        return str(default)
+
+    raise FileNotFoundError(f"{default} not found. Copy an example config from config/examples/ to config/models.yaml.")
+
+
+def load_yaml_config(arg_path: str | None) -> ModelshipConfig:
+    with open(resolve_config_path(arg_path)) as f:
+        return parse_yaml_raw_as(ModelshipConfig, f)
+
+
+def load_raw_models(arg_path: str | None) -> list[dict]:
+    """Read the user's models.yaml as raw, pre-validation dicts.
+
+    The effective-config store keeps raw dicts (not validated configs, which don't
+    round-trip through num_gpus/tp normalization), so the deploy path merges at the
+    raw-dict level and validates only the merged result."""
+    with open(resolve_config_path(arg_path)) as f:
+        data = yaml.safe_load(f) or {}
+    if not isinstance(data, dict):
+        raise ValueError("models.yaml: top-level document must be a mapping with a 'models' key.")
+    models = data.get("models", [])
+    if not isinstance(models, list):
+        raise ValueError("models.yaml: 'models' must be a list.")
+    return models
+
+
+def resolve_all_plugin_wheels(yml_conf: ModelshipConfig) -> dict[str, Path]:
+    """Pre-flight: resolve every referenced plugin wheel up front so a missing
+    wheel fails the whole startup before any Ray deploy is attempted."""
+    wheels: dict[str, Path] = {}
+    for cfg in yml_conf.models:
+        if cfg.loader == ModelLoader.custom and cfg.plugin and cfg.plugin not in wheels:
+            wheels[cfg.plugin] = resolve_plugin_wheel(cfg.plugin)
+    return wheels
+
+
+def resolve_all_model_sources(yml_conf: ModelshipConfig) -> None:
+    """Pre-flight: check every built-in-loader model's source, without
+    downloading any weight bytes.
+
+    Populates `_pinned_source` (and, for llama_server, the mmproj pin) on each
+    config in place; actual download happens per-replica in
+    `BaseInfer.ensure_downloaded`. Raises on the first failure (auth,
+    missing repo, missing file, glob-no-match) so the operator sees it before
+    any Ray actor spins up.
+
+    Plugins (`loader=custom`) are skipped — they manage their own download.
+
+    Note: HF_HOME / VLLM_CACHE_ROOT / FLASHINFER_CACHE_DIR are set at module
+    load time in mship_deploy.py — `huggingface_hub.HF_HOME` is latched at
+    import, so setting them later doesn't take effect.
+    """
+    for cfg in yml_conf.models:
+        if cfg.loader == ModelLoader.custom:
+            continue
+        assert cfg.model is not None  # validator guarantees this for built-in loaders
+        trust_remote_code = bool(cfg.vllm_engine_kwargs and cfg.vllm_engine_kwargs.trust_remote_code)
+        logger.info("Checking model source for '%s': %s", cfg.name, cfg.model)
+        cfg._pinned_source = check_model_source(cfg.model, trust_remote_code=trust_remote_code)
+        logger.info("Checked '%s' (revision=%s)", cfg.name, cfg._pinned_source.revision or "local")
+
+        if cfg.loader == ModelLoader.llama_server and cfg.llama_server_config and cfg.llama_server_config.mmproj:
+            logger.info("Checking mmproj source for '%s': %s", cfg.name, cfg.llama_server_config.mmproj)
+            cfg.llama_server_config._pinned_mmproj = check_model_source(
+                cfg.llama_server_config.mmproj, trust_remote_code=trust_remote_code
+            )
+
+        # GGUF is not supported on the vllm loader (vLLM 0.24 dropped in-tree
+        # GGUF). Reject early using the listed filename, before any download.
+        if cfg.loader == ModelLoader.vllm and cfg._pinned_source.resolves_to_gguf:
+            raise ValueError(
+                f"Model '{cfg.name}' resolves to a GGUF file, which the vllm loader does not support "
+                f"(vLLM 0.24 dropped in-tree GGUF). Use `loader: llama_server` for GGUF models, or point "
+                f"the vllm loader at a non-GGUF checkpoint (safetensors, or an AWQ/GPTQ/FP8 quant)."
+            )

modelship/deploy/effective_config.py

@@ -0,0 +1,101 @@
+"""Per-gateway *effective config* — the durable desired-state for deploys.
+
+Every ``mship_deploy`` invocation, whatever its mode, folds the user's input into
+the gateway's effective set (additive = union; reconcile = replace), then
+the deploy ALWAYS reconciles the live cluster to that effective set. Self-heal is
+then just "re-run the deploy": it reads the persisted effective set and reconciles
+onto an empty cluster, restoring the TRUE live set after the cluster dies — not
+just whatever the last user input happened to contain.
+
+The store holds **raw, user-equivalent model dicts**, NOT serialized validated
+configs: ``ModelshipModelConfig``'s ``num_gpus``/``tensor_parallel_size``
+normalization is not idempotent, so a dumped validated config fails (or silently
+mutates its fingerprint) on reload. Raw input dicts reload exactly as written.
+
+This is the deploy-domain layer over the generic ``modelship.state`` store.
+"""
+
+from typing import Literal
+
+from modelship.infer.infer_config import ModelshipConfig, ModelshipModelConfig
+from modelship.logging import get_logger
+from modelship.state import StateStore
+
+logger = get_logger("startup")
+
+DeployMode = Literal["additive", "reconcile"]
+
+# State-store namespace; one key per gateway: "effective/<gateway-name>".
+_NAMESPACE = "effective"
+
+
+def resolve_mode(*, reconcile: bool) -> DeployMode:
+    """Map the CLI flags to the effective-config merge verb."""
+    return "reconcile" if reconcile else "additive"
+
+
+def _deployment_name(raw: dict, gateway_name: str) -> str:
+    """Deployment name (name + fingerprint) for a raw model dict — the identity
+    key for additive de-dup and fatal-failure eviction. Validates the dict
+    (running normalization) so two raw dicts that normalize identically map to the
+    same deployment."""
+    return ModelshipModelConfig.model_validate(raw).deployment_name(gateway_name)
+
+
+def merge(
+    effective_raw: list[dict],
+    input_raw: list[dict],
+    gateway_name: str,
+    mode: DeployMode,
+) -> list[dict]:
+    """Fold the user's input into the effective raw model set under *mode*.
+
+    - additive: union — append input dicts whose deployment name isn't already
+      present (identical config = idempotent skip; same name + different config =
+      a distinct deployment the gateway round-robins, preserved as today).
+    - reconcile: input replaces the effective set entirely.
+    """
+    if mode == "reconcile":
+        return list(input_raw)
+
+    present = {_deployment_name(d, gateway_name) for d in effective_raw}
+    merged = list(effective_raw)
+    for d in input_raw:
+        name = _deployment_name(d, gateway_name)
+        if name not in present:
+            merged.append(d)
+            present.add(name)
+    return merged
+
+
+def deployment_names(raw_models: list[dict], gateway_name: str) -> set[str]:
+    """The deployment-name set for raw model dicts — the identity set of what's
+    under this gateway's effective management. Passed to the deploy plan so a
+    reconcile only removes deployments that WERE effective-managed (never legacy /
+    un-tracked deployments or another gateway's apps). Relies on the effective
+    config being per-gateway and the gateway being folded into each fingerprint."""
+    return {_deployment_name(d, gateway_name) for d in raw_models}
+
+
+def to_config(raw_models: list[dict]) -> ModelshipConfig:
+    """Validate raw model dicts into a ModelshipConfig for the deploy path."""
+    return ModelshipConfig.model_validate({"models": raw_models})
+
+
+def read_effective(store: StateStore, gateway_name: str) -> list[dict]:
+    """Return the persisted effective raw model set for *gateway_name* (empty if
+    none yet)."""
+    data = store.get(f"{_NAMESPACE}/{gateway_name}")
+    if not isinstance(data, dict):
+        return []
+    models = data.get("models", [])
+    if not isinstance(models, list):
+        logger.warning("Effective config for gateway %r has non-list 'models'; treating as empty.", gateway_name)
+        return []
+    return models
+
+
+def write_effective(store: StateStore, gateway_name: str, raw_models: list[dict]) -> None:
+    """Persist the effective raw model set for *gateway_name*."""
+    store.set(f"{_NAMESPACE}/{gateway_name}", {"models": raw_models})
+    logger.info("Effective config for gateway %r now has %d model(s).", gateway_name, len(raw_models))