msdev 0.9.0__py3-none-any.whl

msdev/core/limits.py

@@ -0,0 +1,44 @@
+"""Shared practical ingress and model-import work limits."""
+
+import math
+from typing import Any
+
+MAX_JSON_RPC_REQUEST_BYTES = 64 * 1024 * 1024
+# A maximum 16 MiB workspace read expands to about 21.4 MiB in base64.
+# 32 MiB preserves that response plus its JSON envelope with finite headroom.
+MAX_RPC_RESPONSE_BYTES = 32 * 1024 * 1024
+MAX_RPC_ERROR_BYTES = 1 * 1024 * 1024
+
+DEFAULT_EXEC_TIMEOUT_SECONDS = 300.0
+EXEC_RPC_COMPLETION_GRACE_SECONDS = 10.0
+RPC_PROCESS_CLEANUP_GRACE_SECONDS = 2.0
+
+
+def validate_exec_timeout_seconds(value: Any) -> float:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, (int, float))
+        or not math.isfinite(value)
+        or (value <= 0 and value != -1)
+    ):
+        raise ValueError(
+            "timeout_seconds must be a positive finite number or -1"
+        )
+    return float(value)
+
+# Budget for conservative six-byte JSON escaping:
+# 8 MiB content * 6 + 64 maps * 2 paths * 4 KiB * 6 = 51 MiB.
+# The remaining 13 MiB covers the JSON-RPC envelope and structural overhead.
+MAX_INVENTORY_CONTENT_BYTES = 8 * 1024 * 1024
+MAX_INVENTORY_LINE_BYTES = 8 * 1024 * 1024
+
+MAX_INVENTORY_ARTIFACTS = 10_000
+MAX_INVENTORY_ALIASES_PER_ARTIFACT = 1_000
+MAX_INVENTORY_TAGS_PER_ARTIFACT = 1_000
+MAX_INVENTORY_REPLICAS_PER_ARTIFACT = 1_000
+MAX_INVENTORY_TOTAL_ALIASES = 10_000
+MAX_INVENTORY_TOTAL_TAGS = 10_000
+MAX_INVENTORY_TOTAL_REPLICAS = 10_000
+
+MAX_ROOT_MAPS = 64
+MAX_ROOT_MAP_PATH_BYTES = 4 * 1024

msdev/core/resources.py

@@ -0,0 +1,300 @@
+"""Local registry for connection nodes and execution environments."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import tempfile
+from dataclasses import asdict, dataclass, replace
+from pathlib import Path
+from typing import Any
+
+from .config import _exclusive_file_lock, config_home, parse_environment_spec
+
+
+RESOURCE_CONFIG_VERSION = 2
+_VARIABLE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def _required(value: str, label: str) -> str:
+    if not isinstance(value, str) or not value.strip() or "\0" in value:
+        raise ValueError(f"{label} must be a non-empty string without NUL")
+    return value
+
+
+def parse_variable_assignment(value: str) -> tuple[str, str]:
+    if not isinstance(value, str) or "\0" in value:
+        raise ValueError("environment assignment must be a string without NUL")
+    name, separator, item = value.partition("=")
+    if not separator or not _VARIABLE_NAME.fullmatch(name):
+        raise ValueError(f"invalid environment variable name: {name!r}")
+    return name, item
+
+
+def normalize_variables(values: tuple[str, ...] | list[str]) -> tuple[str, ...]:
+    normalized: dict[str, str] = {}
+    for value in values:
+        name, item = parse_variable_assignment(value)
+        if name in normalized:
+            raise ValueError(f"duplicate environment variable: {name}")
+        normalized[name] = item
+    return tuple(f"{name}={normalized[name]}" for name in sorted(normalized))
+
+
+@dataclass(frozen=True)
+class Node:
+    name: str
+    ssh_host: str
+    remote_bin: str = "~/.local/bin/msdevd"
+    auto_bootstrap: bool = True
+    variables: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        _required(self.name, "node name")
+        _required(self.ssh_host, "SSH host")
+        _required(self.remote_bin, "remote binary")
+        if type(self.auto_bootstrap) is not bool:
+            raise ValueError("auto_bootstrap must be a boolean")
+        object.__setattr__(
+            self,
+            "variables",
+            normalize_variables(tuple(self.variables)),
+        )
+
+
+@dataclass(frozen=True)
+class Environment:
+    name: str
+    node: str
+    runtime: str = "host"
+    container: str | None = None
+    layers: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        _required(self.name, "environment name")
+        _required(self.node, "environment node")
+        object.__setattr__(self, "layers", tuple(self.layers))
+        if self.runtime not in {"host", "docker"}:
+            raise ValueError(f"unsupported environment runtime: {self.runtime}")
+        if self.runtime == "docker":
+            _required(self.container or "", "docker container")
+        elif self.container is not None:
+            raise ValueError("host environment cannot define a container")
+        for layer in self.layers:
+            parse_environment_spec(layer)
+
+
+class _ResourceStore:
+    def __init__(self, path: Path | None = None):
+        self.path = path or config_home() / "resources.json"
+        self.lock_path = self.path.with_name(f".{self.path.name}.lock")
+
+    @staticmethod
+    def _empty() -> dict[str, Any]:
+        return {
+            "version": RESOURCE_CONFIG_VERSION,
+            "nodes": {},
+            "environments": {},
+        }
+
+    def _load_unlocked(self) -> dict[str, Any]:
+        if not self.path.exists():
+            return self._empty()
+        try:
+            value = json.loads(self.path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise ValueError(f"invalid resource config {self.path}: {exc}") from exc
+        if (
+            not isinstance(value, dict)
+            or set(value) != {"version", "nodes", "environments"}
+            or value.get("version") != RESOURCE_CONFIG_VERSION
+            or not isinstance(value.get("nodes"), dict)
+            or not isinstance(value.get("environments"), dict)
+        ):
+            raise ValueError(f"unsupported resource config schema in {self.path}")
+        nodes: dict[str, dict[str, Any]] = {}
+        environments: dict[str, dict[str, Any]] = {}
+        try:
+            for name, raw in value["nodes"].items():
+                node = Node(**raw)
+                if node.name != name:
+                    raise ValueError(f"node key {name!r} does not match its name")
+                nodes[name] = asdict(node)
+            for name, raw in value["environments"].items():
+                environment = Environment(**raw)
+                if environment.name != name:
+                    raise ValueError(
+                        f"environment key {name!r} does not match its name"
+                    )
+                if environment.node != "local" and environment.node not in nodes:
+                    raise ValueError(
+                        f"environment {name!r} references unknown node "
+                        f"{environment.node!r}"
+                    )
+                environments[name] = asdict(environment)
+        except (TypeError, ValueError) as exc:
+            raise ValueError(
+                f"invalid resource entry in {self.path}: {exc}"
+            ) from exc
+        return {
+            "version": RESOURCE_CONFIG_VERSION,
+            "nodes": nodes,
+            "environments": environments,
+        }
+
+    def load(self) -> dict[str, Any]:
+        with _exclusive_file_lock(self.lock_path):
+            return self._load_unlocked()
+
+    def _save(self, value: dict[str, Any]) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        descriptor, temporary = tempfile.mkstemp(
+            prefix=".resources-",
+            suffix=".json",
+            dir=self.path.parent,
+        )
+        try:
+            with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+                json.dump(value, handle, ensure_ascii=False, indent=2, sort_keys=True)
+                handle.write("\n")
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary, self.path)
+        finally:
+            try:
+                os.unlink(temporary)
+            except FileNotFoundError:
+                pass
+
+
+class NodeStore(_ResourceStore):
+    def add(
+        self,
+        node: Node,
+        *,
+        force: bool = False,
+        create_host_environment: bool = False,
+    ) -> None:
+        if node.name == "local":
+            raise ValueError("node name 'local' is reserved")
+        with _exclusive_file_lock(self.lock_path):
+            value = self._load_unlocked()
+            if node.name in value["nodes"] and not force:
+                raise ValueError(f"node {node.name!r} already exists")
+            if (
+                create_host_environment
+                and node.name in value["environments"]
+                and not force
+            ):
+                raise ValueError(
+                    f"environment {node.name!r} already exists"
+                )
+            value["nodes"][node.name] = asdict(node)
+            if create_host_environment:
+                value["environments"][node.name] = asdict(
+                    Environment(node.name, node.name)
+                )
+            self._save(value)
+
+    def get(self, name: str) -> Node:
+        raw = self.load()["nodes"].get(name)
+        if raw is None:
+            raise KeyError(f"unknown node: {name}")
+        return Node(**raw)
+
+    def list(self) -> list[Node]:
+        values = self.load()["nodes"]
+        return [Node(**values[name]) for name in sorted(values)]
+
+    def update_variables(
+        self,
+        name: str,
+        *,
+        set_values: dict[str, str],
+        unset_keys: tuple[str, ...],
+    ) -> Node:
+        with _exclusive_file_lock(self.lock_path):
+            value = self._load_unlocked()
+            raw = value["nodes"].get(name)
+            if raw is None:
+                raise KeyError(f"unknown node: {name}")
+            node = Node(**raw)
+            variables = dict(
+                parse_variable_assignment(item)
+                for item in node.variables
+            )
+            for key, item in set_values.items():
+                parsed_key, parsed_value = parse_variable_assignment(
+                    f"{key}={item}"
+                )
+                variables[parsed_key] = parsed_value
+            for key in unset_keys:
+                parsed_key, _ = parse_variable_assignment(f"{key}=")
+                variables.pop(parsed_key, None)
+            updated = replace(
+                node,
+                variables=tuple(
+                    f"{key}={variables[key]}" for key in sorted(variables)
+                ),
+            )
+            value["nodes"][name] = asdict(updated)
+            self._save(value)
+            return updated
+
+    def remove(self, name: str, *, cascade: bool = False) -> list[str]:
+        with _exclusive_file_lock(self.lock_path):
+            value = self._load_unlocked()
+            if name not in value["nodes"]:
+                raise KeyError(f"unknown node: {name}")
+            references = sorted(
+                environment_name
+                for environment_name, raw in value["environments"].items()
+                if raw["node"] == name
+            )
+            if references and not cascade:
+                raise ValueError(
+                    f"node {name!r} is referenced by environments: "
+                    + ", ".join(references)
+                )
+            for environment_name in references:
+                del value["environments"][environment_name]
+            del value["nodes"][name]
+            self._save(value)
+            return references
+
+
+class EnvironmentStore(_ResourceStore):
+    def add(self, environment: Environment, *, force: bool = False) -> None:
+        if environment.name == "local":
+            raise ValueError("environment name 'local' is reserved")
+        with _exclusive_file_lock(self.lock_path):
+            value = self._load_unlocked()
+            if environment.node != "local" and environment.node not in value["nodes"]:
+                raise KeyError(f"unknown node: {environment.node}")
+            if environment.name in value["environments"] and not force:
+                raise ValueError(
+                    f"environment {environment.name!r} already exists"
+                )
+            value["environments"][environment.name] = asdict(environment)
+            self._save(value)
+
+    def get(self, name: str) -> Environment:
+        if name == "local":
+            return Environment("local", "local")
+        raw = self.load()["environments"].get(name)
+        if raw is None:
+            raise KeyError(f"unknown environment: {name}")
+        return Environment(**raw)
+
+    def list(self) -> list[Environment]:
+        values = self.load()["environments"]
+        return [Environment(**values[name]) for name in sorted(values)]
+
+    def remove(self, name: str) -> None:
+        with _exclusive_file_lock(self.lock_path):
+            value = self._load_unlocked()
+            if name not in value["environments"]:
+                raise KeyError(f"unknown environment: {name}")
+            del value["environments"][name]
+            self._save(value)

msdev/core/services/__init__.py

@@ -0,0 +1,111 @@
+"""Typed contracts for shared msdev application services."""
+
+from .context import (
+    BootstrapNode,
+    ServiceContext,
+    TransportFactory,
+    create_default_transport,
+)
+from .environment import (
+    EnvironmentAddRequest,
+    EnvironmentListResult,
+    EnvironmentService,
+)
+from .execution import ExecRequest, ExecResult, ExecutionService
+from .model import (
+    ModelAuditRequest,
+    ModelDiscoverRequest,
+    ModelExportResult,
+    ModelImportRequest,
+    ModelListRequest,
+    ModelListResult,
+    ModelNodeError,
+    ModelNodeResult,
+    ModelOperationResult,
+    ModelRefRequest,
+    ModelRegisterRequest,
+    ModelRebindRequest,
+    ModelRefreshRequest,
+    ModelReplicaAddRequest,
+    ModelReplicaRemoveRequest,
+    ModelService,
+    ModelUnregisterRequest,
+    ModelUpdateRequest,
+    ModelValidateRequest,
+    ModelVerifyRequest,
+    inventory_from_jsonl,
+    inventory_to_jsonl,
+)
+from .npu import NpuListResult, NpuNodeResult, NpuService
+from .node import (
+    NodeAddRequest,
+    NodeConnectionBatch,
+    NodeConnectionResult,
+    NodeListResult,
+    NodeService,
+)
+from .workspace import (
+    WorkspaceAddRequest,
+    WorkspaceGlobResult,
+    WorkspaceGitResult,
+    WorkspaceInspectResult,
+    WorkspaceListResult,
+    WorkspacePathResult,
+    WorkspaceReadResult,
+    WorkspaceRemoveResult,
+    WorkspaceSearchResult,
+    WorkspaceService,
+)
+
+__all__ = [
+    "BootstrapNode",
+    "ExecRequest",
+    "ExecResult",
+    "ExecutionService",
+    "EnvironmentAddRequest",
+    "EnvironmentListResult",
+    "EnvironmentService",
+    "NpuListResult",
+    "NpuService",
+    "NpuNodeResult",
+    "ModelAuditRequest",
+    "ModelDiscoverRequest",
+    "ModelExportResult",
+    "ModelImportRequest",
+    "ModelListRequest",
+    "ModelListResult",
+    "ModelNodeError",
+    "ModelNodeResult",
+    "ModelOperationResult",
+    "ModelRefRequest",
+    "ModelRegisterRequest",
+    "ModelRebindRequest",
+    "ModelRefreshRequest",
+    "ModelReplicaAddRequest",
+    "ModelReplicaRemoveRequest",
+    "ModelService",
+    "ModelUnregisterRequest",
+    "ModelUpdateRequest",
+    "ModelValidateRequest",
+    "ModelVerifyRequest",
+    "ServiceContext",
+    "NodeAddRequest",
+    "NodeConnectionBatch",
+    "NodeConnectionResult",
+    "NodeListResult",
+    "NodeService",
+    "TransportFactory",
+    "WorkspaceAddRequest",
+    "WorkspaceGlobResult",
+    "WorkspaceGitResult",
+    "WorkspaceInspectResult",
+    "WorkspaceListResult",
+    "WorkspacePathResult",
+    "WorkspaceReadResult",
+    "WorkspaceRemoveResult",
+    "WorkspaceSearchResult",
+    "WorkspaceService",
+    "create_default_transport",
+    "inventory_from_jsonl",
+    "inventory_to_jsonl",
+]

msdev/core/services/context.py

@@ -0,0 +1,61 @@
+"""Dependencies shared by application services."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Protocol
+
+from ..resources import Environment, EnvironmentStore, Node, NodeStore
+from ..transport import (
+    RpcTransport,
+    SshRpcTransport,
+    UnixRpcTransport,
+    bootstrap_node,
+)
+from ..workspace import WorkspaceStore
+
+
+class TransportFactory(Protocol):
+    def __call__(
+        self,
+        node: Node | None,
+        environment: Environment | None = None,
+    ) -> RpcTransport: ...
+
+
+class BootstrapNode(Protocol):
+    def __call__(
+        self,
+        node: Node,
+        package_path: Path | None = None,
+    ) -> dict[str, Any]: ...
+
+
+def create_default_transport(
+    node: Node | None,
+    environment: Environment | None = None,
+) -> RpcTransport:
+    if node is None:
+        return UnixRpcTransport()
+    return SshRpcTransport(node, environment)
+
+
+@dataclass(frozen=True)
+class ServiceContext:
+    node_store: NodeStore = field(default_factory=NodeStore)
+    environment_store: EnvironmentStore = field(default_factory=EnvironmentStore)
+    workspace_store: WorkspaceStore = field(default_factory=WorkspaceStore)
+    create_transport: TransportFactory = create_default_transport
+    bootstrap: BootstrapNode = bootstrap_node
+
+    def transport_for_node(self, node_name: str) -> RpcTransport:
+        node = None if node_name == "local" else self.node_store.get(node_name)
+        return self.create_transport(node)
+
+    def transport_for_environment(self, environment_name: str) -> RpcTransport:
+        environment = self.environment_store.get(environment_name)
+        if environment.node == "local":
+            return self.create_transport(None, environment)
+        node = self.node_store.get(environment.node)
+        return self.create_transport(node, environment)

msdev/core/services/environment.py

@@ -0,0 +1,84 @@
+"""Execution environment registry and interactive shell service."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any
+
+from ..resources import Environment
+from .context import ServiceContext
+
+
+@dataclass(frozen=True)
+class EnvironmentAddRequest:
+    name: str
+    node: str
+    docker_container: str | None = None
+    layers: tuple[str, ...] = ()
+    force: bool = False
+
+
+@dataclass(frozen=True)
+class EnvironmentListResult:
+    environments: tuple[Environment, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "environments": [
+                asdict(environment) for environment in self.environments
+            ]
+        }
+
+
+class EnvironmentService:
+    def __init__(self, context: ServiceContext):
+        self.context = context
+
+    @staticmethod
+    def _required(value: str, label: str) -> str:
+        if not isinstance(value, str) or not value.strip():
+            raise ValueError(f"{label} must be a non-empty string")
+        return value
+
+    def add(self, request: EnvironmentAddRequest) -> Environment:
+        if type(request.force) is not bool:
+            raise ValueError("force must be a boolean")
+        if not isinstance(request.layers, (tuple, list)) or not all(
+            isinstance(layer, str) for layer in request.layers
+        ):
+            raise ValueError("layers must be a string array")
+        environment = Environment(
+            name=self._required(request.name, "environment name"),
+            node=self._required(request.node, "node name"),
+            runtime="docker" if request.docker_container else "host",
+            container=request.docker_container,
+            layers=tuple(request.layers),
+        )
+        self.context.node_store.get(environment.node)
+        self.context.environment_store.add(environment, force=request.force)
+        return environment
+
+    def list(self) -> EnvironmentListResult:
+        return EnvironmentListResult(
+            tuple(self.context.environment_store.list())
+        )
+
+    def inspect(self, name: str) -> Environment:
+        return self.context.environment_store.get(
+            self._required(name, "environment name")
+        )
+
+    def remove(self, name: str) -> str:
+        environment_name = self._required(name, "environment name")
+        self.context.environment_store.remove(environment_name)
+        return environment_name
+
+    def shell(self, name: str, *, cwd: str | None = None) -> int:
+        environment_name = self._required(name, "environment name")
+        if environment_name == "local":
+            raise ValueError("environment 'local' is already the current shell")
+        transport = self.context.transport_for_environment(environment_name)
+        operation = getattr(transport, "open_shell", None)
+        if operation is None:
+            raise TypeError("environment transport does not support a shell")
+        return int(operation(cwd=cwd))