simulo 0.26.0__py3-none-any.whl

simulo/_client/_entrypoint.py

@@ -0,0 +1,313 @@
+"""Map ``simulo run`` command-line arguments onto a function's signature.
+
+Two submit paths share this module's introspection, pointed at different
+targets:
+
+* **The entrypoint path** — ``simulo run app.py`` maps ``--flag value``
+  arguments onto the registered ``@app.entrypoint``'s parameters and
+  invokes it; the entrypoint's own ``spawn()`` calls end at exactly one
+  submit (:func:`run_entrypoint` / :func:`parse_args`). (A direct ``python
+  app.py`` never reaches this module: :meth:`App.entrypoint` prints the
+  exact equivalent ``simulo run`` command to stderr and exits 2.)
+* **The no-entrypoint path** — with no ``@app.entrypoint`` declared, the
+  CLI maps the same ``--flag value`` arguments directly onto the chosen
+  ``@app.job``'s own signature (:func:`parse_explicit_args`) and spawns it.
+  Only EXPLICITLY-passed flags are returned there: a defaulted-but-unpassed
+  parameter must stay out of the submitted ``args`` so an upgraded client
+  submits the byte-identical package (same ``canonical_args`` → same
+  ``package_id``) for the no-flags case it always produced ``args={}`` for.
+
+An argument parser is built from the target's signature: each parameter
+``foo_bar`` becomes ``--foo-bar`` (dash for underscore), coerced from the
+parameter's default type, then its annotation. A parameter without a default is
+required. This keeps authoring frictionless — ``def main(num_envs: int = 4096)``
+just works as ``--num-envs 4096`` — with no argparse boilerplate in the app file.
+"""
+
+from __future__ import annotations
+
+import argparse
+import inspect
+import types
+import typing
+from typing import Any, Callable, Collection, Mapping, Optional, Sequence
+
+_STR_TO_TYPE: Mapping[str, type] = {"int": int, "float": float, "bool": bool, "str": str}
+
+_SKIP_KINDS = (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
+
+#: The only types a ``--flag value`` string can be conclusively coerced to.
+_SCALAR_TYPES: tuple[type, ...] = (int, float, bool, str)
+
+#: Sentinel default for :func:`parse_explicit_args`'s parser: a parameter whose
+#: parsed value is still this object was never passed on the command line, so it
+#: is omitted from the returned kwargs (the job body applies its own default at
+#: execution). ``None`` cannot play this role — it is a legitimate flag value.
+_UNSET: Any = object()
+
+
+def _param_type(param: inspect.Parameter) -> type:
+    """Best-effort scalar type for a parameter: default's type, then annotation.
+
+    A concrete (non-``None``) default is the most reliable signal — ``= 4096``
+    means ``int``. Otherwise fall back to the annotation (a real type, or a PEP
+    563 string like ``"int"``). Defaults to ``str`` when nothing is conclusive.
+    """
+    default = param.default
+    if default is not inspect.Parameter.empty and default is not None:
+        return type(default)
+    ann = param.annotation
+    if isinstance(ann, type):
+        return ann
+    if isinstance(ann, str):
+        return _STR_TO_TYPE.get(ann, str)
+    return str
+
+
+def _str_to_bool(raw: str) -> bool:
+    """Parse a CLI ``--flag value`` boolean (``true``/``1``/``yes`` → ``True``)."""
+    lowered = raw.strip().lower()
+    if lowered in ("true", "1", "yes", "y", "on"):
+        return True
+    if lowered in ("false", "0", "no", "n", "off"):
+        return False
+    raise argparse.ArgumentTypeError(f"expected a boolean (true/false), got {raw!r}")
+
+
+def flag_for_param(name: str) -> str:
+    """The ``--flag`` a signature parameter ``name`` maps to (dash for underscore)."""
+    return "--" + name.replace("_", "-")
+
+
+def mappable_params(fn: Callable[..., object]) -> list[tuple[str, inspect.Parameter]]:
+    """The parameters :func:`build_parser` maps to ``--flags``.
+
+    ``VAR_POSITIONAL`` (``*args``) and ``VAR_KEYWORD`` (``**kwargs``) are
+    skipped, exactly as ``build_parser`` skips them — signature guards must use
+    the same filter, or a harmless ``def train(**job)`` would be rejected for a
+    flag it never gets.
+    """
+    return [(name, param) for name, param in inspect.signature(fn).parameters.items() if param.kind not in _SKIP_KINDS]
+
+
+def _strip_optional_annotation(ann: Any) -> Any:
+    """Unwrap ``Optional[X]`` / ``X | None`` / ``Union[X, None]`` to ``X``.
+
+    Handles both representations of each spelling: under ``from __future__
+    import annotations`` (which every shipped scaffold enables) an annotation
+    is a STRING like ``"int | None"``, ``"Optional[int]"``, or
+    ``"Union[int, None]"``; without it, a typing object (whose ``Optional``
+    and ``Union`` spellings are the same object, handled by ``get_origin``).
+    All are common, deliberate ways to declare "this scalar, or unset" — the
+    wrapper must not hide the scalar, and no SPELLING may work in one
+    representation and fail in the other. Anything that is not exactly "one
+    type or None" is returned unchanged.
+    """
+    if isinstance(ann, str):
+        text = ann.strip()
+        if text.startswith("typing."):
+            text = text[len("typing.") :]
+        if text.startswith("Optional[") and text.endswith("]"):
+            return text[len("Optional[") : -1].strip()
+        if text.startswith("Union[") and text.endswith("]"):
+            # Naive comma split is safe here: a nested generic like
+            # "Union[dict[str, int], None]" yields 3 parts and falls through
+            # unchanged — its inner type is not a scalar anyway.
+            parts = [part.strip() for part in text[len("Union[") : -1].split(",")]
+            non_none = [part for part in parts if part != "None"]
+            if len(parts) == 2 and len(non_none) == 1:
+                return non_none[0]
+            return ann
+        if "|" in text:
+            parts = [part.strip() for part in text.split("|")]
+            non_none = [part for part in parts if part != "None"]
+            if len(parts) == 2 and len(non_none) == 1:
+                return non_none[0]
+        return ann
+    origin = typing.get_origin(ann)
+    if origin is typing.Union or origin is types.UnionType:
+        args = [arg for arg in typing.get_args(ann) if arg is not type(None)]
+        if len(args) == 1:
+            return args[0]
+    return ann
+
+
+def conclusive_param_type(param: inspect.Parameter) -> Optional[type]:
+    """The scalar type a ``--flag`` value is conclusively coerced to, or ``None``.
+
+    Unlike :func:`_param_type` — whose ``str`` FALLBACK is fine for an
+    entrypoint (the entrypoint body runs client-side and can fix things up) —
+    the no-entrypoint job path submits the coerced value straight into the
+    job's ``args``, so a guess is a silent wrong submission: ``tags: list = []``
+    would coerce ``--tags release`` through ``list("release")`` into
+    ``["r", "e", "l", ...]``, and a ``Path``/``set``/enum value would crash
+    ``json.dumps`` at manifest time. ``None`` here means "reject at submit".
+
+    Conclusive: a non-``None`` scalar default (its type), or an ``int`` /
+    ``float`` / ``bool`` / ``str`` annotation — real type, PEP 563 string, or
+    either wrapped in ``Optional[...]`` / ``| None``. An unannotated parameter
+    stays ``str`` (the documented text fallback, unchanged from the entrypoint
+    convention).
+    """
+    default = param.default
+    if default is not inspect.Parameter.empty and default is not None:
+        return type(default) if type(default) in _SCALAR_TYPES else None
+    ann = param.annotation
+    if ann is inspect.Parameter.empty:
+        return str
+    ann = _strip_optional_annotation(ann)
+    if isinstance(ann, type) and ann in _SCALAR_TYPES:
+        return ann
+    if isinstance(ann, str) and ann in _STR_TO_TYPE:
+        return _STR_TO_TYPE[ann]
+    return None
+
+
+def unmappable_params(fn: Callable[..., object]) -> list[tuple[str, str, str, bool]]:
+    """``(name, kind, why, has_default)`` for parameters the CLI cannot map.
+
+    ``kind`` is ``"positional-only"`` (the worker executes ``fn(**args)``, so a
+    keyword can never reach it) or ``"type"`` (no conclusive scalar coercion —
+    see :func:`conclusive_param_type`).
+
+    ``has_default`` is the caller's rejection policy input: without a default
+    the parameter is required-but-unmappable — the submit can never succeed, so
+    it is rejected always. WITH a default the defaults-only submit works today
+    (``fn(**{})`` never touches the parameter), so callers reject only when a
+    flag actually targets it — a blanket rejection would break submits that
+    work on shipped 0.14.x clients.
+    """
+    result: list[tuple[str, str, str, bool]] = []
+    for name, param in mappable_params(fn):
+        has_default = param.default is not inspect.Parameter.empty
+        if param.kind is inspect.Parameter.POSITIONAL_ONLY:
+            result.append((name, "positional-only", "declared positional-only (before a '/')", has_default))
+            continue
+        if conclusive_param_type(param) is not None:
+            continue
+        if param.default is not inspect.Parameter.empty and param.default is not None:
+            why = f"default {param.default!r} of non-scalar type {type(param.default).__name__}"
+        elif param.annotation is not inspect.Parameter.empty:
+            why = f"annotation {param.annotation!r} is not a scalar type"
+        else:  # pragma: no cover - unreachable: no annotation + no default resolves to str
+            why = "no usable type"
+        result.append((name, "type", why, has_default))
+    return result
+
+
+def build_parser(
+    fn: Callable[..., object],
+    *,
+    sentinel_defaults: bool = False,
+    prog: Optional[str] = None,
+    exclude: Collection[str] = (),
+) -> argparse.ArgumentParser:
+    """Build an ``argparse`` parser from a function's signature.
+
+    With ``sentinel_defaults=False`` (the entrypoint path — behavior unchanged),
+    a defaulted parameter's parser default is the signature's own default, so
+    ``parse_args`` yields EVERY parameter, and values coerce via
+    :func:`_param_type` (``str`` fallback included). With
+    ``sentinel_defaults=True`` (the no-entrypoint job path), defaulted
+    parameters default to the private ``_UNSET`` sentinel instead — so
+    :func:`parse_explicit_args` can tell an explicitly-passed flag from an
+    untouched default — and values coerce via :func:`conclusive_param_type`
+    (callers must have rejected inconclusive signatures first). ``prog`` names
+    the parser in ``-h`` output (e.g. ``"simulo run app.py"``); default is the
+    function's own name, as before.
+
+    ``exclude`` drops the named parameters from the parser entirely — the
+    no-entrypoint path passes its DEFERRED unmappable-but-defaulted parameters
+    here (see :func:`unmappable_params`), so a flag targeting one — even via
+    an argparse abbreviation the caller's exact-token scan cannot see — fails
+    at submit as an unrecognized argument rather than being mis-parsed.
+    """
+    sig = inspect.signature(fn)
+    doc = (inspect.getdoc(fn) or "").strip().splitlines()
+    parser = argparse.ArgumentParser(
+        prog=prog if prog is not None else getattr(fn, "__name__", "entrypoint"),
+        description=doc[0] if doc else None,
+    )
+    for name, param in sig.parameters.items():
+        if param.kind in _SKIP_KINDS or name in exclude:
+            continue
+        flag = flag_for_param(name)
+        required = param.default is inspect.Parameter.empty
+        if required:
+            default = None
+        elif sentinel_defaults:
+            default = _UNSET
+        else:
+            default = param.default
+        if sentinel_defaults:
+            resolved = conclusive_param_type(param)
+            typ = resolved if resolved is not None else _param_type(param)
+        else:
+            typ = _param_type(param)
+        if typ is bool:
+            parser.add_argument(
+                flag,
+                dest=name,
+                type=_str_to_bool,
+                required=required,
+                default=default,
+                metavar="BOOL",
+            )
+        else:
+            parser.add_argument(
+                flag,
+                dest=name,
+                type=typ,
+                required=required,
+                default=default,
+            )
+    return parser
+
+
+def parse_args(fn: Callable[..., object], argv: Sequence[str]) -> dict[str, Any]:
+    """Parse ``argv`` against ``fn``'s signature into a keyword-argument dict."""
+    parser = build_parser(fn)
+    namespace = parser.parse_args(list(argv))
+    sig = inspect.signature(fn)
+    kwargs: dict[str, Any] = {}
+    for name, param in sig.parameters.items():
+        if param.kind in _SKIP_KINDS:
+            continue
+        kwargs[name] = getattr(namespace, name)
+    return kwargs
+
+
+def parse_explicit_args(
+    fn: Callable[..., object],
+    argv: Sequence[str],
+    *,
+    prog: Optional[str] = None,
+    exclude: Collection[str] = (),
+) -> dict[str, Any]:
+    """Parse ``argv`` against ``fn``'s signature; return ONLY explicitly-passed kwargs.
+
+    The no-entrypoint submit path maps CLI flags onto a ``@app.job``'s own
+    signature with this: a parameter the user did not pass stays OUT of the
+    returned dict (the job body applies its own default on the worker), so the
+    no-flags case still submits ``args={}`` — exactly what pre-mapping clients
+    always submitted — and ``package_id`` is unchanged across the upgrade.
+    A parameter without a default is required, exactly as on the entrypoint
+    path. ``exclude`` names parameters left out of the parser and the result
+    (the deferred unmappable-but-defaulted ones — see :func:`build_parser`).
+    """
+    parser = build_parser(fn, sentinel_defaults=True, prog=prog, exclude=exclude)
+    namespace = parser.parse_args(list(argv))
+    sig = inspect.signature(fn)
+    kwargs: dict[str, Any] = {}
+    for name, param in sig.parameters.items():
+        if param.kind in _SKIP_KINDS or name in exclude:
+            continue
+        value = getattr(namespace, name)
+        if value is not _UNSET:
+            kwargs[name] = value
+    return kwargs
+
+
+def run_entrypoint(fn: Callable[..., object], argv: Sequence[str]) -> object:
+    """Parse ``argv`` for ``fn`` and invoke it (the single submit happens inside)."""
+    return fn(**parse_args(fn, argv))

simulo/_client/_mounts.py

@@ -0,0 +1,25 @@
+"""Shared naming convention for mount environment variables.
+
+The backend runner (follow-up PR) exports one env var per mount before invoking
+a job body; ``Volume.path`` / ``Asset.path`` read them back. Centralising the
+key derivation here keeps the producer and consumer in lock-step.
+"""
+
+import re
+
+_NON_ALNUM = re.compile(r"[^A-Za-z0-9]+")
+
+
+def _sanitize(segment: str) -> str:
+    """Upper-case, collapse runs of non-alphanumerics to ``_``, strip edges."""
+    return _NON_ALNUM.sub("_", segment).strip("_").upper()
+
+
+def volume_env_key(name: str) -> str:
+    """Env var the runner sets to a volume's local mount path."""
+    return f"SIMULO_VOLUME_{_sanitize(name)}"
+
+
+def asset_env_key(uri: str) -> str:
+    """Env var the runner sets to an asset's local (read-only) mount path."""
+    return f"SIMULO_ASSET_{_sanitize(uri)}"

simulo/_client/_runner.py

@@ -0,0 +1,186 @@
+"""Submit handle for a packaged job — the result of ``JobFunction.spawn``.
+
+Two submit modes now share this one handle type:
+
+* **Local-disk submit** (no credentials, no ``SIMULO_API_URL``/``SIMULO_ENV``):
+  the thin client's only job is to *write the package* to disk; it never trains
+  and never invokes the backend. ``.get()`` raises :class:`JobResultUnavailable`
+  pointing at the package on disk and the execute command
+  (``simulo-backend run-package``), exactly as before this wave.
+* **Cloud submit** (credentials present, or ``SIMULO_API_URL``/``SIMULO_ENV``
+  set): ``spawn`` has already uploaded the package and created a job on the
+  control plane, so the handle carries the SERVER-assigned ``job_id`` plus the
+  base URL/token to observe it. ``.get()`` polls the jobs API client to a
+  terminal status and returns the result on success, or raises
+  :class:`JobFailedError` (with a
+  ``simulo logs <id>`` hint) on failure/cancellation.
+
+The thin client keeps **no** import of and no ``find_spec`` dependency on
+``simulo.backend`` — submit is fully decoupled from execution either way.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any, NoReturn, Optional
+
+from simulo.interfaces.ids import JobId, JobPublicId
+from simulo.interfaces.platform.enums import JobStatus
+from simulo.interfaces.platform.runs import JOB_SCOPE_MINE, TERMINAL_JOB_STATUSES
+
+#: Poll cadence for cloud ``.get()``. Module-level so tests can shrink it.
+_POLL_INTERVAL_S = 1.0
+
+_TERMINAL_STATUS_STRINGS = frozenset(str(status) for status in TERMINAL_JOB_STATUSES)
+_COMPLETED_STATUS = str(JobStatus.COMPLETED)
+
+
+class JobFailedError(RuntimeError):
+    """A cloud job reached a terminal ``failed``/``cancelled`` status.
+
+    Also reserved for the executor (the ``simulo-backend`` runner) to signal a
+    failed job body in local-disk mode, where the thin client never raises it
+    itself (submitting cannot fail a job because it never runs one there).
+    """
+
+
+class JobResultUnavailable(NotImplementedError):
+    """``JobHandle.get()`` was called in local-disk submit mode.
+
+    Submitting writes the package to disk; it does not execute the job, so
+    there is no result to return. Raised with the package path and the command
+    that *does* execute it (the ``simulo-backend`` runner).
+    """
+
+
+class JobTimeoutError(TimeoutError):
+    """``JobHandle.get(timeout=...)`` elapsed before the job reached a terminal status."""
+
+
+class JobHandle:
+    """Handle to a submitted job — local-disk package, or a cloud job id.
+
+    Returned by :meth:`JobFunction.spawn`. In cloud mode it carries the
+    server-assigned machine ``job_id``, human-facing ``public_id``, and enough
+    context (``base_url`` / ``token``) to poll it; ``execute_hint()`` still names the local package path
+    (preserved for debugging) but ``get()`` talks to the platform instead of
+    raising.
+    """
+
+    def __init__(
+        self,
+        job_id: JobId,
+        package_path: Path,
+        job_name: str,
+        *,
+        public_id: Optional[JobPublicId] = None,
+        cloud: bool = False,
+        base_url: Optional[str] = None,
+        token: Optional[str] = None,
+    ) -> None:
+        self._job_id = job_id
+        self._public_id = public_id
+        self._package_path = package_path
+        self._job_name = job_name
+        self._cloud = cloud
+        self._base_url = base_url
+        self._token = token
+
+    @property
+    def job_id(self) -> JobId:
+        """Opaque id for this submission — server-assigned in cloud mode."""
+        return self._job_id
+
+    @property
+    def public_id(self) -> Optional[JobPublicId]:
+        """Human-facing Job ID in cloud mode; ``None`` for local package-only submits."""
+        return self._public_id
+
+    @property
+    def package_path(self) -> Path:
+        """Directory the package was written to (``<out>/<bundle_dir_name(package_id)>/``)."""
+        return self._package_path
+
+    @property
+    def job_name(self) -> str:
+        """The ``@app.job`` recorded as submitted in the package manifest."""
+        return self._job_name
+
+    @property
+    def is_cloud(self) -> bool:
+        """``True`` when this job was submitted to the cloud (not local-disk)."""
+        return self._cloud
+
+    @property
+    def base_url(self) -> Optional[str]:
+        """The API base URL this job was submitted against (cloud mode only)."""
+        return self._base_url
+
+    @property
+    def token(self) -> Optional[str]:
+        """The bearer token to observe this job with (cloud mode only)."""
+        return self._token
+
+    def execute_hint(self) -> str:
+        """The command that executes this submitted package locally (local-disk mode)."""
+        return f"simulo-backend run-package {self._package_path} --job {self._job_name}"
+
+    def logs_hint(self) -> str:
+        """The command that streams this job's logs (cloud mode)."""
+        if self._public_id is None:
+            return "simulo jobs"
+        return f"simulo logs {self._public_id} --follow"
+
+    def get(self, timeout: Optional[float] = None) -> Any:
+        """Return the job's result.
+
+        Cloud mode: poll to a terminal status (optionally bounded by
+        *timeout* seconds; ``None`` waits indefinitely), returning the result
+        JSON on ``completed`` or raising :class:`JobFailedError` on
+        ``failed``/``cancelled``. Local-disk mode: raises
+        :class:`JobResultUnavailable` — submitting never executes a job there.
+        """
+        if not self._cloud:
+            self._raise_local_disk_unavailable()
+        return self._get_cloud(timeout)
+
+    def _get_cloud(self, timeout: Optional[float]) -> Any:
+        from simulo._client.jobs_api import JobsApiClient
+
+        assert self._base_url is not None, "cloud JobHandle must carry a base_url"
+        client = JobsApiClient(self._base_url, token=self._token)
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            record = client.get_job(str(self._job_id), scope=JOB_SCOPE_MINE)
+            status = str(record.get("status"))
+            if status in _TERMINAL_STATUS_STRINGS:
+                if status == _COMPLETED_STATUS:
+                    return client.get_result(str(self._job_id), scope=JOB_SCOPE_MINE)
+                exit_code = record.get("exit_code")
+                raise JobFailedError(
+                    f"{self._display_name()} {status} (exit_code={exit_code}).\n"
+                    f"  View its logs with:\n      {self.logs_hint()}"
+                )
+            if deadline is not None and time.monotonic() >= deadline:
+                raise JobTimeoutError(
+                    f"{self._display_name()} did not reach a terminal status within "
+                    f"{timeout}s (still {status!r}). Check it with:\n      {self.logs_hint()}"
+                )
+            time.sleep(_POLL_INTERVAL_S)
+
+    def _display_name(self) -> str:
+        """A user-facing label that never falls back to the machine UUID."""
+        if self._public_id is None:
+            return f"Job {self._job_name!r}"
+        return f"Job {self._job_name!r} ({self._public_id})"
+
+    def _raise_local_disk_unavailable(self) -> NoReturn:
+        raise JobResultUnavailable(
+            f"No result is available locally for job {self._job_name!r}: submitting writes the "
+            f"package to disk, it does not execute the job.\n"
+            f"  package: {self._package_path}\n"
+            f"  execute it with the backend (the sole executor):\n"
+            f"      {self.execute_hint()}\n"
+            f"(Run `simulo login` to submit to the cloud instead, where get() streams the real result.)"
+        )