haps-plugin-scheduler-runtime 0.3.0__tar.gz

haps_plugin_scheduler_runtime-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Benjamin De Zordo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

haps_plugin_scheduler_runtime-0.3.0/PKG-INFO

@@ -0,0 +1,31 @@
+Metadata-Version: 2.4
+Name: haps-plugin-scheduler-runtime
+Version: 0.3.0
+Summary: Run HAPS jobs on a French HPC center by submitting them to its scheduler over SSH
+Author: Logica Team, Universite de Rennes / IRISA
+Author-email: Benjamin De Zordo <benjamin.de-zordo@irisa.fr>
+License-Expression: MIT
+Project-URL: Homepage, https://www.irisa.fr/equipes/logica
+Keywords: haps,hpc,plugin,distributed-computing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: System :: Clustering
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+License-File: LICENSE
+Requires-Dist: haps-runtime==0.3.*
+Requires-Dist: haps-runtime-executor==0.3.*
+Requires-Dist: remotemanager
+Provides-Extra: dev
+Requires-Dist: ruff>=0.6; extra == "dev"
+Dynamic: license-file

haps_plugin_scheduler_runtime-0.3.0/haps_plugin_scheduler_runtime/__init__.py

@@ -0,0 +1,30 @@
+from .config import BatchResources, HpcConfig, SchedulerConfig
+from .job import SchedulerJob
+from .metadata import RuntimeInfo
+from .profiles import (
+    CinesProfile,
+    IdrisProfile,
+    ProfileError,
+    SchedulerProfile,
+    TgccProfile,
+)
+from .remote import CommandResult, RemoteError, RemoteExecutor
+
+__version__ = "0.3.0"
+
+__all__ = [
+    "RuntimeInfo",
+    "BatchResources",
+    "CinesProfile",
+    "CommandResult",
+    "HpcConfig",
+    "IdrisProfile",
+    "ProfileError",
+    "RemoteError",
+    "RemoteExecutor",
+    "SchedulerConfig",
+    "SchedulerJob",
+    "SchedulerProfile",
+    "TgccProfile",
+    "__version__",
+]

haps_plugin_scheduler_runtime-0.3.0/haps_plugin_scheduler_runtime/config.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import stat
+import tomllib
+from pathlib import Path
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from .profiles import ProfileError, SchedulerProfile, to_seconds
+from .remote import SAFE_ARGUMENT, check_argument
+
+
+class HpcConfig(BaseModel):
+    """The HPC centre, the account used on it, and how to reach it."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    host: str
+    user: str
+    workdir_root: str
+
+    passfile: Path | None = Field(default=None, repr=False)
+    keyfile: Path | None = Field(default=None, repr=False)
+
+    quick_timeout: int = 60
+    submit_timeout: int = 120
+    poll_interval: float = 60.0
+
+    @field_validator("host", "user")
+    @classmethod
+    def _sendable(cls, value: str) -> str:
+        return check_argument(value)
+
+    @field_validator("workdir_root")
+    @classmethod
+    def _sendable_path(cls, value: str) -> str:
+        """Refuse a work directory path the transport could not send safely."""
+        rest = value
+        if value.startswith("$"):
+            head, _, tail = value[1:].partition("/")
+            if not SAFE_ARGUMENT.match(head.strip("{}")):
+                raise ValueError(f"{value!r}: the variable name must be a plain name")
+            rest = tail
+        if rest and not SAFE_ARGUMENT.match(rest):
+            raise ValueError(
+                f"{value!r}: a path sent to the centre may only hold letters, digits and _@%+=:,./-"
+            )
+        return value
+
+    @field_validator("passfile", "keyfile")
+    @classmethod
+    def _readable_by_nobody_else(cls, value: Path | None) -> Path | None:
+        if value is None:
+            return None
+        path = value.expanduser()
+        if not path.is_file():
+            raise ValueError(f"file not found: {path}")
+        if stat.S_IMODE(path.stat().st_mode) & 0o077:
+            raise ValueError(f"{path} is readable by others - chmod 600 it before starting")
+        return path
+
+
+class BatchResources(BaseModel):
+    """What to ask the scheduler for, or the script on the centre that already asks."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    script_path: str | None = None
+
+    account: str | None = None
+    target: str | None = None
+    qos: str | None = None
+    time: str | int | None = None
+    nodes: int | None = None
+    ntasks: int | None = None
+    ntasks_per_node: int | None = None
+    cpus_per_task: int | None = None
+    gpus_per_node: int | None = None
+    hint: str | None = None
+    threads_per_core: int | None = None
+    filesystems: list[str] = Field(default_factory=list)
+
+    modules: list[str] = Field(default_factory=list)
+    module_purge: bool = False
+    environment: dict[str, str] = Field(default_factory=dict)
+
+    #: Fields a profile may turn into a directive. The rest describe the script
+    #: body, and `directives` must not leak them into the header.
+    DIRECTIVE_FIELDS: ClassVar[tuple[str, ...]] = (
+        "account",
+        "target",
+        "qos",
+        "time",
+        "nodes",
+        "ntasks",
+        "ntasks_per_node",
+        "cpus_per_task",
+        "gpus_per_node",
+        "hint",
+        "threads_per_core",
+        "filesystems",
+    )
+
+    @field_validator("script_path")
+    @classmethod
+    def _sendable(cls, value: str | None) -> str | None:
+        return None if value is None else check_argument(value)
+
+    @model_validator(mode="after")
+    def _one_mode_only(self) -> BatchResources:
+        if self.script_path is None:
+            return self
+        set_anyway = [
+            name
+            for name in (*self.DIRECTIVE_FIELDS, "modules", "module_purge", "environment")
+            if getattr(self, name) not in (None, False, [], {})
+        ]
+        if set_anyway:
+            raise ValueError(
+                "script_path means the script on the centre already carries its own "
+                f"directives; remove {', '.join(set_anyway)} from [resources]"
+            )
+        return self
+
+    def directives(self) -> dict[str, Any]:
+        """The resource fields that are set, for a profile to turn into directives."""
+        return {
+            name: getattr(self, name)
+            for name in self.DIRECTIVE_FIELDS
+            if getattr(self, name) not in (None, False, [], "")
+        }
+
+
+class SchedulerConfig(BaseModel):
+    """Everything the maintainer decides, read once at startup and never changed."""
+
+    # arbitrary_types_allowed because `profile` carries a class, not TOML data.
+    model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
+
+    hpc: HpcConfig
+    resources: BatchResources
+    profile: type[SchedulerProfile]
+
+    @field_validator("profile")
+    @classmethod
+    def _is_a_profile_class(cls, value: Any) -> type[SchedulerProfile]:
+        if not (isinstance(value, type) and issubclass(value, SchedulerProfile)):
+            raise ValueError(
+                "profile must be a SchedulerProfile subclass (the class itself, not an instance)"
+            )
+        return value
+
+    @model_validator(mode="after")
+    def _profile_accepts_these_resources(self) -> SchedulerConfig:
+        """Refuse at startup the resources the profile would reject on every job."""
+        if self.resources.script_path is not None:
+            return self  # the script on the centre carries its own header
+        directives = self.resources.directives()
+        missing = [
+            name
+            for name in self.profile.REQUIRED
+            # A REQUIRED name outside DIRECTIVE_FIELDS is derived by the profile
+            # from the job itself - job_name, chdir, output - and is never absent.
+            if name in BatchResources.DIRECTIVE_FIELDS and name not in directives
+        ]
+        if missing:
+            raise ValueError(
+                f"{self.profile.__name__} requires {', '.join(missing)} in [resources]"
+            )
+        if self.resources.time is not None:
+            try:
+                to_seconds(self.resources.time)
+            except ProfileError as exc:
+                raise ValueError(str(exc)) from exc
+        return self
+
+    @classmethod
+    def from_toml(cls, path: str | Path, *, profile: type[SchedulerProfile]) -> SchedulerConfig:
+        """Read a TOML file and validate it into a configuration."""
+        with open(path, "rb") as handle:
+            document = tomllib.load(handle)
+        return cls(profile=profile, **document)

haps_plugin_scheduler_runtime-0.3.0/haps_plugin_scheduler_runtime/job.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import replace
+from pathlib import Path, PurePosixPath
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Any
+
+from haps_runtime_executor import RuntimeJob, RuntimeJobError, RuntimeServiceError
+
+from .metadata import NO_TRACE, RuntimeInfo
+from .profiles import ProfileError, SchedulerProfile
+from .remote import RemoteError, RemoteExecutor
+from .states import (
+    LOST,
+    SACCT_FIELDS,
+    SQUEUE_FIELDS,
+    is_terminal,
+    reason,
+    slurm_infos,
+    succeeded,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+logger = logging.getLogger(__name__)
+
+#: How long the scheduler's stdout and stderr stay readable after a job ends.
+TRACE_LIFESPAN = 12 * 3600
+
+
+class SchedulerJob(RuntimeJob):
+    """One HAPS job, run as one scheduler job on an HPC centre."""
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        # The centre, as an instance: the configuration carries the class (D-09)
+        # and render / submit_argv / trace_paths are instance methods.
+        self.profile: SchedulerProfile = self.config.profile()
+        self.remote: RemoteExecutor | None = None
+        self.slurm_job_id: str | None = None
+        # This job's own directory on the centre.
+        # example: ``workdir_root / job_uuid``.
+        self.workdir: PurePosixPath | None = None
+        self.script: PurePosixPath | None = None
+        self.last_state: str | None = None
+        self._cancel_sent = False
+        # What this job has published so far. Rewritten whole on every call, so
+        # nothing depends on how deeply the server merges an update.
+        self.published = RuntimeInfo()
+
+    def __repr__(self) -> str:
+        return f"<{type(self).__name__} job={self.job_uuid} slurm={self.slurm_job_id or '-'}>"
+
+    # --- The sequence --------------------------------------------------------
+
+    def steps(self) -> tuple[tuple[str, Callable[[], None]], ...]:
+        """The ordered sequence of steps, which a subclass may reorder."""
+        return (
+            ("connect", self.connect),
+            ("setup directories", self.setup_directories),
+            ("fetch arguments", self.fetch_arguments),
+            ("fetch ebuffer inputs", self.fetch_ebinputs),
+            ("stage script", self.stage_script),
+            ("execute", self.execute),
+            ("push results", self.push_results),
+            ("push ebuffer outputs", self.push_eboutputs),
+        )
+
+    # --- Getting there -------------------------------------------------------
+
+    def connect(self) -> None:
+        """Open the SSH connection to the centre."""
+        executor = RemoteExecutor(self.config.hpc)
+        try:
+            executor.connect()
+        except RemoteError as exc:
+            raise RuntimeServiceError(f"HPC centre unreachable: {exc}") from exc
+        self.remote = executor
+
+    def setup_directories(self, path: Path | str | None = None) -> None:
+        """Create this job's own directory on the centre, named after its uuid."""
+        try:
+            root = path if path else self.remote.expand(self.config.hpc.workdir_root)
+            self.workdir = root / self.job_uuid
+            self.remote.mkdir(self.workdir)
+        except RemoteError as exc:
+            raise RuntimeJobError(
+                "the working directory could not be created on the centre",
+                detail=str(exc),
+            ) from exc
+
+    def stage_script(self) -> None:
+        """Write the batch script on the centre, or point at the one already there."""
+        resources = self.config.resources
+
+        # Slurm script already exists on HPC centre (arg script_path set)
+        if resources.script_path is not None:
+            self.script = PurePosixPath(resources.script_path)
+            if not self.remote.exists(self.script):
+                raise RuntimeJobError(
+                    f"the configured script does not exist on the centre: {self.script}",
+                    detail="[resources] script_path",
+                )
+            return
+
+        # There is no script, build slurm script
+        try:
+            text = self.profile.render(
+                resources,
+                job_uuid=self.job_uuid,
+                workdir=self.workdir,
+                core=self.script_core(),
+            )
+        except ProfileError as exc:
+            # The profile knows what is missing, not that a submitter is waiting
+            # for an answer. Translated here, where the blast radius is known -
+            # the same rule that sends `connect` to RuntimeServiceError and
+            # `mkdir` to RuntimeJobError from the one RemoteError.
+            raise RuntimeJobError(f"the batch script cannot be built: {exc}") from exc
+        self.script = self.workdir / "haps_job.sh"
+        try:
+            self.remote.write_text(self.script, text)
+        except RemoteError as exc:
+            raise RuntimeJobError(
+                "the batch script could not be written on the centre", detail=str(exc)
+            ) from exc
+
+    # Nothing a submitter sent belongs in this text: with
+    # arguments = ["data.in; rm -rf $WORK"] the second half would run under the
+    # maintainer's service account. Write such values to a file in self.workdir
+    # and have the script read it. Use self.profile.launch_cmd rather than a
+    # hard-coded srun, and the same class runs on the three centres.
+    def script_core(self) -> str:
+        """The core of the batch script, which every subclass must write."""
+        raise NotImplementedError(
+            f"{type(self).__name__} must implement script_core(): what the batch "
+            "script runs on the compute nodes"
+        )
+
+    # --- Doing the work ------------------------------------------------------
+
+    def execute(self) -> None:
+        """Submit the job, then watch it until the scheduler says it is over."""
+        self.slurm_job_id = self.submit_slurm_job()
+        self.publish(slurm={"job_id": self.slurm_job_id})
+        state: str | None = None
+        while True:
+            found = self.get_slurm_job_infos()
+            state = found.get("state")
+            if state is not None:
+                if state != self.last_state:
+                    self.last_state = state
+                    self.publish(slurm={"job_id": self.slurm_job_id, **found})
+                if is_terminal(state):
+                    break
+            if self.wait(self.config.hpc.poll_interval):
+                return
+        if not succeeded(state):
+            raise RuntimeJobError(
+                f"HPC job {self.slurm_job_id} ended in {state}: {reason(state)} - "
+                f"SchedulerClient.download_slurm_trace() has the output, for "
+                f"{TRACE_LIFESPAN // 3600} h",
+                detail=f"scheduler job {self.slurm_job_id} state={state}",
+            )
+
+    def submit_slurm_job(self) -> str:
+        try:
+            result = self.remote.run(
+                self.profile.submit_argv(self.script),
+                timeout=self.config.hpc.submit_timeout,
+            )
+        except RemoteError as exc:
+            raise RuntimeJobError(
+                "the job could not be submitted to the scheduler", detail=str(exc)
+            ) from exc
+        if not result.ok:
+            raise RuntimeJobError(
+                "the scheduler refused this job - check the resources asked for",
+                detail=f"{self.profile.submit_cmd} rc={result.returncode}: {result.stderr}",
+            )
+        try:
+            slurm_job_id = self.profile.parse_submit(result.stdout)
+        except ProfileError as exc:
+            # Accepted, then unreadable: the job may well be running and nothing
+            # can be cancelled without its id. The output goes to `detail` - it
+            # is the maintainer's clue and means nothing to a submitter.
+            raise RuntimeJobError(
+                "the scheduler accepted the job but printed no job id",
+                detail=str(exc),
+            ) from exc
+        logger.info("%s | submitted as scheduler job %s", self.log_head, slurm_job_id)
+        return slurm_job_id
+
+    def get_slurm_job_infos(self) -> dict[str, str]:
+        """Ask the scheduler about this job, once. Empty when we could not ask."""
+        # Try a squeue to slurm
+        try:
+            queued = self.remote.run(self.profile.queue_argv(self.slurm_job_id))
+        except RemoteError as exc:
+            logger.warning("%s | queue not readable, will ask again: %s", self.log_head, exc)
+            return {}
+        if queued.ok:
+            found = slurm_infos(queued.stdout, SQUEUE_FIELDS)
+            if found:
+                return found
+        # Try a sacct to slurm
+        try:
+            acct = self.remote.run(self.profile.acct_argv(self.slurm_job_id))
+        except RemoteError as exc:
+            logger.warning("%s | accounting not readable, will ask again: %s", self.log_head, exc)
+            return {}
+        if not acct.ok:
+            return {}
+        # An empty answer here is a verdict, not a silence: accounting replied and
+        # knows no such job.
+        return slurm_infos(acct.stdout, SACCT_FIELDS) or {"state": LOST}
+
+    # --- Stopping ------------------------------------------------------------
+
+    def on_stop(self) -> None:
+        """Cancel the scheduler job when this one is stopped or cancelled."""
+        self.cancel_slurm_job()
+
+    def cancel_slurm_job(self) -> None:
+        """Ask the scheduler to stop this job, at most once."""
+        if self.remote is None or self.slurm_job_id is None:
+            return  # nothing was submitted, nothing to cancel
+        if self._cancel_sent:
+            # `on_stop` runs before the terminal status, `cleanup` after: on a
+            # stop both would fire. A second scancel answers "invalid job id"
+            # and would log an error about a cancellation that worked.
+            return
+        self._cancel_sent = True
+        logger.info("%s | cancelling scheduler job %s", self.log_head, self.slurm_job_id)
+        try:
+            result = self.remote.run(self.profile.cancel_argv(self.slurm_job_id))
+        except RemoteError as exc:
+            logger.error("%s | %s not cancelled: %s", self.log_head, self.slurm_job_id, exc)
+            return
+        if not result.ok:
+            logger.error(
+                "%s | %s not cancelled: %s", self.log_head, self.slurm_job_id, result.stderr
+            )
+
+    def cleanup(self) -> None:
+        """Stop what is left running, save the trace, clear the centre, disconnect."""
+        if self.remote is None:
+            return
+        try:
+            # Anything that is not a verdict from the scheduler leaves the
+            # calculation running: an unexpected error, a refused metadata write,
+            # a failure in a later step. It would burn quota to the walltime for
+            # a result nobody collects, and `remove_workdir` below would pull the
+            # ground from under it. Cancelling first makes both go away.
+            if not is_terminal(self.last_state):
+                self.cancel_slurm_job()
+            self.push_slurm_trace()
+            self.remove_workdir()
+        finally:
+            self.remote.close()
+
+    def publish(self, **changes: Any) -> None:
+        """Add to what this job recorded, and rewrite the whole runtime section."""
+        self.published = replace(self.published, **changes)
+        self.set_runtime_metadata(self.published.to_section())
+
+    def push_slurm_trace(self) -> None:
+        """Put the scheduler's stdout and stderr in a buffer each and record where."""
+        # `script_path` set means the header is the maintainer's, so we do not
+        # know where the scheduler was told to write - probing our own naming
+        # would report "absent" when the truth is "we do not know where".
+        if self.config.resources.script_path is not None:
+            return
+        if self.slurm_job_id is None:
+            return
+        out, err = self.profile.trace_paths(self.workdir, self.job_uuid, self.slurm_job_id)
+        recorded: dict[str, str] = {}
+        for name, path in (("stdout_buffer", out), ("stderr_buffer", err)):
+            try:
+                recorded[name] = self._push_to_buffer(path)
+            except Exception:
+                logger.exception("%s | %s not saved", self.log_head, path.name)
+        if recorded:
+            try:
+                self.publish(**recorded)
+            except Exception:
+                logger.exception("%s | log buffers not recorded", self.log_head)
+            # The maintainer's log carries nothing of what the calculation
+            # printed; this is what points at the buffers that do - while they
+            # live, which is TRACE_LIFESPAN and not a minute longer.
+            logger.info("%s | trace saved: %s", self.log_head, recorded)
+
+    def _push_to_buffer(self, path: PurePosixPath) -> str:
+        """Copy one remote file into a buffer sized to it and return its uuid."""
+        with TemporaryDirectory() as folder:
+            local = Path(folder)
+            if self.remote.size_of(path) in (None, 0):
+                return NO_TRACE
+            self.remote.get(path, local)
+            staged = local / path.name
+            size = staged.stat().st_size
+            if size == 0:
+                return NO_TRACE
+            buffer = self.client.buffers.create(
+                reserved_size=size,
+                lifespan=TRACE_LIFESPAN,
+                tags=["haps-scheduler:trace", f"job::{self.job_uuid}"],
+            )
+            buffer.fill(staged)
+            # A reservation above the server's cap is trimmed in silence; only
+            # reading it back says so.
+            if buffer.reserved_size is not None and buffer.reserved_size < size:
+                logger.warning(
+                    "%s | %s truncated by the server: %d asked, %d given",
+                    self.log_head,
+                    path.name,
+                    size,
+                    buffer.reserved_size,
+                )
+            return buffer.uuid
+
+    def remove_workdir(self) -> None:
+        """Delete this job's directory on the centre."""
+        if self.remote is None or self.workdir is None:
+            return
+        # A guard on the one destructive call of this plugin: the directory is
+        # named after the job, and nothing else may be removed by accident.
+        if self.job_uuid not in str(self.workdir):
+            logger.error("%s | refusing to remove %s", self.log_head, self.workdir)
+            return
+        try:
+            self.remote.remove_tree(self.workdir)
+        except RemoteError as exc:
+            logger.error("%s | %s not removed: %s", self.log_head, self.workdir, exc)

haps_plugin_scheduler_runtime-0.3.0/haps_plugin_scheduler_runtime/metadata.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field, fields
+from datetime import datetime
+from typing import Any
+
+#: The server keeps its own event log beside what the writers post, and puts
+#: everything they write under this key. Measured, not assumed: ``extra_metadata``
+#: comes back as {"events": [...], "user_data": {"runtime": {...}, "user": {...}}}.
+USER_DATA = "user_data"
+RUNTIME = "runtime"
+NO_TRACE = "absent"
+
+#: What a microservice must carry to be usable with this plugin, and what the
+#: client half copies onto every job it submits. Tags can never be removed, so a
+#: marker posed here cannot be erased by mistake.
+PLUGIN_TAG = "plugin::scheduler"
+
+
+@dataclass(frozen=True)
+class RuntimeInfo:
+    """What a scheduler runtime records on a job, and the contract between the two halves."""
+
+    # What the scheduler said, under its own key: job_id, state, and whatever
+    # states.py is told to collect. A plain dict on purpose - adding a field there
+    # must not mean editing the contract here.
+    slurm: dict[str, Any] = field(default_factory=dict)
+    #: uuid of an ebuffer, or NO_TRACE. HAPS objects, not scheduler facts, so they
+    #: stay one level up.
+    stdout_buffer: str | None = None
+    stderr_buffer: str | None = None
+
+    def to_section(self) -> dict[str, Any]:
+        """What is set, as the sub-document to hand to set_runtime_metadata."""
+        return {k: v for k, v in asdict(self).items() if v not in (None, {})}
+
+    # Reads only the fields declared above and ignores everything else, so a job
+    # run by another runtime plugin - which writes its own keys under the same
+    # section - comes back empty rather than raising.
+    @classmethod
+    def read(cls, extra_metadata: Any) -> RuntimeInfo:
+        """What the runtime recorded on this job, empty when there is nothing."""
+        section = cls._section(extra_metadata)
+        known = {f.name for f in fields(cls)}
+        found = {k: v for k, v in section.items() if k in known and v is not None}
+        if not isinstance(found.get("slurm"), dict):
+            found.pop("slurm", None)
+        return cls(**found)
+
+    @staticmethod
+    def _section(extra_metadata: Any) -> dict[str, Any]:
+        section = written_by(extra_metadata).get(RUNTIME)
+        return section if isinstance(section, dict) else {}
+
+
+
+def from_utc(text: str | None) -> datetime | None:
+    """Read an instant the runtime wrote, or None when it is absent or damaged."""
+    if not text:
+        return None
+    try:
+        return datetime.fromisoformat(text)
+    except ValueError:
+        return None
+
+
+def written_by(extra_metadata: Any) -> dict[str, Any]:
+    """The document the writers own, under the server's own event log."""
+    # Falls back to the top level so a deployment that does not wrap keeps
+    # working: one lookup either way, and no version to keep track of.
+    if not isinstance(extra_metadata, dict):
+        return {}
+    inner = extra_metadata.get(USER_DATA)
+    return inner if isinstance(inner, dict) else extra_metadata