freesolo-flash-dev 1.2.49__tar.gz → 1.2.52__tar.gz

{freesolo_flash_dev-1.2.49 → freesolo_flash_dev-1.2.52}/CHANGELOG.md

@@ -21,6 +21,16 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Changed
 
+- An attempt's terminal artifacts (its strict marker plus `metrics.json`) are now interpreted by
+  one shared protocol instead of being decoded separately by live instance polling and by each
+  recovery path. The two decoders disagreed about identical bytes. A marker that could not be tied
+  to the attempt was a terminal failure to the live poller but indistinguishable from "no marker
+  yet" to recovery, so the same corrupt artifact produced a different story depending on which
+  layer happened to observe it; recovery now names it rather than logging silence. Neither path
+  ever adopted such a marker as completed work, and neither does now. Recovery also computed a
+  fresh observation window for the marker and then another for `metrics.json`, so the real ceiling
+  was their sum and moved with however long the first read took; both reads now share one window,
+  which makes the bound the cutoff rather than the sum.
 - CI runs the offline test suite on both supported interpreters (3.11 and 3.12) rather than
   3.11 alone, and sets `HF_HUB_OFFLINE` / `TRANSFORMERS_OFFLINE` so a test that reaches the
   network fails deterministically instead of depending on runner connectivity.

{freesolo_flash_dev-1.2.49 → freesolo_flash_dev-1.2.52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.49
+Version: 1.2.52
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.49 → freesolo_flash_dev-1.2.52}/flash/providers/_lifecycle/poll_instance.py

@@ -10,7 +10,6 @@ reference ``time`` as a module attribute so provider poll tests patch the shared
 from __future__ import annotations
 
 import contextlib
-import json
 import math
 import time
 from collections.abc import Callable
@@ -25,12 +24,19 @@ from flash.providers._lifecycle.poll import (
     BOOT_LOG_ABSENT_POLLS,
     FIRST_LIVENESS_OBSERVED_FLOOR_S,
     PollErrorTracker,
-    _attempt_int,
     heartbeat_progress_ts,
     is_training_heartbeat,
     make_say,
     surface_heartbeat,
 )
+from flash.providers._lifecycle.terminal_artifacts import (
+    INVALID_MARKER_DETAIL,
+    AttemptIdentity,
+    ProbeBudget,
+    TerminalKind,
+    read_within,
+    resolve_terminal_artifacts,
+)
 from flash.providers.artifacts.hf import worker_flagged_retriable
 from flash.providers.base import PollResult
 
@@ -51,32 +57,13 @@ _TERMINAL_REREAD_RETRIES = 6
 _TERMINAL_REREAD_WAIT_S = 5.0
 
 
-def _read_with_retries(
-    read,
-    *,
-    tries: int,
-    wait_s: float,
-    say,
-    message: str,
-    deadline_at: float | None = None,
-):
-    """Re-read an artifact with every wait capped by the absolute run deadline."""
-    value = read()
-    while value is None and tries > 0:
-        delay = wait_s
-        if deadline_at is not None:
-            remaining = remaining_seconds(deadline_at)
-            if remaining <= 0:
-                break
-            delay = min(delay, remaining)
-        say(message)
-        if delay > 0:
-            time.sleep(delay)
-        if deadline_at is not None and remaining_seconds(deadline_at) <= 0:
-            break
-        value = read()
-        tries -= 1
-    return value
+def _reread_budget(cutoff_at: float | None) -> ProbeBudget:
+    """The give-up-path reread window, read from the module constants tests patch."""
+    return ProbeBudget(
+        tries=_TERMINAL_REREAD_RETRIES,
+        wait_s=_TERMINAL_REREAD_WAIT_S,
+        cutoff_at=cutoff_at,
+    )
 
 
 @dataclass
@@ -122,69 +109,6 @@ class InstancePollAdapter:
     first_liveness_detail: Callable[[float, float], str]  # (elapsed_s, first_liveness_s) -> str
 
 
-def decode_terminal_marker(
-    raw: str,
-    *,
-    run_id: str,
-    attempt: int,
-    launch_floor: float,
-    deadline_at: float | None = None,
-) -> dict:
-    """Validate one attempt-scoped terminal marker against its durable identity."""
-    marker = json.loads(raw)
-    if not isinstance(marker, dict) or set(marker) != {
-        "attempt",
-        "error",
-        "ok",
-        "retriable",
-        "run_id",
-        "ts",
-    }:
-        raise ValueError("invalid terminal marker schema")
-    marker_attempt = marker["attempt"]
-    ts = marker["ts"]
-    error = marker["error"]
-    now = time.time()
-    if (
-        isinstance(launch_floor, bool)
-        or not isinstance(launch_floor, (int, float))
-        or not math.isfinite(launch_floor)
-        or launch_floor <= 0
-        or isinstance(now, bool)
-        or not isinstance(now, (int, float))
-        or not math.isfinite(now)
-        or now <= 0
-        or _attempt_int(marker_attempt) is None
-        or marker_attempt != _attempt_int(attempt)
-        or type(marker["run_id"]) is not str
-        or marker["run_id"] != run_id
-        or type(marker["ok"]) is not bool
-        or type(marker["retriable"]) is not bool
-        or type(error) is not str
-        or len(error) > 4096
-        or isinstance(ts, bool)
-        or not isinstance(ts, (int, float))
-        or not math.isfinite(ts)
-        or ts < launch_floor
-        or ts > now + 120.0
-        or (deadline_at is not None and ts > deadline_at)
-    ):
-        raise ValueError("invalid terminal marker identity")
-    return marker
-
-
-def _decode_terminal_marker(
-    raw: str,
-    adapter: InstancePollAdapter,
-) -> dict:
-    return decode_terminal_marker(
-        raw,
-        run_id=adapter.run_id,
-        attempt=adapter.current_attempt,
-        launch_floor=adapter.launch_ts,
-    )
-
-
 class _TerminalArtifacts:
     """Reads HF terminal artifacts (marker, done, metrics) and classifies them into a ``PollResult``.
 
@@ -208,57 +132,6 @@ class _TerminalArtifacts:
             **deadline_kwargs(reader, read_deadline_at),
         )
 
-    def finish_ok(
-        self,
-        end_ts_hint: float | str | None = None,
-        *,
-        read_deadline_at: float | None = _UNSET_DEADLINE,
-    ) -> PollResult:
-        if read_deadline_at is _UNSET_DEADLINE:
-            read_deadline_at = self._absolute_deadline
-        # the strict success marker authorizes completion, but metrics.json visibility can lag it on hf.
-        # re-read before falling back to the poll_error retry used for a success without metrics.
-        raw = _read_with_retries(
-            lambda: self._read_artifact(
-                self._adapter.metrics_reader,
-                force=True,
-                read_deadline_at=read_deadline_at,
-            ),
-            tries=_METRICS_AFTER_SUCCESS_RETRIES,
-            wait_s=_METRICS_AFTER_SUCCESS_WAIT_S,
-            say=self._say,
-            message="DONE seen but metrics.json not visible yet; waiting for HF read-after-write",
-            deadline_at=read_deadline_at,
-        )
-        if raw is None:
-            # the strict success marker has authorized completion, so missing metrics after the inline
-            # retries is a transient hf read gap, not a worker error. return the infra-retryable
-            # poll_error so the run gets its bounded infra budget instead of a hard terminal failure.
-            return PollResult(
-                False, failure="poll_error", detail="DONE without metrics.json (transient HF read)"
-            )
-        try:
-            metrics = json.loads(raw)
-        except ValueError:
-            # a present but unparseable metrics.json must not escape as a raw json decode error and
-            # bypass the poll result path. treat this marker-authorized success like missing metrics:
-            # infra-retryable poll_error, not job_failed.
-            return PollResult(
-                False,
-                failure="poll_error",
-                detail="DONE with unparseable metrics.json (transient HF read)",
-            )
-        # end_ts is the worker completion time: prefer the optional fresh done timestamp supplied by
-        # the caller, otherwise use the strict ok marker timestamp. adopt only values in [launch, now].
-        end_ts = time.time()
-        if end_ts_hint is not None:
-            with contextlib.suppress(TypeError, ValueError):
-                ts = float(end_ts_hint)
-                if self._launch_ts <= ts <= end_ts:
-                    end_ts = ts
-        self._adapter.stamp_cost_and_notes(metrics, end_ts=end_ts, launch_ts=self._launch_ts)
-        return PollResult(True, metrics=metrics)
-
     def _done_is_fresh(self, content: str) -> bool:
         # done carries only a finite completion timestamp bound to this attempt and deadline.
         try:
@@ -274,24 +147,27 @@ class _TerminalArtifacts:
             and (self._absolute_deadline is None or ts <= self._absolute_deadline)
         )
 
-    def _finish_from_ok_marker(
-        self,
-        marker: dict,
-        *,
-        read_deadline_at: float | None = _UNSET_DEADLINE,
+    def _succeed(
+        self, marker: dict, metrics: dict, *, read_deadline_at: float | None
     ) -> PollResult:
-        if read_deadline_at is _UNSET_DEADLINE:
-            read_deadline_at = self._absolute_deadline
-        # a strict ok marker authorizes completion even if done is stale or absent. prefer a fresh done
-        # timestamp when available; otherwise use the marker's completion timestamp for the wall note.
+        """Stamp a marker-authorized success with the best completion timestamp available.
+
+        A strict ok marker authorizes completion even if DONE is stale or absent, so DONE only
+        refines the wall note. Adopt a timestamp only inside [launch, now].
+        """
         d = self._read_artifact(
             self._adapter.done_reader,
             force=True,
             read_deadline_at=read_deadline_at,
         )
-        fresh = d is not None and self._done_is_fresh(d)
-        marker_ts = marker["ts"]
-        return self.finish_ok(d if fresh else marker_ts, read_deadline_at=read_deadline_at)
+        hint = d if (d is not None and self._done_is_fresh(d)) else marker["ts"]
+        end_ts = time.time()
+        with contextlib.suppress(TypeError, ValueError):
+            ts = float(hint)
+            if self._launch_ts <= ts <= end_ts:
+                end_ts = ts
+        self._adapter.stamp_cost_and_notes(metrics, end_ts=end_ts, launch_ts=self._launch_ts)
+        return PollResult(True, metrics=metrics)
 
     def _fail_from_marker(self, marker: dict) -> PollResult:
         # a real worker error fails fast unless flagged retriable in the marker or the worker heartbeat.
@@ -318,23 +194,56 @@ class _TerminalArtifacts:
         # marker-authorized success, but it cannot complete a run by itself.
         if read_deadline_at is _UNSET_DEADLINE:
             read_deadline_at = self._absolute_deadline
-        raw = self._read_artifact(
-            self._adapter.marker_reader,
-            force=force,
-            read_deadline_at=read_deadline_at,
+        resolution = resolve_terminal_artifacts(
+            AttemptIdentity(
+                run_id=self._adapter.run_id,
+                attempt=self._adapter.current_attempt,
+                launch_floor=self._adapter.launch_ts,
+            ),
+            read_marker=lambda: self._read_artifact(
+                self._adapter.marker_reader,
+                force=force,
+                read_deadline_at=read_deadline_at,
+            ),
+            read_metrics=lambda: self._read_artifact(
+                self._adapter.metrics_reader,
+                force=True,
+                read_deadline_at=read_deadline_at,
+            ),
+            budget=ProbeBudget(
+                tries=_METRICS_AFTER_SUCCESS_RETRIES,
+                wait_s=_METRICS_AFTER_SUCCESS_WAIT_S,
+                cutoff_at=read_deadline_at,
+            ),
+            say=self._say,
+            metrics_message=(
+                "DONE seen but metrics.json not visible yet; waiting for HF read-after-write"
+            ),
         )
-        if raw is None:
+        if resolution.kind is TerminalKind.ABSENT:
             return None
-        try:
-            marker = _decode_terminal_marker(raw, self._adapter)
-        except (TypeError, ValueError):
+        if resolution.kind is TerminalKind.UNVERIFIABLE:
+            return PollResult(False, failure="job_failed", detail=INVALID_MARKER_DETAIL)
+        if resolution.kind is TerminalKind.SUCCESS:
+            return self._succeed(
+                resolution.marker,
+                resolution.metrics,
+                read_deadline_at=read_deadline_at,
+            )
+        if resolution.kind is TerminalKind.PENDING:
+            # the strict success marker has authorized completion, so unreadable metrics are a
+            # transient hf read gap, not a worker error. return the infra-retryable poll_error so the
+            # run gets its bounded infra budget instead of a hard terminal failure.
             return PollResult(
                 False,
-                failure="job_failed",
-                detail="terminal marker is invalid or unverifiable",
+                failure="poll_error",
+                detail=(
+                    "DONE with unparseable metrics.json (transient HF read)"
+                    if resolution.metrics_unparseable
+                    else "DONE without metrics.json (transient HF read)"
+                ),
             )
-        if marker["ok"]:
-            return self._finish_from_ok_marker(marker, read_deadline_at=read_deadline_at)
+        marker = resolution.marker
         failure = self._fail_from_marker(marker)
         # the watchdog marker can race a successful final marker or lagging done upload. only the
         # deadline exit defers it; every other failure marker still terminates immediately.
@@ -358,13 +267,11 @@ class _TerminalArtifacts:
 
     def reread_before_giving_up(self, message: str) -> PollResult | None:
         """Bounded terminal reread shared by the dead-host, poll-outage, and stall give-up paths."""
-        terminal = _read_with_retries(
+        terminal = read_within(
             self.result,
-            tries=_TERMINAL_REREAD_RETRIES,
-            wait_s=_TERMINAL_REREAD_WAIT_S,
+            _reread_budget(self._absolute_deadline),
             say=self._say,
             message=message,
-            deadline_at=self._absolute_deadline,
         )
         if terminal is not None:
             self.surface_final_heartbeat()
@@ -376,16 +283,14 @@ class _TerminalArtifacts:
         self.deferred_deadline_failure = None
         self.surface_final_heartbeat()
         read_deadline = time.time() + (_TERMINAL_REREAD_RETRIES * _TERMINAL_REREAD_WAIT_S)
-        terminal = _read_with_retries(
+        terminal = read_within(
             lambda: self.result(
                 read_deadline_at=read_deadline,
                 defer_deadline_failure=True,
             ),
-            tries=_TERMINAL_REREAD_RETRIES,
-            wait_s=_TERMINAL_REREAD_WAIT_S,
+            _reread_budget(read_deadline),
             say=self._say,
             message="deadline reached; waiting for HF to expose any terminal DONE/marker before stalled",
-            deadline_at=read_deadline,
         )
         if terminal is not None:
             self.surface_final_heartbeat()

freesolo_flash_dev-1.2.52/flash/providers/_lifecycle/terminal_artifacts.py

@@ -0,0 +1,261 @@
+"""one terminal-artifact protocol, shared by live polling and every recovery path.
+
+an attempt's terminal evidence (the strict attempt marker plus its metrics.json) used to be decoded
+in two places that disagreed about the same bytes: the live instance poller in ``poll_instance`` and
+the reconstruction helpers in ``flash.runner.supervise.recovery``.
+
+two things diverged.
+
+* an unverifiable marker. the live poller classified it as a terminal ``job_failed``; recovery caught
+  the same ``TypeError``/``ValueError`` and returned ``None``, which its callers read as "no evidence
+  yet". neither adopts the corrupt artifact, so this is a classification split rather than a
+  correctness hole, but the same bytes still produced two different stories depending on which layer
+  happened to observe the attempt.
+* the observation window. the live poller shares one absolute read deadline across its rereads, while
+  recovery computed a fresh ~30s window for the marker and then ANOTHER fresh ~30s window for
+  metrics. because the second window started only after the first finished, the effective ceiling was
+  their SUM, and it drifted with however long the marker read happened to take.
+
+this module owns that decode exactly once. it is evidence only: it never mutates run state, never
+performs the exact-remote CAS, and never decides whether a run retries -- the attempt ledger stays
+the sole persisted authority. it answers exactly one question, as a ``TerminalKind``, and callers
+adapt that into their own vocabulary (a ``PollResult`` for the poller, a metrics dict for recovery)
+because finishing a success is provider-specific (cost stamping, done-timestamp refinement) while
+the DECISION is not.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from enum import Enum
+from typing import TypeVar
+
+from flash.providers._lifecycle.deadline import remaining_seconds
+from flash.providers._lifecycle.poll import _attempt_int
+
+T = TypeVar("T")
+
+
+class TerminalKind(Enum):
+    """What one bounded observation of an attempt's terminal artifacts established.
+
+    These five are exhaustive and mutually exclusive, so callers dispatch on the kind alone. In
+    particular UNVERIFIABLE is its OWN kind rather than a flag on FAILURE: recovery used to fold it
+    into "no evidence", and keeping it distinct is what stops that from reappearing.
+    """
+
+    # no marker is visible. the attempt may still be running, or hf has not exposed the upload yet.
+    ABSENT = "absent"
+    # a marker exists but cannot be tied to this attempt, so it cannot be trusted to speak for it.
+    # distinct from ABSENT: absence means nothing settled, this means something settled unreadably.
+    UNVERIFIABLE = "unverifiable"
+    # a strict success marker exists but its metrics are not readable yet. callers must keep
+    # observing rather than tearing down: the attempt already finished its paid work.
+    PENDING = "pending"
+    # a strict success marker plus readable metrics.
+    SUCCESS = "success"
+    # the attempt is terminally settled and did not succeed.
+    FAILURE = "failure"
+
+
+@dataclass(frozen=True)
+class TerminalResolution:
+    """One decided observation. ``marker`` is the validated marker when one was decoded."""
+
+    kind: TerminalKind
+    metrics: dict | None = None
+    marker: dict | None = None
+    # PENDING only, and diagnostic only: metrics were present but unparseable rather than missing.
+    # both are the same visibility gap behind an already-authorized completion, so this narrows an
+    # operator message and nothing else.
+    metrics_unparseable: bool = False
+
+
+ABSENT = TerminalResolution(TerminalKind.ABSENT)
+UNVERIFIABLE = TerminalResolution(TerminalKind.UNVERIFIABLE)
+
+INVALID_MARKER_DETAIL = "terminal marker is invalid or unverifiable"
+
+
+@dataclass(frozen=True)
+class AttemptIdentity:
+    """The durable identity every terminal artifact must match to speak for this attempt."""
+
+    run_id: str
+    attempt: int
+    launch_floor: float
+
+
+@dataclass(frozen=True)
+class ProbeBudget:
+    """One observation window, computed ONCE and shared by every read inside a resolution.
+
+    the marker and its metrics are two uploads of the same completion racing the same hf
+    read-after-write lag, so they are one budget, not two. ``cutoff_at`` is absolute and is threaded
+    through both reads: a marker read that burns most of the window leaves metrics the remainder
+    instead of starting a fresh allowance. that makes the ceiling the cutoff rather than the sum of
+    however many reads a path happens to make.
+
+    ``tries`` and ``wait_s`` stay per-read because provider tests patch the module constants they
+    come from; the shared absolute cutoff is what bounds the total.
+    """
+
+    tries: int
+    wait_s: float
+    cutoff_at: float | None = None
+
+
+def read_within(
+    read: Callable[[], T | None],
+    budget: ProbeBudget,
+    *,
+    say: Callable[[str], None] | None = None,
+    message: str = "",
+) -> T | None:
+    """Re-read one artifact until it is visible or the SHARED budget is spent.
+
+    Every wait is clipped to the budget's absolute cutoff, so the window cannot be restarted by a
+    later read in the same resolution.
+    """
+    value = read()
+    tries = budget.tries
+    while value is None and tries > 0:
+        delay = budget.wait_s
+        if budget.cutoff_at is not None:
+            remaining = remaining_seconds(budget.cutoff_at)
+            if remaining <= 0:
+                break
+            delay = min(delay, remaining)
+        if say is not None and message:
+            say(message)
+        if delay > 0:
+            time.sleep(delay)
+        if budget.cutoff_at is not None and remaining_seconds(budget.cutoff_at) <= 0:
+            break
+        value = read()
+        tries -= 1
+    return value
+
+
+def decode_terminal_marker(
+    raw: str,
+    *,
+    run_id: str,
+    attempt: int,
+    launch_floor: float,
+    deadline_at: float | None = None,
+) -> dict:
+    """Validate one attempt-scoped terminal marker against its durable identity.
+
+    Fails closed on anything it cannot tie to exactly this attempt: an unknown schema, another run,
+    another attempt, a timestamp before the launch this marker claims to end, or one past the bound
+    the caller allows. A marker that cannot be verified must never settle an attempt.
+    """
+    marker = json.loads(raw)
+    if not isinstance(marker, dict) or set(marker) != {
+        "attempt",
+        "error",
+        "ok",
+        "retriable",
+        "run_id",
+        "ts",
+    }:
+        raise ValueError("invalid terminal marker schema")
+    marker_attempt = marker["attempt"]
+    ts = marker["ts"]
+    error = marker["error"]
+    now = time.time()
+    if (
+        isinstance(launch_floor, bool)
+        or not isinstance(launch_floor, (int, float))
+        or not math.isfinite(launch_floor)
+        or launch_floor <= 0
+        or isinstance(now, bool)
+        or not isinstance(now, (int, float))
+        or not math.isfinite(now)
+        or now <= 0
+        or _attempt_int(marker_attempt) is None
+        or marker_attempt != _attempt_int(attempt)
+        or type(marker["run_id"]) is not str
+        or marker["run_id"] != run_id
+        or type(marker["ok"]) is not bool
+        or type(marker["retriable"]) is not bool
+        or type(error) is not str
+        or len(error) > 4096
+        or isinstance(ts, bool)
+        or not isinstance(ts, (int, float))
+        or not math.isfinite(ts)
+        or ts < launch_floor
+        or ts > now + 120.0
+        or (deadline_at is not None and ts > deadline_at)
+    ):
+        raise ValueError("invalid terminal marker identity")
+    return marker
+
+
+def _parse_metrics(raw: str | None) -> dict | None:
+    if raw is None:
+        return None
+    try:
+        metrics = json.loads(raw)
+    except (TypeError, ValueError):
+        return None
+    return metrics if isinstance(metrics, dict) else None
+
+
+def resolve_terminal_artifacts(
+    identity: AttemptIdentity,
+    *,
+    read_marker: Callable[[], str | None],
+    read_metrics: Callable[[], str | None],
+    budget: ProbeBudget,
+    say: Callable[[str], None] | None = None,
+    marker_deadline_at: float | None = None,
+    marker_wait_message: str | None = None,
+    metrics_message: str = "successful marker seen; waiting for metrics.json",
+) -> TerminalResolution:
+    """Decide what an attempt's marker plus metrics establish, under ONE shared budget.
+
+    The strict marker is the sole terminal authority. A DONE file may later refine a
+    marker-authorized success's completion timestamp, but it can never complete an attempt on its
+    own, so it is deliberately not consulted here.
+
+    ``marker_wait_message`` both enables and narrates waiting for the marker itself: live polling
+    reads it once per iteration (the loop is the retry), while recovery re-reads it inside the shared
+    budget. Passing the message is what asks for that wait, so neither can be set without the other.
+    """
+    raw = (
+        read_marker()
+        if marker_wait_message is None
+        else read_within(read_marker, budget, say=say, message=marker_wait_message)
+    )
+    if raw is None:
+        return ABSENT
+    try:
+        marker = decode_terminal_marker(
+            raw,
+            run_id=identity.run_id,
+            attempt=identity.attempt,
+            launch_floor=identity.launch_floor,
+            deadline_at=marker_deadline_at,
+        )
+    except (TypeError, ValueError):
+        return UNVERIFIABLE
+    if not marker["ok"]:
+        return TerminalResolution(TerminalKind.FAILURE, marker=marker)
+    raw_metrics = read_within(read_metrics, budget, say=say, message=metrics_message)
+    metrics = _parse_metrics(raw_metrics)
+    if metrics is None:
+        # the marker already authorized completion, so unreadable metrics are a visibility gap in the
+        # SECOND upload, not a worker error. callers keep observing within the budget instead of
+        # tearing down an attempt that already finished its paid work.
+        return TerminalResolution(
+            TerminalKind.PENDING,
+            marker=marker,
+            metrics_unparseable=raw_metrics is not None,
+        )
+    return TerminalResolution(TerminalKind.SUCCESS, metrics=metrics, marker=marker)