freesolo-flash-dev 0.2.25__py3-none-any.whl

flash/server/environment_registry.py

@@ -0,0 +1,102 @@
+"""Best-effort reporting of published Flash environments to the Freesolo backend."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+import urllib.error
+import urllib.request
+
+from .auth import INTERNAL_KEY_ENV, freesolo_base_url
+
+_LOG = logging.getLogger("flash.server.environments")
+_TIMEOUT_S = 10.0
+_PATH = "/api/flash/environments/internal"
+_USE_PATH = "/api/flash/environments/use/internal"
+_DEFAULT_HUB_REPO = "freesolo-co/environment-hub"
+_DEFAULT_HUB_REF = "main"
+
+
+def record_published_environment(*, slug: str, name: str, key: dict) -> bool:
+    """Persist Hub metadata in the platform backend.
+
+    The GitHub publish is the source of truth for the environment package. This
+    metadata write exists so the web UI can list Flash environments, so it is
+    deliberately best-effort and never blocks `flash env push`.
+    """
+    internal_key = os.environ.get(INTERNAL_KEY_ENV)
+    org_id = str(key.get("org_id") or "").strip()
+    if not internal_key or not org_id:
+        return False
+
+    body = {
+        "orgId": org_id,
+        "slug": slug,
+        "name": name,
+        "hubRepo": _DEFAULT_HUB_REPO,
+        "hubRef": _DEFAULT_HUB_REF,
+        "hubPath": f"{slug}/environment.py",
+        "publishedByUserId": key.get("user_id"),
+        "apiKeyId": key.get("api_key_id"),
+        "metadata": {"source": "flash.env.push"},
+    }
+    req = urllib.request.Request(
+        f"{freesolo_base_url()}{_PATH}",
+        data=json.dumps(body).encode("utf-8"),
+        method="POST",
+        headers={
+            "Authorization": f"Bearer {internal_key}",
+            "Content-Type": "application/json",
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=_TIMEOUT_S) as resp:
+            return 200 <= resp.status < 300
+    except urllib.error.HTTPError as exc:
+        detail = ""
+        with contextlib.suppress(Exception):
+            detail = exc.read().decode("utf-8", "replace")[:500]
+        _LOG.warning(
+            "failed to record published environment %s: HTTP %s %s",
+            slug,
+            exc.code,
+            detail,
+        )
+    except (urllib.error.URLError, OSError) as exc:
+        _LOG.warning("failed to record published environment %s: %s", slug, exc)
+    return False
+
+
+def record_environment_use(*, slug: str, run_id: str, key: dict) -> bool:
+    internal_key = os.environ.get(INTERNAL_KEY_ENV)
+    org_id = str(key.get("org_id") or "").strip()
+    if not internal_key or not org_id:
+        return False
+    req = urllib.request.Request(
+        f"{freesolo_base_url()}{_USE_PATH}",
+        data=json.dumps({"orgId": org_id, "slug": slug, "runId": run_id}).encode("utf-8"),
+        method="POST",
+        headers={
+            "Authorization": f"Bearer {internal_key}",
+            "Content-Type": "application/json",
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=_TIMEOUT_S) as resp:
+            return 200 <= resp.status < 300
+    except urllib.error.HTTPError as exc:
+        detail = ""
+        with contextlib.suppress(Exception):
+            detail = exc.read().decode("utf-8", "replace")[:500]
+        _LOG.warning(
+            "failed to record environment use %s for run %s: HTTP %s %s",
+            slug,
+            run_id,
+            exc.code,
+            detail,
+        )
+    except (urllib.error.URLError, OSError) as exc:
+        _LOG.warning("failed to record environment use %s for run %s: %s", slug, run_id, exc)
+    return False

flash/server/envs.py

@@ -0,0 +1,360 @@
+"""Managed Freesolo environment publishing.
+
+``POST /v1/envs`` accepts a packaged Freesolo environment and uploads it to the
+managed environment hub. The returned id is a Freesolo environment slug
+(``namespace/name``) that Flash resolves internally.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import os
+import re
+import shutil
+import subprocess
+import tarfile
+import tempfile
+import time
+import urllib.parse
+from pathlib import Path
+
+_MAX_UPLOAD_BYTES = 64 * 1024 * 1024
+_MAX_UNCOMPRESSED_BYTES = 256 * 1024 * 1024
+_MAX_MEMBERS = 5000
+_DEFAULT_GITHUB_REPO = "freesolo-co/environment-hub"
+_GITHUB_BRANCH = "main"
+_DEFAULT_ENVIRONMENT_FILE = "environment.py"
+_BLOCKED_TOP_LEVEL_PATHS = {
+    ".github",
+    ".git",
+    "source",
+}
+_GIT_TIMEOUT_S = 180
+_GIT_PUSH_RETRY_DELAYS_SECONDS = (2.0, 5.0)
+
+
+def _human_mb(n: int) -> str:
+    return f"{n / (1024 * 1024):.0f} MB"
+
+
+class EnvPublishError(Exception):
+    def __init__(self, message: str, *, status: int = 400):
+        super().__init__(message)
+        self.status = status
+
+
+# Reserved hub namespace for the operator/internal service key. Its /v1/me identity is synthetic
+# and shared (no per-user email), so it can't derive a namespace from an email like a user key does —
+# but it IS a trusted identity that can submit runs and read everything, so it must be able to
+# publish environments too. Matches the slug `internal@freesolo.co` would yield (see
+# flash.server.db.ensure_internal_key) so the namespace is stable regardless of the row's email.
+_INTERNAL_NAMESPACE = "internal-freesolo-co"
+
+
+def namespace_for(key: dict) -> str:
+    # Special case RESERVED for the internal service key ONLY (auth_kind == "internal"): give it a
+    # fixed namespace instead of requiring an email. Every other key is a freesolo USER key and still
+    # must carry a real email — we never loosen that, so two different users can't collide on one
+    # namespace.
+    if key.get("auth_kind") == "internal":
+        return _INTERNAL_NAMESPACE
+    email = str(key.get("email") or "")
+    if "@" not in email:
+        raise EnvPublishError(
+            "authenticated Freesolo key must include an email (used to derive the hub namespace) — "
+            "publish with a key created at https://freesolo.co/sign-in (`flash login`)"
+        )
+    slug = re.sub(r"[^a-z0-9]+", "-", email.lower()).strip("-")
+    return slug or "user"
+
+
+def _sanitize_name(name: str) -> str:
+    slug = re.sub(r"[^a-z0-9._-]+", "-", name.lower()).strip("-")
+    if slug in {".", ".."} or not re.search(r"[a-z0-9]", slug):
+        return "env"
+    return slug or "env"
+
+
+def _safe_extract(tar_bytes: bytes, dest: Path) -> None:
+    root = dest.resolve()
+    try:
+        with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:gz") as tar:
+            total = 0
+            for count, member in enumerate(tar, start=1):
+                if count > _MAX_MEMBERS:
+                    raise EnvPublishError(
+                        f"env package has too many members (limit {_MAX_MEMBERS})"
+                    )
+                segments: list[str] = []
+                for segment in member.name.replace("\\", "/").split("/"):
+                    if not segment or segment == ".":
+                        continue
+                    if segment == "..":
+                        raise EnvPublishError(f"unsafe path in env package: {member.name!r}")
+                    segments.append(segment)
+                if not segments:
+                    continue
+                normalized_name = "/".join(segments)
+                target = (dest / normalized_name).resolve()
+                if target != root and root not in target.parents:
+                    raise EnvPublishError(f"unsafe path in env package: {member.name!r}")
+                if segments[0] in _BLOCKED_TOP_LEVEL_PATHS:
+                    raise EnvPublishError(
+                        "env packages must not contain repo-control or source top-level paths"
+                    )
+                if member.islnk() or member.issym():
+                    raise EnvPublishError(f"links are not allowed in env packages: {member.name!r}")
+                if not (member.isreg() or member.isdir()):
+                    raise EnvPublishError(
+                        f"only regular files and directories are allowed in env packages, "
+                        f"but {member.name!r} is a special file"
+                    )
+                total += max(0, member.size)
+                if total > _MAX_UNCOMPRESSED_BYTES:
+                    raise EnvPublishError(
+                        "env package is too large uncompressed "
+                        f"(limit {_human_mb(_MAX_UNCOMPRESSED_BYTES)})"
+                    )
+                member.name = normalized_name
+                tar.extract(member, dest)
+    except tarfile.TarError as exc:
+        raise EnvPublishError(f"env package is not a valid .tar.gz archive: {exc}") from exc
+    except OSError as exc:
+        raise EnvPublishError(f"env package could not be extracted: {exc}") from exc
+
+
+def _github_repo() -> str:
+    return _DEFAULT_GITHUB_REPO
+
+
+def _github_token() -> str | None:
+    return os.environ.get("GITHUB_TOKEN")
+
+
+def _redact(value: str, token: str) -> str:
+    if not token:
+        return value
+    return value.replace(token, "<redacted>").replace(
+        urllib.parse.quote(token, safe=""), "<redacted>"
+    )
+
+
+def _credentialed_repo_url(repo: str, token: str) -> str:
+    quoted = urllib.parse.quote(token, safe="")
+    return f"https://x-access-token:{quoted}@github.com/{repo}.git"
+
+
+def _run_git(cwd: Path, args: list[str], *, token: str) -> subprocess.CompletedProcess[str]:
+    env = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
+    try:
+        proc = subprocess.run(
+            ["git", *args],
+            cwd=cwd,
+            env=env,
+            capture_output=True,
+            text=True,
+            timeout=_GIT_TIMEOUT_S,
+        )
+    except FileNotFoundError as exc:
+        raise EnvPublishError(
+            "git is required to upload environments to Freesolo", status=503
+        ) from exc
+    except subprocess.TimeoutExpired as exc:
+        raise EnvPublishError(
+            f"Freesolo environment upload git command timed out after {_GIT_TIMEOUT_S}s",
+            status=504,
+        ) from exc
+    if proc.returncode != 0:
+        output = f"{proc.stdout or ''}\n{proc.stderr or ''}".strip()
+        cmd = "git " + " ".join(args)
+        raise EnvPublishError(
+            _redact(f"Freesolo environment upload failed during `{cmd}`: {output[:1000]}", token),
+            status=502,
+        )
+    return proc
+
+
+def _is_retryable_git_publish_error(message: str) -> bool:
+    lowered = message.lower()
+    permanent = (
+        "authentication failed",
+        "could not read username",
+        "repository not found",
+        "permission denied",
+        "403",
+        "401",
+    )
+    if any(marker in lowered for marker in permanent):
+        return False
+    retryable = (
+        "failed to push some refs",
+        "fetch first",
+        "non-fast-forward",
+        "stale info",
+        "cannot lock ref",
+        "connection reset",
+        "operation timed out",
+        "the remote end hung up",
+        "early eof",
+        "index.lock",
+        "rebase",
+    )
+    return any(marker in lowered for marker in retryable)
+
+
+def _copy_package_to_checkout(*, source: Path, checkout: Path, publish_root: str) -> None:
+    target = checkout / publish_root
+    checkout_root = checkout.resolve()
+    target_root = target.resolve()
+    if target_root != checkout_root and checkout_root not in target_root.parents:
+        raise EnvPublishError("unsafe environment publish path")
+    shutil.rmtree(target, ignore_errors=True)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copytree(source, target)
+
+
+def _commit_environment_update(
+    *, checkout: Path, publish_root: str, message: str, token: str
+) -> bool:
+    _run_git(checkout, ["config", "user.name", "freesolo-bot"], token=token)
+    _run_git(checkout, ["config", "user.email", "bot@freesolo.co"], token=token)
+    _run_git(checkout, ["add", "-A", "--", publish_root], token=token)
+    try:
+        proc = subprocess.run(
+            ["git", "diff", "--cached", "--quiet", "--", publish_root],
+            cwd=checkout,
+            capture_output=True,
+            text=True,
+            timeout=_GIT_TIMEOUT_S,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise EnvPublishError(
+            f"Freesolo environment upload git command timed out after {_GIT_TIMEOUT_S}s",
+            status=504,
+        ) from exc
+    if proc.returncode == 0:
+        return False
+    if proc.returncode != 1:
+        output = f"{proc.stdout or ''}\n{proc.stderr or ''}".strip()
+        raise EnvPublishError(
+            _redact(
+                f"Freesolo environment upload failed during staged diff check: {output}", token
+            ),
+            status=502,
+        )
+    _run_git(checkout, ["commit", "-m", message], token=token)
+    return True
+
+
+def _push_environment_commit(*, checkout: Path, token: str) -> None:
+    _run_git(checkout, ["pull", "--rebase", "origin", _GITHUB_BRANCH], token=token)
+    _run_git(checkout, ["push", "origin", f"HEAD:{_GITHUB_BRANCH}"], token=token)
+
+
+def _github_publish_once(
+    *,
+    dest: Path,
+    repo: str,
+    token: str,
+    publish_root: str,
+    message: str,
+) -> None:
+    with tempfile.TemporaryDirectory(prefix="flash-env-hub-") as tmp:
+        tmp_path = Path(tmp)
+        checkout = tmp_path / "environment-hub"
+        _run_git(
+            tmp_path,
+            [
+                "clone",
+                "--branch",
+                _GITHUB_BRANCH,
+                "--single-branch",
+                _credentialed_repo_url(repo, token),
+                str(checkout),
+            ],
+            token=token,
+        )
+        _copy_package_to_checkout(source=dest, checkout=checkout, publish_root=publish_root)
+        if _commit_environment_update(
+            checkout=checkout,
+            publish_root=publish_root,
+            message=message,
+            token=token,
+        ):
+            _push_environment_commit(checkout=checkout, token=token)
+
+
+def _environment_file_relative_path(root: Path) -> str:
+    canonical = root / _DEFAULT_ENVIRONMENT_FILE
+    if canonical.is_file():
+        return _DEFAULT_ENVIRONMENT_FILE
+    raise EnvPublishError("env package must contain environment.py")
+
+
+def _github_publish(dest: Path, *, name: str, key: dict) -> str:
+    token = _github_token()
+    if not token:
+        raise EnvPublishError(
+            "GITHUB_TOKEN is required to upload environments to Freesolo",
+            status=503,
+        )
+    repo = _github_repo()
+    ns = namespace_for(key)
+    clean = _sanitize_name(name)
+    publish_root = f"{ns}/{clean}"
+    _environment_file_relative_path(dest)
+    if not any(path.is_file() for path in dest.rglob("*")):
+        raise EnvPublishError("env package contains no files")
+    message = f"Upload Flash environment {ns}/{clean}"
+
+    last_error: EnvPublishError | None = None
+    max_attempts = len(_GIT_PUSH_RETRY_DELAYS_SECONDS) + 1
+    for attempt in range(max_attempts):
+        if attempt:
+            time.sleep(_GIT_PUSH_RETRY_DELAYS_SECONDS[attempt - 1])
+        try:
+            _github_publish_once(
+                dest=dest,
+                repo=repo,
+                token=token,
+                publish_root=publish_root,
+                message=message,
+            )
+            return f"{ns}/{clean}"
+        except EnvPublishError as exc:
+            last_error = exc
+            if attempt == max_attempts - 1 or not _is_retryable_git_publish_error(str(exc)):
+                raise
+    assert last_error is not None
+    raise last_error
+
+
+def publish_package(*, package_b64: str, name: str, key: dict) -> str:
+    if not isinstance(name, str):
+        raise EnvPublishError("env name must be a string")
+    if not isinstance(package_b64, str):
+        raise EnvPublishError("env package must be a base64 string")
+    if not name:
+        raise EnvPublishError("missing env name")
+    max_encoded = ((_MAX_UPLOAD_BYTES + 2) // 3) * 4 + 3
+    if len(package_b64) > max_encoded:
+        raise EnvPublishError(
+            f"env package upload is too large (limit {_human_mb(_MAX_UPLOAD_BYTES)} compressed)",
+            status=413,
+        )
+    try:
+        tar_bytes = base64.b64decode(package_b64, validate=True)
+    except Exception as exc:
+        raise EnvPublishError("env package is not valid base64") from exc
+    if not tar_bytes:
+        raise EnvPublishError("empty env package")
+    if len(tar_bytes) > _MAX_UPLOAD_BYTES:
+        raise EnvPublishError(
+            f"env package upload is too large (limit {_human_mb(_MAX_UPLOAD_BYTES)} compressed)",
+            status=413,
+        )
+    with tempfile.TemporaryDirectory(prefix="flash-env-publish-") as tmp:
+        dest = Path(tmp)
+        _safe_extract(tar_bytes, dest)
+        return _github_publish(dest, name=name, key=key)

flash/server/reconcile.py

@@ -0,0 +1,163 @@
+"""Daily realized-cost reconciliation: pull what the GPU provider actually billed for each
+finished run and report it to the freesolo backend for estimator accuracy tracking.
+
+Flash charges customer-facing training usage from the completed run's final ``cost_usd``. This
+job is the COGS side: the realized provider invoice (RunPod /v1/billing/endpoints).
+The backend's training_cost_accuracy view joins the two per run to surface
+charged-vs-realized error.
+
+Best-effort and entirely off the run hot path: it runs in a background loop (see the server
+lifespan), never blocks request handling, and any failure is swallowed and retried next cycle.
+Realized cost is reported with the operator INTERNAL key (this is COGS, not a customer charge),
+which also gates the whole feature -- with no FREESOLO_INTERNAL_KEY set, reconciliation is off.
+
+Scope note (v1): cost is attributed from the run's last persisted handle (RunStatus.remote),
+which is exact for the common single-seed run. A multi-seed run keeps only its final seed's
+handle, so its realized cost is currently under-counted -- a known limitation to extend by
+persisting every seed's resource id.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import time
+import urllib.error
+import urllib.request
+
+from flash import runner
+from flash.providers.realized import realized_cost_for_remote
+from flash.server.auth import INTERNAL_KEY_ENV, freesolo_base_url
+
+_REPORT_PATH = "/api/billing/training-cost"
+_REPORT_TIMEOUT_S = 10.0
+# Provider billing lags; wait this long after a run goes terminal before pulling (so the
+# invoice has settled) and stop retrying once a run is older than the window.
+_SETTLE_SECONDS = 3600.0  # 1h
+_WINDOW_SECONDS = 7 * 86400.0  # only reconcile runs that finished within the last 7 days
+# States that incur no GPU cost -> never reconciled.
+_FREE_TERMINAL_STATES = frozenset({"dry_run"})
+# States whose training is finished and whose GPU cost is therefore final -> eligible for
+# reconciliation. The terminal billable states plus `deployed`: a deployed run finished
+# training (its training invoice has settled) before serving was stood up on top of it, so
+# its realized training cost is final and must be reconciled like any other finished run.
+# (`deployed` is intentionally NOT in runner.TERMINAL_STATES -- it's a live, undeployable-back
+# state -- so it has to be added explicitly here.) Excludes the free states (e.g. dry_run).
+_RECONCILABLE_STATES = (runner.TERMINAL_STATES | {"deployed"}) - _FREE_TERMINAL_STATES
+
+
+def reconcile_enabled() -> bool:
+    """Reconciliation (and its reporting) is on only when the operator internal key is set."""
+    return bool(os.environ.get(INTERNAL_KEY_ENV))
+
+
+def _report(body: dict) -> bool:
+    """POST realized cost to the backend with the internal key (Bearer). Best-effort: returns
+    True on a 2xx, False on any failure (never raises). Mirrors ``billing._post_billing`` but
+    swallows errors -- a metering report must never affect anything."""
+    key = os.environ.get(INTERNAL_KEY_ENV)
+    if not key:
+        return False
+    req = urllib.request.Request(
+        f"{freesolo_base_url()}{_REPORT_PATH}",
+        data=json.dumps(body).encode("utf-8"),
+        method="POST",
+        headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=_REPORT_TIMEOUT_S) as resp:
+            resp.read()
+        return True
+    except (urllib.error.URLError, OSError):
+        return False
+
+
+def _terminal_ts(status: runner.RunStatus) -> float:
+    """The run's training-teardown time, used for both billing (``run_end``) and eligibility
+    (settle delay + window). Prefer the frozen ``finished_at`` over the mutable ``updated_at``:
+    deploy / late heartbeat / reconcile all move ``updated_at`` past teardown, which would both
+    DELAY the settle gate (it counts from the bump, not the finish) and let a long-finished run
+    that was merely bumped look "recent" and slip back inside ``_WINDOW_SECONDS``. ``finished_at``
+    is stamped once at the terminal transition and never moved; falls back to ``updated_at`` for
+    pre-feature runs. ``is not None`` (not truthiness) so a legitimate ``finished_at == 0.0`` is
+    honored rather than silently falling back to ``updated_at``."""
+    return float(status.finished_at if status.finished_at is not None else status.updated_at)
+
+
+def _due(status: runner.RunStatus, now: float) -> bool:
+    """Whether a run should be reconciled this pass: a billable run whose training is finished
+    (a terminal billable state, or `deployed` -- see _RECONCILABLE_STATES), not yet reconciled,
+    past the settle delay, still within the window, and carrying a provider handle."""
+    if status.state not in _RECONCILABLE_STATES:
+        return False
+    if status.reconciled_at:
+        return False
+    age = now - _terminal_ts(status)  # from teardown, not a later updated_at bump (see _terminal_ts)
+    if age < _SETTLE_SECONDS or age > _WINDOW_SECONDS:
+        return False
+    return bool(status.remote)
+
+
+def reconcile_run(status: runner.RunStatus, *, now: float | None = None) -> bool:
+    """Pull + report realized cost for one run; mark it reconciled on success. Returns True when
+    a positive realized cost was reported. A zero/None result leaves the run unreconciled so a
+    later cycle (within the window) retries once the provider invoice settles."""
+    now = time.time() if now is None else now
+    remote = status.remote or {}
+    # Truthiness (`or`), NOT `is not None`: this started_ts comes from a persisted provider handle
+    # whose from_dict coerces a MISSING started_ts to 0.0 (see Lambda/HyperstackJobHandle.from_dict),
+    # so 0.0 means "unknown launch", not a 1970 epoch launch. Billing the flat $/hr from 0.0 would
+    # massively inflate realized cost, so fall back to created_at when started_ts is falsey/missing.
+    start = float(remote.get("started_ts") or status.created_at)
+    # The run's true terminal time (~teardown / billing stop); see _terminal_ts for why this is
+    # the frozen finished_at rather than the mutable updated_at (which deploy/heartbeat move past
+    # teardown and would make the instance providers' flat $/hr bill until that later event).
+    run_end = _terminal_ts(status)
+    # RunPod's billing query pads past run end so the settled invoice is in range; the instance
+    # providers bill flat $/hr to teardown, so they get the UN-padded run_end (no extra settle hour).
+    realized = realized_cost_for_remote(remote, start=start, end=run_end + _SETTLE_SECONDS, run_end=run_end)
+    if realized is None or realized.realized_usd <= 0:
+        return False
+
+    body = {
+        "runId": status.run_id,
+        "realizedCostUsd": realized.realized_usd,
+        "provider": realized.provider,
+        "gpu": remote.get("allocated_gpu") or remote.get("gpu"),
+        "costByResource": realized.by_resource,
+        "wallSeconds": realized.wall_seconds,
+        "costBasis": "realized",
+        "source": realized.source,
+    }
+    if not _report(body):
+        return False
+
+    # Persist locally so we don't re-pull/re-report, and so `flash status` can show realized vs
+    # estimated. COST-FIELDS-ONLY: record_realized_cost re-reads the run under the lock and writes
+    # only the realized-cost columns, never `state`. The `status` here is an earlier snapshot, so
+    # writing its `state` back could REVERT a run that advanced since (e.g. to `deployed`) -- which
+    # the terminal-sticky CAS does NOT protect against, since `deployed` is non-terminal. Updating
+    # only the cost columns keeps the run's current state intact.
+    with contextlib.suppress(Exception):
+        runner.record_realized_cost(
+            status.run_id,
+            realized_cost_usd=realized.realized_usd,
+            reconciled_at=now,
+        )
+    return True
+
+
+def reconcile_once(*, now: float | None = None) -> int:
+    """One sweep over local runs: reconcile every due run. Returns how many were reported."""
+    if not reconcile_enabled():
+        return 0
+    now = time.time() if now is None else now
+    reported = 0
+    for status in runner.list_runs():
+        if not _due(status, now):
+            continue
+        with contextlib.suppress(Exception):
+            if reconcile_run(status, now=now):
+                reported += 1
+    return reported