precisionai-pace 0.0.1__py3-none-any.whl

precisionai/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2026 Precision AI
+# SPDX-License-Identifier: Apache-2.0

precisionai/pace/__init__.py

@@ -0,0 +1,52 @@
+# Copyright 2026 Precision AI
+# SPDX-License-Identifier: Apache-2.0
+
+"""Pace — a logger with progress reporting whose output says when the work will finish.
+
+Each progress record is a readable log line that also carries a short, versioned
+trailer, so the same output serves a person watching a terminal and a scheduler
+estimating when the job will release its hardware. See ``docs/design.md``.
+"""
+
+from precisionai.pace.capture import PaceHandler, capture_logging
+from precisionai.pace.core import Logger, logger
+from precisionai.pace.forecast import (
+    Forecast,
+    ForecastConfig,
+    ForecastState,
+    JobEstimate,
+    JobForecast,
+    ScopeForecast,
+)
+from precisionai.pace.protocol import Event, LineSplitter, Record, format_trailer, parse_line
+from precisionai.pace.scopes import Phase, Scope, phase, plan, progress, timed, track
+from precisionai.pace.version import get_version
+
+__version__ = get_version()
+
+__all__ = [
+    "Event",
+    "Forecast",
+    "ForecastConfig",
+    "ForecastState",
+    "JobEstimate",
+    "JobForecast",
+    "LineSplitter",
+    "Logger",
+    "PaceHandler",
+    "Phase",
+    "Record",
+    "Scope",
+    "ScopeForecast",
+    "__version__",
+    "capture_logging",
+    "format_trailer",
+    "get_version",
+    "logger",
+    "parse_line",
+    "phase",
+    "plan",
+    "progress",
+    "timed",
+    "track",
+]

precisionai/pace/_adapters.py

@@ -0,0 +1,166 @@
+# Copyright 2026 Precision AI
+# SPDX-License-Identifier: Apache-2.0
+
+"""The scope bookkeeping the framework adapters share.
+
+An adapter reports one main scope (the training run) and, while it is open, at most one
+child scope (a validation or evaluation pass) whose parent is the main scope. This module
+holds that bookkeeping with the standard library only, so it is tested without any
+framework installed.
+"""
+
+from __future__ import annotations
+
+from precisionai.pace.core import Logger
+from precisionai.pace.scopes import Scope
+
+
+class AdapterScopes:
+    """Own the main scope of a run and the child scope of the pass currently running inside it.
+
+    Every method is safe to call in any order and any number of times: a scope that was
+    never started, or has already ended, is left alone. Only the first scope this object
+    starts is marked as the job's root, because a process has at most one root.
+
+    Parameters
+    ----------
+    description : str
+        Label of the main scope.
+    unit : str
+        Unit of the main scope's counts.
+    scope : str, optional
+        Explicit scope id for the main scope.
+    root : bool
+        Whether the first scope this object starts stands for the whole job.
+    logger : Logger, optional
+        Output to use; the module-level logger by default.
+    """
+
+    def __init__(
+        self,
+        description: str,
+        unit: str = "step",
+        *,
+        scope: str | None = None,
+        root: bool = True,
+        logger: Logger | None = None,
+    ) -> None:
+        self.description = description
+        self.unit = unit
+        self._scope_id = scope
+        self._root_allowed = root
+        self._logger = logger
+        self._main: Scope | None = None
+        self._child: Scope | None = None
+
+    @property
+    def main(self) -> Scope | None:
+        """The main scope while it is open, otherwise ``None``."""
+        return self._main
+
+    @property
+    def child(self) -> Scope | None:
+        """The child scope while it is open, otherwise ``None``."""
+        return self._child
+
+    def begin(self, total: float | None, completed: float = 0.0, resumed: bool = False) -> Scope:
+        """Start the main scope, ending any main scope that is still open.
+
+        Parameters
+        ----------
+        total : float, optional
+            Total amount; ``None`` when unknown.
+        completed : float
+            Amount already done, above zero when resuming.
+        resumed : bool
+            Whether this resumes earlier work.
+
+        Returns
+        -------
+        Scope
+            The started scope.
+        """
+        self.end()
+        scope = Scope(
+            self.description,
+            total,
+            self.unit,
+            scope=self._scope_id,
+            root=self._take_root(),
+            completed=completed,
+            resumed=resumed,
+            logger=self._logger,
+        )
+        scope.start()
+        self._main = scope
+        return scope
+
+    def update(self, completed: float) -> None:
+        """Set the main scope's completed amount."""
+        if self._main is not None:
+            self._main.update(completed)
+
+    def begin_child(self, description: str, total: float | None, unit: str) -> Scope:
+        """Start a child scope, ending the previous one if it is still open.
+
+        Its parent is the main scope while that is open. With no main scope it is a
+        top-level scope but never the root: an evaluation run before training must
+        not mark the job complete when it ends, so only training can be the root.
+
+        Parameters
+        ----------
+        description : str
+            Label of the child scope.
+        total : float, optional
+            Total amount; ``None`` when unknown.
+        unit : str
+            Unit of the child scope's counts.
+
+        Returns
+        -------
+        Scope
+            The started scope.
+        """
+        self.end_child()
+        scope = Scope(description, total, unit, logger=self._logger)
+        if self._main is None:
+            scope.start()
+        else:
+            self._main.bind()
+            try:
+                scope.start()
+            finally:
+                self._main.unbind()
+        self._child = scope
+        return scope
+
+    def advance_child(self, n: float = 1) -> None:
+        """Add ``n`` to the child scope's completed amount."""
+        if self._child is not None:
+            self._child.advance(n)
+
+    def end_child(self) -> None:
+        """End the child scope if one is open."""
+        child, self._child = self._child, None
+        if child is not None:
+            child.end()
+
+    def end(self) -> None:
+        """End the child scope and then the main scope, whichever are open."""
+        self.end_child()
+        main, self._main = self._main, None
+        if main is not None:
+            main.end()
+
+    def fail(self) -> None:
+        """Mark the child scope and then the main scope failed, whichever are open."""
+        child, self._child = self._child, None
+        if child is not None:
+            child.fail()
+        main, self._main = self._main, None
+        if main is not None:
+            main.fail()
+
+    def _take_root(self) -> bool:
+        taken, self._root_allowed = self._root_allowed, False
+        return taken

precisionai/pace/_env.py

@@ -0,0 +1,208 @@
+# Copyright 2026 Precision AI
+# SPDX-License-Identifier: Apache-2.0
+
+"""Configuration, process rank and the clock behind the logger and progress APIs.
+
+Every setting resolves in the same order: a value passed to ``Logger.configure``,
+then the ``PACE_*`` environment variable, then the built-in default. Environment
+variables are read each time they are needed, so a scheduler (or a test) can change
+them without restarting the process.
+"""
+
+from __future__ import annotations
+
+import math
+import os
+import sys
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+DEFAULT_MIN_INTERVAL = 15.0
+DEFAULT_HEARTBEAT = 120.0
+
+LEVELS: dict[str, int] = {
+    "DEBUG": 10,
+    "INFO": 20,
+    "SUCCESS": 25,
+    "WARNING": 30,
+    "ERROR": 40,
+    "CRITICAL": 50,
+}
+DEFAULT_LEVEL = LEVELS["INFO"]
+
+_RANK_VARIABLES = ("RANK", "LOCAL_RANK", "SLURM_PROCID")
+_COLOR_MODES = ("auto", "always", "never")
+_TRUE_WORDS = frozenset({"1", "true", "yes", "on"})
+
+
+@dataclass(frozen=True)
+class Clock:
+    """The two clocks the package reads.
+
+    Parameters
+    ----------
+    monotonic : callable
+        Seconds on a monotonic clock. Used for every rate, interval and ``e=`` value.
+    wall : callable
+        Seconds since the epoch. Used only for the printed timestamp.
+    """
+
+    monotonic: Callable[[], float] = time.monotonic
+    wall: Callable[[], float] = time.time
+
+
+def parse_level(value: Any) -> int | None:
+    """Return the numeric level for a name or number, or ``None`` if it is not one.
+
+    Parameters
+    ----------
+    value : str or int
+        A level name (any case) or a number.
+
+    Returns
+    -------
+    int or None
+        The numeric level.
+    """
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str):
+        return LEVELS.get(value.strip().upper())
+    return None
+
+
+def level_name(number: int) -> str:
+    """Return the name of the highest level that does not exceed ``number``.
+
+    Parameters
+    ----------
+    number : int
+        A numeric level, such as the one standard ``logging`` uses.
+
+    Returns
+    -------
+    str
+        A level name; ``DEBUG`` for anything below it.
+    """
+    name = "DEBUG"
+    for candidate, value in LEVELS.items():
+        if value <= number:
+            name = candidate
+    return name
+
+
+def current_rank(env: dict[str, str] | None = None) -> int:
+    """Return this process's rank from ``RANK``, ``LOCAL_RANK`` or ``SLURM_PROCID``.
+
+    Parameters
+    ----------
+    env : mapping, optional
+        Environment to read; defaults to ``os.environ``.
+
+    Returns
+    -------
+    int
+        The first variable that holds an integer, or ``0`` when none does.
+    """
+    source = os.environ if env is None else env
+    for name in _RANK_VARIABLES:
+        rank = _parse_int(source.get(name))
+        if rank is not None:
+            return rank
+    return 0
+
+
+def _parse_int(text: str | None) -> int | None:
+    try:
+        return int((text or "").strip())
+    except ValueError:
+        return None
+
+
+def _env_text(name: str) -> str | None:
+    value = os.environ.get(name)
+    if value is None or not value.strip():
+        return None
+    return value.strip().lower()
+
+
+def _env_seconds(name: str) -> float | None:
+    text = _env_text(name)
+    if text is None:
+        return None
+    try:
+        value = float(text)
+    except ValueError:
+        return None
+    return value if math.isfinite(value) and value >= 0 else None
+
+
+def _choice(explicit: str | None, variable: str) -> str:
+    for candidate in (explicit, _env_text(variable)):
+        if candidate in _COLOR_MODES:
+            return str(candidate)
+    return "auto"
+
+
+@dataclass
+class Settings:
+    """Explicit settings, each ``None`` until ``configure`` sets it.
+
+    A ``None`` field defers to the environment variable, then to the default.
+    """
+
+    level: int | None = None
+    stream: Any = None
+    color: str | None = None
+    bar: str | None = None
+    min_interval: float | None = None
+    heartbeat: float | None = None
+    all_ranks: bool | None = None
+
+    def resolve_level(self) -> int:
+        """Return the minimum level that is written."""
+        if self.level is not None:
+            return self.level
+        text = _env_text("PACE_LEVEL")
+        parsed = None if text is None else parse_level(text)
+        return DEFAULT_LEVEL if parsed is None else parsed
+
+    def resolve_stream(self) -> Any:
+        """Return the stream to write to, looked up now so redirection is honoured."""
+        if self.stream is not None and not isinstance(self.stream, str):
+            return self.stream
+        name = self.stream if isinstance(self.stream, str) else _env_text("PACE_STREAM")
+        return sys.stderr if name == "stderr" else sys.stdout
+
+    def resolve_color_mode(self) -> str:
+        """Return ``auto``, ``always`` or ``never``."""
+        return _choice(self.color, "PACE_COLOR")
+
+    def resolve_bar_mode(self) -> str:
+        """Return whether to draw an in-place bar: ``auto`` (on a terminal), ``always`` or ``never``."""
+        return _choice(self.bar, "PACE_BAR")
+
+    def resolve_min_interval(self) -> float:
+        """Return the minimum seconds between progress lines of one scope."""
+        if self.min_interval is not None:
+            return self.min_interval
+        value = _env_seconds("PACE_MIN_INTERVAL")
+        return DEFAULT_MIN_INTERVAL if value is None else value
+
+    def resolve_heartbeat(self) -> float:
+        """Return the seconds after which a progress line is written whatever the progress."""
+        if self.heartbeat is not None:
+            return self.heartbeat
+        value = _env_seconds("PACE_HEARTBEAT")
+        return DEFAULT_HEARTBEAT if value is None else value
+
+    def progress_enabled(self) -> bool:
+        """Return whether this process writes progress lines (rank 0, or every rank if allowed)."""
+        if self.all_ranks is not None:
+            allowed = self.all_ranks
+        else:
+            allowed = _env_text("PACE_ALL_RANKS") in _TRUE_WORDS
+        return allowed or current_rank() == 0

precisionai/pace/_render.py

@@ -0,0 +1,234 @@
+# Copyright 2026 Precision AI
+# SPDX-License-Identifier: Apache-2.0
+
+"""Turn log messages and progress records into the readable part of an output line.
+
+Nothing here writes; it only builds strings. The text is for people and may change
+between releases. Only the ``[pace:1 ...]`` trailer that follows it is a contract.
+"""
+
+from __future__ import annotations
+
+import codecs
+import re
+import time
+from dataclasses import dataclass
+from typing import Sequence
+
+from precisionai.pace.forecast import Forecast, ForecastState
+from precisionai.pace.protocol import Event, Record
+
+PROGRESS_LABEL = "PROGRESS"
+LABEL_WIDTH = 8
+BAR_WIDTH = 20
+MAX_DESCRIPTION_CHARS = 80
+SEPARATOR = "  "
+
+
+@dataclass(frozen=True)
+class Charset:
+    """The glyphs used in the readable part of a line."""
+
+    dot: str
+    ellipsis: str
+    bar_full: str
+    bar_empty: str
+
+
+UNICODE = Charset(dot="·", ellipsis="…", bar_full="━", bar_empty="─")
+ASCII = Charset(dot="|", ellipsis="...", bar_full="#", bar_empty="-")
+
+_COLOR_CODES: dict[str, str] = {
+    "DEBUG": "2",
+    "SUCCESS": "32",
+    "WARNING": "33",
+    "ERROR": "31",
+    "CRITICAL": "1;31",
+    PROGRESS_LABEL: "36",
+}
+_ESCAPE = "\x1b["
+_WHITESPACE = re.compile(r"\s+")
+# The same escape sequences precisionai.pace.protocol strips before parsing.
+_ANSI_SEQUENCE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")
+_FORGED_MARKER = re.compile(r"\[(?=pace:)")
+
+
+def is_utf8(encoding: str) -> bool:
+    """Return whether ``encoding`` is UTF-8, so Unicode glyphs are safe to print."""
+    try:
+        return codecs.lookup(encoding).name == "utf-8"
+    except LookupError:
+        return False
+
+
+def charset_for(encoding: str) -> Charset:
+    """Return the Unicode glyphs for a UTF-8 stream and the ASCII ones for any other."""
+    return UNICODE if is_utf8(encoding) else ASCII
+
+
+def encode_safely(text: str, encoding: str) -> str:
+    """Return ``text`` with every character the encoding cannot represent replaced.
+
+    Parameters
+    ----------
+    text : str
+        Text about to be written.
+    encoding : str
+        The stream's encoding. An unknown encoding is treated as ASCII.
+
+    Returns
+    -------
+    str
+        Text the stream can write without raising ``UnicodeEncodeError``.
+    """
+    try:
+        return text.encode(encoding, "replace").decode(encoding, "replace")
+    except LookupError:
+        return text.encode("ascii", "replace").decode("ascii")
+
+
+def format_timestamp(wall: float) -> str:
+    """Return ``wall`` (seconds since the epoch) as ISO-8601 UTC with a ``Z``, to the second."""
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(wall))
+
+
+def colorize(label: str, enabled: bool) -> str:
+    """Wrap a level label in its colour when ``enabled``; the trailer is never coloured."""
+    code = _COLOR_CODES.get(label)
+    if not enabled or code is None:
+        return label
+    return f"{_ESCAPE}{code}m{label}{_ESCAPE}0m"
+
+
+def log_line(timestamp: str, label: str, message: str, color: bool) -> str:
+    """Return a log line: timestamp, level padded to eight characters, message."""
+    pad = " " * max(LABEL_WIDTH - len(label), 0)
+    return f"{timestamp} {colorize(label, color)}{pad} {neutralize(message)}"
+
+
+def neutralize(message: str) -> str:
+    """Break up any ``[pace:`` in a user message so it cannot pass for a progress trailer.
+
+    A reader removes terminal escape sequences before it looks for a trailer, so they
+    are removed here first: ``[pa`` + ESC ``[31m`` + ``ce:`` must not survive as a
+    marker. Any ESC left over is dropped too.
+    """
+    plain = _ANSI_SEQUENCE.sub("", message).replace("\x1b", "")
+    return _FORGED_MARKER.sub("[ ", plain)
+
+
+def clean_text(text: str, limit: int = MAX_DESCRIPTION_CHARS) -> str:
+    """Return ``text`` on one line (whitespace and control characters become one space), cut to ``limit``."""
+    one_line = _WHITESPACE.sub(" ", "".join(" " if ord(char) < 32 else char for char in text)).strip()
+    return neutralize(one_line)[:limit]
+
+
+def format_duration(seconds: float) -> str:
+    """Return a short duration such as ``45s``, ``21m57s``, ``3h05m`` or ``2d4h``."""
+    total = max(int(seconds + 0.5), 0)
+    minutes, secs = divmod(total, 60)
+    hours, minutes = divmod(minutes, 60)
+    days, hours = divmod(hours, 24)
+    if days:
+        return f"{days}d{hours}h"
+    if hours:
+        return f"{hours}h{minutes:02d}m"
+    if minutes:
+        return f"{minutes}m{secs:02d}s"
+    return f"{secs}s"
+
+
+def format_count(value: float) -> str:
+    """Return a count without a trailing ``.0``."""
+    if float(value).is_integer():
+        return str(int(value))
+    return f"{value:.4g}"
+
+
+def format_rate(rate: float, unit: str) -> str:
+    """Return a rate such as ``2.4 step/s``."""
+    shown = f"{rate:.0f}" if rate >= 100 else f"{rate:.3g}"
+    return f"{shown} {unit}/s"
+
+
+def format_bar(fraction: float, charset: Charset) -> str:
+    """Return a fixed-width bar, ``fraction`` of it filled."""
+    filled = min(max(int(fraction * BAR_WIDTH), 0), BAR_WIDTH)
+    return charset.bar_full * filled + charset.bar_empty * (BAR_WIDTH - filled)
+
+
+def estimate_text(record: Record, forecast: Forecast, charset: Charset) -> str:
+    """Describe the outcome of a finished scope, or its time remaining, or why there is no estimate."""
+    if record.event is Event.END:
+        return f"done in {format_duration(record.elapsed)}"
+    if record.event is Event.FAIL:
+        return f"failed after {format_duration(record.elapsed)}"
+    remaining = forecast.remaining_seconds
+    state = forecast.state
+    if remaining is not None and state in (ForecastState.LIVE, ForecastState.STALE, ForecastState.BOOTSTRAP):
+        note = {ForecastState.STALE: " (stale)", ForecastState.BOOTSTRAP: " (guess)"}.get(state, "")
+        return f"~{format_duration(remaining)}{note}"
+    if state is ForecastState.LEARNING:
+        return f"estimating{charset.ellipsis}"
+    return "no estimate"
+
+
+def progress_text(
+    record: Record,
+    forecast: Forecast,
+    description: str,
+    unit: str,
+    charset: Charset,
+    bar: bool = False,
+) -> str:
+    """Return the readable part of a progress line, for example ``Train  36.8%  1840/5000 step · 2.4 step/s · ~21m57s``.
+
+    Parameters
+    ----------
+    record : Record
+        The record being written.
+    forecast : Forecast
+        The scope's estimate after ``record``.
+    description : str
+        What the scope is doing; the scope id is used when empty.
+    unit : str
+        Unit of the counts.
+    charset : Charset
+        Unicode or ASCII glyphs.
+    bar : bool
+        Whether to include a bar after the description (the in-place terminal line).
+
+    Returns
+    -------
+    str
+        One line of text with no timestamp, level or trailer.
+    """
+    completed = record.completed or 0.0
+    total = record.total
+    parts: list[str] = [clean_text(description) or record.scope]
+    if total is not None:
+        if bar:
+            parts.append(format_bar(completed / total, charset))
+        parts.append(f"{100 * completed / total:.1f}%")
+    count = format_count(completed) if total is None else f"{format_count(completed)}/{format_count(total)}"
+    details = [f"{count} {clean_text(unit, 32)}".rstrip()]
+    if forecast.rate is not None:
+        details.append(format_rate(forecast.rate, clean_text(unit, 32) or "it"))
+    details.append(estimate_text(record, forecast, charset))
+    parts.append(f" {charset.dot} ".join(details))
+    return SEPARATOR.join(parts)
+
+
+def plan_text(phases: Sequence[tuple[str, float]]) -> str:
+    """Return the readable part of a ``plan`` line, for example ``Plan  prep 5%, train 90%``."""
+    whole = sum(weight for _, weight in phases) or 1.0
+    shares = ", ".join(f"{name} {round(100 * weight / whole)}%" for name, weight in phases)
+    return f"Plan{SEPARATOR}{shares}"
+
+
+def fit(text: str, width: int, charset: Charset | None = None) -> str:
+    """Return ``text`` cut so it fits ``width`` columns, ending in an ellipsis when it was cut."""
+    if len(text) <= width:
+        return text
+    mark = (charset or ASCII).ellipsis
+    return (text[: max(width - len(mark), 0)] + mark)[:width]