envloop-cli-experiment 1.23.0__py3-none-any.whl

envloop_experiment/train_log_cli.py

@@ -0,0 +1,130 @@
+"""Job log rendering and bounded polling; reading never changes the remote Job."""
+
+import argparse
+import json
+import sys
+import time
+from collections import OrderedDict
+from typing import Any
+
+from envloop.http import TRANSIENT_CODES, RemoteError
+from envloop.terminal import color_enabled, compact_time, safe_text, status_text, style
+from envloop_experiment.train_progress import job_progress, progress_line
+
+# While following a Train Job, re-read its progress at most this often (the platform observes it about every 30 s).
+PROGRESS_POLL_SEC = 30
+
+
+def log_line(item: dict[str, Any], *, color: bool = False):
+    timestamp = compact_time(safe_text(item.get("at") or ""))
+    level = safe_text(item.get("level") or "info")
+    message = safe_text(item["content"])
+    # Captured lines keep their own line break; print adds one, so drop it rather than double-space.
+    if message.endswith("\n"):
+        message = message[:-1]
+    if item.get("code"):
+        diagnostic = safe_text(item["code"])
+        if item.get("phase"):
+            diagnostic += f"; phase={safe_text(item['phase'])}"
+        message = f"[{diagnostic}] {message}"
+    if color:
+        timestamp = style(timestamp, "2") if timestamp else ""
+        level = status_text(level)
+    prefix = " ".join(part for part in (timestamp, level) if part)
+    return f"{prefix} {message}"
+
+
+def capture_summary(page: dict[str, Any]):
+    details = []
+    status = page.get("collection_status")
+    if status in {"partial", "failed", "unavailable"}:
+        details.append(f"collection={status}")
+    if page.get("truncated"):
+        details.append(f"truncated; dropped_bytes={page.get('dropped_bytes', 0)}")
+    source_reasons = set()
+    for source in page.get("sources", []):
+        if source.get("missing_reason"):
+            source_reasons.add(source["missing_reason"])
+            name = safe_text(source.get("name", "source"))
+            if source.get("process"):
+                name += f" ({safe_text(source['process'])})"
+            details.append(f"{name}: {safe_text(source['missing_reason'])}")
+    if page.get("missing_reason") and page["missing_reason"] not in source_reasons:
+        details.append(safe_text(page["missing_reason"]))
+    if page.get("has_more"):
+        details.append("more records available")
+    return "Log collection: " + "; ".join(details) if details else ""
+
+
+def show_progress(train: Any, job_id: str, state: dict[str, Any]):
+    """Print a Train Job's progress line to stderr when it changes; other Job types stop the polling."""
+    state["next"] = time.monotonic() + PROGRESS_POLL_SEC
+    try:
+        job = train.status(job_id)
+    except RemoteError:
+        return  # Progress is an observation; a failed read never interrupts the log stream.
+    if not isinstance(job, dict) or job.get("type") not in {None, "train"}:
+        state["enabled"] = False
+        return
+    line = progress_line(job_progress(job))
+    if line and line != state["line"]:
+        print(line, file=sys.stderr, flush=True)
+        state["line"] = line
+
+
+def read_logs(train: Any, args: argparse.Namespace):
+    color = color_enabled()
+    cursor = args.cursor
+    tail = args.tail if args.tail is not None else (200 if cursor is None else None)
+    retries = 0
+    previous_summary = None
+    seen = OrderedDict()
+    progress = {"next": 0.0, "line": None, "enabled": args.follow and not args.json}
+    try:
+        while True:
+            try:
+                page = train.logs(args.job_id, source=args.source, cursor=cursor, tail=tail, limit=args.limit,
+                                  **({"process": args.process} if getattr(args, "process", None) is not None else {}))
+            except RemoteError as error:
+                transient = error.code in TRANSIENT_CODES or error.status_code in {429, 500, 502, 503, 504}
+                if not args.follow or not transient or retries >= 3:
+                    raise
+                retries += 1
+                print("Log read temporarily unavailable; retrying.", file=sys.stderr)
+                delay = getattr(error, "retry_after", None)
+                time.sleep(delay if delay is not None else 2)
+                continue
+            retries = 0
+            if args.follow:
+                items = []
+                for item in page["items"]:
+                    identity = item["id"]
+                    if identity not in seen:
+                        items.append(item)
+                        seen[identity] = None
+                        # Retain a bounded overlap window, independent of log text.
+                        if len(seen) > 10000:
+                            seen.popitem(last=False)
+                page = {**page, "items": items}
+            if args.json:
+                # JSON escaping also prevents control sequences from executing in terminals.
+                print(json.dumps(page, ensure_ascii=True), flush=True)
+            else:
+                for item in page["items"]:
+                    print(log_line(item, color=color), flush=True)
+            if progress["enabled"] and time.monotonic() >= progress["next"]:
+                show_progress(train, args.job_id, progress)
+            summary = capture_summary(page)
+            if summary and summary != previous_summary:
+                print(summary, file=sys.stderr)
+            previous_summary = summary
+            if not args.follow or (page.get("eof") and not page.get("has_more")):
+                return 0
+            next_cursor = page.get("next_cursor")
+            if not isinstance(next_cursor, str) or not next_cursor or (page.get("has_more") and next_cursor == cursor):
+                raise RemoteError("INVALID_LOG_PAGE")
+            cursor, tail = next_cursor, None
+            if not page.get("has_more"):
+                time.sleep(2)
+    except KeyboardInterrupt:
+        return 130

envloop_experiment/train_progress.py

@@ -0,0 +1,52 @@
+"""One-line view of a Train Job's live ``training.progress``; reading never changes the Job."""
+
+import math
+from typing import Any
+
+from envloop.terminal import compact_time, safe_text
+
+
+def _number(value: Any) -> float | None:
+    return value if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) else None
+
+
+def _duration(seconds: float) -> str:
+    seconds = round(seconds)
+    hours, rest = divmod(seconds, 3600)
+    minutes, seconds = divmod(rest, 60)
+    return f"{hours}h{minutes:02d}m" if hours else f"{minutes}m{seconds:02d}s" if minutes else f"{seconds}s"
+
+
+def job_progress(job: Any) -> dict[str, Any] | None:
+    """The ``training.progress`` mapping of a Job response, or None."""
+    training = job.get("training") if isinstance(job, dict) else None
+    progress = training.get("progress") if isinstance(training, dict) else None
+    return progress if isinstance(progress, dict) else None
+
+
+def progress_line(progress: dict[str, Any] | None) -> str | None:
+    """``Training progress: step 32/66 (48.5%), loss 0.2118, lr 0.0001315, ETA 22m41s, updated ...``.
+
+    None until the platform has observed a step; a value never observed is omitted, not shown as zero.
+    """
+    if not isinstance(progress, dict) or _number(progress.get("step")) is None:
+        return None
+    step, total = _number(progress.get("step")), _number(progress.get("total_steps"))
+    part = f"step {step:g}/{total:g}" if total else f"step {step:g}"
+    percent = _number(progress.get("percent"))
+    parts = [part + (f" ({percent:g}%)" if percent is not None else "")]
+    loss = _number(progress.get("loss"))
+    if loss is not None:
+        logged = _number(progress.get("metrics_step"))
+        # The latest logged loss can belong to an earlier step than the step bar.
+        parts.append(f"loss {loss:.4g}" + (f" (step {logged:g})" if logged is not None and logged != step else ""))
+    for key, label in (("learning_rate", "lr"), ("grad_norm", "grad_norm")):
+        if _number(progress.get(key)) is not None:
+            parts.append(f"{label} {progress[key]:.4g}")
+    if _number(progress.get("tokens_seen")) is not None:
+        parts.append(f"tokens {int(progress['tokens_seen']):,}")
+    if _number(progress.get("eta_seconds")) is not None:
+        parts.append(f"ETA {_duration(progress['eta_seconds'])}")
+    if isinstance(progress.get("updated_at"), str):
+        parts.append(f"updated {safe_text(compact_time(progress['updated_at']))}")
+    return "Training progress: " + ", ".join(parts)

envloop_experiment/train_runner_cli.py

@@ -0,0 +1,66 @@
+"""Translate Runner Train CLI shortcuts into the public Job request schema."""
+
+import argparse
+import re
+from typing import Any
+from uuid import uuid4
+
+SHORTCUT_FIELDS = ('docker', 'trails', 'gput_type', 'gpu', 'cpu', 'memory', 'disk', 'shm_size')
+
+
+def positive_integer(value: str):
+    count = int(value)
+    if count <= 0:
+        raise ValueError('expected a positive integer')
+    return count
+
+
+def memory_mebibytes(value: str):
+    """CLI binary units map exactly to the platform's historical *_mb fields."""
+    match = re.fullmatch(r'([1-9][0-9]*)(Mi|Gi|Ti)(?:B)?', value)
+    if not match:
+        raise ValueError('expected a positive integer with Mi, Gi or Ti units (for example 64Gi)')
+    return int(match[1]) * {'Mi': 1, 'Gi': 1024, 'Ti': 1024 * 1024}[match[2]]
+
+
+def runner_request(args: argparse.Namespace):
+    from envloop_experiment.client import runner_selector
+    from envloop_experiment.sandbox import validate_image
+
+    if not args.runner or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*@[^\s@]+', args.runner):
+        raise ValueError('Train Runner requires --runner namespace/name@REVISION')
+    runner = runner_selector(args.runner)
+    if args.docker is not None and args.image is not None:
+        raise ValueError('--docker and --image cannot be combined')
+    count = args.trails if args.trails is not None else 1
+    if not 1 <= count <= 16:
+        raise ValueError('--trails must be from 1 to 16')
+    spec: dict[str, Any] = {'runner': runner}
+    image = args.docker if args.docker is not None else args.image
+    if image is not None:
+        try:
+            spec['image'] = validate_image(image)
+        except ValueError:
+            raise ValueError('Train image must be a Docker image name, optionally with a tag or sha256 digest') from None
+    if args.command is not None:
+        try:
+            valid_command = bool(args.command.strip()) and '\0' not in args.command and len(args.command.encode('utf-8')) <= 65536
+        except UnicodeEncodeError:
+            valid_command = False
+        if not valid_command:
+            raise ValueError('--command must be non-empty UTF-8, without NUL, at most 65536 bytes')
+        spec['command'] = args.command
+    resources = {field: getattr(args, option) for option, field in (
+        ('gput_type', 'gpu_type'), ('gpu', 'gpus'), ('cpu', 'cpus'),
+        ('memory', 'memory_mb'), ('disk', 'storage_mb'), ('shm_size', 'shm_mb'))
+        if getattr(args, option) is not None}
+    if args.gput_type is not None and not args.gput_type.strip():
+        raise ValueError('--gput-type must be non-empty')
+    if resources:
+        spec['resources'] = resources
+    if args.timeout_sec is not None:
+        if args.timeout_sec <= 0 or not args.timeout_sec.is_integer():
+            raise ValueError('Train Runner --timeout-sec must be positive whole seconds')
+        spec['timeout_sec'] = int(args.timeout_sec)
+    return {'type': 'train', 'request_id': args.request_id if args.request_id is not None else str(uuid4()),
+            'trials': [{'name': f'trial-{index + 1}', 'spec': dict(spec)} for index in range(count)]}

envloop_experiment/training/__init__.py

@@ -0,0 +1,4 @@
+"""Removed: the public interactive training SDK. Training is managed Jobs only."""
+from envloop.client import INTERACTIVE_TRAINING_REMOVED
+
+raise ModuleNotFoundError(INTERACTIVE_TRAINING_REMOVED, name="envloop.training")