tines 0.0.145 → 0.0.147

package/dist/index.js

@@ -3560,6 +3560,9 @@ var selfRef = () => ({ kind: "self-ref" });
 function str(v) {
   return v === null || v === void 0 ? "" : String(v);
 }
+function isoTime(v) {
+  return typeof v === "number" && Number.isFinite(v) ? new Date(v).toISOString() : "an unknown time";
+}
 function joinChanged(v, sep2) {
   return Array.isArray(v) ? v.join(sep2) : "";
 }
@@ -3659,6 +3662,13 @@ var DESCRIBERS = {
     // runner dropped after launch. The cause is in the error text.
     text(`fail (${str(p.consecutive_failures)} consecutive): ${str(p.error)}`)
   ],
+  "runner.rate_limited": (_ev, p) => [
+    text("saw runner"),
+    name(p.runner_name),
+    // A provider condition, not a failure: the runner holds itself until the
+    // reported reset and resumes with nobody in the loop.
+    text(`hit its usage limit \u2014 resumes ${isoTime(p.resets_at)}: ${str(p.error)}`)
+  ],
   "routing_rule.created": (ev, p) => [
     text(`${action(ev.type)} the ${str(p.scope_label)} routing rule`)
   ],
@@ -4227,6 +4237,8 @@ function scheduleRef(s) {
 }
 function runnerStatusLabel(runner) {
   if (runner.status === "paused") return "paused";
+  if (runner.backoff_reason === "rate_limit" && (runner.backoff_until ?? 0) > Date.now())
+    return "rate limited";
   return runner.online ? "online" : "offline";
 }
 function ruleTargetsLabel(rule) {
@@ -6725,15 +6737,27 @@ function renderStreamEvent(event) {
       if (event.is_error && event.result) lines.push(`[error] ${clip(event.result, 2e3)}`);
       return lines;
     }
+    case "rate_limit_event": {
+      const info = event.rate_limit_info;
+      if (info?.status !== "rejected") return [];
+      const resets = typeof info.resetsAt === "number" && Number.isFinite(info.resetsAt) ? new Date(info.resetsAt < 1e12 ? info.resetsAt * 1e3 : info.resetsAt).toISOString() : "unknown";
+      return [`[session] rate limit: ${info.rateLimitType ?? "usage"} rejected \u2014 resets ${resets}`];
+    }
     default:
       return [];
   }
 }
 var ClaudeStreamRenderer = class {
-  constructor(emit) {
+  /**
+   * @param onEvent every successfully parsed event, before rendering — so the
+   * rate-limit detector can read the stream without parsing it a second time.
+   */
+  constructor(emit, onEvent) {
     this.emit = emit;
+    this.onEvent = onEvent;
   }
   emit;
+  onEvent;
   pending = "";
   write(chunk) {
     this.pending += chunk;
@@ -6767,11 +6791,172 @@ var ClaudeStreamRenderer = class {
 `);
       return;
     }
+    this.onEvent?.(event);
     for (const line of renderStreamEvent(event)) this.emit(`${line}
 `);
   }
 };
 
+// src/daemon/rate-limit.ts
+var USAGE_LIMIT_PREFIXES = [
+  "You've hit your",
+  "You've reached your",
+  "You're out of usage credits",
+  "Your org is out of usage",
+  "Your seat type doesn't include usage credits",
+  "Your usage allocation has been disabled by your admin",
+  "Your group's usage limit is set to $0",
+  "You're out of extra usage"
+];
+var USAGE_LIMIT_PATTERN = /^Fable(?: [^·\n]{1,40})? requires usage credits\./;
+var ANSI = /\x1b\[[0-9;]*m/g;
+function toEpochMs(value) {
+  return value < 1e12 ? Math.round(value * 1e3) : Math.round(value);
+}
+function rateLimitFromStreamEvent(event) {
+  if (typeof event !== "object" || event === null) return null;
+  const ev = event;
+  if (ev.type !== "rate_limit_event") return null;
+  const info = ev.rate_limit_info;
+  if (typeof info !== "object" || info === null) return null;
+  const { status, resetsAt, rateLimitType } = info;
+  if (status !== "rejected") return null;
+  const limit = typeof rateLimitType === "string" && rateLimitType ? rateLimitType : null;
+  const resumeAt = typeof resetsAt === "number" && Number.isFinite(resetsAt) && resetsAt > 0 ? toEpochMs(resetsAt) : null;
+  return {
+    resumeAt,
+    source: "stream",
+    limit,
+    detail: `${limit ?? "usage"} limit rejected${resumeAt === null ? "" : ` \xB7 resets ${new Date(resumeAt).toISOString()}`}`
+  };
+}
+function rateLimitFromStderrLine(line, now) {
+  const clean = line.replace(ANSI, "").trim();
+  if (!clean) return null;
+  const matches = USAGE_LIMIT_PREFIXES.some((prefix) => clean.startsWith(prefix)) || USAGE_LIMIT_PATTERN.test(clean);
+  if (!matches) return null;
+  const [, reset] = clean.split("\xB7");
+  const resumeAt = reset ? parseResetTime(reset.replace(/^\s*resets\s*/i, ""), now) : null;
+  return { resumeAt, source: "stderr", limit: null, detail: clean };
+}
+var MONTHS = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"];
+function zoneOffsetMs(zone, at) {
+  const parts = new Intl.DateTimeFormat("en-US", {
+    timeZone: zone,
+    hour12: false,
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit"
+  }).formatToParts(new Date(at));
+  const get = (type) => Number(parts.find((p) => p.type === type)?.value ?? "0");
+  const hour = get("hour") % 24;
+  const asUtc = Date.UTC(
+    get("year"),
+    get("month") - 1,
+    get("day"),
+    hour,
+    get("minute"),
+    get("second")
+  );
+  return asUtc - Math.floor(at / 1e3) * 1e3;
+}
+function zonedTimeToEpoch(zone, y, m, d, hh, mm) {
+  const naive = Date.UTC(y, m, d, hh, mm, 0);
+  let guess = naive - zoneOffsetMs(zone, naive);
+  guess = naive - zoneOffsetMs(zone, guess);
+  return guess;
+}
+function zonedDate(zone, at) {
+  const parts = new Intl.DateTimeFormat("en-US", {
+    timeZone: zone,
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit"
+  }).formatToParts(new Date(at));
+  const get = (type) => Number(parts.find((p) => p.type === type)?.value ?? "0");
+  return { y: get("year"), m: get("month") - 1, d: get("day") };
+}
+var DURATION = /(\d+)\s*([dhms])/g;
+function parseResetTime(text2, now) {
+  const raw = text2.trim();
+  if (!raw) return null;
+  const relative = raw.match(/^in\s+(.+)$/i);
+  if (relative) {
+    let ms = 0;
+    let matched = false;
+    for (const [, amount, unit] of relative[1].matchAll(DURATION)) {
+      const n = Number(amount);
+      ms += n * { d: 864e5, h: 36e5, m: 6e4, s: 1e3 }[unit];
+      matched = true;
+    }
+    return matched ? now + ms : null;
+  }
+  const zoned = raw.match(/^(.*?)\s*\(([A-Za-z_]+\/[A-Za-z_+-]+|UTC)\)$/);
+  if (!zoned) return null;
+  const [, head, zone] = zoned;
+  try {
+    zoneOffsetMs(zone, now);
+  } catch {
+    return null;
+  }
+  const clock = head.match(/(\d{1,2})(?::(\d{2}))?\s*(am|pm)$/i);
+  if (!clock) return null;
+  const meridiem = clock[3].toLowerCase();
+  const hour12 = Number(clock[1]) % 12;
+  const hh = meridiem === "pm" ? hour12 + 12 : hour12;
+  const mm = Number(clock[2] ?? "0");
+  const dated = head.match(/^([A-Za-z]{3})[a-z]*\s+(\d{1,2})(?:,\s*(\d{4}))?,/);
+  if (dated) {
+    const month = MONTHS.indexOf(dated[1].toLowerCase());
+    if (month === -1) return null;
+    const here = zonedDate(zone, now);
+    const year = dated[3] ? Number(dated[3]) : here.y;
+    let at2 = zonedTimeToEpoch(zone, year, month, Number(dated[2]), hh, mm);
+    if (!dated[3] && at2 < now - 864e5)
+      at2 = zonedTimeToEpoch(zone, year + 1, month, Number(dated[2]), hh, mm);
+    return at2;
+  }
+  const today = zonedDate(zone, now);
+  const at = zonedTimeToEpoch(zone, today.y, today.m, today.d, hh, mm);
+  return at > now ? at : zonedTimeToEpoch(zone, today.y, today.m, today.d + 1, hh, mm);
+}
+var RateLimitDetector = class {
+  stream = null;
+  stderr = null;
+  pending = "";
+  noteStreamEvent(event) {
+    const signal = rateLimitFromStreamEvent(event);
+    if (signal) this.stream = signal;
+  }
+  noteStderr(chunk, now = Date.now()) {
+    this.pending += chunk;
+    let nl = this.pending.indexOf("\n");
+    while (nl !== -1) {
+      this.line(this.pending.slice(0, nl), now);
+      this.pending = this.pending.slice(nl + 1);
+      nl = this.pending.indexOf("\n");
+    }
+  }
+  /** Flushes a trailing partial line. Call once the harness has exited. */
+  finish(now = Date.now()) {
+    if (this.pending) {
+      this.line(this.pending, now);
+      this.pending = "";
+    }
+  }
+  /** The structured signal when there is one; the prose only as a fallback. */
+  signal() {
+    return this.stream ?? this.stderr;
+  }
+  line(raw, now) {
+    const signal = rateLimitFromStderrLine(raw, now);
+    if (signal) this.stderr = signal;
+  }
+};
+
 // src/daemon/cli-refresh.ts
 import { spawn } from "node:child_process";
 import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync7 } from "node:fs";
@@ -6901,11 +7086,13 @@ var RunTable = class {
    * this still cleans up — the slot, workspace, and state entry must never
    * outlive the run — but reports nothing.
    *
-   * Pass `judgment: 'interrupted'` when the daemon itself ended the run
+   * Pass `{ judgment: 'interrupted' }` when the daemon itself ended the run
    * (shutdown, restart) rather than the work failing: the supervisor then
    * spares the issue a strike. Everything a run can do wrong to itself —
    * a non-zero harness exit, a workspace that would not set up — must not
-   * set it.
+   * set it. The one exception is `{ judgment: 'rate_limited' }`: a non-zero
+   * exit whose cause was the provider refusing on a usage limit is the
+   * runner's condition, not the work's.
    */
   async finishAndCleanup(run, status, error, judgment) {
     if (run.settled) return this.cleanup(run);
@@ -7442,7 +7629,7 @@ async function runDaemon(opts) {
         await client2.finishRun(run.runId, {
           status,
           ...error ? { error } : {},
-          ...judgment ? { judgment } : {}
+          ...judgment
         });
       },
       release: (run, { keep, outcome }) => {
@@ -7615,7 +7802,12 @@ async function runDaemon(opts) {
       table2.persist();
       log(`run ${runId}: launched ${invocation.file} (pid ${child.pid})`);
       if (opts.harness === "claude_code") {
-        const renderer = new ClaudeStreamRenderer((line) => run.batcher.append(line));
+        const limiter = new RateLimitDetector();
+        run.limiter = limiter;
+        const renderer = new ClaudeStreamRenderer(
+          (line) => run.batcher.append(line),
+          (event) => limiter.noteStreamEvent(event)
+        );
         run.renderer = renderer;
         run.drain = () => renderer.finish();
         const spoolPath = join5(opts.configDir, "rawlogs", `${runId}.ndjson`);
@@ -7630,7 +7822,11 @@ async function runDaemon(opts) {
       } else {
         child.stdout?.on("data", (data) => run.batcher.append(data.toString("utf8")));
       }
-      child.stderr?.on("data", (data) => run.batcher.append(data.toString("utf8")));
+      child.stderr?.on("data", (data) => {
+        const text2 = data.toString("utf8");
+        run.batcher.append(text2);
+        run.limiter?.noteStderr(text2);
+      });
       run.timeout = setTimeout(() => {
         if (run.settled) return;
         log(`run ${runId} hit its ${assignment.timeout_minutes}m timeout; killing`);
@@ -7643,6 +7839,7 @@ async function runDaemon(opts) {
       });
       child.on("exit", (code, signal) => {
         run.drain?.();
+        run.limiter?.finish();
         const closing = exitLineForRun(run, {
           code,
           signal,
@@ -7658,11 +7855,22 @@ async function runDaemon(opts) {
         } else if (code === 0) {
           void table2.finishAndCleanup(run, "completed");
         } else {
-          void table2.finishAndCleanup(
-            run,
-            "failed",
-            signal ? `harness killed by ${signal}` : `harness exited with code ${code}`
-          );
+          const limited = signal ? null : run.limiter?.signal() ?? null;
+          if (limited) {
+            log(
+              `run ${runId}: harness rate limited (${limited.detail}); reporting without a strike`
+            );
+            void table2.finishAndCleanup(run, "failed", `rate limited: ${limited.detail}`, {
+              judgment: "rate_limited",
+              ...limited.resumeAt !== null ? { resume_at: limited.resumeAt } : {}
+            });
+          } else {
+            void table2.finishAndCleanup(
+              run,
+              "failed",
+              signal ? `harness killed by ${signal}` : `harness exited with code ${code}`
+            );
+          }
         }
       });
     } catch (err) {
@@ -7676,7 +7884,9 @@ async function runDaemon(opts) {
     await Promise.all(
       table2.values().map(async (run) => {
         if (run.child?.pid) killTree(run.child.pid, "SIGTERM");
-        await table2.finishAndCleanup(run, "failed", "daemon shut down", "interrupted");
+        await table2.finishAndCleanup(run, "failed", "daemon shut down", {
+          judgment: "interrupted"
+        });
       })
     );
     process.exit(0);
@@ -7815,6 +8025,11 @@ function register8(program3) {
         `active runs: ${runner.active_runs}/${runner.max_concurrent}  timeout: ${runner.max_run_minutes}m  default tier: ${runner.default_tier}`
       );
       if (runner.last_seen_at) console.log(`last seen: ${timestamp(runner.last_seen_at)}`);
+      if (runner.backoff_reason === "rate_limit" && runner.backoff_until) {
+        console.log(
+          `rate limited: the harness account is out of usage; resumes ${timestamp(runner.backoff_until)}`
+        );
+      }
       if (runner.launch_failures > 0) {
         console.log(
           `consecutive failures: ${runner.launch_failures}${runner.backoff_until ? ` (backing off until ${timestamp(runner.backoff_until)})` : ""}`
@@ -8416,6 +8631,8 @@ function queueHeadline(block) {
       return `${runner} draining`;
     case "backing_off":
       return `${runner} backing off`;
+    case "rate_limited":
+      return `${runner} rate limited`;
     case "no_rule":
       return "no matching routing rule";
     case "no_targets":
@@ -8452,6 +8669,8 @@ function queueFix(block) {
       return "tines supervisor enable";
     case "backing_off":
       return "retries automatically; check the daemon log if it keeps failing";
+    case "rate_limited":
+      return "resumes automatically when the usage window resets; nothing to do";
     default:
       return null;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tines",
-  "version": "0.0.145",
+  "version": "0.0.147",
   "description": "CLI for Tines, an orchestration layer for AI agents",
   "repository": {
     "type": "git",