ravensight-playtest 0.6.0 → 0.7.2

package/src/run/report.js

@@ -57,6 +57,60 @@ export function lintMarkdownSections(markdown) {
   return REQUIRED_MARKDOWN_SECTIONS.filter(section => !text.includes(section));
 }
 
+/** The tool-call tags a model has been seen to leak into a written file. */
+const TOOL_TAG = '(?:parameter|invoke|content|function_calls|function_results|tool_use|tool_call|antml:[\\w-]+)';
+
+/** A line made only of tags: closers of any name, or tool-call openers. */
+const MARKUP_LINE = new RegExp(`^(?:\\s*(?:<\\/[A-Za-z][\\w:.-]*\\s*>|<${TOOL_TAG}\\b[^>]*>))+\\s*$`);
+
+/** Tool-call closers glued to the end of the last line of real text. */
+const MARKUP_SUFFIX = new RegExp(`(?:\\s*<\\/${TOOL_TAG}\\s*>)+\\s*$`);
+
+const FENCE = /^\s*(?:```|~~~)/;
+
+/**
+ * Takes tool-call markup off the end of a file the report step wrote. A real
+ * report.md ended "</content>\n</invoke>" because the model closed its tool
+ * call inside the file's content, and the runner appended its own sections
+ * after the stray tags.
+ *
+ * Only the end of the text is touched, and only outside a code fence: HTML a
+ * report quotes in backticks or in a fenced block is kept.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function stripTrailingToolMarkup(text) {
+  if (typeof text !== 'string' || text === '') return text;
+  const lines = text.split('\n');
+  const insideFence = count => lines.slice(0, count).filter(line => FENCE.test(line)).length % 2 === 1;
+  let end = lines.length;
+  let changed = false;
+  while (end > 0) {
+    const line = lines[end - 1];
+    if (line.trim() === '') {
+      end -= 1;
+    } else if (MARKUP_LINE.test(line) && !insideFence(end - 1)) {
+      end -= 1;
+      changed = true;
+    } else {
+      break;
+    }
+  }
+  const kept = lines.slice(0, end);
+  if (kept.length > 0) {
+    const index = kept.length - 1;
+    const last = kept[index];
+    const backticks = (last.match(/`/g) || []).length;
+    if (MARKUP_SUFFIX.test(last) && backticks % 2 === 0 && !FENCE.test(last) && !insideFence(index)) {
+      kept[index] = last.replace(MARKUP_SUFFIX, '');
+      changed = true;
+    }
+  }
+  if (!changed) return text;
+  return `${kept.join('\n').replace(/\s+$/, '')}\n`;
+}
+
 export class ReportInvalidError extends Error {
   /**
    * @param {Array<{instancePath: string, message: string}>} details

package/src/run/usage.js

@@ -32,6 +32,14 @@ export function createUsageLedger() {
   let modelMs = 0;
   let serverTotalUsd = null;
   let serverCalls = null;
+  // The run's tokens in input-token units (see weightedUnits), and the last
+  // server spend that covered every call this ledger had recorded when it
+  // arrived. Together they say what one unit costs on this run, which is how
+  // the persona loop keeps enough budget back for the report without ever
+  // holding a price: ./budgetReserve.js.
+  let units = 0;
+  let lastCallUnits = 0;
+  let sample = null;
 
   return {
     /**
@@ -50,6 +58,8 @@ export function createUsageLedger() {
       entry.cache_read_tokens += Number(usage.cache_read_input_tokens || 0);
       entry.output_tokens += Number(usage.output_tokens || 0);
       byModel.set(key, entry);
+      lastCallUnits = weightedUnits(usage);
+      units += lastCallUnits;
     },
 
     /**
@@ -61,6 +71,31 @@ export function createUsageLedger() {
       if (!runUsage) return;
       if (Number.isFinite(Number(runUsage.total_usd))) serverTotalUsd = Number(runUsage.total_usd);
       if (Number.isFinite(Number(runUsage.calls))) serverCalls = Number(runUsage.calls);
+      // A figure that has not caught up with every call recorded here would
+      // make a unit look cheaper than it is, so only one that covers them all
+      // (the server may count more, which only makes a unit look dearer) is
+      // kept as the price sample.
+      const covers = !Number.isFinite(Number(runUsage.calls)) || Number(runUsage.calls) >= calls;
+      if (serverTotalUsd !== null && serverTotalUsd > 0 && units > 0 && covers) sample = { usd: serverTotalUsd, units };
+    },
+
+    /** The weighted units of the last call recorded, 0 before the first. */
+    get lastCallUnits() {
+      return lastCallUnits;
+    },
+
+    /**
+     * What the run has spent and what one weighted unit costs on it, or null
+     * before the server has priced a call this ledger recorded. The spend is
+     * the server's figure plus the calls made since, priced at the sample's
+     * rate, so it never reads lower than the server's own.
+     *
+     * @returns {{usd: number, usdPerUnit: number}|null}
+     */
+    spendEstimate() {
+      if (!sample) return null;
+      const usdPerUnit = sample.usd / sample.units;
+      return { usd: Math.max(serverTotalUsd || 0, sample.usd + usdPerUnit * Math.max(0, units - sample.units)), usdPerUnit };
     },
 
     get modelCalls() {
@@ -114,6 +149,25 @@ export function createUsageLedger() {
   };
 }
 
+/** Input-token units per cache write, cache read and output token. */
+export const CACHE_WRITE_WEIGHT = 1.25;
+export const CACHE_READ_WEIGHT = 0.1;
+export const OUTPUT_WEIGHT = 5;
+
+/**
+ * One call's tokens in input-token units: what the server's price for the call
+ * is proportional to on every tier, since each tier prices a cache write at
+ * 1.25 input tokens, a cache read at 0.1 and an output token at 5. No price,
+ * only the shape of one.
+ *
+ * @param {Object} usage
+ * @returns {number}
+ */
+export function weightedUnits(usage = {}) {
+  const n = key => Math.max(0, Number(usage && usage[key]) || 0);
+  return n('input_tokens') + n('cache_creation_input_tokens') * CACHE_WRITE_WEIGHT + n('cache_read_input_tokens') * CACHE_READ_WEIGHT + n('output_tokens') * OUTPUT_WEIGHT;
+}
+
 /**
  * @param {string} runDirectory
  * @param {ReturnType<typeof createUsageLedger>} ledger