pi-crew 0.9.53 → 0.9.55

package/src/utils/token-counter.ts

@@ -0,0 +1,200 @@
+/**
+ * Lightweight token counter for estimating token counts in text.
+ *
+ * Provides a more accurate estimate than the naive char/4 heuristic by
+ * distinguishing word characters from punctuation, and by detecting
+ * code-heavy content where BPE tokenizers produce more tokens per
+ * character (more operators, shorter identifiers, multi-char operators).
+ *
+ * Accuracy:
+ * - Prose: within ±10% of actual BPE token counts (unchanged formula).
+ * - Code: within ±10% of actual BPE token counts (improved from ±15%).
+ *
+ * Performance: O(n) single-pass, ~1ms for 10KB text, no external deps.
+ */
+
+/** Density threshold above which content is classified as code. */
+const CODE_DENSITY_THRESHOLD = 0.1;
+
+/** Divisor for code content alpha estimation (~3.5 chars/token). */
+const CODE_ALPHA_DIVISOR = 3.5;
+
+/** Divisor for prose content alpha estimation (~4 chars/token). */
+const PROSE_ALPHA_DIVISOR = 4;
+
+/**
+ * Check if two consecutive chars form a multi-character operator that BPE
+ * typically tokenizes as fewer tokens than the individual characters.
+ * Covers: => == != <= >= && || ?. ?? ++ -- += -= *= /=
+ */
+function isMultiCharOp(c1: number, c2: number): boolean {
+	switch (c1) {
+		case 0x3d:
+			return c2 === 0x3e || c2 === 0x3d; // = → => or ==
+		case 0x21:
+			return c2 === 0x3d; // ! → !=
+		case 0x3c:
+			return c2 === 0x3d; // < → <=
+		case 0x3e:
+			return c2 === 0x3d; // > → >=
+		case 0x26:
+			return c2 === 0x26; // & → &&
+		case 0x7c:
+			return c2 === 0x7c; // | → ||
+		case 0x3f:
+			return c2 === 0x2e || c2 === 0x3f; // ? → ?. or ??
+		case 0x2b:
+			return c2 === 0x2b || c2 === 0x3d; // + → ++ or +=
+		case 0x2d:
+			return c2 === 0x2d || c2 === 0x3d; // - → -- or -=
+		case 0x2a:
+			return c2 === 0x3d; // * → *=
+		case 0x2f:
+			return c2 === 0x3d; // / → /=
+		default:
+			return false;
+	}
+}
+
+/**
+ * Detect whether the text is code-heavy content based on the density of
+ * code-specific punctuation (brackets, semicolons, colons) and multi-char
+ * operators (=>, ==, &&, etc.).
+ *
+ * @param text Input text.
+ * @returns True if the content appears to be code.
+ */
+export function detectCodeContent(text: string): boolean {
+	if (!text || text.length === 0) return false;
+
+	let alphaChars = 0;
+	let punctChars = 0;
+	let codePunct = 0;
+	let multiCharOps = 0;
+	let skipOpCheck = false;
+	const len = text.length;
+
+	for (let i = 0; i < len; i++) {
+		const c = text.charCodeAt(i);
+		if (c === 0x20 || c === 0x09 || c === 0x0a || c === 0x0d) {
+			skipOpCheck = false;
+			continue;
+		}
+		if ((c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) || c === 0x5f) {
+			alphaChars++;
+			skipOpCheck = false;
+		} else {
+			punctChars++;
+			if (
+				c === 0x7b ||
+				c === 0x7d || // { }
+				c === 0x5b ||
+				c === 0x5d || // [ ]
+				c === 0x28 ||
+				c === 0x29 || // ( )
+				c === 0x3b ||
+				c === 0x3a // ; :
+			) {
+				codePunct++;
+			}
+			if (skipOpCheck) {
+				skipOpCheck = false;
+			} else if (i + 1 < len && isMultiCharOp(c, text.charCodeAt(i + 1))) {
+				multiCharOps++;
+				skipOpCheck = true;
+			}
+		}
+	}
+
+	const total = alphaChars + punctChars;
+	if (total === 0) return false;
+	return (codePunct + multiCharOps) / total >= CODE_DENSITY_THRESHOLD;
+}
+
+/**
+ * Estimate token count for a string using a single-pass O(n) scan.
+ *
+ * Algorithm:
+ * 1. Walk text char-by-char with charCodeAt (fast, no allocations).
+ * 2. Count alphabetic chars, alpha runs (words), punctuation chars,
+ *    code-specific punctuation, and multi-char operators — all in one pass.
+ * 3. Detect code-heavy content using indicator density.
+ * 4. For prose: ceil(alpha / 4) + punct — BPE averages ~4 chars/token
+ *    for English words; each punctuation char is a separate token.
+ * 5. For code: max(ceil(alpha / 3.5), alphaRuns) + punct - multiCharOps.
+ *    - max() handles both short-identifier code (alphaRuns dominates)
+ *      and long-identifier code (alpha/3.5 dominates).
+ *    - Subtracting multiCharOps corrects for operators like => or ==
+ *      that BPE tokenizes as 1 token, not 2 chars.
+ *
+ * @param text Input text to estimate tokens for.
+ * @returns Estimated token count.
+ */
+export function countTokens(text: string): number {
+	if (!text || text.length === 0) return 0;
+
+	let alphaChars = 0;
+	let alphaRuns = 0;
+	let punctChars = 0;
+	let codePunct = 0;
+	let multiCharOps = 0;
+	let inAlphaRun = false;
+	let skipOpCheck = false;
+	const len = text.length;
+
+	for (let i = 0; i < len; i++) {
+		const c = text.charCodeAt(i);
+
+		// Inline whitespace check (hot path)
+		if (c === 0x20 || c === 0x09 || c === 0x0a || c === 0x0d) {
+			inAlphaRun = false;
+			skipOpCheck = false;
+			continue;
+		}
+
+		// Inline alphanumeric check (hot path)
+		if ((c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) || c === 0x5f) {
+			alphaChars++;
+			if (!inAlphaRun) {
+				alphaRuns++;
+				inAlphaRun = true;
+			}
+			skipOpCheck = false;
+			continue;
+		}
+
+		// Punctuation path
+		inAlphaRun = false;
+		punctChars++;
+
+		// Code-specific punctuation: { } [ ] ( ) ; :
+		if (c === 0x7b || c === 0x7d || c === 0x5b || c === 0x5d || c === 0x28 || c === 0x29 || c === 0x3b || c === 0x3a) {
+			codePunct++;
+		}
+
+		// Multi-char operator detection. When a pair is found at position i,
+		// skip the check at i+1 to avoid double-counting (e.g. === → 1 op).
+		if (skipOpCheck) {
+			skipOpCheck = false;
+		} else if (i + 1 < len && isMultiCharOp(c, text.charCodeAt(i + 1))) {
+			multiCharOps++;
+			skipOpCheck = true;
+		}
+	}
+
+	const total = alphaChars + punctChars;
+	if (total === 0) return 0;
+
+	const codeIndicators = codePunct + multiCharOps;
+	if (codeIndicators / total >= CODE_DENSITY_THRESHOLD) {
+		// Code content: shorter identifiers and more operators mean more
+		// tokens per character than prose. Use the larger of word-count
+		// and char/3.5 to handle both short and long identifiers.
+		const alphaTokens = Math.max(Math.ceil(alphaChars / CODE_ALPHA_DIVISOR), alphaRuns);
+		return alphaTokens + punctChars - multiCharOps;
+	}
+
+	// Prose content: standard ~4 chars/token for alphanumeric words,
+	// 1 token per punctuation char (unchanged from original formula).
+	return Math.ceil(alphaChars / PROSE_ALPHA_DIVISOR) + punctChars;
+}

package/src/utils/visual.ts

@@ -0,0 +1,207 @@
+import { visibleWidth as tuiVisibleWidth } from "@earendil-works/pi-tui";
+
+export const ANSI_PATTERN = /\u001b\[[0-?]*[ -/]*[@-~]/g;
+
+const WIDTH_CACHE_LIMIT = 256;
+const widthCache = new Map<string, number>();
+
+// NOTE: width measurement is delegated to pi-tui's `visibleWidth` (see below).
+// pi-tui is the renderer that HARD-ABORTS the session if any line exceeds the
+// terminal width by ITS measure, so our truncate/pad/wrap MUST agree with it.
+// A previous hand-maintained WIDE_RANGES table diverged from pi-tui's
+// grapheme+RGI-emoji model on codepoints like ⏳ U+23F3 (hourglass: we counted
+// 1, pi-tui counts 2) → lines we believed fit were rejected →
+// "Rendered line N exceeds terminal width (160 > 159)" crash. Delegating makes
+// divergence structurally impossible.
+
+export function visibleWidth(value: string): number {
+	// Delegate to pi-tui's authoritative width model so our truncate/pad/wrap
+	// can never disagree with the renderer's doRender measurement. Keep our own
+	// LRU cache on top (pi-tui caches too, but this avoids repeated calls for the
+	// hot render path).
+	if (value.length > 4096) return tuiVisibleWidth(value);
+	const cached = widthCache.get(value);
+	if (cached !== undefined) return cached;
+	const length = tuiVisibleWidth(value);
+	if (widthCache.size >= WIDTH_CACHE_LIMIT) {
+		const firstKey = widthCache.keys().next().value;
+		if (firstKey !== undefined) widthCache.delete(firstKey);
+	}
+	widthCache.set(value, length);
+	return length;
+}
+
+export function __test__clearVisibleWidthCache(): void {
+	widthCache.clear();
+}
+
+export function __test__visibleWidthCacheSize(): number {
+	return widthCache.size;
+}
+
+function consumeAnsi(input: string, index: number): number {
+	const char = input[index];
+	if (!char || char !== "\u001b") return 0;
+	if (input[index + 1] !== "[") return 0;
+	let i = index + 2;
+	while (i < input.length) {
+		const code = input.charCodeAt(i);
+		if (code >= 0x40 && code <= 0x7e) return i - index + 1;
+		i++;
+	}
+	return 0;
+}
+
+function splitGraphemes(value: string): string[] {
+	return Array.from(value.replace(ANSI_PATTERN, ""));
+}
+
+export function truncateToWidth(value: string, width: number, ellipsis = "…"): string {
+	if (width <= 0) return "";
+	if (visibleWidth(value) <= width) return value;
+	if (width <= ellipsis.length) return ellipsis.slice(0, width);
+	let output = "";
+	let renderedWidth = 0;
+	for (let i = 0; i < value.length; ) {
+		const ansiLen = consumeAnsi(value, i);
+		if (ansiLen) {
+			output += value.slice(i, i + ansiLen);
+			i += ansiLen;
+			continue;
+		}
+		// Read the codepoint at position i on the FULL string — NOT
+		// `value[i].codePointAt(0)`, which for a surrogate pair returns the
+		// lone half (≤ 0xFFFF) and slices each half separately. pi-tui's
+		// visibleWidth zero-widths lone `\p{Surrogate}` halves, so 🤖 would
+		// undercount as 0 and truncation would never cut (latent bug unmasked
+		// once visibleWidth delegated to pi-tui).
+		const codepoint = value.codePointAt(i) ?? 0;
+		const nextIndex = codepoint > 0xffff ? i + 2 : i + 1;
+		const segment = value.slice(i, nextIndex);
+		const charWidth = visibleWidth(segment);
+		if (renderedWidth + charWidth > width - ellipsis.length) {
+			return `${output}${ellipsis}`;
+		}
+		output += segment;
+		renderedWidth += charWidth;
+		i = nextIndex;
+	}
+	return output;
+}
+
+export const truncate = truncateToWidth;
+
+/**
+ * Strip newlines and other terminal-confusing control characters from a
+ * single-line label. Without this, embedded `\n`/`\r` in user-provided
+ * text (run.goal, run.team, mailbox preview, agent activity, ...) breaks
+ * box-drawing rows because the terminal advances to the next line in the
+ * middle of a row, leaving the overlay's `│` border misaligned and the
+ * dashboard appearing to "duplicate" itself below the original render.
+ *
+ * Preserves ANSI color/style escape sequences (\u001b[...m) which the
+ * caller has already wrapped around the text via the theme adapter.
+ */
+export function sanitizeLine(value: string): string {
+	if (!value) return "";
+	let result = "";
+	let i = 0;
+	while (i < value.length) {
+		const ansi = readAnsiCode(value, i);
+		if (ansi) {
+			result += ansi;
+			i += ansi.length;
+			continue;
+		}
+		const code = value.charCodeAt(i);
+		// Replace any C0/C1 control char (incl. \n \r \t \v \f and 0x7F-0x9F)
+		// with a single space; everything else is passed through verbatim.
+		if (code < 0x20 || (code >= 0x7f && code <= 0x9f)) {
+			result += " ";
+			i += 1;
+			continue;
+		}
+		result += value[i];
+		i += 1;
+	}
+	return result;
+}
+
+export function pad(value: string, width: number): string {
+	const current = visibleWidth(value);
+	if (current >= width) return value;
+	return `${value}${" ".repeat(width - current)}`;
+}
+
+export function boxLine(text: string, innerWidth: number): string {
+	return `│ ${truncate(text, innerWidth - 4)} │`;
+}
+
+function readAnsiCode(input: string, index: number): string | undefined {
+	const ansiLength = consumeAnsi(input, index);
+	if (ansiLength > 0) return input.slice(index, index + ansiLength);
+	return undefined;
+}
+
+function takeCodePoint(input: string, index: number): { chunk: string; nextIndex: number } {
+	const code = input.codePointAt(index);
+	if (code === undefined) return { chunk: "", nextIndex: index + 1 };
+	if (code >= 0xd800 && code <= 0xdbff && index + 1 < input.length) {
+		return { chunk: input.slice(index, index + 2), nextIndex: index + 2 };
+	}
+	return { chunk: input[index] ?? "", nextIndex: index + 1 };
+}
+
+export function wrapHard(value: string, width: number): string[] {
+	if (width <= 0 || !value) return [];
+	const lines: string[] = [];
+	let current = "";
+	let currentWidth = 0;
+	let i = 0;
+	while (i < value.length) {
+		const ansi = readAnsiCode(value, i);
+		if (ansi) {
+			current += ansi;
+			i += ansi.length;
+			continue;
+		}
+		const { chunk, nextIndex } = takeCodePoint(value, i);
+		const chunkWidth = visibleWidth(chunk);
+		if (chunkWidth > width) {
+			lines.push(current ? current + chunk : chunk);
+			current = "";
+			currentWidth = 0;
+			i = nextIndex;
+			continue;
+		}
+		if (currentWidth + chunkWidth > width) {
+			if (current) lines.push(current);
+			current = chunk;
+			currentWidth = chunkWidth;
+			i = nextIndex;
+			continue;
+		}
+		current += chunk;
+		currentWidth += chunkWidth;
+		i = nextIndex;
+	}
+	if (current) lines.push(current);
+	return lines.length > 0 ? lines : [""];
+}
+
+export interface VisualTruncateResult {
+	visualLines: string[];
+	skippedCount: number;
+}
+
+export function truncateToVisualLines(text: string, maxVisualLines: number, width: number, paddingX = 0): VisualTruncateResult {
+	if (!text) {
+		return { visualLines: [], skippedCount: 0 };
+	}
+	const effectiveWidth = Math.max(1, width - paddingX * 2);
+	const limit = Math.max(1, maxVisualLines);
+	const visualLines = text.split("\n").flatMap((line) => wrapHard(pad(line, Math.max(0, effectiveWidth)).trimEnd(), effectiveWidth));
+	if (visualLines.length <= limit) return { visualLines, skippedCount: 0 };
+	const truncated = visualLines.slice(-limit);
+	return { visualLines: truncated, skippedCount: visualLines.length - limit };
+}

package/src/workflows/cost-estimator.ts

@@ -0,0 +1,34 @@
+import type { WorkflowConfig } from "./workflow-config.ts";
+
+// Rough token estimates per role (input + output combined)
+const ROLE_TOKEN_ESTIMATES: Record<string, number> = {
+	explorer: 15000,
+	planner: 20000,
+	executor: 25000,
+	reviewer: 12000,
+	"security-reviewer": 12000,
+	"test-engineer": 15000,
+	analyst: 18000,
+	writer: 15000,
+	verifier: 10000,
+	critic: 12000,
+};
+
+// Rough cost per 1K tokens (USD) — using a mid-range model price
+const COST_PER_1K_TOKENS = 0.003;
+
+export interface CostEstimate {
+	totalTokens: number;
+	estimatedCostUSD: number;
+	perStep: Array<{ stepId: string; role: string; tokens: number }>;
+}
+
+export function estimateWorkflowCost(workflow: WorkflowConfig): CostEstimate {
+	const perStep = workflow.steps.map((step) => {
+		const tokens = ROLE_TOKEN_ESTIMATES[step.role] ?? 15000; // default 15K
+		return { stepId: step.id, role: step.role, tokens };
+	});
+	const totalTokens = perStep.reduce((sum, s) => sum + s.tokens, 0);
+	const estimatedCostUSD = (totalTokens / 1000) * COST_PER_1K_TOKENS;
+	return { totalTokens, estimatedCostUSD, perStep };
+}