pi-crew 0.1.46 → 0.1.49

package/src/utils/ids.ts

@@ -1,12 +1,17 @@
-import { randomBytes } from "node:crypto";
-
-export function createRunId(prefix = "team"): string {
-	const stamp = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14);
-	const suffix = randomBytes(8).toString("hex");
-	return `${prefix}_${stamp}_${suffix}`;
-}
-
-export function createTaskId(stepId: string, index: number): string {
-	const normalized = stepId.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "") || "task";
-	return `${String(index + 1).padStart(2, "0")}_${normalized}`;
-}
+import { randomBytes } from "node:crypto";
+import { generateTaskName } from "./task-name-generator.ts";
+
+export function createRunId(prefix = "team"): string {
+	const stamp = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14);
+	const suffix = randomBytes(8).toString("hex");
+	return `${prefix}_${stamp}_${suffix}`;
+}
+
+export function createTaskId(stepId: string, index: number): string {
+	const normalized = stepId.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "") || "task";
+	return `${String(index + 1).padStart(2, "0")}_${normalized}`;
+}
+
+export function createDisplayName(): string {
+	return generateTaskName();
+}

package/src/utils/incremental-reader.ts

@@ -0,0 +1,104 @@
+import * as fs from "node:fs";
+
+export interface IncrementalReadState {
+	byteOffset: number;
+	lineCount: number;
+}
+
+export interface IncrementalReadResult {
+	lines: string[];
+	state: IncrementalReadState;
+	eof: boolean;
+}
+
+const CHUNK_SIZE = 64 * 1024;
+
+/**
+ * Read new lines from a text file since last known byte offset.
+ * Uses fs.openSync + fs.readSync for efficient incremental reading.
+ */
+export function readLinesSince(filePath: string, state: IncrementalReadState): IncrementalReadResult {
+	let fd: number | undefined;
+	try {
+		fd = fs.openSync(filePath, "r");
+	} catch {
+		return { lines: [], state: { byteOffset: state.byteOffset, lineCount: state.lineCount }, eof: true };
+	}
+
+	try {
+		const stat = fs.fstatSync(fd);
+		const fileSize = stat.size;
+
+		if (fileSize <= state.byteOffset) {
+			return {
+				lines: [],
+				state: { byteOffset: fileSize, lineCount: state.lineCount },
+				eof: true,
+			};
+		}
+
+		const bytesToRead = fileSize - state.byteOffset;
+		const buf = Buffer.alloc(bytesToRead);
+		let totalRead = 0;
+
+		while (totalRead < bytesToRead) {
+			const chunkSize = Math.min(CHUNK_SIZE, bytesToRead - totalRead);
+			const bytesRead = fs.readSync(fd, buf, totalRead, chunkSize, state.byteOffset + totalRead);
+			if (bytesRead === 0) break;
+			totalRead += bytesRead;
+		}
+
+		const content = buf.toString("utf-8", 0, totalRead);
+		const lines: string[] = [];
+		let lineCount = state.lineCount;
+		let committedOffset = state.byteOffset;
+
+		let searchFrom = 0;
+		let newlineIdx: number;
+
+		while ((newlineIdx = content.indexOf("\n", searchFrom)) !== -1) {
+			const lineText = content.slice(searchFrom, newlineIdx);
+			committedOffset = state.byteOffset + newlineIdx + 1;
+			searchFrom = newlineIdx + 1;
+			if (lineText.length > 0) {
+				lines.push(lineText);
+				lineCount++;
+			}
+		}
+
+		const eof = committedOffset >= fileSize;
+
+		return {
+			lines,
+			state: { byteOffset: committedOffset, lineCount },
+			eof,
+		};
+	} finally {
+		if (fd !== undefined) {
+			try { fs.closeSync(fd); } catch { /* ignore */ }
+		}
+	}
+}
+
+/**
+ * Read parsed JSON objects from a JSONL file since last known byte offset.
+ * Skips malformed lines.
+ */
+export function readJsonlSince<T>(filePath: string, state: IncrementalReadState): { items: T[]; state: IncrementalReadState; eof: boolean } {
+	const result = readLinesSince(filePath, state);
+	const items: T[] = [];
+
+	for (const line of result.lines) {
+		try {
+			items.push(JSON.parse(line) as T);
+		} catch {
+			// Skip malformed lines
+		}
+	}
+
+	return {
+		items,
+		state: result.state,
+		eof: result.eof,
+	};
+}

package/src/utils/names.ts

@@ -1,27 +1,27 @@
-export function sanitizeName(name: string): string {
-	const result = name.toLowerCase().trim().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/-+/g, "-").replace(/^-+|-+$/g, "");
-	return result || "unnamed";
-}
-
-export function requireString(value: unknown, label: string): { value?: string; error?: string } {
-	if (typeof value !== "string" || !value.trim()) return { error: `${label} must be a non-empty string.` };
-	return { value: value.trim() };
-}
-
-export function parseConfigObject(config: unknown): { value?: Record<string, unknown>; error?: string } {
-	let parsed = config;
-	if (typeof parsed === "string") {
-		try {
-			parsed = JSON.parse(parsed) as unknown;
-		} catch (error) {
-			const message = error instanceof Error ? error.message : String(error);
-			return { error: `config must be valid JSON: ${message}` };
-		}
-	}
-	if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return { error: "config must be an object." };
-	return { value: parsed as Record<string, unknown> };
-}
-
-export function hasOwn(obj: Record<string, unknown>, key: string): boolean {
-	return Object.prototype.hasOwnProperty.call(obj, key);
-}
+export function sanitizeName(name: string): string {
+	const result = name.toLowerCase().trim().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/-+/g, "-").replace(/^-+|-+$/g, "");
+	return result || "unnamed";
+}
+
+export function requireString(value: unknown, label: string): { value?: string; error?: string } {
+	if (typeof value !== "string" || !value.trim()) return { error: `${label} must be a non-empty string.` };
+	return { value: value.trim() };
+}
+
+export function parseConfigObject(config: unknown): { value?: Record<string, unknown>; error?: string } {
+	let parsed = config;
+	if (typeof parsed === "string") {
+		try {
+			parsed = JSON.parse(parsed) as unknown;
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			return { error: `config must be valid JSON: ${message}` };
+		}
+	}
+	if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return { error: "config must be an object." };
+	return { value: parsed as Record<string, unknown> };
+}
+
+export function hasOwn(obj: Record<string, unknown>, key: string): boolean {
+	return Object.prototype.hasOwnProperty.call(obj, key);
+}

package/src/utils/redaction.ts

@@ -1,44 +1,44 @@
-const SECRET_KEY_PATTERN = /(?:^|[_.-])(token|api[-_]?key|password|passwd|secret|credential|authorization|private[-_]?key)(?:$|[_.-])/i;
-const INLINE_SECRET_PATTERN = /(^|[\s,{])(([A-Za-z0-9_.-]*(?:api[-_]?key|token|password|passwd|secret|credential|authorization|private[-_]?key)[A-Za-z0-9_.-]*)\s*[=:]\s*)([^\s,;"'}]+)/gi;
-const AUTH_HEADER_PATTERN = /\b(Authorization\s*:\s*(?:Bearer|Basic|Token)?\s*)([^\r\n]+)/gi;
-const BEARER_PATTERN = /\b(Bearer\s+)([A-Za-z0-9._~+/=-]{8,})\b/g;
-const PEM_PRIVATE_KEY_PATTERN = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g;
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-	if (!value || typeof value !== "object" || Array.isArray(value)) return false;
-	// Exclude built-in types whose Object.entries() would produce empty arrays.
-	if (value instanceof Date || value instanceof RegExp || value instanceof Error || value instanceof Map || value instanceof Set) return false;
-	return true;
-}
-
-function isSecretKey(keyName: string): boolean {
-	return SECRET_KEY_PATTERN.test(keyName) || /^(token|apiKey|api_key|password|secret|credential|authorization|privateKey|private_key)$/i.test(keyName);
-}
-
-export function redactSecretString(value: string): string {
-	return value
-		.replace(PEM_PRIVATE_KEY_PATTERN, "***")
-		.replace(AUTH_HEADER_PATTERN, "$1***")
-		.replace(BEARER_PATTERN, "$1***")
-		.replace(INLINE_SECRET_PATTERN, "$1$2***");
-}
-
-export function redactSecrets(value: unknown, keyName = ""): unknown {
-	if (keyName && isSecretKey(keyName)) return "***";
-	if (typeof value === "string") return redactSecretString(value);
-	if (Array.isArray(value)) return value.map((item) => redactSecrets(item));
-	if (isRecord(value)) {
-		const output: Record<string, unknown> = {};
-		for (const [key, entry] of Object.entries(value)) output[key] = redactSecrets(entry, key);
-		return output;
-	}
-	return value;
-}
-
-export function redactJsonLine(line: string): string {
-	try {
-		return JSON.stringify(redactSecrets(JSON.parse(line) as unknown));
-	} catch {
-		return redactSecretString(line);
-	}
-}
+const SECRET_KEY_PATTERN = /(?:^|[_.-])(token|api[-_]?key|password|passwd|secret|credential|authorization|private[-_]?key)(?:$|[_.-])/i;
+const INLINE_SECRET_PATTERN = /(^|[\s,{])(([A-Za-z0-9_.-]*(?:api[-_]?key|token|password|passwd|secret|credential|authorization|private[-_]?key)[A-Za-z0-9_.-]*)\s*[=:]\s*)([^\s,;"'}]+)/gi;
+const AUTH_HEADER_PATTERN = /\b(Authorization\s*:\s*(?:Bearer|Basic|Token)?\s*)([^\r\n]+)/gi;
+const BEARER_PATTERN = /\b(Bearer\s+)([A-Za-z0-9._~+/=-]{8,})\b/g;
+const PEM_PRIVATE_KEY_PATTERN = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+	// Exclude built-in types whose Object.entries() would produce empty arrays.
+	if (value instanceof Date || value instanceof RegExp || value instanceof Error || value instanceof Map || value instanceof Set) return false;
+	return true;
+}
+
+function isSecretKey(keyName: string): boolean {
+	return SECRET_KEY_PATTERN.test(keyName) || /^(token|apiKey|api_key|password|secret|credential|authorization|privateKey|private_key)$/i.test(keyName);
+}
+
+export function redactSecretString(value: string): string {
+	return value
+		.replace(PEM_PRIVATE_KEY_PATTERN, "***")
+		.replace(AUTH_HEADER_PATTERN, "$1***")
+		.replace(BEARER_PATTERN, "$1***")
+		.replace(INLINE_SECRET_PATTERN, "$1$2***");
+}
+
+export function redactSecrets(value: unknown, keyName = ""): unknown {
+	if (keyName && isSecretKey(keyName)) return "***";
+	if (typeof value === "string") return redactSecretString(value);
+	if (Array.isArray(value)) return value.map((item) => redactSecrets(item));
+	if (isRecord(value)) {
+		const output: Record<string, unknown> = {};
+		for (const [key, entry] of Object.entries(value)) output[key] = redactSecrets(entry, key);
+		return output;
+	}
+	return value;
+}
+
+export function redactJsonLine(line: string): string {
+	try {
+		return JSON.stringify(redactSecrets(JSON.parse(line) as unknown));
+	} catch {
+		return redactSecretString(line);
+	}
+}

package/src/utils/safe-paths.ts

@@ -1,47 +1,47 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-
-export function isSafePathId(value: string): boolean {
-	return /^[A-Za-z0-9_-]+$/.test(value);
-}
-
-export function assertSafePathId(kind: string, value: string): string {
-	if (!isSafePathId(value)) throw new Error(`Invalid ${kind}: ${value}`);
-	return value;
-}
-
-export function resolveContainedPath(baseDir: string, targetPath: string): string {
-	const base = path.resolve(baseDir);
-	const resolved = path.isAbsolute(targetPath) ? path.resolve(targetPath) : path.resolve(base, targetPath);
-	const relative = path.relative(base, resolved);
-	if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error(`Path is outside ${baseDir}: ${targetPath}`);
-	return resolved;
-}
-
-export function resolveRealContainedPath(baseDir: string, targetPath: string): string {
-	const resolved = resolveContainedPath(baseDir, targetPath);
-	let realBase: string;
-	let realTarget: string;
-	try {
-		realBase = fs.realpathSync.native(baseDir);
-	} catch (baseError) {
-		throw new Error(`Cannot resolve real path of base directory ${baseDir}: ${baseError instanceof Error ? baseError.message : String(baseError)}`);
-	}
-	try {
-		realTarget = fs.realpathSync.native(resolved);
-	} catch (targetError) {
-		if ((targetError as NodeJS.ErrnoException).code === "ENOENT") {
-			throw new Error(`Path does not exist: ${resolved}`);
-		}
-		throw new Error(`Cannot resolve real path of ${resolved}: ${targetError instanceof Error ? targetError.message : String(targetError)}`);
-	}
-	const relative = path.relative(realBase, realTarget);
-	if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error(`Path is outside ${baseDir}: ${targetPath}`);
-	return realTarget;
-}
-
-export function resolveContainedRelativePath(baseDir: string, relativePath: string, kind = "path"): string {
-	const normalized = relativePath.replaceAll("\\", "/").replace(/^\.\/+/, "");
-	if (!normalized || normalized.split("/").some((segment) => segment === "..") || path.isAbsolute(normalized)) throw new Error(`Invalid ${kind}: ${relativePath}`);
-	return resolveContainedPath(baseDir, path.resolve(baseDir, normalized));
-}
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+export function isSafePathId(value: string): boolean {
+	return /^[A-Za-z0-9_-]+$/.test(value);
+}
+
+export function assertSafePathId(kind: string, value: string): string {
+	if (!isSafePathId(value)) throw new Error(`Invalid ${kind}: ${value}`);
+	return value;
+}
+
+export function resolveContainedPath(baseDir: string, targetPath: string): string {
+	const base = path.resolve(baseDir);
+	const resolved = path.isAbsolute(targetPath) ? path.resolve(targetPath) : path.resolve(base, targetPath);
+	const relative = path.relative(base, resolved);
+	if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error(`Path is outside ${baseDir}: ${targetPath}`);
+	return resolved;
+}
+
+export function resolveRealContainedPath(baseDir: string, targetPath: string): string {
+	const resolved = resolveContainedPath(baseDir, targetPath);
+	let realBase: string;
+	let realTarget: string;
+	try {
+		realBase = fs.realpathSync.native(baseDir);
+	} catch (baseError) {
+		throw new Error(`Cannot resolve real path of base directory ${baseDir}: ${baseError instanceof Error ? baseError.message : String(baseError)}`);
+	}
+	try {
+		realTarget = fs.realpathSync.native(resolved);
+	} catch (targetError) {
+		if ((targetError as NodeJS.ErrnoException).code === "ENOENT") {
+			throw new Error(`Path does not exist: ${resolved}`);
+		}
+		throw new Error(`Cannot resolve real path of ${resolved}: ${targetError instanceof Error ? targetError.message : String(targetError)}`);
+	}
+	const relative = path.relative(realBase, realTarget);
+	if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error(`Path is outside ${baseDir}: ${targetPath}`);
+	return realTarget;
+}
+
+export function resolveContainedRelativePath(baseDir: string, relativePath: string, kind = "path"): string {
+	const normalized = relativePath.replaceAll("\\", "/").replace(/^\.\/+/, "");
+	if (!normalized || normalized.split("/").some((segment) => segment === "..") || path.isAbsolute(normalized)) throw new Error(`Invalid ${kind}: ${relativePath}`);
+	return resolveContainedPath(baseDir, path.resolve(baseDir, normalized));
+}

package/src/utils/scan-cache.ts

@@ -0,0 +1,137 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+export interface ScanEntry {
+	/** Unique key for this entry (e.g., runId, artifactPath) */
+	key: string;
+	/** Filesystem path */
+	path: string;
+	/** Raw content (parsed JSON or text) */
+	raw: unknown;
+	/** File modification time */
+	mtimeMs: number;
+	/** File size in bytes */
+	sizeBytes: number;
+	/** When this entry was loaded */
+	loadedAtMs: number;
+}
+
+export interface ScanCacheOptions {
+	/** TTL in ms for cached entries. Default 1000. */
+	ttlMs?: number;
+	/** Maximum number of entries. Default 100. */
+	maxEntries?: number;
+}
+
+interface CacheBucket {
+	entries: Map<string, ScanEntry>;
+	expireAtMs: number;
+}
+
+/**
+ * Shared raw scan-entry cache for runs, artifacts, mailbox, transcripts.
+ * Provides deterministic sort order and invalidation on mutation.
+ */
+export class SharedScanCache {
+	#buckets = new Map<string, CacheBucket>();
+	#ttlMs: number;
+	#maxEntries: number;
+	#now: () => number;
+
+	constructor(options: ScanCacheOptions = {}) {
+		this.#ttlMs = options.ttlMs ?? 1000;
+		this.#maxEntries = options.maxEntries ?? 100;
+		this.#now = () => Date.now();
+	}
+
+	/** Get a cached entry by bucket and key. Returns undefined if not cached or expired. */
+	get(bucket: string, key: string): ScanEntry | undefined {
+		const b = this.#buckets.get(bucket);
+		if (!b) return undefined;
+		if (this.#now() > b.expireAtMs) {
+			this.#buckets.delete(bucket);
+			return undefined;
+		}
+		return b.entries.get(key);
+	}
+
+	/** Get all entries in a bucket. Returns empty array if expired or missing. */
+	list(bucket: string): ScanEntry[] {
+		const b = this.#buckets.get(bucket);
+		if (!b) return [];
+		if (this.#now() > b.expireAtMs) {
+			this.#buckets.delete(bucket);
+			return [];
+		}
+		return [...b.entries.values()].sort((a, b) => a.key.localeCompare(b.key));
+	}
+
+	/** Set an entry in a bucket. */
+	set(bucket: string, entry: ScanEntry): void {
+		let b = this.#buckets.get(bucket);
+		if (!b || this.#now() > b.expireAtMs) {
+			b = { entries: new Map(), expireAtMs: this.#now() + this.#ttlMs };
+			this.#buckets.set(bucket, b);
+		}
+		if (b.entries.size >= this.#maxEntries) {
+			// Evict oldest entry
+			const firstKey = b.entries.keys().next().value;
+			if (firstKey !== undefined) b.entries.delete(firstKey);
+		}
+		b.entries.set(entry.key, entry);
+	}
+
+	/** Invalidate a specific key in a bucket. */
+	invalidate(bucket: string, key: string): void {
+		const b = this.#buckets.get(bucket);
+		if (b) b.entries.delete(key);
+	}
+
+	/** Invalidate an entire bucket. */
+	invalidateBucket(bucket: string): void {
+		this.#buckets.delete(bucket);
+	}
+
+	/** Invalidate all buckets. */
+	clear(): void {
+		this.#buckets.clear();
+	}
+
+	/** Read a file, parse if JSON, and cache the result. */
+	readAndCache(bucket: string, key: string, filePath: string, parseJson = true): ScanEntry | undefined {
+		try {
+			if (!fs.existsSync(filePath)) return undefined;
+			const stat = fs.statSync(filePath);
+			const cached = this.get(bucket, key);
+			if (cached && cached.mtimeMs >= stat.mtimeMs && cached.sizeBytes === stat.size) return cached;
+			const content = fs.readFileSync(filePath, "utf-8");
+			const raw = parseJson ? JSON.parse(content) : content;
+			const entry: ScanEntry = { key, path: filePath, raw, mtimeMs: stat.mtimeMs, sizeBytes: stat.size, loadedAtMs: this.#now() };
+			this.set(bucket, entry);
+			return entry;
+		} catch {
+			return undefined;
+		}
+	}
+
+	/** Read a directory and cache entries for each file. */
+	scanAndCache(bucket: string, dirPath: string, parseJson = true): ScanEntry[] {
+		try {
+			if (!fs.existsSync(dirPath)) return [];
+			const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+			const results: ScanEntry[] = [];
+			for (const entry of entries) {
+				if (!entry.isFile()) continue;
+				const filePath = path.join(dirPath, entry.name);
+				const cached = this.readAndCache(bucket, entry.name, filePath, parseJson);
+				if (cached) results.push(cached);
+			}
+			return results.sort((a, b) => a.key.localeCompare(b.key));
+		} catch {
+			return [];
+		}
+	}
+}
+
+/** Global shared scan cache instance. */
+export const sharedScanCache = new SharedScanCache();

package/src/utils/sleep.ts

@@ -1,32 +1,32 @@
-/**
- * Sleep helper that respects abort signal.
- */
-export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
-	return new Promise((resolve, reject) => {
-		if (signal?.aborted) {
-			reject(signal.reason instanceof Error ? signal.reason : new Error("Aborted"));
-			return;
-		}
-
-		let settled = false;
-		const cleanup = (): void => {
-			if (signal) signal.removeEventListener("abort", onAbort);
-		};
-		const timeout = setTimeout(() => {
-			if (settled) return;
-			settled = true;
-			cleanup();
-			resolve();
-		}, ms);
-
-		const onAbort = (): void => {
-			if (settled) return;
-			settled = true;
-			clearTimeout(timeout);
-			cleanup();
-			reject(signal?.reason instanceof Error ? signal.reason : new Error("Aborted"));
-		};
-
-		signal?.addEventListener("abort", onAbort);
-	});
-}
+/**
+ * Sleep helper that respects abort signal.
+ */
+export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+	return new Promise((resolve, reject) => {
+		if (signal?.aborted) {
+			reject(signal.reason instanceof Error ? signal.reason : new Error("Aborted"));
+			return;
+		}
+
+		let settled = false;
+		const cleanup = (): void => {
+			if (signal) signal.removeEventListener("abort", onAbort);
+		};
+		const timeout = setTimeout(() => {
+			if (settled) return;
+			settled = true;
+			cleanup();
+			resolve();
+		}, ms);
+
+		const onAbort = (): void => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timeout);
+			cleanup();
+			reject(signal?.reason instanceof Error ? signal.reason : new Error("Aborted"));
+		};
+
+		signal?.addEventListener("abort", onAbort);
+	});
+}