testeiya 0.2.0 → 0.3.9

package/src/shell-env.ts

@@ -1,45 +0,0 @@
-import { Settings } from "@oh-my-pi/pi-coding-agent";
-import { userEnvKeys, projectEnvVars } from "./user-env.js";
-
-// Make the agent's shell commands see the CURRENT Testomat.io credentials.
-// The SDK's bash executor reads its shell env from `Settings.getShellConfig()`,
-// which delegates to pi-utils' procmgr — and procmgr snapshots `{...Bun.env}`
-// into a module-global cache on the FIRST bash command and never rebuilds, so
-// `process.env.TESTOMATIO` set at session creation (connection.ts) or by the
-// TUI's /connect never reaches agent shells. The SDK exposes no per-session
-// shell env (CreateAgentSessionOptions has no env field), so — like the
-// SOURCE_PATHS patch in session-factory.ts — this is a deliberate in-process
-// monkey patch: wrap the prototype method and overlay the live values of the
-// Testomat.io keys on every call. Absent keys are DELETED from the copy so a
-// token baked into the frozen snapshot at process start (e.g. a dev `.env`)
-// can't leak into a workspace that has none. The returned objects are spread
-// copies — procmgr hands out its cached object by reference; never mutate it.
-// The user's own env vars (Settings → ENV Variables, saved to ~/.testeiya/.env
-// via user-env.ts) are overlaid alongside these, so secrets like API keys reach
-// every agent bash command too. The workspace's project-scope vars
-// (`<cwd>/.testeiya/.env`, resolved per session via Settings.getCwd()) are
-// overlaid last and read live from disk on each call, so a project value wins
-// over a global one and a Settings save applies to the very next command.
-const SHELL_ENV_KEYS = ["TESTOMATIO", "TESTOMATIO_URL", "TESTOMATIO_PROJECT_ID"];
-
-const originalGetShellConfig = Settings.prototype.getShellConfig;
-if (originalGetShellConfig) {
-  Settings.prototype.getShellConfig = function (this: Settings) {
-    const config = originalGetShellConfig.call(this);
-    const env = { ...config.env };
-    for (const key of [...SHELL_ENV_KEYS, ...userEnvKeys()]) {
-      const value = process.env[key];
-      if (value) env[key] = value;
-      else delete env[key];
-    }
-    for (const [key, value] of Object.entries(projectEnvVars(this.getCwd()))) {
-      if (value) env[key] = value;
-    }
-    return { ...config, env };
-  };
-} else {
-  console.warn(
-    "[shell-env] Settings.prototype.getShellConfig is missing (SDK change?) — " +
-    "TESTOMATIO env vars may not reach agent shell commands"
-  );
-}

package/src/skills.ts

@@ -1,163 +0,0 @@
-import { join } from "node:path";
-import { readdirSync, readFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
-import { createRequire } from "node:module";
-import { parse as parseYaml } from "yaml";
-import dedent from "dedent";
-import type { Skill } from "@oh-my-pi/pi-coding-agent";
-import { CUSTOM_SKILLS_DIR, projectSkillsDir, resolveBundledSkillsDir } from "./project-dir.js";
-
-const require = createRequire(import.meta.url);
-
-/**
- * Whether the Playwright browser CLI (`@playwright/cli`) is installed. This gates
- * the browser-automation guidance in the system prompt so it's only advertised
- * when the tool is actually present. The `playwright-cli` *skill* is vendored
- * from GitHub (`microsoft/playwright-cli`) like every other prebuilt skill — it
- * is not read from the package.
- */
-export function hasPlaywrightCli(): boolean {
-  try {
-    require.resolve("@playwright/cli/package.json");
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * The skills tree shipped with the CLI package / desktop bundle, organized by
- * vendor folder: external vendors written by `bunosh skills:update` plus
- * first-party folders authored in this repo (e.g. `testeiya/`). Inside a vendor
- * folder — a root SKILL.md makes the folder one flat skill; a child folder with
- * a SKILL.md is a skill whose `category` is the (prettified) vendor folder; a
- * child folder without one is a category folder whose skill children get that
- * folder as `category`. Source `"bundled"`.
- */
-export function loadBundledSkills(): CategorizedSkill[] {
-  const dir = resolveBundledSkillsDir();
-  if (!existsSync(dir)) return [];
-  const skills: CategorizedSkill[] = [];
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    if (!entry.isDirectory()) continue;
-    skills.push(...readVendorDir(join(dir, entry.name), entry.name));
-  }
-  return skills;
-}
-
-/**
- * User-added skills from the global `~/.testeiya/skills` folder and, when a
- * workspace is open, its per-project `<cwd>/.testeiya/skills`. Directory entries
- * that are symlinks are followed so a skill can live in another repo and be
- * linked in. Deduped by name so a per-project or custom skill overrides one from
- * a broader scope.
- */
-export function loadCustomSkills(cwd?: string): CategorizedSkill[] {
-  const dirs = [CUSTOM_SKILLS_DIR];
-  if (cwd) dirs.push(projectSkillsDir(cwd));
-  const skills: CategorizedSkill[] = [];
-  for (const dir of dirs) skills.push(...readSkillsDir(dir, "custom"));
-  return dedupeSkillsByName(skills);
-}
-
-/** Create the global custom-skills folder and seed a README the first time. */
-export function ensureCustomSkillsDir(): string {
-  mkdirSync(CUSTOM_SKILLS_DIR, { recursive: true });
-  const readme = join(CUSTOM_SKILLS_DIR, "README.md");
-  if (!existsSync(readme)) writeFileSync(readme, CUSTOM_SKILLS_README);
-  return CUSTOM_SKILLS_DIR;
-}
-
-/** Keep one skill per name; the last occurrence wins. */
-export function dedupeSkillsByName<T extends Skill>(skills: T[]): T[] {
-  const byName = new Map<string, T>();
-  for (const skill of skills) byName.set(skill.name, skill);
-  return [...byName.values()];
-}
-
-/** One vendor folder of the bundled tree — see `loadBundledSkills` for the layout. */
-function readVendorDir(vendorDir: string, vendorName: string): CategorizedSkill[] {
-  const vendorCategory = prettifyCategory(vendorName);
-  const single = readSkill(vendorDir, vendorName, "bundled");
-  if (single) {
-    single.category = vendorCategory;
-    return [single];
-  }
-  const skills: CategorizedSkill[] = [];
-  for (const entry of readdirSync(vendorDir, { withFileTypes: true })) {
-    if (!entry.isDirectory() && !entry.isSymbolicLink()) continue;
-    const entryDir = join(vendorDir, entry.name);
-    const skill = readSkill(entryDir, entry.name, "bundled");
-    if (skill) {
-      skill.category = vendorCategory;
-      skills.push(skill);
-      continue;
-    }
-    for (const nested of readSkillsDir(entryDir, "bundled")) {
-      nested.category = prettifyCategory(entry.name);
-      skills.push(nested);
-    }
-  }
-  return skills;
-}
-
-/**
- * Scan one flat skills folder: every child directory that holds a `SKILL.md` is
- * a skill (symlinks are followed, so a skill can be linked in). Entries without
- * a `SKILL.md` are ignored. Used per-category for the vendored bundle, and for
- * the (flat) user skills dirs.
- */
-function readSkillsDir(dir: string, source: string): CategorizedSkill[] {
-  if (!existsSync(dir)) return [];
-  const skills: CategorizedSkill[] = [];
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    if (!entry.isDirectory() && !entry.isSymbolicLink()) continue;
-    const skill = readSkill(join(dir, entry.name), entry.name, source);
-    if (skill) skills.push(skill);
-  }
-  return skills;
-}
-
-/** Turn a category folder slug (`test-management`) into a label (`Test Management`). */
-function prettifyCategory(folder: string): string {
-  return folder
-    .split(/[-_\s]+/)
-    .filter(Boolean)
-    .map((word) => word[0].toUpperCase() + word.slice(1))
-    .join(" ");
-}
-
-function readSkill(skillDir: string, fallbackName: string, source: string): CategorizedSkill | null {
-  const skillFile = join(skillDir, "SKILL.md");
-  if (!existsSync(skillFile)) return null;
-  const content = readFileSync(skillFile, "utf-8");
-  const fmMatch = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
-  const fm = fmMatch ? parseYaml(fmMatch[1]) : null;
-  const fmName = fm?.name?.toString().trim();
-  // A skill name is used as a mention token, so it must be a slug; fall back to
-  // the folder name when the frontmatter name isn't one (e.g. a spaced title).
-  const name = fmName && /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(fmName) ? fmName : fallbackName;
-  const description = fm?.description?.toString().trim() || "";
-  return { name, description, filePath: skillFile, baseDir: skillDir, source };
-}
-
-const CUSTOM_SKILLS_README = dedent`
-  # Testeiya custom skills
-
-  Drop a skill here to make it available to the agent. A skill is a folder that
-  contains a \`SKILL.md\` file — YAML frontmatter with a \`name\` and
-  \`description\`, followed by the skill's instructions:
-
-      ~/.testeiya/skills/my-skill/SKILL.md
-
-  You can also symlink a skill that already lives in another repository:
-
-      ln -s /path/to/repo/skills/my-skill ~/.testeiya/skills/my-skill
-
-  After adding or linking a skill, click **Refresh** in the Skills menu.
-
-  Skills placed here are available in every workspace. To scope a skill to a
-  single project, put it in that workspace's \`.testeiya/skills/\` folder instead.
-` + "\n";
-
-/** An SDK `Skill` plus the category folder it was discovered under, if any. */
-export type CategorizedSkill = Skill & { category?: string };

package/src/sync-snapshot.ts

@@ -1,140 +0,0 @@
-// Tracks which manual-test files changed since the last Testomat.io sync.
-//
-// After every successful `check-tests` pull/push the synced files are
-// content-hashed into `.testeiya/sync-snapshot.json` (the baseline = the state
-// the remote agrees with). The file tree then diffs the current files against
-// that baseline so a created/edited-but-not-pushed file can be flagged.
-
-import crypto from "node:crypto";
-import fs from "node:fs";
-import path from "node:path";
-import { readJson, writeJson } from "./json-store.js";
-import { resolveManualTestsDir, VENDOR_DIRS } from "./workspace-model.js";
-import { syncSnapshotPath } from "./project-dir.js";
-
-const TEST_MD_RE = /\.test\.md$/i;
-const MAX_FILES = 5000;
-
-/**
- * Snapshot the manual-test files `check-tests` just synced. `syncedDir` is the
- * dir it ran in, relative to `cwd` (`"."` for a root project). Only that subtree
- * is re-hashed, so multi-project workspaces (one snapshot, many synced subdirs)
- * keep each other's entries.
- *
- * `onlyFiles` (cwd-relative paths) scopes a *partial* sync: a push/pull of a
- * single suite or just the changed files re-baselines only those files, so the
- * other un-pushed files keep their "changed" markers.
- */
-export function writeSyncSnapshot(cwd: string, syncedDir: string, onlyFiles?: string[]): void {
-  const snapshot = readJson<SyncSnapshot>(syncSnapshotPath(cwd), emptySnapshot());
-  if (onlyFiles) {
-    for (const rel of onlyFiles) {
-      const hash = hashFile(path.resolve(cwd, rel));
-      if (hash) snapshot.files[rel] = hash;
-      else delete snapshot.files[rel];
-    }
-    snapshot.syncedAt = new Date().toISOString();
-    writeJson(syncSnapshotPath(cwd), snapshot);
-    return;
-  }
-  const base = syncedDir === "." ? "" : syncedDir;
-  for (const key of Object.keys(snapshot.files)) {
-    if (isUnder(key, base)) delete snapshot.files[key];
-  }
-  hashDir(path.resolve(cwd, base), base, snapshot.files, { count: 0 });
-  snapshot.syncedAt = new Date().toISOString();
-  writeJson(syncSnapshotPath(cwd), snapshot);
-}
-
-/**
- * Establish a baseline if the workspace has manual tests but never recorded one
- * (e.g. a folder opened without a sync, or pulled before tracking existed). The
- * current state becomes "the last synced state", so edits from now on are
- * detected without forcing the user to push first.
- */
-export function ensureSyncBaseline(cwd: string): void {
-  const manualDir = resolveManualTestsDir(cwd);
-  if (manualDir === null) return;
-  if (fs.existsSync(syncSnapshotPath(cwd))) return;
-  writeSyncSnapshot(cwd, manualDir === "" ? "." : manualDir);
-}
-
-/**
- * Map of `cwd`-relative path → status for manual-test files that differ from the
- * last sync. Empty when nothing is loaded or no snapshot exists yet (so a
- * freshly pulled workspace shows clean rather than lighting up entirely).
- */
-export function computeChangedFiles(cwd: string): Record<string, FileStatus> {
-  const manualDir = resolveManualTestsDir(cwd);
-  if (manualDir === null) return {};
-  const snapshot = readJson<SyncSnapshot | null>(syncSnapshotPath(cwd), null);
-  if (!snapshot) return {};
-
-  const current: Record<string, string> = {};
-  hashDir(path.resolve(cwd, manualDir), manualDir, current, { count: 0 });
-
-  const changed: Record<string, FileStatus> = {};
-  for (const rel of Object.keys(current)) {
-    const before = snapshot.files[rel];
-    if (!before) {
-      changed[rel] = "created";
-      continue;
-    }
-    if (before !== current[rel]) changed[rel] = "changed";
-  }
-  return changed;
-}
-
-function hashDir(
-  absDir: string,
-  relDir: string,
-  out: Record<string, string>,
-  state: { count: number }
-): void {
-  if (state.count >= MAX_FILES) return;
-  let entries: fs.Dirent[];
-  try {
-    entries = fs.readdirSync(absDir, { withFileTypes: true });
-  } catch {
-    return;
-  }
-  for (const entry of entries) {
-    if (state.count >= MAX_FILES) return;
-    if (entry.name.startsWith(".") || VENDOR_DIRS.has(entry.name)) continue;
-    const childAbs = path.join(absDir, entry.name);
-    const childRel = relDir ? `${relDir}/${entry.name}` : entry.name;
-    if (entry.isDirectory()) {
-      hashDir(childAbs, childRel, out, state);
-      continue;
-    }
-    if (!entry.isFile() || !TEST_MD_RE.test(entry.name)) continue;
-    const hash = hashFile(childAbs);
-    if (!hash) continue;
-    out[childRel] = hash;
-    state.count++;
-  }
-}
-
-function hashFile(absFile: string): string | null {
-  try {
-    return crypto.createHash("sha1").update(fs.readFileSync(absFile)).digest("hex");
-  } catch {
-    return null;
-  }
-}
-
-function isUnder(key: string, base: string): boolean {
-  if (base === "") return true;
-  return key === base || key.startsWith(`${base}/`);
-}
-
-function emptySnapshot(): SyncSnapshot {
-  return { syncedAt: "", files: {} };
-}
-
-export type FileStatus = "created" | "changed";
-
-interface SyncSnapshot {
-  syncedAt: string;
-  files: Record<string, string>;
-}

package/src/telemetry.ts

@@ -1,239 +0,0 @@
-// OpenTelemetry → Langfuse observability for the agent.
-//
-// Two parts:
-//   1. initTelemetry() — one-time NodeSDK + LangfuseSpanProcessor bootstrap,
-//      gated on LANGFUSE_PUBLIC_KEY + LANGFUSE_SECRET_KEY. A no-op (and no
-//      dependency cost at runtime) when the keys are absent. Called from
-//      app-server.ts right after the .env files are loaded.
-//   2. createTelemetryExtension() — a per-session extension that turns the
-//      agent's hook events into a Langfuse trace tree:
-//        before_agent_start → root (agent) observation = one trace per prompt
-//          message_start/_end → child generation (LLM call) with usage + cost
-//          tool_execution_start/_end → child tool observation (args + result)
-//        agent_end → close the trace and flush
-//      All prompts of one agent session share the SDK sessionId, so Langfuse
-//      groups them into a single session.
-
-import { NodeSDK } from "@opentelemetry/sdk-node";
-import { LangfuseSpanProcessor } from "@langfuse/otel";
-import { startObservation, LangfuseOtelSpanAttributes } from "@langfuse/tracing";
-import type {
-  LangfuseAgent,
-  LangfuseGeneration,
-  LangfuseTool,
-  ObservationLevel,
-} from "@langfuse/tracing";
-import type { ExtensionFactory } from "@oh-my-pi/pi-coding-agent";
-
-let processor: LangfuseSpanProcessor | undefined;
-let sdk: NodeSDK | undefined;
-let enabled = false;
-
-export function initTelemetry(): void {
-  if (sdk) return;
-
-  const publicKey = process.env.LANGFUSE_PUBLIC_KEY;
-  const secretKey = process.env.LANGFUSE_SECRET_KEY;
-  if (!publicKey || !secretKey) {
-    console.log(
-      "[telemetry] Langfuse disabled — set LANGFUSE_PUBLIC_KEY + LANGFUSE_SECRET_KEY to enable",
-    );
-    return;
-  }
-
-  const baseUrl = process.env.LANGFUSE_BASE_URL ?? process.env.LANGFUSE_BASEURL;
-  processor = new LangfuseSpanProcessor({ publicKey, secretKey, baseUrl });
-  sdk = new NodeSDK({ spanProcessors: [processor] });
-  sdk.start();
-  enabled = true;
-  console.log(`[telemetry] Langfuse enabled → ${baseUrl ?? "https://cloud.langfuse.com"}`);
-}
-
-export function isTelemetryEnabled(): boolean {
-  return enabled;
-}
-
-export async function flushTelemetry(): Promise<void> {
-  if (!processor) return;
-  try {
-    await processor.forceFlush();
-  } catch (err) {
-    console.error(
-      "[telemetry] flush failed:",
-      err instanceof Error ? err.message : err,
-    );
-  }
-}
-
-export function createTelemetryExtension(meta: TelemetryMeta): ExtensionFactory {
-  return ((pi: any) => {
-    let root: LangfuseAgent | undefined;
-    let generation: LangfuseGeneration | undefined;
-    let contextMessages: unknown;
-    const tools = new Map<string, LangfuseTool>();
-
-    pi.on("before_agent_start", (event: any, ctx: any) => {
-      if (root) endTrace();
-      const prompt = event?.prompt ?? "";
-      const name = traceName(prompt);
-      root = startObservation(name, { input: prompt }, { asType: "agent" });
-      root.update({
-        metadata: {
-          cwd: meta.cwd,
-          model: meta.modelId,
-          provider: meta.provider,
-          projectIds: meta.projectIds,
-        },
-      });
-      const span = root.otelSpan;
-      span.setAttribute(LangfuseOtelSpanAttributes.TRACE_NAME, name);
-      span.setAttribute(LangfuseOtelSpanAttributes.TRACE_INPUT, serialize(prompt));
-      span.setAttribute(
-        LangfuseOtelSpanAttributes.TRACE_TAGS,
-        JSON.stringify(["testeiya", meta.mode]),
-      );
-      const sessionId = ctx?.sessionManager?.getSessionId?.();
-      if (sessionId) {
-        span.setAttribute(LangfuseOtelSpanAttributes.TRACE_SESSION_ID, sessionId);
-      }
-    });
-
-    pi.on("context", (event: any) => {
-      contextMessages = event?.messages;
-    });
-
-    pi.on("message_start", (event: any) => {
-      if (!root) return;
-      if (event?.message?.role !== "assistant") return;
-      generation = root.startObservation(
-        "llm-generation",
-        { input: contextMessages ?? [], model: meta.modelId },
-        { asType: "generation" },
-      );
-    });
-
-    pi.on("message_end", (event: any) => {
-      const msg = event?.message;
-      if (!generation || msg?.role !== "assistant") return;
-      generation.update({
-        model: msg.model ?? meta.modelId,
-        output: msg.content,
-        usageDetails: usageDetails(msg.usage),
-        costDetails: costDetails(msg.usage),
-      });
-      generation.end();
-      generation = undefined;
-    });
-
-    pi.on("tool_execution_start", (event: any) => {
-      if (!root) return;
-      const name = event?.toolName ?? "tool";
-      tools.set(
-        event.toolCallId,
-        root.startObservation(
-          name,
-          { input: event?.args, metadata: { toolName: name, intent: event?.intent } },
-          { asType: "tool" },
-        ),
-      );
-    });
-
-    pi.on("tool_execution_end", (event: any) => {
-      const span = tools.get(event?.toolCallId);
-      if (!span) return;
-      let level: ObservationLevel = "DEFAULT";
-      if (event?.isError) level = "ERROR";
-      span.update({ output: event?.result, level });
-      span.end();
-      tools.delete(event.toolCallId);
-    });
-
-    pi.on("agent_end", (event: any) => {
-      if (!root) return;
-      const output = finalText(event?.messages);
-      root.otelSpan.setAttribute(
-        LangfuseOtelSpanAttributes.TRACE_OUTPUT,
-        serialize(output),
-      );
-      root.update({ output });
-      endTrace();
-      void flushTelemetry();
-    });
-
-    function endTrace(): void {
-      for (const span of tools.values()) span.end();
-      tools.clear();
-      if (generation) {
-        generation.end();
-        generation = undefined;
-      }
-      if (root) {
-        root.end();
-        root = undefined;
-      }
-    }
-  }) as ExtensionFactory;
-}
-
-function usageDetails(usage: any): Record<string, number> | undefined {
-  if (!usage) return undefined;
-  return {
-    input: usage.input,
-    output: usage.output,
-    cache_read_input_tokens: usage.cacheRead,
-    cache_creation_input_tokens: usage.cacheWrite,
-    total: usage.totalTokens,
-  };
-}
-
-function costDetails(usage: any): Record<string, number> | undefined {
-  if (!usage?.cost) return undefined;
-  return {
-    input: usage.cost.input,
-    output: usage.cost.output,
-    total: usage.cost.total,
-  };
-}
-
-function finalText(messages: any): string | undefined {
-  if (!Array.isArray(messages)) return undefined;
-  for (let i = messages.length - 1; i >= 0; i--) {
-    if (messages[i]?.role === "assistant") return textOf(messages[i].content);
-  }
-  return undefined;
-}
-
-function textOf(content: any): string | undefined {
-  if (typeof content === "string") return content;
-  if (!Array.isArray(content)) return undefined;
-  return content
-    .filter((c) => c?.type === "text")
-    .map((c) => c.text)
-    .join("\n");
-}
-
-function traceName(prompt: unknown): string {
-  let text = "";
-  if (typeof prompt === "string") text = prompt;
-  const firstLine = text.split("\n")[0]?.trim() ?? "";
-  if (!firstLine) return "agent-run";
-  if (firstLine.length <= 60) return firstLine;
-  return `${firstLine.slice(0, 60)}…`;
-}
-
-function serialize(value: unknown): string {
-  if (typeof value === "string") return value;
-  try {
-    return JSON.stringify(value) ?? "";
-  } catch {
-    return String(value);
-  }
-}
-
-interface TelemetryMeta {
-  cwd: string;
-  mode: string;
-  modelId?: string;
-  provider?: string;
-  projectIds?: string[];
-}