testeiya 0.2.0

package/src/skills.ts

@@ -0,0 +1,163 @@
+import { join } from "node:path";
+import { readdirSync, readFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { parse as parseYaml } from "yaml";
+import dedent from "dedent";
+import type { Skill } from "@oh-my-pi/pi-coding-agent";
+import { CUSTOM_SKILLS_DIR, projectSkillsDir, resolveBundledSkillsDir } from "./project-dir.js";
+
+const require = createRequire(import.meta.url);
+
+/**
+ * Whether the Playwright browser CLI (`@playwright/cli`) is installed. This gates
+ * the browser-automation guidance in the system prompt so it's only advertised
+ * when the tool is actually present. The `playwright-cli` *skill* is vendored
+ * from GitHub (`microsoft/playwright-cli`) like every other prebuilt skill — it
+ * is not read from the package.
+ */
+export function hasPlaywrightCli(): boolean {
+  try {
+    require.resolve("@playwright/cli/package.json");
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * The skills tree shipped with the CLI package / desktop bundle, organized by
+ * vendor folder: external vendors written by `bunosh skills:update` plus
+ * first-party folders authored in this repo (e.g. `testeiya/`). Inside a vendor
+ * folder — a root SKILL.md makes the folder one flat skill; a child folder with
+ * a SKILL.md is a skill whose `category` is the (prettified) vendor folder; a
+ * child folder without one is a category folder whose skill children get that
+ * folder as `category`. Source `"bundled"`.
+ */
+export function loadBundledSkills(): CategorizedSkill[] {
+  const dir = resolveBundledSkillsDir();
+  if (!existsSync(dir)) return [];
+  const skills: CategorizedSkill[] = [];
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    skills.push(...readVendorDir(join(dir, entry.name), entry.name));
+  }
+  return skills;
+}
+
+/**
+ * User-added skills from the global `~/.testeiya/skills` folder and, when a
+ * workspace is open, its per-project `<cwd>/.testeiya/skills`. Directory entries
+ * that are symlinks are followed so a skill can live in another repo and be
+ * linked in. Deduped by name so a per-project or custom skill overrides one from
+ * a broader scope.
+ */
+export function loadCustomSkills(cwd?: string): CategorizedSkill[] {
+  const dirs = [CUSTOM_SKILLS_DIR];
+  if (cwd) dirs.push(projectSkillsDir(cwd));
+  const skills: CategorizedSkill[] = [];
+  for (const dir of dirs) skills.push(...readSkillsDir(dir, "custom"));
+  return dedupeSkillsByName(skills);
+}
+
+/** Create the global custom-skills folder and seed a README the first time. */
+export function ensureCustomSkillsDir(): string {
+  mkdirSync(CUSTOM_SKILLS_DIR, { recursive: true });
+  const readme = join(CUSTOM_SKILLS_DIR, "README.md");
+  if (!existsSync(readme)) writeFileSync(readme, CUSTOM_SKILLS_README);
+  return CUSTOM_SKILLS_DIR;
+}
+
+/** Keep one skill per name; the last occurrence wins. */
+export function dedupeSkillsByName<T extends Skill>(skills: T[]): T[] {
+  const byName = new Map<string, T>();
+  for (const skill of skills) byName.set(skill.name, skill);
+  return [...byName.values()];
+}
+
+/** One vendor folder of the bundled tree — see `loadBundledSkills` for the layout. */
+function readVendorDir(vendorDir: string, vendorName: string): CategorizedSkill[] {
+  const vendorCategory = prettifyCategory(vendorName);
+  const single = readSkill(vendorDir, vendorName, "bundled");
+  if (single) {
+    single.category = vendorCategory;
+    return [single];
+  }
+  const skills: CategorizedSkill[] = [];
+  for (const entry of readdirSync(vendorDir, { withFileTypes: true })) {
+    if (!entry.isDirectory() && !entry.isSymbolicLink()) continue;
+    const entryDir = join(vendorDir, entry.name);
+    const skill = readSkill(entryDir, entry.name, "bundled");
+    if (skill) {
+      skill.category = vendorCategory;
+      skills.push(skill);
+      continue;
+    }
+    for (const nested of readSkillsDir(entryDir, "bundled")) {
+      nested.category = prettifyCategory(entry.name);
+      skills.push(nested);
+    }
+  }
+  return skills;
+}
+
+/**
+ * Scan one flat skills folder: every child directory that holds a `SKILL.md` is
+ * a skill (symlinks are followed, so a skill can be linked in). Entries without
+ * a `SKILL.md` are ignored. Used per-category for the vendored bundle, and for
+ * the (flat) user skills dirs.
+ */
+function readSkillsDir(dir: string, source: string): CategorizedSkill[] {
+  if (!existsSync(dir)) return [];
+  const skills: CategorizedSkill[] = [];
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (!entry.isDirectory() && !entry.isSymbolicLink()) continue;
+    const skill = readSkill(join(dir, entry.name), entry.name, source);
+    if (skill) skills.push(skill);
+  }
+  return skills;
+}
+
+/** Turn a category folder slug (`test-management`) into a label (`Test Management`). */
+function prettifyCategory(folder: string): string {
+  return folder
+    .split(/[-_\s]+/)
+    .filter(Boolean)
+    .map((word) => word[0].toUpperCase() + word.slice(1))
+    .join(" ");
+}
+
+function readSkill(skillDir: string, fallbackName: string, source: string): CategorizedSkill | null {
+  const skillFile = join(skillDir, "SKILL.md");
+  if (!existsSync(skillFile)) return null;
+  const content = readFileSync(skillFile, "utf-8");
+  const fmMatch = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  const fm = fmMatch ? parseYaml(fmMatch[1]) : null;
+  const fmName = fm?.name?.toString().trim();
+  // A skill name is used as a mention token, so it must be a slug; fall back to
+  // the folder name when the frontmatter name isn't one (e.g. a spaced title).
+  const name = fmName && /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(fmName) ? fmName : fallbackName;
+  const description = fm?.description?.toString().trim() || "";
+  return { name, description, filePath: skillFile, baseDir: skillDir, source };
+}
+
+const CUSTOM_SKILLS_README = dedent`
+  # Testeiya custom skills
+
+  Drop a skill here to make it available to the agent. A skill is a folder that
+  contains a \`SKILL.md\` file — YAML frontmatter with a \`name\` and
+  \`description\`, followed by the skill's instructions:
+
+      ~/.testeiya/skills/my-skill/SKILL.md
+
+  You can also symlink a skill that already lives in another repository:
+
+      ln -s /path/to/repo/skills/my-skill ~/.testeiya/skills/my-skill
+
+  After adding or linking a skill, click **Refresh** in the Skills menu.
+
+  Skills placed here are available in every workspace. To scope a skill to a
+  single project, put it in that workspace's \`.testeiya/skills/\` folder instead.
+` + "\n";
+
+/** An SDK `Skill` plus the category folder it was discovered under, if any. */
+export type CategorizedSkill = Skill & { category?: string };

package/src/sync-snapshot.ts

@@ -0,0 +1,140 @@
+// Tracks which manual-test files changed since the last Testomat.io sync.
+//
+// After every successful `check-tests` pull/push the synced files are
+// content-hashed into `.testeiya/sync-snapshot.json` (the baseline = the state
+// the remote agrees with). The file tree then diffs the current files against
+// that baseline so a created/edited-but-not-pushed file can be flagged.
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { readJson, writeJson } from "./json-store.js";
+import { resolveManualTestsDir, VENDOR_DIRS } from "./workspace-model.js";
+import { syncSnapshotPath } from "./project-dir.js";
+
+const TEST_MD_RE = /\.test\.md$/i;
+const MAX_FILES = 5000;
+
+/**
+ * Snapshot the manual-test files `check-tests` just synced. `syncedDir` is the
+ * dir it ran in, relative to `cwd` (`"."` for a root project). Only that subtree
+ * is re-hashed, so multi-project workspaces (one snapshot, many synced subdirs)
+ * keep each other's entries.
+ *
+ * `onlyFiles` (cwd-relative paths) scopes a *partial* sync: a push/pull of a
+ * single suite or just the changed files re-baselines only those files, so the
+ * other un-pushed files keep their "changed" markers.
+ */
+export function writeSyncSnapshot(cwd: string, syncedDir: string, onlyFiles?: string[]): void {
+  const snapshot = readJson<SyncSnapshot>(syncSnapshotPath(cwd), emptySnapshot());
+  if (onlyFiles) {
+    for (const rel of onlyFiles) {
+      const hash = hashFile(path.resolve(cwd, rel));
+      if (hash) snapshot.files[rel] = hash;
+      else delete snapshot.files[rel];
+    }
+    snapshot.syncedAt = new Date().toISOString();
+    writeJson(syncSnapshotPath(cwd), snapshot);
+    return;
+  }
+  const base = syncedDir === "." ? "" : syncedDir;
+  for (const key of Object.keys(snapshot.files)) {
+    if (isUnder(key, base)) delete snapshot.files[key];
+  }
+  hashDir(path.resolve(cwd, base), base, snapshot.files, { count: 0 });
+  snapshot.syncedAt = new Date().toISOString();
+  writeJson(syncSnapshotPath(cwd), snapshot);
+}
+
+/**
+ * Establish a baseline if the workspace has manual tests but never recorded one
+ * (e.g. a folder opened without a sync, or pulled before tracking existed). The
+ * current state becomes "the last synced state", so edits from now on are
+ * detected without forcing the user to push first.
+ */
+export function ensureSyncBaseline(cwd: string): void {
+  const manualDir = resolveManualTestsDir(cwd);
+  if (manualDir === null) return;
+  if (fs.existsSync(syncSnapshotPath(cwd))) return;
+  writeSyncSnapshot(cwd, manualDir === "" ? "." : manualDir);
+}
+
+/**
+ * Map of `cwd`-relative path → status for manual-test files that differ from the
+ * last sync. Empty when nothing is loaded or no snapshot exists yet (so a
+ * freshly pulled workspace shows clean rather than lighting up entirely).
+ */
+export function computeChangedFiles(cwd: string): Record<string, FileStatus> {
+  const manualDir = resolveManualTestsDir(cwd);
+  if (manualDir === null) return {};
+  const snapshot = readJson<SyncSnapshot | null>(syncSnapshotPath(cwd), null);
+  if (!snapshot) return {};
+
+  const current: Record<string, string> = {};
+  hashDir(path.resolve(cwd, manualDir), manualDir, current, { count: 0 });
+
+  const changed: Record<string, FileStatus> = {};
+  for (const rel of Object.keys(current)) {
+    const before = snapshot.files[rel];
+    if (!before) {
+      changed[rel] = "created";
+      continue;
+    }
+    if (before !== current[rel]) changed[rel] = "changed";
+  }
+  return changed;
+}
+
+function hashDir(
+  absDir: string,
+  relDir: string,
+  out: Record<string, string>,
+  state: { count: number }
+): void {
+  if (state.count >= MAX_FILES) return;
+  let entries: fs.Dirent[];
+  try {
+    entries = fs.readdirSync(absDir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    if (state.count >= MAX_FILES) return;
+    if (entry.name.startsWith(".") || VENDOR_DIRS.has(entry.name)) continue;
+    const childAbs = path.join(absDir, entry.name);
+    const childRel = relDir ? `${relDir}/${entry.name}` : entry.name;
+    if (entry.isDirectory()) {
+      hashDir(childAbs, childRel, out, state);
+      continue;
+    }
+    if (!entry.isFile() || !TEST_MD_RE.test(entry.name)) continue;
+    const hash = hashFile(childAbs);
+    if (!hash) continue;
+    out[childRel] = hash;
+    state.count++;
+  }
+}
+
+function hashFile(absFile: string): string | null {
+  try {
+    return crypto.createHash("sha1").update(fs.readFileSync(absFile)).digest("hex");
+  } catch {
+    return null;
+  }
+}
+
+function isUnder(key: string, base: string): boolean {
+  if (base === "") return true;
+  return key === base || key.startsWith(`${base}/`);
+}
+
+function emptySnapshot(): SyncSnapshot {
+  return { syncedAt: "", files: {} };
+}
+
+export type FileStatus = "created" | "changed";
+
+interface SyncSnapshot {
+  syncedAt: string;
+  files: Record<string, string>;
+}

package/src/telemetry.ts

@@ -0,0 +1,239 @@
+// OpenTelemetry → Langfuse observability for the agent.
+//
+// Two parts:
+//   1. initTelemetry() — one-time NodeSDK + LangfuseSpanProcessor bootstrap,
+//      gated on LANGFUSE_PUBLIC_KEY + LANGFUSE_SECRET_KEY. A no-op (and no
+//      dependency cost at runtime) when the keys are absent. Called from
+//      app-server.ts right after the .env files are loaded.
+//   2. createTelemetryExtension() — a per-session extension that turns the
+//      agent's hook events into a Langfuse trace tree:
+//        before_agent_start → root (agent) observation = one trace per prompt
+//          message_start/_end → child generation (LLM call) with usage + cost
+//          tool_execution_start/_end → child tool observation (args + result)
+//        agent_end → close the trace and flush
+//      All prompts of one agent session share the SDK sessionId, so Langfuse
+//      groups them into a single session.
+
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { LangfuseSpanProcessor } from "@langfuse/otel";
+import { startObservation, LangfuseOtelSpanAttributes } from "@langfuse/tracing";
+import type {
+  LangfuseAgent,
+  LangfuseGeneration,
+  LangfuseTool,
+  ObservationLevel,
+} from "@langfuse/tracing";
+import type { ExtensionFactory } from "@oh-my-pi/pi-coding-agent";
+
+let processor: LangfuseSpanProcessor | undefined;
+let sdk: NodeSDK | undefined;
+let enabled = false;
+
+export function initTelemetry(): void {
+  if (sdk) return;
+
+  const publicKey = process.env.LANGFUSE_PUBLIC_KEY;
+  const secretKey = process.env.LANGFUSE_SECRET_KEY;
+  if (!publicKey || !secretKey) {
+    console.log(
+      "[telemetry] Langfuse disabled — set LANGFUSE_PUBLIC_KEY + LANGFUSE_SECRET_KEY to enable",
+    );
+    return;
+  }
+
+  const baseUrl = process.env.LANGFUSE_BASE_URL ?? process.env.LANGFUSE_BASEURL;
+  processor = new LangfuseSpanProcessor({ publicKey, secretKey, baseUrl });
+  sdk = new NodeSDK({ spanProcessors: [processor] });
+  sdk.start();
+  enabled = true;
+  console.log(`[telemetry] Langfuse enabled → ${baseUrl ?? "https://cloud.langfuse.com"}`);
+}
+
+export function isTelemetryEnabled(): boolean {
+  return enabled;
+}
+
+export async function flushTelemetry(): Promise<void> {
+  if (!processor) return;
+  try {
+    await processor.forceFlush();
+  } catch (err) {
+    console.error(
+      "[telemetry] flush failed:",
+      err instanceof Error ? err.message : err,
+    );
+  }
+}
+
+export function createTelemetryExtension(meta: TelemetryMeta): ExtensionFactory {
+  return ((pi: any) => {
+    let root: LangfuseAgent | undefined;
+    let generation: LangfuseGeneration | undefined;
+    let contextMessages: unknown;
+    const tools = new Map<string, LangfuseTool>();
+
+    pi.on("before_agent_start", (event: any, ctx: any) => {
+      if (root) endTrace();
+      const prompt = event?.prompt ?? "";
+      const name = traceName(prompt);
+      root = startObservation(name, { input: prompt }, { asType: "agent" });
+      root.update({
+        metadata: {
+          cwd: meta.cwd,
+          model: meta.modelId,
+          provider: meta.provider,
+          projectIds: meta.projectIds,
+        },
+      });
+      const span = root.otelSpan;
+      span.setAttribute(LangfuseOtelSpanAttributes.TRACE_NAME, name);
+      span.setAttribute(LangfuseOtelSpanAttributes.TRACE_INPUT, serialize(prompt));
+      span.setAttribute(
+        LangfuseOtelSpanAttributes.TRACE_TAGS,
+        JSON.stringify(["testeiya", meta.mode]),
+      );
+      const sessionId = ctx?.sessionManager?.getSessionId?.();
+      if (sessionId) {
+        span.setAttribute(LangfuseOtelSpanAttributes.TRACE_SESSION_ID, sessionId);
+      }
+    });
+
+    pi.on("context", (event: any) => {
+      contextMessages = event?.messages;
+    });
+
+    pi.on("message_start", (event: any) => {
+      if (!root) return;
+      if (event?.message?.role !== "assistant") return;
+      generation = root.startObservation(
+        "llm-generation",
+        { input: contextMessages ?? [], model: meta.modelId },
+        { asType: "generation" },
+      );
+    });
+
+    pi.on("message_end", (event: any) => {
+      const msg = event?.message;
+      if (!generation || msg?.role !== "assistant") return;
+      generation.update({
+        model: msg.model ?? meta.modelId,
+        output: msg.content,
+        usageDetails: usageDetails(msg.usage),
+        costDetails: costDetails(msg.usage),
+      });
+      generation.end();
+      generation = undefined;
+    });
+
+    pi.on("tool_execution_start", (event: any) => {
+      if (!root) return;
+      const name = event?.toolName ?? "tool";
+      tools.set(
+        event.toolCallId,
+        root.startObservation(
+          name,
+          { input: event?.args, metadata: { toolName: name, intent: event?.intent } },
+          { asType: "tool" },
+        ),
+      );
+    });
+
+    pi.on("tool_execution_end", (event: any) => {
+      const span = tools.get(event?.toolCallId);
+      if (!span) return;
+      let level: ObservationLevel = "DEFAULT";
+      if (event?.isError) level = "ERROR";
+      span.update({ output: event?.result, level });
+      span.end();
+      tools.delete(event.toolCallId);
+    });
+
+    pi.on("agent_end", (event: any) => {
+      if (!root) return;
+      const output = finalText(event?.messages);
+      root.otelSpan.setAttribute(
+        LangfuseOtelSpanAttributes.TRACE_OUTPUT,
+        serialize(output),
+      );
+      root.update({ output });
+      endTrace();
+      void flushTelemetry();
+    });
+
+    function endTrace(): void {
+      for (const span of tools.values()) span.end();
+      tools.clear();
+      if (generation) {
+        generation.end();
+        generation = undefined;
+      }
+      if (root) {
+        root.end();
+        root = undefined;
+      }
+    }
+  }) as ExtensionFactory;
+}
+
+function usageDetails(usage: any): Record<string, number> | undefined {
+  if (!usage) return undefined;
+  return {
+    input: usage.input,
+    output: usage.output,
+    cache_read_input_tokens: usage.cacheRead,
+    cache_creation_input_tokens: usage.cacheWrite,
+    total: usage.totalTokens,
+  };
+}
+
+function costDetails(usage: any): Record<string, number> | undefined {
+  if (!usage?.cost) return undefined;
+  return {
+    input: usage.cost.input,
+    output: usage.cost.output,
+    total: usage.cost.total,
+  };
+}
+
+function finalText(messages: any): string | undefined {
+  if (!Array.isArray(messages)) return undefined;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (messages[i]?.role === "assistant") return textOf(messages[i].content);
+  }
+  return undefined;
+}
+
+function textOf(content: any): string | undefined {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return undefined;
+  return content
+    .filter((c) => c?.type === "text")
+    .map((c) => c.text)
+    .join("\n");
+}
+
+function traceName(prompt: unknown): string {
+  let text = "";
+  if (typeof prompt === "string") text = prompt;
+  const firstLine = text.split("\n")[0]?.trim() ?? "";
+  if (!firstLine) return "agent-run";
+  if (firstLine.length <= 60) return firstLine;
+  return `${firstLine.slice(0, 60)}…`;
+}
+
+function serialize(value: unknown): string {
+  if (typeof value === "string") return value;
+  try {
+    return JSON.stringify(value) ?? "";
+  } catch {
+    return String(value);
+  }
+}
+
+interface TelemetryMeta {
+  cwd: string;
+  mode: string;
+  modelId?: string;
+  provider?: string;
+  projectIds?: string[];
+}