pipeline-moe 0.1.0

package/src/custom-tools/arxiv-search.ts

@@ -0,0 +1,186 @@
+// arxiv_search — arXiv academic paper search tool definition
+// Free, no API key. Returns paper titles, authors, abstracts, categories,
+// and PDF links from the arXiv Atom feed API.
+
+import { Type } from "typebox"
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent"
+import type { AgentToolResult } from "@earendil-works/pi-coding-agent"
+
+// Minimal text content type.
+interface TextContent {
+  type: "text"
+  text: string
+}
+
+const arxivSearchSchema = Type.Object({
+  query: Type.String({
+    description: "Search terms (title, abstract, author, etc.)",
+  }),
+  max_results: Type.Optional(
+    Type.Integer({
+      minimum: 1,
+      maximum: 50,
+      default: 5,
+      description: "Maximum results to return (1-50, default 5)",
+    })
+  ),
+  categories: Type.Optional(
+    Type.Array(Type.String(), {
+      default: [],
+      description: "arXiv categories to filter (e.g., cs.AI, cs.LG, cs.CL)",
+    })
+  ),
+  sort_by: Type.Optional(
+    Type.Enum({ relevance: "relevance", lastUpdatedDate: "lastUpdatedDate" }, {
+      default: "relevance",
+      description: "Sort order: relevance or lastUpdatedDate",
+    })
+  ),
+})
+
+const ARXIV_API = "http://export.arxiv.org/api/query"
+const TIMEOUT_MS = 15_000
+const ABSTRACT_MAX = 300
+
+// Extract text between XML tags (simple, no dependency needed).
+function extractTag(xml: string, tag: string): string | null {
+  const re = new RegExp(`<${tag}[^>]*>([^<]*)</${tag}>`, "i")
+  const match = xml.match(re)
+  return match ? match[1].replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"') : null
+}
+
+// Extract all tags of a given name.
+function extractAllTags(xml: string, tag: string): string[] {
+  const re = new RegExp(`<${tag}[^>]*>([^<]*)</${tag}>`, "gi")
+  const results: string[] = []
+  let m
+  while ((m = re.exec(xml)) !== null) {
+    results.push(m[1].replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"'))
+  }
+  return results
+}
+
+// Extract text between <entry> tags.
+function extractEntries(xml: string): string[] {
+  const re = /<entry>([\s\S]*?)<\/entry>/gi
+  const results: string[] = []
+  let m
+  while ((m = re.exec(xml)) !== null) {
+    results.push(m[1])
+  }
+  return results
+}
+
+async function searchArxiv(query: string, maxResults: number, categories: string[], sortBy: string): Promise<AgentToolResult<undefined>> {
+  // Build search query with category filters.
+  const searchTerms: string[] = []
+  if (query) {
+    searchTerms.push(query)
+  }
+  if (categories.length > 0) {
+    const catQuery = categories.map((c) => `cat:${c}`).join(" OR ")
+    searchTerms.push(`(${catQuery})`)
+  }
+  const searchQuery = searchTerms.join(" AND ")
+
+  // Build URL.
+  const params = new URLSearchParams({
+    search_query: searchQuery,
+    max_results: String(Math.min(maxResults, 50)),
+    sort_by: sortBy,
+    sort_order: "descending",
+  })
+  const url = `${ARXIV_API}?${params}`
+
+  const controller = new AbortController()
+  const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS)
+  try {
+    const response = await fetch(url, { signal: controller.signal })
+
+    if (!response.ok) {
+      throw new Error(`arXiv API returned ${response.status} ${response.statusText}`)
+    }
+
+    const xml = await response.text()
+    const entries = extractEntries(xml)
+
+    if (entries.length === 0) {
+      return {
+        content: [{ type: "text", text: `No results found for: "${query}"` }],
+        details: undefined,
+      }
+    }
+
+    // Parse each entry.
+    const results = entries.map((entryXml, i) => {
+      const title = extractTag(entryXml, "title") ?? "No title"
+      const abstract = (extractTag(entryXml, "summary") ?? "").replace(/\n/g, " ").trim()
+      const authors = extractAllTags(entryXml, "name")
+      const categories = extractAllTags(entryXml, "term")
+      const id = extractTag(entryXml, "id") ?? ""
+      const published = extractTag(entryXml, "published") ?? ""
+
+      // Extract PDF URL from <link> tags.
+      let pdfUrl = ""
+      const linkRe = /<link[^>]*title="pdf"[^>]*href="([^"]*)"/i
+      const pdfMatch = entryXml.match(linkRe)
+      if (pdfMatch) pdfUrl = pdfMatch[1]
+      else {
+        // Fallback: construct from arXiv ID.
+        const idRe = /arxiv\.org\/abs\/(.+)$/
+        const idMatch = id.match(idRe)
+        if (idMatch) pdfUrl = `https://arxiv.org/pdf/${idMatch[1]}`
+      }
+
+      // Truncate abstract.
+      const truncatedAbstract = abstract.length > ABSTRACT_MAX
+        ? abstract.slice(0, ABSTRACT_MAX) + "..."
+        : abstract
+
+      return [
+        `${i + 1}. ${title}`,
+        `   Authors: ${authors.join(", ")}`,
+        `   Published: ${published}`,
+        `   Categories: ${categories.join(", ")}`,
+        `   Abstract: ${truncatedAbstract}`,
+        `   PDF: ${pdfUrl}`,
+      ].join("\n")
+    })
+
+    const header = `arXiv search results for "${query}" (${results.length} papers found)`
+    return {
+      content: [{ type: "text", text: header + "\n\n" + results.join("\n\n") }],
+      details: undefined,
+    }
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err)
+    return {
+      content: [{
+        type: "text",
+        text: `arxiv_search error: ${msg}. Ensure the arXiv API is reachable at ${ARXIV_API}.`,
+      }],
+      details: undefined,
+    }
+  } finally {
+    clearTimeout(timeout)
+  }
+}
+
+export function createArxivSearchToolDefinition(): ToolDefinition<typeof arxivSearchSchema, undefined> {
+  return {
+    name: "arxiv_search",
+    label: "arXiv Search",
+    description:
+      "Search academic papers on arXiv (2.3M+ papers). " +
+      "Free, no API key. Returns titles, authors, abstracts, categories, and PDF links. " +
+      "Abstracts are truncated to ~300 chars.",
+    parameters: arxivSearchSchema,
+    execute: async (_toolCallId, params) => {
+      const query = params.query
+      const maxResults = params.max_results ?? 5
+      const categories = params.categories ?? []
+      const sortBy = params.sort_by ?? "relevance"
+      return searchArxiv(query, maxResults, categories, sortBy)
+    },
+  }
+}

package/src/custom-tools/check-room.ts

@@ -0,0 +1,45 @@
+// check_room — read a sub-room's goal status and recent transcript so the
+// spawning agent can decide whether to keep waiting or synthesize the result.
+
+import { Type } from "typebox"
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent"
+import type { RoomOrchestrator } from "../orchestrator.js"
+
+const checkRoomSchema = Type.Object({
+  roomId: Type.String({
+    description: "The roomId returned by spawn_room.",
+  }),
+})
+
+export function createCheckRoomToolDefinition(
+  orchestrator: RoomOrchestrator,
+): ToolDefinition<typeof checkRoomSchema, undefined> {
+  return {
+    name: "check_room",
+    label: "Check Sub-Room",
+    description:
+      "Read a sub-room's current goal status (idle/running/completed/failed) and its last few " +
+      "transcript messages. Poll this after spawn_room until the goal is completed or failed, " +
+      "then synthesize the result and destroy the room.",
+    parameters: checkRoomSchema,
+    execute: async (_toolCallId, params) => {
+      const r = orchestrator.checkRoom(params.roomId)
+      if (!r.found) {
+        return {
+          content: [{ type: "text", text: `check_room: no room with id "${params.roomId}".` }],
+          details: undefined,
+        }
+      }
+      const header =
+        `Room "${r.name}" (${r.roomId}) — goal status: ${r.goalStatus}` +
+        (r.goalText ? `\nGoal: ${r.goalText}` : "")
+      const body = r.lastMessages && r.lastMessages.length > 0
+        ? `\n\nLast messages:\n${r.lastMessages.join("\n")}`
+        : "\n\n(no transcript yet)"
+      return {
+        content: [{ type: "text", text: header + body }],
+        details: undefined,
+      }
+    },
+  }
+}

package/src/custom-tools/destroy-room.ts

@@ -0,0 +1,45 @@
+// destroy_room — tear down a sub-room after its goal is done. Unmounts any
+// sshfs target the room was scoped to. The default room cannot be destroyed.
+
+import { Type } from "typebox"
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent"
+import type { RoomOrchestrator } from "../orchestrator.js"
+
+const destroyRoomSchema = Type.Object({
+  roomId: Type.String({
+    description: "The roomId of the sub-room to destroy.",
+  }),
+})
+
+export function createDestroyRoomToolDefinition(
+  orchestrator: RoomOrchestrator,
+): ToolDefinition<typeof destroyRoomSchema, undefined> {
+  return {
+    name: "destroy_room",
+    label: "Destroy Sub-Room",
+    description:
+      "Destroy a sub-room once you have collected its result. Unmounts any sshfs target. " +
+      "Call this after check_room shows the goal is completed or failed, to free resources.",
+    parameters: destroyRoomSchema,
+    execute: async (_toolCallId, params) => {
+      try {
+        const ok = await orchestrator.destroyRoom(params.roomId)
+        return {
+          content: [{
+            type: "text",
+            text: ok
+              ? `Destroyed room "${params.roomId}".`
+              : `destroy_room: no room with id "${params.roomId}" (or it cannot be destroyed).`,
+          }],
+          details: undefined,
+        }
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err)
+        return {
+          content: [{ type: "text", text: `destroy_room error: ${msg}` }],
+          details: undefined,
+        }
+      }
+    },
+  }
+}

package/src/custom-tools/index.ts

@@ -0,0 +1,75 @@
+// Custom tools registry — extension point for Pipeline-MoE.
+// Each custom tool is a standalone ToolDefinition. Add new tools here
+// and register them in the index. The allowlist in `persona.tools` gates
+// which tools each agent gets.
+
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent"
+import { createWebSearchToolDefinition } from "./web-search.js"
+import { createWebReadToolDefinition } from "./web-read.js"
+import { createYoutubeTranscriptToolDefinition } from "./youtube-transcript.js"
+import { createArxivSearchToolDefinition } from "./arxiv-search.js"
+import { createYoucomSearchToolDefinition } from "./youcom-search.js"
+import { createSpawnRoomToolDefinition } from "./spawn-room.js"
+import { createCheckRoomToolDefinition } from "./check-room.js"
+import { createStopRoomToolDefinition } from "./stop-room.js"
+import { createDestroyRoomToolDefinition } from "./destroy-room.js"
+import type { RoomOrchestrator } from "../orchestrator.js"
+
+/** Runtime context the tool registry needs to build context-dependent tools.
+ *  Orchestration tools (spawn/check/destroy room) require a live orchestrator;
+ *  they are only built when one is supplied. */
+export interface ToolContext {
+  orchestrator?: RoomOrchestrator
+}
+
+/** Orchestration tool names — gated on a RoomOrchestrator being present, not on
+ *  the static TOOLS registry. Only orchestrator personas (the planner) get them. */
+export const ORCHESTRATION_TOOLS = ["spawn_room", "check_room", "stop_room", "destroy_room"] as const
+
+// Registry of tool name → factory function.
+// Add new tools here — each tool is a self-contained module.
+// Factory is typed loosely to avoid contravariance issues with ToolDefinition
+// generics — the actual type safety is enforced in each tool's definition.
+const TOOLS: Array<{ name: string; factory: () => unknown }> = [
+  { name: "web_search", factory: createWebSearchToolDefinition },
+  { name: "web_read", factory: createWebReadToolDefinition },
+  { name: "youtube_transcript", factory: createYoutubeTranscriptToolDefinition },
+  { name: "arxiv_search", factory: createArxivSearchToolDefinition },
+  { name: "youcom_search", factory: createYoucomSearchToolDefinition },
+]
+
+/** Build custom tool definitions for the given tool name allowlist.
+ *  Only returns tools whose name appears in the allowlist. Orchestration tools
+ *  (spawn/check/destroy room) are only built when `ctx.orchestrator` is supplied. */
+export function buildCustomTools(toolNames: string[], ctx?: ToolContext): ToolDefinition[] {
+  const wanted = new Set(toolNames)
+  const tools: ToolDefinition[] = []
+
+  for (const { name, factory } of TOOLS) {
+    if (wanted.has(name)) {
+      tools.push(factory() as ToolDefinition)
+    }
+  }
+
+  // Orchestration tools — context-gated. Require a live orchestrator reference
+  // captured at execution time. Absent orchestrator → these names are silently
+  // ignored (same contract as unknown tool names).
+  const wantsOrch = [...ORCHESTRATION_TOOLS].filter(n => wanted.has(n))
+  if (wantsOrch.length > 0) {
+    console.log(`[buildCustomTools] orchestration tools requested: ${wantsOrch.join(", ")}; ctx?.orchestrator = ${!!ctx?.orchestrator}`)
+  }
+  if (ctx?.orchestrator) {
+    const orch = ctx.orchestrator
+    if (wanted.has("spawn_room")) tools.push(createSpawnRoomToolDefinition(orch) as ToolDefinition)
+    if (wanted.has("check_room")) tools.push(createCheckRoomToolDefinition(orch) as ToolDefinition)
+    if (wanted.has("stop_room")) tools.push(createStopRoomToolDefinition(orch) as ToolDefinition)
+    if (wanted.has("destroy_room")) tools.push(createDestroyRoomToolDefinition(orch) as ToolDefinition)
+  }
+
+  return tools
+}
+
+/** Get all available custom tool names (for validation / UI). */
+export function availableCustomTools(): string[] {
+  return TOOLS.map((t) => t.name)
+}

package/src/custom-tools/spawn-room.ts

@@ -0,0 +1,99 @@
+// spawn_room — create a parallel sub-room with its own agents and a goal.
+// The room runs independently in the background; this tool returns immediately
+// with the roomId. Poll progress with check_room, clean up with destroy_room.
+
+import { Type } from "typebox"
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent"
+import type { RoomOrchestrator } from "../orchestrator.js"
+
+const spawnRoomSchema = Type.Object({
+  name: Type.String({
+    description: "Display name for the sub-room (e.g. 'audit-auth-flow').",
+  }),
+  goal: Type.String({
+    description:
+      "The goal the sub-room's agents will work on autonomously. Be specific and self-contained — the sub-room does not share your conversation context.",
+  }),
+  preset: Type.Optional(
+    Type.String({
+      description:
+        "Preset roster name (from presets/, e.g. 'local-default'). Omit to use the default roster.",
+    }),
+  ),
+  workspaceDir: Type.Optional(
+    Type.String({
+      description:
+        "Working directory scope — a local path or an sshfs target (user@host:/path). Omit for the pipeline workspace.",
+    }),
+  ),
+  goalMode: Type.Optional(
+    Type.Union([Type.Literal("auto"), Type.Literal("eval")], {
+      description:
+        "Goal completion mode. 'auto' (default): the goal completes when the sub-room's pipeline " +
+        "drains. 'eval': after each pass the evaluator agent re-enters the sub-room, verifies " +
+        "the goal independently with tools, and either dispatches more work or declares GOAL_MET. Use " +
+        "'eval' for iterative goals that need a verification loop.",
+    }),
+  ),
+  goalEvaluator: Type.Optional(
+    Type.String({
+      description:
+        "Agent id that evaluates the goal in 'eval' mode (e.g. 'planner', 'auditor'). Defaults to " +
+        "'planner'. Must match a persona id in the sub-room's roster, or the goal will auto-complete " +
+        "without verification.",
+    }),
+  ),
+  maxGoalIterations: Type.Optional(
+    Type.Integer({
+      minimum: 1,
+      maximum: 50,
+      description:
+        "Max evaluation iterations before the goal auto-fails in 'eval' mode. Default 10. Each " +
+        "iteration is one evaluator verification pass plus any agents it dispatches.",
+    }),
+  ),
+})
+
+export function createSpawnRoomToolDefinition(
+  orchestrator: RoomOrchestrator,
+): ToolDefinition<typeof spawnRoomSchema, undefined> {
+  return {
+    name: "spawn_room",
+    label: "Spawn Sub-Room",
+    description:
+      "Create a new parallel room with its own agents and give it a goal. The room runs " +
+      "independently in the background — use this to delegate a bounded, self-contained workstream. " +
+      "Returns the roomId; poll progress with check_room and clean up with destroy_room. " +
+      "The sub-room does NOT share your conversation, so the goal must be fully self-contained.",
+    parameters: spawnRoomSchema,
+    execute: async (_toolCallId, params) => {
+      try {
+        const r = await orchestrator.spawnRoom({
+          name: params.name,
+          goal: params.goal,
+          preset: params.preset,
+          workspaceDir: params.workspaceDir,
+          goalMode: params.goalMode,
+          goalEvaluator: params.goalEvaluator,
+          maxGoalIterations: params.maxGoalIterations,
+        })
+        return {
+          content: [{
+            type: "text",
+            text:
+              `Spawned room "${r.name}" — roomId: ${r.roomId}, status: ${r.goalStatus}.\n` +
+              `Poll with check_room({ roomId: "${r.roomId}" }) until status is "completed" or "failed", ` +
+              `then destroy_room({ roomId: "${r.roomId}" }) to clean up.`,
+          }],
+          details: undefined,
+        }
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err)
+        return {
+          content: [{ type: "text", text: `spawn_room error: ${msg}` }],
+          details: undefined,
+        }
+      }
+    },
+  }
+}

package/src/custom-tools/stop-room.ts

@@ -0,0 +1,50 @@
+// stop_room — halt a sub-room's in-flight work WITHOUT destroying it. Aborts
+// running agents and cancels the goal (status → "cancelled"); the room and its
+// transcript survive so you can inspect what went wrong, then decide whether to
+// re-dispatch or destroy_room to free resources. The default room is protected.
+
+import { Type } from "typebox"
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent"
+import type { RoomOrchestrator } from "../orchestrator.js"
+
+const stopRoomSchema = Type.Object({
+  roomId: Type.String({
+    description: "The roomId of the sub-room to stop.",
+  }),
+})
+
+export function createStopRoomToolDefinition(
+  orchestrator: RoomOrchestrator,
+): ToolDefinition<typeof stopRoomSchema, undefined> {
+  return {
+    name: "stop_room",
+    label: "Stop Sub-Room",
+    description:
+      "Halt a sub-room that is running away or no longer needed, WITHOUT destroying it. Aborts its " +
+      "running agents and cancels its goal (status becomes 'cancelled'); the room and its transcript " +
+      "remain intact so you can inspect what happened with check_room. Call destroy_room afterwards " +
+      "to free resources. Cannot stop the default room.",
+    parameters: stopRoomSchema,
+    execute: async (_toolCallId, params) => {
+      try {
+        const ok = await orchestrator.stopRoom(params.roomId)
+        return {
+          content: [{
+            type: "text",
+            text: ok
+              ? `Stopped room "${params.roomId}" — goal cancelled, agents aborted. The room and its ` +
+                `transcript are intact: check_room to inspect, or destroy_room to free resources.`
+              : `stop_room: no room with id "${params.roomId}" (or it cannot be stopped — the default room is protected).`,
+          }],
+          details: undefined,
+        }
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err)
+        return {
+          content: [{ type: "text", text: `stop_room error: ${msg}` }],
+          details: undefined,
+        }
+      }
+    },
+  }
+}

package/src/custom-tools/web-read.ts

@@ -0,0 +1,104 @@
+// web_read — Jina Reader tool definition
+// Extracts clean markdown content from any URL via Jina Reader.
+// Free, no API key, 100 req/min. The natural follow-up to web_search:
+// search finds URLs, web_read extracts their content.
+
+import { Type } from "typebox"
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent"
+import type { AgentToolResult } from "@earendil-works/pi-coding-agent"
+
+// Minimal text content type (mirrors pi-ai TextContent — not re-exported).
+interface TextContent {
+  type: "text"
+  text: string
+}
+
+const JinaResponse = Type.Object({
+  code: Type.Number(),
+  status: Type.Number(),
+  data: Type.Optional(Type.Object({
+    title: Type.String(),
+    content: Type.String(),
+    url: Type.String(),
+  })),
+})
+
+const webReadSchema = Type.Object({
+  url: Type.String({
+    description: "The URL to extract content from",
+  }),
+})
+
+const JINA_URL = "https://r.jina.ai"
+const TIMEOUT_MS = 15_000
+const MAX_CONTENT_LENGTH = 16000
+
+async function readUrl(url: string): Promise<AgentToolResult<undefined>> {
+  const jinaUrl = `${JINA_URL}/${url}`
+
+  const controller = new AbortController()
+  const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS)
+  try {
+    const response = await fetch(jinaUrl, {
+      signal: controller.signal,
+      headers: { "Accept": "application/json" },
+    })
+
+    if (!response.ok) {
+      throw new Error(`Jina Reader returned ${response.status} ${response.statusText}`)
+    }
+
+    const body = await response.json()
+    const data = (body as { data?: { title: string; content: string; url: string } }).data
+
+    if (!data) {
+      throw new Error("No content returned by Jina Reader")
+    }
+
+    // Truncate to avoid context explosion.
+    const content = data.content.length > MAX_CONTENT_LENGTH
+      ? data.content.slice(0, MAX_CONTENT_LENGTH) + "\n\n[content truncated — " + data.content.length + " chars total]"
+      : data.content
+
+    const lines = [
+      `# ${data.title}`,
+      `Source: ${data.url}`,
+      "",
+      content,
+    ]
+
+    return {
+      content: [{ type: "text", text: lines.join("\n") }],
+      details: undefined,
+    }
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err)
+    return {
+      content: [{
+        type: "text",
+        text: `web_read error: ${msg}. Ensure the URL is accessible and Jina Reader is reachable at ${JINA_URL}.`,
+      }],
+      details: undefined,
+    }
+  } finally {
+    clearTimeout(timeout)
+  }
+}
+
+export function createWebReadToolDefinition(): ToolDefinition<typeof webReadSchema, undefined> {
+  return {
+    name: "web_read",
+    label: "Web Read",
+    description:
+      "Extract clean markdown content from any web page via Jina Reader. " +
+      "Free, no API key, 100 req/min. The natural follow-up to web_search: " +
+      "search finds URLs, web_read extracts their content. " +
+      "Content is truncated to ~16000 chars to avoid context explosion — " +
+      "ask for a specific section if you need more detail.",
+    parameters: webReadSchema,
+    execute: async (_toolCallId, params) => {
+      const url = params.url
+      return readUrl(url)
+    },
+  }
+}