yzcode-cli 1.0.0

package/tools/TodoWriteTool/prompt.ts

@@ -0,0 +1,184 @@
+import { FILE_EDIT_TOOL_NAME } from '../FileEditTool/constants.js'
+
+export const PROMPT = `Use this tool to create and manage a structured task list for your current coding session. This helps you track progress, organize complex tasks, and demonstrate thoroughness to the user.
+It also helps the user understand the progress of the task and overall progress of their requests.
+
+## When to Use This Tool
+Use this tool proactively in these scenarios:
+
+1. Complex multi-step tasks - When a task requires 3 or more distinct steps or actions
+2. Non-trivial and complex tasks - Tasks that require careful planning or multiple operations
+3. User explicitly requests todo list - When the user directly asks you to use the todo list
+4. User provides multiple tasks - When users provide a list of things to be done (numbered or comma-separated)
+5. After receiving new instructions - Immediately capture user requirements as todos
+6. When you start working on a task - Mark it as in_progress BEFORE beginning work. Ideally you should only have one todo as in_progress at a time
+7. After completing a task - Mark it as completed and add any new follow-up tasks discovered during implementation
+
+## When NOT to Use This Tool
+
+Skip using this tool when:
+1. There is only a single, straightforward task
+2. The task is trivial and tracking it provides no organizational benefit
+3. The task can be completed in less than 3 trivial steps
+4. The task is purely conversational or informational
+
+NOTE that you should not use this tool if there is only one trivial task to do. In this case you are better off just doing the task directly.
+
+## Examples of When to Use the Todo List
+
+<example>
+User: I want to add a dark mode toggle to the application settings. Make sure you run the tests and build when you're done!
+Assistant: *Creates todo list with the following items:*
+1. Creating dark mode toggle component in Settings page
+2. Adding dark mode state management (context/store)
+3. Implementing CSS-in-JS styles for dark theme
+4. Updating existing components to support theme switching
+5. Running tests and build process, addressing any failures or errors that occur
+*Begins working on the first task*
+
+<reasoning>
+The assistant used the todo list because:
+1. Adding dark mode is a multi-step feature requiring UI, state management, and styling changes
+2. The user explicitly requested tests and build be run afterward
+3. The assistant inferred that tests and build need to pass by adding "Ensure tests and build succeed" as the final task
+</reasoning>
+</example>
+
+<example>
+User: Help me rename the function getCwd to getCurrentWorkingDirectory across my project
+Assistant: *Uses grep or search tools to locate all instances of getCwd in the codebase*
+I've found 15 instances of 'getCwd' across 8 different files.
+*Creates todo list with specific items for each file that needs updating*
+
+<reasoning>
+The assistant used the todo list because:
+1. First, the assistant searched to understand the scope of the task
+2. Upon finding multiple occurrences across different files, it determined this was a complex task with multiple steps
+3. The todo list helps ensure every instance is tracked and updated systematically
+4. This approach prevents missing any occurrences and maintains code consistency
+</reasoning>
+</example>
+
+
+<example>
+User: I need to implement these features for my e-commerce site: user registration, product catalog, shopping cart, and checkout flow.
+Assistant: *Creates a todo list breaking down each feature into specific tasks based on the project architecture*
+Let's start with user registration. This will involve creating database models, API endpoints, and frontend forms that integrate with your existing authentication system.
+
+<reasoning>
+The assistant used the todo list because:
+1. The user provided multiple complex features to implement in a comma separated list
+2. The todo list helps organize these large features into manageable tasks
+3. This approach allows for tracking progress across the entire implementation
+</reasoning>
+</example>
+
+<example>
+User: Can you help optimize my React application? It's rendering slowly and has performance issues.
+Assistant: *Reviews component structure, render patterns, state management, and data fetching*
+After analyzing your codebase, I've identified several performance issues.
+*Creates todo list with items like: 1) Implementing memoization for expensive calculations in ProductList, 2) Adding virtualization for long lists in Dashboard, 3) Optimizing image loading in Gallery component, 4) Fixing state update loops in ShoppingCart, 5) Reviewing bundle size and implementing code splitting*
+Let's start by implementing memoization for the expensive calculations in your ProductList component.
+
+<reasoning>
+The assistant used the todo list because:
+1. First, the assistant examined the codebase to identify specific performance issues
+2. Based on this analysis, it identified multiple optimization opportunities
+3. Performance optimization is a non-trivial task requiring multiple steps
+4. The todo list helps methodically track improvements across different components
+5. This systematic approach ensures all performance bottlenecks are addressed
+</reasoning>
+</example>
+
+## Examples of When NOT to Use the Todo List
+
+<example>
+User: How do I print 'Hello World' in Python?
+Assistant: In Python, you can print "Hello World" with this simple code:
+
+python
+print("Hello World")
+
+This will output the text "Hello World" to the console when executed.
+
+<reasoning>
+The assistant did not use the todo list because this is a single, trivial task that can be completed in one step. There's no need to track multiple tasks or steps for such a straightforward request.
+</reasoning>
+</example>
+
+<example>
+User: What does the git status command do?
+Assistant: The git status command shows the current state of your working directory and staging area. It displays which changes have been staged, which haven't, and which files aren't being tracked by Git. This is a fundamental command for understanding the current state of your repository before making commits.
+
+<reasoning>
+The assistant did not use the todo list because this is an informational request with no actual coding task to complete. The user is simply asking for an explanation, not for the assistant to perform multiple steps or tasks.
+</reasoning>
+</example>
+
+<example>
+User: Can you add a comment to the calculateTotal function to explain what it does?
+Assistant: Sure, let me add a comment to the calculateTotal function to explain what it does.
+* Uses the ${FILE_EDIT_TOOL_NAME} tool to add a comment to the calculateTotal function *
+
+<reasoning>
+The assistant did not use the todo list because this is a single, straightforward task confined to one location in the code. Adding a comment doesn't require tracking multiple steps or systematic organization.
+</reasoning>
+</example>
+
+<example>
+User: Run npm install for me and tell me what happens.
+Assistant: I'll run the npm install command for you.
+
+*Executes: npm install*
+
+The command completed successfully. Here's the output:
+[Output of npm install command]
+
+All dependencies have been installed according to your package.json file.
+
+<reasoning>
+The assistant did not use the todo list because this is a single command execution with immediate results. There are no multiple steps to track or organize, making the todo list unnecessary for this straightforward task.
+</reasoning>
+</example>
+
+## Task States and Management
+
+1. **Task States**: Use these states to track progress:
+   - pending: Task not yet started
+   - in_progress: Currently working on (limit to ONE task at a time)
+   - completed: Task finished successfully
+
+   **IMPORTANT**: Task descriptions must have two forms:
+   - content: The imperative form describing what needs to be done (e.g., "Run tests", "Build the project")
+   - activeForm: The present continuous form shown during execution (e.g., "Running tests", "Building the project")
+
+2. **Task Management**:
+   - Update task status in real-time as you work
+   - Mark tasks complete IMMEDIATELY after finishing (don't batch completions)
+   - Exactly ONE task must be in_progress at any time (not less, not more)
+   - Complete current tasks before starting new ones
+   - Remove tasks that are no longer relevant from the list entirely
+
+3. **Task Completion Requirements**:
+   - ONLY mark a task as completed when you have FULLY accomplished it
+   - If you encounter errors, blockers, or cannot finish, keep the task as in_progress
+   - When blocked, create a new task describing what needs to be resolved
+   - Never mark a task as completed if:
+     - Tests are failing
+     - Implementation is partial
+     - You encountered unresolved errors
+     - You couldn't find necessary files or dependencies
+
+4. **Task Breakdown**:
+   - Create specific, actionable items
+   - Break complex tasks into smaller, manageable steps
+   - Use clear, descriptive task names
+   - Always provide both forms:
+     - content: "Fix authentication bug"
+     - activeForm: "Fixing authentication bug"
+
+When in doubt, use this tool. Being proactive with task management demonstrates attentiveness and ensures you complete all requirements successfully.
+`
+
+export const DESCRIPTION =
+  'Update the todo list for the current session. To be used proactively and often to track progress and pending tasks. Make sure that at least one task is in_progress at all times. Always provide both content (imperative) and activeForm (present continuous) for each task.'

package/tools/ToolSearchTool/ToolSearchTool.ts

@@ -0,0 +1,471 @@
+import type { ToolResultBlockParam } from '@anthropic-ai/sdk/resources/index.mjs'
+import memoize from 'lodash-es/memoize.js'
+import { z } from 'zod/v4'
+import {
+  type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+  logEvent,
+} from '../../services/analytics/index.js'
+import {
+  buildTool,
+  findToolByName,
+  type Tool,
+  type ToolDef,
+  type Tools,
+} from '../../Tool.js'
+import { logForDebugging } from '../../utils/debug.js'
+import { lazySchema } from '../../utils/lazySchema.js'
+import { escapeRegExp } from '../../utils/stringUtils.js'
+import { isToolSearchEnabledOptimistic } from '../../utils/toolSearch.js'
+import { getPrompt, isDeferredTool, TOOL_SEARCH_TOOL_NAME } from './prompt.js'
+
+export const inputSchema = lazySchema(() =>
+  z.object({
+    query: z
+      .string()
+      .describe(
+        'Query to find deferred tools. Use "select:<tool_name>" for direct selection, or keywords to search.',
+      ),
+    max_results: z
+      .number()
+      .optional()
+      .default(5)
+      .describe('Maximum number of results to return (default: 5)'),
+  }),
+)
+type InputSchema = ReturnType<typeof inputSchema>
+
+export const outputSchema = lazySchema(() =>
+  z.object({
+    matches: z.array(z.string()),
+    query: z.string(),
+    total_deferred_tools: z.number(),
+    pending_mcp_servers: z.array(z.string()).optional(),
+  }),
+)
+type OutputSchema = ReturnType<typeof outputSchema>
+
+export type Output = z.infer<OutputSchema>
+
+// Track deferred tool names to detect when cache should be cleared
+let cachedDeferredToolNames: string | null = null
+
+/**
+ * Get a cache key representing the current set of deferred tools.
+ */
+function getDeferredToolsCacheKey(deferredTools: Tools): string {
+  return deferredTools
+    .map(t => t.name)
+    .sort()
+    .join(',')
+}
+
+/**
+ * Get tool description, memoized by tool name.
+ * Used for keyword search scoring.
+ */
+const getToolDescriptionMemoized = memoize(
+  async (toolName: string, tools: Tools): Promise<string> => {
+    const tool = findToolByName(tools, toolName)
+    if (!tool) {
+      return ''
+    }
+    return tool.prompt({
+      getToolPermissionContext: async () => ({
+        mode: 'default' as const,
+        additionalWorkingDirectories: new Map(),
+        alwaysAllowRules: {},
+        alwaysDenyRules: {},
+        alwaysAskRules: {},
+        isBypassPermissionsModeAvailable: false,
+      }),
+      tools,
+      agents: [],
+    })
+  },
+  (toolName: string) => toolName,
+)
+
+/**
+ * Invalidate the description cache if deferred tools have changed.
+ */
+function maybeInvalidateCache(deferredTools: Tools): void {
+  const currentKey = getDeferredToolsCacheKey(deferredTools)
+  if (cachedDeferredToolNames !== currentKey) {
+    logForDebugging(
+      `ToolSearchTool: cache invalidated - deferred tools changed`,
+    )
+    getToolDescriptionMemoized.cache.clear?.()
+    cachedDeferredToolNames = currentKey
+  }
+}
+
+export function clearToolSearchDescriptionCache(): void {
+  getToolDescriptionMemoized.cache.clear?.()
+  cachedDeferredToolNames = null
+}
+
+/**
+ * Build the search result output structure.
+ */
+function buildSearchResult(
+  matches: string[],
+  query: string,
+  totalDeferredTools: number,
+  pendingMcpServers?: string[],
+): { data: Output } {
+  return {
+    data: {
+      matches,
+      query,
+      total_deferred_tools: totalDeferredTools,
+      ...(pendingMcpServers && pendingMcpServers.length > 0
+        ? { pending_mcp_servers: pendingMcpServers }
+        : {}),
+    },
+  }
+}
+
+/**
+ * Parse tool name into searchable parts.
+ * Handles both MCP tools (mcp__server__action) and regular tools (CamelCase).
+ */
+function parseToolName(name: string): {
+  parts: string[]
+  full: string
+  isMcp: boolean
+} {
+  // Check if it's an MCP tool
+  if (name.startsWith('mcp__')) {
+    const withoutPrefix = name.replace(/^mcp__/, '').toLowerCase()
+    const parts = withoutPrefix.split('__').flatMap(p => p.split('_'))
+    return {
+      parts: parts.filter(Boolean),
+      full: withoutPrefix.replace(/__/g, ' ').replace(/_/g, ' '),
+      isMcp: true,
+    }
+  }
+
+  // Regular tool - split by CamelCase and underscores
+  const parts = name
+    .replace(/([a-z])([A-Z])/g, '$1 $2') // CamelCase to spaces
+    .replace(/_/g, ' ')
+    .toLowerCase()
+    .split(/\s+/)
+    .filter(Boolean)
+
+  return {
+    parts,
+    full: parts.join(' '),
+    isMcp: false,
+  }
+}
+
+/**
+ * Pre-compile word-boundary regexes for all search terms.
+ * Called once per search instead of tools×terms×2 times.
+ */
+function compileTermPatterns(terms: string[]): Map<string, RegExp> {
+  const patterns = new Map<string, RegExp>()
+  for (const term of terms) {
+    if (!patterns.has(term)) {
+      patterns.set(term, new RegExp(`\\b${escapeRegExp(term)}\\b`))
+    }
+  }
+  return patterns
+}
+
+/**
+ * Keyword-based search over tool names and descriptions.
+ * Handles both MCP tools (mcp__server__action) and regular tools (CamelCase).
+ *
+ * The model typically queries with:
+ * - Server names when it knows the integration (e.g., "slack", "github")
+ * - Action words when looking for functionality (e.g., "read", "list", "create")
+ * - Tool-specific terms (e.g., "notebook", "shell", "kill")
+ */
+async function searchToolsWithKeywords(
+  query: string,
+  deferredTools: Tools,
+  tools: Tools,
+  maxResults: number,
+): Promise<string[]> {
+  const queryLower = query.toLowerCase().trim()
+
+  // Fast path: if query matches a tool name exactly, return it directly.
+  // Handles models using a bare tool name instead of select: prefix (seen
+  // from subagents/post-compaction). Checks deferred first, then falls back
+  // to the full tool set — selecting an already-loaded tool is a harmless
+  // no-op that lets the model proceed without retry churn.
+  const exactMatch =
+    deferredTools.find(t => t.name.toLowerCase() === queryLower) ??
+    tools.find(t => t.name.toLowerCase() === queryLower)
+  if (exactMatch) {
+    return [exactMatch.name]
+  }
+
+  // If query looks like an MCP tool prefix (mcp__server), find matching tools.
+  // Handles models searching by server name with mcp__ prefix.
+  if (queryLower.startsWith('mcp__') && queryLower.length > 5) {
+    const prefixMatches = deferredTools
+      .filter(t => t.name.toLowerCase().startsWith(queryLower))
+      .slice(0, maxResults)
+      .map(t => t.name)
+    if (prefixMatches.length > 0) {
+      return prefixMatches
+    }
+  }
+
+  const queryTerms = queryLower.split(/\s+/).filter(term => term.length > 0)
+
+  // Partition into required (+prefixed) and optional terms
+  const requiredTerms: string[] = []
+  const optionalTerms: string[] = []
+  for (const term of queryTerms) {
+    if (term.startsWith('+') && term.length > 1) {
+      requiredTerms.push(term.slice(1))
+    } else {
+      optionalTerms.push(term)
+    }
+  }
+
+  const allScoringTerms =
+    requiredTerms.length > 0 ? [...requiredTerms, ...optionalTerms] : queryTerms
+  const termPatterns = compileTermPatterns(allScoringTerms)
+
+  // Pre-filter to tools matching ALL required terms in name or description
+  let candidateTools = deferredTools
+  if (requiredTerms.length > 0) {
+    const matches = await Promise.all(
+      deferredTools.map(async tool => {
+        const parsed = parseToolName(tool.name)
+        const description = await getToolDescriptionMemoized(tool.name, tools)
+        const descNormalized = description.toLowerCase()
+        const hintNormalized = tool.searchHint?.toLowerCase() ?? ''
+        const matchesAll = requiredTerms.every(term => {
+          const pattern = termPatterns.get(term)!
+          return (
+            parsed.parts.includes(term) ||
+            parsed.parts.some(part => part.includes(term)) ||
+            pattern.test(descNormalized) ||
+            (hintNormalized && pattern.test(hintNormalized))
+          )
+        })
+        return matchesAll ? tool : null
+      }),
+    )
+    candidateTools = matches.filter((t): t is Tool => t !== null)
+  }
+
+  const scored = await Promise.all(
+    candidateTools.map(async tool => {
+      const parsed = parseToolName(tool.name)
+      const description = await getToolDescriptionMemoized(tool.name, tools)
+      const descNormalized = description.toLowerCase()
+      const hintNormalized = tool.searchHint?.toLowerCase() ?? ''
+
+      let score = 0
+      for (const term of allScoringTerms) {
+        const pattern = termPatterns.get(term)!
+
+        // Exact part match (high weight for MCP server names, tool name parts)
+        if (parsed.parts.includes(term)) {
+          score += parsed.isMcp ? 12 : 10
+        } else if (parsed.parts.some(part => part.includes(term))) {
+          score += parsed.isMcp ? 6 : 5
+        }
+
+        // Full name fallback (for edge cases)
+        if (parsed.full.includes(term) && score === 0) {
+          score += 3
+        }
+
+        // searchHint match — curated capability phrase, higher signal than prompt
+        if (hintNormalized && pattern.test(hintNormalized)) {
+          score += 4
+        }
+
+        // Description match - use word boundary to avoid false positives
+        if (pattern.test(descNormalized)) {
+          score += 2
+        }
+      }
+
+      return { name: tool.name, score }
+    }),
+  )
+
+  return scored
+    .filter(item => item.score > 0)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, maxResults)
+    .map(item => item.name)
+}
+
+export const ToolSearchTool = buildTool({
+  isEnabled() {
+    return isToolSearchEnabledOptimistic()
+  },
+  isConcurrencySafe() {
+    return true
+  },
+  isReadOnly() {
+    return true
+  },
+  name: TOOL_SEARCH_TOOL_NAME,
+  maxResultSizeChars: 100_000,
+  async description() {
+    return getPrompt()
+  },
+  async prompt() {
+    return getPrompt()
+  },
+  get inputSchema(): InputSchema {
+    return inputSchema()
+  },
+  get outputSchema(): OutputSchema {
+    return outputSchema()
+  },
+  async call(input, { options: { tools }, getAppState }) {
+    const { query, max_results = 5 } = input
+
+    const deferredTools = tools.filter(isDeferredTool)
+    maybeInvalidateCache(deferredTools)
+
+    // Check for MCP servers still connecting
+    function getPendingServerNames(): string[] | undefined {
+      const appState = getAppState()
+      const pending = appState.mcp.clients.filter(c => c.type === 'pending')
+      return pending.length > 0 ? pending.map(s => s.name) : undefined
+    }
+
+    // Helper to log search outcome
+    function logSearchOutcome(
+      matches: string[],
+      queryType: 'select' | 'keyword',
+    ): void {
+      logEvent('tengu_tool_search_outcome', {
+        query:
+          query as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+        queryType:
+          queryType as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+        matchCount: matches.length,
+        totalDeferredTools: deferredTools.length,
+        maxResults: max_results,
+        hasMatches: matches.length > 0,
+      })
+    }
+
+    // Check for select: prefix — direct tool selection.
+    // Supports comma-separated multi-select: `select:A,B,C`.
+    // If a name isn't in the deferred set but IS in the full tool set,
+    // we still return it — the tool is already loaded, so "selecting" it
+    // is a harmless no-op that lets the model proceed without retry churn.
+    const selectMatch = query.match(/^select:(.+)$/i)
+    if (selectMatch) {
+      const requested = selectMatch[1]!
+        .split(',')
+        .map(s => s.trim())
+        .filter(Boolean)
+
+      const found: string[] = []
+      const missing: string[] = []
+      for (const toolName of requested) {
+        const tool =
+          findToolByName(deferredTools, toolName) ??
+          findToolByName(tools, toolName)
+        if (tool) {
+          if (!found.includes(tool.name)) found.push(tool.name)
+        } else {
+          missing.push(toolName)
+        }
+      }
+
+      if (found.length === 0) {
+        logForDebugging(
+          `ToolSearchTool: select failed — none found: ${missing.join(', ')}`,
+        )
+        logSearchOutcome([], 'select')
+        const pendingServers = getPendingServerNames()
+        return buildSearchResult(
+          [],
+          query,
+          deferredTools.length,
+          pendingServers,
+        )
+      }
+
+      if (missing.length > 0) {
+        logForDebugging(
+          `ToolSearchTool: partial select — found: ${found.join(', ')}, missing: ${missing.join(', ')}`,
+        )
+      } else {
+        logForDebugging(`ToolSearchTool: selected ${found.join(', ')}`)
+      }
+      logSearchOutcome(found, 'select')
+      return buildSearchResult(found, query, deferredTools.length)
+    }
+
+    // Keyword search
+    const matches = await searchToolsWithKeywords(
+      query,
+      deferredTools,
+      tools,
+      max_results,
+    )
+
+    logForDebugging(
+      `ToolSearchTool: keyword search for "${query}", found ${matches.length} matches`,
+    )
+
+    logSearchOutcome(matches, 'keyword')
+
+    // Include pending server info when search finds no matches
+    if (matches.length === 0) {
+      const pendingServers = getPendingServerNames()
+      return buildSearchResult(
+        matches,
+        query,
+        deferredTools.length,
+        pendingServers,
+      )
+    }
+
+    return buildSearchResult(matches, query, deferredTools.length)
+  },
+  renderToolUseMessage() {
+    return null
+  },
+  userFacingName: () => '',
+  /**
+   * Returns a tool_result with tool_reference blocks.
+   * This format works on 1P/Foundry. Bedrock/Vertex may not support
+   * client-side tool_reference expansion yet.
+   */
+  mapToolResultToToolResultBlockParam(
+    content: Output,
+    toolUseID: string,
+  ): ToolResultBlockParam {
+    if (content.matches.length === 0) {
+      let text = 'No matching deferred tools found'
+      if (
+        content.pending_mcp_servers &&
+        content.pending_mcp_servers.length > 0
+      ) {
+        text += `. Some MCP servers are still connecting: ${content.pending_mcp_servers.join(', ')}. Their tools will become available shortly — try searching again.`
+      }
+      return {
+        type: 'tool_result',
+        tool_use_id: toolUseID,
+        content: text,
+      }
+    }
+    return {
+      type: 'tool_result',
+      tool_use_id: toolUseID,
+      content: content.matches.map(name => ({
+        type: 'tool_reference' as const,
+        tool_name: name,
+      })),
+    } as unknown as ToolResultBlockParam
+  },
+} satisfies ToolDef<InputSchema, Output>)

package/tools/ToolSearchTool/constants.ts

@@ -0,0 +1 @@
+export const TOOL_SEARCH_TOOL_NAME = 'ToolSearch'