retell-sync-cli 3.6.0 → 3.7.0

package/dist/cli.js

@@ -135137,6 +135137,193 @@ async function canonicalizeFromFiles(files) {
     conversationFlows
   };
 }
+var InputMatchRuleSchema = zod_default.discriminatedUnion("type", [
+  zod_default.object({ type: zod_default.literal("any") }),
+  zod_default.object({
+    type: zod_default.literal("exact"),
+    input: zod_default.record(zod_default.string(), zod_default.unknown())
+  }),
+  zod_default.object({
+    type: zod_default.literal("partial"),
+    input: zod_default.record(zod_default.string(), zod_default.unknown())
+  })
+]);
+var ToolMockSchema = zod_default.object({
+  tool_name: zod_default.string(),
+  input_match_rule: InputMatchRuleSchema,
+  output: zod_default.string(),
+  result: zod_default.boolean()
+});
+var TestCaseResponseEngineSchema = zod_default.discriminatedUnion("type", [
+  zod_default.object({
+    type: zod_default.literal("retell-llm"),
+    llm_id: zod_default.string(),
+    version: zod_default.number().optional()
+  }),
+  zod_default.object({
+    type: zod_default.literal("conversation-flow"),
+    conversation_flow_id: zod_default.string(),
+    version: zod_default.number().optional()
+  })
+]);
+var TestCaseDefinitionSchema = zod_default.object({
+  test_case_definition_id: zod_default.string(),
+  name: zod_default.string(),
+  response_engine: TestCaseResponseEngineSchema,
+  dynamic_variables: zod_default.record(zod_default.string(), zod_default.unknown()).optional().default({}),
+  metrics: zod_default.array(zod_default.string()).optional().default([]),
+  user_prompt: zod_default.string(),
+  creation_timestamp: zod_default.number(),
+  user_modified_timestamp: zod_default.number(),
+  type: zod_default.literal("simulation"),
+  tool_mocks: zod_default.array(ToolMockSchema).optional().default([]),
+  llm_model: zod_default.string()
+});
+async function getTestCaseDefinitions(responseEngine) {
+  const params = new URLSearchParams({ type: responseEngine.type });
+  if (responseEngine.type === "retell-llm") {
+    params.set("llm_id", responseEngine.llm_id);
+    if (responseEngine.version != null) {
+      params.set("version", String(responseEngine.version));
+    }
+  } else {
+    params.set("conversation_flow_id", responseEngine.conversation_flow_id);
+    if (responseEngine.version != null) {
+      params.set("version", String(responseEngine.version));
+    }
+  }
+  const response = await fetch(`https://api.retellai.com/list-test-case-definitions?${params}`, {
+    headers: {
+      Authorization: `Bearer ${process.env.RETELL_API_KEY}`,
+      "Content-Type": "application/json"
+    }
+  });
+  if (!response.ok) {
+    const errorText = await response.text();
+    if (response.status === 404) {
+      return [];
+    }
+    throw new Error(`Failed to fetch test cases: ${response.status} ${errorText}`);
+  }
+  const data = await response.json();
+  const result = zod_default.array(TestCaseDefinitionSchema).safeParse(data);
+  if (!result.success) {
+    console.warn("Warning: Some test case fields failed validation:", result.error.issues);
+    return [];
+  }
+  return result.data;
+}
+async function fetchAndWriteTestCases({
+  state,
+  agentsDir = DEFAULT_AGENTS_DIR,
+  configFormat = DEFAULT_CONFIG_FORMAT
+}) {
+  const results = [];
+  const allAgents = [
+    ...state.voiceAgents.map((a7) => ({
+      ...a7,
+      agentType: "voice"
+    })),
+    ...state.chatAgents.map((a7) => ({
+      ...a7,
+      agentType: "chat"
+    }))
+  ];
+  for (const agent of allAgents) {
+    const agentName = agent.agent_name ?? agent._id;
+    const hash2 = agent._id.slice(-FILE_HASH_LENGTH);
+    const agentDirName = `${toSnakeCase(agentName)}_${hash2}`;
+    const agentDirPath = path15.join(agentsDir, agentDirName);
+    if (agent.response_engine.type !== "retell-llm" && agent.response_engine.type !== "conversation-flow") {
+      continue;
+    }
+    const normalizedEngine = agent.response_engine.type === "retell-llm" ? {
+      type: "retell-llm",
+      llm_id: agent.response_engine.llm_id,
+      version: agent.response_engine.version ?? undefined
+    } : {
+      type: "conversation-flow",
+      conversation_flow_id: agent.response_engine.conversation_flow_id,
+      version: agent.response_engine.version ?? undefined
+    };
+    let testCases;
+    try {
+      testCases = await getTestCaseDefinitions(normalizedEngine);
+    } catch (err) {
+      console.warn(`Warning: Could not fetch test cases for ${agentName}: ${err}`);
+      continue;
+    }
+    if (testCases.length === 0) {
+      const testsDir = path15.join(agentDirPath, "tests");
+      await fs8.rm(testsDir, { recursive: true, force: true }).catch(() => {});
+      continue;
+    }
+    await writeTestCases(testCases, {
+      agentDirPath,
+      responseEngine: normalizedEngine,
+      configFormat
+    });
+    results.push({ agentDir: agentDirName, testCount: testCases.length });
+  }
+  return results;
+}
+async function writeTestCases(testCases, {
+  agentDirPath,
+  responseEngine,
+  configFormat = DEFAULT_CONFIG_FORMAT
+}) {
+  const testsDir = path15.join(agentDirPath, "tests");
+  await fs8.mkdir(testsDir, { recursive: true });
+  const isYaml = configFormat === "yaml" || configFormat === "yml";
+  const isJsonc = configFormat === "jsonc";
+  const configExt = configFormat;
+  const metadata = {
+    response_engine: responseEngine,
+    test_cases: testCases.map((tc3) => ({
+      id: tc3.test_case_definition_id,
+      name: tc3.name
+    }))
+  };
+  await Bun.write(path15.join(testsDir, ".tests.json"), await writeJson(metadata));
+  const writtenFiles = new Set([".tests.json"]);
+  for (const testCase of testCases) {
+    const {
+      test_case_definition_id: _id,
+      response_engine: _engine,
+      creation_timestamp: _created,
+      user_modified_timestamp: _modified,
+      type: _type,
+      ...config2
+    } = testCase;
+    const testCaseName = toSnakeCase(testCase.name);
+    const promptFileName = `${testCaseName}_prompt.md`;
+    const promptFilePath = path15.join(testsDir, promptFileName);
+    await Bun.write(promptFilePath, await writeMarkdown(config2.user_prompt));
+    writtenFiles.add(promptFileName);
+    const configWithFileRef = {
+      ...config2,
+      user_prompt: `file://./${promptFileName}`
+    };
+    const configFileName = `${testCaseName}.${configExt}`;
+    const configContent = isYaml ? await writeYaml(configWithFileRef, { comments: testCaseFieldDocs }) : isJsonc ? await writeJsonc(configWithFileRef, { comments: testCaseFieldDocs }) : await writeJson(configWithFileRef);
+    await Bun.write(path15.join(testsDir, configFileName), configContent);
+    writtenFiles.add(configFileName);
+  }
+  const existingFiles = await fs8.readdir(testsDir).catch(() => []);
+  for (const file2 of existingFiles) {
+    if (!writtenFiles.has(file2)) {
+      await fs8.rm(path15.join(testsDir, file2)).catch(() => {});
+    }
+  }
+}
+var testCaseFieldDocs = {
+  name: "Name of the test case",
+  user_prompt: "Prompt describing simulated user behavior (file reference)",
+  metrics: "Array of evaluation criteria to check",
+  dynamic_variables: "Variables injected into the agent during test",
+  tool_mocks: "Mock responses for tool/function calls",
+  llm_model: "LLM model used to simulate the user"
+};
 
 // src/lib/logger.ts
 var quietMode = false;
@@ -135515,7 +135702,8 @@ async function pullCommand(agentIdArgs, opts, cmd) {
       agentsDir: globalOpts.agentsDir,
       configFormat: globalOpts.configFormat,
       agentIds,
-      yes: opts.yes
+      yes: opts.yes,
+      tests: opts.tests ?? true
     });
   } catch (err) {
     if (err instanceof ExitPromptError) {
@@ -135529,7 +135717,8 @@ async function pull({
   agentsDir = DEFAULT_AGENTS_DIR,
   configFormat = DEFAULT_CONFIG_FORMAT,
   agentIds = null,
-  yes = false
+  yes = false,
+  tests = true
 } = {}) {
   const scopeLabel = agentIds ? `${agentIds.length} agent(s)` : "all agents";
   console.log(source_default.bold(`Pulling ${scopeLabel} from Retell...`));
@@ -135555,6 +135744,17 @@ async function pull({
   const writeSpinner = createSpinner("Writing files...");
   await writeState(remoteState, { agentsDir, configFormat, agentIds });
   writeSpinner.stop(source_default.green("Done"));
+  if (tests) {
+    const testSpinner = createSpinner("Fetching test cases...");
+    const testResults = await fetchAndWriteTestCases({
+      state: remoteState,
+      agentsDir,
+      configFormat
+    });
+    const totalTests = testResults.reduce((sum, r10) => sum + r10.testCount, 0);
+    const agentsWithTests = testResults.filter((r10) => r10.testCount > 0).length;
+    testSpinner.stop(source_default.dim(`${pluralize("test case", totalTests, true)} across ${pluralize("agent", agentsWithTests, true)}`));
+  }
   console.log(source_default.dim(`Files written to ${source_default.bold(agentsDir)}. Review with git diff.`));
 }
 
@@ -136173,7 +136373,7 @@ async function updatePhoneNumberVersions(publishedAgentIds, agentNames) {
 // src/cli.ts
 var program2 = new Command;
 program2.name("retell").description("Retell AI agent management CLI").option("-w, --agents-dir <dir>", "Directory for agent files", DEFAULT_AGENTS_DIR).option("-f, --config-format <format>", `Config file format (${CONFIG_FORMATS.join(", ")})`, DEFAULT_CONFIG_FORMAT);
-program2.command("pull [agentIds...]").description("Pull agents from Retell API (always pulls latest draft state)").option("-a, --all", "Pull all agents in the account").option("-s, --select", "Force interactive agent selection").option("-y, --yes", "Skip confirmation prompts").action(pullCommand);
+program2.command("pull [agentIds...]").description("Pull agents from Retell API (always pulls latest draft state)").option("-a, --all", "Pull all agents in the account").option("-s, --select", "Force interactive agent selection").option("-y, --yes", "Skip confirmation prompts").option("--no-tests", "Skip pulling test case definitions").action(pullCommand);
 program2.command("deploy [agentIds...]").description("Deploy local changes to Retell draft").option("-a, --all", "Deploy all agents in the account").option("-s, --select", "Force interactive agent selection").option("-n, --dry-run", "Show changes without applying").option("-v, --verbose", "Show full diff details (use with --dry-run)").option("-q, --quiet", "Output only affected agent IDs (for piping)").action(deployCommand);
 program2.command("publish [agentIds...]").description("Publish agents with unpublished draft changes").option("-a, --all", "Publish all agents in the account").option("-s, --select", "Force interactive agent selection").option("-n, --dry-run", "Show what would be published without publishing").option("-q, --quiet", "Output only published agent IDs (for piping)").action(publishCommand);
 program2.parse();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "retell-sync-cli",
-  "version": "3.6.0",
+  "version": "3.7.0",
   "description": "CLI tool for syncing Retell AI agents between local filesystem and API",
   "keywords": [
     "agents",