struere 0.5.3 → 0.5.5

package/dist/bin/struere.js

@@ -19269,32 +19269,6 @@ function getApiKey() {
 
 // src/cli/utils/convex.ts
 var CONVEX_URL = process.env.STRUERE_CONVEX_URL || "https://rapid-wildebeest-172.convex.cloud";
-async function listAgents() {
-  const credentials = loadCredentials();
-  const apiKey = getApiKey();
-  const token = apiKey || credentials?.token;
-  if (!token) {
-    return { agents: [], error: "Not authenticated" };
-  }
-  const response = await fetch(`${CONVEX_URL}/api/query`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${token}`
-    },
-    body: JSON.stringify({
-      path: "agents:list",
-      args: {}
-    })
-  });
-  if (!response.ok) {
-    const error = await response.text();
-    return { agents: [], error };
-  }
-  const result = await response.json();
-  const agents = Array.isArray(result) ? result : result?.value || [];
-  return { agents };
-}
 async function listMyOrganizations(token) {
   const response = await fetch(`${CONVEX_URL}/api/query`, {
     method: "POST",
@@ -19481,18 +19455,24 @@ async function syncOrganization(payload) {
     }
     return { success: false, error: `Network error: ${err instanceof Error ? err.message : String(err)}` };
   }
+  const text = await response.text();
+  let json;
+  try {
+    json = JSON.parse(text);
+  } catch {
+    return { success: false, error: text || `HTTP ${response.status}` };
+  }
   if (!response.ok) {
-    const error = await response.text();
-    return { success: false, error };
+    const msg = json.errorMessage || json.message || text;
+    return { success: false, error: msg };
   }
-  const json = await response.json();
   if (json.status === "success" && json.value) {
     return json.value;
   }
   if (json.status === "error") {
     return { success: false, error: json.errorMessage || "Unknown error from Convex" };
   }
-  return { success: false, error: `Unexpected response: ${JSON.stringify(json)}` };
+  return { success: false, error: `Unexpected response: ${text}` };
 }
 async function getSyncState(organizationId, environment) {
   const credentials = loadCredentials();
@@ -19556,267 +19536,6 @@ async function getPullState(organizationId, environment = "development", include
   }
   return { error: `Unexpected response: ${JSON.stringify(result)}` };
 }
-async function resolveAgentId(agentSlug) {
-  const { agents } = await listAgents();
-  const agent = agents.find((a) => a.slug === agentSlug || a._id === agentSlug);
-  return agent?._id;
-}
-async function syncEvalSuites(suites) {
-  const credentials = loadCredentials();
-  const apiKey = getApiKey();
-  const token = apiKey || credentials?.token;
-  if (!token) {
-    return { error: "Not authenticated" };
-  }
-  const suiteIds = {};
-  for (const suite of suites) {
-    const agentId = await resolveAgentId(suite.agent);
-    if (!agentId) {
-      return { error: `Agent "${suite.agent}" not found. Make sure the agent exists before syncing evals.` };
-    }
-    const listResponse = await fetch(`${CONVEX_URL}/api/query`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${token}`
-      },
-      body: JSON.stringify({
-        path: "evals:listAllSuites",
-        args: { environment: "development" }
-      })
-    });
-    let existingSuiteId;
-    if (listResponse.ok) {
-      const listResult = await listResponse.json();
-      const existing = (listResult.value || []).find((s) => s.slug === suite.slug);
-      if (existing) {
-        existingSuiteId = existing._id;
-      }
-    }
-    if (!existingSuiteId) {
-      const createResponse = await fetch(`${CONVEX_URL}/api/mutation`, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          Authorization: `Bearer ${token}`
-        },
-        body: JSON.stringify({
-          path: "evals:createSuite",
-          args: {
-            agentId,
-            name: suite.suite,
-            slug: suite.slug,
-            description: suite.description,
-            tags: suite.tags,
-            judgeModel: suite.judgeModel ? { provider: "anthropic", name: suite.judgeModel } : undefined,
-            environment: "development"
-          }
-        })
-      });
-      if (!createResponse.ok) {
-        const error = await createResponse.text();
-        return { error: `Failed to create suite "${suite.suite}": ${error}` };
-      }
-      const createResult = await createResponse.json();
-      existingSuiteId = createResult.value;
-    } else {
-      const updateResponse = await fetch(`${CONVEX_URL}/api/mutation`, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          Authorization: `Bearer ${token}`
-        },
-        body: JSON.stringify({
-          path: "evals:updateSuite",
-          args: {
-            id: existingSuiteId,
-            name: suite.suite,
-            description: suite.description,
-            tags: suite.tags,
-            judgeModel: suite.judgeModel ? { provider: "anthropic", name: suite.judgeModel } : undefined
-          }
-        })
-      });
-      if (!updateResponse.ok) {
-        const error = await updateResponse.text();
-        return { error: `Failed to update suite "${suite.suite}": ${error}` };
-      }
-      const deleteResponse = await fetch(`${CONVEX_URL}/api/mutation`, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          Authorization: `Bearer ${token}`
-        },
-        body: JSON.stringify({
-          path: "evals:deleteCasesBySuite",
-          args: { suiteId: existingSuiteId }
-        })
-      });
-      if (!deleteResponse.ok) {
-        const error = await deleteResponse.text();
-        return { error: `Failed to clear cases for suite "${suite.suite}": ${error}` };
-      }
-    }
-    if (existingSuiteId) {
-      suiteIds[suite.slug] = existingSuiteId;
-      for (let i = 0;i < suite.cases.length; i++) {
-        const c = suite.cases[i];
-        const turns = c.turns.map((t) => ({
-          userMessage: t.user,
-          assertions: t.assertions?.map((a) => ({
-            type: a.type,
-            ...a.criteria ? { criteria: a.criteria } : {},
-            ...a.value ? { value: a.value } : {},
-            ...a.weight ? { weight: a.weight } : {}
-          }))
-        }));
-        const finalAssertions = c.finalAssertions?.map((a) => ({
-          type: a.type,
-          ...a.criteria ? { criteria: a.criteria } : {},
-          ...a.value ? { value: a.value } : {},
-          ...a.weight ? { weight: a.weight } : {}
-        }));
-        const caseResponse = await fetch(`${CONVEX_URL}/api/mutation`, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            Authorization: `Bearer ${token}`
-          },
-          body: JSON.stringify({
-            path: "evals:createCase",
-            args: {
-              suiteId: existingSuiteId,
-              name: c.name,
-              description: c.description,
-              tags: c.tags,
-              turns,
-              finalAssertions
-            }
-          })
-        });
-        if (!caseResponse.ok) {
-          const error = await caseResponse.text();
-          return { error: `Failed to create case "${c.name}": ${error}` };
-        }
-      }
-    }
-  }
-  return { suiteIds };
-}
-async function startEvalRun(suiteSlug) {
-  const credentials = loadCredentials();
-  const apiKey = getApiKey();
-  const token = apiKey || credentials?.token;
-  if (!token) {
-    return { error: "Not authenticated" };
-  }
-  const listResponse = await fetch(`${CONVEX_URL}/api/query`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${token}`
-    },
-    body: JSON.stringify({
-      path: "evals:listAllSuites",
-      args: { environment: "development" }
-    })
-  });
-  if (!listResponse.ok) {
-    return { error: "Failed to list suites" };
-  }
-  const listResult = await listResponse.json();
-  const suite = (listResult.value || []).find((s) => s.slug === suiteSlug);
-  if (!suite) {
-    return { error: `Suite "${suiteSlug}" not found` };
-  }
-  const response = await fetch(`${CONVEX_URL}/api/mutation`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${token}`
-    },
-    body: JSON.stringify({
-      path: "evals:startRun",
-      args: {
-        suiteId: suite._id,
-        triggerSource: "cli"
-      }
-    })
-  });
-  if (!response.ok) {
-    const error = await response.text();
-    return { error };
-  }
-  const result = await response.json();
-  return { runId: result.value, suiteId: suite._id };
-}
-async function pollEvalRun(runId, onProgress) {
-  const maxAttempts = 300;
-  const pollInterval = 2000;
-  for (let i = 0;i < maxAttempts; i++) {
-    const currentCredentials = loadCredentials();
-    const currentApiKey = getApiKey();
-    const token = currentApiKey || currentCredentials?.token;
-    if (!token) {
-      return { error: 'Authentication expired. Please run "struere login" again.' };
-    }
-    const response = await fetch(`${CONVEX_URL}/api/query`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${token}`
-      },
-      body: JSON.stringify({
-        path: "evals:getRun",
-        args: { id: runId }
-      })
-    });
-    if (!response.ok) {
-      if (response.status === 401) {
-        return { error: 'Authentication expired. Please run "struere login" again.' };
-      }
-      return { error: `Failed to poll run status (HTTP ${response.status})` };
-    }
-    const result = await response.json();
-    const run = result.value;
-    if (!run) {
-      return { error: "Run not found" };
-    }
-    if (onProgress) {
-      onProgress(run);
-    }
-    if (run.status === "completed" || run.status === "failed" || run.status === "cancelled") {
-      return { run };
-    }
-    await new Promise((resolve) => setTimeout(resolve, pollInterval));
-  }
-  return { error: "Run timed out after 10 minutes" };
-}
-async function getEvalRunResults(runId) {
-  const credentials = loadCredentials();
-  const apiKey = getApiKey();
-  const token = apiKey || credentials?.token;
-  if (!token) {
-    return { error: "Not authenticated" };
-  }
-  const response = await fetch(`${CONVEX_URL}/api/query`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${token}`
-    },
-    body: JSON.stringify({
-      path: "evals:getRunResults",
-      args: { runId }
-    })
-  });
-  if (!response.ok) {
-    const error = await response.text();
-    return { error };
-  }
-  const result = await response.json();
-  return { results: result.value || [] };
-}
 
 // src/cli/commands/login.ts
 var AUTH_CALLBACK_PORT = 9876;
@@ -21248,7 +20967,8 @@ import { join as join5 } from "path";
 import { existsSync as existsSync5, writeFileSync as writeFileSync4 } from "fs";
 
 // src/cli/utils/loader.ts
-import { existsSync as existsSync4, readdirSync } from "fs";
+var import_yaml = __toESM(require_dist(), 1);
+import { existsSync as existsSync4, readdirSync, readFileSync as readFileSync4 } from "fs";
 import { join as join4 } from "path";
 async function loadAllResources(cwd) {
   const errors2 = [];
@@ -21259,7 +20979,9 @@ async function loadAllResources(cwd) {
   if (toolsError) {
     errors2.push(toolsError);
   }
-  return { agents, entityTypes, roles, customTools, errors: errors2 };
+  const { suites: evalSuites, errors: evalErrors } = loadAllEvalSuites(join4(cwd, "evals"));
+  errors2.push(...evalErrors);
+  return { agents, entityTypes, roles, customTools, evalSuites, errors: errors2 };
 }
 async function loadAllAgents(dir) {
   if (!existsSync4(dir)) {
@@ -21349,16 +21071,42 @@ async function loadFromDirectory(dir) {
   }
   return items;
 }
+function loadAllEvalSuites(dir) {
+  const suites = [];
+  const errors2 = [];
+  if (!existsSync4(dir)) {
+    return { suites, errors: errors2 };
+  }
+  const files = readdirSync(dir).filter((f) => f.endsWith(".eval.yaml") || f.endsWith(".eval.yml"));
+  for (const file of files) {
+    try {
+      const content = readFileSync4(join4(dir, file), "utf-8");
+      const parsed = import_yaml.default.parse(content);
+      suites.push(parsed);
+    } catch (err) {
+      errors2.push(`Failed to parse ${file}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  return { suites, errors: errors2 };
+}
 function getResourceDirectories(cwd) {
   return {
     agents: join4(cwd, "agents"),
     entityTypes: join4(cwd, "entity-types"),
     roles: join4(cwd, "roles"),
-    tools: join4(cwd, "tools")
+    tools: join4(cwd, "tools"),
+    evals: join4(cwd, "evals")
   };
 }
 
 // src/cli/utils/extractor.ts
+function inferProvider(modelName) {
+  if (modelName.startsWith("gpt-") || modelName.startsWith("o1") || modelName.startsWith("o3") || modelName.startsWith("o4"))
+    return "openai";
+  if (modelName.startsWith("gemini"))
+    return "google";
+  return "anthropic";
+}
 var BUILTIN_TOOLS = [
   "entity.create",
   "entity.get",
@@ -21407,7 +21155,25 @@ function extractSyncPayload(resources) {
       maskConfig: fm.maskConfig
     }))
   }));
-  return { agents, entityTypes, roles };
+  const evalSuites = resources.evalSuites.length > 0 ? resources.evalSuites.map((suite) => ({
+    name: suite.suite,
+    slug: suite.slug,
+    agentSlug: suite.agent,
+    description: suite.description,
+    tags: suite.tags,
+    judgeModel: suite.judgeModel ? { provider: inferProvider(suite.judgeModel), name: suite.judgeModel } : undefined,
+    cases: suite.cases.map((c) => ({
+      name: c.name,
+      description: c.description,
+      tags: c.tags,
+      turns: c.turns.map((t) => ({
+        userMessage: t.user,
+        assertions: t.assertions
+      })),
+      finalAssertions: c.finalAssertions
+    }))
+  })) : undefined;
+  return { agents, entityTypes, roles, evalSuites };
 }
 function extractAgentPayload(agent, customToolsMap) {
   let systemPrompt;
@@ -21723,7 +21489,7 @@ var devCommand = new Command("dev").description("Sync all resources to developme
   spinner.start("Loading resources");
   try {
     const resources = await loadAllResources(cwd);
-    spinner.succeed(`Loaded ${resources.agents.length} agents, ${resources.entityTypes.length} entity types, ${resources.roles.length} roles, ${resources.customTools.length} custom tools`);
+    spinner.succeed(`Loaded ${resources.agents.length} agents, ${resources.entityTypes.length} entity types, ${resources.roles.length} roles, ${resources.customTools.length} custom tools, ${resources.evalSuites.length} eval suites`);
     for (const err of resources.errors) {
       console.log(source_default.red("  \u2716"), err);
     }
@@ -21781,6 +21547,7 @@ var devCommand = new Command("dev").description("Sync all resources to developme
     dirs.entityTypes,
     dirs.roles,
     dirs.tools,
+    dirs.evals,
     join5(cwd, "struere.config.ts")
   ].filter((p) => existsSync5(p));
   const watcher = import_chokidar.default.watch(watchPaths, {
@@ -22055,7 +21822,7 @@ var buildCommand = new Command("build").description("Build and validate agent fo
 });
 
 // src/cli/commands/test.ts
-var import_yaml = __toESM(require_dist(), 1);
+var import_yaml2 = __toESM(require_dist(), 1);
 import { join as join9 } from "path";
 import { readdir, readFile } from "fs/promises";
 var testCommand = new Command("test").description("Run test conversations").argument("[pattern]", "Test file pattern", "*.test.yaml").option("-v, --verbose", "Show detailed output").option("--dry-run", "Parse tests without executing (no API calls)").action(async (pattern, options) => {
@@ -22107,7 +21874,7 @@ var testCommand = new Command("test").description("Run test conversations").argu
   for (const file of testFiles) {
     const filePath = join9(testsDir, file);
     const content = await readFile(filePath, "utf-8");
-    const testCase = import_yaml.default.parse(content);
+    const testCase = import_yaml2.default.parse(content);
     if (options.verbose) {
       console.log();
       console.log(source_default.gray("Running:"), testCase.name);
@@ -22240,174 +22007,6 @@ function formatAssertionError(assertion, context) {
   }
 }
 
-// src/cli/commands/eval.ts
-var import_yaml2 = __toESM(require_dist(), 1);
-import { join as join10 } from "path";
-import { readdir as readdir2, readFile as readFile2 } from "fs/promises";
-var evalCommand = new Command("eval").description("Run agent evaluations with LLM judge support").option("-s, --suite <name>", "Run a specific suite by name").option("-v, --verbose", "Show detailed output including judge reasoning").option("--dry-run", "Parse eval files without executing").option("--json", "Output results as JSON").option("--no-sync", "Skip syncing eval files to Convex").action(async (options) => {
-  const spinner = ora();
-  const cwd = process.cwd();
-  if (!options.json) {
-    console.log();
-    console.log(source_default.bold("Running Evaluations"));
-    console.log();
-  }
-  if (!hasProject(cwd)) {
-    console.log(source_default.yellow("No struere.json found"));
-    console.log();
-    console.log(source_default.gray("Run"), source_default.cyan("struere init"), source_default.gray("to initialize this project"));
-    console.log();
-    process.exit(1);
-  }
-  const project = loadProject(cwd);
-  if (!project) {
-    console.log(source_default.red("Failed to load struere.json"));
-    process.exit(1);
-  }
-  spinner.start("Finding eval files");
-  const evalsDir = join10(cwd, "evals");
-  let evalFiles = [];
-  try {
-    const files = await readdir2(evalsDir);
-    evalFiles = files.filter((f) => f.endsWith(".eval.yaml") || f.endsWith(".eval.yml"));
-  } catch {
-    spinner.warn("No evals directory found");
-    console.log();
-    console.log(source_default.gray("Create eval files in"), source_default.cyan("evals/*.eval.yaml"));
-    console.log();
-    return;
-  }
-  if (evalFiles.length === 0) {
-    spinner.warn("No eval files found");
-    console.log();
-    return;
-  }
-  spinner.succeed(`Found ${evalFiles.length} eval file(s)`);
-  const suites = [];
-  for (const file of evalFiles) {
-    const filePath = join10(evalsDir, file);
-    const content = await readFile2(filePath, "utf-8");
-    const parsed = import_yaml2.default.parse(content);
-    suites.push(parsed);
-  }
-  if (options.suite) {
-    const filtered = suites.filter((s) => s.suite.toLowerCase() === options.suite.toLowerCase() || s.slug === options.suite);
-    if (filtered.length === 0) {
-      console.log(source_default.red(`Suite "${options.suite}" not found`));
-      process.exit(1);
-    }
-    suites.length = 0;
-    suites.push(...filtered);
-  }
-  if (options.dryRun) {
-    console.log();
-    console.log(source_default.yellow("Dry run mode \u2014 parsed successfully"));
-    console.log();
-    for (const suite of suites) {
-      console.log(source_default.cyan(`  ${suite.suite}`), source_default.gray(`(${suite.cases.length} cases)`));
-      for (const c of suite.cases) {
-        const assertionCount = (c.turns || []).reduce((sum, t) => sum + (t.assertions?.length || 0), 0) + (c.finalAssertions?.length || 0);
-        console.log(source_default.gray(`    - ${c.name}`), source_default.gray(`(${c.turns.length} turns, ${assertionCount} assertions)`));
-      }
-    }
-    console.log();
-    return;
-  }
-  if (options.sync !== false) {
-    spinner.start("Syncing eval suites to Convex");
-    const syncResult = await syncEvalSuites(suites);
-    if (syncResult.error) {
-      spinner.fail(`Sync failed: ${syncResult.error}`);
-      process.exit(1);
-    }
-    spinner.succeed("Eval suites synced");
-  }
-  const allResults = [];
-  for (const suite of suites) {
-    if (!options.json) {
-      console.log();
-      console.log(source_default.bold(`  ${suite.suite}`), source_default.gray(`(${suite.cases.length} cases)`));
-    }
-    spinner.start(`Starting run for "${suite.suite}"`);
-    const { runId, suiteId, error: startError } = await startEvalRun(suite.slug);
-    if (startError || !runId) {
-      spinner.fail(`Failed to start: ${startError}`);
-      continue;
-    }
-    spinner.succeed(`Run started`);
-    spinner.start("Executing cases...");
-    const { run, error: pollError } = await pollEvalRun(runId, (status) => {
-      spinner.text = `Executing cases... ${status.completedCases}/${status.totalCases}`;
-    });
-    if (pollError || !run) {
-      spinner.fail(`Run failed: ${pollError}`);
-      continue;
-    }
-    spinner.succeed(`Run completed: ${run.passedCases}/${run.totalCases} passed`);
-    const { results, error: resultsError } = await getEvalRunResults(runId);
-    if (resultsError) {
-      console.log(source_default.red(`  Failed to get results: ${resultsError}`));
-      continue;
-    }
-    allResults.push({ suite: suite.suite, run, results: results || [] });
-    if (!options.json) {
-      for (let ri = 0;ri < (results || []).length; ri++) {
-        const result = results[ri];
-        const caseName = ri < suite.cases.length ? suite.cases[ri].name : result.caseId;
-        if (result.overallPassed) {
-          console.log(source_default.green("    \u2713"), caseName, result.overallScore !== undefined ? source_default.gray(`(${result.overallScore.toFixed(1)}/5)`) : "");
-        } else {
-          console.log(source_default.red("    \u2717"), caseName, result.overallScore !== undefined ? source_default.gray(`(${result.overallScore.toFixed(1)}/5)`) : "");
-          if (result.errorMessage) {
-            console.log(source_default.red("      Error:"), result.errorMessage);
-          }
-          if (options.verbose && result.turnResults) {
-            for (const turn of result.turnResults) {
-              if (turn.assertionResults) {
-                for (const ar of turn.assertionResults) {
-                  if (!ar.passed) {
-                    console.log(source_default.red(`      [${ar.type}]`), ar.reason || "");
-                    if (ar.criteria) {
-                      console.log(source_default.gray(`        Criteria: ${ar.criteria}`));
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-  if (options.json) {
-    console.log(JSON.stringify(allResults, null, 2));
-    const anyFailed = allResults.some((r) => r.run.failedCases > 0);
-    process.exit(anyFailed ? 1 : 0);
-    return;
-  }
-  const totalPassed = allResults.reduce((sum, r) => sum + r.run.passedCases, 0);
-  const totalCases = allResults.reduce((sum, r) => sum + r.run.totalCases, 0);
-  const totalFailed = allResults.reduce((sum, r) => sum + r.run.failedCases, 0);
-  console.log();
-  if (totalFailed === 0) {
-    console.log(source_default.green("All evaluations passed!"), source_default.gray(`(${totalPassed}/${totalCases})`));
-  } else {
-    console.log(source_default.red("Evaluations failed:"), source_default.gray(`${totalPassed}/${totalCases} passed`));
-  }
-  const totalTokens = allResults.reduce((sum, r) => {
-    if (r.run.totalTokens)
-      return sum + r.run.totalTokens.agent + r.run.totalTokens.judge;
-    return sum;
-  }, 0);
-  if (totalTokens > 0) {
-    console.log(source_default.gray(`  Total tokens used: ${totalTokens.toLocaleString()}`));
-  }
-  console.log();
-  if (totalFailed > 0) {
-    process.exit(1);
-  }
-});
-
 // src/cli/commands/deploy.ts
 var isAuthError = (error) => {
   const message = error instanceof Error ? error.message : String(error);
@@ -22463,7 +22062,7 @@ var deployCommand = new Command("deploy").description("Deploy all resources to p
   let resources;
   try {
     resources = await loadAllResources(cwd);
-    spinner.succeed(`Loaded ${resources.agents.length} agents, ${resources.entityTypes.length} entity types, ${resources.roles.length} roles, ${resources.customTools.length} custom tools`);
+    spinner.succeed(`Loaded ${resources.agents.length} agents, ${resources.entityTypes.length} entity types, ${resources.roles.length} roles, ${resources.customTools.length} custom tools, ${resources.evalSuites.length} eval suites`);
     for (const err of resources.errors) {
       console.log(source_default.red("  \u2716"), err);
     }
@@ -22501,6 +22100,13 @@ var deployCommand = new Command("deploy").description("Deploy all resources to p
     for (const role of resources.roles) {
       console.log(source_default.gray("  -"), source_default.cyan(role.name));
     }
+    if (resources.evalSuites.length > 0) {
+      console.log();
+      console.log("Eval suites:");
+      for (const suite of resources.evalSuites) {
+        console.log(source_default.gray("  -"), source_default.cyan(suite.suite), source_default.gray(`(${suite.cases.length} cases)`));
+      }
+    }
     console.log();
     return;
   }
@@ -22937,7 +22543,7 @@ var statusCommand = new Command("status").description("Compare local vs remote s
   let localResources;
   try {
     localResources = await loadAllResources(cwd);
-    spinner.succeed(`Loaded ${localResources.agents.length} agents, ${localResources.entityTypes.length} entity types, ${localResources.roles.length} roles, ${localResources.customTools.length} custom tools`);
+    spinner.succeed(`Loaded ${localResources.agents.length} agents, ${localResources.entityTypes.length} entity types, ${localResources.roles.length} roles, ${localResources.customTools.length} custom tools, ${localResources.evalSuites.length} eval suites`);
     for (const err of localResources.errors) {
       console.log(source_default.red("  \u2716"), err);
     }
@@ -23044,7 +22650,7 @@ var statusCommand = new Command("status").description("Compare local vs remote s
 
 // src/cli/commands/pull.ts
 import { existsSync as existsSync6, mkdirSync as mkdirSync3, writeFileSync as writeFileSync5 } from "fs";
-import { join as join11 } from "path";
+import { join as join10 } from "path";
 
 // src/cli/utils/generator.ts
 var BUILTIN_TOOLS2 = [
@@ -23351,7 +22957,7 @@ var pullCommand = new Command("pull").description("Pull remote resources to loca
     }
   };
   const writeOrSkip = (relativePath, content) => {
-    const fullPath = join11(cwd, relativePath);
+    const fullPath = join10(cwd, relativePath);
     if (existsSync6(fullPath) && !options.force) {
       skipped.push(relativePath);
       return false;
@@ -23360,15 +22966,15 @@ var pullCommand = new Command("pull").description("Pull remote resources to loca
       created.push(relativePath);
       return true;
     }
-    ensureDir2(join11(cwd, relativePath.split("/").slice(0, -1).join("/")));
+    ensureDir2(join10(cwd, relativePath.split("/").slice(0, -1).join("/")));
     writeFileSync5(fullPath, content);
     created.push(relativePath);
     return true;
   };
-  ensureDir2(join11(cwd, "agents"));
-  ensureDir2(join11(cwd, "entity-types"));
-  ensureDir2(join11(cwd, "roles"));
-  ensureDir2(join11(cwd, "tools"));
+  ensureDir2(join10(cwd, "agents"));
+  ensureDir2(join10(cwd, "entity-types"));
+  ensureDir2(join10(cwd, "roles"));
+  ensureDir2(join10(cwd, "tools"));
   const agentSlugs = [];
   for (const agent of state.agents) {
     if (!agent.systemPrompt && agent.tools.length === 0)
@@ -23454,7 +23060,7 @@ var pullCommand = new Command("pull").description("Pull remote resources to loca
 // package.json
 var package_default = {
   name: "struere",
-  version: "0.5.3",
+  version: "0.5.5",
   description: "Build, test, and deploy AI agents",
   keywords: [
     "ai",
@@ -23556,7 +23162,6 @@ program.addCommand(whoamiCommand);
 program.addCommand(devCommand);
 program.addCommand(buildCommand);
 program.addCommand(testCommand);
-program.addCommand(evalCommand);
 program.addCommand(deployCommand);
 program.addCommand(validateCommand);
 program.addCommand(logsCommand);