tarsk 0.5.65 → 0.5.67

package/dist/bundled-skills/skill-creator/scripts/improve_description.js

@@ -32,28 +32,82 @@ function callTarsk(prompt, model, timeoutMs = 300000) {
   return result.stdout;
 }
 
-function improveDescription(
+function parseDescription(text) {
+  const match = text.match(/<new_description>([\s\S]*?)<\/new_description>/);
+  return match
+    ? match[1].trim().replace(/^["']|["']$/g, "")
+    : text.trim().replace(/^["']|["']$/g, "");
+}
+
+function computeScoresSummary(evalResults, testResults) {
+  const trainScore = `${evalResults.summary.passed}/${evalResults.summary.total}`;
+  if (testResults) {
+    return `Train: ${trainScore}, Test: ${testResults.summary.passed}/${testResults.summary.total}`;
+  }
+  return `Train: ${trainScore}`;
+}
+
+function buildFailuresText(evalResults) {
+  const failedTriggers = evalResults.results.filter((r) => r.should_trigger && !r.pass);
+  const falseTriggers = evalResults.results.filter((r) => !r.should_trigger && !r.pass);
+
+  let text = "";
+  if (failedTriggers.length) {
+    text += "FAILED TO TRIGGER (should have triggered but didn't):\n";
+    for (const r of failedTriggers) {
+      text += `  - "${r.query}" (triggered ${r.triggers}/${r.runs} times)\n`;
+    }
+    text += "\n";
+  }
+
+  if (falseTriggers.length) {
+    text += "FALSE TRIGGERS (triggered but shouldn't have):\n";
+    for (const r of falseTriggers) {
+      text += `  - "${r.query}" (triggered ${r.triggers}/${r.runs} times)\n`;
+    }
+    text += "\n";
+  }
+
+  return text;
+}
+
+function buildHistoryText(history) {
+  if (!history.length) {
+    return "";
+  }
+
+  let text = "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n";
+  for (const h of history) {
+    const trainS = `${h.train_passed ?? h.passed ?? 0}/${h.train_total ?? h.total ?? 0}`;
+    const testS = h.test_passed != null ? `${h.test_passed}/${h.test_total ?? "?"}` : null;
+    const scoreStr = `train=${trainS}` + (testS ? `, test=${testS}` : "");
+    text += `<attempt ${scoreStr}>\n`;
+    text += `Description: "${h.description}"\n`;
+    if (h.results) {
+      text += "Train results:\n";
+      for (const r of h.results) {
+        const status = r.pass ? "PASS" : "FAIL";
+        text += `  [${status}] "${r.query.slice(0, 80)}" (triggered ${r.triggers}/${r.runs})\n`;
+      }
+    }
+    if (h.note) {
+      text += `Note: ${h.note}\n`;
+    }
+    text += "</attempt>\n\n";
+  }
+
+  return text;
+}
+
+function buildImprovePrompt(
   skillName,
   skillContent,
   currentDescription,
   evalResults,
   history,
-  model,
-  testResults = null,
-  logDir = null,
-  iteration = null,
+  testResults,
 ) {
-  const failedTriggers = evalResults.results.filter((r) => r.should_trigger && !r.pass);
-  const falseTriggers = evalResults.results.filter((r) => !r.should_trigger && !r.pass);
-
-  const trainScore = `${evalResults.summary.passed}/${evalResults.summary.total}`;
-  let scoresSummary;
-  if (testResults) {
-    const testScore = `${testResults.summary.passed}/${testResults.summary.total}`;
-    scoresSummary = `Train: ${trainScore}, Test: ${testScore}`;
-  } else {
-    scoresSummary = `Train: ${trainScore}`;
-  }
+  const scoresSummary = computeScoresSummary(evalResults, testResults);
 
   let prompt = `You are optimizing a skill description for a Tarsk Code skill called "${skillName}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Tarsk sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples.
 
@@ -68,43 +122,8 @@ Current scores (${scoresSummary}):
 <scores_summary>
 `;
 
-  if (failedTriggers.length) {
-    prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n";
-    for (const r of failedTriggers) {
-      prompt += `  - "${r.query}" (triggered ${r.triggers}/${r.runs} times)\n`;
-    }
-    prompt += "\n";
-  }
-
-  if (falseTriggers.length) {
-    prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n";
-    for (const r of falseTriggers) {
-      prompt += `  - "${r.query}" (triggered ${r.triggers}/${r.runs} times)\n`;
-    }
-    prompt += "\n";
-  }
-
-  if (history.length) {
-    prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n";
-    for (const h of history) {
-      const trainS = `${h.train_passed ?? h.passed ?? 0}/${h.train_total ?? h.total ?? 0}`;
-      const testS = h.test_passed != null ? `${h.test_passed}/${h.test_total ?? "?"}` : null;
-      const scoreStr = `train=${trainS}` + (testS ? `, test=${testS}` : "");
-      prompt += `<attempt ${scoreStr}>\n`;
-      prompt += `Description: "${h.description}"\n`;
-      if (h.results) {
-        prompt += "Train results:\n";
-        for (const r of h.results) {
-          const status = r.pass ? "PASS" : "FAIL";
-          prompt += `  [${status}] "${r.query.slice(0, 80)}" (triggered ${r.triggers}/${r.runs})\n`;
-        }
-      }
-      if (h.note) {
-        prompt += `Note: ${h.note}\n`;
-      }
-      prompt += "</attempt>\n\n";
-    }
-  }
+  prompt += buildFailuresText(evalResults);
+  prompt += buildHistoryText(history);
 
   prompt += `</scores_summary>
 
@@ -130,11 +149,49 @@ I'd encourage you to be creative and mix up the style in different iterations si
 
 Please respond with only the new description text in <new_description> tags, nothing else.`;
 
+  return prompt;
+}
+
+function shortenDescription(prompt, description, model) {
+  const shortenPrompt =
+    `${prompt}\n\n` +
+    `---\n\n` +
+    `A previous attempt produced this description, which at ` +
+    `${description.length} characters is over the 1024-character hard limit:\n\n` +
+    `"${description}"\n\n` +
+    `Rewrite it to be under 1024 characters while keeping the most ` +
+    `important trigger words and intent coverage. Respond with only ` +
+    `the new description in <new_description> tags.`;
+  const shortenText = callTarsk(shortenPrompt, model);
+  return {
+    shortenPrompt,
+    shortenText,
+    shortened: parseDescription(shortenText),
+  };
+}
+
+function improveDescription(
+  skillName,
+  skillContent,
+  currentDescription,
+  evalResults,
+  history,
+  model,
+  testResults = null,
+  logDir = null,
+  iteration = null,
+) {
+  const prompt = buildImprovePrompt(
+    skillName,
+    skillContent,
+    currentDescription,
+    evalResults,
+    history,
+    testResults,
+  );
+
   let text = callTarsk(prompt, model);
-  let match = text.match(/<new_description>([\s\S]*?)<\/new_description>/);
-  let description = match
-    ? match[1].trim().replace(/^["']|["']$/g, "")
-    : text.trim().replace(/^["']|["']$/g, "");
+  let description = parseDescription(text);
 
   const transcript = {
     iteration,
@@ -146,21 +203,11 @@ Please respond with only the new description text in <new_description> tags, not
   };
 
   if (description.length > 1024) {
-    const shortenPrompt =
-      `${prompt}\n\n` +
-      `---\n\n` +
-      `A previous attempt produced this description, which at ` +
-      `${description.length} characters is over the 1024-character hard limit:\n\n` +
-      `"${description}"\n\n` +
-      `Rewrite it to be under 1024 characters while keeping the most ` +
-      `important trigger words and intent coverage. Respond with only ` +
-      `the new description in <new_description> tags.`;
-    const shortenText = callTarsk(shortenPrompt, model);
-    match = shortenText.match(/<new_description>([\s\S]*?)<\/new_description>/);
-    const shortened = match
-      ? match[1].trim().replace(/^["']|["']$/g, "")
-      : shortenText.trim().replace(/^["']|["']$/g, "");
-
+    const { shortenPrompt, shortenText, shortened } = shortenDescription(
+      prompt,
+      description,
+      model,
+    );
     transcript.rewrite_prompt = shortenPrompt;
     transcript.rewrite_response = shortenText;
     transcript.rewrite_description = shortened;

package/dist/bundled-skills/skill-creator/scripts/run_eval.js

@@ -29,6 +29,72 @@ function buildEnv() {
   return env;
 }
 
+function handleStreamEventType(se, seType, state) {
+  if (seType === "content_block_start") {
+    const cb = se.content_block ?? {};
+    if (cb.type === "tool_use") {
+      const toolName = cb.name ?? "";
+      if (toolName === "Skill" || toolName === "Read") {
+        state.pendingToolName = toolName;
+        state.accumulatedJson = "";
+      } else {
+        state.finish(false);
+        return true;
+      }
+    }
+  } else if (seType === "content_block_delta" && state.pendingToolName) {
+    const delta = se.delta ?? {};
+    if (delta.type === "input_json_delta") {
+      state.accumulatedJson += delta.partial_json ?? "";
+      if (state.accumulatedJson.includes(state.cleanName)) {
+        state.finish(true);
+        return true;
+      }
+    }
+  } else if (seType === "content_block_stop" || seType === "message_stop") {
+    if (state.pendingToolName) {
+      state.finish(state.accumulatedJson.includes(state.cleanName));
+      return true;
+    }
+    if (seType === "message_stop") {
+      state.finish(false);
+      return true;
+    }
+  }
+  return false;
+}
+
+function handleAssistantEvent(message, state) {
+  for (const contentItem of message.content ?? []) {
+    if (contentItem.type !== "tool_use") continue;
+    const toolName = contentItem.name ?? "";
+    const toolInput = contentItem.input ?? {};
+    if (toolName === "Skill" && (toolInput.skill ?? "").includes(state.cleanName)) {
+      state.triggered = true;
+    } else if (toolName === "Read" && (toolInput.file_path ?? "").includes(state.cleanName)) {
+      state.triggered = true;
+    }
+    state.finish(state.triggered);
+    return true;
+  }
+  return false;
+}
+
+function handleStreamEvent(event, state) {
+  if (event.type === "stream_event") {
+    const se = event.event ?? {};
+    return handleStreamEventType(se, se.type ?? "", state);
+  }
+  if (event.type === "assistant") {
+    return handleAssistantEvent(event.message ?? {}, state);
+  }
+  if (event.type === "result") {
+    state.finish(state.triggered);
+    return true;
+  }
+  return false;
+}
+
 function runSingleQuery(query, skillName, skillDescription, timeoutSec, projectRoot, model = null) {
   return new Promise((resolve) => {
     const uniqueId = crypto.randomBytes(4).toString("hex").slice(0, 8);
@@ -79,6 +145,14 @@ function runSingleQuery(query, skillName, skillDescription, timeoutSec, projectR
       let accumulatedJson = "";
       let finished = false;
 
+      const state = {
+        cleanName,
+        pendingToolName,
+        accumulatedJson,
+        triggered,
+        finish,
+      };
+
       function finish(value) {
         if (finished) return;
         finished = true;
@@ -108,59 +182,7 @@ function runSingleQuery(query, skillName, skillDescription, timeoutSec, projectR
             continue;
           }
 
-          if (event.type === "stream_event") {
-            const se = event.event ?? {};
-            const seType = se.type ?? "";
-
-            if (seType === "content_block_start") {
-              const cb = se.content_block ?? {};
-              if (cb.type === "tool_use") {
-                const toolName = cb.name ?? "";
-                if (toolName === "Skill" || toolName === "Read") {
-                  pendingToolName = toolName;
-                  accumulatedJson = "";
-                } else {
-                  finish(false);
-                  return;
-                }
-              }
-            } else if (seType === "content_block_delta" && pendingToolName) {
-              const delta = se.delta ?? {};
-              if (delta.type === "input_json_delta") {
-                accumulatedJson += delta.partial_json ?? "";
-                if (accumulatedJson.includes(cleanName)) {
-                  finish(true);
-                  return;
-                }
-              }
-            } else if (seType === "content_block_stop" || seType === "message_stop") {
-              if (pendingToolName) {
-                finish(accumulatedJson.includes(cleanName));
-                return;
-              }
-              if (seType === "message_stop") {
-                finish(false);
-                return;
-              }
-            }
-          } else if (event.type === "assistant") {
-            const message = event.message ?? {};
-            for (const contentItem of message.content ?? []) {
-              if (contentItem.type !== "tool_use") continue;
-              const toolName = contentItem.name ?? "";
-              const toolInput = contentItem.input ?? {};
-              if (toolName === "Skill" && (toolInput.skill ?? "").includes(cleanName)) {
-                triggered = true;
-              } else if (toolName === "Read" && (toolInput.file_path ?? "").includes(cleanName)) {
-                triggered = true;
-              }
-              finish(triggered);
-              return;
-            }
-          } else if (event.type === "result") {
-            finish(triggered);
-            return;
-          }
+          if (handleStreamEvent(event, state)) return;
         }
       });
 
@@ -342,4 +364,4 @@ if (require.main === module) {
   });
 }
 
-module.exports = { findProjectRoot, runEval, runSingleQuery };
+module.exports = { findProjectRoot, runEval };