taskmonkey-cli 0.1.0 → 0.2.0

package/bin/tm.js

@@ -8,6 +8,8 @@ import { pull } from '../src/commands/pull.js';
 import { watch } from '../src/commands/watch.js';
 import { logs } from '../src/commands/logs.js';
 import { chat } from '../src/commands/chat.js';
+import { testChat } from '../src/commands/test-chat.js';
+import { testConversations } from '../src/commands/test-conversations.js';
 
 const program = new Command();
 
@@ -57,4 +59,17 @@ program
   .option('-p, --public', 'Public chat (no auth, no tools)')
   .action(chat);
 
+program
+  .command('test-chat <message>')
+  .description('Single-shot chat test (send message, see response + tool calls)')
+  .option('-t, --task <slug>', 'Monkey task context')
+  .action(testChat);
+
+program
+  .command('test-conversations')
+  .description('Run all conversation_tests from config')
+  .option('-t, --task <slug>', 'Monkey task slug')
+  .option('-v, --verbose', 'Show response excerpts')
+  .action(testConversations);
+
 program.parse();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "taskmonkey-cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "TaskMonkey CLI — Remote dev tools for tenant config editing and tool testing",
   "bin": {
     "tm": "./bin/tm.js",

package/skills/test-chat.md

@@ -0,0 +1,14 @@
+---
+description: Einzelne Chat-Nachricht testen und Antwort + Tool-Calls sehen
+---
+
+Sende eine einzelne Nachricht an den Chat und zeige die Antwort mit allen Tool-Calls.
+Nützlich um Prompt-Änderungen schnell zu testen.
+
+```bash
+tm test-chat "welche erdbeeren habt ihr?"
+tm test-chat "Tomate 15" --task inventur
+```
+
+Zeige dem User das Ergebnis. Wenn Tool-Calls gemacht wurden, liste sie auf.
+Wenn das Ergebnis nicht den Erwartungen entspricht, schlage Prompt-Änderungen vor.

package/skills/test-conversations.md

@@ -0,0 +1,25 @@
+---
+description: Automatische Konversationstests aus der Config ausführen
+---
+
+Führt alle conversation_tests aus der Tenant-Config aus und zeigt das Ergebnis.
+
+```bash
+tm test-conversations
+tm test-conversations --task inventur --verbose
+```
+
+Test-Cases werden in der Config definiert:
+```php
+'conversation_tests' => [
+    [
+        'description' => 'Erdbeere suchen',
+        'user_message' => 'welche erdbeeren habt ihr?',
+        'expect_tool_calls' => ['getProducts'],
+        'expect_response_contains' => ['Erdbeere', 'Preis'],
+    ],
+]
+```
+
+Bei fehlgeschlagenen Tests: Analysiere warum und schlage Prompt-Änderungen vor.
+Nach Änderungen: `tm sync` und dann `tm test-conversations` erneut ausführen.

package/src/commands/test-chat.js

@@ -0,0 +1,43 @@
+import chalk from 'chalk';
+import ora from 'ora';
+import { createClient } from '../lib/api.js';
+
+export async function testChat(message, options) {
+  const client = createClient();
+
+  const spinner = ora('Sending test message...').start();
+
+  try {
+    const result = await client.post('/api/test/chat', {
+      message,
+      task: options.task || null,
+    });
+
+    spinner.stop();
+
+    // Tool calls
+    if (result.tool_calls?.length > 0) {
+      for (const tool of result.tool_calls) {
+        const icon = tool.success !== false ? chalk.green('✓') : chalk.red('✗');
+        const args = tool.args ? ' ' + chalk.gray(JSON.stringify(tool.args).substring(0, 100)) : '';
+        console.log(`  ${icon} ${chalk.yellow(tool.tool)}${args}`);
+      }
+      console.log();
+    }
+
+    // Response
+    if (result.response) {
+      console.log(result.response.replace(/\n{3,}/g, '\n\n').trim());
+    }
+
+    // Suggestions
+    if (result.suggestions?.length > 0) {
+      console.log();
+      console.log(result.suggestions.map(s => chalk.bgGray.white(` ${s} `)).join(' '));
+    }
+
+  } catch (err) {
+    spinner.fail(err.message);
+    process.exit(1);
+  }
+}

package/src/commands/test-conversations.js

@@ -0,0 +1,56 @@
+import chalk from 'chalk';
+import ora from 'ora';
+import { createClient } from '../lib/api.js';
+
+export async function testConversations(options) {
+  const client = createClient();
+
+  const spinner = ora('Running conversation tests...').start();
+
+  try {
+    const result = await client.post('/api/test/conversations', {
+      task: options.task || null,
+    });
+
+    spinner.stop();
+
+    if (result.total === 0) {
+      console.log(chalk.yellow('Keine conversation_tests definiert.'));
+      console.log(chalk.gray('Füge conversation_tests zur Tenant-Config hinzu.'));
+      return;
+    }
+
+    // Results
+    for (const test of result.results) {
+      const icon = test.status === 'pass' ? chalk.green('✓')
+        : test.status === 'fail' ? chalk.red('✗')
+        : chalk.gray('○');
+
+      console.log(`${icon} ${test.description}`);
+
+      if (test.tools_called?.length > 0) {
+        console.log(chalk.gray(`    Tools: ${test.tools_called.join(', ')}`));
+      }
+
+      if (test.errors?.length > 0) {
+        for (const err of test.errors) {
+          console.log(chalk.red(`    ✗ ${err}`));
+        }
+      }
+
+      if (options.verbose && test.response_excerpt) {
+        console.log(chalk.gray(`    Response: ${test.response_excerpt}`));
+      }
+    }
+
+    // Summary
+    console.log();
+    const rate = result.total > 0 ? Math.round((result.passed / result.total) * 100) : 0;
+    const color = rate === 100 ? chalk.green : rate >= 70 ? chalk.yellow : chalk.red;
+    console.log(color(`${result.passed}/${result.total} bestanden (${rate}%)`));
+
+  } catch (err) {
+    spinner.fail(err.message);
+    process.exit(1);
+  }
+}