qualty 0.1.2 → 0.1.4

package/bin/qualty.js

@@ -3,8 +3,8 @@
 import { spawn } from "node:child_process";
 import process from "node:process";
 
-/** When unset, CLI talks to Qualty cloud. Override for self-hosted or local dev, e.g. QUALTY_API_URL=http://localhost:8000 */
-const DEFAULT_QUALTY_API_URL = "https://qualty-api-production.up.railway.app";
+/** When unset, CLI uses Qualty development API. Override for production: QUALTY_API_URL=https://qualty-api-production.up.railway.app or local: http://localhost:8000 */
+const DEFAULT_QUALTY_API_URL = "https://qualty-api-development.up.railway.app";
 
 function resolveApiUrl(args) {
   return String(args.api || process.env.QUALTY_API_URL || DEFAULT_QUALTY_API_URL).replace(/\/$/, "");
@@ -39,7 +39,10 @@ function usage() {
       "Usage:",
       "  qualty connect --project <project-id> [--port 3000] [--api https://your-api] [--token <bearer-token>]",
       "  qualty run --project <project-id> [--suite-id <suite-id>] [--ids <id1,id2>] [--api https://your-api] [--token <bearer-token>]",
-      "             [--poll-interval 5] [--timeout 30] [--fail-on-failure true]",
+      "             [--poll-interval 5] [--timeout 30] [--fail-on-failure true] [--no-view-logs]",
+      "",
+      "  After each run, logs include step results, explanation, and final evaluator output (dashboard \"View logs\" data).",
+      "  Pass --no-view-logs for a short log only (e.g. very large evaluator text).",
       "  qualty resolve --project <project-id> [--suite-id <suite-id>] [--ids <id1,id2>] [--json] [--api https://your-api] [--token <bearer-token>]",
       "",
       "Env vars:",
@@ -171,6 +174,97 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function truncateForCiLog(text, maxLen) {
+  const s = String(text ?? "");
+  if (s.length <= maxLen) return s;
+  return `${s.slice(0, maxLen)}\n[qualty] … (truncated, ${s.length - maxLen} more chars)`;
+}
+
+function logPrefixedLines(prefix, text) {
+  const body = String(text ?? "").trimEnd();
+  if (!body) return;
+  for (const line of body.split("\n")) {
+    // eslint-disable-next-line no-console
+    console.log(`${prefix}${line}`);
+  }
+}
+
+/**
+ * Print the same kind of detail as Qualty "View logs": steps, explanation, evaluator (agent) output.
+ * Data comes from GET status payload fields on each combination (steps_json, explanation, agent_output).
+ */
+function printQualtyViewLogsReport(executionId, status) {
+  // eslint-disable-next-line no-console
+  console.log(`\n[qualty] ━━━ View logs: ${executionId} (${status.episode_name || "run"}) ━━━`);
+  if (status.url) {
+    // eslint-disable-next-line no-console
+    console.log(`[qualty] URL: ${status.url}`);
+  }
+  if (status.error) {
+    // eslint-disable-next-line no-console
+    console.log(`[qualty] Run error: ${truncateForCiLog(status.error, 4000)}`);
+  }
+  if (status.expected_behavior) {
+    // eslint-disable-next-line no-console
+    console.log(`[qualty] Expected behavior:\n[qualty] ${truncateForCiLog(status.expected_behavior, 6000).split("\n").join("\n[qualty] ")}`);
+  }
+  const combos = Array.isArray(status.combinations) ? status.combinations : [];
+  if (combos.length === 0) {
+    // eslint-disable-next-line no-console
+    console.log(
+      "[qualty] (No per-device breakdown in API response yet — open this run in the Qualty dashboard for full logs.)"
+    );
+    // eslint-disable-next-line no-console
+    console.log(`[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+    return;
+  }
+  for (let i = 0; i < combos.length; i += 1) {
+    const c = combos[i];
+    const device = c.device ?? "?";
+    const comboStatus = c.status ?? (c.success === true ? "passed" : c.success === false ? "failed" : "?");
+    // eslint-disable-next-line no-console
+    console.log(`\n[qualty] --- Combination ${i + 1}/${combos.length} (${device} · ${comboStatus}) ---`);
+    const steps = Array.isArray(c.steps_json) ? c.steps_json : [];
+    if (steps.length > 0) {
+      // eslint-disable-next-line no-console
+      console.log(`[qualty] Steps (${steps.length}):`);
+      for (let j = 0; j < steps.length; j += 1) {
+        const s = steps[j];
+        const label =
+          (s.name && String(s.name).trim()) ||
+          (s.description && String(s.description).trim().slice(0, 100)) ||
+          `Step ${j + 1}`;
+        const st = s.status != null ? s.status : "?";
+        // eslint-disable-next-line no-console
+        console.log(`[qualty]   ${j + 1}. [${st}] ${label}`);
+        if (s.description && String(s.description).trim() && String(s.description) !== String(s.name)) {
+          logPrefixedLines("[qualty]      ", truncateForCiLog(s.description, 4000));
+        }
+        if (s.action) {
+          // eslint-disable-next-line no-console
+          console.log(`[qualty]      action: ${truncateForCiLog(s.action, 2000)}`);
+        }
+      }
+    } else if (c.total_steps) {
+      // eslint-disable-next-line no-console
+      console.log(`[qualty] (Step list not available yet; ${c.total_steps} step(s) reported.)`);
+    }
+    if (c.explanation) {
+      // eslint-disable-next-line no-console
+      console.log(`[qualty] Explanation:\n[qualty] ${truncateForCiLog(c.explanation, 12000).split("\n").join("\n[qualty] ")}`);
+    }
+    const evaluator = c.agent_output ?? c.gpt_output;
+    if (evaluator) {
+      // eslint-disable-next-line no-console
+      console.log(
+        `[qualty] Final evaluator output:\n[qualty] ${truncateForCiLog(evaluator, 16000).split("\n").join("\n[qualty] ")}`
+      );
+    }
+  }
+  // eslint-disable-next-line no-console
+  console.log(`\n[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+}
+
 async function runCi(args) {
   const apiUrl = resolveApiUrl(args);
   const token = args.token || process.env.QUALTY_API_TOKEN;
@@ -183,6 +277,7 @@ async function runCi(args) {
   const pollIntervalSec = Number(args["poll-interval"] || 5);
   const timeoutMin = Number(args.timeout || 30);
   const failOnFailure = parseBoolean(args["fail-on-failure"], true);
+  const noViewLogs = parseBoolean(args["no-view-logs"], false);
 
   if (!token) {
     throw new Error("Missing auth token. Pass --token or set QUALTY_API_TOKEN.");
@@ -282,6 +377,14 @@ async function runCi(args) {
     );
   }
 
+  if (!noViewLogs) {
+    // eslint-disable-next-line no-console
+    console.log(`[qualty] Detailed run output (same fields as dashboard "View logs"):`);
+    for (const executionId of executionJobIds) {
+      printQualtyViewLogsReport(executionId, finalStatuses[executionId] || {});
+    }
+  }
+
   // eslint-disable-next-line no-console
   console.log(`[qualty] Summary: ${passed} passed, ${failed} failed.`);
   if (failOnFailure && failed > 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qualty",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Qualty CLI for localhost and CI test runs",
   "bin": {
     "qualty": "bin/qualty.js"