qualty 0.1.3 → 0.1.5

package/bin/qualty.js

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 
+import { appendFileSync } from "node:fs";
 import { spawn } from "node:child_process";
 import process from "node:process";
 
-/** When unset, CLI talks to Qualty cloud. Override for self-hosted or local dev, e.g. QUALTY_API_URL=http://localhost:8000 */
-const DEFAULT_QUALTY_API_URL = "https://qualty-api-production.up.railway.app";
+/** When unset, CLI uses Qualty development API. Override for production: QUALTY_API_URL=https://qualty-api-production.up.railway.app or local: http://localhost:8000 */
+const DEFAULT_QUALTY_API_URL = "https://qualty-api-development.up.railway.app";
 
 function resolveApiUrl(args) {
   return String(args.api || process.env.QUALTY_API_URL || DEFAULT_QUALTY_API_URL).replace(/\/$/, "");
@@ -174,18 +175,143 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function isGithubActions() {
+  return process.env.GITHUB_ACTIONS === "true";
+}
+
+/** Safe one-line text for a GitHub-flavored Markdown table cell. */
+function mdTableCell(text, maxLen = 200) {
+  return String(text ?? "")
+    .replace(/\r\n/g, "\n")
+    .replace(/\n/g, " ")
+    .replace(/\\/g, "\\\\")
+    .replace(/\|/g, "\\|")
+    .trim()
+    .slice(0, maxLen) || "—";
+}
+
+function githubWorkflowRunUrl() {
+  const server = process.env.GITHUB_SERVER_URL || "https://github.com";
+  const repo = process.env.GITHUB_REPOSITORY;
+  const runId = process.env.GITHUB_RUN_ID;
+  if (!repo || !runId) return "";
+  return `${server.replace(/\/$/, "")}/${repo}/actions/runs/${runId}`;
+}
+
+function executionOutcome(status) {
+  const st = status?.status;
+  const failed = Number(status?.failed_tests ?? 0);
+  if (st === "completed" && failed === 0) return { ok: true, label: "Pass" };
+  if (st === "completed" && failed > 0) return { ok: false, label: "Fail" };
+  if (st === "cancelled") return { ok: false, label: "Cancelled" };
+  if (st === "failed") return { ok: false, label: "Failed" };
+  return { ok: false, label: mdTableCell(st || "?", 24) };
+}
+
+/**
+ * Rich Markdown for the job "Summary" tab (tables, links). Step logs stay plain text.
+ * https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
+ */
+function appendGithubJobSummaryMarkdown(markdown) {
+  const path = process.env.GITHUB_STEP_SUMMARY;
+  if (!path || !markdown) return;
+  try {
+    appendFileSync(path, `${markdown}\n`, "utf8");
+  } catch {
+    // best effort — never fail the job for summary I/O
+  }
+}
+
+function writeQualtyGithubJobSummary({ executionJobIds, finalStatuses, passed, failed }) {
+  if (!isGithubActions()) return;
+
+  const total = executionJobIds.length;
+  const lines = [];
+  lines.push("## Qualty");
+  lines.push("");
+  lines.push(
+    `**${passed} passed**, **${failed} failed** · ${total} run${total === 1 ? "" : "s"}.`
+  );
+  lines.push("");
+  lines.push("| Test | Execution | Status | Failed | Result | Qualty |");
+  lines.push("| --- | --- | --- | ---: | --- | --- |");
+
+  for (const executionId of executionJobIds) {
+    const status = finalStatuses[executionId] || {};
+    const title = mdTableCell(status.episode_name || "—", 72);
+    const idCell = `\`${mdTableCell(executionId, 80)}\``;
+    const apiStatus = mdTableCell(status.status ?? "—", 20);
+    const failedN = status.failed_tests != null ? String(status.failed_tests) : "—";
+    const outcome = executionOutcome(status);
+    const resultCell = outcome.ok ? `✅ **${outcome.label}**` : `❌ **${outcome.label}**`;
+    const url = String(status.url || "").trim();
+    const linkCell = url ? `[Open run](${url})` : "—";
+    lines.push(
+      `| ${title} | ${idCell} | ${apiStatus} | ${failedN} | ${resultCell} | ${linkCell} |`
+    );
+  }
+
+  const comboRows = [];
+  for (const executionId of executionJobIds) {
+    const status = finalStatuses[executionId] || {};
+    const combos = Array.isArray(status.combinations) ? status.combinations : [];
+    const shortTitle = mdTableCell(
+      status.episode_name || (executionId ? String(executionId).slice(0, 8) : "—"),
+      40
+    );
+    for (const c of combos) {
+      const device = mdTableCell(c.device ?? "?", 32);
+      const comboSt = mdTableCell(
+        c.status ?? (c.success === true ? "passed" : c.success === false ? "failed" : "?"),
+        16
+      );
+      comboRows.push(`| \`${mdTableCell(executionId, 36)}\` | ${shortTitle} | ${device} | ${comboSt} |`);
+    }
+  }
+
+  if (comboRows.length > 0) {
+    lines.push("");
+    lines.push("<details>");
+    lines.push("<summary><strong>Per device / combination</strong></summary>");
+    lines.push("");
+    lines.push("| Execution | Test | Device | Result |");
+    lines.push("| --- | --- | --- | --- |");
+    lines.push(...comboRows);
+    lines.push("");
+    lines.push("</details>");
+  }
+
+  const runUrl = githubWorkflowRunUrl();
+  if (runUrl) {
+    lines.push("");
+    lines.push(`[This workflow run on GitHub](${runUrl})`);
+  }
+
+  appendGithubJobSummaryMarkdown(lines.join("\n"));
+}
+
 function truncateForCiLog(text, maxLen) {
   const s = String(text ?? "");
   if (s.length <= maxLen) return s;
-  return `${s.slice(0, maxLen)}\n[qualty] … (truncated, ${s.length - maxLen} more chars)`;
+  const tail = isGithubActions()
+    ? `… (truncated, ${s.length - maxLen} more chars)`
+    : `[qualty] … (truncated, ${s.length - maxLen} more chars)`;
+  return `${s.slice(0, maxLen)}\n${tail}`;
+}
+
+/** One log line: `[qualty]` prefix only outside GitHub Actions (inside ::group::, plain text reads better). */
+function viewOut(line) {
+  // eslint-disable-next-line no-console
+  console.log(isGithubActions() ? line : `[qualty] ${line}`);
 }
 
-function logPrefixedLines(prefix, text) {
+function logPrefixedLines(ghaIndent, nonGhaPrefix, text) {
   const body = String(text ?? "").trimEnd();
   if (!body) return;
-  for (const line of body.split("\n")) {
+  const prefix = isGithubActions() ? ghaIndent : nonGhaPrefix;
+  for (const ln of body.split("\n")) {
     // eslint-disable-next-line no-console
-    console.log(`${prefix}${line}`);
+    console.log(`${prefix}${ln}`);
   }
 }
 
@@ -194,40 +320,51 @@ function logPrefixedLines(prefix, text) {
  * Data comes from GET status payload fields on each combination (steps_json, explanation, agent_output).
  */
 function printQualtyViewLogsReport(executionId, status) {
-  // eslint-disable-next-line no-console
-  console.log(`\n[qualty] ━━━ View logs: ${executionId} (${status.episode_name || "run"}) ━━━`);
-  if (status.url) {
+  const gha = isGithubActions();
+  const title = `${status.episode_name || "run"} (${executionId})`;
+  if (gha) {
+    // Collapsible section in GitHub Actions (no rich tables — stdout is plain text).
+    // eslint-disable-next-line no-console
+    console.log(`::group::Qualty · ${title}`);
+  } else {
     // eslint-disable-next-line no-console
-    console.log(`[qualty] URL: ${status.url}`);
+    console.log(`\n[qualty] ━━━ View logs: ${executionId} (${status.episode_name || "run"}) ━━━`);
   }
+  if (status.url) viewOut(`URL: ${status.url}`);
   if (status.error) {
-    // eslint-disable-next-line no-console
-    console.log(`[qualty] Run error: ${truncateForCiLog(status.error, 4000)}`);
+    viewOut("Run error:");
+    logPrefixedLines("  ", "[qualty]   ", truncateForCiLog(status.error, 4000));
   }
   if (status.expected_behavior) {
-    // eslint-disable-next-line no-console
-    console.log(`[qualty] Expected behavior:\n[qualty] ${truncateForCiLog(status.expected_behavior, 6000).split("\n").join("\n[qualty] ")}`);
+    viewOut("Expected behavior:");
+    logPrefixedLines("  ", "[qualty]   ", truncateForCiLog(status.expected_behavior, 6000));
   }
   const combos = Array.isArray(status.combinations) ? status.combinations : [];
   if (combos.length === 0) {
-    // eslint-disable-next-line no-console
-    console.log(
-      "[qualty] (No per-device breakdown in API response yet — open this run in the Qualty dashboard for full logs.)"
-    );
-    // eslint-disable-next-line no-console
-    console.log(`[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+    viewOut("(No per-device breakdown in API response yet — open this run in the Qualty dashboard for full logs.)");
+    if (gha) {
+      // eslint-disable-next-line no-console
+      console.log("::endgroup::");
+    } else {
+      // eslint-disable-next-line no-console
+      console.log(`[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+    }
     return;
   }
   for (let i = 0; i < combos.length; i += 1) {
     const c = combos[i];
     const device = c.device ?? "?";
     const comboStatus = c.status ?? (c.success === true ? "passed" : c.success === false ? "failed" : "?");
-    // eslint-disable-next-line no-console
-    console.log(`\n[qualty] --- Combination ${i + 1}/${combos.length} (${device} · ${comboStatus}) ---`);
+    if (gha) {
+      // eslint-disable-next-line no-console
+      console.log(`::group::${device} · ${comboStatus} (${i + 1}/${combos.length})`);
+    } else {
+      // eslint-disable-next-line no-console
+      console.log(`\n[qualty] --- Combination ${i + 1}/${combos.length} (${device} · ${comboStatus}) ---`);
+    }
     const steps = Array.isArray(c.steps_json) ? c.steps_json : [];
     if (steps.length > 0) {
-      // eslint-disable-next-line no-console
-      console.log(`[qualty] Steps (${steps.length}):`);
+      viewOut(`Steps (${steps.length}):`);
       for (let j = 0; j < steps.length; j += 1) {
         const s = steps[j];
         const label =
@@ -235,34 +372,38 @@ function printQualtyViewLogsReport(executionId, status) {
           (s.description && String(s.description).trim().slice(0, 100)) ||
           `Step ${j + 1}`;
         const st = s.status != null ? s.status : "?";
-        // eslint-disable-next-line no-console
-        console.log(`[qualty]   ${j + 1}. [${st}] ${label}`);
+        viewOut(`  ${j + 1}. [${st}] ${label}`);
         if (s.description && String(s.description).trim() && String(s.description) !== String(s.name)) {
-          logPrefixedLines("[qualty]      ", truncateForCiLog(s.description, 4000));
+          logPrefixedLines("    ", "[qualty]      ", truncateForCiLog(s.description, 4000));
         }
         if (s.action) {
-          // eslint-disable-next-line no-console
-          console.log(`[qualty]      action: ${truncateForCiLog(s.action, 2000)}`);
+          logPrefixedLines("    ", "[qualty]      ", `action: ${truncateForCiLog(s.action, 2000)}`);
         }
       }
     } else if (c.total_steps) {
-      // eslint-disable-next-line no-console
-      console.log(`[qualty] (Step list not available yet; ${c.total_steps} step(s) reported.)`);
+      viewOut(`(Step list not available yet; ${c.total_steps} step(s) reported.)`);
     }
     if (c.explanation) {
-      // eslint-disable-next-line no-console
-      console.log(`[qualty] Explanation:\n[qualty] ${truncateForCiLog(c.explanation, 12000).split("\n").join("\n[qualty] ")}`);
+      viewOut("Explanation:");
+      logPrefixedLines("  ", "[qualty]   ", truncateForCiLog(c.explanation, 12000));
     }
     const evaluator = c.agent_output ?? c.gpt_output;
     if (evaluator) {
+      viewOut("Final evaluator output:");
+      logPrefixedLines("  ", "[qualty]   ", truncateForCiLog(evaluator, 16000));
+    }
+    if (gha) {
       // eslint-disable-next-line no-console
-      console.log(
-        `[qualty] Final evaluator output:\n[qualty] ${truncateForCiLog(evaluator, 16000).split("\n").join("\n[qualty] ")}`
-      );
+      console.log("::endgroup::");
     }
   }
-  // eslint-disable-next-line no-console
-  console.log(`\n[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+  if (gha) {
+    // eslint-disable-next-line no-console
+    console.log("::endgroup::");
+  } else {
+    // eslint-disable-next-line no-console
+    console.log(`\n[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+  }
 }
 
 async function runCi(args) {
@@ -377,9 +518,18 @@ async function runCi(args) {
     );
   }
 
+  writeQualtyGithubJobSummary({ executionJobIds, finalStatuses, passed, failed });
+
   if (!noViewLogs) {
-    // eslint-disable-next-line no-console
-    console.log(`[qualty] Detailed run output (same fields as dashboard "View logs"):`);
+    if (isGithubActions()) {
+      // eslint-disable-next-line no-console
+      console.log(
+        "[qualty] Open the job Summary tab for a results table; expand the Qualty groups below for full step logs."
+      );
+    } else {
+      // eslint-disable-next-line no-console
+      console.log(`[qualty] Detailed run output (same fields as dashboard "View logs"):`);
+    }
     for (const executionId of executionJobIds) {
       printQualtyViewLogsReport(executionId, finalStatuses[executionId] || {});
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qualty",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Qualty CLI for localhost and CI test runs",
   "bin": {
     "qualty": "bin/qualty.js"