qualty 0.1.4 → 0.1.6

package/bin/qualty.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+import { appendFileSync } from "node:fs";
 import { spawn } from "node:child_process";
 import process from "node:process";
 
@@ -174,18 +175,232 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function isGithubActions() {
+  return process.env.GITHUB_ACTIONS === "true";
+}
+
+/** Safe one-line text for a GitHub-flavored Markdown table cell. */
+function mdTableCell(text, maxLen = 200) {
+  return String(text ?? "")
+    .replace(/\r\n/g, "\n")
+    .replace(/\n/g, " ")
+    .replace(/\\/g, "\\\\")
+    .replace(/\|/g, "\\|")
+    .trim()
+    .slice(0, maxLen) || "—";
+}
+
+function githubWorkflowRunUrl() {
+  const server = process.env.GITHUB_SERVER_URL || "https://github.com";
+  const repo = process.env.GITHUB_REPOSITORY;
+  const runId = process.env.GITHUB_RUN_ID;
+  if (!repo || !runId) return "";
+  return `${server.replace(/\/$/, "")}/${repo}/actions/runs/${runId}`;
+}
+
+function executionOutcome(status) {
+  const st = status?.status;
+  const failed = Number(status?.failed_tests ?? 0);
+  if (st === "completed" && failed === 0) return { ok: true, label: "Pass" };
+  if (st === "completed" && failed > 0) return { ok: false, label: "Fail" };
+  if (st === "cancelled") return { ok: false, label: "Cancelled" };
+  if (st === "failed") return { ok: false, label: "Failed" };
+  return { ok: false, label: mdTableCell(st || "?", 24) };
+}
+
+/**
+ * Rich Markdown for the job "Summary" tab (tables, links). Step logs stay plain text.
+ * https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
+ */
+function appendGithubJobSummaryMarkdown(markdown) {
+  const path = process.env.GITHUB_STEP_SUMMARY;
+  if (!path || !markdown) return;
+  try {
+    appendFileSync(path, `${markdown}\n`, "utf8");
+  } catch {
+    // best effort — never fail the job for summary I/O
+  }
+}
+
+/**
+ * Markdown for per-step labels, thoughts (description), actions, explanation, evaluator.
+ * Kept short so the job Summary stays under GitHub size limits; full text stays in step logs.
+ */
+function buildStepDetailsSummaryMarkdown(finalStatuses, executionJobIds) {
+  const MAX_STEPS = 50;
+  const parts = [];
+  let any = false;
+
+  for (const executionId of executionJobIds) {
+    const status = finalStatuses[executionId] || {};
+    const combos = Array.isArray(status.combinations) ? status.combinations : [];
+    const episode = status.episode_name || "Run";
+
+    for (const c of combos) {
+      const steps = Array.isArray(c.steps_json) ? c.steps_json : [];
+      const device = String(c.device ?? "?");
+      const evaluator = c.agent_output ?? c.gpt_output;
+      if (
+        steps.length === 0 &&
+        !String(c.explanation || "").trim() &&
+        !String(evaluator || "").trim()
+      ) {
+        continue;
+      }
+      any = true;
+      parts.push(
+        `#### ${mdTableCell(episode, 72)} · ${mdTableCell(device, 36)} · \`${mdTableCell(executionId, 36)}\``
+      );
+      parts.push("");
+
+      const shown = steps.slice(0, MAX_STEPS);
+      for (let j = 0; j < shown.length; j += 1) {
+        const s = shown[j];
+        const label =
+          (s.name && String(s.name).trim()) ||
+          (s.description && String(s.description).trim().slice(0, 100)) ||
+          `Step ${j + 1}`;
+        const st = s.status != null ? s.status : "?";
+        parts.push(`${j + 1}. **${mdTableCell(st, 16)}** ${mdTableCell(label, 140)}`);
+        const desc = s.description && String(s.description).trim();
+        if (desc && desc !== String(s.name)) {
+          parts.push(`   - *Thoughts:* ${mdTableCell(desc, 900)}`);
+        }
+        if (s.action) {
+          parts.push(`   - *Action:* \`${mdTableCell(String(s.action), 500)}\``);
+        }
+        parts.push("");
+      }
+      if (steps.length > MAX_STEPS) {
+        parts.push(
+          `*…and ${steps.length - MAX_STEPS} more steps — expand **Qualty** groups in the job log or open the run in Qualty for the full list.*`
+        );
+        parts.push("");
+      }
+
+      if (c.explanation && String(c.explanation).trim()) {
+        parts.push("**Explanation**");
+        parts.push("");
+        parts.push(`> ${mdTableCell(c.explanation, 4500)}`);
+        parts.push("");
+      }
+      if (evaluator && String(evaluator).trim()) {
+        parts.push("**Final evaluator**");
+        parts.push("");
+        parts.push(`> ${mdTableCell(evaluator, 8000)}`);
+        parts.push("");
+      }
+    }
+  }
+
+  if (!any) return "";
+  parts.push(
+    "*Truncated for the Summary tab. Uncapped step lines and evaluator text are in the job log (expand the Qualty groups) or in the Qualty dashboard.*"
+  );
+  return parts.join("\n");
+}
+
+function writeQualtyGithubJobSummary({ executionJobIds, finalStatuses, passed, failed }) {
+  if (!isGithubActions()) return;
+
+  const total = executionJobIds.length;
+  const lines = [];
+  lines.push("## Qualty");
+  lines.push("");
+  lines.push(
+    `**${passed} passed**, **${failed} failed** · ${total} run${total === 1 ? "" : "s"}.`
+  );
+  lines.push("");
+  lines.push("| Test | Execution | Status | Failed | Result | Qualty |");
+  lines.push("| --- | --- | --- | ---: | --- | --- |");
+
+  for (const executionId of executionJobIds) {
+    const status = finalStatuses[executionId] || {};
+    const title = mdTableCell(status.episode_name || "—", 72);
+    const idCell = `\`${mdTableCell(executionId, 80)}\``;
+    const apiStatus = mdTableCell(status.status ?? "—", 20);
+    const failedN = status.failed_tests != null ? String(status.failed_tests) : "—";
+    const outcome = executionOutcome(status);
+    const resultCell = outcome.ok ? `✅ **${outcome.label}**` : `❌ **${outcome.label}**`;
+    const url = String(status.url || "").trim();
+    const linkCell = url ? `[Open run](${url})` : "—";
+    lines.push(
+      `| ${title} | ${idCell} | ${apiStatus} | ${failedN} | ${resultCell} | ${linkCell} |`
+    );
+  }
+
+  const comboRows = [];
+  for (const executionId of executionJobIds) {
+    const status = finalStatuses[executionId] || {};
+    const combos = Array.isArray(status.combinations) ? status.combinations : [];
+    const shortTitle = mdTableCell(
+      status.episode_name || (executionId ? String(executionId).slice(0, 8) : "—"),
+      40
+    );
+    for (const c of combos) {
+      const device = mdTableCell(c.device ?? "?", 32);
+      const comboSt = mdTableCell(
+        c.status ?? (c.success === true ? "passed" : c.success === false ? "failed" : "?"),
+        16
+      );
+      comboRows.push(`| \`${mdTableCell(executionId, 36)}\` | ${shortTitle} | ${device} | ${comboSt} |`);
+    }
+  }
+
+  if (comboRows.length > 0) {
+    lines.push("");
+    lines.push("<details>");
+    lines.push("<summary><strong>Per device / combination</strong></summary>");
+    lines.push("");
+    lines.push("| Execution | Test | Device | Result |");
+    lines.push("| --- | --- | --- | --- |");
+    lines.push(...comboRows);
+    lines.push("");
+    lines.push("</details>");
+  }
+
+  const stepDetailsMd = buildStepDetailsSummaryMarkdown(finalStatuses, executionJobIds);
+  if (stepDetailsMd) {
+    lines.push("");
+    lines.push("<details>");
+    lines.push("<summary><strong>Steps &amp; agent notes (truncated)</strong></summary>");
+    lines.push("");
+    lines.push(stepDetailsMd);
+    lines.push("");
+    lines.push("</details>");
+  }
+
+  const runUrl = githubWorkflowRunUrl();
+  if (runUrl) {
+    lines.push("");
+    lines.push(`[This workflow run on GitHub](${runUrl})`);
+  }
+
+  appendGithubJobSummaryMarkdown(lines.join("\n"));
+}
+
 function truncateForCiLog(text, maxLen) {
   const s = String(text ?? "");
   if (s.length <= maxLen) return s;
-  return `${s.slice(0, maxLen)}\n[qualty] … (truncated, ${s.length - maxLen} more chars)`;
+  const tail = isGithubActions()
+    ? `… (truncated, ${s.length - maxLen} more chars)`
+    : `[qualty] … (truncated, ${s.length - maxLen} more chars)`;
+  return `${s.slice(0, maxLen)}\n${tail}`;
 }
 
-function logPrefixedLines(prefix, text) {
+/** One log line: `[qualty]` prefix only outside GitHub Actions (inside ::group::, plain text reads better). */
+function viewOut(line) {
+  // eslint-disable-next-line no-console
+  console.log(isGithubActions() ? line : `[qualty] ${line}`);
+}
+
+function logPrefixedLines(ghaIndent, nonGhaPrefix, text) {
   const body = String(text ?? "").trimEnd();
   if (!body) return;
-  for (const line of body.split("\n")) {
+  const prefix = isGithubActions() ? ghaIndent : nonGhaPrefix;
+  for (const ln of body.split("\n")) {
     // eslint-disable-next-line no-console
-    console.log(`${prefix}${line}`);
+    console.log(`${prefix}${ln}`);
   }
 }
 
@@ -194,40 +409,51 @@ function logPrefixedLines(prefix, text) {
  * Data comes from GET status payload fields on each combination (steps_json, explanation, agent_output).
  */
 function printQualtyViewLogsReport(executionId, status) {
-  // eslint-disable-next-line no-console
-  console.log(`\n[qualty] ━━━ View logs: ${executionId} (${status.episode_name || "run"}) ━━━`);
-  if (status.url) {
+  const gha = isGithubActions();
+  const title = `${status.episode_name || "run"} (${executionId})`;
+  if (gha) {
+    // Collapsible section in GitHub Actions (no rich tables — stdout is plain text).
+    // eslint-disable-next-line no-console
+    console.log(`::group::Qualty · ${title}`);
+  } else {
     // eslint-disable-next-line no-console
-    console.log(`[qualty] URL: ${status.url}`);
+    console.log(`\n[qualty] ━━━ View logs: ${executionId} (${status.episode_name || "run"}) ━━━`);
   }
+  if (status.url) viewOut(`URL: ${status.url}`);
   if (status.error) {
-    // eslint-disable-next-line no-console
-    console.log(`[qualty] Run error: ${truncateForCiLog(status.error, 4000)}`);
+    viewOut("Run error:");
+    logPrefixedLines("  ", "[qualty]   ", truncateForCiLog(status.error, 4000));
   }
   if (status.expected_behavior) {
-    // eslint-disable-next-line no-console
-    console.log(`[qualty] Expected behavior:\n[qualty] ${truncateForCiLog(status.expected_behavior, 6000).split("\n").join("\n[qualty] ")}`);
+    viewOut("Expected behavior:");
+    logPrefixedLines("  ", "[qualty]   ", truncateForCiLog(status.expected_behavior, 6000));
   }
   const combos = Array.isArray(status.combinations) ? status.combinations : [];
   if (combos.length === 0) {
-    // eslint-disable-next-line no-console
-    console.log(
-      "[qualty] (No per-device breakdown in API response yet — open this run in the Qualty dashboard for full logs.)"
-    );
-    // eslint-disable-next-line no-console
-    console.log(`[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+    viewOut("(No per-device breakdown in API response yet — open this run in the Qualty dashboard for full logs.)");
+    if (gha) {
+      // eslint-disable-next-line no-console
+      console.log("::endgroup::");
+    } else {
+      // eslint-disable-next-line no-console
+      console.log(`[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+    }
     return;
   }
   for (let i = 0; i < combos.length; i += 1) {
     const c = combos[i];
     const device = c.device ?? "?";
     const comboStatus = c.status ?? (c.success === true ? "passed" : c.success === false ? "failed" : "?");
-    // eslint-disable-next-line no-console
-    console.log(`\n[qualty] --- Combination ${i + 1}/${combos.length} (${device} · ${comboStatus}) ---`);
+    if (gha) {
+      // eslint-disable-next-line no-console
+      console.log(`::group::${device} · ${comboStatus} (${i + 1}/${combos.length})`);
+    } else {
+      // eslint-disable-next-line no-console
+      console.log(`\n[qualty] --- Combination ${i + 1}/${combos.length} (${device} · ${comboStatus}) ---`);
+    }
     const steps = Array.isArray(c.steps_json) ? c.steps_json : [];
     if (steps.length > 0) {
-      // eslint-disable-next-line no-console
-      console.log(`[qualty] Steps (${steps.length}):`);
+      viewOut(`Steps (${steps.length}):`);
       for (let j = 0; j < steps.length; j += 1) {
         const s = steps[j];
         const label =
@@ -235,34 +461,38 @@ function printQualtyViewLogsReport(executionId, status) {
           (s.description && String(s.description).trim().slice(0, 100)) ||
           `Step ${j + 1}`;
         const st = s.status != null ? s.status : "?";
-        // eslint-disable-next-line no-console
-        console.log(`[qualty]   ${j + 1}. [${st}] ${label}`);
+        viewOut(`  ${j + 1}. [${st}] ${label}`);
         if (s.description && String(s.description).trim() && String(s.description) !== String(s.name)) {
-          logPrefixedLines("[qualty]      ", truncateForCiLog(s.description, 4000));
+          logPrefixedLines("    ", "[qualty]      ", truncateForCiLog(s.description, 4000));
         }
         if (s.action) {
-          // eslint-disable-next-line no-console
-          console.log(`[qualty]      action: ${truncateForCiLog(s.action, 2000)}`);
+          logPrefixedLines("    ", "[qualty]      ", `action: ${truncateForCiLog(s.action, 2000)}`);
         }
       }
     } else if (c.total_steps) {
-      // eslint-disable-next-line no-console
-      console.log(`[qualty] (Step list not available yet; ${c.total_steps} step(s) reported.)`);
+      viewOut(`(Step list not available yet; ${c.total_steps} step(s) reported.)`);
     }
     if (c.explanation) {
-      // eslint-disable-next-line no-console
-      console.log(`[qualty] Explanation:\n[qualty] ${truncateForCiLog(c.explanation, 12000).split("\n").join("\n[qualty] ")}`);
+      viewOut("Explanation:");
+      logPrefixedLines("  ", "[qualty]   ", truncateForCiLog(c.explanation, 12000));
     }
     const evaluator = c.agent_output ?? c.gpt_output;
     if (evaluator) {
+      viewOut("Final evaluator output:");
+      logPrefixedLines("  ", "[qualty]   ", truncateForCiLog(evaluator, 16000));
+    }
+    if (gha) {
       // eslint-disable-next-line no-console
-      console.log(
-        `[qualty] Final evaluator output:\n[qualty] ${truncateForCiLog(evaluator, 16000).split("\n").join("\n[qualty] ")}`
-      );
+      console.log("::endgroup::");
     }
   }
-  // eslint-disable-next-line no-console
-  console.log(`\n[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+  if (gha) {
+    // eslint-disable-next-line no-console
+    console.log("::endgroup::");
+  } else {
+    // eslint-disable-next-line no-console
+    console.log(`\n[qualty] ━━━ End view logs: ${executionId} ━━━\n`);
+  }
 }
 
 async function runCi(args) {
@@ -377,9 +607,18 @@ async function runCi(args) {
     );
   }
 
+  writeQualtyGithubJobSummary({ executionJobIds, finalStatuses, passed, failed });
+
   if (!noViewLogs) {
-    // eslint-disable-next-line no-console
-    console.log(`[qualty] Detailed run output (same fields as dashboard "View logs"):`);
+    if (isGithubActions()) {
+      // eslint-disable-next-line no-console
+      console.log(
+        "[qualty] Open the job Summary tab for a results table; expand the Qualty groups below for full step logs."
+      );
+    } else {
+      // eslint-disable-next-line no-console
+      console.log(`[qualty] Detailed run output (same fields as dashboard "View logs"):`);
+    }
     for (const executionId of executionJobIds) {
       printQualtyViewLogsReport(executionId, finalStatuses[executionId] || {});
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qualty",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Qualty CLI for localhost and CI test runs",
   "bin": {
     "qualty": "bin/qualty.js"