qaify-playwright-reporter 0.5.0 → 0.6.0

package/dist/ai.d.ts

@@ -1,3 +1,4 @@
+import { type ClaudeUsage } from './usage.js';
 export declare const CATEGORIES: readonly ["app_bug", "outdated_test", "flaky", "infra", "unknown"];
 /** --json-schema untuk claude -p: hasilnya ada di field structured_output. */
 export declare const RESULT_SCHEMA: {
@@ -40,11 +41,7 @@ export interface AiResult {
     summary: string;
     confidence: number;
 }
-export interface AiUsage {
-    model: string | null;
-    input_tokens: number;
-    output_tokens: number;
-}
+export type AiUsage = ClaudeUsage;
 /**
  * Baca output `claude -p --output-format json --json-schema …`.
  * Output kosong / rusak / tanpa structured_output (gagal login, kuota habis, ditolak) → results kosong.

package/dist/ai.js

@@ -1,5 +1,6 @@
 // Mode actions: kegagalan dianalisis Claude Code (claude -p) di job analyze dengan langganan Claude Pro/Max.
 // Backend tidak memanggil AI; reporter hanya mengambil antrean dan mengirim balik hasilnya.
+import { parseClaudeUsage, readClaudeUsage } from './usage.js';
 export const CATEGORIES = ['app_bug', 'outdated_test', 'flaky', 'infra', 'unknown'];
 /** --json-schema untuk claude -p: hasilnya ada di field structured_output. */
 export const RESULT_SCHEMA = {
@@ -43,21 +44,9 @@ export function parseClaudeOutput(raw) {
         parsed = JSON.parse(raw);
     }
     catch {
-        return { results: [], usage: null };
+        return { results: [], usage: readClaudeUsage(raw) };
     }
     const items = Array.isArray(parsed?.structured_output?.results) ? parsed.structured_output.results : [];
     const results = items.filter((r) => Number.isInteger(r?.failure_id) && typeof r?.category === 'string' && typeof r?.summary === 'string' && typeof r?.confidence === 'number');
-    // modelUsage: { "<model-id>": {...} }; ambil model dengan token keluaran terbanyak
-    const models = Object.entries((parsed?.modelUsage ?? {}));
-    const model = models.sort(([, a], [, b]) => (b.outputTokens ?? 0) - (a.outputTokens ?? 0))[0]?.[0] ?? null;
-    // token input Claude Code sebagian besar tercatat sebagai cache (system prompt); semuanya dijumlah
-    const u = parsed?.usage;
-    const usage = u
-        ? {
-            model,
-            input_tokens: Number(u.input_tokens ?? 0) + Number(u.cache_creation_input_tokens ?? 0) + Number(u.cache_read_input_tokens ?? 0),
-            output_tokens: Number(u.output_tokens ?? 0),
-        }
-        : null;
-    return { results, usage };
+    return { results, usage: parseClaudeUsage(parsed) };
 }

package/dist/audit.d.ts

@@ -1,4 +1,5 @@
 import { type IngestClient } from './client.js';
+import { type ClaudeUsage } from './usage.js';
 export declare const AREAS: readonly ["ui", "business_logic", "api_contract", "validation", "error_handling", "security", "performance"];
 export declare const SEVERITIES: readonly ["critical", "high", "medium", "low"];
 export declare const ROLES: readonly ["fe", "be"];
@@ -262,18 +263,11 @@ export interface AuditPage {
     file?: string;
     line?: number;
 }
-export interface AuditUsage {
-    model: string | null;
-    input_tokens: number;
-    output_tokens: number;
-    num_turns: number | null;
-    duration_ms: number | null;
-}
 export interface ParsedAudit {
     summary: Record<string, unknown>;
     findings: AuditFinding[];
     pages: AuditPage[];
-    usage: AuditUsage | null;
+    usage: ClaudeUsage | null;
 }
 /**
  * Baca output `claude -p --output-format json --json-schema AUDIT_SCHEMA`.
@@ -285,6 +279,8 @@ export declare const defaultExec: Exec;
 export interface AuditContextResult {
     ok: boolean;
     commits: Record<string, string>;
+    /** alasan gagal; juga ditulis ke context-error.txt agar audit-submit bisa melaporkannya */
+    error?: string;
 }
 /**
  * GET /api/ingest/audits/{id}/context → clone apps/<role> → cabut token.
@@ -297,6 +293,8 @@ export declare function auditContext(client: IngestClient, auditId: string, dir:
     fetch?: typeof fetch;
     log?: (msg: string) => void;
 }): Promise<AuditContextResult>;
+/** Alasan gagal dari audit-context (context-error.txt di samping context.json), atau null. */
+export declare function readContextError(contextFile: string): Promise<string | null>;
 /** Token GitHub untuk git lewat header dari environment: tidak pernah di argumen, URL remote, atau .git/config. */
 export declare function gitAuthEnv(token: string): NodeJS.ProcessEnv;
 /** DELETE /installation/token: token tidak bisa dipakai lagi oleh langkah berikutnya. */
@@ -305,7 +303,9 @@ export declare function revokeTokens(tokens: string[], doFetch?: typeof fetch):
  * Kirim hasil audit. Tanpa hasil terstruktur → audit ditandai gagal dengan penyebab yang mungkin.
  * Screenshot (temuan + galeri halaman) hanya diambil dari dalam folder evidence (path di luar folder itu diabaikan).
  */
-export declare function auditSubmit(client: IngestClient, auditId: string, claudeOutput: string, evidenceDir: string, commits: Record<string, string>, log?: (msg: string) => void): Promise<any>;
+export declare function auditSubmit(client: IngestClient, auditId: string, claudeOutput: string, evidenceDir: string, commits: Record<string, string>, log?: (msg: string) => void, 
+/** dari audit-context: auditor tidak dijalankan sama sekali */
+contextError?: string | null): Promise<any>;
 /** Penyebab screenshot tidak terunggah, agar laporan "Tanpa screenshot" tidak terjadi diam-diam. */
 export declare function uploadProblems(res: any, unsigned: number, rejected: number, putError: string | null): string[];
 /** Commit per role dari context.json (dibuat audit-context). */

package/dist/audit.js

@@ -4,9 +4,10 @@
 // audit-submit: unggah screenshot bukti (presigned) lalu kirim hasil ke Qaify.
 import { execFile } from 'node:child_process';
 import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
-import { extname, join, resolve, sep } from 'node:path';
+import { dirname, extname, join, resolve, sep } from 'node:path';
 import { promisify } from 'node:util';
 import { warn } from './client.js';
+import { parseClaudeUsage, readClaudeUsage } from './usage.js';
 export const AREAS = ['ui', 'business_logic', 'api_contract', 'validation', 'error_handling', 'security', 'performance'];
 export const SEVERITIES = ['critical', 'high', 'medium', 'low'];
 export const ROLES = ['fe', 'be'];
@@ -168,23 +169,12 @@ export function parseAuditOutput(raw) {
         return null;
     const findings = out.findings.filter((f) => ROLES.includes(f?.repo) && typeof f?.title === 'string' && typeof f?.description === 'string');
     const pages = (Array.isArray(out.pages) ? out.pages : []).filter((p) => typeof p?.path === 'string' && VIEWPORTS.includes(p?.viewport));
-    const models = Object.entries((parsed?.modelUsage ?? {}));
-    const model = models.sort(([, a], [, b]) => (b.outputTokens ?? 0) - (a.outputTokens ?? 0))[0]?.[0] ?? null;
-    const u = parsed?.usage;
-    const usage = u
-        ? {
-            model,
-            input_tokens: Number(u.input_tokens ?? 0) + Number(u.cache_creation_input_tokens ?? 0) + Number(u.cache_read_input_tokens ?? 0),
-            output_tokens: Number(u.output_tokens ?? 0),
-            num_turns: Number.isInteger(parsed?.num_turns) ? parsed.num_turns : null,
-            duration_ms: Number.isInteger(parsed?.duration_ms) ? parsed.duration_ms : null,
-        }
-        : null;
-    return { summary: out.summary && typeof out.summary === 'object' ? out.summary : {}, findings, pages, usage };
+    return { summary: out.summary && typeof out.summary === 'object' ? out.summary : {}, findings, pages, usage: parseClaudeUsage(parsed) };
 }
 const execFileAsync = promisify(execFile);
 export const defaultExec = async (file, args, env) => (await execFileAsync(file, args, { env: { ...process.env, ...env }, maxBuffer: 10 * 1024 * 1024 })).stdout;
 const branchOf = (repo) => repo.branch || repo.default_branch;
+const CONTEXT_ERROR_FILE = 'context-error.txt';
 /**
  * GET /api/ingest/audits/{id}/context → clone apps/<role> → cabut token.
  * Token hanya lewat header git dari environment (tidak pernah di argumen, URL remote, atau disk)
@@ -196,8 +186,7 @@ export async function auditContext(client, auditId, dir, opts = {}) {
     const query = opts.githubRunId ? `?github_run_id=${opts.githubRunId}` : '';
     const ctx = await client.get(`/api/ingest/audits/${encodeURIComponent(auditId)}/context${query}`);
     if (!ctx || !Array.isArray(ctx.repos)) {
-        log('qa-factory: konteks audit tidak bisa diambil (audit tidak dikenal / sudah selesai / token runner salah)');
-        return { ok: false, commits: {} };
+        return contextFailed(dir, 'Konteks audit tidak bisa diambil (audit tidak dikenal / sudah selesai / token runner salah).', {});
     }
     const repos = ctx.repos;
     // GitHub Actions menyamarkan token di log
@@ -219,9 +208,10 @@ export async function auditContext(client, auditId, dir, opts = {}) {
         }
     }
     catch (e) {
-        log(`qa-factory: clone gagal: ${e.message.split('\n')[0]}`);
         await revoke(repos, opts.fetch);
-        return { ok: false, commits };
+        // baris terakhir stderr git berisi penyebabnya (mis. "Remote branch x not found"); token tidak pernah ada di pesan
+        const detail = (e.stderr || e.message).trim().split('\n').pop();
+        return contextFailed(dir, `Clone repo gagal: ${detail}`, commits);
     }
     await revoke(repos, opts.fetch);
     await writeFile(join(dir, 'context.json'), JSON.stringify({
@@ -231,6 +221,17 @@ export async function auditContext(client, auditId, dir, opts = {}) {
     }, null, 2));
     return { ok: true, commits };
 }
+/** Peringatan (annotation di ringkasan run) + context-error.txt untuk audit-submit. */
+async function contextFailed(dir, error, commits) {
+    warn(error);
+    await mkdir(dir, { recursive: true });
+    await writeFile(join(dir, CONTEXT_ERROR_FILE), error);
+    return { ok: false, commits, error };
+}
+/** Alasan gagal dari audit-context (context-error.txt di samping context.json), atau null. */
+export async function readContextError(contextFile) {
+    return (await readFile(join(dirname(contextFile), CONTEXT_ERROR_FILE), 'utf8').catch(() => '')).trim() || null;
+}
 /** Token GitHub untuk git lewat header dari environment: tidak pernah di argumen, URL remote, atau .git/config. */
 export function gitAuthEnv(token) {
     return {
@@ -257,14 +258,21 @@ export async function revokeTokens(tokens, doFetch = fetch) {
  * Kirim hasil audit. Tanpa hasil terstruktur → audit ditandai gagal dengan penyebab yang mungkin.
  * Screenshot (temuan + galeri halaman) hanya diambil dari dalam folder evidence (path di luar folder itu diabaikan).
  */
-export async function auditSubmit(client, auditId, claudeOutput, evidenceDir, commits, log = console.log) {
+export async function auditSubmit(client, auditId, claudeOutput, evidenceDir, commits, log = console.log, 
+/** dari audit-context: auditor tidak dijalankan sama sekali */
+contextError = null) {
     const base = `/api/ingest/audits/${encodeURIComponent(auditId)}`;
     const parsed = parseAuditOutput(claudeOutput);
     if (!parsed) {
         log('qa-factory: auditor tidak menghasilkan hasil audit terstruktur');
+        // kuota habis / max turns tetap dicatat di pemakaian Claude
+        const usage = contextError ? null : readClaudeUsage(claudeOutput);
         return client.post(`${base}/results`, {
             status: 'failed',
-            error: 'Auditor tidak menghasilkan hasil audit. Cek log step "Audit tanpa AI (Playwright)" atau "Audit dengan Claude Code" di repo runner; untuk mode Claude, cek juga secret CLAUDE_CODE_OAUTH_TOKEN dan kuota langganan.',
+            ...(usage ? { usage } : {}),
+            error: contextError
+                ? `${contextError} Cek log step "Ambil konteks & clone repo FE + BE" di repo runner (branch ada? GitHub App Qaify terpasang di repo FE + BE?).`
+                : 'Auditor tidak menghasilkan hasil audit. Cek log step "Audit tanpa AI (Playwright)" atau "Audit dengan Claude Code" di repo runner; untuk mode Claude, cek juga secret CLAUDE_CODE_OAUTH_TOKEN dan kuota langganan.',
             commits,
         });
     }

package/dist/cli.js

@@ -22,7 +22,7 @@ import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
 import { INSTRUCTIONS, RESULT_SCHEMA } from './ai.js';
 import { aiQueue, aiSubmit, analyze, CONCLUSIONS, parseArgs } from './analyze.js';
-import { AUDIT_INSTRUCTIONS, AUDIT_PROMPT, AUDIT_SCHEMA, auditContext, auditSubmit, readCommits } from './audit.js';
+import { AUDIT_INSTRUCTIONS, AUDIT_PROMPT, AUDIT_SCHEMA, auditContext, auditSubmit, readCommits, readContextError } from './audit.js';
 import { auditStatic, defaultLaunch, defaultShell, toAuditOutput } from './audit-static.js';
 import { createClient, describeProblem, MISSING_CONFIG, runFromEnv, warn } from './client.js';
 import { FIX_INSTRUCTIONS, FIX_SCHEMA, fixContext, fixDiff, fixSubmit, writeOutputs } from './fix.js';
@@ -127,6 +127,8 @@ switch (args.command) {
             const problem = describeProblem(client);
             if (problem)
                 warn(problem);
+            // step merah: tanpa ini auditor & kirim hasil dilewati diam-diam dan audit macet "running"
+            process.exitCode = 1;
         }
         if (process.env.GITHUB_OUTPUT)
             await appendFile(process.env.GITHUB_OUTPUT, `ok=${result.ok}\n`);
@@ -170,7 +172,8 @@ switch (args.command) {
         // file tidak ada (auditor dilewati / gagal) → audit ditandai gagal
         const output = await readFile(args.results, 'utf8').catch(() => '');
         const commits = args.context ? await readCommits(args.context) : {};
-        await auditSubmit(client, args.audit, output, args.evidence, commits);
+        const contextError = args.context ? await readContextError(args.context) : null;
+        await auditSubmit(client, args.audit, output, args.evidence, commits, console.log, contextError);
         const problem = describeProblem(client);
         if (problem)
             warn(problem);

package/dist/fix.d.ts

@@ -1,5 +1,6 @@
 import { type Exec } from './audit.js';
 import type { IngestClient } from './client.js';
+import { type ClaudeUsage } from './usage.js';
 export declare const FIX_SCHEMA: {
     readonly type: "object";
     readonly properties: {
@@ -104,13 +105,7 @@ export interface ParsedFix {
     summary: string;
     reason: string | null;
     tests_run: string | null;
-    usage: {
-        model: string | null;
-        input_tokens: number;
-        output_tokens: number;
-        num_turns: number | null;
-        duration_ms: number | null;
-    } | null;
+    usage: ClaudeUsage | null;
 }
 /** Output `claude -p --output-format json --json-schema FIX_SCHEMA`; null = tidak ada hasil terstruktur. */
 export declare function parseFixOutput(raw: string): ParsedFix | null;

package/dist/fix.js

@@ -10,6 +10,7 @@
 import { appendFile, mkdir, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { defaultExec, gitAuthEnv, revokeTokens } from './audit.js';
+import { parseClaudeUsage, readClaudeUsage } from './usage.js';
 /** Model untuk --model Claude Code: alias atau ID versi; selain itu ditolak (nilainya masuk ke shell workflow). */
 const MODEL_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/;
 /** Berkas yang tidak boleh diubah perbaikan: workflow (bisa membaca secret repo), rahasia, lockfile. */
@@ -172,23 +173,12 @@ export function parseFixOutput(raw) {
     const out = parsed?.structured_output;
     if (!out || typeof out.fixed !== 'boolean' || typeof out.summary !== 'string')
         return null;
-    const models = Object.entries((parsed?.modelUsage ?? {}));
-    const model = models.sort(([, a], [, b]) => (b.outputTokens ?? 0) - (a.outputTokens ?? 0))[0]?.[0] ?? null;
-    const u = parsed?.usage;
     return {
         fixed: out.fixed,
         summary: out.summary,
         reason: typeof out.reason === 'string' && out.reason ? out.reason : null,
         tests_run: typeof out.tests_run === 'string' && out.tests_run ? out.tests_run : null,
-        usage: u
-            ? {
-                model,
-                input_tokens: Number(u.input_tokens ?? 0) + Number(u.cache_creation_input_tokens ?? 0) + Number(u.cache_read_input_tokens ?? 0),
-                output_tokens: Number(u.output_tokens ?? 0),
-                num_turns: Number.isInteger(parsed?.num_turns) ? parsed.num_turns : null,
-                duration_ms: Number.isInteger(parsed?.duration_ms) ? parsed.duration_ms : null,
-            }
-            : null,
+        usage: parseClaudeUsage(parsed),
     };
 }
 /**
@@ -207,8 +197,11 @@ export async function fixSubmit(client, fixId, claudeOutput, patch, workDir, opt
     };
     const parsed = parseFixOutput(claudeOutput);
     if (!parsed) {
+        // kuota habis / max turns tetap dicatat di pemakaian Claude
+        const usage = readClaudeUsage(claudeOutput);
         return send({
             status: 'failed',
+            ...(usage ? { usage } : {}),
             reason: 'Claude tidak menghasilkan hasil perbaikan terstruktur. Cek log step "Perbaiki dengan Claude Code" di repo runner: secret CLAUDE_CODE_OAUTH_TOKEN, kuota langganan, model yang tersedia di paket Anda, atau batas giliran.',
         });
     }

package/dist/usage.d.ts

@@ -0,0 +1,18 @@
+export interface ClaudeUsage {
+    model: string | null;
+    /** termasuk cache (system prompt Claude Code tercatat sebagai cache) */
+    input_tokens: number;
+    output_tokens: number;
+    cache_read_tokens: number;
+    /** total_cost_usd: nilai setara harga API, bukan tagihan langganan */
+    cost_usd: number | null;
+    num_turns: number | null;
+    duration_ms: number | null;
+    is_error: boolean;
+    limit_hit: boolean;
+    limit_message: string | null;
+}
+/** Dari objek hasil `claude -p` yang sudah di-parse; null bila tidak ada data pemakaian sama sekali. */
+export declare function parseClaudeUsage(parsed: any): ClaudeUsage | null;
+/** Dari teks mentah claude.json; teks non-JSON yang menyebut limit tetap dilaporkan sebagai limit. */
+export declare function readClaudeUsage(raw: string): ClaudeUsage | null;

package/dist/usage.js

@@ -0,0 +1,60 @@
+// Pemakaian Claude dari output `claude -p --output-format json`, dikirim ke Qaify untuk panel "Pemakaian Claude".
+// Sisa kuota resmi langganan tidak ada di output ini; yang tercatat hanya pemakaian per panggilan + tanda kena limit.
+// "Claude AI usage limit reached|1762160400", "5-hour limit reached ∙ resets 3pm", "You've hit your limit · resets …"
+const LIMIT_PATTERN = /usage limit|limit reached|hit your (?:usage )?limit|rate[ _-]?limit|resets? (?:at )?\d/i;
+/** Teks limit yang mudah dibaca: epoch setelah "|" diganti waktu reset ISO. */
+function limitMessage(text) {
+    return text
+        .replace(/\|(\d{10})\b/, (_, epoch) => ` · reset ${new Date(Number(epoch) * 1000).toISOString()}`)
+        .trim()
+        .slice(0, 500);
+}
+/** Dari objek hasil `claude -p` yang sudah di-parse; null bila tidak ada data pemakaian sama sekali. */
+export function parseClaudeUsage(parsed) {
+    if (!parsed || typeof parsed !== 'object')
+        return null;
+    const isError = parsed.is_error === true || (typeof parsed.subtype === 'string' && parsed.subtype.startsWith('error'));
+    const text = typeof parsed.result === 'string' ? parsed.result : '';
+    const limitHit = isError && LIMIT_PATTERN.test(text);
+    const u = parsed.usage;
+    if (!u && !isError)
+        return null;
+    // modelUsage: { "<model-id>": {...} }; ambil model dengan token keluaran terbanyak
+    const models = Object.entries((parsed.modelUsage ?? {}));
+    const model = models.sort(([, a], [, b]) => (b.outputTokens ?? 0) - (a.outputTokens ?? 0))[0]?.[0] ?? null;
+    return {
+        model,
+        input_tokens: Number(u?.input_tokens ?? 0) + Number(u?.cache_creation_input_tokens ?? 0) + Number(u?.cache_read_input_tokens ?? 0),
+        output_tokens: Number(u?.output_tokens ?? 0),
+        cache_read_tokens: Number(u?.cache_read_input_tokens ?? 0),
+        cost_usd: typeof parsed.total_cost_usd === 'number' ? parsed.total_cost_usd : null,
+        num_turns: Number.isInteger(parsed.num_turns) ? parsed.num_turns : null,
+        duration_ms: Number.isInteger(parsed.duration_ms) ? parsed.duration_ms : null,
+        is_error: isError,
+        limit_hit: limitHit,
+        limit_message: limitHit ? limitMessage(text) : null,
+    };
+}
+/** Dari teks mentah claude.json; teks non-JSON yang menyebut limit tetap dilaporkan sebagai limit. */
+export function readClaudeUsage(raw) {
+    try {
+        return parseClaudeUsage(JSON.parse(raw));
+    }
+    catch {
+        const text = raw.trim();
+        if (!text || !LIMIT_PATTERN.test(text))
+            return null;
+        return {
+            model: null,
+            input_tokens: 0,
+            output_tokens: 0,
+            cache_read_tokens: 0,
+            cost_usd: null,
+            num_turns: null,
+            duration_ms: null,
+            is_error: true,
+            limit_hit: true,
+            limit_message: limitMessage(text.split('\n').find((line) => LIMIT_PATTERN.test(line)) ?? text),
+        };
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qaify-playwright-reporter",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Reporter Playwright untuk QA Factory (Qaify): kirim hasil test dan artefak ke /api/ingest, tanpa pernah menggagalkan test.",
   "license": "MIT",
   "author": "ahmdkenz",

package/templates/qaify-audit.yml

@@ -106,9 +106,9 @@ jobs:
             --output-format json --json-schema "$(cat .qaify/schema.json)" \
             > .qaify/claude.json
 
-      # selalu jalan: tanpa hasil auditor, audit ditandai gagal beserta penyebab yang mungkin
+      # selalu jalan (juga saat konteks/clone gagal): tanpa hasil auditor, audit ditandai gagal beserta penyebabnya
       - name: Kirim hasil audit
-        if: always() && steps.ctx.outputs.ok == 'true'
+        if: always()
         env:
           QA_FACTORY_URL: ${{ vars.QA_FACTORY_URL }}
           QA_FACTORY_TOKEN: ${{ secrets.QA_FACTORY_TOKEN }}