subto 3.0.2 → 4.0.0

package/dist/package/README.md

@@ -40,7 +40,7 @@ Commands
 {
   "url": "https://example.com",
   "source": "cli",
-  "client": { "name": "subto-cli", "version": "3.0.2" }
+  "client": { "name": "subto-cli", "version": "4.0.0" }
 }
 ```
 
@@ -98,11 +98,11 @@ Download
 After publishing or packing, a distributable tarball will be available under `./dist/` e.g.:
 
 ```
-./dist/subto-3.0.2.tgz
+./dist/subto-4.0.0.tgz
 ```
 
 You can download that file directly and install locally with:
 
 ```bash
-npm install -g ./dist/subto-3.0.2.tgz
+npm install -g ./dist/subto-4.0.0.tgz
 ```

package/dist/package/index.js

@@ -11,7 +11,7 @@ const chalk = (_chalk && _chalk.default) ? _chalk.default : _chalk;
 const CONFIG_DIR = path.join(os.homedir(), '.subto');
 const CONFIG_PATH = path.join(CONFIG_DIR, 'config.json');
 const DEFAULT_API_BASE = 'https://subto.one';
-const CLIENT_META = { name: 'subto-cli', version: '3.0.2' };
+const CLIENT_META = { name: 'subto-cli', version: '4.0.0' };
 
 function configFilePath() { return CONFIG_PATH; }
 
@@ -91,7 +91,7 @@ function printScanSummary(obj) {
 
 async function run(argv) {
   const program = new Command();
-  program.name('subto').description('Subto CLI — wrapper around Subto.One API').version(CLIENT_META.version || '3.0.2');
+  program.name('subto').description('Subto CLI — wrapper around Subto.One API').version(CLIENT_META.version || '4.0.0');
 
   program.command('login').description('Store your API key in ~/.subto/config.json').action(async () => {
     try {

package/index.js

@@ -11,7 +11,7 @@ const chalk = (_chalk && _chalk.default) ? _chalk.default : _chalk;
 const CONFIG_DIR = path.join(os.homedir(), '.subto');
 const CONFIG_PATH = path.join(CONFIG_DIR, 'config.json');
 const DEFAULT_API_BASE = 'https://subto.one';
-const CLIENT_META = { name: 'subto-cli', version: '3.0.2' };
+const CLIENT_META = { name: 'subto-cli', version: '4.0.0' };
 
 function configFilePath() { return CONFIG_PATH; }
 
@@ -89,9 +89,94 @@ function printScanSummary(obj) {
   if (keys.length) console.log(chalk.dim('  Additional keys: ' + keys.join(', ')));
 }
 
+function printFullReport(data) {
+  const scan = (data && data.results) ? data.results : data;
+  if (!scan || typeof scan !== 'object') { console.log(JSON.stringify(data, null, 2)); return; }
+
+  const lh = scan.lighthouse || scan.lhr || (scan.results && scan.results.lighthouse) || null;
+  const vt = scan.virustotal || (scan.results && scan.results.virustotal) || null;
+  const timings = scan.timings || scan.coreWebVitals || (scan.results && scan.results.timings) || null;
+
+  console.log(chalk.bold.underline('\nOverview'));
+  if (scan.scanId || scan.id) console.log(chalk.green('  ID:'), scan.scanId || scan.id);
+  if (scan.url) console.log(chalk.green('  URL:'), scan.url);
+  if (scan.status) console.log(chalk.green('  Status:'), scan.status);
+  if (scan.completedAt) console.log(chalk.green('  Completed:'), new Date(scan.completedAt).toString());
+
+  console.log(chalk.bold.underline('\nSystem Analysis'));
+  if (lh) console.log('  Performance:', lh.performance ?? lh.performanceScore ?? '--');
+  if (lh) console.log('  Accessibility:', lh.accessibility ?? lh.accessibilityScore ?? '--');
+  if (lh) console.log('  SEO:', lh.seo ?? lh.seoScore ?? '--');
+  if (lh) console.log('  Best Practices:', lh.bestPractices ?? lh.bestPracticesScore ?? '--');
+  if (scan.jsDependency || scan.jsDependencyPercent) console.log('  JS Dependency:', scan.jsDependencyPercent ?? scan.jsDependency ?? '--');
+  if (scan.securityGrade) console.log('  Security:', scan.securityGrade);
+
+  if (timings) {
+    console.log(chalk.bold.underline('\nCore Web Vitals'));
+    const fcp = timings.firstContentfulPaint || timings.fcp || timings.FirstContentfulPaint;
+    const lcp = timings.largestContentfulPaint || timings.lcp || timings.LargestContentfulPaint;
+    const cls = timings.cumulativeLayoutShift || timings.cls;
+    const tti = timings.timeToInteractive || timings.tti;
+    const si = timings.speedIndex || timings.SpeedIndex;
+    if (fcp !== undefined) console.log('  First Contentful Paint:', String(fcp));
+    if (lcp !== undefined) console.log('  Largest Contentful Paint:', String(lcp));
+    if (cls !== undefined) console.log('  Cumulative Layout Shift:', String(cls));
+    if (tti !== undefined) console.log('  Time to Interactive:', String(tti));
+    if (si !== undefined) console.log('  Speed Index:', String(si));
+  }
+
+  // Execution Map / Dependencies heuristics
+  console.log(chalk.bold.underline('\nExecution Map'));
+  const listCandidates = [];
+  ['scripts','resources','dependencies','files','sourceFiles','assets','externalScripts'].forEach(k => { if (scan[k] && Array.isArray(scan[k])) listCandidates.push({k,arr:scan[k]}); if (scan.results && scan.results[k] && Array.isArray(scan.results[k])) listCandidates.push({k,arr:scan.results[k]}); });
+  if (listCandidates.length === 0) {
+    // try to guess from validations
+    const jsPaths = scan.validations && scan.validations.javascript && scan.validations.javascript.scripts;
+    if (Array.isArray(jsPaths)) listCandidates.push({k:'javascript files', arr: jsPaths});
+  }
+  if (listCandidates.length) {
+    for (const c of listCandidates.slice(0,3)) {
+      console.log(chalk.dim(`  ${c.k}:`));
+      for (const item of c.arr.slice(0,25)) console.log('    -', (typeof item === 'string') ? item : JSON.stringify(item));
+    }
+  } else {
+    console.log('  (no dependency list available in scan payload)');
+  }
+
+  // Issues summary
+  console.log(chalk.bold.underline('\nIssues'));
+  if (Array.isArray(scan.issues) && scan.issues.length) {
+    const byImpact = scan.issues.slice(0,50);
+    for (const it of byImpact) console.log('  -', it.title || it.message || JSON.stringify(it).slice(0,120));
+  } else if (scan.issuesCount) {
+    console.log('  Issues Count:', scan.issuesCount);
+  } else {
+    console.log('  No issues reported');
+  }
+
+  // Malware / VirusTotal
+  if (vt) {
+    console.log(chalk.bold.underline('\nMalware Scan (VirusTotal)'));
+    if (vt.scanDate) console.log('  Scan Date:', vt.scanDate);
+    console.log('  Harmless:', vt.harmless ?? vt.undetected ?? '--');
+    console.log('  Malicious:', vt.malicious ?? '--');
+    console.log('  Suspicious:', vt.suspicious ?? '--');
+    if (vt.resultUrl) console.log('  View Full VirusTotal Report:', vt.resultUrl.replace('https://www.virustotal.com/gui/url/','https://www.virustotal.com/gui/url/'));
+  }
+
+  // Video link
+  if (scan.videoUrl || scan.videoPath || scan.hasVideo) {
+    const vurl = scan.videoUrl || ((scan.videoPath) ? `${process.env.SUBTO_API_BASE_URL || DEFAULT_API_BASE}/video/${scan.scanId || scan.id}` : null);
+    if (vurl) console.log(chalk.bold('\nSession video:'), chalk.cyan(vurl));
+  }
+
+  // AI assistant hint
+  console.log(chalk.dim('\nTo inspect full JSON output, run with --json')); 
+}
+
 async function run(argv) {
   const program = new Command();
-  program.name('subto').description('Subto CLI — wrapper around Subto.One API').version(CLIENT_META.version || '3.0.2');
+  program.name('subto').description('Subto CLI — wrapper around Subto.One API').version(CLIENT_META.version || '4.0.0');
 
   program.command('login').description('Store your API key in ~/.subto/config.json').action(async () => {
     try {
@@ -151,9 +236,153 @@ async function run(argv) {
 
           const sleep = ms => new Promise(r => setTimeout(r, ms));
           let interval = 5000;
-          console.log(chalk.blue('Queued scan. Polling for progress...'));
+          const unicodeFrames = ['⠋','⠙','⠹','⠸','⠼','⠴','⠦','⠧','⠇','⠏'];
+          const asciiFrames = ['|','/','-','\\'];
+          const spinnerFrames = (process.stdout && process.stdout.isTTY) ? unicodeFrames : asciiFrames;
+          let spinnerIndex = 0;
+          let spinnerTimer = null;
+          // Percent smoothing: last displayed percent and target from server
+          let lastDisplayedPercent = 0;
+          let targetPercent = 0;
+          let lastPollTs = Date.now();
+          const terminalStates = ['finished', 'completed', 'done', 'complete', 'success', 'succeeded'];
+
+          // Render a single-line progress: spinner + label + [###-------] NN%
+          let lastRender = '';
+          function computePercentFromData(d){
+            if (!d) return 0;
+            // Prefer explicit progress when it's > 0; otherwise fall back to stage/status mapping
+            if (typeof d.progress === 'number' && !Number.isNaN(d.progress) && d.progress > 0) return Math.max(0, Math.min(100, Math.round(d.progress)));
+
+            // stage-aware mapping: try `stage` first (more granular), then `status`
+            const normalize = s => String(s || '').toLowerCase().replace(/[_-]+/g,' ').trim();
+            const stage = d && d.stage ? normalize(d.stage) : '';
+            const status = d && d.status ? normalize(d.status) : '';
+
+            // Exact stage mapping tuned to pipeline:
+            // - index fetch/extraction: 5%
+            // - assets fetch: 10%
+            // - render/DOM processing: 50%
+            // - screenshots/capture: 70%
+            // - analysis/audit/reporting: 90%
+            // - recording/video: 99%
+            // - finished: 100%
+
+            const mapStage = s => {
+              if (!s) return null;
+              if (s.includes('index') || s.includes('extract') || s.includes('fetch index') || s.includes('fetch-main')) return 5;
+              if (s.includes('asset') || s.includes('fetch') || s.includes('download')) return 10;
+              if (s.includes('render') || s.includes('dom') || s.includes('parse') || s.includes('layout')) return 50;
+              if (s.includes('screenshot') || s.includes('capture') || s.includes('screen')) return 70;
+              if (s.includes('analyse') || s.includes('analy') || s.includes('audit') || s.includes('report')) return 90;
+              if (s.includes('record') || s.includes('video') || s.includes('recording')) return 99;
+              if (s.includes('final') || s.includes('finish') || s.includes('complete') || s.includes('success') || s === 'done') return 100;
+              return null;
+            };
+
+            // prefer stage mapping for granularity
+            const stagePct = mapStage(stage);
+            if (stagePct !== null) return stagePct;
+
+            const statusPct = mapStage(status);
+            if (statusPct !== null) return statusPct;
+
+            // Prefer verbatim pipeline messages from server/modules/scan-pipeline.js
+            // Map known server messages to percentages (more accurate):
+            const msg = (stage + ' ' + status).trim();
+            if (/fetching initial html/i.test(msg)) return 5;
+            if (/parsing document structure/i.test(msg)) return 15;
+            if (/analyzing javascript/i.test(msg)) return 20;
+            if (/building dependency graph/i.test(msg)) return 30;
+            if (/capturing network requests/i.test(msg)) return 35;
+            if (/analyzing resource loading/i.test(msg)) return 45;
+            if (/pagespeed/i.test(msg)) return 50;
+            if (/mozilla observatory/i.test(msg)) return 65;
+            if (/detecting interactive/i.test(msg)) return 70;
+            if (/mapping user interactions/i.test(msg)) return 80;
+            if (/comparing no-?js/i.test(msg)) return 85;
+            if (/calculating js dependency/i.test(msg)) return 92;
+            if (/recording session video/i.test(msg)) return 99;
+            if (/analysis complete/i.test(msg) || /finaliz/i.test(msg)) return 100;
+
+            // sensible fallbacks
+            if (status.includes('queued') || status.includes('pending') || status.includes('accepted')) return 1;
+            if (status.includes('start') || status.includes('running') || status.includes('in progress') || status.includes('inprogress')) return 20;
+            return 0;
+          }
+
+          function renderLine(d){
+            const spinner = spinnerFrames[spinnerIndex % spinnerFrames.length];
+            spinnerIndex += 1;
+
+            // robustly extract status string
+            let rawLabel = 'waiting';
+            if (d && d.status) {
+              if (typeof d.status === 'string') rawLabel = d.status;
+              else if (typeof d.status === 'object' && (d.status.state || d.status.name)) rawLabel = d.status.state || d.status.name;
+              else rawLabel = String(d.status);
+            }
+            const label = String(rawLabel).replace(/[_-]+/g, ' ').trim().replace(/\b\w/g, c => c.toUpperCase());
+
+            // Update targetPercent from latest data but don't immediately drop lastDisplayedPercent
+            const serverPct = computePercentFromData(d);
+            if (serverPct >= 100) {
+              targetPercent = 100;
+            } else if (serverPct > lastDisplayedPercent) {
+              targetPercent = Math.max(targetPercent, serverPct);
+            } else if (serverPct > 0 && lastDisplayedPercent === 0) {
+              targetPercent = Math.max(targetPercent, serverPct);
+            }
+
+            // If explicit numeric progress provided and > lastDisplayed, accept it as target
+            if (d && typeof d.progress === 'number' && d.progress > lastDisplayedPercent) {
+              targetPercent = Math.max(targetPercent, Math.max(0, Math.min(100, Math.round(d.progress))));
+            }
+
+            // Smooth display: approach targetPercent gradually each tick
+            if (lastDisplayedPercent < targetPercent) {
+              const diff = targetPercent - lastDisplayedPercent;
+              const step = Math.max(1, Math.ceil(diff / 4));
+              lastDisplayedPercent = Math.min(targetPercent, lastDisplayedPercent + step);
+            }
+
+            // If target is behind (rare), don't regress unless finished
+            if (targetPercent === 100) lastDisplayedPercent = 100;
 
+            const pct = lastDisplayedPercent;
+            const barWidth = 20;
+            const filled = Math.round((pct/100) * barWidth);
+            const fillStr = filled ? chalk.green('#'.repeat(filled)) : '';
+            const emptyStr = chalk.dim('-'.repeat(Math.max(0, barWidth - filled)));
+            const bar = '[' + fillStr + emptyStr + ']';
+            const qp = (d && typeof d.queuePosition === 'number' && d.queuePosition > 0) ? ` Q#${d.queuePosition}` : '';
+            const line = `${chalk.cyan(spinner)} ${chalk.bold(label)} ${bar} ${chalk.yellow(String(pct).padStart(3) + '%')}${qp}`;
+            if (line !== lastRender) {
+              try {
+                readline.clearLine(process.stdout, 0);
+                readline.cursorTo(process.stdout, 0);
+                process.stdout.write(line);
+              } catch (e) {
+                console.log(line);
+              }
+              lastRender = line;
+            }
+            lastPollTs = Date.now();
+          }
+
+          console.log(chalk.blue('Queued scan. Polling for progress...'));
+          let headerState = 'queued';
+          function startSpinner(){
+            if (spinnerTimer) return;
+            if (!process.stdout || !process.stdout.isTTY) return;
+            spinnerTimer = setInterval(() => {
+              try { renderLine(data); } catch (e) { /* ignore */ }
+            }, 80);
+          }
+          function stopSpinner(){ if (spinnerTimer) { clearInterval(spinnerTimer); spinnerTimer = null; } }
           let recheckedAfterTerminal = false;
+          let data = null;
+          startSpinner();
           while (true) {
             const statusUrl = new URL(`/api/v1/scan/${scanId}`, base).toString();
             const r = await fetchFn(statusUrl, { headers: { 'Authorization': `Bearer ${cfg.apiKey}`, 'Accept': 'application/json' } });
@@ -163,41 +392,56 @@ async function run(argv) {
               try { const j = await r.json(); if (j && j.retry_after) retrySeconds = j.retry_after; } catch (e) {}
               try { const ra = r.headers.get && r.headers.get('retry-after'); if (!retrySeconds && ra) retrySeconds = parseInt(ra, 10); } catch (e) {}
               if (retrySeconds) console.error(chalk.yellow(`Rate limit reached. Try again in ${retrySeconds} seconds.`)); else console.error(chalk.yellow('Rate limit reached. Try again later.'));
+              stopSpinner();
               process.exit(1);
             }
 
-            let data = null;
             try { data = await r.json(); } catch (e) { data = null; }
 
-            if (data) {
-              if (data.queuePosition !== undefined) console.log(chalk.cyan('Queue position:'), data.queuePosition);
-              if (data.progress !== undefined) console.log(chalk.cyan('Progress:'), String(data.progress) + '%');
-              const rawStatus = data.status !== undefined && data.status !== null ? String(data.status) : '';
-              const displayStatus = rawStatus.trim();
-              if (displayStatus) console.log(chalk.cyan('Status:'), displayStatus);
-              if (data.stage) console.log(chalk.dim('Stage:'), data.stage);
-            } else {
-              console.log(chalk.dim('Waiting for server response...'));
+            // If we moved out of queue, replace the queued header with "Started scan."
+            const nonQueueStates = ['started','running','in progress','inprogress','processing','scanning'];
+            const statusStr = data && data.status ? String(data.status).toLowerCase().trim() : '';
+            if (headerState === 'queued' && statusStr && !['queued','pending','accepted'].includes(statusStr)) {
+              headerState = 'started';
+              try {
+                if (process.stdout && process.stdout.isTTY) {
+                  readline.moveCursor(process.stdout, 0, -1);
+                  readline.clearLine(process.stdout, 0);
+                  console.log(chalk.blue('Started scan.'));
+                } else {
+                  console.log(chalk.blue('Started scan.'));
+                }
+              } catch (e) {
+                console.log(chalk.blue('Started scan.'));
+              }
             }
 
-            const statusStr = data && data.status ? String(data.status).toLowerCase().trim() : '';
-            const terminalStates = ['finished', 'completed', 'done', 'complete', 'success', 'succeeded'];
-            const done = statusStr && terminalStates.includes(statusStr);
+            // render progress line
+            renderLine(data);
+
+            // consider terminal if status matches known terminal states OR percent maps to 100
+            const pctNow = computePercentFromData(data);
+            const done = (statusStr && terminalStates.includes(statusStr)) || (typeof pctNow === 'number' && pctNow === 100);
 
             if (done) {
-              // If server signals completion but hasn't populated result fields yet,
-              // re-check once after a short delay to allow finalization.
               const payloadKeys = data ? Object.keys(data).filter(k => !['status','queuePosition','progress','stage'].includes(k)) : [];
               const hasPayload = payloadKeys.length > 0;
               if (!hasPayload && !recheckedAfterTerminal) {
                 recheckedAfterTerminal = true;
-                console.log(chalk.yellow('Status is terminal but results not yet available — rechecking shortly...'));
-                await new Promise(r => setTimeout(r, 2000));
-                continue; // loop will fetch again
+                // give the server a moment to populate results
+                await sleep(2000);
+                // continue to refresh once more
+                try { const r2 = await fetchFn(statusUrl, { headers: { 'Authorization': `Bearer ${cfg.apiKey}`, 'Accept': 'application/json' } }); data = await r2.json(); } catch (e) {}
+                // final render
+                renderLine(data);
               }
 
+              // finish
+              stopSpinner();
+              try { readline.clearLine(process.stdout, 0); readline.cursorTo(process.stdout, 0); } catch (e) { /* ignore */ }
               console.log(chalk.green('Scan finished. Full results:'));
-              console.log(JSON.stringify(data, null, 2));
+              if (opts && opts.json) console.log(JSON.stringify(data, null, 2));
+              else printFullReport(data);
               return;
             }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "subto",
-  "version": "3.0.2",
+  "version": "4.0.0",
   "description": "Subto CLI — thin wrapper around the Subto.One API",
   "bin": {
     "subto": "bin/subto.js"