smippo 0.1.0 → 0.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smippo",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "S.M.I.P.P.O. — Structured Mirroring of Internet Pages and Public Objects. Modern website copier that captures sites exactly as they appear in your browser.",
   "main": "src/index.js",
   "bin": {

package/src/cli.js

@@ -89,8 +89,52 @@ export function run() {
       'Wait strategy: networkidle|load|domcontentloaded',
       'networkidle',
     )
-    .option('--wait-time <ms>', 'Additional wait time after network idle', '0')
+    .option(
+      '--wait-time <ms>',
+      'Additional wait time after network idle',
+      '500',
+    )
     .option('--timeout <ms>', 'Page load timeout', '30000')
+
+    // Scroll and reveal options (for capturing dynamic content)
+    .option(
+      '--scroll',
+      'Pre-scroll page to trigger lazy content (default: true)',
+    )
+    .option('--no-scroll', 'Disable pre-scroll behavior')
+    .option(
+      '--scroll-wait <ms>',
+      'Wait time after scrolling for animations',
+      '1000',
+    )
+    .option(
+      '--scroll-step <px>',
+      'Pixels per scroll increment (default: 200)',
+      '200',
+    )
+    .option(
+      '--scroll-delay <ms>',
+      'Delay between scroll steps (default: 50)',
+      '50',
+    )
+    .option(
+      '--scroll-behavior <type>',
+      'Scroll behavior: smooth|instant (default: smooth)',
+      'smooth',
+    )
+    .option(
+      '--reveal-all',
+      'Force reveal scroll-triggered content like GSAP, AOS (default: true)',
+    )
+    .option(
+      '--no-reveal-all',
+      'Disable force-reveal of scroll-triggered content',
+    )
+    .option(
+      '--reduced-motion',
+      'Use prefers-reduced-motion for accessibility (default: true)',
+    )
+    .option('--no-reduced-motion', 'Disable reduced motion preference')
     .option('--user-agent <string>', 'Custom user agent')
     .option('--viewport <WxH>', 'Viewport size', '1920x1080')
     .option('--device <name>', 'Emulate device (e.g., "iPhone 13")')
@@ -217,6 +261,18 @@ export function run() {
       });
     });
 
+  // Delete command - remove captured sites
+  program
+    .command('delete')
+    .alias('rm')
+    .description('Delete captured sites from local storage')
+    .option('-a, --all', 'Delete all captured sites')
+    .option('-y, --yes', 'Skip confirmation prompt')
+    .action(async options => {
+      const {deleteSites} = await import('./delete.js');
+      await deleteSites(options);
+    });
+
   // Screenshot capture command
   program
     .command('capture <url>')
@@ -312,6 +368,13 @@ async function capture(url, options) {
     wait: options.wait,
     waitTime: parseInt(options.waitTime, 10),
     timeout: parseInt(options.timeout, 10),
+    scroll: options.scroll,
+    scrollWait: parseInt(options.scrollWait, 10),
+    scrollStep: parseInt(options.scrollStep, 10),
+    scrollDelay: parseInt(options.scrollDelay, 10),
+    scrollBehavior: options.scrollBehavior,
+    revealAll: options.revealAll,
+    reducedMotion: options.reducedMotion,
     userAgent: options.userAgent,
     viewport: parseViewport(options.viewport),
     device: options.device,

package/src/crawler.js

@@ -270,6 +270,13 @@ export class Crawler extends EventEmitter {
         mimeExclude: this.options.mimeExclude,
         maxSize: this.options.maxSize,
         minSize: this.options.minSize,
+        scroll: this.options.scroll,
+        scrollWait: this.options.scrollWait,
+        scrollStep: this.options.scrollStep,
+        scrollDelay: this.options.scrollDelay,
+        scrollBehavior: this.options.scrollBehavior,
+        revealAll: this.options.revealAll,
+        reducedMotion: this.options.reducedMotion,
       });
 
       const result = await capture.capture(url);

package/src/delete.js

@@ -0,0 +1,188 @@
+// @flow
+import * as p from '@clack/prompts';
+import chalk from 'chalk';
+import fs from 'fs-extra';
+import path from 'path';
+import {readGlobalManifest, writeGlobalManifest} from './utils/home.js';
+
+/**
+ * Format file size for display
+ */
+function formatSize(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+/**
+ * Calculate directory size recursively
+ */
+async function getDirectorySize(dirPath) {
+  let totalSize = 0;
+
+  try {
+    const entries = await fs.readdir(dirPath, {withFileTypes: true});
+    for (const entry of entries) {
+      const fullPath = path.join(dirPath, entry.name);
+      if (entry.isDirectory()) {
+        totalSize += await getDirectorySize(fullPath);
+      } else {
+        const stats = await fs.stat(fullPath);
+        totalSize += stats.size;
+      }
+    }
+  } catch (e) {
+    // Ignore errors
+  }
+
+  return totalSize;
+}
+
+/**
+ * Interactive delete command for captured sites
+ */
+export async function deleteSites(options = {}) {
+  // Read global manifest
+  const manifest = await readGlobalManifest();
+
+  if (!manifest.sites || manifest.sites.length === 0) {
+    console.log(chalk.yellow('\nNo captured sites found.\n'));
+    return;
+  }
+
+  // Show header
+  console.log('');
+  p.intro(chalk.bgRed.white(' Delete Captured Sites '));
+
+  // Get site info with sizes
+  const sitesWithInfo = await Promise.all(
+    manifest.sites.map(async site => {
+      const exists = await fs.pathExists(site.path);
+      let size = 0;
+      if (exists) {
+        size = await getDirectorySize(site.path);
+      }
+      return {
+        ...site,
+        exists,
+        size,
+        displaySize: formatSize(size),
+      };
+    }),
+  );
+
+  // Filter to only existing sites
+  const existingSites = sitesWithInfo.filter(s => s.exists);
+
+  if (existingSites.length === 0) {
+    console.log(chalk.yellow('No captured sites found on disk.\n'));
+
+    // Clean up manifest
+    manifest.sites = [];
+    await writeGlobalManifest(manifest);
+    console.log(chalk.dim('Cleaned up manifest.\n'));
+    return;
+  }
+
+  let sitesToDelete = [];
+
+  if (options.all) {
+    // Delete all sites
+    sitesToDelete = existingSites;
+  } else {
+    // Interactive selection
+    const selected = await p.multiselect({
+      message: 'Select sites to delete (space to select, enter to confirm):',
+      options: existingSites.map(site => ({
+        value: site,
+        label: `${site.domain}`,
+        hint: `${site.displaySize} • ${site.path}`,
+      })),
+      required: false,
+    });
+
+    if (p.isCancel(selected) || !selected || selected.length === 0) {
+      p.cancel('No sites selected.');
+      return;
+    }
+
+    sitesToDelete = selected;
+  }
+
+  // Show what will be deleted
+  console.log('');
+  console.log(chalk.bold('Sites to delete:'));
+  for (const site of sitesToDelete) {
+    console.log(chalk.red(`  • ${site.domain} (${site.displaySize})`));
+    console.log(chalk.dim(`    ${site.path}`));
+  }
+  console.log('');
+
+  // Calculate total size
+  const totalSize = sitesToDelete.reduce((sum, s) => sum + s.size, 0);
+  console.log(
+    chalk.bold(
+      `Total: ${sitesToDelete.length} site(s), ${formatSize(totalSize)}`,
+    ),
+  );
+  console.log('');
+
+  // Confirmation
+  let confirmed = options.yes;
+  if (!confirmed) {
+    confirmed = await p.confirm({
+      message: 'Are you sure you want to delete these sites?',
+      initialValue: false,
+    });
+  }
+
+  if (p.isCancel(confirmed) || !confirmed) {
+    p.cancel('Deletion cancelled.');
+    return;
+  }
+
+  // Delete the sites
+  const spinner = p.spinner();
+  spinner.start('Deleting sites...');
+
+  let deletedCount = 0;
+  let failedCount = 0;
+  const deletedDomains = new Set();
+
+  for (const site of sitesToDelete) {
+    try {
+      await fs.remove(site.path);
+      deletedCount++;
+      deletedDomains.add(site.domain);
+    } catch (error) {
+      failedCount++;
+      console.error(
+        chalk.red(`\n  Failed to delete ${site.domain}: ${error.message}`),
+      );
+    }
+  }
+
+  // Update manifest - remove deleted sites
+  manifest.sites = manifest.sites.filter(
+    s =>
+      !deletedDomains.has(s.domain) ||
+      !sitesToDelete.some(d => d.path === s.path),
+  );
+  await writeGlobalManifest(manifest);
+
+  spinner.stop('Deletion complete!');
+
+  // Summary
+  console.log('');
+  if (deletedCount > 0) {
+    console.log(
+      chalk.green(
+        `✓ Deleted ${deletedCount} site(s), freed ${formatSize(totalSize)}`,
+      ),
+    );
+  }
+  if (failedCount > 0) {
+    console.log(chalk.red(`✗ Failed to delete ${failedCount} site(s)`));
+  }
+  console.log('');
+}

package/src/page-capture.js

@@ -3,6 +3,8 @@ import {extractLinks} from './link-extractor.js';
 
 /**
  * Capture a single page with all its rendered content
+ * Uses best-in-class techniques including accessibility features,
+ * reduced motion, and human-like scrolling behavior
  */
 export class PageCapture {
   constructor(page, options = {}) {
@@ -22,6 +24,12 @@ export class PageCapture {
       await this._collectResource(response);
     });
 
+    // Step 0: Enable reduced motion to disable CSS animations (accessibility feature)
+    // This causes many sites to show final animation states immediately
+    if (this.options.reducedMotion !== false) {
+      await this.page.emulateMedia({reducedMotion: 'reduce'});
+    }
+
     // Navigate to the page
     try {
       await this.page.goto(url, {
@@ -35,11 +43,40 @@ export class PageCapture {
       }
     }
 
-    // Additional wait time if specified
-    if (this.options.waitTime > 0) {
-      await this.page.waitForTimeout(this.options.waitTime);
+    // Additional wait time if specified (default 500ms for animations to start)
+    const waitTime = this.options.waitTime ?? 500;
+    if (waitTime > 0) {
+      await this.page.waitForTimeout(waitTime);
+    }
+
+    // Step 1: Force all scroll-triggered animations to their END state (100% progress)
+    // This is different from killing them - we want their final visual state
+    if (this.options.revealAll !== false) {
+      await this._completeAllAnimations();
+    }
+
+    // Step 2: Human-like scrolling to trigger lazy content and intersection observers
+    if (this.options.scroll !== false) {
+      await this._humanLikeScroll();
     }
 
+    // Step 3: Complete animations again after scroll (new elements may have appeared)
+    if (this.options.revealAll !== false) {
+      await this._completeAllAnimations();
+    }
+
+    // Step 4: Final wait for any remaining animations/network requests
+    const scrollWait = this.options.scrollWait ?? 1000;
+    if (scrollWait > 0) {
+      await this.page.waitForTimeout(scrollWait);
+    }
+
+    // Step 5: Wait for network to be truly idle (no pending requests)
+    await this._waitForNetworkIdle();
+
+    // Step 6: Force reveal any remaining hidden elements
+    await this._forceRevealHiddenElements();
+
     // Get the rendered HTML
     const html = await this.page.content();
 
@@ -148,4 +185,347 @@ export class PageCapture {
       return type === filter || type.startsWith(filter + ';');
     });
   }
+
+  /**
+   * Wait for network to be truly idle (no pending requests for a period)
+   */
+  async _waitForNetworkIdle() {
+    try {
+      await this.page.waitForLoadState('networkidle', {timeout: 5000});
+    } catch {
+      // Timeout is ok, continue anyway
+    }
+  }
+
+  /**
+   * Human-like scrolling behavior with pauses and mouse movements
+   * Triggers intersection observers and lazy loading more naturally
+   */
+  async _humanLikeScroll() {
+    const scrollStep = this.options.scrollStep || 300;
+    const scrollDelay = this.options.scrollDelay || 100;
+
+    /* eslint-disable no-undef */
+    await this.page.evaluate(
+      async ({step, delay}) => {
+        // Human-like smooth scroll with easing
+        const smoothScrollTo = (targetY, duration = 500) => {
+          return new Promise(resolve => {
+            const startY = window.scrollY;
+            const distance = targetY - startY;
+            const startTime = performance.now();
+
+            const easeInOutCubic = t =>
+              t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2;
+
+            const animate = () => {
+              const elapsed = performance.now() - startTime;
+              const progress = Math.min(elapsed / duration, 1);
+              const eased = easeInOutCubic(progress);
+
+              window.scrollTo(0, startY + distance * eased);
+
+              if (progress < 1) {
+                requestAnimationFrame(animate);
+              } else {
+                resolve();
+              }
+            };
+            requestAnimationFrame(animate);
+          });
+        };
+
+        // Wait helper
+        const wait = ms => new Promise(r => setTimeout(r, ms));
+
+        // Get viewport and page dimensions
+        const viewportHeight = window.innerHeight;
+        let pageHeight = document.body.scrollHeight;
+        let currentY = 0;
+        let lastHeight = pageHeight;
+
+        // Phase 1: Scroll down slowly, pausing at each "section"
+        while (currentY < pageHeight) {
+          // Scroll one viewport at a time (like a human reading)
+          const targetY = Math.min(currentY + step, pageHeight);
+          await smoothScrollTo(targetY, delay * 3);
+
+          // Pause longer at section boundaries (every viewport height)
+          if (
+            Math.floor(currentY / viewportHeight) !==
+            Math.floor(targetY / viewportHeight)
+          ) {
+            await wait(delay * 2); // Pause to "read" the section
+          } else {
+            await wait(delay);
+          }
+
+          currentY = targetY;
+
+          // Check for lazy-loaded content increasing page height
+          const newHeight = document.body.scrollHeight;
+          if (newHeight > lastHeight) {
+            pageHeight = newHeight;
+            lastHeight = newHeight;
+          }
+        }
+
+        // Phase 2: Ensure we're at the very bottom
+        await smoothScrollTo(document.body.scrollHeight, 300);
+        await wait(500); // Wait at bottom
+
+        // Check one more time for lazy content
+        if (document.body.scrollHeight > pageHeight) {
+          await smoothScrollTo(document.body.scrollHeight, 300);
+          await wait(300);
+        }
+
+        // Phase 3: Scroll back to top (faster)
+        await smoothScrollTo(0, 800);
+        await wait(300);
+      },
+      {step: scrollStep, delay: scrollDelay},
+    );
+    /* eslint-enable no-undef */
+  }
+
+  /**
+   * Complete all animations to their final state (100% progress)
+   * Instead of killing animations, we progress them to completion
+   */
+  async _completeAllAnimations() {
+    /* eslint-disable no-undef */
+    await this.page.evaluate(() => {
+      // Helper to safely access nested properties
+      const safeGet = (obj, path) => {
+        try {
+          return path.split('.').reduce((o, k) => o?.[k], obj);
+        } catch {
+          return undefined;
+        }
+      };
+
+      // 1. GSAP - Progress ALL tweens and timelines to 100%
+      const gsap = safeGet(window, 'gsap');
+      if (gsap) {
+        try {
+          // Get all global tweens and progress them to end
+          const tweens =
+            gsap.globalTimeline?.getChildren?.(true, true, true) || [];
+          tweens.forEach(tween => {
+            try {
+              // Progress to 100% (end state)
+              tween.progress?.(1, true);
+              // Also try totalProgress for timelines
+              tween.totalProgress?.(1, true);
+            } catch (e) {
+              /* ignore */
+            }
+          });
+        } catch (e) {
+          /* ignore */
+        }
+      }
+
+      // 2. GSAP ScrollTrigger - Progress all triggers to their END state
+      const ScrollTrigger =
+        safeGet(window, 'ScrollTrigger') ||
+        safeGet(window, 'gsap.ScrollTrigger');
+      if (ScrollTrigger) {
+        try {
+          const triggers = ScrollTrigger.getAll?.() || [];
+          triggers.forEach(trigger => {
+            try {
+              // Progress the trigger to 100% (fully scrolled past)
+              if (trigger.animation) {
+                trigger.animation.progress?.(1, true);
+                trigger.animation.totalProgress?.(1, true);
+              }
+              // Also set the trigger's own progress
+              trigger.progress?.(1, true);
+
+              // Disable the trigger so it doesn't reset
+              trigger.disable?.();
+            } catch (e) {
+              /* ignore */
+            }
+          });
+
+          // Don't refresh - we want to keep the end states
+        } catch (e) {
+          /* ignore */
+        }
+      }
+
+      // 3. Web Animations API - Complete all animations
+      document.getAnimations?.().forEach(animation => {
+        try {
+          animation.finish();
+        } catch (e) {
+          try {
+            animation.currentTime =
+              animation.effect?.getTiming?.()?.duration || 0;
+          } catch (e2) {
+            /* ignore */
+          }
+        }
+      });
+
+      // 4. AOS - Mark all as animated
+      document.querySelectorAll('[data-aos]').forEach(el => {
+        el.classList.add('aos-animate');
+        el.setAttribute('data-aos-animate', 'true');
+      });
+
+      // 5. WOW.js elements
+      document.querySelectorAll('.wow').forEach(el => {
+        el.classList.add('animated');
+        el.style.visibility = 'visible';
+      });
+
+      // 6. ScrollReveal elements
+      document.querySelectorAll('[data-sr-id]').forEach(el => {
+        el.style.visibility = 'visible';
+        el.style.opacity = '1';
+      });
+
+      // 7. Anime.js - Complete all animations
+      const anime = safeGet(window, 'anime');
+      if (anime?.running) {
+        anime.running.forEach(anim => {
+          try {
+            anim.seek?.(anim.duration);
+          } catch (e) {
+            /* ignore */
+          }
+        });
+      }
+
+      // 8. Lottie animations - Go to last frame
+      document.querySelectorAll('lottie-player, [data-lottie]').forEach(el => {
+        try {
+          el.goToAndStop?.(el.totalFrames - 1, true);
+          el.pause?.();
+        } catch (e) {
+          /* ignore */
+        }
+      });
+    });
+    /* eslint-enable no-undef */
+  }
+
+  /**
+   * Force reveal any remaining hidden elements
+   * This is the final cleanup pass
+   */
+  async _forceRevealHiddenElements() {
+    /* eslint-disable no-undef */
+    await this.page.evaluate(() => {
+      // 1. Force load all lazy images
+      document.querySelectorAll('img').forEach(img => {
+        // Load from data attributes
+        const lazySrc =
+          img.dataset.src || img.dataset.lazy || img.dataset.lazySrc;
+        if (lazySrc && !img.src.includes(lazySrc)) {
+          img.src = lazySrc;
+        }
+        // Remove lazy loading attribute
+        img.removeAttribute('loading');
+        // Trigger load if not loaded
+        if (!img.complete) {
+          img.loading = 'eager';
+        }
+      });
+
+      // 2. Load lazy srcset
+      document.querySelectorAll('source[data-srcset]').forEach(source => {
+        if (source.dataset.srcset) {
+          source.srcset = source.dataset.srcset;
+        }
+      });
+
+      // 3. Load lazy background images
+      document
+        .querySelectorAll(
+          '[data-bg], [data-background], [data-background-image]',
+        )
+        .forEach(el => {
+          const bg =
+            el.dataset.bg ||
+            el.dataset.background ||
+            el.dataset.backgroundImage;
+          if (bg) {
+            el.style.backgroundImage = `url("${bg}")`;
+          }
+        });
+
+      // 4. Load lazy iframes
+      document.querySelectorAll('iframe[data-src]').forEach(iframe => {
+        if (iframe.dataset.src) {
+          iframe.src = iframe.dataset.src;
+        }
+      });
+
+      // 5. Fix elements with opacity: 0 (likely animation end states)
+      document.querySelectorAll('*').forEach(el => {
+        const computed = window.getComputedStyle(el);
+
+        // Only fix visible containers that have hidden content
+        if (computed.opacity === '0' && !el.getAttribute('aria-hidden')) {
+          // Check if this is likely an animated element
+          const hasAnimClass =
+            el.className && /anim|fade|slide|reveal|show/i.test(el.className);
+          const hasAnimAttr =
+            el.hasAttribute('data-aos') ||
+            el.hasAttribute('data-animate') ||
+            el.hasAttribute('data-scroll');
+
+          if (hasAnimClass || hasAnimAttr || el.style.opacity === '0') {
+            el.style.setProperty('opacity', '1', 'important');
+          }
+        }
+
+        // Fix visibility
+        if (
+          computed.visibility === 'hidden' &&
+          !el.getAttribute('aria-hidden')
+        ) {
+          const hasAnimClass =
+            el.className && /anim|fade|slide|reveal|show/i.test(el.className);
+          if (hasAnimClass || el.style.visibility === 'hidden') {
+            el.style.setProperty('visibility', 'visible', 'important');
+          }
+        }
+
+        // Fix transforms that look like animation starting positions
+        if (computed.transform && computed.transform !== 'none') {
+          const transform = computed.transform;
+          // Check for translateY/translateX that might be animation starting positions
+          if (/translate[XY]\s*\(\s*[+-]?\d+/.test(transform)) {
+            const hasAnimClass =
+              el.className && /anim|fade|slide|reveal|show/i.test(el.className);
+            if (hasAnimClass) {
+              el.style.setProperty('transform', 'none', 'important');
+            }
+          }
+        }
+      });
+
+      // 6. Force CSS animations to end state
+      document.querySelectorAll('*').forEach(el => {
+        const computed = window.getComputedStyle(el);
+        if (computed.animationName && computed.animationName !== 'none') {
+          el.style.setProperty('animation', 'none', 'important');
+        }
+        if (
+          computed.transition &&
+          computed.transition !== 'none' &&
+          computed.transition !== 'all 0s ease 0s'
+        ) {
+          el.style.setProperty('transition', 'none', 'important');
+        }
+      });
+    });
+    /* eslint-enable no-undef */
+  }
 }