smippo 0.0.1

package/src/progress.js

@@ -0,0 +1,190 @@
+// @flow
+import cliProgress from 'cli-progress';
+import chalk from 'chalk';
+
+/**
+ * Create a styled multi-bar progress display for Smippo
+ */
+export function createProgressDisplay(options = {}) {
+  const {quiet = false, verbose = false} = options;
+
+  if (quiet) {
+    return {
+      start: () => {},
+      stop: () => {},
+      updatePage: () => {},
+      updateAsset: () => {},
+      log: () => {},
+    };
+  }
+
+  // Create multi-bar container
+  const multibar = new cliProgress.MultiBar(
+    {
+      clearOnComplete: false,
+      hideCursor: true,
+      format: (options, params, payload) => {
+        const {type, name} = payload;
+        const bar = options.barCompleteChar.repeat(
+          Math.round(params.progress * 30),
+        );
+        const empty = options.barIncompleteChar.repeat(
+          30 - Math.round(params.progress * 30),
+        );
+        const percent = Math.round(params.progress * 100);
+
+        if (type === 'pages') {
+          return `  ${chalk.cyan('Pages')}   ${chalk.cyan('[')}${chalk.cyan(bar)}${chalk.dim(empty)}${chalk.cyan(']')} ${chalk.bold(params.value)}/${params.total} ${chalk.dim(`(${percent}%)`)}`;
+        } else if (type === 'assets') {
+          return `  ${chalk.magenta('Assets')}  ${chalk.magenta('[')}${chalk.magenta(bar)}${chalk.dim(empty)}${chalk.magenta(']')} ${chalk.bold(params.value)}/${params.total} ${chalk.dim(`(${percent}%)`)}`;
+        } else if (type === 'current') {
+          const truncated = name.length > 50 ? '...' + name.slice(-47) : name;
+          return `  ${chalk.dim('Current:')} ${chalk.white(truncated)}`;
+        }
+        return '';
+      },
+      barCompleteChar: '█',
+      barIncompleteChar: '░',
+    },
+    cliProgress.Presets.shades_classic,
+  );
+
+  let pageBar = null;
+  let assetBar = null;
+  let currentBar = null;
+  let totalPages = 0;
+  let totalAssets = 0;
+  let completedPages = 0;
+  let completedAssets = 0;
+
+  return {
+    /**
+     * Start the progress display
+     */
+    start(estimatedPages = 1, estimatedAssets = 10) {
+      totalPages = estimatedPages;
+      totalAssets = estimatedAssets;
+      completedPages = 0;
+      completedAssets = 0;
+
+      console.log('');
+      pageBar = multibar.create(totalPages, 0, {type: 'pages'});
+      assetBar = multibar.create(totalAssets, 0, {type: 'assets'});
+      currentBar = multibar.create(100, 0, {
+        type: 'current',
+        name: 'Initializing...',
+      });
+    },
+
+    /**
+     * Stop the progress display
+     */
+    stop() {
+      if (multibar) {
+        multibar.stop();
+      }
+      console.log('');
+    },
+
+    /**
+     * Update page progress
+     */
+    updatePage(url, total = null) {
+      completedPages++;
+      if (total && total > totalPages) {
+        totalPages = total;
+        if (pageBar) pageBar.setTotal(totalPages);
+      }
+      if (pageBar) pageBar.update(completedPages);
+      if (currentBar) currentBar.update(0, {name: url});
+    },
+
+    /**
+     * Update asset progress
+     */
+    updateAsset(url, total = null) {
+      completedAssets++;
+      if (total && total > totalAssets) {
+        totalAssets = total;
+        if (assetBar) assetBar.setTotal(totalAssets);
+      }
+      if (assetBar) assetBar.update(completedAssets);
+      if (currentBar && verbose) currentBar.update(0, {name: url});
+    },
+
+    /**
+     * Increment total assets estimate
+     */
+    addAssets(count) {
+      totalAssets += count;
+      if (assetBar) assetBar.setTotal(totalAssets);
+    },
+
+    /**
+     * Log a message (pauses bars)
+     */
+    log(message) {
+      if (verbose) {
+        multibar.log(message + '\n');
+      }
+    },
+
+    /**
+     * Update current status
+     */
+    setStatus(status) {
+      if (currentBar) currentBar.update(0, {name: status});
+    },
+  };
+}
+
+/**
+ * Simple spinner-based progress for non-TTY environments
+ */
+export function createSimpleProgress(options = {}) {
+  const {quiet = false, verbose = false} = options;
+
+  let pageCount = 0;
+  let assetCount = 0;
+  let lastUpdate = Date.now();
+
+  return {
+    start: () => {
+      if (!quiet) console.log(chalk.cyan('\n  Starting capture...\n'));
+    },
+    stop: () => {},
+    updatePage: url => {
+      pageCount++;
+      if (!quiet && (verbose || Date.now() - lastUpdate > 500)) {
+        process.stdout.write(
+          `\r  ${chalk.cyan('Pages:')} ${pageCount}  ${chalk.magenta('Assets:')} ${assetCount}  ${chalk.dim('Current:')} ${url.slice(0, 50)}${url.length > 50 ? '...' : ''}    `,
+        );
+        lastUpdate = Date.now();
+      }
+    },
+    updateAsset: () => {
+      assetCount++;
+      if (!quiet && verbose) {
+        process.stdout.write(
+          `\r  ${chalk.cyan('Pages:')} ${pageCount}  ${chalk.magenta('Assets:')} ${assetCount}    `,
+        );
+      }
+    },
+    addAssets: () => {},
+    log: message => {
+      if (verbose) console.log(message);
+    },
+    setStatus: () => {},
+  };
+}
+
+/**
+ * Choose the appropriate progress display based on environment
+ */
+export function createProgress(options = {}) {
+  // Use simple progress if not a TTY or if specifically requested
+  if (!process.stdout.isTTY || options.simple) {
+    return createSimpleProgress(options);
+  }
+  return createProgressDisplay(options);
+}

package/src/resource-saver.js

@@ -0,0 +1,210 @@
+// @flow
+import fs from 'fs-extra';
+import path from 'path';
+import {urlToPath} from './utils/url.js';
+import {sanitizePath, joinPath} from './utils/path.js';
+import mime from 'mime-types';
+
+/**
+ * Save resources to disk
+ */
+export class ResourceSaver {
+  constructor(options = {}) {
+    this.outputDir = options.output || './site';
+    this.structure = options.structure || 'original';
+    this.savedFiles = new Map(); // URL -> local path
+    this.stats = {
+      files: 0,
+      bytes: 0,
+    };
+  }
+
+  /**
+   * Get the local path for a URL
+   */
+  getLocalPath(url) {
+    const relativePath = urlToPath(url, this.structure);
+    return joinPath(this.outputDir, sanitizePath(relativePath));
+  }
+
+  /**
+   * Save HTML content
+   */
+  async saveHtml(url, html, _options = {}) {
+    const relativePath = urlToPath(url, this.structure);
+    const localPath = joinPath(this.outputDir, sanitizePath(relativePath));
+
+    // Ensure directory exists
+    await fs.ensureDir(path.dirname(localPath));
+
+    // Write the file
+    await fs.writeFile(localPath, html, 'utf8');
+
+    // Store the relative path (not full path) for link rewriting
+    this.savedFiles.set(url, sanitizePath(relativePath));
+    this.stats.files++;
+    this.stats.bytes += Buffer.byteLength(html, 'utf8');
+
+    return localPath;
+  }
+
+  /**
+   * Save a resource (binary or text)
+   */
+  async saveResource(url, resource) {
+    let relativePath = urlToPath(url, this.structure);
+    relativePath = sanitizePath(relativePath);
+    let localPath = joinPath(this.outputDir, relativePath);
+
+    // Fix extension based on content type if needed
+    localPath = this._fixExtension(localPath, resource.contentType);
+    relativePath = this._fixExtension(relativePath, resource.contentType);
+
+    // Ensure directory exists
+    await fs.ensureDir(path.dirname(localPath));
+
+    // Write the file
+    await fs.writeFile(localPath, resource.body);
+
+    // Store the relative path (not full path) for link rewriting
+    this.savedFiles.set(url, relativePath);
+    this.stats.files++;
+    this.stats.bytes += resource.size;
+
+    return localPath;
+  }
+
+  /**
+   * Save multiple resources
+   */
+  async saveResources(resources) {
+    const saved = [];
+
+    for (const [url, resource] of resources) {
+      try {
+        const localPath = await this.saveResource(url, resource);
+        saved.push({url, localPath, size: resource.size});
+      } catch (error) {
+        // Continue saving other resources
+      }
+    }
+
+    return saved;
+  }
+
+  /**
+   * Save a screenshot
+   */
+  async saveScreenshot(url, screenshot) {
+    const basePath = this.getLocalPath(url);
+    const screenshotPath = basePath.replace(/\.html?$/i, '.png');
+
+    await fs.ensureDir(path.dirname(screenshotPath));
+    await fs.writeFile(screenshotPath, screenshot);
+
+    return screenshotPath;
+  }
+
+  /**
+   * Save a PDF
+   */
+  async savePdf(url, pdf) {
+    const basePath = this.getLocalPath(url);
+    const pdfPath = basePath.replace(/\.html?$/i, '.pdf');
+
+    await fs.ensureDir(path.dirname(pdfPath));
+    await fs.writeFile(pdfPath, pdf);
+
+    return pdfPath;
+  }
+
+  /**
+   * Get URL to local path mapping
+   */
+  getUrlMap() {
+    return new Map(this.savedFiles);
+  }
+
+  /**
+   * Get relative path from output directory
+   */
+  getRelativePath(localPath) {
+    return path.relative(this.outputDir, localPath);
+  }
+
+  /**
+   * Fix file extension based on content type
+   */
+  _fixExtension(filePath, contentType) {
+    if (!contentType) return filePath;
+
+    const mimeType = contentType.split(';')[0].trim();
+    const expectedExt = mime.extension(mimeType);
+
+    if (!expectedExt) return filePath;
+
+    const currentExt = path.extname(filePath).slice(1).toLowerCase();
+
+    // Don't change if extension seems correct
+    const equivalentExtensions = {
+      jpeg: ['jpg', 'jpeg'],
+      htm: ['html', 'htm'],
+      js: ['js', 'mjs', 'cjs'],
+    };
+
+    const isEquivalent = Object.values(equivalentExtensions).some(
+      group => group.includes(currentExt) && group.includes(expectedExt),
+    );
+
+    if (isEquivalent || currentExt === expectedExt) {
+      return filePath;
+    }
+
+    // Only fix if current extension is wrong or missing
+    if (!currentExt || !isKnownExtension(currentExt)) {
+      return `${filePath}.${expectedExt}`;
+    }
+
+    return filePath;
+  }
+}
+
+/**
+ * Check if extension is known
+ */
+function isKnownExtension(ext) {
+  const known = [
+    'html',
+    'htm',
+    'css',
+    'js',
+    'mjs',
+    'json',
+    'xml',
+    'png',
+    'jpg',
+    'jpeg',
+    'gif',
+    'webp',
+    'svg',
+    'ico',
+    'bmp',
+    'woff',
+    'woff2',
+    'ttf',
+    'eot',
+    'otf',
+    'mp3',
+    'mp4',
+    'webm',
+    'ogg',
+    'wav',
+    'avi',
+    'pdf',
+    'zip',
+    'tar',
+    'gz',
+  ];
+
+  return known.includes(ext.toLowerCase());
+}

package/src/robots.js

@@ -0,0 +1,104 @@
+// @flow
+import robotsParser from 'robots-parser';
+
+/**
+ * robots.txt handler
+ */
+export class RobotsHandler {
+  constructor(options = {}) {
+    this.enabled = !options.ignoreRobots;
+    this.userAgent = options.userAgent || 'Smippo/0.0.1';
+    this.cache = new Map();
+  }
+
+  /**
+   * Check if a URL is allowed by robots.txt
+   */
+  async isAllowed(url, fetchFn) {
+    if (!this.enabled) return true;
+
+    try {
+      const robots = await this.getRobots(url, fetchFn);
+      if (!robots) return true;
+
+      return robots.isAllowed(url, this.userAgent);
+    } catch {
+      // If we can't fetch/parse robots.txt, allow access
+      return true;
+    }
+  }
+
+  /**
+   * Get robots.txt parser for a URL
+   */
+  async getRobots(url, fetchFn) {
+    try {
+      const parsed = new URL(url);
+      const robotsUrl = `${parsed.origin}/robots.txt`;
+
+      // Check cache
+      if (this.cache.has(robotsUrl)) {
+        return this.cache.get(robotsUrl);
+      }
+
+      // Fetch robots.txt
+      const robotsContent = await fetchFn(robotsUrl);
+
+      if (!robotsContent) {
+        this.cache.set(robotsUrl, null);
+        return null;
+      }
+
+      // Parse robots.txt
+      const robots = robotsParser(robotsUrl, robotsContent);
+      this.cache.set(robotsUrl, robots);
+
+      return robots;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Get crawl delay for a domain
+   */
+  getCrawlDelay(url) {
+    if (!this.enabled) return 0;
+
+    try {
+      const parsed = new URL(url);
+      const robotsUrl = `${parsed.origin}/robots.txt`;
+      const robots = this.cache.get(robotsUrl);
+
+      if (!robots) return 0;
+
+      return robots.getCrawlDelay(this.userAgent) || 0;
+    } catch {
+      return 0;
+    }
+  }
+
+  /**
+   * Get sitemap URLs from robots.txt
+   */
+  getSitemaps(url) {
+    try {
+      const parsed = new URL(url);
+      const robotsUrl = `${parsed.origin}/robots.txt`;
+      const robots = this.cache.get(robotsUrl);
+
+      if (!robots) return [];
+
+      return robots.getSitemaps() || [];
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Clear the cache
+   */
+  clearCache() {
+    this.cache.clear();
+  }
+}

package/src/screenshot.js

@@ -0,0 +1,185 @@
+// @flow
+import {chromium} from 'playwright';
+import fs from 'fs-extra';
+import path from 'path';
+import chalk from 'chalk';
+
+/**
+ * Capture a screenshot of a URL
+ * Based on Playwright Screenshots API: https://playwright.dev/docs/screenshots
+ */
+export async function captureScreenshot(url, options = {}) {
+  const {
+    output,
+    fullPage = false,
+    format = 'png',
+    quality,
+    viewport = {width: 1920, height: 1080},
+    device,
+    selector,
+    wait = 'networkidle',
+    waitTime = 0,
+    timeout = 30000,
+    userAgent,
+    darkMode = false,
+    scale = 'device',
+    omitBackground = false,
+    verbose = false,
+    quiet = false,
+  } = options;
+
+  // Normalize URL
+  if (!url.startsWith('http://') && !url.startsWith('https://')) {
+    url = 'https://' + url;
+  }
+
+  // Determine output path
+  let outputPath = output;
+  if (!outputPath) {
+    const urlObj = new URL(url);
+    const timestamp = new Date()
+      .toISOString()
+      .replace(/[:.]/g, '-')
+      .slice(0, 19);
+    const ext = format === 'jpeg' ? 'jpg' : 'png';
+    outputPath = `${urlObj.hostname}-${timestamp}.${ext}`;
+  }
+
+  // Ensure output directory exists
+  const outputDir = path.dirname(outputPath);
+  if (outputDir && outputDir !== '.') {
+    await fs.ensureDir(outputDir);
+  }
+
+  if (!quiet) {
+    console.log('');
+    console.log(chalk.cyan('  📸 Smippo Screenshot'));
+    console.log(chalk.dim(`  URL: ${url}`));
+    console.log('');
+  }
+
+  const browser = await chromium.launch();
+
+  try {
+    // Set up context options
+    const contextOptions = {
+      viewport,
+      userAgent: userAgent || 'Smippo/0.0.1 Screenshot',
+    };
+
+    // Device emulation
+    if (device) {
+      const {devices} = await import('playwright');
+      if (devices[device]) {
+        Object.assign(contextOptions, devices[device]);
+        if (!quiet) console.log(chalk.dim(`  Device: ${device}`));
+      } else {
+        console.warn(
+          chalk.yellow(
+            `  Warning: Unknown device "${device}", using default viewport`,
+          ),
+        );
+      }
+    }
+
+    // Dark mode
+    if (darkMode) {
+      contextOptions.colorScheme = 'dark';
+    }
+
+    const context = await browser.newContext(contextOptions);
+    const page = await context.newPage();
+
+    // Navigate
+    if (verbose) console.log(chalk.dim(`  Navigating to ${url}...`));
+
+    await page.goto(url, {
+      waitUntil: wait,
+      timeout,
+    });
+
+    // Additional wait time
+    if (waitTime > 0) {
+      if (verbose) console.log(chalk.dim(`  Waiting ${waitTime}ms...`));
+      await page.waitForTimeout(waitTime);
+    }
+
+    // Screenshot options
+    const screenshotOptions = {
+      path: outputPath,
+      type: format,
+      fullPage,
+      scale,
+      omitBackground,
+    };
+
+    // JPEG quality (only for jpeg format)
+    if (format === 'jpeg' && quality) {
+      screenshotOptions.quality = quality;
+    }
+
+    // Take screenshot
+    if (selector) {
+      // Element screenshot
+      if (verbose) console.log(chalk.dim(`  Capturing element: ${selector}`));
+      const element = page.locator(selector);
+      await element.screenshot(screenshotOptions);
+    } else {
+      // Page screenshot
+      if (verbose) {
+        console.log(
+          chalk.dim(`  Capturing ${fullPage ? 'full page' : 'viewport'}...`),
+        );
+      }
+      await page.screenshot(screenshotOptions);
+    }
+
+    // Get file size
+    const stats = await fs.stat(outputPath);
+    const fileSize = formatFileSize(stats.size);
+
+    if (!quiet) {
+      console.log(chalk.green(`  ✓ Screenshot saved`));
+      console.log(chalk.dim(`  File: ${outputPath}`));
+      console.log(chalk.dim(`  Size: ${fileSize}`));
+      if (fullPage) {
+        const dimensions = await page.evaluate(() => ({
+          // eslint-disable-next-line no-undef
+          width: document.documentElement.scrollWidth,
+          // eslint-disable-next-line no-undef
+          height: document.documentElement.scrollHeight,
+        }));
+        console.log(
+          chalk.dim(`  Dimensions: ${dimensions.width}x${dimensions.height}px`),
+        );
+      }
+      console.log('');
+    }
+
+    return {
+      path: outputPath,
+      size: stats.size,
+      url,
+    };
+  } finally {
+    await browser.close();
+  }
+}
+
+/**
+ * Format file size for display
+ */
+function formatFileSize(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+/**
+ * Parse viewport string (e.g., "1920x1080")
+ */
+export function parseViewport(viewportStr) {
+  if (!viewportStr) return {width: 1920, height: 1080};
+  const [width, height] = viewportStr.split('x').map(Number);
+  return {width: width || 1920, height: height || 1080};
+}