smippo 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Smippo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,116 @@
+<p align="center">
+  <img src="assets/logo.svg" alt="Smippo Logo" width="120" height="100">
+</p>
+
+<h1 align="center">SMIPPO</h1>
+
+<p align="center">
+  <strong>S.M.I.P.P.O.</strong> = Structured Mirroring of Internet Pages and Public Objects
+</p>
+
+<p align="center">
+  Modern website copier — consumes everything fast. Hippos don't nibble. They vacuum.
+</p>
+
+<p align="center">
+  <a href="https://smippo.com"><img src="https://img.shields.io/badge/docs-smippo.com-blue" alt="Documentation"></a>
+  <a href="https://www.npmjs.com/package/smippo"><img src="https://img.shields.io/npm/v/smippo?color=cb0000&label=npm" alt="npm version"></a>
+  <a href="https://www.npmjs.com/package/smippo"><img src="https://img.shields.io/npm/dm/smippo?color=cb0000" alt="npm downloads"></a>
+  <a href="./LICENSE"><img src="https://img.shields.io/npm/l/smippo?color=blue" alt="license"></a>
+  <a href="https://nodejs.org"><img src="https://img.shields.io/node/v/smippo?color=339933" alt="node"></a>
+  <a href="https://github.com/pouyanafisi/smippo/pulls"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg" alt="PRs Welcome"></a>
+</p>
+
+---
+
+**S.M.I.P.P.O.** (Structured Mirroring of Internet Pages and Public Objects) is a command-line website copier and scraper that captures websites exactly as they appear in your browser. Create complete offline mirrors with all assets, styles, and dynamic content preserved. Perfect for website duplication, archiving, and offline browsing.
+
+📚 **[View complete documentation →](https://smippo.com)**
+
+## Quick Start
+
+Install globally:
+
+```bash
+npm install -g smippo
+```
+
+Capture a single page:
+
+```bash
+smippo https://example.com
+```
+
+Mirror a site (3 levels deep):
+
+```bash
+smippo https://example.com --depth 3
+```
+
+Or use without installing:
+
+```bash
+npx smippo https://example.com
+```
+
+## Commands
+
+Smippo provides several commands for different use cases:
+
+- **`smippo <url>`** — Capture and mirror websites with full fidelity
+- **`smippo capture <url>`** — Take screenshots of web pages
+- **`smippo serve <directory>`** — Serve captured sites locally
+- **`smippo continue`** — Resume an interrupted capture
+- **`smippo update`** — Update an existing mirror
+
+Run `smippo` with no arguments to start the interactive guided mode.
+
+## Features
+
+- **🚀 Vacuum Architecture** — Parallel workers consume sites rapidly
+- **📸 Complete Fidelity** — Captures pages exactly as rendered, including CSS-in-JS, dynamic content, and lazy-loaded images
+- **🎯 Smart Filtering** — Filter by URL patterns, MIME types, and file sizes. Respects robots.txt
+- **🌐 Built-in Server** — Serve captured sites locally with directory browsing
+- **📊 HAR Files** — Generates HTTP Archive files for debugging and replay
+- **💻 Programmatic API** — Use Smippo in your Node.js applications
+
+## Documentation
+
+For complete documentation, guides, and API reference, visit **[smippo.com](https://smippo.com)**:
+
+- **[Installation Guide](https://smippo.com/getting-started/installation)** — Detailed installation instructions
+- **[Commands Reference](https://smippo.com/commands)** — All available commands and options
+- **[Configuration](https://smippo.com/configuration)** — Filtering, scope control, performance tuning
+- **[Guides](https://smippo.com/guides)** — Output structure, link rewriting, troubleshooting
+- **[Programmatic API](https://smippo.com/api/programmatic)** — Use Smippo in your Node.js code
+- **[Examples](https://smippo.com/getting-started/examples)** — Real-world use cases
+
+## Requirements
+
+- Node.js 18 or later
+- Chromium (automatically downloaded on first install)
+
+## Contributing
+
+Contributions are welcome! Whether it's bug reports, feature requests, or pull requests — all contributions help make Smippo better.
+
+Please read our [Contributing Guide](CONTRIBUTING.md) for details on development setup, code style guidelines, and the pull request process.
+
+Quick start:
+
+```bash
+git clone https://github.com/pouyanafisi/smippo.git
+cd smippo
+npm install
+npm test
+```
+
+## License
+
+[MIT](./LICENSE) — feel free to use this in your own projects.
+
+## Acknowledgments
+
+- Built with [Playwright](https://playwright.dev/) for reliable browser automation
+- CLI powered by [Commander.js](https://github.com/tj/commander.js) and [@clack/prompts](https://github.com/natemoo-re/clack)
+- Inspired by classic website copiers like [HTTrack](https://www.httrack.com/)

package/bin/smippo.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+
+import {run} from '../src/cli.js';
+
+run();

package/package.json

@@ -0,0 +1,100 @@
+{
+  "name": "smippo",
+  "version": "0.0.1",
+  "description": "S.M.I.P.P.O. — Structured Mirroring of Internet Pages and Public Objects. Modern website copier that captures sites exactly as they appear in your browser.",
+  "main": "src/index.js",
+  "bin": {
+    "smippo": "bin/smippo.js"
+  },
+  "type": "module",
+  "scripts": {
+    "start": "node bin/smippo.js",
+    "dev": "node bin/smippo.js",
+    "test": "mocha",
+    "test:unit": "mocha 'test/**/*.test.js'",
+    "test:integration": "mocha 'test/integration/**/*.test.js'",
+    "test:watch": "mocha --watch",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "typecheck": "tsc --noEmit",
+    "prepare": "husky install",
+    "postinstall": "npx playwright install chromium",
+    "prepublishOnly": "npm run lint && npm run format:check && npm test",
+    "verdaccio": "verdaccio --config verdaccio.yml",
+    "verdaccio:publish": "node verdaccioPublish.js",
+    "publish:github": "npm publish --registry=https://npm.pkg.github.com"
+  },
+  "keywords": [
+    "website",
+    "copier",
+    "mirror",
+    "offline",
+    "browser",
+    "playwright",
+    "crawler",
+    "scraper",
+    "archiver",
+    "smippo",
+    "website-mirror",
+    "site-copier",
+    "web-scraper",
+    "offline-browsing"
+  ],
+  "author": "Pouyan Afisi",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pouyanafisi/smippo.git"
+  },
+  "bugs": {
+    "url": "https://github.com/pouyanafisi/smippo/issues"
+  },
+  "homepage": "https://smippo.dev",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "LICENSE",
+    "README.md"
+  ],
+  "lint-staged": {
+    "*.js": [
+      "eslint --fix",
+      "prettier --write"
+    ],
+    "*.{json,md}": [
+      "prettier --write"
+    ]
+  },
+  "dependencies": {
+    "@clack/prompts": "^0.11.0",
+    "chalk": "^5.3.0",
+    "cheerio": "^1.0.0-rc.12",
+    "cli-progress": "^3.12.0",
+    "commander": "^12.0.0",
+    "figlet": "^1.9.4",
+    "fs-extra": "^11.2.0",
+    "glob": "^10.3.10",
+    "gradient-string": "^3.0.0",
+    "mime-types": "^2.1.35",
+    "minimatch": "^10.1.1",
+    "ora": "^8.0.1",
+    "p-queue": "^8.0.1",
+    "playwright": "^1.41.0",
+    "robots-parser": "^3.0.1"
+  },
+  "devDependencies": {
+    "eslint": "^8.57.0",
+    "husky": "^9.0.0",
+    "lint-staged": "^15.2.0",
+    "mocha": "^10.4.0",
+    "prettier": "^3.2.5"
+  }
+}

package/src/cli.js

@@ -0,0 +1,437 @@
+// @flow
+import {Command} from 'commander';
+import chalk from 'chalk';
+import ora from 'ora';
+import {Crawler} from './crawler.js';
+import {readManifest, manifestExists} from './manifest.js';
+import {version} from './utils/version.js';
+import {
+  showHelp,
+  runInteractiveCapture,
+  shouldRunInteractive,
+} from './interactive.js';
+
+const program = new Command();
+
+export function run() {
+  // Check for help command first
+  const args = process.argv.slice(2);
+  if (args.includes('help') || args.includes('--help') || args.includes('-h')) {
+    showHelp();
+    return;
+  }
+
+  // Check if we should run interactive mode
+  if (shouldRunInteractive(args)) {
+    runInteractiveCapture()
+      .then(options => {
+        return capture(options.url, {
+          output: options.output,
+          depth: options.depth,
+          scope: options.scope,
+          externalAssets: options.externalAssets,
+          static: options.static,
+          screenshot: options.screenshot,
+          workers: options.workers,
+        });
+      })
+      .catch(error => {
+        console.error(chalk.red(`\n✗ Error: ${error.message}`));
+        process.exit(1);
+      });
+    return;
+  }
+
+  program
+    .name('smippo')
+    .description(
+      'Modern website copier powered by Playwright - capture JS-rendered pages for offline viewing',
+    )
+    .version(version);
+
+  // Main capture command
+  program
+    .argument('[url]', 'URL to capture')
+    .option('-o, --output <dir>', 'Output directory', './site')
+    .option('-d, --depth <n>', 'Recursion depth (0 = single page)', '0')
+    .option('--no-crawl', 'Disable link following (same as -d 0)')
+    .option('--dry-run', 'Show what would be captured without downloading')
+
+    // Scope options
+    .option(
+      '-s, --scope <type>',
+      'Link scope: subdomain|domain|tld|all',
+      'domain',
+    )
+    .option('--stay-in-dir', 'Only follow links in same directory or subdirs')
+    .option('--external-assets', 'Capture assets from external domains')
+
+    // Filter options
+    .option('-I, --include <glob...>', 'Include URLs matching pattern')
+    .option('-E, --exclude <glob...>', 'Exclude URLs matching pattern')
+    .option('--mime-include <type...>', 'Include MIME types')
+    .option('--mime-exclude <type...>', 'Exclude MIME types')
+    .option('--max-size <size>', 'Maximum file size (e.g., 10MB)')
+    .option('--min-size <size>', 'Minimum file size (e.g., 1KB)')
+
+    // Browser options
+    .option(
+      '--wait <strategy>',
+      'Wait strategy: networkidle|load|domcontentloaded',
+      'networkidle',
+    )
+    .option('--wait-time <ms>', 'Additional wait time after network idle', '0')
+    .option('--timeout <ms>', 'Page load timeout', '30000')
+    .option('--user-agent <string>', 'Custom user agent')
+    .option('--viewport <WxH>', 'Viewport size', '1920x1080')
+    .option('--device <name>', 'Emulate device (e.g., "iPhone 13")')
+
+    // Network options
+    .option('--proxy <url>', 'Proxy server URL')
+    .option('--cookies <file>', 'Load cookies from JSON file')
+    .option('--headers <json>', 'Custom headers as JSON')
+    .option('--capture-auth', 'Interactive authentication capture')
+
+    // Output options
+    .option(
+      '--structure <type>',
+      'Output structure: original|flat|domain',
+      'original',
+    )
+    .option('--har', 'Generate HAR file', true)
+    .option('--no-har', 'Disable HAR file generation')
+    .option('--screenshot', 'Take screenshot of each page')
+    .option('--pdf', 'Save PDF of each page')
+    .option('--static', 'Remove scripts for static offline viewing')
+    .option('--inline-css', 'Inline CSS into HTML for single-file output')
+
+    // Performance options
+    .option('-w, --workers <n>', 'Parallel workers/pages (default: 8)', '8')
+    .option('-c, --concurrency <n>', 'Alias for --workers', '8')
+    .option('--max-pages <n>', 'Maximum pages to capture')
+    .option('--max-time <seconds>', 'Maximum total time')
+    .option('--rate-limit <ms>', 'Delay between requests')
+
+    // Robots options
+    .option('--ignore-robots', 'Ignore robots.txt')
+    .option('--respect-robots', 'Respect robots.txt', true)
+
+    // Cache options
+    .option('--no-cache', "Don't use cache")
+
+    // Logging options
+    .option('-v, --verbose', 'Verbose output')
+    .option('-q, --quiet', 'Minimal output')
+    .option('--log-file <path>', 'Write logs to file')
+    .option('--debug', 'Debug mode with visible browser')
+
+    // Interaction options
+    .option('--no-interaction', 'Non-interactive mode (for CI/scripts)')
+    .option('-y, --yes', 'Skip prompts, use defaults')
+
+    .action(async (url, options) => {
+      if (!url) {
+        showHelp();
+        return;
+      }
+
+      try {
+        await capture(url, options);
+      } catch (error) {
+        console.error(chalk.red(`\n✗ Error: ${error.message}`));
+        if (options.verbose || options.debug) {
+          console.error(error.stack);
+        }
+        process.exit(1);
+      }
+    });
+
+  // Continue command
+  program
+    .command('continue')
+    .description('Resume an interrupted capture')
+    .option('-o, --output <dir>', 'Output directory', './site')
+    .option('-v, --verbose', 'Verbose output')
+    .action(async options => {
+      try {
+        await continueCapture(options);
+      } catch (error) {
+        console.error(chalk.red(`\n✗ Error: ${error.message}`));
+        process.exit(1);
+      }
+    });
+
+  // Update command
+  program
+    .command('update')
+    .description('Update an existing mirror')
+    .option('-o, --output <dir>', 'Output directory', './site')
+    .option('-v, --verbose', 'Verbose output')
+    .action(async options => {
+      try {
+        await updateCapture(options);
+      } catch (error) {
+        console.error(chalk.red(`\n✗ Error: ${error.message}`));
+        process.exit(1);
+      }
+    });
+
+  // Serve command
+  program
+    .command('serve [directory]')
+    .description('Serve a captured site locally')
+    .option(
+      '-p, --port <port>',
+      'Port to serve on (auto-finds available)',
+      '8080',
+    )
+    .option('-H, --host <host>', 'Host to bind to', '127.0.0.1')
+    .option('-o, --open', 'Open browser automatically')
+    .option('--no-cors', 'Disable CORS headers')
+    .option('-v, --verbose', 'Show all requests')
+    .option('-q, --quiet', 'Minimal output')
+    .action(async (directory, options) => {
+      const {serve} = await import('./server.js');
+      await serve({
+        directory: directory || './site',
+        port: options.port,
+        host: options.host,
+        open: options.open,
+        cors: options.cors,
+        verbose: options.verbose,
+        quiet: options.quiet,
+      });
+    });
+
+  // Screenshot capture command
+  program
+    .command('capture <url>')
+    .description('Take a screenshot of a URL')
+    .option(
+      '-O, --out <file>',
+      'Output file path (auto-generated if not specified)',
+    )
+    .option('-f, --full-page', 'Capture full scrollable page')
+    .option('--format <type>', 'Image format: png|jpeg', 'png')
+    .option('--quality <n>', 'JPEG quality (1-100)', '80')
+    .option('--viewport <WxH>', 'Viewport size', '1920x1080')
+    .option('--device <name>', 'Emulate device (e.g., "iPhone 13", "iPad Pro")')
+    .option('--selector <css>', 'Capture specific element by CSS selector')
+    .option(
+      '--wait <strategy>',
+      'Wait strategy: networkidle|load|domcontentloaded',
+      'networkidle',
+    )
+    .option('--wait-time <ms>', 'Additional wait time after load', '0')
+    .option('--timeout <ms>', 'Page load timeout', '30000')
+    .option('--dark-mode', 'Use dark color scheme')
+    .option('--no-background', 'Transparent background (PNG only)')
+    .option('-v, --verbose', 'Verbose output')
+    .option('-q, --quiet', 'Minimal output')
+    .action(async (url, options) => {
+      try {
+        const {captureScreenshot, parseViewport} =
+          await import('./screenshot.js');
+        await captureScreenshot(url, {
+          output: options.out,
+          fullPage: options.fullPage,
+          format: options.format,
+          quality: options.quality ? parseInt(options.quality, 10) : undefined,
+          viewport: parseViewport(options.viewport),
+          device: options.device,
+          selector: options.selector,
+          wait: options.wait,
+          waitTime: parseInt(options.waitTime, 10),
+          timeout: parseInt(options.timeout, 10),
+          darkMode: options.darkMode,
+          omitBackground: !options.background,
+          verbose: options.verbose,
+          quiet: options.quiet,
+        });
+      } catch (error) {
+        console.error(chalk.red(`\n✗ Error: ${error.message}`));
+        if (options.verbose) {
+          console.error(error.stack);
+        }
+        process.exit(1);
+      }
+    });
+
+  // Help command
+  program
+    .command('help')
+    .description('Show detailed help')
+    .action(() => {
+      showHelp();
+    });
+
+  program.parse();
+}
+
+async function capture(url, options) {
+  const spinner = ora({
+    text: 'Initializing browser...',
+    isSilent: options.quiet,
+  }).start();
+
+  const crawler = new Crawler({
+    url,
+    output: options.output,
+    depth: parseInt(options.depth, 10),
+    scope: options.scope,
+    stayInDir: options.stayInDir,
+    externalAssets: options.externalAssets,
+    include: options.include || [],
+    exclude: options.exclude || [],
+    mimeInclude: options.mimeInclude || [],
+    mimeExclude: options.mimeExclude || [],
+    maxSize: parseSize(options.maxSize),
+    minSize: parseSize(options.minSize),
+    wait: options.wait,
+    waitTime: parseInt(options.waitTime, 10),
+    timeout: parseInt(options.timeout, 10),
+    userAgent: options.userAgent,
+    viewport: parseViewport(options.viewport),
+    device: options.device,
+    proxy: options.proxy,
+    cookies: options.cookies,
+    headers: options.headers ? JSON.parse(options.headers) : {},
+    captureAuth: options.captureAuth,
+    structure: options.structure,
+    har: options.har,
+    screenshot: options.screenshot,
+    pdf: options.pdf,
+    noJs: options.static,
+    inlineCss: options.inlineCss,
+    concurrency: parseInt(options.workers || options.concurrency, 10),
+    maxPages: options.maxPages ? parseInt(options.maxPages, 10) : undefined,
+    maxTime: options.maxTime ? parseInt(options.maxTime, 10) * 1000 : undefined,
+    rateLimit: options.rateLimit ? parseInt(options.rateLimit, 10) : 0,
+    ignoreRobots: options.ignoreRobots,
+    useCache: options.cache,
+    verbose: options.verbose,
+    quiet: options.quiet,
+    logFile: options.logFile,
+    debug: options.debug,
+    dryRun: options.dryRun,
+    spinner,
+  });
+
+  crawler.on('page:start', ({url}) => {
+    spinner.text = `Capturing: ${truncateUrl(url, 60)}`;
+  });
+
+  crawler.on('page:complete', ({url, size}) => {
+    if (options.verbose) {
+      spinner.succeed(
+        `Captured: ${truncateUrl(url, 50)} (${formatSize(size)})`,
+      );
+      spinner.start();
+    }
+  });
+
+  crawler.on('asset:save', ({url, size}) => {
+    if (options.verbose) {
+      spinner.text = `Asset: ${truncateUrl(url, 60)} (${formatSize(size)})`;
+    }
+  });
+
+  crawler.on('error', ({url, error}) => {
+    if (!options.quiet) {
+      spinner.warn(`Failed: ${truncateUrl(url, 50)} - ${error.message}`);
+      spinner.start();
+    }
+  });
+
+  const result = await crawler.start();
+
+  spinner.succeed(chalk.green(`Capture complete!`));
+  console.log('');
+  console.log(chalk.cyan('  Summary:'));
+  console.log(`    Pages captured:  ${result.stats.pagesCapt}`);
+  console.log(`    Assets saved:    ${result.stats.assetsCapt}`);
+  console.log(`    Total size:      ${formatSize(result.stats.totalSize)}`);
+  console.log(`    Duration:        ${formatDuration(result.stats.duration)}`);
+  if (result.stats.errors > 0) {
+    console.log(chalk.yellow(`    Errors:          ${result.stats.errors}`));
+  }
+  console.log('');
+  console.log(`  Output: ${chalk.underline(options.output)}`);
+}
+
+async function continueCapture(options) {
+  if (!manifestExists(options.output)) {
+    throw new Error(
+      'No capture found in the specified directory. Start a new capture first.',
+    );
+  }
+
+  const manifest = await readManifest(options.output);
+  console.log(chalk.cyan(`Continuing capture of ${manifest.rootUrl}...`));
+
+  await capture(manifest.rootUrl, {
+    ...manifest.options,
+    ...options,
+    useCache: true,
+  });
+}
+
+async function updateCapture(options) {
+  if (!manifestExists(options.output)) {
+    throw new Error(
+      'No capture found in the specified directory. Start a new capture first.',
+    );
+  }
+
+  const manifest = await readManifest(options.output);
+  console.log(chalk.cyan(`Updating mirror of ${manifest.rootUrl}...`));
+
+  await capture(manifest.rootUrl, {
+    ...manifest.options,
+    ...options,
+    useCache: true,
+    update: true,
+  });
+}
+
+function parseSize(sizeStr) {
+  if (!sizeStr) return undefined;
+  const match = sizeStr.match(/^(\d+(?:\.\d+)?)\s*(KB|MB|GB|B)?$/i);
+  if (!match) return undefined;
+
+  const num = parseFloat(match[1]);
+  const unit = (match[2] || 'B').toUpperCase();
+
+  const multipliers = {B: 1, KB: 1024, MB: 1024 * 1024, GB: 1024 * 1024 * 1024};
+  return num * (multipliers[unit] || 1);
+}
+
+function parseViewport(viewportStr) {
+  if (!viewportStr) return {width: 1920, height: 1080};
+  const [width, height] = viewportStr.split('x').map(Number);
+  return {width: width || 1920, height: height || 1080};
+}
+
+function truncateUrl(url, maxLen) {
+  if (url.length <= maxLen) return url;
+  return url.slice(0, maxLen - 3) + '...';
+}
+
+function formatSize(bytes) {
+  if (!bytes) return '0 B';
+  const units = ['B', 'KB', 'MB', 'GB'];
+  let i = 0;
+  while (bytes >= 1024 && i < units.length - 1) {
+    bytes /= 1024;
+    i++;
+  }
+  return `${bytes.toFixed(i > 0 ? 1 : 0)} ${units[i]}`;
+}
+
+function formatDuration(ms) {
+  if (ms < 1000) return `${ms}ms`;
+  if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
+  const mins = Math.floor(ms / 60000);
+  const secs = Math.round((ms % 60000) / 1000);
+  return `${mins}m ${secs}s`;
+}