smippo 0.0.1

package/src/link-rewriter.js

@@ -0,0 +1,293 @@
+// @flow
+import {load} from 'cheerio';
+import {getRelativePath} from './utils/path.js';
+import {urlToPath, resolveUrl} from './utils/url.js';
+
+/**
+ * Rewrite links in HTML to point to local files
+ */
+export function rewriteLinks(html, pageUrl, urlMap, _options = {}) {
+  const $ = load(html, {decodeEntities: false});
+  const pagePath = urlToPath(pageUrl, _options.structure);
+
+  // Strip all scripts if --no-js flag is set
+  if (_options.noJs) {
+    // Remove script tags
+    $('script').remove();
+    // Remove event handlers
+    $(
+      '[onclick], [onload], [onerror], [onmouseover], [onmouseout], [onkeydown], [onkeyup], [onsubmit], [onchange], [onfocus], [onblur]',
+    ).each((_, el) => {
+      $(el)
+        .removeAttr('onclick')
+        .removeAttr('onload')
+        .removeAttr('onerror')
+        .removeAttr('onmouseover')
+        .removeAttr('onmouseout')
+        .removeAttr('onkeydown')
+        .removeAttr('onkeyup')
+        .removeAttr('onsubmit')
+        .removeAttr('onchange')
+        .removeAttr('onfocus')
+        .removeAttr('onblur');
+    });
+    // Remove module preloads
+    $('link[rel="modulepreload"]').remove();
+  }
+
+  // Helper to get relative path for a URL
+  const getLocalPath = url => {
+    if (!url) return null;
+
+    // Resolve relative URLs (including absolute paths starting with /)
+    const absoluteUrl = resolveUrl(url, pageUrl);
+
+    // Check if we have this URL in our map
+    if (urlMap.has(absoluteUrl)) {
+      const targetPath = urlMap.get(absoluteUrl);
+      return getRelativePath(pagePath, targetPath);
+    }
+
+    // Check without trailing slash
+    const normalizedUrl = absoluteUrl.replace(/\/$/, '');
+    if (urlMap.has(normalizedUrl)) {
+      const targetPath = urlMap.get(normalizedUrl);
+      return getRelativePath(pagePath, targetPath);
+    }
+
+    // Check with index.html appended
+    if (absoluteUrl.endsWith('/')) {
+      const indexUrl = absoluteUrl + 'index.html';
+      if (urlMap.has(indexUrl)) {
+        const targetPath = urlMap.get(indexUrl);
+        return getRelativePath(pagePath, targetPath);
+      }
+    }
+
+    // Check without query string
+    const urlWithoutQuery = absoluteUrl.split('?')[0];
+    if (urlWithoutQuery !== absoluteUrl && urlMap.has(urlWithoutQuery)) {
+      const targetPath = urlMap.get(urlWithoutQuery);
+      return getRelativePath(pagePath, targetPath);
+    }
+
+    return null;
+  };
+
+  // Rewrite <a href>
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (shouldSkipUrl(href)) return;
+
+    const localPath = getLocalPath(href);
+    if (localPath) {
+      $(el).attr('href', localPath);
+    }
+  });
+
+  // Rewrite <link href>
+  $('link[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (shouldSkipUrl(href)) return;
+
+    const localPath = getLocalPath(href);
+    if (localPath) {
+      $(el).attr('href', localPath);
+    }
+  });
+
+  // Rewrite <script src>
+  $('script[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    if (shouldSkipUrl(src)) return;
+
+    const localPath = getLocalPath(src);
+    if (localPath) {
+      $(el).attr('src', localPath);
+    }
+  });
+
+  // Rewrite <img src>
+  $('img[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    if (shouldSkipUrl(src)) return;
+
+    const localPath = getLocalPath(src);
+    if (localPath) {
+      $(el).attr('src', localPath);
+    }
+  });
+
+  // Rewrite <img srcset>
+  $('img[srcset], source[srcset]').each((_, el) => {
+    const srcset = $(el).attr('srcset');
+    if (!srcset) return;
+
+    const newSrcset = rewriteSrcset(srcset, pageUrl, urlMap, _options);
+    $(el).attr('srcset', newSrcset);
+  });
+
+  // Rewrite <video src>, <audio src>, <source src>
+  $('video[src], audio[src], source[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    if (shouldSkipUrl(src)) return;
+
+    const localPath = getLocalPath(src);
+    if (localPath) {
+      $(el).attr('src', localPath);
+    }
+  });
+
+  // Rewrite <video poster>
+  $('video[poster]').each((_, el) => {
+    const poster = $(el).attr('poster');
+    if (shouldSkipUrl(poster)) return;
+
+    const localPath = getLocalPath(poster);
+    if (localPath) {
+      $(el).attr('poster', localPath);
+    }
+  });
+
+  // Rewrite <iframe src>
+  $('iframe[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    if (shouldSkipUrl(src)) return;
+
+    const localPath = getLocalPath(src);
+    if (localPath) {
+      $(el).attr('src', localPath);
+    }
+  });
+
+  // Rewrite <object data>
+  $('object[data]').each((_, el) => {
+    const data = $(el).attr('data');
+    if (shouldSkipUrl(data)) return;
+
+    const localPath = getLocalPath(data);
+    if (localPath) {
+      $(el).attr('data', localPath);
+    }
+  });
+
+  // Rewrite style attributes
+  $('[style]').each((_, el) => {
+    const style = $(el).attr('style');
+    const newStyle = rewriteCssUrls(style, pageUrl, urlMap, pagePath, _options);
+    $(el).attr('style', newStyle);
+  });
+
+  // Rewrite inline <style> tags
+  $('style').each((_, el) => {
+    const css = $(el).html();
+    const newCss = rewriteCssUrls(css, pageUrl, urlMap, pagePath, _options);
+    $(el).html(newCss);
+  });
+
+  return $.html();
+}
+
+/**
+ * Rewrite URLs in CSS content
+ */
+export function rewriteCssUrls(css, baseUrl, urlMap, pagePath, _options = {}) {
+  if (!css) return css;
+
+  // Helper to find local path for a URL
+  const findLocalPath = url => {
+    const absoluteUrl = resolveUrl(url, baseUrl);
+
+    // Direct match
+    if (urlMap.has(absoluteUrl)) {
+      return urlMap.get(absoluteUrl);
+    }
+
+    // Try without query string
+    const urlWithoutQuery = absoluteUrl.split('?')[0];
+    if (urlWithoutQuery !== absoluteUrl && urlMap.has(urlWithoutQuery)) {
+      return urlMap.get(urlWithoutQuery);
+    }
+
+    // Try without trailing slash
+    const normalizedUrl = absoluteUrl.replace(/\/$/, '');
+    if (urlMap.has(normalizedUrl)) {
+      return urlMap.get(normalizedUrl);
+    }
+
+    return null;
+  };
+
+  // Rewrite url() references
+  css = css.replace(/url\s*\(\s*['"]?([^'")]+)['"]?\s*\)/gi, (match, url) => {
+    if (shouldSkipUrl(url)) return match;
+
+    const targetPath = findLocalPath(url);
+    if (targetPath) {
+      const relativePath = getRelativePath(pagePath, targetPath);
+      return `url("${relativePath}")`;
+    }
+
+    return match;
+  });
+
+  // Rewrite @import references
+  css = css.replace(/@import\s+['"]([^'"]+)['"]/gi, (match, url) => {
+    if (shouldSkipUrl(url)) return match;
+
+    const targetPath = findLocalPath(url);
+    if (targetPath) {
+      const relativePath = getRelativePath(pagePath, targetPath);
+      return `@import "${relativePath}"`;
+    }
+
+    return match;
+  });
+
+  return css;
+}
+
+/**
+ * Rewrite srcset attribute
+ */
+function rewriteSrcset(srcset, pageUrl, urlMap, options) {
+  const pagePath = urlToPath(pageUrl, options.structure);
+
+  return srcset
+    .split(',')
+    .map(part => {
+      const [url, descriptor] = part.trim().split(/\s+/);
+      if (shouldSkipUrl(url)) return part;
+
+      const absoluteUrl = resolveUrl(url, pageUrl);
+      if (urlMap.has(absoluteUrl)) {
+        const targetPath = urlMap.get(absoluteUrl);
+        const relativePath = getRelativePath(pagePath, targetPath);
+        return descriptor ? `${relativePath} ${descriptor}` : relativePath;
+      }
+
+      return part;
+    })
+    .join(', ');
+}
+
+/**
+ * Check if URL should be skipped
+ */
+function shouldSkipUrl(url) {
+  if (!url) return true;
+
+  const skipPrefixes = [
+    'javascript:',
+    'mailto:',
+    'tel:',
+    'data:',
+    '#',
+    'blob:',
+    'about:',
+  ];
+
+  return skipPrefixes.some(prefix =>
+    url.trim().toLowerCase().startsWith(prefix),
+  );
+}

package/src/manifest.js

@@ -0,0 +1,163 @@
+import fs from 'fs-extra';
+import path from 'path';
+
+const SMIPPO_DIR = '.smippo';
+const MANIFEST_FILE = 'manifest.json';
+const CACHE_FILE = 'cache.json';
+
+/**
+ * Check if a manifest exists
+ */
+export function manifestExists(outputDir) {
+  const manifestPath = path.join(outputDir, SMIPPO_DIR, MANIFEST_FILE);
+  return fs.existsSync(manifestPath);
+}
+
+/**
+ * Read the manifest file
+ */
+export async function readManifest(outputDir) {
+  const manifestPath = path.join(outputDir, SMIPPO_DIR, MANIFEST_FILE);
+
+  if (!fs.existsSync(manifestPath)) {
+    return null;
+  }
+
+  const content = await fs.readFile(manifestPath, 'utf8');
+  return JSON.parse(content);
+}
+
+/**
+ * Write the manifest file
+ */
+export async function writeManifest(outputDir, manifest) {
+  const smippoDir = path.join(outputDir, SMIPPO_DIR);
+  const manifestPath = path.join(smippoDir, MANIFEST_FILE);
+
+  await fs.ensureDir(smippoDir);
+  await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
+}
+
+/**
+ * Read the cache file
+ */
+export async function readCache(outputDir) {
+  const cachePath = path.join(outputDir, SMIPPO_DIR, CACHE_FILE);
+
+  if (!fs.existsSync(cachePath)) {
+    return {
+      etags: {},
+      lastModified: {},
+      contentTypes: {},
+    };
+  }
+
+  const content = await fs.readFile(cachePath, 'utf8');
+  return JSON.parse(content);
+}
+
+/**
+ * Write the cache file
+ */
+export async function writeCache(outputDir, cache) {
+  const smippoDir = path.join(outputDir, SMIPPO_DIR);
+  const cachePath = path.join(smippoDir, CACHE_FILE);
+
+  await fs.ensureDir(smippoDir);
+  await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), 'utf8');
+}
+
+/**
+ * Create initial manifest
+ */
+export function createManifest(url, options) {
+  return {
+    version: '0.0.1',
+    created: new Date().toISOString(),
+    updated: new Date().toISOString(),
+    rootUrl: url,
+    options: {
+      depth: options.depth,
+      scope: options.scope,
+      stayInDir: options.stayInDir,
+      externalAssets: options.externalAssets,
+      filters: {
+        include: options.include || [],
+        exclude: options.exclude || [],
+      },
+    },
+    stats: {
+      pagesCapt: 0,
+      assetsCapt: 0,
+      totalSize: 0,
+      duration: 0,
+      errors: 0,
+    },
+    pages: [],
+    assets: [],
+  };
+}
+
+/**
+ * Update manifest with captured page
+ */
+export function addPageToManifest(manifest, page) {
+  manifest.pages.push({
+    url: page.url,
+    localPath: page.localPath,
+    status: page.status || 200,
+    captured: new Date().toISOString(),
+    size: page.size,
+    title: page.title,
+  });
+
+  manifest.stats.pagesCapt++;
+  manifest.stats.totalSize += page.size || 0;
+  manifest.updated = new Date().toISOString();
+}
+
+/**
+ * Update manifest with captured asset
+ */
+export function addAssetToManifest(manifest, asset) {
+  manifest.assets.push({
+    url: asset.url,
+    localPath: asset.localPath,
+    mimeType: asset.mimeType,
+    size: asset.size,
+  });
+
+  manifest.stats.assetsCapt++;
+  manifest.stats.totalSize += asset.size || 0;
+  manifest.updated = new Date().toISOString();
+}
+
+/**
+ * Record an error in manifest
+ */
+export function addErrorToManifest(manifest, _url, _error) {
+  manifest.stats.errors++;
+  manifest.updated = new Date().toISOString();
+}
+
+/**
+ * Finalize manifest with duration
+ */
+export function finalizeManifest(manifest, duration) {
+  manifest.stats.duration = duration;
+  manifest.updated = new Date().toISOString();
+}
+
+/**
+ * Get HAR file path
+ */
+export function getHarPath(outputDir) {
+  return path.join(outputDir, SMIPPO_DIR, 'network.har');
+}
+
+/**
+ * Get log file path
+ */
+export function getLogPath(outputDir) {
+  return path.join(outputDir, SMIPPO_DIR, 'log.txt');
+}

package/src/page-capture.js

@@ -0,0 +1,151 @@
+// @flow
+import {extractLinks} from './link-extractor.js';
+
+/**
+ * Capture a single page with all its rendered content
+ */
+export class PageCapture {
+  constructor(page, options = {}) {
+    this.page = page;
+    this.options = options;
+    this.resources = new Map();
+  }
+
+  /**
+   * Capture the page content and resources
+   */
+  async capture(url) {
+    const startTime = Date.now();
+
+    // Set up resource collection
+    this.page.on('response', async response => {
+      await this._collectResource(response);
+    });
+
+    // Navigate to the page
+    try {
+      await this.page.goto(url, {
+        waitUntil: this.options.wait || 'networkidle',
+        timeout: this.options.timeout || 30000,
+      });
+    } catch (error) {
+      // Handle navigation errors but continue if we got some content
+      if (!this.page.url().startsWith('http')) {
+        throw error;
+      }
+    }
+
+    // Additional wait time if specified
+    if (this.options.waitTime > 0) {
+      await this.page.waitForTimeout(this.options.waitTime);
+    }
+
+    // Get the rendered HTML
+    const html = await this.page.content();
+
+    // Get page metadata
+    const title = await this.page.title();
+    const finalUrl = this.page.url();
+
+    // Extract links from the rendered page
+    const links = await extractLinks(this.page, finalUrl, this.options);
+
+    // Take screenshot if requested
+    let screenshot = null;
+    if (this.options.screenshot) {
+      screenshot = await this.page.screenshot({
+        fullPage: true,
+        type: 'png',
+      });
+    }
+
+    // Generate PDF if requested
+    let pdf = null;
+    if (this.options.pdf) {
+      pdf = await this.page.pdf({
+        format: 'A4',
+        printBackground: true,
+      });
+    }
+
+    return {
+      url: finalUrl,
+      requestedUrl: url,
+      html,
+      title,
+      links,
+      resources: this.resources,
+      screenshot,
+      pdf,
+      duration: Date.now() - startTime,
+    };
+  }
+
+  /**
+   * Collect a resource from a network response
+   */
+  async _collectResource(response) {
+    try {
+      const url = response.url();
+      const status = response.status();
+      const headers = response.headers();
+      const contentType = headers['content-type'] || '';
+
+      // Skip failed requests
+      if (status < 200 || status >= 400) return;
+
+      // Skip the main HTML page (captured separately)
+      if (contentType.includes('text/html')) return;
+
+      // Skip data URLs
+      if (url.startsWith('data:')) return;
+
+      // Apply MIME type filters
+      if (this.options.mimeExclude?.length) {
+        if (this._matchesMimeFilter(contentType, this.options.mimeExclude)) {
+          return;
+        }
+      }
+
+      if (this.options.mimeInclude?.length) {
+        if (!this._matchesMimeFilter(contentType, this.options.mimeInclude)) {
+          return;
+        }
+      }
+
+      // Get the response body
+      const body = await response.body().catch(() => null);
+      if (!body) return;
+
+      // Apply size filters
+      if (this.options.maxSize && body.length > this.options.maxSize) return;
+      if (this.options.minSize && body.length < this.options.minSize) return;
+
+      this.resources.set(url, {
+        url,
+        status,
+        contentType,
+        size: body.length,
+        body,
+        headers,
+      });
+    } catch (error) {
+      // Ignore resource collection errors
+    }
+  }
+
+  /**
+   * Check if a content type matches a filter pattern
+   */
+  _matchesMimeFilter(contentType, filters) {
+    const type = contentType.split(';')[0].trim().toLowerCase();
+
+    return filters.some(filter => {
+      filter = filter.toLowerCase();
+      if (filter.endsWith('/*')) {
+        return type.startsWith(filter.slice(0, -1));
+      }
+      return type === filter || type.startsWith(filter + ';');
+    });
+  }
+}