portapack 0.2.1

package/src/core/web-fetcher.ts

@@ -0,0 +1,292 @@
+/**
+ * @file src/core/web-fetcher.ts
+ * @description Provides functions for fetching web page content using Puppeteer,
+ * including recursive site crawling capabilities.
+ */
+
+import * as puppeteer from 'puppeteer';
+import * as fs from 'fs/promises';
+import { Logger } from '../utils/logger'; // Assuming logger is in ../utils
+import { BuildResult, PageEntry } from '../types'; // Assuming types are defined here
+import { bundleMultiPageHTML } from './bundler'; // Assuming bundler is here
+
+/**
+ * @typedef {object} CrawlResult
+ * @property {string} url - The URL of the crawled page.
+ * @property {string} html - The HTML content of the crawled page.
+ */
+
+/**
+ * Fetches the rendered HTML content and basic metadata for a single web page URL.
+ * Manages its own browser instance lifecycle (launch and close).
+ *
+ * @param {string} url - The fully qualified URL to fetch.
+ * @param {Logger} [logger] - Optional logger instance for debug/info messages.
+ * @param {number} [timeout=30000] - Navigation timeout in milliseconds.
+ * @returns {Promise<BuildResult>} A promise that resolves with the fetched HTML
+ * and metadata, or rejects on critical errors.
+ * @throws {Error} Throws errors from Puppeteer launch, page creation, or navigation failures.
+ */
+export async function fetchAndPackWebPage(
+    url: string,
+    logger?: Logger,
+    timeout: number = 30000
+): Promise<BuildResult> {
+    let browser: puppeteer.Browser | null = null; // Initialize browser to null
+    const start = Date.now();
+    logger?.debug(`Initiating fetch for single page: ${url}`);
+
+    try {
+        browser = await puppeteer.launch({ headless: true });
+        logger?.debug(`Browser launched for ${url}`);
+        const page = await browser.newPage();
+        logger?.debug(`Page created for ${url}`);
+
+        try {
+            logger?.debug(`Navigating to ${url} with timeout ${timeout}ms`);
+            await page.goto(url, { waitUntil: 'networkidle2', timeout: timeout });
+            logger?.debug(`Navigation successful for ${url}`);
+            const html = await page.content();
+            logger?.debug(`Content retrieved for ${url}`);
+
+            const metadata: BuildResult['metadata'] = {
+                input: url,
+                outputSize: Buffer.byteLength(html, 'utf-8'),
+                assetCount: 0, // Basic fetch doesn't track assets
+                buildTimeMs: Date.now() - start,
+                errors: [], // No errors if we reached this point
+            };
+
+            await page.close(); // Close the page specifically
+            logger?.debug(`Page closed for ${url}`);
+            // await browser.close(); // Close the browser instance
+            logger?.debug(`Browser closed for ${url}`);
+            browser = null; // Ensure browser is marked as closed
+
+            return { html, metadata };
+
+        } catch (pageError: any) {
+            logger?.error(`Error during page processing for ${url}: ${pageError.message}`);
+             // Ensure page is closed even if an error occurred during processing
+            try { await page.close();
+
+             } catch (closeErr) { 
+                throw closeErr;
+            }
+            throw pageError; // Re-throw the original page processing error
+        }
+    } catch (launchError: any) {
+        logger?.error(`Critical error during browser launch or page creation for ${url}: ${launchError.message}`);
+        // Ensure browser is closed if launch succeeded but newPage failed, etc.
+        // Although if launch fails, browser might be null.
+        if (browser) {
+            try { await browser.close(); } catch (closeErr) { /* Ignore browser close error */ }
+        }
+        throw launchError; // Re-throw the original launch/setup error
+    } finally {
+        // Final check: If browser somehow wasn't closed and isn't null, attempt closure.
+        // This handles edge cases where errors might bypass earlier closes.
+        if (browser) {
+             logger?.warn(`Closing browser in final cleanup for ${url}. This might indicate an unusual error path.`);
+             try { await browser.close(); } catch (closeErr) { /* Ignore final browser close error */ }
+        }
+    }
+}
+
+/**
+ * Internal function to recursively crawl a website starting from a given URL.
+ * Uses a single browser instance and manages pages for efficiency during crawl.
+ * Implements Breadth-First Search (BFS) using a queue.
+ *
+ * @private
+ * @param {string} startUrl - The initial URL to start crawling from.
+ * @param {number} maxDepth - The maximum depth of links to follow (1 means only the start URL).
+ * @param {Logger} [logger] - Optional logger instance.
+ * @returns {Promise<PageEntry[]>} A promise resolving to an array of PageEntry objects
+ * containing the URL and HTML for each successfully crawled page.
+ */
+async function crawlWebsite(
+    startUrl: string,
+    maxDepth: number,
+    logger?: Logger
+): Promise<PageEntry[]> {
+    logger?.info(`Starting crawl for ${startUrl} with maxDepth ${maxDepth}`);
+    
+    // Don't even start a browser if maxDepth is 0
+    if (maxDepth <= 0) {
+        logger?.info('maxDepth is 0 or negative, no pages will be crawled.');
+        return [];
+    }
+    
+    const browser = await puppeteer.launch({ headless: true });
+    const visited = new Set<string>();
+    const results: PageEntry[] = [];
+    // Queue stores URLs to visit and their corresponding depth
+    const queue: { url: string; depth: number }[] = [];
+    
+    // Initialize startOrigin for same-origin check
+    let startOrigin: string;
+    try {
+        startOrigin = new URL(startUrl).origin;
+    } catch (e: any) {
+        logger?.error(`Invalid start URL: ${startUrl}. ${e.message}`);
+        await browser.close();
+        return []; // Cannot start crawl with invalid URL
+    }
+
+    // Normalize start URL (remove fragment) and add to queue/visited if depth allows
+    let normalizedStartUrl: string;
+    try {
+        const parsedStartUrl = new URL(startUrl);
+        parsedStartUrl.hash = ''; // Remove fragment for consistent visited checks
+        normalizedStartUrl = parsedStartUrl.href;
+    } catch (e: any) {
+        logger?.error(`Invalid start URL: ${startUrl}. ${e.message}`);
+        await browser.close();
+        return []; // Cannot start crawl with invalid URL
+    }
+
+    visited.add(normalizedStartUrl);
+    queue.push({ url: normalizedStartUrl, depth: 1 });
+    logger?.debug(`Queued initial URL: ${normalizedStartUrl} (depth 1)`);
+
+    while (queue.length > 0) {
+        const { url, depth } = queue.shift()!; // Non-null assertion ok due to queue.length check
+        logger?.info(`Processing: ${url} (depth ${depth})`);
+        let page: puppeteer.Page | null = null;
+
+        try {
+            page = await browser.newPage();
+            // Set a reasonable viewport, sometimes helps with rendering/layout dependent scripts
+            await page.setViewport({ width: 1280, height: 800 });
+            await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+            const html = await page.content();
+
+            // Add successfully fetched page to results
+            // Ensure the object structure matches your PageEntry type definition
+            results.push({ url, html });
+            logger?.debug(`Successfully fetched content for ${url}`);
+
+            // --- Link Discovery ---
+            // Only look for more links if we haven't reached the maximum depth
+            if (depth < maxDepth) {
+                logger?.debug(`Discovering links on ${url} (current depth ${depth}, maxDepth ${maxDepth})`);
+                // Use page.evaluate to get all href attributes directly from the DOM
+                const hrefs = await page.evaluate(() =>
+                    Array.from(document.querySelectorAll('a[href]'), a => a.getAttribute('href'))
+                );
+                logger?.debug(`Found ${hrefs.length} potential hrefs on ${url}`);
+
+                let linksAdded = 0;
+                for (const href of hrefs) {
+                    if (!href) continue; // Skip empty hrefs like href=""
+
+                    let absoluteUrl: string;
+                    try {
+                        // Resolve the href relative to the current page's URL
+                        const resolved = new URL(href, url);
+                        // Remove fragment (#) for visited checks and queueing consistency
+                        resolved.hash = '';
+                        absoluteUrl = resolved.href;
+                    } catch (e) {
+                        // Ignore URLs that fail to parse (e.g., "javascript:void(0)")
+                        logger?.debug(`Ignoring invalid URL syntax: "${href}" on page ${url}`);
+                        continue;
+                    }
+
+                    // --- Filtering and Queueing ---
+                    // 1. Check if it belongs to the same origin as the start URL
+                    // 2. Check if it has already been visited (or is in the queue)
+                    if (absoluteUrl.startsWith(startOrigin) && !visited.has(absoluteUrl)) {
+                        visited.add(absoluteUrl); // Mark as visited *before* adding to queue
+                        queue.push({ url: absoluteUrl, depth: depth + 1 });
+                        linksAdded++;
+                        // logger?.debug(`Queueing: ${absoluteUrl} (depth ${depth + 1})`); // Verbose
+                    } else {
+                        // logger?.debug(`Skipping (external, visited, or invalid): ${absoluteUrl}`); // Verbose
+                    }
+                }
+                logger?.debug(`Added ${linksAdded} new unique internal links to queue from ${url}`);
+            } else {
+                logger?.debug(`Max depth (${maxDepth}) reached, not discovering links on ${url}`);
+            }
+
+        } catch (err: any) {
+            // Log errors encountered during page processing (goto, content, evaluate)
+            logger?.warn(`❌ Failed to process ${url}: ${err.message}`);
+            // Optionally add error details to results or a separate error list if needed
+        } finally {
+            // Ensure the page is closed reliably after processing or error
+            if (page) {
+                try {
+                    await page.close();
+                } catch (pageCloseError: any) {
+                    // Log if closing the page fails, but don't let it stop the crawl
+                    logger?.error(`Failed to close page for ${url}: ${pageCloseError.message}`);
+                }
+            }
+        }
+    } // End while loop
+
+    logger?.info(`Crawl finished. Closing browser.`);
+    await browser.close();
+    logger?.info(`Found ${results.length} pages.`);
+    return results;
+}
+
+/**
+ * Fetches all internal pages of a website recursively starting from a given URL,
+ * bundles them into a single HTML string using the bundler module, and writes
+ * the result to a file.
+ *
+ * @export
+ * @param {string} startUrl - The fully qualified URL to begin crawling from.
+ * @param {string} outputFile - The path where the bundled HTML file should be saved.
+ * @param {number} [maxDepth=1] - The maximum depth to crawl links (default: 1, only the start page).
+ * @returns {Promise<{ pages: number; html: string }>} A promise resolving to an object containing
+ * the number of pages successfully crawled and the final bundled HTML string.
+ * @throws {Error} Throws errors if the crawl initiation fails, bundling fails, or file writing fails.
+ */
+export async function recursivelyBundleSite(
+    startUrl: string,
+    outputFile: string,
+    maxDepth = 1
+): Promise<{ pages: number; html: string }> {
+    // Create a logger instance specifically for this operation
+    const logger = new Logger();
+    logger.info(`Starting recursive site bundle for ${startUrl} to ${outputFile} (maxDepth: ${maxDepth})`);
+
+    try {
+        // Step 1: Crawl the website
+        const pages: PageEntry[] = await crawlWebsite(startUrl, maxDepth, logger);
+
+        if (pages.length === 0) {
+            logger.warn("Crawl completed but found 0 pages. Output file may be empty or reflect an empty bundle.");
+        } else {
+            logger.info(`Crawl successful, found ${pages.length} pages. Starting bundling.`);
+        }
+
+        // Step 2: Bundle the HTML content
+        const bundledHtml = bundleMultiPageHTML(pages, logger); // Passing logger for consistency
+        logger.info(`Bundling complete. Output size: ${Buffer.byteLength(bundledHtml, 'utf-8')} bytes.`);
+
+        // Step 3: Write the bundled HTML to the output file
+        logger.info(`Writing bundled HTML to ${outputFile}`);
+        await fs.writeFile(outputFile, bundledHtml, 'utf-8');
+        logger.info(`Successfully wrote bundled output to ${outputFile}`);
+
+        // Step 4: Return the results
+        return {
+            pages: pages.length,
+            html: bundledHtml
+        };
+    } catch (error: any) {
+        logger.error(`Error during recursive site bundle: ${error.message}`);
+        // Log the stack trace for better debugging if available
+        if (error.stack) {
+            logger.error(`Stack trace: ${error.stack}`);
+        }
+        // Re-throw the error to signal failure to the caller
+        throw error;
+    }
+}

package/src/index.ts

@@ -0,0 +1,262 @@
+/**
+ * @file src/index.ts
+ * @description
+ * Main public API for the PortaPack library.
+ * Provides functions to create portable HTML files from local paths or URLs,
+ * including single-page fetching, recursive site crawling, and multi-page bundling.
+ * It coordinates calls to various core modules (parser, extractor, minifier, packer, web-fetcher, bundler).
+ */
+
+// Core processing modules
+import { parseHTML } from './core/parser';
+import { extractAssets } from './core/extractor';
+import { minifyAssets } from './core/minifier';
+import { packHTML } from './core/packer';
+// Core web fetching modules (imported with aliases)
+import {
+    fetchAndPackWebPage as coreFetchAndPack,
+    recursivelyBundleSite as coreRecursivelyBundleSite
+} from './core/web-fetcher';
+// Core bundler module (for multi-page)
+import { bundleMultiPageHTML as coreBundleMultiPageHTML } from './core/bundler';
+// Utilities
+import { BuildTimer } from './utils/meta';
+import { Logger } from './utils/logger';
+
+// Types
+import type {
+    BundleOptions,
+    BuildResult,
+    PageEntry,
+    BundleMetadata // Type used in return values
+} from './types';
+
+/**
+ * Generates a single, portable HTML file from a local file path or a remote URL.
+ *
+ * - **For local files:** Reads the file, parses it, discovers linked assets (CSS, JS, images, fonts),
+ * fetches/reads asset content, optionally embeds assets as data URIs (default),
+ * optionally minifies HTML/CSS/JS (default), and packs everything into a single HTML string.
+ * - **For remote URLs:** Fetches the HTML content of the single specified URL using the core web-fetcher.
+ * *Note: This does not process/embed assets for single remote URLs; it returns the fetched HTML as-is.*
+ *
+ * @export
+ * @param {string} input - The local file path or remote http(s) URL of the HTML document.
+ * @param {BundleOptions} [options={}] - Configuration options controlling embedding, minification,
+ * base URL, logging level, etc. See `BundleOptions` type for details.
+ * @param {Logger} [loggerInstance] - Optional pre-configured logger instance to use.
+ * @returns {Promise<BuildResult>} A promise resolving to an object containing the final HTML string
+ * and metadata (`BundleMetadata`) about the bundling process (input, size, time, assets, errors).
+ * @throws {Error} Throws errors if file reading, parsing, required asset fetching, or processing fails critically.
+ */
+export async function generatePortableHTML(
+    input: string,
+    options: BundleOptions = {},
+    loggerInstance?: Logger // Allow passing logger
+): Promise<BuildResult> {
+    // Use passed logger or create one based on options. Defaults to LogLevel.INFO.
+    const logger = loggerInstance || new Logger(options.logLevel);
+    logger.info(`Generating portable HTML for: ${input}`);
+    const timer = new BuildTimer(input); // Start timing
+
+    // --- Handle Remote URLs ---
+    const isRemote = /^https?:\/\//i.test(input);
+    if (isRemote) {
+        logger.info(`Input is a remote URL. Fetching page content directly...`);
+        try {
+            // Call the specific public API wrapper for fetching, passing logger and options
+            const result = await fetchAndPackWebPage(input, options, logger);
+            logger.info(`Remote fetch complete. Input: ${input}, Size: ${result.metadata.outputSize} bytes, Time: ${result.metadata.buildTimeMs}ms`);
+            // Forward the result (which includes metadata finalized by fetchAndPackWebPage)
+            return result;
+        } catch (error: any) {
+            logger.error(`Failed to fetch remote URL ${input}: ${error.message}`);
+            throw error; // Re-throw to signal failure
+        }
+    }
+
+    // --- Handle Local Files ---
+    logger.info(`Input is a local file path. Starting local processing pipeline...`);
+    // Determine base path for resolving relative assets. Default to input file's path.
+    const basePath = options.baseUrl || input;
+    logger.debug(`Using base path for asset resolution: ${basePath}`);
+
+    try {
+        // Execute the core processing steps sequentially, passing the logger
+        const parsed = await parseHTML(input, logger);
+        const enriched = await extractAssets(parsed, options.embedAssets ?? true, basePath, logger);
+        const minified = await minifyAssets(enriched, options, logger); // Pass full options
+        const finalHtml = packHTML(minified, logger);
+
+        // Finalize metadata using the timer.
+        // Pass assetCount calculated from the final list of processed assets.
+        const metadata = timer.finish(finalHtml, {
+            assetCount: minified.assets.length
+            // FIX: Removed incorrect attempt to get errors from logger
+            // Errors collected by the timer itself (via timer.addError) will be included automatically.
+        });
+        logger.info(`Local processing complete. Input: ${input}, Size: ${metadata.outputSize} bytes, Assets: ${metadata.assetCount}, Time: ${metadata.buildTimeMs}ms`);
+        if (metadata.errors && metadata.errors.length > 0) {
+             logger.warn(`Completed with ${metadata.errors.length} warning(s) logged in metadata.`);
+        }
+
+        // Include any errors collected *by the timer* in the result
+        return { html: finalHtml, metadata };
+
+    } catch (error: any) {
+        logger.error(`Error during local processing for ${input}: ${error.message}`);
+        throw error; // Re-throw critical errors
+    }
+}
+
+/**
+ * Crawls a website starting from a given URL up to a specified depth,
+ * bundles all discovered internal HTML pages into a single multi-page file,
+ * and returns the result.
+ *
+ * @export
+ * @param {string} url - The entry point URL to start crawling. Must be http or https.
+ * @param {number} [depth=1] - The maximum link depth to crawl (1 means only the starting page).
+ * @param {BundleOptions} [options={}] - Configuration options. Primarily used for `logLevel`.
+ * @param {Logger} [loggerInstance] - Optional pre-configured logger instance to use.
+ * @returns {Promise<BuildResult>} A promise resolving to an object containing the bundled multi-page HTML string
+ * and metadata (`BundleMetadata`) about the crawl and bundling process.
+ * @throws {Error} Throws errors if the initial URL is invalid, crawling fails, or bundling fails.
+ */
+export async function generateRecursivePortableHTML(
+    url: string,
+    depth = 1,
+    options: BundleOptions = {},
+    loggerInstance?: Logger // Allow passing logger
+): Promise<BuildResult> {
+    // Use passed logger or create one
+    const logger = loggerInstance || new Logger(options.logLevel);
+    logger.info(`Generating recursive portable HTML for: ${url}, Max Depth: ${depth}`);
+    const timer = new BuildTimer(url);
+
+    if (!/^https?:\/\//i.test(url)) {
+        const errMsg = `Invalid input URL for recursive bundling: ${url}. Must start with http(s)://`;
+        logger.error(errMsg);
+        throw new Error(errMsg);
+    }
+
+    // Placeholder output path for core function (consider removing if core doesn't need it)
+    const internalOutputPathPlaceholder = `${new URL(url).hostname}_recursive.html`;
+
+    try {
+        // Call the CORE recursive site function
+        // Assuming coreRecursivelyBundleSite accepts logger as an optional argument
+        const { html, pages } = await coreRecursivelyBundleSite(url, internalOutputPathPlaceholder, depth); // Pass logger if accepted
+        logger.info(`Recursive crawl complete. Discovered and bundled ${pages} pages.`);
+
+        // Finalize metadata
+        timer.setPageCount(pages); // Store page count
+        const metadata = timer.finish(html, {
+            assetCount: 0, // NOTE: Asset count across multiple pages is not currently aggregated.
+            pagesBundled: pages
+            // TODO: Potentially collect errors from the core function if it returns them
+        });
+        logger.info(`Recursive bundling complete. Input: ${url}, Size: ${metadata.outputSize} bytes, Pages: ${metadata.pagesBundled}, Time: ${metadata.buildTimeMs}ms`);
+        if (metadata.errors && metadata.errors.length > 0) {
+             logger.warn(`Completed with ${metadata.errors.length} warning(s) logged in metadata.`);
+        }
+
+        return { html, metadata };
+
+    } catch (error: any) {
+        logger.error(`Error during recursive generation for ${url}: ${error.message}`);
+        if (error.cause instanceof Error) { // Log cause if it's an Error
+            logger.error(`Cause: ${error.cause.message}`);
+        }
+        throw error; // Re-throw
+    }
+}
+
+/**
+ * Fetches the HTML content of a single remote URL using the core web-fetcher.
+ * This function acts as a public wrapper, primarily adding standardized timing and metadata.
+ * It does *not* process assets within the fetched HTML.
+ *
+ * @export
+ * @param {string} url - The remote http(s) URL to fetch.
+ * @param {BundleOptions} [options={}] - Configuration options, mainly for `logLevel`.
+ * @param {Logger} [loggerInstance] - Optional pre-configured logger instance to use.
+ * @returns {Promise<BuildResult>} A promise resolving to the BuildResult containing the fetched HTML
+ * and metadata from the fetch operation.
+ * @throws {Error} Propagates errors directly from the core fetching function or if URL is invalid.
+ */
+export async function fetchAndPackWebPage(
+    url: string,
+    options: BundleOptions = {},
+    loggerInstance?: Logger // Allow passing an existing logger
+): Promise<BuildResult> {
+    // Use the passed logger or create a new one based on options
+    const logger = loggerInstance || new Logger(options.logLevel);
+    logger.info(`Workspaceing single remote page: ${url}`);
+    const timer = new BuildTimer(url);
+
+    if (!/^https?:\/\//i.test(url)) {
+        const errMsg = `Invalid input URL for fetchAndPackWebPage: ${url}. Must start with http(s)://`;
+        logger.error(errMsg);
+        throw new Error(errMsg);
+    }
+
+    try {
+        // Call the CORE fetcher function, passing the logger
+        // Assuming coreFetchAndPack accepts logger as an optional second argument
+        const result = await coreFetchAndPack(url, logger);
+
+        // Finalize metadata using timer and data from the core result
+        const metadata = timer.finish(result.html, {
+            // Use assetCount and errors from core metadata if available
+            assetCount: result.metadata?.assetCount ?? 0,
+            errors: result.metadata?.errors ?? [] // Ensure errors array exists
+        });
+        logger.info(`Single page fetch complete. Input: ${url}, Size: ${metadata.outputSize} bytes, Assets: ${metadata.assetCount}, Time: ${metadata.buildTimeMs}ms`);
+        if (metadata.errors && metadata.errors.length > 0) {
+             logger.warn(`Completed with ${metadata.errors.length} warning(s) logged in metadata.`);
+        }
+
+        // Return HTML from core result, but use metadata finalized by this wrapper
+        return { html: result.html, metadata };
+    } catch (error: any) {
+        logger.error(`Error during single page fetch for ${url}: ${error.message}`);
+        throw error; // Re-throw original error
+    }
+}
+
+/**
+ * Bundles an array of pre-fetched/generated HTML pages into a single static HTML file
+ * using `<template>` tags and a simple client-side hash-based router.
+ * This function does not perform any asset processing on the input HTML strings.
+ *
+ * @export
+ * @param {PageEntry[]} pages - An array of page objects, where each object has a `url` (for slug generation)
+ * and `html` (the content for that page).
+ * @param {BundleOptions} [options={}] - Configuration options, primarily used for `logLevel`.
+ * @param {Logger} [loggerInstance] - Optional pre-configured logger instance.
+ * @returns {string} A single HTML string representing the bundled multi-page document.
+ */
+export function bundleMultiPageHTML(
+    pages: PageEntry[],
+    options: BundleOptions = {},
+    loggerInstance?: Logger // Allow passing an existing logger
+): string {
+    // Use passed logger or create a new one
+    const logger = loggerInstance || new Logger(options.logLevel);
+    logger.info(`Bundling ${pages.length} provided pages into multi-page HTML...`);
+
+    try {
+        // Directly call the CORE multi-page bundler function, passing the logger
+        // Assuming coreBundleMultiPageHTML accepts logger as an optional second argument
+        const bundledHtml = coreBundleMultiPageHTML(pages, logger);
+        logger.info(`Multi-page bundling complete.`);
+        return bundledHtml;
+    } catch (error: any) {
+         logger.error(`Error during multi-page bundling: ${error.message}`);
+         throw error; // Re-throw error
+    }
+}
+
+// Optional: Export core types directly from index for easier consumption?
+export * from './types';