portapack 0.3.2 → 0.3.3

package/docs/.vitepress/config.ts

@@ -52,6 +52,8 @@ export default defineConfig({
         ],
       },
       { text: 'Contributing', link: '/contributing' },
+      { text: 'Architecture', link: '/architecture'},
+      { text: 'Roadmap', link: 'roadmap'}
     ],
 
     sidebar: {

package/docs/architecture.md

@@ -0,0 +1,186 @@
+# PortaPack Architecture
+
+## Overview
+
+PortaPack is a sophisticated tool that bundles entire websites—HTML, CSS, JavaScript, images, and fonts—into self-contained HTML files for offline access. This document outlines the architectural components that make up the system.
+
+```mermaid
+graph TD
+    CLI[CLI Entry Point] --> Options[Options Parser]
+    Options --> Core
+    API[API Entry Point] --> Core
+    
+    subgraph Core ["Core Pipeline"]
+        Parser[HTML Parser] --> Extractor[Asset Extractor]
+        Extractor --> Minifier[Asset Minifier]
+        Minifier --> Packer[HTML Packer]
+    end
+    
+    subgraph Recursive ["Advanced Features"]
+        WebFetcher[Web Fetcher] --> MultipageBundler[Multipage Bundler]
+    end
+    
+    WebFetcher --> Parser
+    Core --> Output[Bundled HTML]
+    MultipageBundler --> Output
+    
+    subgraph Utilities ["Utilities"]
+        Logger[Logger]
+        MimeUtils[MIME Utilities]
+        BuildTimer[Build Timer]
+        Slugify[URL Slugifier]
+    end
+    
+    Logger -.-> CLI
+    Logger -.-> Core
+    Logger -.-> Recursive
+    MimeUtils -.-> Extractor
+    MimeUtils -.-> Parser
+    BuildTimer -.-> CLI
+    BuildTimer -.-> API
+    Slugify -.-> MultipageBundler
+```
+
+## Entry Points
+
+### CLI Interface
+
+The command-line interface provides a convenient way to use PortaPack through terminal commands:
+
+| Component | Purpose |
+|-----------|---------|
+| `cli-entry.ts` | Executable entry point with shebang support |
+| `cli.ts` | Main runner that processes args and manages execution |
+| `options.ts` | Parses command-line arguments and normalizes options |
+
+### API Interface
+
+The programmatic API enables developers to integrate PortaPack into their applications:
+
+| Component | Purpose |
+|-----------|---------|
+| `index.ts` | Exports public functions like `pack()` with TypeScript types |
+| `types.ts` | Defines shared interfaces and types for the entire system |
+
+## Core Pipeline
+
+The bundling process follows a clear four-stage pipeline:
+
+### 1. HTML Parser (`parser.ts`)
+
+The parser reads and analyzes the input HTML:
+- Uses Cheerio for robust HTML parsing
+- Identifies linked assets through element attributes (href, src, etc.)
+- Creates an initial asset list with URLs and inferred types
+- Handles both local file paths and remote URLs
+
+### 2. Asset Extractor (`extractor.ts`)
+
+The extractor resolves and fetches all referenced assets:
+- Resolves relative URLs against the base context
+- Fetches content for all discovered assets
+- Recursively extracts nested assets from CSS (@import, url())
+- Handles protocol-relative URLs and different origins
+- Provides detailed logging of asset discovery
+
+### 3. Asset Minifier (`minifier.ts`)
+
+The minifier reduces the size of all content:
+- Minifies HTML using html-minifier-terser
+- Minifies CSS using clean-css
+- Minifies JavaScript using terser
+- Preserves original content if minification fails
+- Configurable through command-line flags
+
+### 4. HTML Packer (`packer.ts`)
+
+The packer combines everything into a single file:
+- Inlines CSS into `<style>` tags
+- Embeds JavaScript into `<script>` tags
+- Converts binary assets to data URIs
+- Handles srcset attributes properly
+- Ensures proper HTML structure with base tag
+
+## Advanced Features
+
+### Web Fetcher (`web-fetcher.ts`)
+
+For remote content, the web fetcher provides crawling capabilities:
+- Uses Puppeteer for fully-rendered page capture
+- Crawls websites recursively to specified depth
+- Respects same-origin policy by default
+- Manages browser instances efficiently
+- Provides detailed logging of the crawl process
+
+### Multipage Bundler (`bundler.ts`)
+
+For bundling multiple pages into a single file:
+- Combines multiple HTML documents into one
+- Creates a client-side router for navigation
+- Generates a navigation interface
+- Uses slugs for routing between pages
+- Handles page templates and content swapping
+
+## Utilities
+
+### Logger (`logger.ts`)
+- Customizable log levels (debug, info, warn, error)
+- Consistent logging format across the codebase
+- Optional timestamps and colored output
+
+### MIME Utilities (`mime.ts`)
+- Maps file extensions to correct MIME types
+- Categorizes assets by type (CSS, JS, image, font)
+- Provides fallbacks for unknown extensions
+
+### Build Timer (`meta.ts`)
+- Tracks build performance metrics
+- Records asset counts and page counts
+- Captures output size and build duration
+- Collects errors and warnings for reporting
+
+### URL Slugifier (`slugify.ts`)
+- Converts URLs to safe HTML IDs
+- Handles special characters and normalization
+- Prevents slug collisions in multipage bundles
+
+## Asynchronous Processing
+
+PortaPack uses modern async patterns throughout:
+
+- **Promise-based Pipeline**: Each stage returns promises that are awaited
+- **Sequential Processing**: Assets are processed in order to avoid overwhelming resources
+- **Error Boundaries**: Individual asset failures don't break the entire pipeline
+- **Resource Management**: Browser instances and file handles are properly closed
+
+## Build System
+
+PortaPack uses a dual build configuration:
+
+| Build Target | Format | Purpose |
+|--------------|--------|---------|
+| CLI | CommonJS (.cjs) | Works with Node.js and npx |
+| API | ESModule (.js) | Modern import/export support |
+
+TypeScript declarations (.d.ts) are generated for API consumers, and source maps support debugging.
+
+## Current Limitations
+
+### Script Execution Issues
+
+- Inlined scripts with `async`/`defer` attributes lose their intended loading behavior
+- ES Modules with import/export statements may fail after bundling
+- Script execution order can change, breaking dependencies
+
+### Content Limitations
+
+- CORS policies may prevent access to some cross-origin resources
+- Only initially rendered content from SPAs is captured by default
+- Very large sites produce impractically large HTML files
+
+### Technical Constraints
+
+- No streaming API or WebSocket support
+- Service worker capabilities are not preserved
+- Memory pressure with large sites
+- Limited support for authenticated content

package/docs/getting-started.md

@@ -77,8 +77,7 @@ For more specific use cases, you can access individual components:
 import {
   generatePortableHTML,
   generateRecursivePortableHTML,
-  bundleMultiPageHTML,
-  fetchAndPackWebPage,
+  bundleMultiPageHTML
 } from 'portapack';
 
 // Bundle a single HTML file or URL

package/docs/roadmap.md

@@ -0,0 +1,233 @@
+# PortaPack Roadmap
+
+## Version 2: Enhanced Bundling Capabilities
+
+Version 2 focuses on addressing core limitations and expanding compatibility with modern web applications.
+
+### 🎯 Script Execution Enhancement
+
+| Feature | Description | Priority |
+|---------|-------------|----------|
+| **Script Execution Manager** | Preserve loading order of async/defer scripts | High |
+| **Dependency Analysis** | Detect and maintain script dependencies | High |
+| **Script Initialization Sequencing** | Ensure scripts initialize in the correct order | Medium |
+
+**Implementation:** Add a lightweight runtime script that:
+- Maintains a queue of scripts to execute
+- Respects original async/defer behavior
+- Adds proper event listeners for load/error events
+- Enforces correct execution order
+
+### 🔄 Module Support
+
+| Feature | Description | Priority |
+|---------|-------------|----------|
+| **ES Module Transformation** | Convert ES modules to browser-compatible format | High |
+| **Import Resolution** | Resolve and inline imported modules | High |
+| **Export Management** | Create namespace for module exports | Medium |
+
+**Implementation:**
+- Parse import statements using an AST parser
+- Resolve modules relative to source files
+- Rewrite as namespaced functions
+- Create a runtime module resolution system
+
+### 📦 Resource Optimization
+
+| Feature | Description | Priority |
+|---------|-------------|----------|
+| **Bundle Chunking** | Split large bundles into multiple linked files | Medium |
+| **Lazy Loading** | Load assets only when needed | Medium |
+| **Selective Embedding** | Configure thresholds for embedding vs. linking | Low |
+
+**Implementation:**
+- Create a manifest system for chunked resources
+- Add intersection observer for lazy loading
+- Implement size-based decision logic for embedding
+
+### 🖥️ Enhanced SPA Support
+
+| Feature | Description | Priority |
+|---------|-------------|----------|
+| **Rendered State Capture** | Wait for JavaScript rendering before capture | High |
+| **Route Detection** | Automatically discover SPA routes | Medium |
+| **State Interaction** | Simulate user interactions to capture states | Medium |
+
+**Implementation:**
+- Add configurable wait strategies
+- Implement navigation state detection
+- Create event simulation system
+
+### 🔒 Authentication Support
+
+| Feature | Description | Priority |
+|---------|-------------|----------|
+| **Authentication Configuration** | Pass credentials to the crawler | High |
+| **Login Sequence** | Define authentication steps | Medium |
+| **Session Management** | Maintain authenticated state during crawling | Medium |
+
+**Implementation:**
+- Add cookie and header configuration options
+- Create login sequence definition format
+- Implement session persistence
+
+### 💼 Developer Experience
+
+| Feature | Description | Priority |
+|---------|-------------|----------|
+| **Enhanced Diagnostics** | Improved logging and error reporting | Medium |
+| **Preview Server** | Built-in server for bundle testing | Medium |
+| **Bundle Analysis** | Visual report of bundle composition | Low |
+
+**Implementation:**
+- Expand logging with visualization options
+- Create lightweight preview server
+- Implement size and composition analyzer
+
+## Version 3: Universal Content Platform
+
+Version 3 transforms PortaPack from a bundling tool into a comprehensive offline content platform.
+
+### 📱 Cross-Platform Applications
+
+| Platform | Key Features |
+|----------|-------------|
+| **Desktop** (macOS, Windows, Linux) | Native app with system integration, background bundling |
+| **Mobile** (iOS, Android) | Touch-optimized interface, efficient storage management |
+| **Browser Extensions** | One-click saving, context menu integration |
+
+**Implementation:**
+- Use Electron for desktop applications
+- React Native for mobile platforms
+- Extension APIs for major browsers
+
+### ☁️ Synchronization System
+
+| Feature | Description |
+|---------|-------------|
+| **Encrypted Sync** | End-to-end encrypted content synchronization |
+| **Delta Updates** | Bandwidth-efficient incremental synchronization |
+| **Reading State Sync** | Preserve reading position across devices |
+| **Selective Sync** | Choose what content syncs to which devices |
+
+**Implementation:**
+- Create secure synchronization protocol
+- Implement conflict resolution system
+- Build metadata synchronization service
+
+### 🧠 Content Intelligence
+
+| Feature | Description |
+|---------|-------------|
+| **Automatic Summarization** | AI-generated summaries of saved content |
+| **Smart Tagging** | Automatic categorization and organization |
+| **Content Relationships** | Identify connections between saved items |
+| **Content Extraction** | Convert complex pages to readable format |
+
+**Implementation:**
+- Integrate NLP models for content understanding
+- Develop concept extraction algorithms
+- Create relationship graph between content
+- Build advanced readability transformations
+
+### 🔍 Advanced Search & Organization
+
+| Feature | Description |
+|---------|-------------|
+| **Full-Text Search** | Search across all content |
+| **Semantic Search** | Find content by meaning, not just keywords |
+| **Smart Collections** | Automatically organize related content |
+| **Timeline Views** | Chronological content organization |
+
+**Implementation:**
+- Build full-text search engine with indexing
+- Implement vector-based semantic search
+- Create automatic collection generation
+- Develop flexible visualization components
+
+### ✏️ Interactive Features
+
+| Feature | Description |
+|---------|-------------|
+| **Annotation System** | Highlights, notes, and comments |
+| **Content Transformations** | Dark mode, font adjustment, text-to-speech |
+| **Social Sharing** | Controlled sharing with privacy options |
+| **Export Capabilities** | Convert to PDF, EPUB, and other formats |
+
+**Implementation:**
+- Create cross-platform annotation framework
+- Build content adaptation engine
+- Implement secure sharing mechanism
+- Develop export converters for multiple formats
+
+### 🔧 Technical Architecture Expansion
+
+| Component | Purpose |
+|-----------|---------|
+| **Sync Service** | Handle cross-device synchronization |
+| **Auth System** | Manage user accounts and security |
+| **Content Processing** | Pipeline for intelligent content handling |
+| **Analytics** | Privacy-focused usage tracking |
+
+**Implementation:**
+- Build scalable backend services
+- Create secure authentication system
+- Develop modular processing pipeline
+- Implement privacy-preserving analytics
+
+### 🧩 Developer Platform
+
+| Feature | Description |
+|---------|-------------|
+| **Plugin System** | Custom processors and content handlers |
+| **API** | Third-party integration capabilities |
+| **Webhooks** | Automation triggers and notifications |
+| **Theme Engine** | Customization of the reading experience |
+
+**Implementation:**
+- Create plugin architecture with sandboxing
+- Develop comprehensive API documentation
+- Implement webhook system with security
+- Build theme and template engine
+
+### 🤖 Machine Learning Capabilities
+
+| Feature | Description |
+|---------|-------------|
+| **Topic Extraction** | Identify main topics in content |
+| **Entity Recognition** | Detect people, places, organizations |
+| **Recommendation Engine** | Suggest related content |
+| **On-Device Processing** | Local ML for privacy and performance |
+
+**Implementation:**
+- Deploy NLP models for content analysis
+- Create entity linking system
+- Develop recommendation algorithms
+- Optimize ML models for on-device usage
+
+## Development Timeline
+
+### Version 2 Milestones
+
+1. **Phase 1:** Script Execution Manager & Module Support
+2. **Phase 2:** Resource Optimization & SPA Support
+3. **Phase 3** Authentication Support & Developer Experience 
+4. **Phase 4** Stabilization & Release
+
+### Version 3 Phased Approach
+
+1. **Foundation Phase:**
+   - Cross-platform application architecture
+   - Core synchronization system
+   - Basic content intelligence
+
+2. **Expansion Phase:**
+   - Advanced search and organization
+   - Interactive features
+   - Developer platform beta
+
+3. **Intelligence Phase:**
+   - Full machine learning capabilities
+   - Recommendation engine
+   - Advanced content relationships
+

package/examples/main.ts

@@ -13,7 +13,6 @@ import {
   generatePortableHTML,
   bundleMultiPageHTML,
   generateRecursivePortableHTML,
-  fetchAndPackWebPage,
 } from '../src/index'; // 🔧 use '../src/index' for dev, '../dist/index' for built
 
 const TEMP_DIR = path.join(os.tmpdir(), 'portapack-example');
@@ -67,17 +66,6 @@ async function timedBundle(name: string, task: () => Promise<{ html: string; met
     })
   );
 
-  // 🔹 Fetch and display raw HTML from remote site (no metadata)
-  console.log(chalk.cyan('\n⏳ Fetch and Pack Web Page (raw)'));
-  try {
-    const { html, metadata } = await fetchAndPackWebPage('https://getbootstrap.com');
-    const filePath = await writeTempFile('fetched-page.html', html);
-    console.log(chalk.green('✅ Saved fetched HTML:'), `file://${filePath}`);
-    console.log(`📦 Size: ${(metadata.outputSize / 1024).toFixed(2)} KB`);
-  } catch (err) {
-    console.error(chalk.red('❌ Failed to fetch web page:'), err);
-  }
-
   // 🔹 Multi-page manual bundle
   await timedBundle('Multi-Page Site Bundling', async () => {
     const pages = [
@@ -101,15 +89,5 @@ async function timedBundle(name: string, task: () => Promise<{ html: string; met
     generateRecursivePortableHTML('https://getbootstrap.com', 2)
   );
 
-  // 🔹 Broken page test
-  console.log(chalk.cyan('\n⏳ Broken Page Test'));
-  try {
-    const { html, metadata } = await fetchAndPackWebPage('https://example.com/404');
-    const brokenOut = await writeTempFile('broken-page.html', html);
-    console.log(chalk.yellow('⚠️ Page returned something, saved to:'), `file://${brokenOut}`);
-  } catch {
-    console.log(chalk.red('🚫 Could not fetch broken page as expected.'));
-  }
-
   console.log(chalk.gray(`\n📁 Output directory: ${TEMP_DIR}\n`));
 })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "portapack",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "📦 A tool to bundle and minify HTML and all its dependencies into a single portable file.",
   "main": "dist/index.js",
   "module": "dist/index.js",

package/src/core/bundler.ts

@@ -215,8 +215,6 @@ export function bundleMultiPageHTML(pages: PageEntry[], logger?: Logger): string
         `Could not determine a valid base slug for "${page.url}", using generated fallback "${baseSlug}".`
       );
     }
-    // --- END REVISED SLUG LOGIC ---
-
     // --- Collision Handling ---
     let slug = baseSlug;
     let collisionCounter = 1;

package/src/core/extractor.ts

@@ -288,7 +288,6 @@ async function fetchAsset(
       logger?.debug(
         `Workspaceed remote asset ${resolvedUrl.href} (Status: ${response.status}, Type: ${response.headers['content-type'] || 'N/A'}, Size: ${response.data?.byteLength ?? 0} bytes)`
       );
-      // console.log(`[DEBUG fetchAsset] HTTP fetch SUCCESS for: ${resolvedUrl.href}, Status: ${response.status}`); // Keep debug log commented unless needed
       // Return the fetched data as a Node.js Buffer
       return Buffer.from(response.data);
     }
@@ -300,7 +299,6 @@ async function fetchAsset(
         // IMPORTANT: This strips query params and fragments from the URL
         filePath = fileURLToPath(resolvedUrl);
       } catch (e: any) {
-        // console.error(`[DEBUG fetchAsset] fileURLToPath FAILED for: ${resolvedUrl.href}`, e); // Keep debug log commented unless needed
         logger?.error(
           `Could not convert file URL to path: ${resolvedUrl.href}. Error: ${e.message}`
         );
@@ -308,12 +306,9 @@ async function fetchAsset(
       }
 
       const normalizedForLog = path.normalize(filePath);
-      // console.log(`[DEBUG fetchAsset] Attempting readFile with path: "${normalizedForLog}" (Original from URL: "${filePath}")`); // Keep debug log commented unless needed
 
       // Read file content using fs/promises
       const data = await readFile(filePath); // This call uses the mock in tests
-
-      // console.log(`[DEBUG fetchAsset] readFile call SUCCEEDED for path: "${normalizedForLog}". Data length: ${data?.byteLength}`); // Keep debug log commented unless needed
       logger?.debug(`Read local file ${filePath} (${data.byteLength} bytes)`);
       // Return the file content as a Buffer
       return data;

package/tsup.config.ts

@@ -7,7 +7,6 @@
  *   - CommonJS format (`cjs`) for CLI compatibility with Node/npx
  *   - .cjs file extension to avoid ESM interpretation issues
  *   - Shebang (`#!/usr/bin/env node`) for executability
- *   - No type declarations
  *
  * 🔹 API Build:
  *   - ESModule format (`esm`) for modern module usage