wayback-dl 1.0.0

package/README.md

@@ -0,0 +1,73 @@
+# wayback-dl
+
+Download archived websites from the [Internet Archive Wayback Machine](https://web.archive.org/).
+
+## Install
+
+```bash
+npm install -g wayback-dl
+```
+
+Or run without installing:
+
+```bash
+npx wayback-dl https://example.com
+```
+
+## Usage
+
+```bash
+wayback-dl <url> [options]
+```
+
+### Examples
+
+```bash
+# Download a site to ./websites/example.com/
+wayback-dl https://example.com
+
+# Download to a custom directory
+wayback-dl https://example.com -d ./my-archive/
+
+# Use 20 concurrent downloads
+wayback-dl https://example.com -c 20
+
+# Only download images
+wayback-dl https://example.com -o "/\.(jpg|png|gif)$/i"
+
+# Exclude certain paths
+wayback-dl https://example.com -x "/ads/"
+
+# Restrict by timestamp (YYYYMMDDHHmmss)
+wayback-dl https://example.com -f 20060101000000 -t 20101231235959
+
+# List files as JSON without downloading
+wayback-dl https://example.com -l
+```
+
+### Options
+
+| Flag | Description |
+|------|-------------|
+| `-d, --directory <path>` | Output directory (default: `./websites/{domain}/`) |
+| `-c, --concurrency <n>` | Parallel downloads (default: 5) |
+| `-f, --from <timestamp>` | Only snapshots from this timestamp |
+| `-t, --to <timestamp>` | Only snapshots up to this timestamp |
+| `-o, --only <filter>` | Only download URLs matching filter (use `/regex/` for regex) |
+| `-x, --exclude <filter>` | Exclude URLs matching filter |
+| `-a, --all` | Include non-200 responses (errors, redirects) |
+| `-s, --all-timestamps` | Download all snapshot versions |
+| `-e, --exact-url` | Download only the exact URL, not the full site |
+| `-l, --list` | List files as JSON without downloading |
+| `--max-pages <n>` | Max CDX API pages (default: 100) |
+| `--overwrite` | Re-download existing files |
+| `--retry <n>` | Max retries per file (default: 5) |
+| `--no-color` | Disable colored output |
+
+## Requirements
+
+- Node.js 18+
+
+## License
+
+MIT

package/bin/wayback-dl.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+import '../dist/cli.js';

package/dist/cdx.d.ts

@@ -0,0 +1,24 @@
+import type { Snapshot } from './types.js';
+export interface CdxOptions {
+    url: string;
+    fromTimestamp?: number;
+    toTimestamp?: number;
+    all?: boolean;
+    page?: number;
+}
+/**
+ * Fetch a single page of snapshots from the CDX API.
+ */
+export declare function fetchCdxPage(options: CdxOptions): Promise<Snapshot[]>;
+/**
+ * Fetch all snapshots for a URL, paginating through the CDX API.
+ */
+export declare function fetchAllSnapshots(baseUrl: string, options?: {
+    exactUrl?: boolean;
+    fromTimestamp?: number;
+    toTimestamp?: number;
+    all?: boolean;
+    maxPages?: number;
+    onPage?: (page: number, count: number) => void;
+}): Promise<Snapshot[]>;
+//# sourceMappingURL=cdx.d.ts.map

package/dist/cdx.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cdx.d.ts","sourceRoot":"","sources":["../src/cdx.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAK3C,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AA2BD;;GAEG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CA4D3E;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAC3C,GACL,OAAO,CAAC,QAAQ,EAAE,CAAC,CAkCrB"}

package/dist/cdx.js

@@ -0,0 +1,113 @@
+import { retry } from './retry.js';
+const CDX_BASE = 'https://web.archive.org/cdx/search/xd';
+function buildCdxUrl(options) {
+    const params = new URLSearchParams({
+        output: 'json',
+        url: options.url,
+        fl: 'timestamp,original',
+        collapse: 'digest',
+        gzip: 'false',
+    });
+    if (!options.all) {
+        params.set('filter', 'statuscode:200');
+    }
+    if (options.fromTimestamp !== undefined && options.fromTimestamp !== 0) {
+        params.set('from', String(options.fromTimestamp));
+    }
+    if (options.toTimestamp !== undefined && options.toTimestamp !== 0) {
+        params.set('to', String(options.toTimestamp));
+    }
+    if (options.page !== undefined) {
+        params.set('page', String(options.page));
+    }
+    return `${CDX_BASE}?${params.toString()}`;
+}
+/**
+ * Fetch a single page of snapshots from the CDX API.
+ */
+export async function fetchCdxPage(options) {
+    const url = buildCdxUrl(options);
+    let response;
+    try {
+        response = await retry(async () => {
+            const res = await fetch(url);
+            if (res.status === 400 || res.status === 404) {
+                return '';
+            }
+            if (!res.ok) {
+                throw new Error(`CDX API error: ${res.status} ${res.statusText}`);
+            }
+            return res.text();
+        }, {
+            retries: 5,
+            baseDelay: 1000,
+            maxDelay: 30000,
+        });
+    }
+    catch {
+        return [];
+    }
+    if (response === '') {
+        return [];
+    }
+    let json;
+    try {
+        json = JSON.parse(response);
+    }
+    catch {
+        return [];
+    }
+    if (!Array.isArray(json) || json.length === 0) {
+        return [];
+    }
+    const header = json[0];
+    if (Array.isArray(header) &&
+        header[0] === 'timestamp' &&
+        header[1] === 'original') {
+        json.shift();
+    }
+    const snapshots = [];
+    for (const row of json) {
+        if (Array.isArray(row) && row.length >= 2) {
+            snapshots.push({
+                timestamp: String(row[0]),
+                url: String(row[1]),
+            });
+        }
+    }
+    return snapshots;
+}
+/**
+ * Fetch all snapshots for a URL, paginating through the CDX API.
+ */
+export async function fetchAllSnapshots(baseUrl, options = {}) {
+    const maxPages = options.maxPages ?? 100;
+    const all = [];
+    const baseOptions = {
+        url: baseUrl,
+        fromTimestamp: options.fromTimestamp,
+        toTimestamp: options.toTimestamp,
+        all: options.all,
+    };
+    const firstPage = await fetchCdxPage(baseOptions);
+    all.push(...firstPage);
+    options.onPage?.(0, firstPage.length);
+    if (options.exactUrl || firstPage.length === 0) {
+        return all;
+    }
+    const wildcardUrl = baseUrl.endsWith('/') ? `${baseUrl}*` : `${baseUrl}/*`;
+    for (let page = 0; page < maxPages; page++) {
+        const pageSnapshots = await fetchCdxPage({
+            ...baseOptions,
+            url: wildcardUrl,
+            page,
+        });
+        if (pageSnapshots.length === 0) {
+            break;
+        }
+        all.push(...pageSnapshots);
+        options.onPage?.(page + 1, pageSnapshots.length);
+    }
+    return all;
+}
+//# sourceMappingURL=cdx.js.map

package/dist/cdx.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cdx.js","sourceRoot":"","sources":["../src/cdx.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,MAAM,QAAQ,GAAG,uCAAuC,CAAC;AAUzD,SAAS,WAAW,CAAC,OAAmB;IACtC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;QACjC,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,EAAE,EAAE,oBAAoB;QACxB,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,OAAO;KACd,CAAC,CAAC;IAEH,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;QACvE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,WAAW,KAAK,CAAC,EAAE,CAAC;QACnE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IAChD,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,GAAG,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAmB;IACpD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEjC,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CACpB,KAAK,IAAI,EAAE;YACT,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC7C,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC,EACD;YACE,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,IAAI;YACf,QAAQ,EAAE,KAAK;SAChB,CACF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACvB,IACE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QACrB,MAAM,CAAC,CAAC,CAAC,KAAK,WAAW;QACzB,MAAM,CAAC,CAAC,CAAC,KAAK,UAAU,EACxB,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC1C,SAAS,CAAC,IAAI,CAAC;gBACb,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACzB,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aACpB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAe,EACf,UAOI,EAAE;IAEN,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;IACzC,MAAM,GAAG,GAAe,EAAE,CAAC;IAE3B,MAAM,WAAW,GAA6B;QAC5C,GAAG,EAAE,OAAO;QACZ,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACvB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC;IAC3E,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;QAC3C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC;YACvC,GAAG,WAAW;YACd,GAAG,EAAE,WAAW;YAChB,IAAI;SACL,CAAC,CAAC;QACH,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM;QACR,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;QAC3B,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,GAAG,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,105 @@
+import { createRequire } from 'node:module';
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { download, listFiles } from './downloader.js';
+import { createProgressBar, createSnapshotProgressBar, printSummary } from './progress.js';
+import { getBackupName } from './utils.js';
+const require = createRequire(import.meta.url);
+const pkg = require('../package.json');
+const program = new Command();
+program
+    .name('wayback-dl')
+    .description('Download archived websites from the Internet Archive Wayback Machine')
+    .version(pkg.version)
+    .argument('<url>', 'Website URL to download (e.g. https://example.com)')
+    .option('-d, --directory <path>', 'Output directory (default: ./websites/{domain}/)')
+    .option('-c, --concurrency <n>', 'Number of parallel downloads', '5')
+    .option('-f, --from <timestamp>', 'Only snapshots from this timestamp (e.g. 20060716231334)')
+    .option('-t, --to <timestamp>', 'Only snapshots up to this timestamp')
+    .option('-o, --only <filter>', 'Only download URLs matching filter (use /pattern/ for regex)')
+    .option('-x, --exclude <filter>', 'Exclude URLs matching filter')
+    .option('-a, --all', 'Include non-200 responses (errors, redirects)')
+    .option('-s, --all-timestamps', 'Download all snapshot versions')
+    .option('-e, --exact-url', 'Download only the exact URL, not the full site')
+    .option('-l, --list', 'List files as JSON without downloading')
+    .option('--max-pages <n>', 'Max CDX API pages to fetch', '100')
+    .option('--overwrite', 'Re-download existing files')
+    .option('--retry <n>', 'Max retries per file', '5')
+    .option('--no-color', 'Disable colored output')
+    .action(async (url, opts) => {
+    const useColors = opts.color !== false;
+    const options = {
+        baseUrl: url,
+        directory: opts.directory,
+        exactUrl: opts.exactUrl === true,
+        allTimestamps: opts.allTimestamps === true,
+        fromTimestamp: opts.from ? parseInt(String(opts.from), 10) : undefined,
+        toTimestamp: opts.to ? parseInt(String(opts.to), 10) : undefined,
+        onlyFilter: opts.only,
+        excludeFilter: opts.exclude,
+        all: opts.all === true,
+        maxPages: parseInt(String(opts.maxPages), 10) || 100,
+        concurrency: parseInt(String(opts.concurrency), 10) || 5,
+        overwrite: opts.overwrite === true,
+        maxRetries: parseInt(String(opts.retry), 10) || 5,
+    };
+    try {
+        if (opts.list === true) {
+            const files = await listFiles(options);
+            console.log(JSON.stringify(files, null, 2));
+            return;
+        }
+        const backupPath = options.directory ?? `websites/${getBackupName(url)}/`;
+        console.log(chalk.cyan(`Downloading ${chalk.bold(url)} to ${chalk.bold(backupPath)} from Wayback Machine.`));
+        console.log();
+        const snapshotBar = createSnapshotProgressBar(useColors);
+        snapshotBar.start();
+        const snapshotBarUpdate = (page, count) => {
+            snapshotBar.update(`... page ${page + 1} (${count} snapshots)`);
+        };
+        const progress = createProgressBar(useColors);
+        const { files, stats } = await download({
+            ...options,
+            onSnapshotPage: snapshotBarUpdate,
+            onFileListReady: (count) => {
+                snapshotBar.stop();
+                if (count > 0) {
+                    console.log(chalk.cyan(`${count} files to download.`));
+                    console.log();
+                    progress.start(count);
+                }
+            },
+            onProgress: (s) => {
+                progress.update(s.downloaded + s.skipped + s.errors, s.downloaded, s.skipped, s.errors);
+            },
+        });
+        if (files.length === 0) {
+            snapshotBar.stop();
+            console.log(chalk.yellow('No files to download.'));
+            console.log(chalk.gray('Possible reasons:'));
+            console.log(chalk.gray('  • Site is not in Wayback Machine Archive.'));
+            if (options.fromTimestamp) {
+                console.log(chalk.gray('  • From timestamp too far in the future.'));
+            }
+            if (options.toTimestamp) {
+                console.log(chalk.gray('  • To timestamp too far in the past.'));
+            }
+            if (options.onlyFilter) {
+                console.log(chalk.gray(`  • Only filter too restrictive: ${options.onlyFilter}`));
+            }
+            if (options.excludeFilter) {
+                console.log(chalk.gray(`  • Exclude filter too wide: ${options.excludeFilter}`));
+            }
+            return;
+        }
+        progress.stop();
+        printSummary(stats);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.error(chalk.red(`\nError: ${message}`));
+        process.exit(1);
+    }
+});
+program.parse();
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAwB,CAAC;AAE9D,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,sEAAsE,CAAC;KACnF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;KACpB,QAAQ,CAAC,OAAO,EAAE,oDAAoD,CAAC;KACvE,MAAM,CAAC,wBAAwB,EAAE,kDAAkD,CAAC;KACpF,MAAM,CAAC,uBAAuB,EAAE,8BAA8B,EAAE,GAAG,CAAC;KACpE,MAAM,CAAC,wBAAwB,EAAE,0DAA0D,CAAC;KAC5F,MAAM,CAAC,sBAAsB,EAAE,qCAAqC,CAAC;KACrE,MAAM,CAAC,qBAAqB,EAAE,8DAA8D,CAAC;KAC7F,MAAM,CAAC,wBAAwB,EAAE,8BAA8B,CAAC;KAChE,MAAM,CAAC,WAAW,EAAE,+CAA+C,CAAC;KACpE,MAAM,CAAC,sBAAsB,EAAE,gCAAgC,CAAC;KAChE,MAAM,CAAC,iBAAiB,EAAE,gDAAgD,CAAC;KAC3E,MAAM,CAAC,YAAY,EAAE,wCAAwC,CAAC;KAC9D,MAAM,CAAC,iBAAiB,EAAE,4BAA4B,EAAE,KAAK,CAAC;KAC9D,MAAM,CAAC,aAAa,EAAE,4BAA4B,CAAC;KACnD,MAAM,CAAC,aAAa,EAAE,sBAAsB,EAAE,GAAG,CAAC;KAClD,MAAM,CAAC,YAAY,EAAE,wBAAwB,CAAC;KAC9C,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAkD,EAAE,EAAE;IAChF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC;IAEvC,MAAM,OAAO,GAAG;QACd,OAAO,EAAE,GAAG;QACZ,SAAS,EAAE,IAAI,CAAC,SAA+B;QAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,KAAK,IAAI;QAChC,aAAa,EAAE,IAAI,CAAC,aAAa,KAAK,IAAI;QAC1C,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QACtE,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QAChE,UAAU,EAAE,IAAI,CAAC,IAA0B;QAC3C,aAAa,EAAE,IAAI,CAAC,OAA6B;QACjD,GAAG,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI;QACtB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG;QACpD,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC;QACxD,SAAS,EAAE,IAAI,CAAC,SAAS,KAAK,IAAI;QAClC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC;KAClD,CAAC;IAEF,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,IAAI,YAAY,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;QAC1E,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAChG,CAAC;QACF,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,MAAM,WAAW,GAAG,yBAAyB,CAAC,SAAS,CAAC,CAAC;QACzD,WAAW,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;YACxD,WAAW,CAAC,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC,KAAK,KAAK,aAAa,CAAC,CAAC;QAClE,CAAC,CAAC;QAEF,MAAM,QAAQ,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAE9C,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC;YACtC,GAAG,OAAO;YACV,cAAc,EAAE,iBAAiB;YACjC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;gBACzB,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAC,CAAC;oBACvD,OAAO,CAAC,GAAG,EAAE,CAAC;oBACd,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE;gBAChB,QAAQ,CAAC,MAAM,CACb,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,MAAM,EACnC,CAAC,CAAC,UAAU,EACZ,CAAC,CAAC,OAAO,EACT,CAAC,CAAC,MAAM,CACT,CAAC;YACJ,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,WAAW,CAAC,IAAI,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC,CAAC;YACvE,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC,CAAC;YACvE,CAAC;YACD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC,CAAC;YACnE,CAAC;YACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,oCAAoC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACpF,CAAC;YACD,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gCAAgC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;YACnF,CAAC;YACD,OAAO;QACT,CAAC;QAED,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChB,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/downloader.d.ts

@@ -0,0 +1,19 @@
+import type { FileToDownload, DownloadOptions } from './types.js';
+import type { ProgressStats } from './progress.js';
+export interface DownloadResult {
+    files: FileToDownload[];
+    stats: ProgressStats;
+}
+export type ProgressCallback = (stats: {
+    downloaded: number;
+    skipped: number;
+    errors: number;
+}) => void;
+export type SnapshotPageCallback = (page: number, count: number) => void;
+export declare function download(options: DownloadOptions & {
+    onProgress?: ProgressCallback;
+    onSnapshotPage?: SnapshotPageCallback;
+    onFileListReady?: (count: number) => void;
+}): Promise<DownloadResult>;
+export declare function listFiles(options: DownloadOptions): Promise<FileToDownload[]>;
+//# sourceMappingURL=downloader.d.ts.map

package/dist/downloader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"downloader.d.ts","sourceRoot":"","sources":["../src/downloader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,cAAc,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAU5E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,KAAK,IAAI,CAAC;AAEX,MAAM,MAAM,oBAAoB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;AAyDzE,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,eAAe,GAAG;IACzB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAC3C,GACA,OAAO,CAAC,cAAc,CAAC,CAmEzB;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAcnF"}

package/dist/downloader.js

@@ -0,0 +1,118 @@
+import PQueue from 'p-queue';
+import { fetchAllSnapshots } from './cdx.js';
+import { downloadFile } from './file-manager.js';
+import { getBackupName, extractFileId, tidyBytes, matchOnlyFilter, matchExcludeFilter, } from './utils.js';
+function getBackupPath(directory, baseUrl) {
+    if (directory) {
+        return directory.endsWith('/') ? directory : `${directory}/`;
+    }
+    return `websites/${getBackupName(baseUrl)}/`;
+}
+function curateFileList(snapshots, options) {
+    const map = new Map();
+    for (const { timestamp, url } of snapshots) {
+        const fileId = extractFileId(url);
+        if (fileId === null) {
+            continue;
+        }
+        const tidiedId = fileId === '' ? fileId : tidyBytes(fileId);
+        if (matchExcludeFilter(url, options.excludeFilter)) {
+            continue;
+        }
+        if (!matchOnlyFilter(url, options.onlyFilter)) {
+            continue;
+        }
+        const key = options.allTimestamps ? `${timestamp}/${tidiedId}` : tidiedId;
+        const existing = map.get(key);
+        if (existing) {
+            if (!options.allTimestamps && existing.timestamp < timestamp) {
+                map.set(key, { fileUrl: url, timestamp });
+            }
+        }
+        else {
+            map.set(key, { fileUrl: url, timestamp });
+        }
+    }
+    const result = [];
+    for (const [key, { fileUrl, timestamp }] of map) {
+        result.push({
+            fileId: key,
+            fileUrl,
+            timestamp,
+        });
+    }
+    return options.allTimestamps
+        ? result
+        : result.sort((a, b) => (b.timestamp > a.timestamp ? 1 : -1));
+}
+export async function download(options) {
+    const backupPath = getBackupPath(options.directory, options.baseUrl);
+    const concurrency = options.concurrency ?? 5;
+    const maxRetries = options.maxRetries ?? 5;
+    const snapshots = await fetchAllSnapshots(options.baseUrl, {
+        exactUrl: options.exactUrl,
+        fromTimestamp: options.fromTimestamp,
+        toTimestamp: options.toTimestamp,
+        all: options.all,
+        maxPages: options.maxPages,
+        onPage: options.onSnapshotPage,
+    });
+    const files = curateFileList(snapshots, {
+        onlyFilter: options.onlyFilter,
+        excludeFilter: options.excludeFilter,
+        allTimestamps: options.allTimestamps,
+    });
+    const stats = {
+        total: files.length,
+        downloaded: 0,
+        skipped: 0,
+        errors: 0,
+        retries: 0,
+        durationMs: 0,
+    };
+    options.onFileListReady?.(files.length);
+    const startTime = Date.now();
+    const queue = new PQueue({ concurrency });
+    await Promise.all(files.map((file) => queue.add(async () => {
+        const result = await downloadFile(file, {
+            backupPath,
+            overwrite: options.overwrite,
+            maxRetries,
+            saveErrors: options.all,
+            onRetry: () => {
+                stats.retries++;
+            },
+        });
+        if (result.status === 'downloaded') {
+            stats.downloaded++;
+        }
+        else if (result.status === 'skipped') {
+            stats.skipped++;
+        }
+        else {
+            stats.errors++;
+        }
+        options.onProgress?.({
+            downloaded: stats.downloaded,
+            skipped: stats.skipped,
+            errors: stats.errors,
+        });
+    })));
+    stats.durationMs = Date.now() - startTime;
+    return { files, stats };
+}
+export async function listFiles(options) {
+    const snapshots = await fetchAllSnapshots(options.baseUrl, {
+        exactUrl: options.exactUrl,
+        fromTimestamp: options.fromTimestamp,
+        toTimestamp: options.toTimestamp,
+        all: options.all,
+        maxPages: options.maxPages,
+    });
+    return curateFileList(snapshots, {
+        onlyFilter: options.onlyFilter,
+        excludeFilter: options.excludeFilter,
+        allTimestamps: options.allTimestamps,
+    });
+}
+//# sourceMappingURL=downloader.js.map

package/dist/downloader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"downloader.js","sourceRoot":"","sources":["../src/downloader.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EACL,aAAa,EACb,aAAa,EACb,SAAS,EACT,eAAe,EACf,kBAAkB,GACnB,MAAM,YAAY,CAAC;AAgBpB,SAAS,aAAa,CAAC,SAA6B,EAAE,OAAe;IACnE,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC;IAC/D,CAAC;IACD,OAAO,YAAY,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC;AAC/C,CAAC;AAED,SAAS,cAAc,CACrB,SAAqB,EACrB,OAGC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkD,CAAC;IAEtE,KAAK,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,IAAI,SAAS,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAI,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACnD,SAAS;QACX,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9C,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC1E,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC,SAAS,GAAG,SAAS,EAAE,CAAC;gBAC7D,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,KAAK,MAAM,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC;QAChD,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,GAAG;YACX,OAAO;YACP,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,aAAa;QAC1B,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,OAIC;IAED,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACrE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;IAE3C,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE;QACzD,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,OAAO,CAAC,cAAc;KAC/B,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,cAAc,CAAC,SAAS,EAAE;QACtC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,aAAa,EAAE,OAAO,CAAC,aAAa;KACrC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAkB;QAC3B,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,UAAU,EAAE,CAAC;QACb,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC;QACT,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,CAAC;KACd,CAAC;IAEF,OAAO,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;IAE1C,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjB,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE;YACtC,UAAU;YACV,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,UAAU;YACV,UAAU,EAAE,OAAO,CAAC,GAAG;YACvB,OAAO,EAAE,GAAG,EAAE;gBACZ,KAAK,CAAC,OAAO,EAAE,CAAC;YAClB,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACnC,KAAK,CAAC,UAAU,EAAE,CAAC;QACrB,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACvC,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;QACD,OAAO,CAAC,UAAU,EAAE,CAAC;YACnB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;IACL,CAAC,CAAC,CACH,CACF,CAAC;IAEF,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAE1C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAwB;IACtD,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE;QACzD,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC,SAAS,EAAE;QAC/B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,aAAa,EAAE,OAAO,CAAC,aAAa;KACrC,CAAC,CAAC;AACL,CAAC"}

package/dist/file-manager.d.ts

@@ -0,0 +1,15 @@
+import type { FileToDownload } from './types.js';
+export interface FileManagerOptions {
+    backupPath: string;
+    overwrite?: boolean;
+    maxRetries?: number;
+    saveErrors?: boolean;
+    onRetry?: (url: string, error: Error, attempt: number) => void;
+}
+export interface DownloadResult {
+    status: 'downloaded' | 'skipped' | 'error';
+    filePath?: string;
+    error?: Error;
+}
+export declare function downloadFile(file: FileToDownload, options: FileManagerOptions): Promise<DownloadResult>;
+//# sourceMappingURL=file-manager.d.ts.map

package/dist/file-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-manager.d.ts","sourceRoot":"","sources":["../src/file-manager.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAMjD,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAChE;AA0DD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,OAAO,CAAC;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,cAAc,CAAC,CAsEzB"}

package/dist/file-manager.js

@@ -0,0 +1,119 @@
+import { mkdir, writeFile, stat, unlink, rename } from 'node:fs/promises';
+import { retry } from './retry.js';
+import { sanitizePath } from './utils.js';
+const WAYBACK_BASE = 'https://web.archive.org/web';
+function buildWaybackUrl(timestamp, originalUrl) {
+    return `${WAYBACK_BASE}/${timestamp}id_/${originalUrl}`;
+}
+function resolvePaths(backupPath, fileId, fileUrl) {
+    const pathElements = fileId.split('/');
+    const isDir = fileUrl.endsWith('/') || !pathElements[pathElements.length - 1]?.includes('.');
+    let dirPath;
+    let filePath;
+    if (fileId === '') {
+        dirPath = backupPath;
+        filePath = `${backupPath}index.html`;
+    }
+    else if (isDir) {
+        dirPath = `${backupPath}${pathElements.join('/')}`;
+        filePath = `${dirPath}/index.html`;
+    }
+    else {
+        dirPath = `${backupPath}${pathElements.slice(0, -1).join('/')}`;
+        filePath = `${backupPath}${pathElements.join('/')}`;
+    }
+    if (process.platform === 'win32') {
+        dirPath = sanitizePath(dirPath);
+        filePath = sanitizePath(filePath);
+    }
+    return { dirPath, filePath };
+}
+async function ensureDirectory(dirPath) {
+    try {
+        await mkdir(dirPath, { recursive: true });
+    }
+    catch (err) {
+        const error = err;
+        if (error.code === 'EEXIST') {
+            const existingPath = error.path ?? dirPath;
+            const stats = await stat(existingPath);
+            if (stats.isFile()) {
+                const tempPath = `${existingPath}.temp`;
+                const permanentPath = `${existingPath}/index.html`;
+                await rename(existingPath, tempPath);
+                await mkdir(existingPath, { recursive: true });
+                await rename(tempPath, permanentPath);
+                await ensureDirectory(dirPath);
+            }
+        }
+        else {
+            throw err;
+        }
+    }
+}
+export async function downloadFile(file, options) {
+    const { backupPath, overwrite = false, maxRetries = 5, saveErrors = false } = options;
+    const { dirPath, filePath } = resolvePaths(backupPath, file.fileId, file.fileUrl);
+    if (!overwrite) {
+        try {
+            await stat(filePath);
+            return { status: 'skipped', filePath };
+        }
+        catch {
+            // File doesn't exist, proceed
+        }
+    }
+    await ensureDirectory(dirPath);
+    try {
+        await retry(async () => {
+            const url = buildWaybackUrl(file.timestamp, file.fileUrl);
+            const response = await fetch(url, {
+                headers: { 'Accept-Encoding': 'identity' },
+            });
+            if (!response.ok && !saveErrors) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            const buffer = Buffer.from(await response.arrayBuffer());
+            await writeFile(filePath, buffer);
+        }, {
+            retries: maxRetries,
+            baseDelay: 1000,
+            maxDelay: 30000,
+            onRetry: (err, attempt) => {
+                options.onRetry?.(file.fileUrl, err, attempt);
+            },
+        });
+    }
+    catch (err) {
+        try {
+            const s = await stat(filePath);
+            if (s.size === 0 && !saveErrors) {
+                await unlink(filePath);
+            }
+        }
+        catch {
+            // Ignore
+        }
+        return {
+            status: 'error',
+            filePath,
+            error: err instanceof Error ? err : new Error(String(err)),
+        };
+    }
+    try {
+        const s = await stat(filePath);
+        if (s.size === 0 && !saveErrors) {
+            await unlink(filePath);
+            return {
+                status: 'error',
+                filePath,
+                error: new Error('Downloaded file was empty'),
+            };
+        }
+    }
+    catch {
+        // Ignore
+    }
+    return { status: 'downloaded', filePath };
+}
+//# sourceMappingURL=file-manager.js.map

package/dist/file-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-manager.js","sourceRoot":"","sources":["../src/file-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG1E,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,YAAY,GAAG,6BAA6B,CAAC;AAUnD,SAAS,eAAe,CAAC,SAAiB,EAAE,WAAmB;IAC7D,OAAO,GAAG,YAAY,IAAI,SAAS,OAAO,WAAW,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,YAAY,CACnB,UAAkB,EAClB,MAAc,EACd,OAAe;IAEf,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAE7F,IAAI,OAAe,CAAC;IACpB,IAAI,QAAgB,CAAC;IAErB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAClB,OAAO,GAAG,UAAU,CAAC;QACrB,QAAQ,GAAG,GAAG,UAAU,YAAY,CAAC;IACvC,CAAC;SAAM,IAAI,KAAK,EAAE,CAAC;QACjB,OAAO,GAAG,GAAG,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnD,QAAQ,GAAG,GAAG,OAAO,aAAa,CAAC;IACrC,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,GAAG,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAChE,QAAQ,GAAG,GAAG,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACtD,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAChC,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAAe;IAC5C,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,GAA4B,CAAC;QAC3C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,IAAI,OAAO,CAAC;YAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;YACvC,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,GAAG,YAAY,OAAO,CAAC;gBACxC,MAAM,aAAa,GAAG,GAAG,YAAY,aAAa,CAAC;gBACnD,MAAM,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;gBACrC,MAAM,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC/C,MAAM,MAAM,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBACtC,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAoB,EACpB,OAA2B;IAE3B,MAAM,EAAE,UAAU,EAAE,SAAS,GAAG,KAAK,EAAE,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACtF,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,YAAY,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAElF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,8BAA8B;QAChC,CAAC;IACH,CAAC;IAED,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;IAE/B,IAAI,CAAC;QACH,MAAM,KAAK,CACT,KAAK,IAAI,EAAE;YACT,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;aAC3C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC,EACD;YACE,OAAO,EAAE,UAAU;YACnB,SAAS,EAAE,IAAI;YACf,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;gBACxB,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;SACF,CACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,OAAO;YACL,MAAM,EAAE,OAAO;YACf,QAAQ;YACR,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;SAC3D,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACvB,OAAO;gBACL,MAAM,EAAE,OAAO;gBACf,QAAQ;gBACR,KAAK,EAAE,IAAI,KAAK,CAAC,2BAA2B,CAAC;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,SAAS;IACX,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;AAC5C,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { download, listFiles } from './downloader.js';
+export type { DownloadOptions, FileToDownload, Snapshot } from './types.js';
+export type { DownloadResult, ProgressCallback, SnapshotPageCallback } from './downloader.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACtD,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC5E,YAAY,EAAE,cAAc,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/index.js

@@ -0,0 +1,2 @@
+export { download, listFiles } from './downloader.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/progress.d.ts

@@ -0,0 +1,30 @@
+export interface ProgressCallbacks {
+    onStart?: (total: number) => void;
+    onFileStart?: (url: string) => void;
+    onFileComplete?: (url: string, status: 'downloaded' | 'skipped') => void;
+    onFileError?: (url: string, error: Error) => void;
+    onFileRetry?: (url: string, attempt: number) => void;
+    onComplete?: (stats: ProgressStats) => void;
+}
+export interface ProgressStats {
+    total: number;
+    downloaded: number;
+    skipped: number;
+    errors: number;
+    retries: number;
+    durationMs: number;
+}
+export declare function createProgressBar(useColors?: boolean): {
+    start: (total: number) => void;
+    update: (completed: number, downloaded: number, skipped: number, errors: number) => void;
+    stop: () => void;
+    increment: (status: 'downloaded' | 'skipped' | 'error') => void;
+    log: (message: string, type?: 'success' | 'warn' | 'error' | 'info') => void;
+};
+export declare function createSnapshotProgressBar(useColors?: boolean): {
+    start: () => void;
+    update: (message: string) => void;
+    stop: () => void;
+};
+export declare function printSummary(stats: ProgressStats): void;
+//# sourceMappingURL=progress.d.ts.map

package/dist/progress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../src/progress.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,iBAAiB;IAChC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,cAAc,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,SAAS,KAAK,IAAI,CAAC;IACzE,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAClD,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACrD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wBAAgB,iBAAiB,CAAC,SAAS,UAAO,GAAG;IACnD,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC/B,MAAM,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACzF,IAAI,EAAE,MAAM,IAAI,CAAC;IACjB,SAAS,EAAE,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,OAAO,KAAK,IAAI,CAAC;IAChE,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,KAAK,IAAI,CAAC;CAC9E,CA6EA;AAED,wBAAgB,yBAAyB,CAAC,SAAS,UAAO,GAAG;IAC3D,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,IAAI,EAAE,MAAM,IAAI,CAAC;CAClB,CA8BA;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAYvD"}

package/dist/progress.js

@@ -0,0 +1,115 @@
+import cliProgress from 'cli-progress';
+import chalk from 'chalk';
+export function createProgressBar(useColors = true) {
+    let multibar = null;
+    let mainBar = null;
+    let total = 0;
+    let downloaded = 0;
+    let skipped = 0;
+    let errors = 0;
+    let startTime = 0;
+    const format = useColors
+        ? `  ${chalk.cyan('{bar}')} | {percentage}% | {value}/{total} files | ${chalk.green('✓')} {downloaded} ${chalk.gray('○')} {skipped} ${chalk.red('✗')} {errors}`
+        : '  {bar} | {percentage}% | {value}/{total} files | ✓ {downloaded} ○ {skipped} ✗ {errors}';
+    return {
+        start(t) {
+            total = t;
+            startTime = Date.now();
+            multibar = new cliProgress.MultiBar({
+                clearOnComplete: false,
+                hideCursor: true,
+                format,
+                barCompleteChar: useColors ? chalk.green('█') : '█',
+                barIncompleteChar: useColors ? chalk.gray('░') : '░',
+            });
+            mainBar = multibar.create(total, 0, {
+                downloaded: 0,
+                skipped: 0,
+                errors: 0,
+            });
+            mainBar.setTotal(total);
+        },
+        update(completed, d, s, e) {
+            downloaded = d;
+            skipped = s;
+            errors = e;
+            mainBar?.update(completed, { downloaded: d, skipped: s, errors: e });
+        },
+        increment(status) {
+            if (status === 'downloaded')
+                downloaded++;
+            else if (status === 'skipped')
+                skipped++;
+            else
+                errors++;
+            const completed = downloaded + skipped + errors;
+            mainBar?.increment(1, { downloaded, skipped, errors });
+        },
+        stop() {
+            multibar?.stop();
+            multibar = null;
+            mainBar = null;
+        },
+        log(message, type = 'info') {
+            if (mainBar) {
+                multibar?.log(`${getPrefix(type)} ${message}\n`);
+            }
+            else {
+                console.log(`${getPrefix(type)} ${message}`);
+            }
+        },
+    };
+    function getPrefix(t) {
+        if (!useColors) {
+            return t === 'error' ? '[ERROR]' : t === 'warn' ? '[WARN]' : t === 'success' ? '[OK]' : '[INFO]';
+        }
+        switch (t) {
+            case 'success':
+                return chalk.green('✓');
+            case 'warn':
+                return chalk.yellow('⚠');
+            case 'error':
+                return chalk.red('✗');
+            default:
+                return chalk.blue('ℹ');
+        }
+    }
+}
+export function createSnapshotProgressBar(useColors = true) {
+    let bar = null;
+    const format = useColors
+        ? `  ${chalk.cyan('Fetching snapshots')} {message}`
+        : '  Fetching snapshots {message}';
+    return {
+        start() {
+            bar = new cliProgress.SingleBar({
+                format,
+                barCompleteChar: ' ',
+                barIncompleteChar: ' ',
+                hideCursor: true,
+            }, cliProgress.Presets.shades_classic);
+            bar.start(1, 0, { message: '' });
+        },
+        update(message) {
+            bar?.update(1, { message });
+        },
+        stop() {
+            bar?.stop();
+            bar = null;
+        },
+    };
+}
+export function printSummary(stats) {
+    const durationSec = (stats.durationMs / 1000).toFixed(2);
+    console.log();
+    console.log(chalk.bold('Download complete'));
+    console.log(chalk.gray(`  Duration: ${durationSec}s`));
+    console.log(chalk.green(`  Downloaded: ${stats.downloaded}`));
+    if (stats.skipped > 0) {
+        console.log(chalk.gray(`  Skipped (already exist): ${stats.skipped}`));
+    }
+    if (stats.errors > 0) {
+        console.log(chalk.red(`  Errors: ${stats.errors}`));
+    }
+}
+//# sourceMappingURL=progress.js.map

package/dist/progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["../src/progress.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAoB1B,MAAM,UAAU,iBAAiB,CAAC,SAAS,GAAG,IAAI;IAOhD,IAAI,QAAQ,GAAgC,IAAI,CAAC;IACjD,IAAI,OAAO,GAAiC,IAAI,CAAC;IACjD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,MAAM,GAAG,SAAS;QACtB,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,8CAA8C,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW;QAC/J,CAAC,CAAC,yFAAyF,CAAC;IAE9F,OAAO;QACL,KAAK,CAAC,CAAS;YACb,KAAK,GAAG,CAAC,CAAC;YACV,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,QAAQ,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC;gBAClC,eAAe,EAAE,KAAK;gBACtB,UAAU,EAAE,IAAI;gBAChB,MAAM;gBACN,eAAe,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;gBACnD,iBAAiB,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;aACrD,CAAC,CAAC;YACH,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE;gBAClC,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,CAAC;aACV,CAAC,CAAC;YACH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,CAAC,SAAiB,EAAE,CAAS,EAAE,CAAS,EAAE,CAAS;YACvD,UAAU,GAAG,CAAC,CAAC;YACf,OAAO,GAAG,CAAC,CAAC;YACZ,MAAM,GAAG,CAAC,CAAC;YACX,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,SAAS,CAAC,MAA0C;YAClD,IAAI,MAAM,KAAK,YAAY;gBAAE,UAAU,EAAE,CAAC;iBACrC,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,EAAE,CAAC;;gBACpC,MAAM,EAAE,CAAC;YACd,MAAM,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;YAChD,OAAO,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,IAAI;YACF,QAAQ,EAAE,IAAI,EAAE,CAAC;YACjB,QAAQ,GAAG,IAAI,CAAC;YAChB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;QAED,GAAG,CAAC,OAAe,EAAE,OAA8C,MAAM;YACvE,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;KACF,CAAC;IAEF,SAAS,SAAS,CAAC,CAAwC;QACzD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnG,CAAC;QACD,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC1B,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3B,KAAK,OAAO;gBACV,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxB;gBACE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,SAAS,GAAG,IAAI;IAKxD,IAAI,GAAG,GAAiC,IAAI,CAAC;IAE7C,MAAM,MAAM,GAAG,SAAS;QACtB,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,YAAY;QACnD,CAAC,CAAC,gCAAgC,CAAC;IAErC,OAAO;QACL,KAAK;YACH,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CAC7B;gBACE,MAAM;gBACN,eAAe,EAAE,GAAG;gBACpB,iBAAiB,EAAE,GAAG;gBACtB,UAAU,EAAE,IAAI;aACjB,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;YACF,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,CAAC,OAAe;YACpB,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9B,CAAC;QAED,IAAI;YACF,GAAG,EAAE,IAAI,EAAE,CAAC;YACZ,GAAG,GAAG,IAAI,CAAC;QACb,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAoB;IAC/C,MAAM,WAAW,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,WAAW,GAAG,CAAC,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,iBAAiB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC9D,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC"}

package/dist/retry.d.ts

@@ -0,0 +1,11 @@
+export interface RetryOptions {
+    retries?: number;
+    baseDelay?: number;
+    maxDelay?: number;
+    onRetry?: (error: Error, attempt: number) => void;
+}
+/**
+ * Execute an async function with exponential backoff on failure.
+ */
+export declare function retry<T>(fn: () => Promise<T>, options?: RetryOptions): Promise<T>;
+//# sourceMappingURL=retry.d.ts.map

package/dist/retry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACnD;AAqBD;;GAEG;AACH,wBAAsB,KAAK,CAAC,CAAC,EAC3B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,CAAC,CAAC,CAsBZ"}

package/dist/retry.js

@@ -0,0 +1,43 @@
+const DEFAULT_RETRIES = 5;
+const DEFAULT_BASE_DELAY = 1000;
+const DEFAULT_MAX_DELAY = 30000;
+/**
+ * Sleep for a given number of milliseconds.
+ */
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Add jitter to a delay to avoid thundering herd.
+ */
+function jitter(delay) {
+    const jitterRange = delay * 0.2;
+    return delay + (Math.random() * 2 - 1) * jitterRange;
+}
+/**
+ * Execute an async function with exponential backoff on failure.
+ */
+export async function retry(fn, options = {}) {
+    const retries = options.retries ?? DEFAULT_RETRIES;
+    const baseDelay = options.baseDelay ?? DEFAULT_BASE_DELAY;
+    const maxDelay = options.maxDelay ?? DEFAULT_MAX_DELAY;
+    const onRetry = options.onRetry;
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            lastError = err instanceof Error ? err : new Error(String(err));
+            if (attempt === retries) {
+                throw lastError;
+            }
+            const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
+            const delayWithJitter = Math.max(0, jitter(delay));
+            onRetry?.(lastError, attempt + 1);
+            await sleep(delayWithJitter);
+        }
+    }
+    throw lastError;
+}
+//# sourceMappingURL=retry.js.map

package/dist/retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.js","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAOA,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAEhC;;GAEG;AACH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,SAAS,MAAM,CAAC,KAAa;IAC3B,MAAM,WAAW,GAAG,KAAK,GAAG,GAAG,CAAC;IAChC,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC;AACvD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,EAAoB,EACpB,UAAwB,EAAE;IAE1B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACvD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAEhC,IAAI,SAA4B,CAAC;IACjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBACxB,MAAM,SAAS,CAAC;YAClB,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;YACnE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACnD,OAAO,EAAE,CAAC,SAAS,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YAClC,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,MAAM,SAAS,CAAC;AAClB,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,45 @@
+/**
+ * A single snapshot from the Wayback Machine CDX API.
+ */
+export interface Snapshot {
+    timestamp: string;
+    url: string;
+}
+/**
+ * A file to download, derived from a snapshot after filtering and deduplication.
+ */
+export interface FileToDownload {
+    fileId: string;
+    fileUrl: string;
+    timestamp: string;
+}
+/**
+ * Options for the download orchestrator.
+ */
+export interface DownloadOptions {
+    baseUrl: string;
+    directory?: string;
+    exactUrl?: boolean;
+    allTimestamps?: boolean;
+    fromTimestamp?: number;
+    toTimestamp?: number;
+    onlyFilter?: string;
+    excludeFilter?: string;
+    all?: boolean;
+    maxPages?: number;
+    concurrency?: number;
+    overwrite?: boolean;
+    maxRetries?: number;
+}
+/**
+ * Progress event emitted during download.
+ */
+export interface ProgressEvent {
+    type: 'started' | 'completed' | 'skipped' | 'error' | 'retry';
+    fileUrl?: string;
+    filePath?: string;
+    error?: Error;
+    completed: number;
+    total: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,OAAO,GAAG,OAAO,CAAC;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACf"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/utils.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Extract the domain/host from a URL for use as backup name.
+ * e.g. "https://example.com/path" -> "example.com"
+ */
+export declare function getBackupName(url: string): string;
+/**
+ * Extract the file path from a full URL (path after domain).
+ * e.g. "https://example.com/foo/bar.html" -> "foo/bar.html"
+ */
+export declare function extractFileId(url: string): string | null;
+/**
+ * Fix invalid UTF-8 bytes (similar to Ruby tidy_bytes).
+ * Interprets string as latin1 and re-encodes to UTF-8.
+ */
+export declare function tidyBytes(str: string): string;
+/**
+ * Parse a filter string into a RegExp or null for literal matching.
+ * Supports /pattern/ and /pattern/i notation.
+ */
+export declare function parseFilterToRegex(filter: string): RegExp | null;
+/**
+ * Check if a URL matches the "only" filter (include).
+ */
+export declare function matchOnlyFilter(fileUrl: string, onlyFilter: string | undefined): boolean;
+/**
+ * Check if a URL matches the "exclude" filter.
+ */
+export declare function matchExcludeFilter(fileUrl: string, excludeFilter: string | undefined): boolean;
+/**
+ * Sanitize a path for the filesystem (Windows-incompatible chars).
+ */
+export declare function sanitizePath(path: string): string;
+//# sourceMappingURL=utils.d.ts.map

package/dist/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMjD;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAexD;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO7C;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAehE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CASxF;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAS9F;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEjD"}

package/dist/utils.js

@@ -0,0 +1,99 @@
+/**
+ * Extract the domain/host from a URL for use as backup name.
+ * e.g. "https://example.com/path" -> "example.com"
+ */
+export function getBackupName(url) {
+    if (url.includes('//')) {
+        const parts = url.split('/');
+        return parts[2] ?? url;
+    }
+    return url;
+}
+/**
+ * Extract the file path from a full URL (path after domain).
+ * e.g. "https://example.com/foo/bar.html" -> "foo/bar.html"
+ */
+export function extractFileId(url) {
+    if (!url.includes('/')) {
+        return null;
+    }
+    const parts = url.split('/');
+    const pathParts = parts.slice(3);
+    if (pathParts.length === 0) {
+        return '';
+    }
+    const joined = pathParts.join('/');
+    try {
+        return decodeURIComponent(joined);
+    }
+    catch {
+        return tidyBytes(joined);
+    }
+}
+/**
+ * Fix invalid UTF-8 bytes (similar to Ruby tidy_bytes).
+ * Interprets string as latin1 and re-encodes to UTF-8.
+ */
+export function tidyBytes(str) {
+    try {
+        const buf = Buffer.from(str, 'latin1');
+        return buf.toString('utf8');
+    }
+    catch {
+        return str;
+    }
+}
+/**
+ * Parse a filter string into a RegExp or null for literal matching.
+ * Supports /pattern/ and /pattern/i notation.
+ */
+export function parseFilterToRegex(filter) {
+    if (!filter || filter.length < 2) {
+        return null;
+    }
+    if (filter.startsWith('/') && filter.includes('/')) {
+        const lastSlash = filter.lastIndexOf('/');
+        const pattern = filter.slice(1, lastSlash).replace(/\\\//g, '/');
+        const flags = filter.slice(lastSlash + 1);
+        try {
+            return new RegExp(pattern, flags);
+        }
+        catch {
+            return null;
+        }
+    }
+    return null;
+}
+/**
+ * Check if a URL matches the "only" filter (include).
+ */
+export function matchOnlyFilter(fileUrl, onlyFilter) {
+    if (!onlyFilter) {
+        return true;
+    }
+    const regex = parseFilterToRegex(onlyFilter);
+    if (regex !== null) {
+        return regex.test(fileUrl);
+    }
+    return fileUrl.toLowerCase().includes(onlyFilter.toLowerCase());
+}
+/**
+ * Check if a URL matches the "exclude" filter.
+ */
+export function matchExcludeFilter(fileUrl, excludeFilter) {
+    if (!excludeFilter) {
+        return false;
+    }
+    const regex = parseFilterToRegex(excludeFilter);
+    if (regex !== null) {
+        return regex.test(fileUrl);
+    }
+    return fileUrl.toLowerCase().includes(excludeFilter.toLowerCase());
+}
+/**
+ * Sanitize a path for the filesystem (Windows-incompatible chars).
+ */
+export function sanitizePath(path) {
+    return path.replace(/[:*?&=<>\\|]/g, (s) => '%' + s.charCodeAt(0).toString(16));
+}
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC;QACH,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACvC,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAc;IAC/C,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QAC1C,IAAI,CAAC;YACH,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe,EAAE,UAA8B;IAC7E,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAC7C,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;AAClE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,aAAiC;IACnF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,KAAK,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;AACrE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;AAClF,CAAC"}

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "wayback-dl",
+  "version": "1.0.0",
+  "description": "Download archived websites from the Internet Archive Wayback Machine",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "wayback-dl": "bin/wayback-dl.js"
+  },
+  "files": [
+    "dist",
+    "bin"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "prepare": "pnpm run build",
+    "test": "node --import tsx --test test/*.test.ts",
+    "start": "node --import tsx bin/wayback-dl.js",
+    "dev": "node --import tsx src/cli.ts"
+  },
+  "keywords": [
+    "wayback-machine",
+    "internet-archive",
+    "archive",
+    "download",
+    "cli"
+  ],
+  "author": "Scott Weaver <scottmweaver@gmail.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/tdlm/wayback-machine-downloader.git"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "chalk": "^5.3.0",
+    "cli-progress": "^3.12.0",
+    "commander": "^12.1.0",
+    "p-queue": "^8.0.1"
+  },
+  "devDependencies": {
+    "@types/cli-progress": "^3.11.6",
+    "@types/node": "^22.10.1",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2"
+  }
+}