single-file-cli 2.1.3 → 2.2.0

package/.github/workflows/publish.yml

@@ -0,0 +1,141 @@
+# .github/workflows/publish.yml
+
+name: Publish
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  # nothing else guarantees that the released commit was verified: the test workflow runs on push,
+  # so this waits for its run on that exact commit and refuses to publish unless it succeeded
+  check_tests:
+    name: 'Check tests'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      # only used to resolve the tag to a commit, so it runs no code from the repository and keeps no
+      # git credentials behind
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.release.tag_name }}
+          persist-credentials: false
+      # the tag is typed in the release UI, so nothing ties it to the version that is about to be
+      # published: a mistyped tag, or a bump that was never pushed, publishes a version nobody asked for
+      - name: Verify the tag matches the declared version
+        env:
+          TAG_NAME: ${{ github.event.release.tag_name }}
+        run: |
+          version=$(jq -r .version package.json)
+          if [ "$TAG_NAME" != "v$version" ]; then
+            echo "::error::release $TAG_NAME but package.json declares $version"
+            exit 1
+          fi
+          deno_version=$(jq -r .version deno.json)
+          if [ "$deno_version" != "$version" ]; then
+            echo "::error::package.json declares $version but deno.json declares $deno_version"
+            exit 1
+          fi
+          if ! grep -q "\"$version\"" lib/version.js; then
+            echo "::error::lib/version.js does not declare $version"
+            exit 1
+          fi
+      - name: Wait for the test run of the released commit
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          sha=$(git rev-parse HEAD)
+          echo "released commit: $sha"
+          missing=0
+          for attempt in $(seq 1 60); do
+            run=$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?head_sha=$sha&per_page=1" \
+              --jq 'if (.workflow_runs | length) == 0 then "none none none"
+                    else (.workflow_runs[0] | "\(.status) \(.conclusion) \(.html_url)") end')
+            run_status=$(echo "$run" | cut -d' ' -f1)
+            conclusion=$(echo "$run" | cut -d' ' -f2)
+            url=$(echo "$run" | cut -d' ' -f3)
+            if [ "$run_status" = "none" ]; then
+              missing=$((missing + 1))
+              if [ "$missing" -ge 10 ]; then
+                echo "::error::no test run for $sha, run the test workflow on this commit and publish again"
+                exit 1
+              fi
+              echo "no test run found yet for $sha"
+            else
+              echo "test run: status=$run_status conclusion=$conclusion $url"
+              if [ "$run_status" = "completed" ]; then
+                if [ "$conclusion" = "success" ]; then
+                  exit 0
+                fi
+                echo "::error::the tests concluded $conclusion for $sha, see $url"
+                exit 1
+              fi
+            fi
+            sleep 30
+          done
+          echo "::error::timed out waiting for the tests of $sha"
+          exit 1
+
+  publish_npm:
+    needs: check_tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.release.tag_name }}
+      # Setup .npmrc file to publish to npm
+      - uses: actions/setup-node@v5
+        with:
+          node-version: '24'
+          registry-url: 'https://registry.npmjs.org'
+      - run: npm publish
+
+  publish_docker:
+    name: 'Publish Docker image'
+    needs: publish_npm
+    runs-on: ubuntu-latest
+    steps:
+      # the image installs single-file-cli from npm, so building before the registry serves the
+      # released version would silently ship the previous one
+      - name: Wait for npm to serve the released version
+        env:
+          TAG_NAME: ${{ github.event.release.tag_name }}
+        run: |
+          version="${TAG_NAME#v}"
+          for attempt in $(seq 1 40); do
+            available=$(npm view single-file-cli version 2>/dev/null || true)
+            if [ "$available" = "$version" ]; then
+              exit 0
+            fi
+            echo "npm serves ${available:-nothing}, waiting for $version"
+            sleep 30
+          done
+          echo "::error::timed out waiting for npm to serve $version"
+          exit 1
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/singlefile:latest
+            ${{ secrets.DOCKER_USERNAME }}/singlefile:${{ github.event.release.tag_name }}
+          platforms: linux/amd64,linux/arm64

package/Dockerfile

@@ -1,12 +1,16 @@
-FROM zenika/alpine-chrome:with-node
+FROM node:24-alpine
 
-RUN npm install --omit=dev single-file-cli
+RUN apk add --no-cache chromium ttf-freefont font-noto-emoji
+
+USER node
 
 WORKDIR /usr/src/app
 
+RUN npm install --omit=dev single-file-cli
+
 ENTRYPOINT [ \
     "npx", \
     "single-file", \
     "--browser-executable-path", "/usr/bin/chromium-browser", \
     "--output-directory", "./out/", \
-    "--dump-content" ]
+    "--dump-content" ]

package/README.MD

@@ -143,6 +143,12 @@ Make sure Chrome or a Chromium-based browser is installed in the default folder.
   single-file https://www.wikipedia.org --crawl-links=true --crawl-inner-links-only=false --crawl-external-links-max-depth=1 --crawl-rewrite-rule="^.*wikipedia.*$"
   ```
 
+  - Save https://www.wikipedia.org and its internal links into a single self-extracting ZIP file
+
+  ```sh
+  single-file https://www.wikipedia.org wikipedia.html --crawl-links=true --crawl-save-archive=true --compress-content=true
+  ```
+
 ## Compile executables
 
  - Compile executables into `/dist`

package/build.sh

@@ -3,7 +3,7 @@
 mv package.json package.json.tmp
 mv deno.json deno.json.tmp
 mv deno.lock deno.lock.tmp
-deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.86"
+deno install --vendor --quiet --minimum-dependency-age=0 "npm:single-file-core@1.5.92"
 mv package.json.tmp package.json
 mv deno.json.tmp deno.json
 mv deno.lock.tmp deno.lock
@@ -66,6 +66,20 @@ await build({
   plugins: [],
 });
 
+await build({
+  stdin: {
+    contents: \"export * from './processors/compression/compression.js'; export * from './vendor/zip/zip.js';\",
+    resolveDir: 'node_modules/single-file-core',
+  },
+  bundle: true,
+  outfile: 'lib/single-file-archive.js',
+  platform: 'neutral',
+  sourcemap: false,
+  minify: true,
+  format: 'esm',
+  plugins: [],
+});
+
 const SCRIPTS = [
 	'lib/single-file.js',
 	'lib/single-file-bootstrap.js',

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@single-file/single-file-cli",
-  "version": "2.1.3",
+  "version": "2.2.0",
   "description": "SingleFile CLI",
   "exports": {
     ".": "./single-file-cli-api.js"

package/dev-build.sh

@@ -61,6 +61,20 @@ await build({
   plugins: [],
 });
 
+await build({
+  stdin: {
+    contents: \"export * from './processors/compression/compression.js'; export * from './vendor/zip/zip.js';\",
+    resolveDir: 'node_modules/single-file-core',
+  },
+  bundle: true,
+  outfile: 'lib/single-file-archive.js',
+  platform: 'neutral',
+  sourcemap: false,
+  minify: false,
+  format: 'esm',
+  plugins: [],
+});
+
 const SCRIPTS = [
 	'lib/single-file.js',
 	'lib/single-file-bootstrap.js',

package/eslint.config.mjs

@@ -1,6 +1,7 @@
 import js from "@eslint/js";
 
 export default [
+	{ ignores: ["lib/single-file-archive.js"] },
 	js.configs.recommended,
 	{
 		languageOptions: {

package/lib/archive-packager.js

@@ -0,0 +1,225 @@
+/*
+ * Copyright 2010-2026 Gildas Lormeau
+ * contact : gildas.lormeau <at> gmail.com
+ *
+ * This file is part of SingleFile.
+ *
+ *   The code in this file is free software: you can redistribute it and/or
+ *   modify it under the terms of the GNU Affero General Public License
+ *   (GNU AGPL) as published by the Free Software Foundation, either version 3
+ *   of the License, or (at your option) any later version.
+ *
+ *   The code in this file is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+ *   General Public License for more details.
+ *
+ *   As additional permission under GNU AGPL version 3 section 7, you may
+ *   distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
+ *   AGPL normally required by section 4, provided you include this license
+ *   notice and a URL through which recipients can access the Corresponding
+ *   Source.
+ */
+
+/* global URL */
+
+import {
+	configure,
+	createArchive,
+	TextReader,
+	Uint8ArrayReader,
+	Uint8ArrayWriter,
+	ZipReader
+} from "./single-file-archive.js";
+
+const PAGES_PREFIX = "pages/";
+const PAGES_FILENAME = "sfz-pages.json";
+const TOC_FILENAME = "sfz-toc.html";
+const TOC_TITLE = "Table of contents";
+const TOC_STYLE = "body{font-family:system-ui,sans-serif;margin:2em auto;max-width:40em;padding:0 1em;background-color:#fff;color:#000}" +
+	"a{color:#0000ee}a:visited{color:#551a8b}" +
+	"summary{cursor:pointer;font-weight:bold;margin:.5em 0}" +
+	"details{padding-left:1em}ul{margin:.25em 0;padding-left:1.5em}" +
+	"@media(prefers-color-scheme:dark){body{background-color:#111;color:#eee}a{color:#8ab4f8}a:visited{color:#c58af9}}";
+const COMMENT_HEADER = "Page saved with SingleFile";
+const SYMLINK_UNIX_MODE = 0o120777;
+
+export { createPagesArchive };
+
+async function createPagesArchive(pages, options) {
+	configure({ useWebWorkers: false });
+	const manifest = {
+		pages: pages.map((page, pageIndex) => ({
+			path: getPagePath(pageIndex),
+			url: page.url,
+			originalUrls: page.originalUrls,
+			title: page.title
+		}))
+	};
+	if (options.markUnarchivedLinks) {
+		manifest.markUnarchivedLinks = true;
+	}
+	const pageData = {
+		doctype: "<!DOCTYPE html>",
+		content: "",
+		title: pages[0].title || "",
+		comment: options.insertSingleFileComment ? getComment(pages[0].url, options) : undefined,
+		tocContent: getTOCContent(pages)
+	};
+	const archiveOptions = {
+		url: pages[0].url,
+		multiPageArchive: true,
+		selfExtractingArchive: options.selfExtractingArchive,
+		extractDataFromPage: options.extractDataFromPage,
+		preventAppendedData: options.preventAppendedData,
+		includeBOM: options.includeBOM,
+		insertMetaCSP: options.insertMetaCSP,
+		insertCanonicalLink: options.insertCanonicalLink,
+		insertMetaNoIndex: options.insertMetaNoIndex
+	};
+	const writtenEntries = options.dedupPages ? new Map() : undefined;
+	const aliases = {};
+	const blob = await createArchive(pageData, archiveOptions, options.zipScript, async zipWriter => {
+		for (let pageIndex = 0; pageIndex < pages.length; pageIndex++) {
+			const pagePath = getPagePath(pageIndex);
+			const zipReader = new ZipReader(new Uint8ArrayReader(await pages[pageIndex].getData()));
+			for (const entry of await zipReader.getEntries()) {
+				const filename = pagePath + entry.filename;
+				const rawData = await entry.getData(new Uint8ArrayWriter(), { passThrough: true, checkSignature: false });
+				const canonicalFilename = writtenEntries && findDuplicate(writtenEntries, filename, entry, rawData);
+				if (canonicalFilename === undefined) {
+					await zipWriter.add(filename, new Uint8ArrayReader(rawData), {
+						passThrough: true,
+						compressionMethod: entry.compressionMethod,
+						uncompressedSize: entry.uncompressedSize,
+						signature: entry.signature,
+						comment: entry.comment,
+						lastModDate: entry.lastModDate
+					});
+				} else {
+					// the duplicate becomes a symlink entry so that external
+					// extractors still produce complete page folders, the router
+					// resolves it from the manifest alias map instead
+					aliases[filename] = canonicalFilename;
+					await zipWriter.add(filename, new TextReader(getRelativePath(filename, canonicalFilename)), {
+						msDosCompatible: false,
+						unixMode: SYMLINK_UNIX_MODE,
+						level: 0,
+						comment: entry.comment,
+						lastModDate: entry.lastModDate
+					});
+				}
+			}
+			await zipReader.close();
+		}
+		if (Object.keys(aliases).length) {
+			manifest.aliases = aliases;
+		}
+		if (options.tocPage) {
+			await zipWriter.add(TOC_FILENAME, new TextReader(getTOCPageContent(manifest.pages)));
+		}
+		await zipWriter.add(PAGES_FILENAME, new TextReader(JSON.stringify(manifest, null, 2)));
+	});
+	return new Uint8Array(await blob.arrayBuffer());
+}
+
+function findDuplicate(writtenEntries, filename, entry, rawData) {
+	if (entry.directory || !entry.uncompressedSize) {
+		return;
+	}
+	const key = [entry.compressionMethod, entry.uncompressedSize, entry.signature, rawData.length].join(":");
+	const candidates = writtenEntries.get(key);
+	if (candidates) {
+		const match = candidates.find(candidate => equalData(candidate.rawData, rawData));
+		if (match) {
+			return match.filename;
+		}
+		candidates.push({ filename, rawData });
+	} else {
+		writtenEntries.set(key, [{ filename, rawData }]);
+	}
+}
+
+function equalData(dataLeft, dataRight) {
+	return dataLeft.length == dataRight.length && dataLeft.every((value, index) => value == dataRight[index]);
+}
+
+function getRelativePath(filename, targetFilename) {
+	const baseSegments = filename.split("/").slice(0, -1);
+	const targetSegments = targetFilename.split("/");
+	while (baseSegments.length && targetSegments.length > 1 && baseSegments[0] == targetSegments[0]) {
+		baseSegments.shift();
+		targetSegments.shift();
+	}
+	return "../".repeat(baseSegments.length) + targetSegments.join("/");
+}
+
+function getPagePath(pageIndex) {
+	return pageIndex == 0 ? "" : PAGES_PREFIX + (pageIndex + 1) + "/";
+}
+
+function getComment(url, options) {
+	return "\n " + COMMENT_HEADER +
+		" \n url: " + url +
+		(options.removeSavedDate ? " " : " \n saved date: " + new Date()) + "\n";
+}
+
+function getTOCPageContent(pages) {
+	const origins = new Set(pages.map(page => new URL(page.url).origin));
+	const rootGroup = { groups: new Map(), pages: [] };
+	pages.forEach(page => {
+		const url = new URL(page.url);
+		const segments = url.pathname.split("/").slice(1, -1);
+		if (origins.size > 1) {
+			segments.unshift(url.origin);
+		}
+		let group = rootGroup;
+		segments.forEach(segment => {
+			if (!group.groups.has(segment)) {
+				group.groups.set(segment, { groups: new Map(), pages: [] });
+			}
+			group = group.groups.get(segment);
+		});
+		group.pages.push(page);
+	});
+	const title = pages[0].title ? TOC_TITLE + " - " + pages[0].title : TOC_TITLE;
+	return "<!DOCTYPE html><html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">" +
+		"<title>" + escapeUnicodeHTML(title) + "</title><style>" + TOC_STYLE + "</style></head><body><main><h1>" +
+		escapeUnicodeHTML(TOC_TITLE) + "</h1>" + getTOCGroupContent(rootGroup) + "</main></body></html>";
+}
+
+// nested details/summary groups stay collapsible without scripts on purpose,
+// the page must remain usable after a plain unzip
+function getTOCGroupContent(group) {
+	let content = "";
+	if (group.pages.length) {
+		content += "<ul>" + group.pages.map(page =>
+			"<li><a href=\"" + escapeUnicodeHTML(page.path + "index.html") + "\">" + escapeUnicodeHTML(page.title || page.url) + "</a></li>").join("") + "</ul>";
+	}
+	group.groups.forEach((childGroup, segment) => {
+		content += "<details open><summary>" + escapeUnicodeHTML(segment) + "</summary>" + getTOCGroupContent(childGroup) + "</details>";
+	});
+	return content;
+}
+
+// unlike the prelude TOC below, the stored page is a UTF-8 entry: only the
+// markup delimiters need escaping, but crawled titles remain untrusted
+function escapeUnicodeHTML(value) {
+	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+
+function getTOCContent(pages) {
+	return "<nav><ul>" +
+		pages.map(page => "<li><a href=\"" + escapeHTML(page.url) + "\">" + escapeHTML(page.title || page.url) + "</a></li>").join("") +
+		"</ul></nav>";
+}
+
+// the prelude declares the windows-1252 charset, non-ASCII characters must be
+// encoded as HTML entities to survive it
+function escapeHTML(value) {
+	return Array.from(value).map(character => {
+		const codePoint = character.codePointAt(0);
+		return codePoint < 32 || codePoint > 126 || character == "&" || character == "<" || character == ">" || character == "\"" ?
+			"&#" + codePoint + ";" : character;
+	}).join("");
+}