registry-sync 8.0.0 → 8.2.0

package/bin/sync

@@ -1,3 +1,3 @@
 #!/usr/bin/env node
 
-await import('../src/index.ts')
+await import('../src/index.js')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "registry-sync",
-  "version": "8.0.0",
+  "version": "8.2.0",
   "description": "synchronize a remote npm registry for private use",
   "repository": "https://github.com/heikkipora/registry-sync",
   "type": "module",
@@ -13,24 +13,25 @@
     "fix-eslint": "eslint --fix src test release-test/server/src",
     "test": "mocha --timeout 120000 test/*.ts",
     "typecheck": "tsc",
-    "release-test": "cd release-test && ./run-sync-install-cycle.sh"
+    "release-test": "cd release-test && ./run-sync-install-cycle.sh",
+    "refresh-manifests": "cd test/manifests && ./refresh-manifests.sh"
   },
   "author": "Heikki Pora",
   "license": "MIT",
   "dependencies": {
     "@yarnpkg/lockfile": "1.1.0",
-    "axios": "1.13.2",
-    "commander": "14.0.2",
-    "lru-cache": "11.2.4",
-    "semver": "7.7.3",
+    "axios": "1.13.4",
+    "commander": "14.0.3",
+    "lru-cache": "11.2.5",
+    "semver": "7.7.4",
     "ssri": "13.0.0",
     "tar-fs": "3.1.1"
   },
   "devDependencies": {
     "@types/chai": "5.2.3",
-    "@types/lodash": "4.17.21",
+    "@types/lodash": "4.17.23",
     "@types/mocha": "10.0.10",
-    "@types/node": "20.17.32",
+    "@types/node": "22.18.0",
     "@types/semver": "7.7.1",
     "@types/ssri": "7.1.5",
     "@types/tar-fs": "2.0.4",
@@ -39,10 +40,10 @@
     "eslint": "9.39.2",
     "eslint-plugin-mocha": "11.2.0",
     "express": "5.2.1",
-    "globals": "17.0.0",
+    "globals": "17.3.0",
     "mocha": "11.7.5",
     "typescript": "5.9.3",
-    "typescript-eslint": "8.51.0"
+    "typescript-eslint": "8.54.0"
   },
   "keywords": [
     "registry",

package/src/client.js

@@ -0,0 +1,27 @@
+import * as https from 'https';
+import axios from 'axios';
+import { LRUCache } from 'lru-cache';
+const metadataCache = new LRUCache({ max: 100 });
+const client = axios.create({
+    httpsAgent: new https.Agent({ keepAlive: true }),
+    timeout: 30 * 1000
+});
+export async function fetchJsonWithCacheCloned(url, token) {
+    const cached = metadataCache.get(url);
+    if (cached) {
+        return structuredClone(cached);
+    }
+    const value = await fetch(url, 'json', token);
+    metadataCache.set(url, value);
+    return structuredClone(value);
+}
+export function fetchBinaryData(url, token) {
+    return fetch(url, 'arraybuffer', token);
+}
+async function fetch(url, responseType, token) {
+    const config = { responseType };
+    if (token !== '') {
+        config.headers = { authorization: 'Bearer ' + token };
+    }
+    return (await client.get(url, config)).data;
+}

package/src/download.js

@@ -0,0 +1,89 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as semver from 'semver';
+import * as url from 'url';
+import assert from 'assert';
+import { downloadPrebuiltBinaries, hasPrebuiltBinaries } from "./pregyp.js";
+import { fetchBinaryData, fetchJsonWithCacheCloned } from "./client.js";
+import { rewriteMetadataInTarball, rewriteVersionMetadata, tarballFilename } from "./metadata.js";
+import { verifyIntegrity } from "./integrity.js";
+export async function downloadAll(packages, { localUrl, prebuiltBinaryProperties, registryUrl, registryToken, rootFolder, enforceTarballsOverHttps }) {
+    const downloadFromRegistry = download.bind(null, registryUrl, registryToken, localUrl, rootFolder, prebuiltBinaryProperties, enforceTarballsOverHttps);
+    for (const pkg of packages) {
+        await downloadFromRegistry(pkg);
+    }
+}
+async function download(registryUrl, registryToken, localUrl, rootFolder, prebuiltBinaryProperties, enforceTarballsOverHttps, { name, version }) {
+    const registryMetadata = await fetchMetadataCloned(name, registryUrl, registryToken);
+    const versionMetadata = registryMetadata.versions[version];
+    if (!versionMetadata) {
+        throw new Error(`Unknown package version ${name}@${version}`);
+    }
+    const localFolder = await ensureLocalFolderExists(name, rootFolder);
+    let data = await downloadTarball(versionMetadata, enforceTarballsOverHttps, registryToken);
+    if (hasPrebuiltBinaries(versionMetadata)) {
+        const localPregypFolder = await ensureLocalFolderExists(version, localFolder);
+        await downloadPrebuiltBinaries(versionMetadata, localPregypFolder, prebuiltBinaryProperties);
+        data = await rewriteMetadataInTarball(data, versionMetadata, localUrl, localFolder);
+    }
+    await saveTarball(versionMetadata, data, localFolder);
+    rewriteVersionMetadata(versionMetadata, data, localUrl);
+    await updateMetadata(versionMetadata, registryMetadata, registryUrl, localFolder);
+}
+async function downloadTarball({ _id: id, dist }, enforceTarballsOverHttps, registryToken) {
+    const tarballUrl = enforceTarballsOverHttps ? dist.tarball.replace('http://', 'https://') : dist.tarball;
+    const data = await fetchBinaryData(tarballUrl, registryToken);
+    verifyIntegrity(data, id, dist);
+    return data;
+}
+function saveTarball({ name, version }, data, localFolder) {
+    return fs.promises.writeFile(tarballPath(name, version, localFolder), data);
+}
+async function updateMetadata(versionMetadata, defaultMetadata, registryUrl, localFolder) {
+    const { version } = versionMetadata;
+    const localMetadataPath = path.join(localFolder, 'index.json');
+    const localMetadata = await loadMetadata(localMetadataPath, defaultMetadata);
+    localMetadata.versions[version] = versionMetadata;
+    localMetadata.time[version] = defaultMetadata.time[version];
+    localMetadata['dist-tags'] = collectDistTags(localMetadata, defaultMetadata);
+    await saveMetadata(localMetadataPath, localMetadata);
+}
+// Collect dist-tags entries (name -> version) from registry metadata,
+// which point to versions we have locally available.
+// Override 'latest' tag to ensure its validity as we might not have the version
+// that is tagged latest in registry
+function collectDistTags(localMetadata, defaultMetadata) {
+    const availableVersions = Object.keys(localMetadata.versions);
+    const validDistTags = Object.entries(defaultMetadata['dist-tags']).filter(([, version]) => availableVersions.includes(version));
+    const latest = availableVersions.sort(semver.compare).pop();
+    assert(latest, 'At least one version should be locally available to determine "latest" dist-tag');
+    return {
+        ...Object.fromEntries(validDistTags),
+        latest
+    };
+}
+async function loadMetadata(path, defaultMetadata) {
+    try {
+        const json = await fs.promises.readFile(path, 'utf8');
+        return JSON.parse(json);
+    }
+    catch {
+        return { ...defaultMetadata, 'dist-tags': {}, time: {}, versions: {} };
+    }
+}
+function saveMetadata(path, metadata) {
+    const json = JSON.stringify(metadata, null, 2);
+    return fs.promises.writeFile(path, json, 'utf8');
+}
+function tarballPath(name, version, localFolder) {
+    return path.join(localFolder, tarballFilename(name, version));
+}
+async function ensureLocalFolderExists(name, rootFolder) {
+    const localFolder = path.resolve(rootFolder, name);
+    await fs.promises.mkdir(localFolder, { recursive: true });
+    return localFolder;
+}
+function fetchMetadataCloned(name, registryUrl, registryToken) {
+    const urlSafeName = name.replace(/\//g, '%2f');
+    return fetchJsonWithCacheCloned(url.resolve(registryUrl, urlSafeName), registryToken);
+}

package/src/index.js

@@ -0,0 +1,46 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { Command } from 'commander';
+import { synchronize } from "./sync.js";
+import { URL } from 'url';
+const { version } = JSON.parse(fs.readFileSync(path.join(import.meta.dirname, '..', 'package.json'), 'utf-8'));
+const program = new Command();
+program
+    .version(version)
+    .requiredOption('--root <path>', 'Path to save NPM package tarballs and metadata to')
+    .requiredOption('--manifest <file>', 'Path to a package-lock.json or yarn.lock file to use as catalog for mirrored NPM packages.')
+    .requiredOption('--localUrl <url>', 'URL to use as root in stored package metadata (i.e. where folder defined as --root will be exposed at)')
+    .option('--binaryAbi <list>', 'Comma-separated list of node C++ ABI numbers to download pre-built binaries for. See NODE_MODULE_VERSION column in https://nodejs.org/en/download/releases/')
+    .option('--binaryArch <list>', 'Comma-separated list of CPU architectures to download pre-built binaries for. Valid values: arm, ia32, and x64')
+    .option('--binaryPlatform <list>', 'Comma-separated list of OS platforms to download pre-built binaries for. Valid values: linux, darwin, win32, sunos, freebsd, openbsd, and aix')
+    .option('--registryUrl [url]', 'Optional URL to use as NPM registry when fetching packages. Default value is https://registry.npmjs.org')
+    .option('--registryToken [string]', 'Optional Bearer token for the registry.')
+    .option('--dontEnforceHttps', 'Disable the default behavior of downloading tarballs over HTTPS (will use whichever protocol is defined in the registry metadata)')
+    .option('--includeDev', 'Include also packages found from devDependencies section of the --manifest')
+    .option('--dryRun', 'Print packages that would be downloaded but do not download them')
+    .parse(process.argv);
+const rawOptions = program.opts();
+// use current (abi,arch,platform) triplet as default if none is specified
+// so the user doesn't have to look them up if build is always done on the
+// same kind of machine
+const binaryAbi = rawOptions.binaryAbi || process.versions.modules;
+const binaryArch = rawOptions.binaryArch || process.arch;
+const binaryPlatform = rawOptions.binaryPlatform || process.platform;
+const abis = binaryAbi.split(',').map(Number);
+const architectures = binaryArch.split(',');
+const platforms = binaryPlatform.split(',');
+const prebuiltBinaryProperties = abis
+    .map(abi => architectures.map(arch => platforms.map(platform => ({ abi, arch, platform }))).flat())
+    .flat();
+const options = {
+    localUrl: new URL(rawOptions.localUrl),
+    manifest: rawOptions.manifest,
+    prebuiltBinaryProperties,
+    registryUrl: rawOptions.registryUrl || 'https://registry.npmjs.org',
+    registryToken: rawOptions.registryToken || '',
+    rootFolder: rawOptions.root,
+    enforceTarballsOverHttps: Boolean(!rawOptions.dontEnforceHttps),
+    includeDevDependencies: Boolean(rawOptions.includeDev),
+    dryRun: Boolean(rawOptions.dryRun)
+};
+synchronize(options);

package/src/integrity.js

@@ -0,0 +1,20 @@
+import * as ssri from 'ssri';
+export function verifyIntegrity(data, id, { integrity, shasum }) {
+    if (!integrity && !shasum) {
+        throw new Error(`Integrity values not present in metadata for ${id}`);
+    }
+    if (integrity) {
+        if (!ssri.checkData(data, integrity)) {
+            throw new Error(`Integrity check failed for ${id}`);
+        }
+    }
+    else if (sha1(data) != shasum) {
+        throw new Error(`Integrity check with SHA1 failed for failed for ${id}`);
+    }
+}
+export function sha1(data) {
+    return ssri.fromData(data, { algorithms: ['sha1'] }).hexDigest();
+}
+export function sha512(data) {
+    return ssri.fromData(data, { algorithms: ['sha512'] }).toString();
+}

package/src/metadata.js

@@ -0,0 +1,59 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as tar from 'tar-fs';
+import * as zlib from 'zlib';
+import { hasPrebuiltBinaries } from "./pregyp.js";
+import { Readable } from 'stream';
+import { sha1, sha512 } from "./integrity.js";
+export function rewriteVersionMetadata(versionMetadata, data, localUrl) {
+    versionMetadata.dist.tarball = localTarballUrl(versionMetadata, localUrl);
+    if (hasPrebuiltBinaries(versionMetadata)) {
+        versionMetadata.binary.host = localUrl.origin;
+        versionMetadata.binary.remote_path = createPrebuiltBinaryRemotePath(localUrl, versionMetadata);
+        versionMetadata.dist.integrity = sha512(data);
+        versionMetadata.dist.shasum = sha1(data);
+    }
+}
+export async function rewriteMetadataInTarball(data, versionMetadata, localUrl, localFolder) {
+    const tmpFolder = path.join(localFolder, '.tmp');
+    await fs.promises.mkdir(tmpFolder, { recursive: true });
+    await extractTgz(data, tmpFolder);
+    const manifestPath = path.join(tmpFolder, 'package', 'package.json');
+    const json = await fs.promises.readFile(manifestPath, 'utf8');
+    const metadata = JSON.parse(json);
+    metadata.binary.host = localUrl.origin;
+    metadata.binary.remote_path = createPrebuiltBinaryRemotePath(localUrl, versionMetadata);
+    await fs.promises.writeFile(manifestPath, JSON.stringify(metadata, null, 2));
+    const updatedData = await compressTgz(tmpFolder);
+    await fs.promises.rm(tmpFolder, { recursive: true });
+    return updatedData;
+}
+function createPrebuiltBinaryRemotePath(url, versionMetadata) {
+    return `${removeTrailingSlash(url.pathname)}/${versionMetadata.name}/${versionMetadata.version}/`;
+}
+export function extractTgz(data, folder) {
+    return new Promise((resolve, reject) => {
+        const tgz = Readable.from(data).pipe(zlib.createGunzip()).pipe(tar.extract(folder));
+        tgz.on('finish', resolve);
+        tgz.on('error', reject);
+    });
+}
+function compressTgz(folder) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        const tgz = tar.pack(folder).pipe(zlib.createGzip());
+        tgz.on('data', (chunk) => chunks.push(chunk));
+        tgz.on('end', () => resolve(Buffer.concat(chunks)));
+        tgz.on('error', reject);
+    });
+}
+function localTarballUrl({ name, version }, localUrl) {
+    return `${localUrl.origin}${removeTrailingSlash(localUrl.pathname)}/${name}/${tarballFilename(name, version)}`;
+}
+export function tarballFilename(name, version) {
+    const normalized = name.replace(/\//g, '-');
+    return `${normalized}-${version}.tgz`;
+}
+function removeTrailingSlash(str) {
+    return str.replace(/\/$/, '');
+}

package/src/{normalize-yarn-pattern.ts → normalize-yarn-pattern.js}

@@ -27,40 +27,31 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 // From https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/util/normalize-pattern.js#L2
-
-export function normalizeYarnPackagePattern(pattern: string): {
-  hasVersion: boolean
-  name: string
-  range: string
-} {
-  let hasVersion = false
-  let range = 'latest'
-  let name = pattern
-
-  // if we're a scope then remove the @ and add it back later
-  let isScoped = false
-  if (name[0] === '@') {
-    isScoped = true
-    name = name.slice(1)
-  }
-
-  // take first part as the name
-  const parts = name.split('@')
-  if (parts.length > 1) {
-    name = parts.shift()!
-    range = parts.join('@')
-
-    if (range) {
-      hasVersion = true
-    } else {
-      range = '*'
+export function normalizeYarnPackagePattern(pattern) {
+    let hasVersion = false;
+    let range = 'latest';
+    let name = pattern;
+    // if we're a scope then remove the @ and add it back later
+    let isScoped = false;
+    if (name[0] === '@') {
+        isScoped = true;
+        name = name.slice(1);
     }
-  }
-
-  // add back @ scope suffix
-  if (isScoped) {
-    name = `@${name}`
-  }
-
-  return {name, range, hasVersion}
+    // take first part as the name
+    const parts = name.split('@');
+    if (parts.length > 1) {
+        name = parts.shift();
+        range = parts.join('@');
+        if (range) {
+            hasVersion = true;
+        }
+        else {
+            range = '*';
+        }
+    }
+    // add back @ scope suffix
+    if (isScoped) {
+        name = `@${name}`;
+    }
+    return { name, range, hasVersion };
 }

package/src/pregyp.js

@@ -0,0 +1,84 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as semver from 'semver';
+import * as url from 'url';
+import { fetchBinaryData } from "./client.js";
+export function hasPrebuiltBinaries(metadata) {
+    return Boolean(metadata.binary
+        && metadata.binary.host
+        && metadata.binary.module_name
+        && metadata.binary.package_name
+        && metadata.binary.remote_path);
+}
+export async function downloadPrebuiltBinaries(versionMetadata, localFolder, prebuiltBinaryProperties) {
+    const { binary, name, version } = versionMetadata;
+    if (!binary.napi_versions) {
+        for (const { abi, arch, platform } of prebuiltBinaryProperties) {
+            await downloadPrebuiltBinary(localFolder, name, version, binary, abi, platform, arch);
+        }
+        return;
+    }
+    for (const napiVersion of binary.napi_versions) {
+        for (const { abi, arch, platform } of prebuiltBinaryProperties) {
+            await downloadPrebuiltBinary(localFolder, name, version, binary, abi, platform, arch, napiVersion);
+        }
+    }
+}
+async function downloadPrebuiltBinary(localFolder, name, version, binary, abi, platform, arch, napiVersion) {
+    try {
+        const data = await fetchPrebuiltBinary(name, version, binary, abi, platform, arch, napiVersion);
+        await fs.promises.writeFile(prebuiltBinaryFilePath(localFolder, name, version, binary, abi, platform, arch, napiVersion), data);
+    }
+    catch (err) {
+        // pre-built binaries are commonly not available on all platforms (and S3 will commonly respond with 403 for a non-existent file)
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const fileNotFoundError = err.response && (err.response.status == 403 || err.response.status == 404);
+        if (!fileNotFoundError) {
+            console.error(`Unexpected error fetching prebuilt binary for ${name} and ABI v${abi} on ${arch}-${platform} (n-api version ${napiVersion})`);
+            throw err;
+        }
+    }
+}
+function fetchPrebuiltBinary(name, version, binary, abi, platform, arch, napiVersion) {
+    return fetchBinaryData(prebuiltBinaryUrl(name, version, binary, abi, platform, arch, napiVersion), '');
+}
+function prebuiltBinaryFilePath(localFolder, name, version, binary, abi, platform, arch, napiVersion) {
+    return path.join(localFolder, prebuiltBinaryFileName(name, version, binary, abi, platform, arch, napiVersion));
+}
+function prebuiltBinaryUrl(name, version, binary, abi, platform, arch, napiVersion) {
+    const remotePath = prebuiltBinaryRemotePath(name, version, binary, abi, platform, arch, napiVersion).replace(/\/$/, '');
+    const fileName = prebuiltBinaryFileName(name, version, binary, abi, platform, arch, napiVersion);
+    return url.resolve(binary.host, `${remotePath}/${fileName}`);
+}
+function prebuiltBinaryRemotePath(name, version, binary, abi, platform, arch, napiVersion) {
+    return formatPrebuilt(binary.remote_path, name, version, binary.module_name, abi, platform, arch, napiVersion);
+}
+function prebuiltBinaryFileName(name, version, binary, abi, platform, arch, napiVersion) {
+    return formatPrebuilt(binary.package_name, name, version, binary.module_name, abi, platform, arch, napiVersion);
+}
+// see node-pre-gyp: /lib/util/versioning.js for documentation of possible values
+function formatPrebuilt(formatString, name, version, moduleName, abi, platform, arch, napiVersion) {
+    const moduleVersion = semver.parse(version, false, true);
+    const prerelease = (moduleVersion.prerelease || []).join('.');
+    const build = (moduleVersion.build || []).join('.');
+    const formatted = formatString
+        .replace('{name}', name)
+        .replace('{version}', version)
+        .replace('{major}', moduleVersion.major.toString())
+        .replace('{minor}', moduleVersion.minor.toString())
+        .replace('{patch}', moduleVersion.patch.toString())
+        .replace('{prerelease}', prerelease)
+        .replace('{build}', build)
+        .replace('{module_name}', moduleName)
+        .replace('{node_abi}', `node-v${abi}`)
+        .replace('{platform}', platform)
+        .replace('{arch}', arch)
+        .replace('{libc}', libc(platform))
+        .replace('{configuration}', 'Release')
+        .replace('{toolset}', '')
+        .replace(/[/]+/g, '/');
+    return napiVersion ? formatted.replace('{napi_build_version}', napiVersion.toString()) : formatted;
+}
+function libc(platform) {
+    return platform === 'linux' ? 'glibc' : 'unknown';
+}

package/src/resolve.js

@@ -0,0 +1,205 @@
+import * as fs from 'fs';
+import * as pathLib from 'path';
+import * as readline from 'readline';
+import * as url from 'url';
+import assert, { deepStrictEqual } from 'assert';
+import yarnLockfile from '@yarnpkg/lockfile';
+import { normalizeYarnPackagePattern } from "./normalize-yarn-pattern.js";
+const YARN_LOCK_FILENAME = 'yarn.lock';
+export async function updateDependenciesCache(newDependencies, cacheFilePath, prebuiltBinaryProperties) {
+    const { dependencies: cachedDependencies } = await loadCache(cacheFilePath);
+    const dependencies = cachedDependencies.concat(newDependencies).sort(sortById).filter(uniqueById);
+    const data = {
+        dependencies,
+        prebuiltBinaryProperties,
+        prebuiltBinaryNApiSupport: true
+    };
+    return fs.promises.writeFile(cacheFilePath, JSON.stringify(data), 'utf8');
+}
+export async function dependenciesNotInCache(dependencies, cacheFilePath, prebuiltBinaryProperties) {
+    const { dependencies: cachedDependencies, prebuiltBinaryProperties: cachedPrebuiltBinaryProperties, prebuiltBinaryNApiSupport } = await loadCache(cacheFilePath);
+    if (cachedDependencies.length > 0 &&
+        (!isDeepEqual(prebuiltBinaryProperties, cachedPrebuiltBinaryProperties) || !prebuiltBinaryNApiSupport)) {
+        console.log(`Pre-built binary properties changed, re-downloading all current packages`);
+        return dependencies;
+    }
+    const packageIdsInCache = cachedDependencies.map(pkg => pkg.id);
+    return dependencies.filter(pkg => !packageIdsInCache.includes(pkg.id));
+}
+async function loadCache(cacheFilePath) {
+    try {
+        const data = JSON.parse(await fs.promises.readFile(cacheFilePath, 'utf8'));
+        // Migrate V1 legacy cache file schema to V2
+        if (Array.isArray(data)) {
+            return {
+                dependencies: data,
+                prebuiltBinaryProperties: [],
+                prebuiltBinaryNApiSupport: false
+            };
+        }
+        return data;
+    }
+    catch {
+        // empty V2 cache
+        return {
+            dependencies: [],
+            prebuiltBinaryProperties: [],
+            prebuiltBinaryNApiSupport: true
+        };
+    }
+}
+function isNonRegistryYarnPackagePattern(packagePattern) {
+    if (
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/link-resolver.js#L14
+    packagePattern.startsWith('link:') ||
+        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/file-resolver.js#L18
+        packagePattern.startsWith('file:') ||
+        /^\.{1,2}\//.test(packagePattern) ||
+        pathLib.isAbsolute(packagePattern) ||
+        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/tarball-resolver.js#L15
+        packagePattern.startsWith('http://') ||
+        packagePattern.startsWith('https://') ||
+        (packagePattern.indexOf('@') < 0 && (packagePattern.endsWith('.tgz') || packagePattern.endsWith('.tar.gz'))) ||
+        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/github-resolver.js#L6
+        packagePattern.startsWith('github:') ||
+        /^[^:@%/\s.-][^:@%/\s]*[/][^:@\s/%]+(?:#.*)?$/.test(packagePattern) ||
+        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/gitlab-resolver.js#L6
+        packagePattern.startsWith('gitlab:') ||
+        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/bitbucket-resolver.js#L6
+        packagePattern.startsWith('bitbucket:') ||
+        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/gist-resolver.js#L26
+        packagePattern.startsWith('gist:') ||
+        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/git-resolver.js#L19
+        /^git:|^git\+.+:|^ssh:|^https?:.+\.git$|^https?:.+\.git#.+/.test(packagePattern)) {
+        return true;
+    }
+    else {
+        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/git-resolver.js#L19
+        const { hostname, path } = url.parse(packagePattern);
+        if (hostname && path && ['github.com', 'gitlab.com', 'bitbucket.com', 'bitbucket.org'].indexOf(hostname) >= 0) {
+            return path.split('/').filter((p) => !!p).length === 2;
+        }
+    }
+    return false;
+}
+function resolvePackageNameFromRegistryYarnPackagePattern(packagePattern) {
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/registry-resolver.js#L12
+    const match = packagePattern.match(/^(\S+):(@?.*?)(@(.*?)|)$/);
+    if (match) {
+        return match[2];
+    }
+    else {
+        throw new Error(`Failed to resolve yarn package pattern ${packagePattern}, unrecognized format`);
+    }
+}
+function resolveNpmPackagesFromYarnLockDependencies(yarnLockDependencies) {
+    const packages = yarnLockDependencies.reduce((filterMappedDependencies, { packagePattern, version }) => {
+        if (isNonRegistryYarnPackagePattern(packagePattern)) {
+            return filterMappedDependencies;
+        }
+        let packageName;
+        if (packagePattern.startsWith('npm:') || packagePattern.startsWith('yarn:')) {
+            packageName = resolvePackageNameFromRegistryYarnPackagePattern(packagePattern);
+        }
+        else {
+            // Package pattern not yet recognized, continue with parsing logic from
+            // https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/package-request.js#L99
+            const { name: namePart, range: rangePart } = normalizeYarnPackagePattern(packagePattern);
+            if (isNonRegistryYarnPackagePattern(rangePart)) {
+                return filterMappedDependencies;
+            }
+            if (rangePart.startsWith('npm:') || rangePart.startsWith('yarn:')) {
+                packageName = resolvePackageNameFromRegistryYarnPackagePattern(rangePart);
+            }
+            else {
+                // Finally, we just assume that the pattern is a registry pattern,
+                // see https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/package-request.js#L119
+                packageName = namePart;
+            }
+        }
+        filterMappedDependencies.push({ id: `${packageName}@${version}`, name: packageName, version });
+        return filterMappedDependencies;
+    }, []);
+    return packages;
+}
+async function parseDependenciesFromNpmLockFile(lockFilepath, includeDevDependencies) {
+    const packageLock = JSON.parse(await fs.promises.readFile(lockFilepath, 'utf8'));
+    const fileVersion = packageLock.lockfileVersion || 1;
+    if (![2, 3].includes(fileVersion)) {
+        throw new Error(`Unsupported package-lock.json version ${fileVersion}`);
+    }
+    const dependencies = collectNpmLockfileDependencies(packageLock, includeDevDependencies);
+    return dependencies.map(({ name, version }) => ({ id: `${name}@${version}`, name, version }));
+}
+async function parseDependenciesFromYarnLockFile(lockFilepath) {
+    const lockFileStream = fs.createReadStream(lockFilepath);
+    const lockFileReadlineInterface = readline.createInterface({
+        input: lockFileStream,
+        crlfDelay: Infinity
+    });
+    for await (const line of lockFileReadlineInterface) {
+        // https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/lockfile/stringify.js#L111
+        if (/# yarn lockfile v1\s*$/.test(line)) {
+            // lockfile version 1 recognized
+            break;
+        }
+        if (/^\s*$/.test(line) || /^\s*#/.test(line)) {
+            // skip empty or comment lines
+            continue;
+        }
+        throw new Error(`Failed to parse file ${lockFilepath} as yarn lockfile, unrecognized format, only version 1 is supported`);
+    }
+    lockFileStream.destroy();
+    const lockfileContents = await fs.promises.readFile(lockFilepath, 'utf8');
+    const { type: lockfileParseStatus, object: packagePatternToLockedVersion } = yarnLockfile.parse(lockfileContents);
+    if (lockfileParseStatus !== 'success') {
+        throw new Error(`Failed to parse file ${lockFilepath} as yarn lockfile, parse status ${lockfileParseStatus}`);
+    }
+    const yarnLockDependencies = Object.entries(packagePatternToLockedVersion).map(([packagePattern, { version }]) => ({ packagePattern, version }));
+    return resolveNpmPackagesFromYarnLockDependencies(yarnLockDependencies);
+}
+export async function dependenciesFromPackageLock(path, includeDevDependencies) {
+    const filename = pathLib.basename(path);
+    const dependencies = filename === YARN_LOCK_FILENAME
+        ? await parseDependenciesFromYarnLockFile(path)
+        : await parseDependenciesFromNpmLockFile(path, includeDevDependencies);
+    return dependencies.sort(sortById).filter(uniqueById).filter(isNotLocal);
+}
+function sortById(a, b) {
+    return a.id.localeCompare(b.id);
+}
+function uniqueById(value, index, values) {
+    return values.findIndex(v => v.id === value.id) === index;
+}
+function isNotLocal(dependency) {
+    // if the version starts with the url scheme 'file:' that means that
+    // the package is fetched from the local filesystem relative to the
+    // package-lock that we were passed; it could for instance be a git
+    // submodule. this package will not be fetched through the web server
+    // that we set up anyway, so don't attempt to synchronize it
+    return !dependency.version.startsWith('file:');
+}
+function collectNpmLockfileDependencies({ packages }, includeDevDependencies) {
+    return Object.entries(packages)
+        .filter(([name, props]) => name.length > 0 && (includeDevDependencies || !props.dev))
+        .map(([name, props]) => ({
+        name: props.name || pathToName(name),
+        version: props.version
+    }));
+}
+// "node_modules/lodash" -> "lodash"
+// "node_modules/make-dir/node_modules/semver" -> "semver"
+function pathToName(path) {
+    const name = path.split('node_modules/').pop();
+    assert(name, `Failed to extract package name from path ${path}`);
+    return name;
+}
+function isDeepEqual(a, b) {
+    try {
+        deepStrictEqual(a, b);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}