registry-sync 7.1.0 → 8.0.0

package/src/index.js

@@ -1,48 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const fs = require("fs");
-const path = require("path");
-const commander_1 = require("commander");
-const sync_1 = require("./sync");
-const url_1 = require("url");
-const { version } = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf-8'));
-const program = new commander_1.Command();
-program
-    .version(version)
-    .requiredOption('--root <path>', 'Path to save NPM package tarballs and metadata to')
-    .requiredOption('--manifest <file>', 'Path to a package-lock.json or yarn.lock file to use as catalog for mirrored NPM packages.')
-    .requiredOption('--localUrl <url>', 'URL to use as root in stored package metadata (i.e. where folder defined as --root will be exposed at)')
-    .option('--binaryAbi <list>', 'Comma-separated list of node C++ ABI numbers to download pre-built binaries for. See NODE_MODULE_VERSION column in https://nodejs.org/en/download/releases/')
-    .option('--binaryArch <list>', 'Comma-separated list of CPU architectures to download pre-built binaries for. Valid values: arm, ia32, and x64')
-    .option('--binaryPlatform <list>', 'Comma-separated list of OS platforms to download pre-built binaries for. Valid values: linux, darwin, win32, sunos, freebsd, openbsd, and aix')
-    .option('--registryUrl [url]', 'Optional URL to use as NPM registry when fetching packages. Default value is https://registry.npmjs.org')
-    .option('--registryToken [string]', 'Optional Bearer token for the registry.')
-    .option('--dontEnforceHttps', 'Disable the default behavior of downloading tarballs over HTTPS (will use whichever protocol is defined in the registry metadata)')
-    .option('--includeDev', 'Include also packages found from devDependencies section of the --manifest')
-    .option('--dryRun', 'Print packages that would be downloaded but do not download them')
-    .parse(process.argv);
-const rawOptions = program.opts();
-// use current (abi,arch,platform) triplet as default if none is specified
-// so the user doesn't have to look them up if build is always done on the
-// same kind of machine
-const binaryAbi = rawOptions.binaryAbi || process.versions.modules;
-const binaryArch = rawOptions.binaryArch || process.arch;
-const binaryPlatform = rawOptions.binaryPlatform || process.platform;
-const abis = binaryAbi.split(',').map(Number);
-const architectures = binaryArch.split(',');
-const platforms = binaryPlatform.split(',');
-const prebuiltBinaryProperties = abis
-    .map(abi => architectures.map(arch => platforms.map(platform => ({ abi, arch, platform }))).flat())
-    .flat();
-const options = {
-    localUrl: new url_1.URL(rawOptions.localUrl),
-    manifest: rawOptions.manifest,
-    prebuiltBinaryProperties,
-    registryUrl: rawOptions.registryUrl || 'https://registry.npmjs.org',
-    registryToken: rawOptions.registryToken || '',
-    rootFolder: rawOptions.root,
-    enforceTarballsOverHttps: Boolean(!rawOptions.dontEnforceHttps),
-    includeDevDependencies: Boolean(rawOptions.includeDev),
-    dryRun: Boolean(rawOptions.dryRun)
-};
-(0, sync_1.synchronize)(options);

package/src/integrity.js

@@ -1,25 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.verifyIntegrity = verifyIntegrity;
-exports.sha1 = sha1;
-exports.sha512 = sha512;
-const ssri = require("ssri");
-function verifyIntegrity(data, id, { integrity, shasum }) {
-    if (!integrity && !shasum) {
-        throw new Error(`Integrity values not present in metadata for ${id}`);
-    }
-    if (integrity) {
-        if (!ssri.checkData(data, integrity)) {
-            throw new Error(`Integrity check failed for ${id}`);
-        }
-    }
-    else if (sha1(data) != shasum) {
-        throw new Error(`Integrity check with SHA1 failed for failed for ${id}`);
-    }
-}
-function sha1(data) {
-    return ssri.fromData(data, { algorithms: ['sha1'] }).hexDigest();
-}
-function sha512(data) {
-    return ssri.fromData(data, { algorithms: ['sha512'] }).toString();
-}

package/src/metadata.js

@@ -1,65 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.rewriteVersionMetadata = rewriteVersionMetadata;
-exports.rewriteMetadataInTarball = rewriteMetadataInTarball;
-exports.extractTgz = extractTgz;
-exports.tarballFilename = tarballFilename;
-const fs = require("fs");
-const path = require("path");
-const tar = require("tar-fs");
-const zlib = require("zlib");
-const pregyp_1 = require("./pregyp");
-const stream_1 = require("stream");
-const integrity_1 = require("./integrity");
-function rewriteVersionMetadata(versionMetadata, data, localUrl) {
-    versionMetadata.dist.tarball = localTarballUrl(versionMetadata, localUrl);
-    if ((0, pregyp_1.hasPrebuiltBinaries)(versionMetadata)) {
-        versionMetadata.binary.host = localUrl.origin;
-        versionMetadata.binary.remote_path = createPrebuiltBinaryRemotePath(localUrl, versionMetadata);
-        versionMetadata.dist.integrity = (0, integrity_1.sha512)(data);
-        versionMetadata.dist.shasum = (0, integrity_1.sha1)(data);
-    }
-}
-async function rewriteMetadataInTarball(data, versionMetadata, localUrl, localFolder) {
-    const tmpFolder = path.join(localFolder, '.tmp');
-    await fs.promises.mkdir(tmpFolder, { recursive: true });
-    await extractTgz(data, tmpFolder);
-    const manifestPath = path.join(tmpFolder, 'package', 'package.json');
-    const json = await fs.promises.readFile(manifestPath, 'utf8');
-    const metadata = JSON.parse(json);
-    metadata.binary.host = localUrl.origin;
-    metadata.binary.remote_path = createPrebuiltBinaryRemotePath(localUrl, versionMetadata);
-    await fs.promises.writeFile(manifestPath, JSON.stringify(metadata, null, 2));
-    const updatedData = await compressTgz(tmpFolder);
-    await fs.promises.rm(tmpFolder, { recursive: true });
-    return updatedData;
-}
-function createPrebuiltBinaryRemotePath(url, versionMetadata) {
-    return `${removeTrailingSlash(url.pathname)}/${versionMetadata.name}/${versionMetadata.version}/`;
-}
-function extractTgz(data, folder) {
-    return new Promise((resolve, reject) => {
-        const tgz = stream_1.Readable.from(data).pipe(zlib.createGunzip()).pipe(tar.extract(folder));
-        tgz.on('finish', resolve);
-        tgz.on('error', reject);
-    });
-}
-function compressTgz(folder) {
-    return new Promise((resolve, reject) => {
-        const chunks = [];
-        const tgz = tar.pack(folder).pipe(zlib.createGzip());
-        tgz.on('data', (chunk) => chunks.push(chunk));
-        tgz.on('end', () => resolve(Buffer.concat(chunks)));
-        tgz.on('error', reject);
-    });
-}
-function localTarballUrl({ name, version }, localUrl) {
-    return `${localUrl.origin}${removeTrailingSlash(localUrl.pathname)}/${name}/${tarballFilename(name, version)}`;
-}
-function tarballFilename(name, version) {
-    const normalized = name.replace(/\//g, '-');
-    return `${normalized}-${version}.tgz`;
-}
-function removeTrailingSlash(str) {
-    return str.replace(/\/$/, '');
-}

package/src/pregyp.js

@@ -1,83 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.hasPrebuiltBinaries = hasPrebuiltBinaries;
-exports.downloadPrebuiltBinaries = downloadPrebuiltBinaries;
-const fs = require("fs");
-const path = require("path");
-const semver = require("semver");
-const url = require("url");
-const client_1 = require("./client");
-function hasPrebuiltBinaries({ binary }) {
-    return Boolean(binary && binary.module_name);
-}
-async function downloadPrebuiltBinaries(versionMetadata, localFolder, prebuiltBinaryProperties) {
-    const { binary, name, version } = versionMetadata;
-    if (!binary.napi_versions) {
-        for (const { abi, arch, platform } of prebuiltBinaryProperties) {
-            await downloadPrebuiltBinary(localFolder, name, version, binary, abi, platform, arch);
-        }
-        return;
-    }
-    for (const napiVersion of binary.napi_versions) {
-        for (const { abi, arch, platform } of prebuiltBinaryProperties) {
-            await downloadPrebuiltBinary(localFolder, name, version, binary, abi, platform, arch, napiVersion);
-        }
-    }
-}
-async function downloadPrebuiltBinary(localFolder, name, version, binary, abi, platform, arch, napiVersion) {
-    try {
-        const data = await fetchPrebuiltBinary(name, version, binary, abi, platform, arch, napiVersion);
-        await fs.promises.writeFile(prebuiltBinaryFilePath(localFolder, name, version, binary, abi, platform, arch, napiVersion), data);
-    }
-    catch (err) {
-        // pre-built binaries are commonly not available on all platforms (and S3 will commonly respond with 403 for a non-existent file)
-        const fileNotFoundError = err.response && (err.response.status == 403 || err.response.status == 404);
-        if (!fileNotFoundError) {
-            console.error(`Unexpected error fetching prebuilt binary for ${name} and ABI v${abi} on ${arch}-${platform} (n-api version ${napiVersion})`);
-            throw err;
-        }
-    }
-}
-function fetchPrebuiltBinary(name, version, binary, abi, platform, arch, napiVersion) {
-    return (0, client_1.fetchBinaryData)(prebuiltBinaryUrl(name, version, binary, abi, platform, arch, napiVersion), '');
-}
-function prebuiltBinaryFilePath(localFolder, name, version, binary, abi, platform, arch, napiVersion) {
-    return path.join(localFolder, prebuiltBinaryFileName(name, version, binary, abi, platform, arch, napiVersion));
-}
-function prebuiltBinaryUrl(name, version, binary, abi, platform, arch, napiVersion) {
-    const remotePath = prebuiltBinaryRemotePath(name, version, binary, abi, platform, arch, napiVersion).replace(/\/$/, '');
-    const fileName = prebuiltBinaryFileName(name, version, binary, abi, platform, arch, napiVersion);
-    return url.resolve(binary.host, `${remotePath}/${fileName}`);
-}
-function prebuiltBinaryRemotePath(name, version, binary, abi, platform, arch, napiVersion) {
-    return formatPrebuilt(binary.remote_path, name, version, binary.module_name, abi, platform, arch, napiVersion);
-}
-function prebuiltBinaryFileName(name, version, binary, abi, platform, arch, napiVersion) {
-    return formatPrebuilt(binary.package_name, name, version, binary.module_name, abi, platform, arch, napiVersion);
-}
-// see node-pre-gyp: /lib/util/versioning.js for documentation of possible values
-function formatPrebuilt(formatString, name, version, moduleName, abi, platform, arch, napiVersion) {
-    const moduleVersion = semver.parse(version);
-    const prerelease = (moduleVersion.prerelease || []).join('.');
-    const build = (moduleVersion.build || []).join('.');
-    const formatted = formatString
-        .replace('{name}', name)
-        .replace('{version}', version)
-        .replace('{major}', moduleVersion.major.toString())
-        .replace('{minor}', moduleVersion.minor.toString())
-        .replace('{patch}', moduleVersion.patch.toString())
-        .replace('{prerelease}', prerelease)
-        .replace('{build}', build)
-        .replace('{module_name}', moduleName)
-        .replace('{node_abi}', `node-v${abi}`)
-        .replace('{platform}', platform)
-        .replace('{arch}', arch)
-        .replace('{libc}', libc(platform))
-        .replace('{configuration}', 'Release')
-        .replace('{toolset}', '')
-        .replace(/[/]+/g, '/');
-    return napiVersion ? formatted.replace('{napi_build_version}', napiVersion.toString()) : formatted;
-}
-function libc(platform) {
-    return platform === 'linux' ? 'glibc' : 'unknown';
-}

package/src/resolve.js

@@ -1,207 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.updateDependenciesCache = updateDependenciesCache;
-exports.dependenciesNotInCache = dependenciesNotInCache;
-exports.dependenciesFromPackageLock = dependenciesFromPackageLock;
-const fs = require("fs");
-const pathLib = require("path");
-const readline = require("readline");
-const url = require("url");
-const assert_1 = require("assert");
-const yarnLockfile = require("@yarnpkg/lockfile");
-const normalize_yarn_pattern_1 = require("./normalize-yarn-pattern");
-const YARN_LOCK_FILENAME = 'yarn.lock';
-async function updateDependenciesCache(newDependencies, cacheFilePath, prebuiltBinaryProperties) {
-    const { dependencies: cachedDependencies } = await loadCache(cacheFilePath);
-    const dependencies = cachedDependencies.concat(newDependencies).sort(sortById).filter(uniqueById);
-    const data = {
-        dependencies,
-        prebuiltBinaryProperties,
-        prebuiltBinaryNApiSupport: true
-    };
-    return fs.promises.writeFile(cacheFilePath, JSON.stringify(data), 'utf8');
-}
-async function dependenciesNotInCache(dependencies, cacheFilePath, prebuiltBinaryProperties) {
-    const { dependencies: cachedDependencies, prebuiltBinaryProperties: cachedPrebuiltBinaryProperties, prebuiltBinaryNApiSupport } = await loadCache(cacheFilePath);
-    if (cachedDependencies.length > 0 &&
-        (!isDeepEqual(prebuiltBinaryProperties, cachedPrebuiltBinaryProperties) || !prebuiltBinaryNApiSupport)) {
-        console.log(`Pre-built binary properties changed, re-downloading all current packages`);
-        return dependencies;
-    }
-    const packageIdsInCache = cachedDependencies.map(pkg => pkg.id);
-    return dependencies.filter(pkg => !packageIdsInCache.includes(pkg.id));
-}
-async function loadCache(cacheFilePath) {
-    try {
-        const data = JSON.parse(await fs.promises.readFile(cacheFilePath, 'utf8'));
-        // Migrate V1 legacy cache file schema to V2
-        if (Array.isArray(data)) {
-            return {
-                dependencies: data,
-                prebuiltBinaryProperties: [],
-                prebuiltBinaryNApiSupport: false
-            };
-        }
-        return data;
-    }
-    catch {
-        // empty V2 cache
-        return {
-            dependencies: [],
-            prebuiltBinaryProperties: [],
-            prebuiltBinaryNApiSupport: true
-        };
-    }
-}
-function isNonRegistryYarnPackagePattern(packagePattern) {
-    if (
-    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/link-resolver.js#L14
-    packagePattern.startsWith('link:') ||
-        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/file-resolver.js#L18
-        packagePattern.startsWith('file:') ||
-        /^\.{1,2}\//.test(packagePattern) ||
-        pathLib.isAbsolute(packagePattern) ||
-        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/tarball-resolver.js#L15
-        packagePattern.startsWith('http://') ||
-        packagePattern.startsWith('https://') ||
-        (packagePattern.indexOf('@') < 0 && (packagePattern.endsWith('.tgz') || packagePattern.endsWith('.tar.gz'))) ||
-        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/github-resolver.js#L6
-        packagePattern.startsWith('github:') ||
-        /^[^:@%/\s.-][^:@%/\s]*[/][^:@\s/%]+(?:#.*)?$/.test(packagePattern) ||
-        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/gitlab-resolver.js#L6
-        packagePattern.startsWith('gitlab:') ||
-        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/bitbucket-resolver.js#L6
-        packagePattern.startsWith('bitbucket:') ||
-        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/gist-resolver.js#L26
-        packagePattern.startsWith('gist:') ||
-        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/git-resolver.js#L19
-        /^git:|^git\+.+:|^ssh:|^https?:.+\.git$|^https?:.+\.git#.+/.test(packagePattern)) {
-        return true;
-    }
-    else {
-        // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/git-resolver.js#L19
-        const { hostname, path } = url.parse(packagePattern);
-        if (hostname && path && ['github.com', 'gitlab.com', 'bitbucket.com', 'bitbucket.org'].indexOf(hostname) >= 0) {
-            return path.split('/').filter((p) => !!p).length === 2;
-        }
-    }
-}
-function resolvePackageNameFromRegistryYarnPackagePattern(packagePattern) {
-    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/registry-resolver.js#L12
-    const match = packagePattern.match(/^(\S+):(@?.*?)(@(.*?)|)$/);
-    if (match) {
-        return match[2];
-    }
-    else {
-        throw new Error(`Failed to resolve yarn package pattern ${packagePattern}, unrecognized format`);
-    }
-}
-function resolveNpmPackagesFromYarnLockDependencies(yarnLockDependencies) {
-    const packages = yarnLockDependencies.reduce((filterMappedDependencies, { packagePattern, version }) => {
-        if (isNonRegistryYarnPackagePattern(packagePattern)) {
-            return filterMappedDependencies;
-        }
-        let packageName;
-        if (packagePattern.startsWith('npm:') || packagePattern.startsWith('yarn:')) {
-            packageName = resolvePackageNameFromRegistryYarnPackagePattern(packagePattern);
-        }
-        else {
-            // Package pattern not yet recognized, continue with parsing logic from
-            // https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/package-request.js#L99
-            const { name: namePart, range: rangePart } = (0, normalize_yarn_pattern_1.normalizeYarnPackagePattern)(packagePattern);
-            if (isNonRegistryYarnPackagePattern(rangePart)) {
-                return filterMappedDependencies;
-            }
-            if (rangePart.startsWith('npm:') || rangePart.startsWith('yarn:')) {
-                packageName = resolvePackageNameFromRegistryYarnPackagePattern(rangePart);
-            }
-            else {
-                // Finally, we just assume that the pattern is a registry pattern,
-                // see https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/package-request.js#L119
-                packageName = namePart;
-            }
-        }
-        filterMappedDependencies.push({ id: `${packageName}@${version}`, name: packageName, version });
-        return filterMappedDependencies;
-    }, []);
-    return packages;
-}
-async function parseDependenciesFromNpmLockFile(lockFilepath, includeDevDependencies) {
-    const packageLock = JSON.parse(await fs.promises.readFile(lockFilepath, 'utf8'));
-    const fileVersion = packageLock.lockfileVersion || 1;
-    if (![2, 3].includes(packageLock.lockfileVersion)) {
-        throw new Error(`Unsupported package-lock.json version ${fileVersion}`);
-    }
-    const dependencies = collectNpmLockfileDependencies(packageLock, includeDevDependencies);
-    return dependencies.map(({ name, version }) => ({ id: `${name}@${version}`, name, version }));
-}
-async function parseDependenciesFromYarnLockFile(lockFilepath) {
-    const lockFileStream = fs.createReadStream(lockFilepath);
-    const lockFileReadlineInterface = readline.createInterface({
-        input: lockFileStream,
-        crlfDelay: Infinity
-    });
-    for await (const line of lockFileReadlineInterface) {
-        // https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/lockfile/stringify.js#L111
-        if (/# yarn lockfile v1\s*$/.test(line)) {
-            // lockfile version 1 recognized
-            break;
-        }
-        if (/^\s*$/.test(line) || /^\s*#/.test(line)) {
-            // skip empty or comment lines
-            continue;
-        }
-        throw new Error(`Failed to parse file ${lockFilepath} as yarn lockfile, unrecognized format, only version 1 is supported`);
-    }
-    lockFileStream.destroy();
-    const lockfileContents = await fs.promises.readFile(lockFilepath, 'utf8');
-    const { type: lockfileParseStatus, object: packagePatternToLockedVersion } = yarnLockfile.parse(lockfileContents);
-    if (lockfileParseStatus !== 'success') {
-        throw new Error(`Failed to parse file ${lockFilepath} as yarn lockfile, parse status ${lockfileParseStatus}`);
-    }
-    const yarnLockDependencies = Object.entries(packagePatternToLockedVersion).map(([packagePattern, { version }]) => ({ packagePattern, version }));
-    return resolveNpmPackagesFromYarnLockDependencies(yarnLockDependencies);
-}
-async function dependenciesFromPackageLock(path, includeDevDependencies) {
-    const filename = pathLib.basename(path);
-    const dependencies = filename === YARN_LOCK_FILENAME
-        ? await parseDependenciesFromYarnLockFile(path)
-        : await parseDependenciesFromNpmLockFile(path, includeDevDependencies);
-    return dependencies.sort(sortById).filter(uniqueById).filter(isNotLocal);
-}
-function sortById(a, b) {
-    return a.id.localeCompare(b.id);
-}
-function uniqueById(value, index, values) {
-    return values.findIndex(v => v.id === value.id) === index;
-}
-function isNotLocal(dependency) {
-    // if the version starts with the url scheme 'file:' that means that
-    // the package is fetched from the local filesystem relative to the
-    // package-lock that we were passed; it could for instance be a git
-    // submodule. this package will not be fetched through the web server
-    // that we set up anyway, so don't attempt to synchronize it
-    return !dependency.version.startsWith('file:');
-}
-function collectNpmLockfileDependencies({ packages }, includeDevDependencies) {
-    return Object.entries(packages)
-        .filter(([name, props]) => name.length > 0 && (includeDevDependencies || !props.dev))
-        .map(([name, props]) => ({
-        name: props.name || pathToName(name),
-        version: props.version
-    }));
-}
-// "node_modules/lodash" -> "lodash"
-// "node_modules/make-dir/node_modules/semver" -> "semver"
-function pathToName(path) {
-    return path.split('node_modules/').pop();
-}
-function isDeepEqual(a, b) {
-    try {
-        (0, assert_1.deepStrictEqual)(a, b);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}

package/src/sync.js

@@ -1,20 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.synchronize = synchronize;
-const download_1 = require("./download");
-const resolve_1 = require("./resolve");
-async function synchronize(options) {
-    const cacheFilePath = `${options.rootFolder}/.registry-sync-cache.json`;
-    const packages = await (0, resolve_1.dependenciesFromPackageLock)(options.manifest, options.includeDevDependencies);
-    const newPackages = await (0, resolve_1.dependenciesNotInCache)(packages, cacheFilePath, options.prebuiltBinaryProperties);
-    if (options.dryRun) {
-        console.log(newPackages.map(({ name, version }) => `${name}@${version}`).join('\n'));
-        console.log(`\nWould download ${newPackages.length} packages.`);
-    }
-    else {
-        await (0, download_1.downloadAll)(newPackages, options);
-        await (0, resolve_1.updateDependenciesCache)(newPackages, cacheFilePath, options.prebuiltBinaryProperties);
-        console.log(`Downloaded ${newPackages.length} packages`);
-    }
-    return newPackages;
-}