registry-sync 7.0.2 → 8.0.0

package/src/pregyp.ts

@@ -0,0 +1,170 @@
+import * as fs from 'fs'
+import * as path from 'path'
+import * as semver from 'semver'
+import * as url from 'url'
+import {fetchBinaryData} from './client.ts'
+import type {PlatformVariant, PrebuiltBinaryMetadata, VersionMetadata, VersionMetadataWithBinary} from './types.d.ts'
+
+export function hasPrebuiltBinaries(metadata: VersionMetadata): metadata is VersionMetadataWithBinary {
+  return Boolean(metadata.binary
+    && metadata.binary.host
+    && metadata.binary.module_name
+    && metadata.binary.package_name
+    && metadata.binary.remote_path
+  )
+}
+
+export async function downloadPrebuiltBinaries(
+  versionMetadata: VersionMetadataWithBinary,
+  localFolder: string,
+  prebuiltBinaryProperties: PlatformVariant[]
+): Promise<void> {
+  const {binary, name, version} = versionMetadata
+
+  if (!binary.napi_versions) {
+    for (const {abi, arch, platform} of prebuiltBinaryProperties) {
+      await downloadPrebuiltBinary(localFolder, name, version, binary, abi, platform, arch)
+    }
+    return
+  }
+
+  for (const napiVersion of binary.napi_versions) {
+    for (const {abi, arch, platform} of prebuiltBinaryProperties) {
+      await downloadPrebuiltBinary(localFolder, name, version, binary, abi, platform, arch, napiVersion)
+    }
+  }
+}
+
+async function downloadPrebuiltBinary(
+  localFolder: string,
+  name: string,
+  version: string,
+  binary: PrebuiltBinaryMetadata,
+  abi: number,
+  platform: string,
+  arch: string,
+  napiVersion?: number
+): Promise<void> {
+  try {
+    const data = await fetchPrebuiltBinary(name, version, binary, abi, platform, arch, napiVersion)
+    await fs.promises.writeFile(
+      prebuiltBinaryFilePath(localFolder, name, version, binary, abi, platform, arch, napiVersion),
+      data
+    )
+  } catch (err: unknown) {
+    // pre-built binaries are commonly not available on all platforms (and S3 will commonly respond with 403 for a non-existent file)
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const fileNotFoundError = (err as any).response && ((err as any).response.status == 403 || (err as any).response.status == 404)
+    if (!fileNotFoundError) {
+      console.error(
+        `Unexpected error fetching prebuilt binary for ${name} and ABI v${abi} on ${arch}-${platform} (n-api version ${napiVersion})`
+      )
+      throw err
+    }
+  }
+}
+
+function fetchPrebuiltBinary(
+  name: string,
+  version: string,
+  binary: PrebuiltBinaryMetadata,
+  abi: number,
+  platform: string,
+  arch: string,
+  napiVersion?: number
+): Promise<Buffer> {
+  return fetchBinaryData(prebuiltBinaryUrl(name, version, binary, abi, platform, arch, napiVersion), '')
+}
+
+function prebuiltBinaryFilePath(
+  localFolder: string,
+  name: string,
+  version: string,
+  binary: PrebuiltBinaryMetadata,
+  abi: number,
+  platform: string,
+  arch: string,
+  napiVersion?: number
+): string {
+  return path.join(localFolder, prebuiltBinaryFileName(name, version, binary, abi, platform, arch, napiVersion))
+}
+
+function prebuiltBinaryUrl(
+  name: string,
+  version: string,
+  binary: PrebuiltBinaryMetadata,
+  abi: number,
+  platform: string,
+  arch: string,
+  napiVersion?: number
+): string {
+  const remotePath = prebuiltBinaryRemotePath(name, version, binary, abi, platform, arch, napiVersion).replace(
+    /\/$/,
+    ''
+  )
+  const fileName = prebuiltBinaryFileName(name, version, binary, abi, platform, arch, napiVersion)
+  return url.resolve(binary.host, `${remotePath}/${fileName}`)
+}
+
+function prebuiltBinaryRemotePath(
+  name: string,
+  version: string,
+  binary: PrebuiltBinaryMetadata,
+  abi: number,
+  platform: string,
+  arch: string,
+  napiVersion?: number
+): string {
+  return formatPrebuilt(binary.remote_path, name, version, binary.module_name, abi, platform, arch, napiVersion)
+}
+
+function prebuiltBinaryFileName(
+  name: string,
+  version: string,
+  binary: PrebuiltBinaryMetadata,
+  abi: number,
+  platform: string,
+  arch: string,
+  napiVersion?: number
+): string {
+  return formatPrebuilt(binary.package_name, name, version, binary.module_name, abi, platform, arch, napiVersion)
+}
+
+// see node-pre-gyp: /lib/util/versioning.js for documentation of possible values
+function formatPrebuilt(
+  formatString: string,
+  name: string,
+  version: string,
+  moduleName: string,
+  abi: number,
+  platform: string,
+  arch: string,
+  napiVersion?: number
+): string {
+  const moduleVersion = semver.parse(version, false, true)
+  const prerelease = (moduleVersion.prerelease || []).join('.')
+  const build = (moduleVersion.build || []).join('.')
+
+  const formatted = formatString
+    .replace('{name}', name)
+    .replace('{version}', version)
+    .replace('{major}', moduleVersion.major.toString())
+    .replace('{minor}', moduleVersion.minor.toString())
+    .replace('{patch}', moduleVersion.patch.toString())
+    .replace('{prerelease}', prerelease)
+    .replace('{build}', build)
+    .replace('{module_name}', moduleName)
+    .replace('{node_abi}', `node-v${abi}`)
+    .replace('{platform}', platform)
+    .replace('{arch}', arch)
+    .replace('{libc}', libc(platform))
+    .replace('{configuration}', 'Release')
+    .replace('{toolset}', '')
+    .replace(/[/]+/g, '/')
+
+  return napiVersion ? formatted.replace('{napi_build_version}', napiVersion.toString()) : formatted
+}
+
+function libc(platform: string): string {
+  return platform === 'linux' ? 'glibc' : 'unknown'
+}

package/src/resolve.ts

@@ -0,0 +1,276 @@
+import * as fs from 'fs'
+import * as pathLib from 'path'
+import * as readline from 'readline'
+import * as url from 'url'
+import assert, {deepStrictEqual} from 'assert'
+import yarnLockfile from '@yarnpkg/lockfile'
+import {normalizeYarnPackagePattern} from './normalize-yarn-pattern.ts'
+import type {
+  CacheSchemaV1,
+  CacheSchemaV2,
+  Package,
+  PackageLock,
+  PackageWithId,
+  PlatformVariant,
+  YarnLockDependency
+} from './types.d.ts'
+
+const YARN_LOCK_FILENAME = 'yarn.lock'
+
+export async function updateDependenciesCache(
+  newDependencies: PackageWithId[],
+  cacheFilePath: string,
+  prebuiltBinaryProperties: PlatformVariant[]
+): Promise<void> {
+  const {dependencies: cachedDependencies} = await loadCache(cacheFilePath)
+  const dependencies = cachedDependencies.concat(newDependencies).sort(sortById).filter(uniqueById)
+
+  const data: CacheSchemaV2 = {
+    dependencies,
+    prebuiltBinaryProperties,
+    prebuiltBinaryNApiSupport: true
+  }
+  return fs.promises.writeFile(cacheFilePath, JSON.stringify(data), 'utf8')
+}
+
+export async function dependenciesNotInCache(
+  dependencies: PackageWithId[],
+  cacheFilePath: string,
+  prebuiltBinaryProperties: PlatformVariant[]
+): Promise<PackageWithId[]> {
+  const {
+    dependencies: cachedDependencies,
+    prebuiltBinaryProperties: cachedPrebuiltBinaryProperties,
+    prebuiltBinaryNApiSupport
+  } = await loadCache(cacheFilePath)
+  if (
+    cachedDependencies.length > 0 &&
+    (!isDeepEqual(prebuiltBinaryProperties, cachedPrebuiltBinaryProperties) || !prebuiltBinaryNApiSupport)
+  ) {
+    console.log(`Pre-built binary properties changed, re-downloading all current packages`)
+    return dependencies
+  }
+  const packageIdsInCache = cachedDependencies.map(pkg => pkg.id)
+  return dependencies.filter(pkg => !packageIdsInCache.includes(pkg.id))
+}
+
+async function loadCache(cacheFilePath: string): Promise<CacheSchemaV2> {
+  try {
+    const data: CacheSchemaV1 | CacheSchemaV2 = JSON.parse(await fs.promises.readFile(cacheFilePath, 'utf8'))
+    // Migrate V1 legacy cache file schema to V2
+    if (Array.isArray(data)) {
+      return {
+        dependencies: data,
+        prebuiltBinaryProperties: [],
+        prebuiltBinaryNApiSupport: false
+      }
+    }
+    return data
+  } catch {
+    // empty V2 cache
+    return {
+      dependencies: [],
+      prebuiltBinaryProperties: [],
+      prebuiltBinaryNApiSupport: true
+    }
+  }
+}
+
+function isNonRegistryYarnPackagePattern(packagePattern: string): boolean {
+  if (
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/link-resolver.js#L14
+    packagePattern.startsWith('link:') ||
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/file-resolver.js#L18
+    packagePattern.startsWith('file:') ||
+    /^\.{1,2}\//.test(packagePattern) ||
+    pathLib.isAbsolute(packagePattern) ||
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/tarball-resolver.js#L15
+    packagePattern.startsWith('http://') ||
+    packagePattern.startsWith('https://') ||
+    (packagePattern.indexOf('@') < 0 && (packagePattern.endsWith('.tgz') || packagePattern.endsWith('.tar.gz'))) ||
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/github-resolver.js#L6
+    packagePattern.startsWith('github:') ||
+    /^[^:@%/\s.-][^:@%/\s]*[/][^:@\s/%]+(?:#.*)?$/.test(packagePattern) ||
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/gitlab-resolver.js#L6
+    packagePattern.startsWith('gitlab:') ||
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/bitbucket-resolver.js#L6
+    packagePattern.startsWith('bitbucket:') ||
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/gist-resolver.js#L26
+    packagePattern.startsWith('gist:') ||
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/git-resolver.js#L19
+    /^git:|^git\+.+:|^ssh:|^https?:.+\.git$|^https?:.+\.git#.+/.test(packagePattern)
+  ) {
+    return true
+  } else {
+    // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/git-resolver.js#L19
+    const {hostname, path} = url.parse(packagePattern)
+    if (hostname && path && ['github.com', 'gitlab.com', 'bitbucket.com', 'bitbucket.org'].indexOf(hostname) >= 0) {
+      return path.split('/').filter((p): boolean => !!p).length === 2
+    }
+  }
+  return false
+}
+
+function resolvePackageNameFromRegistryYarnPackagePattern(packagePattern: string): string {
+  // See https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/resolvers/exotics/registry-resolver.js#L12
+  const match = packagePattern.match(/^(\S+):(@?.*?)(@(.*?)|)$/)
+  if (match) {
+    return match[2]
+  } else {
+    throw new Error(`Failed to resolve yarn package pattern ${packagePattern}, unrecognized format`)
+  }
+}
+
+function resolveNpmPackagesFromYarnLockDependencies(yarnLockDependencies: YarnLockDependency[]): PackageWithId[] {
+  const packages: PackageWithId[] = yarnLockDependencies.reduce(
+    (filterMappedDependencies: PackageWithId[], {packagePattern, version}) => {
+      if (isNonRegistryYarnPackagePattern(packagePattern)) {
+        return filterMappedDependencies
+      }
+
+      let packageName
+
+      if (packagePattern.startsWith('npm:') || packagePattern.startsWith('yarn:')) {
+        packageName = resolvePackageNameFromRegistryYarnPackagePattern(packagePattern)
+      } else {
+        // Package pattern not yet recognized, continue with parsing logic from
+        // https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/package-request.js#L99
+        const {name: namePart, range: rangePart} = normalizeYarnPackagePattern(packagePattern)
+
+        if (isNonRegistryYarnPackagePattern(rangePart)) {
+          return filterMappedDependencies
+        }
+
+        if (rangePart.startsWith('npm:') || rangePart.startsWith('yarn:')) {
+          packageName = resolvePackageNameFromRegistryYarnPackagePattern(rangePart)
+        } else {
+          // Finally, we just assume that the pattern is a registry pattern,
+          // see https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/package-request.js#L119
+          packageName = namePart
+        }
+      }
+
+      filterMappedDependencies.push({id: `${packageName}@${version}`, name: packageName, version})
+
+      return filterMappedDependencies
+    },
+    []
+  )
+
+  return packages
+}
+
+async function parseDependenciesFromNpmLockFile(
+  lockFilepath: string,
+  includeDevDependencies: boolean
+): Promise<PackageWithId[]> {
+  const packageLock: PackageLock = JSON.parse(await fs.promises.readFile(lockFilepath, 'utf8'))
+  const fileVersion = packageLock.lockfileVersion || 1
+  if (![2, 3].includes(fileVersion)) {
+    throw new Error(`Unsupported package-lock.json version ${fileVersion}`)
+  }
+
+  const dependencies = collectNpmLockfileDependencies(packageLock, includeDevDependencies)
+  return dependencies.map(({name, version}) => ({id: `${name}@${version}`, name, version}))
+}
+
+async function parseDependenciesFromYarnLockFile(lockFilepath: string): Promise<PackageWithId[]> {
+  const lockFileStream = fs.createReadStream(lockFilepath)
+  const lockFileReadlineInterface = readline.createInterface({
+    input: lockFileStream,
+    crlfDelay: Infinity
+  })
+
+  for await (const line of lockFileReadlineInterface) {
+    // https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/lockfile/stringify.js#L111
+    if (/# yarn lockfile v1\s*$/.test(line)) {
+      // lockfile version 1 recognized
+      break
+    }
+
+    if (/^\s*$/.test(line) || /^\s*#/.test(line)) {
+      // skip empty or comment lines
+      continue
+    }
+
+    throw new Error(
+      `Failed to parse file ${lockFilepath} as yarn lockfile, unrecognized format, only version 1 is supported`
+    )
+  }
+  lockFileStream.destroy()
+
+  const lockfileContents = await fs.promises.readFile(lockFilepath, 'utf8')
+  const {
+    type: lockfileParseStatus,
+    object: packagePatternToLockedVersion
+  }: {
+    type: 'success' | 'merge' | 'conflict'
+    object: {[packagePattern: string]: {version: string}}
+  } = yarnLockfile.parse(lockfileContents)
+
+  if (lockfileParseStatus !== 'success') {
+    throw new Error(`Failed to parse file ${lockFilepath} as yarn lockfile, parse status ${lockfileParseStatus}`)
+  }
+
+  const yarnLockDependencies: YarnLockDependency[] = Object.entries(packagePatternToLockedVersion).map(
+    ([packagePattern, {version}]) => ({packagePattern, version})
+  )
+
+  return resolveNpmPackagesFromYarnLockDependencies(yarnLockDependencies)
+}
+
+export async function dependenciesFromPackageLock(
+  path: string,
+  includeDevDependencies: boolean
+): Promise<PackageWithId[]> {
+  const filename = pathLib.basename(path)
+  const dependencies =
+    filename === YARN_LOCK_FILENAME
+      ? await parseDependenciesFromYarnLockFile(path)
+      : await parseDependenciesFromNpmLockFile(path, includeDevDependencies)
+
+  return dependencies.sort(sortById).filter(uniqueById).filter(isNotLocal)
+}
+
+function sortById(a: PackageWithId, b: PackageWithId): number {
+  return a.id.localeCompare(b.id)
+}
+
+function uniqueById(value: PackageWithId, index: number, values: PackageWithId[]): boolean {
+  return values.findIndex(v => v.id === value.id) === index
+}
+
+function isNotLocal(dependency: PackageWithId): boolean {
+  // if the version starts with the url scheme 'file:' that means that
+  // the package is fetched from the local filesystem relative to the
+  // package-lock that we were passed; it could for instance be a git
+  // submodule. this package will not be fetched through the web server
+  // that we set up anyway, so don't attempt to synchronize it
+  return !dependency.version.startsWith('file:')
+}
+
+function collectNpmLockfileDependencies({packages}: PackageLock, includeDevDependencies: boolean): Package[] {
+  return Object.entries(packages)
+    .filter(([name, props]) => name.length > 0 && (includeDevDependencies || !props.dev))
+    .map(([name, props]) => ({
+      name: props.name || pathToName(name),
+      version: props.version
+    }))
+}
+
+// "node_modules/lodash" -> "lodash"
+// "node_modules/make-dir/node_modules/semver" -> "semver"
+function pathToName(path: string) {
+  const name = path.split('node_modules/').pop()
+  assert(name, `Failed to extract package name from path ${path}`)
+  return name
+}
+
+function isDeepEqual(a: PlatformVariant[], b: PlatformVariant[]): boolean {
+  try {
+    deepStrictEqual(a, b)
+    return true
+  } catch {
+    return false
+  }
+}

package/src/sync.ts

@@ -0,0 +1,21 @@
+import {dependenciesFromPackageLock, dependenciesNotInCache, updateDependenciesCache} from './resolve.ts'
+import {downloadAll} from './download.ts'
+import type {CommandLineOptions, PackageWithId} from './types.d.ts'
+
+export async function synchronize(options: CommandLineOptions): Promise<PackageWithId[]> {
+  const cacheFilePath = `${options.rootFolder}/.registry-sync-cache.json`
+
+  const packages = await dependenciesFromPackageLock(options.manifest, options.includeDevDependencies)
+  const newPackages = await dependenciesNotInCache(packages, cacheFilePath, options.prebuiltBinaryProperties)
+
+  if (options.dryRun) {
+    console.log(newPackages.map(({name, version}) => `${name}@${version}`).join('\n'))
+    console.log(`\nWould download ${newPackages.length} packages.`)
+  } else {
+    await downloadAll(newPackages, options)
+    await updateDependenciesCache(newPackages, cacheFilePath, options.prebuiltBinaryProperties)
+    console.log(`Downloaded ${newPackages.length} packages`)
+  }
+
+  return newPackages
+}

package/src/types.d.ts

@@ -0,0 +1,99 @@
+import type {URL} from 'url'
+
+export interface CommandLineOptions {
+  localUrl: URL
+  manifest: string
+  prebuiltBinaryProperties: PlatformVariant[]
+  registryUrl: string
+  registryToken: string
+  rootFolder: string
+  enforceTarballsOverHttps: boolean
+  includeDevDependencies: boolean
+  dryRun: boolean
+}
+
+export interface PackageLock {
+  name: string
+  version: string
+  lockfileVersion?: number
+  packages: {
+    [path: string]: {
+      version: string
+      // only aliased packages have the name property
+      name?: string
+      dev?: true
+    }
+  }
+}
+
+export interface YarnLockDependency {
+  packagePattern: string
+  version: string
+}
+
+export interface PackageWithId extends Package {
+  id: string
+}
+
+export interface Package {
+  name: string
+  version: string
+}
+
+export interface CacheSchemaV2 {
+  dependencies: PackageWithId[]
+  prebuiltBinaryProperties: PlatformVariant[]
+  prebuiltBinaryNApiSupport: boolean
+}
+
+export type CacheSchemaV1 = PackageWithId[]
+
+export interface PlatformVariant {
+  abi: number
+  arch: string
+  platform: string
+}
+
+export interface RegistryMetadata {
+  _id: string
+  versions: {
+    [name: string]: VersionMetadata
+  }
+  time: {
+    modified?: string
+    created?: string
+    [version: string]: string
+  }
+  'dist-tags': {
+    latest?: string
+    [tag: string]: string
+  }
+}
+
+export interface VersionMetadata {
+  _id: string
+  name: string
+  version: string
+  binary?: PrebuiltBinaryMetadata
+  dist: {
+    shasum?: string
+    integrity?: string
+    tarball: string
+  }
+}
+
+// The following properties may be present but are ignored by registry-sync
+// module_path?: string
+// remote_uri?: string
+export interface PrebuiltBinaryMetadata {
+  host: string
+  module_name: string
+  napi_versions?: number[]
+  package_name: string
+  remote_path: string
+  template: string
+}
+
+export interface VersionMetadataWithBinary extends VersionMetadata {
+  binary: PrebuiltBinaryMetadata
+}

package/src/client.js

@@ -1,30 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchJsonWithCacheCloned = fetchJsonWithCacheCloned;
-exports.fetchBinaryData = fetchBinaryData;
-const https = require("https");
-const axios_1 = require("axios");
-const lru_cache_1 = require("lru-cache");
-const metadataCache = new lru_cache_1.LRUCache({ max: 100 });
-const client = axios_1.default.create({
-    httpsAgent: new https.Agent({ keepAlive: true }),
-    timeout: 30 * 1000
-});
-async function fetchJsonWithCacheCloned(url, token) {
-    if (metadataCache.has(url)) {
-        return structuredClone(metadataCache.get(url));
-    }
-    const value = await fetch(url, 'json', token);
-    metadataCache.set(url, value);
-    return structuredClone(value);
-}
-function fetchBinaryData(url, token) {
-    return fetch(url, 'arraybuffer', token);
-}
-async function fetch(url, responseType, token) {
-    const config = { responseType };
-    if (token !== '') {
-        config.headers = { authorization: 'Bearer ' + token };
-    }
-    return (await client.get(url, config)).data;
-}

package/src/download.js

@@ -1,89 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.downloadAll = downloadAll;
-const fs = require("fs");
-const path = require("path");
-const semver = require("semver");
-const url = require("url");
-const integrity_1 = require("./integrity");
-const pregyp_1 = require("./pregyp");
-const client_1 = require("./client");
-const metadata_1 = require("./metadata");
-async function downloadAll(packages, { localUrl, prebuiltBinaryProperties, registryUrl, registryToken, rootFolder, enforceTarballsOverHttps }) {
-    const downloadFromRegistry = download.bind(null, registryUrl, registryToken, localUrl, rootFolder, prebuiltBinaryProperties, enforceTarballsOverHttps);
-    for (const pkg of packages) {
-        await downloadFromRegistry(pkg);
-    }
-}
-async function download(registryUrl, registryToken, localUrl, rootFolder, prebuiltBinaryProperties, enforceTarballsOverHttps, { name, version }) {
-    const registryMetadata = await fetchMetadataCloned(name, registryUrl, registryToken);
-    const versionMetadata = registryMetadata.versions[version];
-    if (!versionMetadata) {
-        throw new Error(`Unknown package version ${name}@${version}`);
-    }
-    const localFolder = await ensureLocalFolderExists(name, rootFolder);
-    let data = await downloadTarball(versionMetadata, enforceTarballsOverHttps, registryToken);
-    if ((0, pregyp_1.hasPrebuiltBinaries)(versionMetadata)) {
-        const localPregypFolder = await ensureLocalFolderExists(version, localFolder);
-        await (0, pregyp_1.downloadPrebuiltBinaries)(versionMetadata, localPregypFolder, prebuiltBinaryProperties);
-        data = await (0, metadata_1.rewriteMetadataInTarball)(data, versionMetadata, localUrl, localFolder);
-    }
-    await saveTarball(versionMetadata, data, localFolder);
-    (0, metadata_1.rewriteVersionMetadata)(versionMetadata, data, localUrl);
-    await updateMetadata(versionMetadata, registryMetadata, registryUrl, localFolder);
-}
-async function downloadTarball({ _id: id, dist }, enforceTarballsOverHttps, registryToken) {
-    const tarballUrl = enforceTarballsOverHttps ? dist.tarball.replace('http://', 'https://') : dist.tarball;
-    const data = await (0, client_1.fetchBinaryData)(tarballUrl, registryToken);
-    (0, integrity_1.verifyIntegrity)(data, id, dist);
-    return data;
-}
-function saveTarball({ name, version }, data, localFolder) {
-    return fs.promises.writeFile(tarballPath(name, version, localFolder), data);
-}
-async function updateMetadata(versionMetadata, defaultMetadata, registryUrl, localFolder) {
-    const { version } = versionMetadata;
-    const localMetadataPath = path.join(localFolder, 'index.json');
-    const localMetadata = await loadMetadata(localMetadataPath, defaultMetadata);
-    localMetadata.versions[version] = versionMetadata;
-    localMetadata.time[version] = defaultMetadata.time[version];
-    localMetadata['dist-tags'] = collectDistTags(localMetadata, defaultMetadata);
-    await saveMetadata(localMetadataPath, localMetadata);
-}
-// Collect thise dist-tags entries (name -> version) from registry metadata,
-// which point to versions we have locally available.
-// Override 'latest' tag to ensure its validity as we might not have the version
-// that is tagged latest in registry
-function collectDistTags(localMetadata, defaultMetadata) {
-    const availableVersions = Object.keys(localMetadata.versions);
-    const validDistTags = Object.entries(defaultMetadata['dist-tags']).filter(([, version]) => availableVersions.includes(version));
-    return {
-        ...Object.fromEntries(validDistTags),
-        latest: availableVersions.sort(semver.compare).pop()
-    };
-}
-async function loadMetadata(path, defaultMetadata) {
-    try {
-        const json = await fs.promises.readFile(path, 'utf8');
-        return JSON.parse(json);
-    }
-    catch {
-        return { ...defaultMetadata, 'dist-tags': {}, time: {}, versions: {} };
-    }
-}
-function saveMetadata(path, metadata) {
-    const json = JSON.stringify(metadata, null, 2);
-    return fs.promises.writeFile(path, json, 'utf8');
-}
-function tarballPath(name, version, localFolder) {
-    return path.join(localFolder, (0, metadata_1.tarballFilename)(name, version));
-}
-async function ensureLocalFolderExists(name, rootFolder) {
-    const localFolder = path.resolve(rootFolder, name);
-    await fs.promises.mkdir(localFolder, { recursive: true });
-    return localFolder;
-}
-function fetchMetadataCloned(name, registryUrl, registryToken) {
-    const urlSafeName = name.replace(/\//g, '%2f');
-    return (0, client_1.fetchJsonWithCacheCloned)(url.resolve(registryUrl, urlSafeName), registryToken);
-}