registry-sync 7.1.0 → 8.0.0

package/README.md

@@ -8,7 +8,7 @@ The local copy can then be used as a simple private NPM registry without publish
 
 ## Pre-requisites
 
-- Node.js v20.17.0 or newer
+- Node.js v22.18.0 or newer
 
 ## Installation
 
@@ -118,5 +118,5 @@ See [releases](https://github.com/heikkipora/registry-sync/releases).
 
 ## Contributing
 
-Pull requests are welcome. Kindly check that your code passes ESLint checks by running `npm run eslint:check` first.
+Pull requests are welcome. Kindly check that your code passes ESLint checks by running `npm run eslint` first.
 Integration tests can be run with `npm test`. Both are anyway run automatically by GitHub Actions.

package/bin/sync

@@ -1,4 +1,3 @@
 #!/usr/bin/env node
 
-const path = require('path')
-require(path.join(__dirname, '..', 'src'))
+await import('../src/index.ts')

package/package.json

@@ -1,19 +1,18 @@
 {
   "name": "registry-sync",
-  "version": "7.1.0",
+  "version": "8.0.0",
   "description": "synchronize a remote npm registry for private use",
   "repository": "https://github.com/heikkipora/registry-sync",
+  "type": "module",
   "bin": {
     "registry-sync": "bin/sync"
   },
   "scripts": {
     "build": "./build-npm",
-    "prettier": "prettier --write .",
-    "prettier:check": "prettier --check --loglevel warn .",
-    "eslint": "eslint --fix --format=codeframe",
-    "eslint:check": "eslint --max-warnings=0 --format=codeframe",
-    "lint-staged": "lint-staged --verbose",
-    "test": "mocha -r ts-node/register --config test/.mocharc.js --timeout 120000 test/*.ts",
+    "eslint": "eslint --max-warnings=0 src test release-test/server/src",
+    "fix-eslint": "eslint --fix src test release-test/server/src",
+    "test": "mocha --timeout 120000 test/*.ts",
+    "typecheck": "tsc",
     "release-test": "cd release-test && ./run-sync-install-cycle.sh"
   },
   "author": "Heikki Pora",
@@ -22,37 +21,28 @@
     "@yarnpkg/lockfile": "1.1.0",
     "axios": "1.13.2",
     "commander": "14.0.2",
-    "lru-cache": "11.2.2",
+    "lru-cache": "11.2.4",
     "semver": "7.7.3",
     "ssri": "13.0.0",
     "tar-fs": "3.1.1"
   },
   "devDependencies": {
-    "@arkweid/lefthook": "0.7.7",
-    "@eslint/eslintrc": "3.3.1",
-    "@eslint/js": "9.39.1",
     "@types/chai": "5.2.3",
-    "@types/lodash": "4.17.20",
+    "@types/lodash": "4.17.21",
     "@types/mocha": "10.0.10",
     "@types/node": "20.17.32",
     "@types/semver": "7.7.1",
     "@types/ssri": "7.1.5",
     "@types/tar-fs": "2.0.4",
     "@types/yarnpkg__lockfile": "1.1.9",
-    "@typescript-eslint/eslint-plugin": "8.46.4",
-    "@typescript-eslint/parser": "8.46.4",
-    "chai": "6.2.1",
-    "eslint": "9.39.1",
-    "eslint-config-prettier": "10.1.8",
-    "eslint-formatter-codeframe": "7.32.2",
+    "chai": "6.2.2",
+    "eslint": "9.39.2",
     "eslint-plugin-mocha": "11.2.0",
-    "express": "5.1.0",
-    "globals": "16.5.0",
-    "lint-staged": "16.2.6",
+    "express": "5.2.1",
+    "globals": "17.0.0",
     "mocha": "11.7.5",
-    "prettier": "3.6.2",
-    "ts-node": "10.9.2",
-    "typescript": "5.9.3"
+    "typescript": "5.9.3",
+    "typescript-eslint": "8.51.0"
   },
   "keywords": [
     "registry",
@@ -62,6 +52,6 @@
     "offline"
   ],
   "engines": {
-    "node": ">=20.17.0"
+    "node": ">=22.18.0"
   }
 }

package/src/client.ts

@@ -0,0 +1,35 @@
+import * as https from 'https'
+import axios from 'axios'
+import {LRUCache} from 'lru-cache'
+import type {AxiosRequestConfig, ResponseType} from 'axios'
+import type {RegistryMetadata} from './types.d.ts'
+
+const metadataCache = new LRUCache<string, RegistryMetadata>({max: 100})
+
+const client = axios.create({
+  httpsAgent: new https.Agent({keepAlive: true}),
+  timeout: 30 * 1000
+})
+
+export async function fetchJsonWithCacheCloned(url: string, token: string): Promise<RegistryMetadata> {
+  const cached = metadataCache.get(url)
+  if (cached) {
+    return structuredClone(cached)
+  }
+
+  const value = await fetch<RegistryMetadata>(url, 'json', token)
+  metadataCache.set(url, value)
+  return structuredClone(value)
+}
+
+export function fetchBinaryData(url: string, token: string): Promise<Buffer> {
+  return fetch<Buffer>(url, 'arraybuffer', token)
+}
+
+async function fetch<T>(url: string, responseType: ResponseType, token: string): Promise<T> {
+  const config: AxiosRequestConfig = {responseType}
+  if (token !== '') {
+    config.headers = {authorization: 'Bearer ' + token}
+  }
+  return (await client.get<T>(url, config)).data
+}

package/src/download.ts

@@ -0,0 +1,141 @@
+import * as fs from 'fs'
+import * as path from 'path'
+import * as semver from 'semver'
+import * as url from 'url'
+import assert from 'assert'
+import {downloadPrebuiltBinaries, hasPrebuiltBinaries} from './pregyp.ts'
+import {fetchBinaryData, fetchJsonWithCacheCloned} from './client.ts'
+import {rewriteMetadataInTarball, rewriteVersionMetadata, tarballFilename} from './metadata.ts'
+import {verifyIntegrity} from './integrity.ts'
+import type {CommandLineOptions, PackageWithId, PlatformVariant, RegistryMetadata, VersionMetadata} from './types.d.ts'
+
+export async function downloadAll(
+  packages: PackageWithId[],
+  {
+    localUrl,
+    prebuiltBinaryProperties,
+    registryUrl,
+    registryToken,
+    rootFolder,
+    enforceTarballsOverHttps
+  }: Omit<CommandLineOptions, 'manifest' | 'includeDevDependencies'>
+): Promise<void> {
+  const downloadFromRegistry = download.bind(
+    null,
+    registryUrl,
+    registryToken,
+    localUrl,
+    rootFolder,
+    prebuiltBinaryProperties,
+    enforceTarballsOverHttps
+  )
+  for (const pkg of packages) {
+    await downloadFromRegistry(pkg)
+  }
+}
+
+async function download(
+  registryUrl: string,
+  registryToken: string,
+  localUrl: url.URL,
+  rootFolder: string,
+  prebuiltBinaryProperties: PlatformVariant[],
+  enforceTarballsOverHttps: boolean,
+  {name, version}: PackageWithId
+): Promise<void> {
+  const registryMetadata = await fetchMetadataCloned(name, registryUrl, registryToken)
+  const versionMetadata: VersionMetadata | undefined = registryMetadata.versions[version]
+  if (!versionMetadata) {
+    throw new Error(`Unknown package version ${name}@${version}`)
+  }
+
+  const localFolder = await ensureLocalFolderExists(name, rootFolder)
+  let data = await downloadTarball(versionMetadata, enforceTarballsOverHttps, registryToken)
+  if (hasPrebuiltBinaries(versionMetadata)) {
+    const localPregypFolder = await ensureLocalFolderExists(version, localFolder)
+    await downloadPrebuiltBinaries(versionMetadata, localPregypFolder, prebuiltBinaryProperties)
+    data = await rewriteMetadataInTarball(data, versionMetadata, localUrl, localFolder)
+  }
+  await saveTarball(versionMetadata, data, localFolder)
+
+  rewriteVersionMetadata(versionMetadata, data, localUrl)
+  await updateMetadata(versionMetadata, registryMetadata, registryUrl, localFolder)
+}
+
+async function downloadTarball(
+  {_id: id, dist}: VersionMetadata,
+  enforceTarballsOverHttps: boolean,
+  registryToken: string
+): Promise<Buffer> {
+  const tarballUrl = enforceTarballsOverHttps ? dist.tarball.replace('http://', 'https://') : dist.tarball
+  const data = await fetchBinaryData(tarballUrl, registryToken)
+  verifyIntegrity(data, id, dist)
+  return data
+}
+
+function saveTarball({name, version}: VersionMetadata, data: Buffer, localFolder: string) {
+  return fs.promises.writeFile(tarballPath(name, version, localFolder), data)
+}
+
+async function updateMetadata(
+  versionMetadata: VersionMetadata,
+  defaultMetadata: RegistryMetadata,
+  registryUrl: string,
+  localFolder: string
+) {
+  const {version} = versionMetadata
+  const localMetadataPath = path.join(localFolder, 'index.json')
+  const localMetadata = await loadMetadata(localMetadataPath, defaultMetadata)
+  localMetadata.versions[version] = versionMetadata
+  localMetadata.time[version] = defaultMetadata.time[version]
+  localMetadata['dist-tags'] = collectDistTags(localMetadata, defaultMetadata)
+  await saveMetadata(localMetadataPath, localMetadata)
+}
+
+// Collect dist-tags entries (name -> version) from registry metadata,
+// which point to versions we have locally available.
+// Override 'latest' tag to ensure its validity as we might not have the version
+// that is tagged latest in registry
+function collectDistTags(localMetadata: RegistryMetadata, defaultMetadata: RegistryMetadata): Record<string, string> {
+  const availableVersions = Object.keys(localMetadata.versions)
+  const validDistTags = Object.entries(defaultMetadata['dist-tags']).filter(([, version]) =>
+    availableVersions.includes(version)
+  )
+
+  const latest = availableVersions.sort(semver.compare).pop()
+  assert(latest, 'At least one version should be locally available to determine "latest" dist-tag')
+
+  return {
+    ...Object.fromEntries(validDistTags),
+    latest
+  }
+}
+
+async function loadMetadata(path: string, defaultMetadata: RegistryMetadata): Promise<RegistryMetadata> {
+  try {
+    const json = await fs.promises.readFile(path, 'utf8')
+    return JSON.parse(json)
+  } catch {
+    return {...defaultMetadata, 'dist-tags': {}, time: {}, versions: {}}
+  }
+}
+
+function saveMetadata(path: string, metadata: RegistryMetadata): Promise<void> {
+  const json = JSON.stringify(metadata, null, 2)
+  return fs.promises.writeFile(path, json, 'utf8')
+}
+
+function tarballPath(name: string, version: string, localFolder: string) {
+  return path.join(localFolder, tarballFilename(name, version))
+}
+
+async function ensureLocalFolderExists(name: string, rootFolder: string): Promise<string> {
+  const localFolder = path.resolve(rootFolder, name)
+  await fs.promises.mkdir(localFolder, {recursive: true})
+  return localFolder
+}
+
+function fetchMetadataCloned(name: string, registryUrl: string, registryToken: string): Promise<RegistryMetadata> {
+  const urlSafeName = name.replace(/\//g, '%2f')
+  return fetchJsonWithCacheCloned(url.resolve(registryUrl, urlSafeName), registryToken)
+}

package/src/index.ts

@@ -0,0 +1,75 @@
+import * as fs from 'fs'
+import * as path from 'path'
+import {Command} from 'commander'
+import {synchronize} from './sync.ts'
+import {URL} from 'url'
+import type {CommandLineOptions, PlatformVariant} from './types.d.ts'
+
+const {version} = JSON.parse(fs.readFileSync(path.join(import.meta.dirname, '..', 'package.json'), 'utf-8'))
+
+const program = new Command()
+program
+  .version(version)
+  .requiredOption('--root <path>', 'Path to save NPM package tarballs and metadata to')
+  .requiredOption(
+    '--manifest <file>',
+    'Path to a package-lock.json or yarn.lock file to use as catalog for mirrored NPM packages.'
+  )
+  .requiredOption(
+    '--localUrl <url>',
+    'URL to use as root in stored package metadata (i.e. where folder defined as --root will be exposed at)'
+  )
+  .option(
+    '--binaryAbi <list>',
+    'Comma-separated list of node C++ ABI numbers to download pre-built binaries for. See NODE_MODULE_VERSION column in https://nodejs.org/en/download/releases/'
+  )
+  .option(
+    '--binaryArch <list>',
+    'Comma-separated list of CPU architectures to download pre-built binaries for. Valid values: arm, ia32, and x64'
+  )
+  .option(
+    '--binaryPlatform <list>',
+    'Comma-separated list of OS platforms to download pre-built binaries for. Valid values: linux, darwin, win32, sunos, freebsd, openbsd, and aix'
+  )
+  .option(
+    '--registryUrl [url]',
+    'Optional URL to use as NPM registry when fetching packages. Default value is https://registry.npmjs.org'
+  )
+  .option('--registryToken [string]', 'Optional Bearer token for the registry.')
+  .option(
+    '--dontEnforceHttps',
+    'Disable the default behavior of downloading tarballs over HTTPS (will use whichever protocol is defined in the registry metadata)'
+  )
+  .option('--includeDev', 'Include also packages found from devDependencies section of the --manifest')
+  .option('--dryRun', 'Print packages that would be downloaded but do not download them')
+  .parse(process.argv)
+
+const rawOptions = program.opts()
+
+// use current (abi,arch,platform) triplet as default if none is specified
+// so the user doesn't have to look them up if build is always done on the
+// same kind of machine
+const binaryAbi: string = rawOptions.binaryAbi || process.versions.modules
+const binaryArch: string = rawOptions.binaryArch || process.arch
+const binaryPlatform: string = rawOptions.binaryPlatform || process.platform
+
+const abis: number[] = binaryAbi.split(',').map(Number)
+const architectures: string[] = binaryArch.split(',')
+const platforms: string[] = binaryPlatform.split(',')
+const prebuiltBinaryProperties: PlatformVariant[] = abis
+  .map(abi => architectures.map(arch => platforms.map(platform => ({abi, arch, platform}))).flat())
+  .flat()
+
+const options: CommandLineOptions = {
+  localUrl: new URL(rawOptions.localUrl),
+  manifest: rawOptions.manifest,
+  prebuiltBinaryProperties,
+  registryUrl: rawOptions.registryUrl || 'https://registry.npmjs.org',
+  registryToken: rawOptions.registryToken || '',
+  rootFolder: rawOptions.root,
+  enforceTarballsOverHttps: Boolean(!rawOptions.dontEnforceHttps),
+  includeDevDependencies: Boolean(rawOptions.includeDev),
+  dryRun: Boolean(rawOptions.dryRun)
+}
+
+synchronize(options)

package/src/integrity.ts

@@ -0,0 +1,27 @@
+import * as ssri from 'ssri'
+
+export function verifyIntegrity(
+  data: Buffer,
+  id: string,
+  {integrity, shasum}: {integrity?: string; shasum?: string}
+): void {
+  if (!integrity && !shasum) {
+    throw new Error(`Integrity values not present in metadata for ${id}`)
+  }
+
+  if (integrity) {
+    if (!ssri.checkData(data, integrity)) {
+      throw new Error(`Integrity check failed for ${id}`)
+    }
+  } else if (sha1(data) != shasum) {
+    throw new Error(`Integrity check with SHA1 failed for failed for ${id}`)
+  }
+}
+
+export function sha1(data: Buffer): string {
+  return ssri.fromData(data, {algorithms: ['sha1']}).hexDigest()
+}
+
+export function sha512(data: Buffer): string {
+  return ssri.fromData(data, {algorithms: ['sha512']}).toString()
+}

package/src/metadata.ts

@@ -0,0 +1,78 @@
+import * as fs from 'fs'
+import * as path from 'path'
+import * as tar from 'tar-fs'
+import * as zlib from 'zlib'
+import {hasPrebuiltBinaries} from './pregyp.ts'
+import {Readable} from 'stream'
+import {sha1, sha512} from './integrity.ts'
+import type {URL} from 'url'
+import type {VersionMetadata} from './types.d.ts'
+
+export function rewriteVersionMetadata(versionMetadata: VersionMetadata, data: Buffer, localUrl: URL): void {
+  versionMetadata.dist.tarball = localTarballUrl(versionMetadata, localUrl)
+
+  if (hasPrebuiltBinaries(versionMetadata)) {
+    versionMetadata.binary.host = localUrl.origin
+    versionMetadata.binary.remote_path = createPrebuiltBinaryRemotePath(localUrl, versionMetadata)
+    versionMetadata.dist.integrity = sha512(data)
+    versionMetadata.dist.shasum = sha1(data)
+  }
+}
+
+export async function rewriteMetadataInTarball(
+  data: Buffer,
+  versionMetadata: VersionMetadata,
+  localUrl: URL,
+  localFolder: string
+): Promise<Buffer> {
+  const tmpFolder = path.join(localFolder, '.tmp')
+  await fs.promises.mkdir(tmpFolder, {recursive: true})
+  await extractTgz(data, tmpFolder)
+
+  const manifestPath = path.join(tmpFolder, 'package', 'package.json')
+  const json = await fs.promises.readFile(manifestPath, 'utf8')
+  const metadata = JSON.parse(json)
+  metadata.binary.host = localUrl.origin
+  metadata.binary.remote_path = createPrebuiltBinaryRemotePath(localUrl, versionMetadata)
+  await fs.promises.writeFile(manifestPath, JSON.stringify(metadata, null, 2))
+
+  const updatedData = await compressTgz(tmpFolder)
+  await fs.promises.rm(tmpFolder, {recursive: true})
+  return updatedData
+}
+
+function createPrebuiltBinaryRemotePath(url: URL, versionMetadata: VersionMetadata): string {
+  return `${removeTrailingSlash(url.pathname)}/${versionMetadata.name}/${versionMetadata.version}/`
+}
+
+export function extractTgz(data: Buffer, folder: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const tgz = Readable.from(data).pipe(zlib.createGunzip()).pipe(tar.extract(folder))
+
+    tgz.on('finish', resolve)
+    tgz.on('error', reject)
+  })
+}
+
+function compressTgz(folder: string): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = []
+    const tgz = tar.pack(folder).pipe(zlib.createGzip())
+    tgz.on('data', (chunk: Buffer) => chunks.push(chunk))
+    tgz.on('end', () => resolve(Buffer.concat(chunks)))
+    tgz.on('error', reject)
+  })
+}
+
+function localTarballUrl({name, version}: {name: string; version: string}, localUrl: URL) {
+  return `${localUrl.origin}${removeTrailingSlash(localUrl.pathname)}/${name}/${tarballFilename(name, version)}`
+}
+
+export function tarballFilename(name: string, version: string): string {
+  const normalized = name.replace(/\//g, '-')
+  return `${normalized}-${version}.tgz`
+}
+
+function removeTrailingSlash(str: string): string {
+  return str.replace(/\/$/, '')
+}

package/src/{normalize-yarn-pattern.js → normalize-yarn-pattern.ts}

@@ -1,4 +1,3 @@
-"use strict";
 /*
 BSD 2-Clause License
 
@@ -28,33 +27,40 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 // From https://github.com/yarnpkg/yarn/blob/953c8b6a20e360b097625d64189e6e56ed813e0f/src/util/normalize-pattern.js#L2
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizeYarnPackagePattern = normalizeYarnPackagePattern;
-function normalizeYarnPackagePattern(pattern) {
-    let hasVersion = false;
-    let range = 'latest';
-    let name = pattern;
-    // if we're a scope then remove the @ and add it back later
-    let isScoped = false;
-    if (name[0] === '@') {
-        isScoped = true;
-        name = name.slice(1);
-    }
-    // take first part as the name
-    const parts = name.split('@');
-    if (parts.length > 1) {
-        name = parts.shift();
-        range = parts.join('@');
-        if (range) {
-            hasVersion = true;
-        }
-        else {
-            range = '*';
-        }
-    }
-    // add back @ scope suffix
-    if (isScoped) {
-        name = `@${name}`;
+
+export function normalizeYarnPackagePattern(pattern: string): {
+  hasVersion: boolean
+  name: string
+  range: string
+} {
+  let hasVersion = false
+  let range = 'latest'
+  let name = pattern
+
+  // if we're a scope then remove the @ and add it back later
+  let isScoped = false
+  if (name[0] === '@') {
+    isScoped = true
+    name = name.slice(1)
+  }
+
+  // take first part as the name
+  const parts = name.split('@')
+  if (parts.length > 1) {
+    name = parts.shift()!
+    range = parts.join('@')
+
+    if (range) {
+      hasVersion = true
+    } else {
+      range = '*'
     }
-    return { name, range, hasVersion };
+  }
+
+  // add back @ scope suffix
+  if (isScoped) {
+    name = `@${name}`
+  }
+
+  return {name, range, hasVersion}
 }