prod-files 0.1.1 → 0.1.3

package/README.md

@@ -63,6 +63,8 @@ Flags:
   -g, --globs   Prints out the default globs.
 
   -n, --noSize  Skips the size calc at the end, saves about 200-1000ms.
+
+  -q, --quiet   Quiet output, suppresses stdout.
 ```
 
 With a package manager:
@@ -122,42 +124,41 @@ RUN node pf.js my-app/foo/node_modules/.pnpm
 
 ## Development
 
-Unit tests are written with node's test utils.
-
 ```sh
-pnpm test
+pnpm i
 ```
 
-There's also a `test-project` directory with dummy `package.json` with some
-random deps. You can run the script against it to see how it fairs in real usage
-and get some timing data.
+### Unit tests
 
-Set it up:
+Unit tests are written with node's test utils.
 
 ```sh
-pnpm test:setup
+pnpm test
 ```
 
-Run `prod-files` on it:
+### End to end tests
+
+In `test-project` directory has Sentry's `package.json`. You can run the script
+against it to see how it fairs in real-world use and get some timing data.
 
 ```sh
-pnpm test:prune
+# Re-installs the packages and runs the script on it
+pnpm test:e2e
+# Disable size reportings since it's pretty slow
+pnpm test:e2e --noSize
 ```
 
-It uses the real file system, so you need to reset it with `pnpm test:setup`
-before running another test.
-
-Or chain it for ease of use (with timing):
+The nuke command removes `node_modules` and prunes the store:
 
 ```sh
-pnpm test:setup && time pnpm test:prune
+pnpm test:e2e:nuke
 ```
 
-There's also a simple script to print the size of `test-project/node_modules/`
-using `du`:
+There's also a simple script to print the weight of `test-project/node_modules/`
+using `du`. You can run it before and after to see more detailed results:
 
 ```sh
-pnpm test:size
+pnpm test:e2e:weight
 ```
 
 ## Prior art

package/index.mjs

@@ -1,14 +1,17 @@
-import childProcess from 'node:child_process'
+// oxlint-disable prefer-spread
+import cp from 'node:child_process'
 import fs from 'node:fs/promises'
-import { matchesGlob, join, isAbsolute, resolve } from 'node:path'
-import { parseArgs, styleText } from 'node:util'
+import { matchesGlob, join, isAbsolute, resolve, dirname } from 'node:path'
+import { parseArgs, promisify, styleText } from 'node:util'
+
+const exec = promisify(cp.exec)
 
 /**
  * A list of glob patterns for files/dirs to be deleted. The globs are matched
  * with node's `matchesGlob()`. With one special rule: globs which end in `/`
  * are marked as directories.
  *
- * Ordered by popularity (educated guess).
+ * Roughly ordered by popularity (educated guess).
  *
  * Partially based on
  * @see {@link https://github.com/duniul/clean-modules/blob/main/.cleanmodules-default}
@@ -21,16 +24,21 @@ const defaultGlobs = [
   '**/*.tsx',
   '**/doc{,s}/',
 
-  // TypeScript
+  // Types
   '**/*tsconfig*.json',
   '**/*.tsbuildinfo',
+  '**/flow-typed/',
+
+  // Sensitive
+  '**/.env*',
 
   // Package mangers
   '**/.npm*',
-  '**/pnpm-*.y{,a}ml',
+  '**/pnpm-{lock,workspace}.yaml',
   '**/.yarn*',
   '**/yarn.lock',
   '**/bun.lock',
+  '**/bunfig.toml',
 
   // IDE
   '**/.idea/',
@@ -50,6 +58,13 @@ const defaultGlobs = [
   // CI/CD
   '**/.github/',
   '**/.circleci/',
+  '**/.vercel',
+  '**/now.json',
+  '**/.travis.yml',
+
+  // Docker
+  '**/Dockerfile*',
+  '**/.dockerignore',
 
   // Tests
   '**/test{,s}/',
@@ -60,31 +75,42 @@ const defaultGlobs = [
   '**/karma.conf.{js,ts}',
   '**/wallaby.conf.{js,ts}',
   '**/wallaby.{js,ts}',
-
-  // Build tools
-  '**/gemfile',
+  '**/playwright.config.{js,ts}',
+  '**/.mocharc*',
+
+  // Build/bundle config
+  '**/{rollup,rolldown,vite}.config.{js,ts,mjs}',
+  '**/webpack.config.{js,mjs,cjs,ts}',
+  '**/babel.config.{js,mjs,cjs,json}',
+  '**/parcel.config.{js,ts,json}',
+  '**/rspack.config.{js,mjs,cjs,ts}',
+  '**/.babelrc*',
+  '**/turbo.json',
+  '**/.browserslist*',
+  '**/metro.config.{js,json}',
   '**/{G,g}runtfile.{js,ts}',
   '**/{G,g}ulpfile.{js,ts}',
   '**/{M,m}akefile',
+  '**/gemfile',
 
   // Images
   '**/*.jp{,e}g',
-  '**/*.png',
-  '**/*.gif',
   '**/*.svg',
+  '**/*.gif',
+  '**/*.png',
 
   // Linters and formatters
-  '**/.jshintrc',
-  '**/.lint',
-  '**/.prettier*',
-  '**/prettier.config*',
-  '**/biome.json{,c}',
-  '**/tslint.json',
-  '**/.eslintrc',
   '**/eslint*.{json,jsonc,ts}',
+  '**/.eslintrc',
+  '**/prettier.config*',
+  '**/.prettier*',
   '**/.ox{lint,fmt}rc.json{,c}',
   '**/ox{lint,fmt}*.{json,jsonc,ts}',
+  '**/biome.json{,c}',
   '**/.dprint.json{,c}',
+  '**/.jshintrc',
+  '**/.lint',
+  '**/tslint.json',
 
   // Git
   '**/.git/',
@@ -93,7 +119,7 @@ const defaultGlobs = [
 
   // Code coverage
   '**/.nyc_output/',
-  '**/.nycrc',
+  '**/.nycrc*',
   '**/.codecov.y{,a}ml',
   '**/coverage/',
 
@@ -123,6 +149,7 @@ const defaultGlobs = [
   '**/*.coffee',
 
   // Misc
+  '**/.jscpd',
   '**/*.jst',
   '**/*.log',
   '**/*.mkd',
@@ -189,9 +216,11 @@ function usage() {
     -g, --globs   Prints out the default globs.
 
     -n, --noSize  Skips the size calc at the end, saves about 200-1000ms.
+
+    -q, --quiet   Quiet output, suppresses stdout.
 `
 
-  console.log(usageText)
+  log.log(usageText)
 }
 
 /**
@@ -213,29 +242,48 @@ function bail(message, error, withUsage = false) {
 
 /**
  * @typedef {Object} Logger
- * @property {( ...args: any[] ) => void} info - Logs information messages in blue
- * @property {( ...args: any[] ) => void} error - Logs error messages in red
- * @property {( ...args: any[] ) => void} success - Logs success messages in green
+ * @property {typeof console.error} error - Logs error messages in red
+ * @property {typeof console.info} info - Logs information messages in blue
+ * @property {typeof console.info} log - Logs with no color
+ * @property {typeof console.log} success - Logs success messages in green
+ * @property {typeof console.table} table - Logs as table
  */
 
+// Quiet mode
+let quiet = /** @type {boolean} */ (false)
+
+/**
+ * @param {import('node:util').InspectColor} color
+ * @param {any[]} args
+ * @returns
+ */
+const style = (color, args) => args.map(a => styleText(color, String(a)))
+
 /**
  * A utility for styled console logs
  * @type {Logger}
  */
-const log = {
-  info: (...x) => console.info(styleText('blue', x.join(' '))),
-  error: (...x) => console.error(styleText('red', x.join(' '))),
-  success: (...x) => console.log(styleText('green', x.join(' '))),
+export const log = {
+  error: (...a) =>
+    quiet ? undefined : console.error.apply(console, style('red', a)),
+  info: (...a) =>
+    quiet ? undefined : console.info.apply(console, style('blue', a)),
+  log: (...a) => (quiet ? undefined : console.log.apply(console, a)),
+  success: (...a) =>
+    quiet ? undefined : console.log.apply(console, style('green', a)),
+  table: (...a) => (quiet ? undefined : console.table.apply(console, a)),
 }
 
 /**
  * Get size of node_modules
  * @param {string} dirPath - Path to node_modules
- * @returns {number}
+ * @returns {Promise<number>}
  */
-function getSize(dirPath) {
-  const stdout = childProcess.execSync(`du -s ${dirPath} | awk '{print $1}'`)
-  return Number(stdout)
+async function getSize(dirPath) {
+  const { stdout, stderr } = await exec(`LC_ALL=C du -s ${dirPath}`)
+  if (stderr.length > 0) bail(stderr)
+  const size = stdout.split('\t')[0]
+  return size ? Number.parseInt(size, 10) : 0
 }
 
 /**
@@ -253,16 +301,25 @@ function calcSize(originalSize, prunedSize) {
 /**
  * Prints a nice diff table
  * @param {object} opts
- * @param {number | undefined} opts.prunedSize
+ * @param {Promise<number> | undefined} opts.prunedSize
  * @param {number} opts.startTime
  * @param {number} opts.itemCount
- * @param {number | undefined} opts.originalSize
+ * @param {Promise<number> | undefined} opts.originalSize
  */
-export function printDiff({ prunedSize, startTime, itemCount, originalSize }) {
-  console.table([
+export async function printDiff({
+  prunedSize,
+  startTime,
+  itemCount,
+  originalSize,
+}) {
+  const [original, pruned] =
+    originalSize && prunedSize
+      ? await Promise.all([originalSize, prunedSize])
+      : [undefined, undefined]
+
+  log.table([
     {
-      Pruned:
-        originalSize && prunedSize ? calcSize(originalSize, prunedSize) : 'n/a',
+      ...(original && pruned && { Pruned: calcSize(original, pruned) }),
       Time: `${((Date.now() - startTime) / 1000).toFixed(1)}s`,
       Items: itemCount,
     },
@@ -272,18 +329,20 @@ export function printDiff({ prunedSize, startTime, itemCount, originalSize }) {
 /**
  * @typedef Args
  * @type {object}
- * @property {string} path - Path to node_modules
+ * @property {string} [path] - Path to node_modules
  * @property {string[]} include - New glob pattern
  * @property {string[]} exclude - Existing glob pattern
  * @property {boolean} help - Prints help
  * @property {boolean} noSize - Don't show size savings
  * @property {boolean} globs - Prints globs
+ * @property {boolean} quiet - Suppress console.log output
  */
 
 /**
  * Parse the command-line arguments into an object
  * @returns {Args}
  */
+
 function handleArgs() {
   try {
     const {
@@ -297,9 +356,9 @@ function handleArgs() {
         help: { type: 'boolean', short: 'h', default: false },
         globs: { type: 'boolean', short: 'g', default: false },
         noSize: { type: 'boolean', short: 'n', default: false },
+        quiet: { type: 'boolean', short: 'q', default: false },
       },
     })
-    if (!path) throw bail('Path not defined', undefined, true)
 
     return { ...values, path }
   } catch (err) {
@@ -327,14 +386,6 @@ export async function validateNodeModulesPath(nodeModulesPath) {
   }
 }
 
-/**
- * Removes a directory or a file
- * @param {string} file - the file or dir to remove
- */
-async function rimraf(file) {
-  await fs.rm(file, { recursive: true, force: true })
-}
-
 /**
  * `file.matchesGlob()` does not match dotfiles, this util replaces leading dots
  * with an underscore
@@ -644,44 +695,131 @@ export function compactPaths(paths) {
   return compact
 }
 
+/**
+ * Checks the rmdir error: ENOTEMPTY means the dir was not empty and the removal
+ * failed, which is what we want
+ * @param {unknown} err
+ * @returns {boolean}
+ */
+function hasContent(err) {
+  return (
+    !!err &&
+    typeof err === 'object' &&
+    'code' in err &&
+    (err.code === 'ENOTEMPTY' || err.code === 'ENOENT')
+  )
+}
+
+/**
+ * Removes a dir if it's empty
+ * @param {string[]} dirs
+ * @returns {Promise<void>[]}
+ */
+function rmEmptyDir(dirs) {
+  return dirs.map(dir =>
+    fs.rmdir(dir).catch(err => {
+      if (hasContent(err)) return
+      throw err
+    })
+  )
+}
+
+/**
+ * Removes a file and collects parent directories for later cleanup
+ * @param {string} file - the file to remove
+ * @param {Set<string>} visited - tracks directories we've already visited
+ * @param {Map<number, Set<string>>} dirDepths - cleanup dirs grouped by depth
+ * @param {string} rootDir - stop collecting once we reach this directory
+ */
+async function rimraf(
+  file,
+  visited = new Set(),
+  dirDepths = new Map(),
+  rootDir = dirname(file)
+) {
+  // Remove the file/dir recursively
+  await fs.rm(file, { recursive: true, force: true })
+
+  // Walk up the tree collecting all the ancestors, we'll use them later on to
+  // delete directories which are left empty
+  let dir = dirname(file)
+  while (dir !== rootDir) {
+    if (visited.has(dir)) break
+    visited.add(dir)
+
+    const depth = dir.split('/').length
+
+    // Group the dirs by depth
+    const dirs = dirDepths.get(depth)
+    if (dirs) dirs.add(dir)
+    else dirDepths.set(depth, new Set([dir]))
+
+    dir = dirname(dir)
+  }
+}
+
+/**
+ * @typedef {Args & { path: string }} ArgsWithRequiredPath
+ */
+
 /**
  * Removes unneeded files from node_modules
- * @param {Args} opts
+ * @param {ArgsWithRequiredPath} opts
  */
 export async function prune(opts) {
   const startTime = Date.now()
   log.info('Pruning:', opts.path)
 
-  const originalSize = opts.noSize ? undefined : getSize(opts.path)
+  // Don't wait
+  const originalSize = getSize(opts.path)
   const excludedGlobs = new Set(opts.exclude)
   const activeGlobs = [...defaultGlobs, ...opts.include].filter(
     glob => !excludedGlobs.has(glob)
   )
   const compiledGlobs = compileGlobs(activeGlobs)
 
-  // This could be slightly faster with optimized walker
+  // TODO: this could be slightly faster with an optimized walker
   const allFiles = await fs.readdir(opts.path, {
     recursive: true,
     withFileTypes: true,
   })
 
-  const junkFiles = findJunkFiles(allFiles, compiledGlobs)
-  const results = compactPaths(junkFiles)
+  const junk = compactPaths(findJunkFiles(allFiles, compiledGlobs))
 
   try {
-    await Promise.all(results.map(x => rimraf(x)))
+    /** @type {Set<string>} */
+    const visited = new Set()
+    /** @type {Map<number, Set<string>>} */
+    const dirDepths = new Map()
+    // Rm & populate visited & dirDepths so dirs can be removed in parallel
+    await Promise.all(junk.map(x => rimraf(x, visited, dirDepths, opts.path)))
+    const depths = [...dirDepths.keys()].sort((a, b) => b - a)
+
+    /**
+     * Remove one depth level at a time, but parallelize within each level
+     * @param {number} i
+     * @returns {Promise<void>}
+     */
+    async function removeDepth(i) {
+      if (i >= depths.length) return
+      const dirs = dirDepths.get(depths[i] || 0) ?? []
+      await Promise.all(rmEmptyDir([...dirs]))
+      await removeDepth(i + 1)
+    }
+
+    await removeDepth(0)
   } catch (err) {
     throw bail(undefined, err)
   }
 
-  printDiff({
-    itemCount: results.length,
+  void printDiff({
+    itemCount: junk.length,
     prunedSize: opts.noSize ? undefined : getSize(opts.path),
-    originalSize,
+    originalSize: opts.noSize ? undefined : originalSize,
     startTime,
   })
 
-  return results
+  return junk
 }
 
 const entry = process.argv[1]
@@ -691,16 +829,28 @@ const runAsScript =
 if (runAsScript) {
   const args = handleArgs()
 
+  quiet = args.quiet
+
   if (args.help) {
     usage()
     process.exit(0)
   }
 
   if (args.globs) {
-    console.log(JSON.stringify(defaultGlobs, null, 2))
+    log.log(JSON.stringify(defaultGlobs, null, 2))
     process.exit(0)
   }
 
-  await validateNodeModulesPath(args.path)
-  await prune(args)
+  // Should have a path by now
+  if (!args.path) {
+    throw bail(
+      undefined,
+      'Path not defined. Usage: prod-files <path-to-node-modules>'
+    )
+  }
+
+  const argsWithPath = /** @type {ArgsWithRequiredPath} */ (args)
+
+  await validateNodeModulesPath(argsWithPath.path)
+  await prune(argsWithPath)
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prod-files",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Keep only prod files by pruning non-prod files from node_modules before deploying",
   "keywords": [
     "clean",
@@ -40,10 +40,8 @@
     "pf": "node ./index.mjs",
     "pub": "(npm whoami || npm login) && pnpm publish --access=public --tag=latest",
     "test": "node --test",
-    "test:nuke": "rm -rf ./test-project/node_modules ./test-project/pnpm-lock.yaml",
-    "test:run": "cd test-project && time node ../index.mjs node_modules/.pnpm",
-    "test:setup": "node --run test:nuke && pnpm -C=test-project i --prod",
-    "test:size": "du -sh ./test-project/node_modules/.pnpm | sort -h",
-    "test:watch": "node --test --watch"
+    "test:e2e": "cd test-project && bash prepare_test.sh && time node ../index.mjs node_modules/.pnpm",
+    "test:e2e:nuke": "trash ./test-project/node_modules ./test-project/pnpm-lock.yaml && pnpm store prune",
+    "test:e2e:weight": "du -sh ./test-project/node_modules/.pnpm | sort -h"
   }
 }