prod-files 0.1.2 → 0.1.4

package/README.md

@@ -11,7 +11,7 @@ projects, or you’re just obsessed with small deployments.
 
 It's relatively fast, prunes
 [Sentry's `node_modules`](https://github.com/getsentry/sentry/blob/master/package.json)
-in 2.1s (M2 MacBook). Prod deps only though, installed with `pnpm i --prod`, but
+in 1.8s (M2 MacBook). Prod deps only though, installed with `pnpm i --prod`, but
 that's the common use-case anyway.
 
 ## Install
@@ -62,7 +62,9 @@ Flags:
 
   -g, --globs   Prints out the default globs.
 
-  -n, --noSize  Skips the size calc at the end, saves about 200-1000ms.
+  -n, --noSize  Skips the size calculation.
+
+  -q, --quiet   Quiet output, suppresses stdout.
 ```
 
 With a package manager:
@@ -122,42 +124,41 @@ RUN node pf.js my-app/foo/node_modules/.pnpm
 
 ## Development
 
-Unit tests are written with node's test utils.
-
 ```sh
-pnpm test
+pnpm i
 ```
 
-There's also a `test-project` directory with dummy `package.json` with some
-random deps. You can run the script against it to see how it fairs in real usage
-and get some timing data.
+### Unit tests
 
-Set it up:
+Unit tests are written with node's test utils.
 
 ```sh
-pnpm test:setup
+pnpm test
 ```
 
-Run `prod-files` on it:
+### End to end tests
+
+In `test-project` directory has Sentry's `package.json`. You can run the script
+against it to see how it does in real-world use and get some timing data.
 
 ```sh
-pnpm test:prune
+# Re-installs the packages and runs the script on it
+pnpm test:e2e
+# Disable size reportings since it adds 200-300ms
+pnpm test:e2e --noSize
 ```
 
-It uses the real file system, so you need to reset it with `pnpm test:setup`
-before running another test.
-
-Or chain it for ease of use (with timing):
+The nuke command removes `node_modules` and prunes the store:
 
 ```sh
-pnpm test:setup && time pnpm test:prune
+pnpm test:e2e:nuke
 ```
 
-There's also a simple script to print the size of `test-project/node_modules/`
-using `du`:
+There's also a simple script to print the weight of `test-project/node_modules/`
+using `du`. You can run it before and after to see more detailed results:
 
 ```sh
-pnpm test:size
+pnpm test:e2e:weight
 ```
 
 ## Prior art

package/index.mjs

@@ -1,14 +1,17 @@
-import childProcess from 'node:child_process'
+// oxlint-disable prefer-spread
+import cp from 'node:child_process'
 import fs from 'node:fs/promises'
 import { matchesGlob, join, isAbsolute, resolve } from 'node:path'
-import { parseArgs, styleText } from 'node:util'
+import { parseArgs, promisify, styleText } from 'node:util'
+
+const exec = promisify(cp.exec)
 
 /**
  * A list of glob patterns for files/dirs to be deleted. The globs are matched
  * with node's `matchesGlob()`. With one special rule: globs which end in `/`
  * are marked as directories.
  *
- * Ordered by popularity (educated guess).
+ * Roughly ordered by popularity (educated guess).
  *
  * Partially based on
  * @see {@link https://github.com/duniul/clean-modules/blob/main/.cleanmodules-default}
@@ -21,16 +24,21 @@ const defaultGlobs = [
   '**/*.tsx',
   '**/doc{,s}/',
 
-  // TypeScript
+  // Types
   '**/*tsconfig*.json',
   '**/*.tsbuildinfo',
+  '**/flow-typed/',
+
+  // Sensitive
+  '**/.env*',
 
   // Package mangers
   '**/.npm*',
-  '**/pnpm-*.y{,a}ml',
+  '**/pnpm-{lock,workspace}.yaml',
   '**/.yarn*',
   '**/yarn.lock',
   '**/bun.lock',
+  '**/bunfig.toml',
 
   // IDE
   '**/.idea/',
@@ -50,6 +58,13 @@ const defaultGlobs = [
   // CI/CD
   '**/.github/',
   '**/.circleci/',
+  '**/.vercel',
+  '**/now.json',
+  '**/.travis.yml',
+
+  // Docker
+  '**/Dockerfile*',
+  '**/.dockerignore',
 
   // Tests
   '**/test{,s}/',
@@ -60,31 +75,42 @@ const defaultGlobs = [
   '**/karma.conf.{js,ts}',
   '**/wallaby.conf.{js,ts}',
   '**/wallaby.{js,ts}',
-
-  // Build tools
-  '**/gemfile',
+  '**/playwright.config.{js,ts}',
+  '**/.mocharc*',
+
+  // Build/bundle config
+  '**/{rollup,rolldown,vite}.config.{js,ts,mjs}',
+  '**/webpack.config.{js,mjs,cjs,ts}',
+  '**/babel.config.{js,mjs,cjs,json}',
+  '**/parcel.config.{js,ts,json}',
+  '**/rspack.config.{js,mjs,cjs,ts}',
+  '**/.babelrc*',
+  '**/turbo.json',
+  '**/.browserslist*',
+  '**/metro.config.{js,json}',
   '**/{G,g}runtfile.{js,ts}',
   '**/{G,g}ulpfile.{js,ts}',
   '**/{M,m}akefile',
+  '**/gemfile',
 
   // Images
   '**/*.jp{,e}g',
-  '**/*.png',
-  '**/*.gif',
   '**/*.svg',
+  '**/*.gif',
+  '**/*.png',
 
   // Linters and formatters
-  '**/.jshintrc',
-  '**/.lint',
-  '**/.prettier*',
-  '**/prettier.config*',
-  '**/biome.json{,c}',
-  '**/tslint.json',
-  '**/.eslintrc',
   '**/eslint*.{json,jsonc,ts}',
+  '**/.eslintrc',
+  '**/prettier.config*',
+  '**/.prettier*',
   '**/.ox{lint,fmt}rc.json{,c}',
   '**/ox{lint,fmt}*.{json,jsonc,ts}',
+  '**/biome.json{,c}',
   '**/.dprint.json{,c}',
+  '**/.jshintrc',
+  '**/.lint',
+  '**/tslint.json',
 
   // Git
   '**/.git/',
@@ -93,7 +119,7 @@ const defaultGlobs = [
 
   // Code coverage
   '**/.nyc_output/',
-  '**/.nycrc',
+  '**/.nycrc*',
   '**/.codecov.y{,a}ml',
   '**/coverage/',
 
@@ -123,6 +149,7 @@ const defaultGlobs = [
   '**/*.coffee',
 
   // Misc
+  '**/.jscpd',
   '**/*.jst',
   '**/*.log',
   '**/*.mkd',
@@ -188,10 +215,12 @@ function usage() {
 
     -g, --globs   Prints out the default globs.
 
-    -n, --noSize  Skips the size calc at the end, saves about 200-1000ms.
+    -n, --noSize  Skips the size calculation.
+
+    -q, --quiet   Quiet output, suppresses stdout.
 `
 
-  console.log(usageText)
+  log.log(usageText)
 }
 
 /**
@@ -213,29 +242,70 @@ function bail(message, error, withUsage = false) {
 
 /**
  * @typedef {Object} Logger
- * @property {( ...args: any[] ) => void} info - Logs information messages in blue
- * @property {( ...args: any[] ) => void} error - Logs error messages in red
- * @property {( ...args: any[] ) => void} success - Logs success messages in green
+ * @property {typeof console.error} error - Logs error messages in red
+ * @property {typeof console.info} info - Logs information messages in blue
+ * @property {typeof console.info} log - Logs with no color
+ * @property {typeof console.log} success - Logs success messages in green
+ * @property {typeof console.table} table - Logs as table
  */
 
+// Quiet mode
+let quiet = /** @type {boolean} */ (false)
+
+/**
+ * @param {import('node:util').InspectColor} color
+ * @param {any[]} args
+ * @returns
+ */
+const style = (color, args) => args.map(a => styleText(color, String(a)))
+
 /**
  * A utility for styled console logs
  * @type {Logger}
  */
-const log = {
-  info: (...x) => console.info(styleText('blue', x.join(' '))),
-  error: (...x) => console.error(styleText('red', x.join(' '))),
-  success: (...x) => console.log(styleText('green', x.join(' '))),
+export const log = {
+  error: (...a) =>
+    quiet ? undefined : console.error.apply(console, style('red', a)),
+  info: (...a) =>
+    quiet ? undefined : console.info.apply(console, style('blue', a)),
+  log: (...a) => (quiet ? undefined : console.log.apply(console, a)),
+  success: (...a) =>
+    quiet ? undefined : console.log.apply(console, style('green', a)),
+  table: (...a) => (quiet ? undefined : console.table.apply(console, a)),
+}
+
+/**
+ * Get disk usage via du (512-byte blocks)
+ * @param {string} dirPath
+ * @returns {Promise<number>}
+ */
+async function getSize(dirPath) {
+  const { stdout, stderr } = await exec(`LC_ALL=C du -s ${dirPath}`)
+  if (stderr.length > 0) bail(stderr)
+  const size = stdout.split('\t')[0]
+  return size ? Number.parseInt(size, 10) : 0
 }
 
 /**
- * Get size of node_modules
- * @param {string} dirPath - Path to node_modules
- * @returns {number}
+ * Sums disk usage of a path using lstat blocks (512-byte blocks, same as du)
+ * @param {string} path
+ * @returns {Promise<number>} Total in 512-byte blocks
  */
-function getSize(dirPath) {
-  const stdout = childProcess.execSync(`du -s ${dirPath} | awk '{print $1}'`)
-  return Number(stdout)
+async function treeSize(path) {
+  /** @type {import('node:fs').Stats} */
+  let stat
+  try {
+    stat = await fs.lstat(path)
+  } catch {
+    // Entry disappeared (concurrent pruning), count as 0
+    return 0
+  }
+  if (!stat.isDirectory()) return stat.blocks
+  const names = await fs.readdir(path).catch(() => [])
+  const sizes = await Promise.all(names.map(n => treeSize(join(path, n))))
+  let total = 0
+  for (const s of sizes) total += s
+  return total
 }
 
 /**
@@ -253,16 +323,23 @@ function calcSize(originalSize, prunedSize) {
 /**
  * Prints a nice diff table
  * @param {object} opts
- * @param {number | undefined} opts.prunedSize
+ * @param {number | undefined} opts.removedBytes
  * @param {number} opts.startTime
  * @param {number} opts.itemCount
  * @param {number | undefined} opts.originalSize
  */
-export function printDiff({ prunedSize, startTime, itemCount, originalSize }) {
-  console.table([
+export function printDiff({
+  removedBytes,
+  startTime,
+  itemCount,
+  originalSize,
+}) {
+  log.table([
     {
-      Pruned:
-        originalSize && prunedSize ? calcSize(originalSize, prunedSize) : 'n/a',
+      ...(originalSize &&
+        removedBytes && {
+          Pruned: calcSize(originalSize, originalSize - removedBytes),
+        }),
       Time: `${((Date.now() - startTime) / 1000).toFixed(1)}s`,
       Items: itemCount,
     },
@@ -278,12 +355,14 @@ export function printDiff({ prunedSize, startTime, itemCount, originalSize }) {
  * @property {boolean} help - Prints help
  * @property {boolean} noSize - Don't show size savings
  * @property {boolean} globs - Prints globs
+ * @property {boolean} quiet - Suppress console.log output
  */
 
 /**
  * Parse the command-line arguments into an object
  * @returns {Args}
  */
+
 function handleArgs() {
   try {
     const {
@@ -297,6 +376,7 @@ function handleArgs() {
         help: { type: 'boolean', short: 'h', default: false },
         globs: { type: 'boolean', short: 'g', default: false },
         noSize: { type: 'boolean', short: 'n', default: false },
+        quiet: { type: 'boolean', short: 'q', default: false },
       },
     })
 
@@ -326,14 +406,6 @@ export async function validateNodeModulesPath(nodeModulesPath) {
   }
 }
 
-/**
- * Removes a directory or a file
- * @param {string} file - the file or dir to remove
- */
-async function rimraf(file) {
-  await fs.rm(file, { recursive: true, force: true })
-}
-
 /**
  * `file.matchesGlob()` does not match dotfiles, this util replaces leading dots
  * with an underscore
@@ -643,6 +715,104 @@ export function compactPaths(paths) {
   return compact
 }
 
+/**
+ * @typedef {object} WalkResult
+ * @property {string[]} removed - Compacted list of removed paths
+ * @property {number} removedBlocks - Removed disk usage in 512-byte blocks
+ */
+
+/**
+ * Parallel walker that finds junk, removes it, and cleans empty dirs in one
+ * pass. Skips recursing into junk directories (implicit path compacting) and
+ * removes empty ancestors bottom-up as the recursion unwinds.
+ * @param {string} rootDir - The directory to walk
+ * @param {CompiledGlobs} compiledGlobs - Precompiled glob matchers
+ * @param {boolean} trackSize - Whether to collect byte sizes of removed items
+ * @returns {Promise<WalkResult>}
+ */
+async function walkAndPrune(rootDir, compiledGlobs, trackSize) {
+  /** @type {string[]} */
+  const removed = []
+  let removedBlocks = 0
+  const hasAnyGlobs = compiledGlobs.any.globs.length > 0
+  const hasDirGlobs = compiledGlobs.dir.globs.length > 0
+
+  /**
+   * Walks a directory in parallel, removes junk, and reports whether the
+   * directory still has content so the caller can clean up empty parents
+   * @param {string} dir
+   * @returns {Promise<boolean>} true when the directory still has content
+   */
+  async function walkDir(dir) {
+    const entries = await fs.readdir(dir, { withFileTypes: true })
+
+    /** @type {string[]} */
+    const junkPaths = []
+    /** @type {string[]} */
+    const keptDirPaths = []
+    let keptFiles = 0
+
+    for (const entry of entries) {
+      const { name } = entry
+      const isDir = entry.isDirectory()
+      const path = join(dir, name)
+
+      // Basename checks are cheapest, try them first
+      if (
+        matchesSet(name, compiledGlobs.any) ||
+        (isDir && matchesSet(name, compiledGlobs.dir))
+      ) {
+        junkPaths.push(path)
+        continue
+      }
+
+      // Full path glob checks only when compiled globs exist
+      if (hasAnyGlobs || (isDir && hasDirGlobs)) {
+        const escapedPath = escapeLeadingDots(isDir ? `${path}/` : path)
+        if (
+          compiledGlobs.any.globs.some(g => matchesGlob(escapedPath, g)) ||
+          (isDir &&
+            compiledGlobs.dir.globs.some(g => matchesGlob(escapedPath, g)))
+        ) {
+          junkPaths.push(path)
+          continue
+        }
+      }
+
+      if (isDir) keptDirPaths.push(path)
+      else keptFiles += 1
+    }
+
+    // Collect removed paths before awaiting (no junk dir recursion = compacting)
+    for (const p of junkPaths) removed.push(p)
+
+    // Size (when tracking), remove junk, and recurse kept subdirs in parallel
+    const [junkSizes, walkResults] = await Promise.all([
+      Promise.all(
+        junkPaths.map(async p => {
+          const size = trackSize ? await treeSize(p) : 0
+          await fs.rm(p, { recursive: true, force: true })
+          return size
+        })
+      ),
+      Promise.all(keptDirPaths.map(walkDir)),
+    ])
+
+    for (const s of junkSizes) removedBlocks += s
+
+    // Subdirs that became empty after pruning their contents
+    const emptyDirs = keptDirPaths.filter((_, i) => !walkResults[i])
+    if (emptyDirs.length > 0) {
+      await Promise.all(emptyDirs.map(d => fs.rmdir(d)))
+    }
+
+    return keptFiles + keptDirPaths.length - emptyDirs.length > 0
+  }
+
+  await walkDir(rootDir)
+  return { removed, removedBlocks }
+}
+
 /**
  * @typedef {Args & { path: string }} ArgsWithRequiredPath
  */
@@ -655,36 +825,30 @@ export async function prune(opts) {
   const startTime = Date.now()
   log.info('Pruning:', opts.path)
 
-  const originalSize = opts.noSize ? undefined : getSize(opts.path)
+  // Fire early so du runs concurrently with the walk
+  const sizePromise = opts.noSize ? undefined : getSize(opts.path)
   const excludedGlobs = new Set(opts.exclude)
   const activeGlobs = [...defaultGlobs, ...opts.include].filter(
     glob => !excludedGlobs.has(glob)
   )
   const compiledGlobs = compileGlobs(activeGlobs)
 
-  // This could be slightly faster with optimized walker
-  const allFiles = await fs.readdir(opts.path, {
-    recursive: true,
-    withFileTypes: true,
-  })
-
-  const junkFiles = findJunkFiles(allFiles, compiledGlobs)
-  const results = compactPaths(junkFiles)
-
+  /** @type {WalkResult} */
+  let result
   try {
-    await Promise.all(results.map(x => rimraf(x)))
+    result = await walkAndPrune(opts.path, compiledGlobs, !opts.noSize)
   } catch (err) {
     throw bail(undefined, err)
   }
 
   printDiff({
-    itemCount: results.length,
-    prunedSize: opts.noSize ? undefined : getSize(opts.path),
-    originalSize,
+    itemCount: result.removed.length,
+    removedBytes: opts.noSize ? undefined : result.removedBlocks,
+    originalSize: sizePromise ? await sizePromise : undefined,
     startTime,
   })
 
-  return results
+  return result.removed
 }
 
 const entry = process.argv[1]
@@ -694,13 +858,15 @@ const runAsScript =
 if (runAsScript) {
   const args = handleArgs()
 
+  quiet = args.quiet
+
   if (args.help) {
     usage()
     process.exit(0)
   }
 
   if (args.globs) {
-    console.log(JSON.stringify(defaultGlobs, null, 2))
+    log.log(JSON.stringify(defaultGlobs, null, 2))
     process.exit(0)
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prod-files",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Keep only prod files by pruning non-prod files from node_modules before deploying",
   "keywords": [
     "clean",
@@ -40,10 +40,8 @@
     "pf": "node ./index.mjs",
     "pub": "(npm whoami || npm login) && pnpm publish --access=public --tag=latest",
     "test": "node --test",
-    "test:nuke": "rm -rf ./test-project/node_modules ./test-project/pnpm-lock.yaml",
-    "test:run": "cd test-project && time node ../index.mjs node_modules/.pnpm",
-    "test:setup": "node --run test:nuke && pnpm -C=test-project i --prod",
-    "test:size": "du -sh ./test-project/node_modules/.pnpm | sort -h",
-    "test:watch": "node --test --watch"
+    "test:e2e": "cd test-project && bash prepare_test.sh && time node ../index.mjs node_modules/.pnpm",
+    "test:e2e:nuke": "trash ./test-project/node_modules ./test-project/pnpm-lock.yaml && pnpm store prune",
+    "test:e2e:weight": "du -sh ./test-project/node_modules/.pnpm | sort -h"
   }
 }