prod-files 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -24
- package/index.mjs +193 -164
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ projects, or you’re just obsessed with small deployments.
|
|
|
11
11
|
|
|
12
12
|
It's relatively fast, prunes
|
|
13
13
|
[Sentry's `node_modules`](https://github.com/getsentry/sentry/blob/master/package.json)
|
|
14
|
-
in
|
|
14
|
+
in 1.8s (M2 MacBook). Prod deps only though, installed with `pnpm i --prod`, but
|
|
15
15
|
that's the common use-case anyway.
|
|
16
16
|
|
|
17
17
|
## Install
|
|
@@ -36,8 +36,7 @@ Examples:
|
|
|
36
36
|
arguments, so don't use equals signs:
|
|
37
37
|
$ pf -i "**/foo" -e "**/*tsconfig*.json" node_modules/.pnpm
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
omitted:
|
|
39
|
+
In short-hand args the space between the key and the value can be omitted:
|
|
41
40
|
$ pf -i"**/foo" node_modules/.pnpm
|
|
42
41
|
|
|
43
42
|
Usage:
|
|
@@ -51,18 +50,24 @@ Arguments:
|
|
|
51
50
|
- yarn: 'node_modules' or 'node_modules/.store'
|
|
52
51
|
|
|
53
52
|
Flags:
|
|
54
|
-
-i, --include
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
-i, --include Extra custom glob pattern. Uses node's path.matchesGlob(),
|
|
54
|
+
with one exception: patterns ending with slash '**/foo/' are
|
|
55
|
+
marked as directories. Can have multiple.
|
|
57
56
|
|
|
58
57
|
-e, --exclude Exclude existing glob patterns if the script is too
|
|
59
|
-
aggressive. Must be exact match.
|
|
58
|
+
aggressive. Must be exact match. Can have multiple.
|
|
59
|
+
|
|
60
|
+
-d, --dryRun Nothing is removed and the paths are printed out.
|
|
60
61
|
|
|
61
62
|
-h, --help Prints out the help.
|
|
62
63
|
|
|
63
|
-
-g, --
|
|
64
|
+
-g, --showGlobs
|
|
65
|
+
Prints out the default globs.
|
|
66
|
+
|
|
67
|
+
--noGlobs Disable default glob patterns, only use patterns from
|
|
68
|
+
--include.
|
|
64
69
|
|
|
65
|
-
-n, --noSize Skips the size
|
|
70
|
+
-n, --noSize Skips the size calculation.
|
|
66
71
|
|
|
67
72
|
-q, --quiet Quiet output, suppresses stdout.
|
|
68
73
|
```
|
|
@@ -88,6 +93,43 @@ Different package manager `node_modules` paths:
|
|
|
88
93
|
| yarn | pnpm | `node_modules/.store` | same as pnpm |
|
|
89
94
|
| yarn | pnp | no-op | no node_modules |
|
|
90
95
|
|
|
96
|
+
### Provide your own globs
|
|
97
|
+
|
|
98
|
+
The default globs can de disabled with `--noGlobs` flag, and only globs in
|
|
99
|
+
`--include` are matched:
|
|
100
|
+
|
|
101
|
+
```sh
|
|
102
|
+
pnpm prod-files --noGlobs -i "**/custom/" -i "**/*.html" node_modules/.pnpm
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### Dy run
|
|
106
|
+
|
|
107
|
+
The `--dry-run` flag does not delete anything and prints out the paths:
|
|
108
|
+
|
|
109
|
+
```sh
|
|
110
|
+
pnpm prod-files --dryRun node_modules/.pnpm
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
### Use as a search
|
|
114
|
+
|
|
115
|
+
You can use `prod-files` to search any dir if you set `--dryRun` and
|
|
116
|
+
`--noGlobs`, and provide the search term in `--include`:
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
pnpm prod-files --noGlobs --dryRun --include="**/bower.json" node_modules/.pnpm
|
|
120
|
+
|
|
121
|
+
Pruning (--dryRun, nothing deleted): node_modules/.pnpm
|
|
122
|
+
node_modules/.pnpm/less@4.3.0/node_modules/less/bower.json
|
|
123
|
+
node_modules/.pnpm/papaparse@5.5.3/node_modules/papaparse/bower.json
|
|
124
|
+
node_modules/.pnpm/reflux@0.4.1_react@19.2.3/node_modules/reflux/bower.json
|
|
125
|
+
node_modules/.pnpm/sprintf-js@1.0.3/node_modules/sprintf-js/bower.json
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Sentry's `node_modules` has 4 bower config files :)
|
|
129
|
+
|
|
130
|
+
> [!CAUTION]\
|
|
131
|
+
> You're wielding `rm -rf` here, always remember to set `--dryRun`!
|
|
132
|
+
|
|
91
133
|
### Dockerfile example
|
|
92
134
|
|
|
93
135
|
Simple yet somewhat realistic example usage in Dockerfile for an app named `foo`
|
|
@@ -101,25 +143,27 @@ RUN pnpm fetch
|
|
|
101
143
|
COPY . ./
|
|
102
144
|
RUN pnpm i --offline --frozen-lockfile
|
|
103
145
|
RUN pnpm build
|
|
104
|
-
RUN pnpm -F=foo --prod deploy /
|
|
105
|
-
# Run it as the last command of the build step.
|
|
106
|
-
# --prod
|
|
107
|
-
|
|
146
|
+
RUN pnpm -F=foo --prod deploy /foo
|
|
147
|
+
# Run it as the last command of the build step.
|
|
148
|
+
# NOTE: with --prod, the script needs to be a prod dep. Or use pnpx/npx/yarn dlx
|
|
149
|
+
WORKDIR /foo
|
|
150
|
+
RUN pnpm prod-files node_modules/.pnpm --noSize
|
|
108
151
|
|
|
109
152
|
# Enjoy your new slimmer image
|
|
110
153
|
FROM node:lts-alpine3.19 AS foo
|
|
111
|
-
COPY --from=base /
|
|
112
|
-
|
|
154
|
+
COPY --from=base foo/build /foo/build
|
|
155
|
+
COPY --from=base foo/node_modules /foo/node_modules
|
|
156
|
+
WORKDIR /foo
|
|
113
157
|
CMD node build/server.js
|
|
114
158
|
```
|
|
115
159
|
|
|
116
|
-
Or use wget
|
|
117
|
-
|
|
160
|
+
Or use `wget` if you don't have a package manager in your env (there are certain
|
|
161
|
+
risks involved when you execute files downloaded from the net, if I get
|
|
118
162
|
comprised that file can have anything):
|
|
119
163
|
|
|
120
164
|
```dockerfile
|
|
121
|
-
RUN wget -O pf.
|
|
122
|
-
RUN node pf.
|
|
165
|
+
RUN wget -O pf.mjs https://raw.githubusercontent.com/hilja/prod-files/refs/heads/main/index.mjs
|
|
166
|
+
RUN node pf.mjs /foo/node_modules/.pnpm
|
|
123
167
|
```
|
|
124
168
|
|
|
125
169
|
## Development
|
|
@@ -138,23 +182,29 @@ pnpm test
|
|
|
138
182
|
|
|
139
183
|
### End to end tests
|
|
140
184
|
|
|
141
|
-
|
|
142
|
-
against it to see how it
|
|
185
|
+
The `test-project` directory has Sentry's `package.json`. You can run the script
|
|
186
|
+
against it to see how it does in real-world use and get some timing data.
|
|
143
187
|
|
|
144
188
|
```sh
|
|
145
189
|
# Re-installs the packages and runs the script on it
|
|
146
190
|
pnpm test:e2e
|
|
147
|
-
# Disable size reportings since it
|
|
191
|
+
# Disable size reportings since it adds 200-300ms
|
|
148
192
|
pnpm test:e2e --noSize
|
|
149
193
|
```
|
|
150
194
|
|
|
151
|
-
|
|
195
|
+
If you're testing `--dryRun`, use `test:e2e:run`, it does not reinstall:
|
|
196
|
+
|
|
197
|
+
```sh
|
|
198
|
+
pnpm test:r2e:run
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The nuke command removes `test-project/node_modules` and prunes the store:
|
|
152
202
|
|
|
153
203
|
```sh
|
|
154
204
|
pnpm test:e2e:nuke
|
|
155
205
|
```
|
|
156
206
|
|
|
157
|
-
There's also a simple script to print the weight of `test-project/node_modules
|
|
207
|
+
There's also a simple script to print the weight of `test-project/node_modules`
|
|
158
208
|
using `du`. You can run it before and after to see more detailed results:
|
|
159
209
|
|
|
160
210
|
```sh
|
package/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// oxlint-disable prefer-spread
|
|
2
2
|
import cp from 'node:child_process'
|
|
3
3
|
import fs from 'node:fs/promises'
|
|
4
|
-
import { matchesGlob, join, isAbsolute, resolve
|
|
4
|
+
import { matchesGlob, join, isAbsolute, resolve } from 'node:path'
|
|
5
5
|
import { parseArgs, promisify, styleText } from 'node:util'
|
|
6
6
|
|
|
7
7
|
const exec = promisify(cp.exec)
|
|
@@ -21,13 +21,16 @@ const defaultGlobs = [
|
|
|
21
21
|
'**/*.md',
|
|
22
22
|
'**/*.map',
|
|
23
23
|
'**/*.{,m,c}ts',
|
|
24
|
-
'**/*.
|
|
24
|
+
'**/*.{j,t}sx',
|
|
25
25
|
'**/doc{,s}/',
|
|
26
26
|
|
|
27
27
|
// Types
|
|
28
28
|
'**/*tsconfig*.json',
|
|
29
29
|
'**/*.tsbuildinfo',
|
|
30
30
|
'**/flow-typed/',
|
|
31
|
+
'**/.flowconfig',
|
|
32
|
+
'**/*.flow',
|
|
33
|
+
'**/__typings__/',
|
|
31
34
|
|
|
32
35
|
// Sensitive
|
|
33
36
|
'**/.env*',
|
|
@@ -39,6 +42,8 @@ const defaultGlobs = [
|
|
|
39
42
|
'**/yarn.lock',
|
|
40
43
|
'**/bun.lock',
|
|
41
44
|
'**/bunfig.toml',
|
|
45
|
+
'**/bower.json',
|
|
46
|
+
'**/node_modules/.bin/',
|
|
42
47
|
|
|
43
48
|
// IDE
|
|
44
49
|
'**/.idea/',
|
|
@@ -54,6 +59,7 @@ const defaultGlobs = [
|
|
|
54
59
|
'**/contributing',
|
|
55
60
|
'**/CONTRIBUTORS',
|
|
56
61
|
'**/contributors',
|
|
62
|
+
'**/node_modules/**/man/',
|
|
57
63
|
|
|
58
64
|
// CI/CD
|
|
59
65
|
'**/.github/',
|
|
@@ -69,6 +75,7 @@ const defaultGlobs = [
|
|
|
69
75
|
// Tests
|
|
70
76
|
'**/test{,s}/',
|
|
71
77
|
'**/spec{,s}/',
|
|
78
|
+
'**/**.{test,spec}.{js,mjs}',
|
|
72
79
|
'**/__{mocks,tests}__/',
|
|
73
80
|
'**/jest.*.{js,ts}',
|
|
74
81
|
'**/vitest.*.ts',
|
|
@@ -77,6 +84,8 @@ const defaultGlobs = [
|
|
|
77
84
|
'**/wallaby.{js,ts}',
|
|
78
85
|
'**/playwright.config.{js,ts}',
|
|
79
86
|
'**/.mocharc*',
|
|
87
|
+
'**/.zuul.yml',
|
|
88
|
+
'**/.coveralls.yml',
|
|
80
89
|
|
|
81
90
|
// Build/bundle config
|
|
82
91
|
'**/{rollup,rolldown,vite}.config.{js,ts,mjs}',
|
|
@@ -100,8 +109,9 @@ const defaultGlobs = [
|
|
|
100
109
|
'**/*.png',
|
|
101
110
|
|
|
102
111
|
// Linters and formatters
|
|
103
|
-
'**/eslint*.{json,jsonc,ts}',
|
|
104
|
-
'**/.eslintrc',
|
|
112
|
+
'**/eslint*.{json,jsonc,ts,js,mjs}',
|
|
113
|
+
'**/.eslintrc*',
|
|
114
|
+
'**/.eslintignore',
|
|
105
115
|
'**/prettier.config*',
|
|
106
116
|
'**/.prettier*',
|
|
107
117
|
'**/.ox{lint,fmt}rc.json{,c}',
|
|
@@ -149,6 +159,11 @@ const defaultGlobs = [
|
|
|
149
159
|
'**/*.coffee',
|
|
150
160
|
|
|
151
161
|
// Misc
|
|
162
|
+
'**/KEYS',
|
|
163
|
+
'**/.spmignore',
|
|
164
|
+
'**/.editorconfig',
|
|
165
|
+
'**/component.json',
|
|
166
|
+
'**/*.jison',
|
|
152
167
|
'**/.jscpd',
|
|
153
168
|
'**/*.jst',
|
|
154
169
|
'**/*.log',
|
|
@@ -189,8 +204,7 @@ function usage() {
|
|
|
189
204
|
arguments, so don't use equals signs:
|
|
190
205
|
$ pf -i "**/foo" -e "**/*tsconfig*.json" node_modules/.pnpm
|
|
191
206
|
|
|
192
|
-
|
|
193
|
-
omitted:
|
|
207
|
+
In short-hand args the space between the key and the value can be omitted:
|
|
194
208
|
$ pf -i"**/foo" node_modules/.pnpm
|
|
195
209
|
|
|
196
210
|
Usage:
|
|
@@ -204,18 +218,24 @@ function usage() {
|
|
|
204
218
|
- yarn: 'node_modules' or 'node_modules/.store'
|
|
205
219
|
|
|
206
220
|
Flags:
|
|
207
|
-
-i, --include
|
|
208
|
-
|
|
209
|
-
|
|
221
|
+
-i, --include Extra custom glob pattern. Uses node's path.matchesGlob(),
|
|
222
|
+
with one exception: patterns ending with slash '**/foo/' are
|
|
223
|
+
marked as directories. Can have multiple.
|
|
210
224
|
|
|
211
225
|
-e, --exclude Exclude existing glob patterns if the script is too
|
|
212
|
-
aggressive. Must be exact match.
|
|
226
|
+
aggressive. Must be exact match. Can have multiple.
|
|
227
|
+
|
|
228
|
+
-d, --dryRun Nothing is removed and the paths are printed out.
|
|
213
229
|
|
|
214
230
|
-h, --help Prints out the help.
|
|
215
231
|
|
|
216
|
-
-g, --
|
|
232
|
+
-g, --showGlobs
|
|
233
|
+
Prints out the default globs.
|
|
234
|
+
|
|
235
|
+
--noGlobs Disable default glob patterns, only use patterns from
|
|
236
|
+
--include.
|
|
217
237
|
|
|
218
|
-
-n, --noSize Skips the size
|
|
238
|
+
-n, --noSize Skips the size calculation.
|
|
219
239
|
|
|
220
240
|
-q, --quiet Quiet output, suppresses stdout.
|
|
221
241
|
`
|
|
@@ -264,8 +284,7 @@ const style = (color, args) => args.map(a => styleText(color, String(a)))
|
|
|
264
284
|
* @type {Logger}
|
|
265
285
|
*/
|
|
266
286
|
export const log = {
|
|
267
|
-
error: (...a) =>
|
|
268
|
-
quiet ? undefined : console.error.apply(console, style('red', a)),
|
|
287
|
+
error: (...a) => console.error.apply(console, style('red', a)),
|
|
269
288
|
info: (...a) =>
|
|
270
289
|
quiet ? undefined : console.info.apply(console, style('blue', a)),
|
|
271
290
|
log: (...a) => (quiet ? undefined : console.log.apply(console, a)),
|
|
@@ -275,8 +294,8 @@ export const log = {
|
|
|
275
294
|
}
|
|
276
295
|
|
|
277
296
|
/**
|
|
278
|
-
* Get
|
|
279
|
-
* @param {string} dirPath
|
|
297
|
+
* Get disk usage via du (512-byte blocks)
|
|
298
|
+
* @param {string} dirPath
|
|
280
299
|
* @returns {Promise<number>}
|
|
281
300
|
*/
|
|
282
301
|
async function getSize(dirPath) {
|
|
@@ -286,14 +305,36 @@ async function getSize(dirPath) {
|
|
|
286
305
|
return size ? Number.parseInt(size, 10) : 0
|
|
287
306
|
}
|
|
288
307
|
|
|
308
|
+
/**
|
|
309
|
+
* Sums disk usage of a path using lstat blocks (512-byte blocks, same as du)
|
|
310
|
+
* @param {string} path
|
|
311
|
+
* @returns {Promise<number>} Total in 512-byte blocks
|
|
312
|
+
*/
|
|
313
|
+
async function treeSize(path) {
|
|
314
|
+
/** @type {import('node:fs').Stats} */
|
|
315
|
+
let stat
|
|
316
|
+
try {
|
|
317
|
+
stat = await fs.lstat(path)
|
|
318
|
+
} catch {
|
|
319
|
+
// Entry disappeared (concurrent pruning), count as 0
|
|
320
|
+
return 0
|
|
321
|
+
}
|
|
322
|
+
if (!stat.isDirectory()) return stat.blocks
|
|
323
|
+
const names = await fs.readdir(path).catch(() => [])
|
|
324
|
+
const sizes = await Promise.all(names.map(n => treeSize(join(path, n))))
|
|
325
|
+
let total = 0
|
|
326
|
+
for (const s of sizes) total += s
|
|
327
|
+
return total
|
|
328
|
+
}
|
|
329
|
+
|
|
289
330
|
/**
|
|
290
331
|
* @param {number} originalSize
|
|
291
332
|
* @param {number} prunedSize
|
|
292
333
|
*/
|
|
293
|
-
function calcSize(originalSize, prunedSize) {
|
|
334
|
+
export function calcSize(originalSize, prunedSize) {
|
|
294
335
|
const diff = originalSize - prunedSize
|
|
295
336
|
const diffMb = `${(diff / 1024).toFixed(1)} MB`
|
|
296
|
-
const diffPercent = `${((diff /
|
|
337
|
+
const diffPercent = `${((diff / originalSize) * 100).toFixed(1)}%`
|
|
297
338
|
|
|
298
339
|
return `${diffPercent} (${diffMb})`
|
|
299
340
|
}
|
|
@@ -301,25 +342,23 @@ function calcSize(originalSize, prunedSize) {
|
|
|
301
342
|
/**
|
|
302
343
|
* Prints a nice diff table
|
|
303
344
|
* @param {object} opts
|
|
304
|
-
* @param {
|
|
345
|
+
* @param {number | undefined} opts.removedBytes
|
|
305
346
|
* @param {number} opts.startTime
|
|
306
347
|
* @param {number} opts.itemCount
|
|
307
|
-
* @param {
|
|
348
|
+
* @param {number | undefined} opts.originalSize
|
|
308
349
|
*/
|
|
309
|
-
export
|
|
310
|
-
|
|
350
|
+
export function printDiff({
|
|
351
|
+
removedBytes,
|
|
311
352
|
startTime,
|
|
312
353
|
itemCount,
|
|
313
354
|
originalSize,
|
|
314
355
|
}) {
|
|
315
|
-
const [original, pruned] =
|
|
316
|
-
originalSize && prunedSize
|
|
317
|
-
? await Promise.all([originalSize, prunedSize])
|
|
318
|
-
: [undefined, undefined]
|
|
319
|
-
|
|
320
356
|
log.table([
|
|
321
357
|
{
|
|
322
|
-
...(
|
|
358
|
+
...(originalSize &&
|
|
359
|
+
removedBytes && {
|
|
360
|
+
Pruned: calcSize(originalSize, originalSize - removedBytes),
|
|
361
|
+
}),
|
|
323
362
|
Time: `${((Date.now() - startTime) / 1000).toFixed(1)}s`,
|
|
324
363
|
Items: itemCount,
|
|
325
364
|
},
|
|
@@ -332,9 +371,11 @@ export async function printDiff({
|
|
|
332
371
|
* @property {string} [path] - Path to node_modules
|
|
333
372
|
* @property {string[]} include - New glob pattern
|
|
334
373
|
* @property {string[]} exclude - Existing glob pattern
|
|
374
|
+
* @property {boolean} dryRun - Print out the files to be removed, no deletion
|
|
335
375
|
* @property {boolean} help - Prints help
|
|
376
|
+
* @property {boolean} showGlobs - Prints globs
|
|
377
|
+
* @property {boolean} noGlobs - Disable default glob patterns
|
|
336
378
|
* @property {boolean} noSize - Don't show size savings
|
|
337
|
-
* @property {boolean} globs - Prints globs
|
|
338
379
|
* @property {boolean} quiet - Suppress console.log output
|
|
339
380
|
*/
|
|
340
381
|
|
|
@@ -342,7 +383,6 @@ export async function printDiff({
|
|
|
342
383
|
* Parse the command-line arguments into an object
|
|
343
384
|
* @returns {Args}
|
|
344
385
|
*/
|
|
345
|
-
|
|
346
386
|
function handleArgs() {
|
|
347
387
|
try {
|
|
348
388
|
const {
|
|
@@ -353,8 +393,10 @@ function handleArgs() {
|
|
|
353
393
|
options: {
|
|
354
394
|
include: { type: 'string', short: 'i', default: [], multiple: true },
|
|
355
395
|
exclude: { type: 'string', short: 'e', default: [], multiple: true },
|
|
396
|
+
dryRun: { type: 'boolean', short: 'd', default: false },
|
|
356
397
|
help: { type: 'boolean', short: 'h', default: false },
|
|
357
|
-
|
|
398
|
+
showGlobs: { type: 'boolean', short: 'g', default: false },
|
|
399
|
+
noGlobs: { type: 'boolean', default: false },
|
|
358
400
|
noSize: { type: 'boolean', short: 'n', default: false },
|
|
359
401
|
quiet: { type: 'boolean', short: 'q', default: false },
|
|
360
402
|
},
|
|
@@ -666,160 +708,144 @@ export function findJunkFiles(files, compiledGlobs = defaultCompiledGlobs) {
|
|
|
666
708
|
}
|
|
667
709
|
|
|
668
710
|
/**
|
|
669
|
-
*
|
|
670
|
-
* @
|
|
671
|
-
* @
|
|
711
|
+
* @typedef {object} WalkResult
|
|
712
|
+
* @property {string[]} removed - Compacted list of removed paths
|
|
713
|
+
* @property {number} removedBlocks - Removed disk usage in 512-byte blocks
|
|
714
|
+
*/
|
|
715
|
+
|
|
716
|
+
/**
|
|
717
|
+
* Parallel walker that finds junk, removes it, and cleans empty dirs in one
|
|
718
|
+
* pass. Skips recursing into junk directories (implicit path compacting) and
|
|
719
|
+
* removes empty ancestors bottom-up as the recursion unwinds.
|
|
720
|
+
* @param {CompiledGlobs} compiledGlobs - Precompiled glob matchers
|
|
721
|
+
* @param {ArgsWithPath} opts - The args object
|
|
722
|
+
* @returns {Promise<WalkResult>}
|
|
672
723
|
*/
|
|
673
|
-
|
|
674
|
-
/** @type {Set<string>} */
|
|
675
|
-
const seen = new Set()
|
|
724
|
+
async function walkAndPrune(compiledGlobs, opts) {
|
|
676
725
|
/** @type {string[]} */
|
|
677
|
-
const
|
|
726
|
+
const removed = []
|
|
727
|
+
let removedBlocks = 0
|
|
728
|
+
const hasAnyGlobs = compiledGlobs.any.globs.length > 0
|
|
729
|
+
const hasDirGlobs = compiledGlobs.dir.globs.length > 0
|
|
730
|
+
|
|
731
|
+
/**
|
|
732
|
+
* Walks a directory in parallel, removes junk, and reports whether the
|
|
733
|
+
* directory still has content so the caller can clean up empty parents
|
|
734
|
+
* @param {string} dir
|
|
735
|
+
* @returns {Promise<boolean>} true when the directory still has content
|
|
736
|
+
*/
|
|
737
|
+
async function walkDir(dir) {
|
|
738
|
+
const entries = await fs.readdir(dir, { withFileTypes: true })
|
|
739
|
+
|
|
740
|
+
/** @type {string[]} */
|
|
741
|
+
const junkPaths = []
|
|
742
|
+
/** @type {string[]} */
|
|
743
|
+
const keptDirPaths = []
|
|
744
|
+
let keptFiles = 0
|
|
745
|
+
|
|
746
|
+
for (const entry of entries) {
|
|
747
|
+
const { name } = entry
|
|
748
|
+
const isDir = entry.isDirectory()
|
|
749
|
+
const path = join(dir, name)
|
|
750
|
+
|
|
751
|
+
// Basename checks are cheapest, try them first
|
|
752
|
+
if (
|
|
753
|
+
matchesSet(name, compiledGlobs.any) ||
|
|
754
|
+
(isDir && matchesSet(name, compiledGlobs.dir))
|
|
755
|
+
) {
|
|
756
|
+
junkPaths.push(path)
|
|
757
|
+
continue
|
|
758
|
+
}
|
|
678
759
|
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
760
|
+
// Full path glob checks only when compiled globs exist
|
|
761
|
+
if (hasAnyGlobs || (isDir && hasDirGlobs)) {
|
|
762
|
+
const escapedPath = escapeLeadingDots(isDir ? `${path}/` : path)
|
|
763
|
+
if (
|
|
764
|
+
compiledGlobs.any.globs.some(g => matchesGlob(escapedPath, g)) ||
|
|
765
|
+
(isDir &&
|
|
766
|
+
compiledGlobs.dir.globs.some(g => matchesGlob(escapedPath, g)))
|
|
767
|
+
) {
|
|
768
|
+
junkPaths.push(path)
|
|
769
|
+
continue
|
|
770
|
+
}
|
|
771
|
+
}
|
|
682
772
|
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
i = path.lastIndexOf('/', i - 1)
|
|
773
|
+
if (isDir) keptDirPaths.push(path)
|
|
774
|
+
else keptFiles += 1
|
|
686
775
|
}
|
|
687
776
|
|
|
688
|
-
//
|
|
689
|
-
|
|
777
|
+
// Collect removed paths before awaiting (no junk dir recursion = compacting)
|
|
778
|
+
for (const p of junkPaths) removed.push(p)
|
|
779
|
+
|
|
780
|
+
// Size (when tracking), remove junk, and recurse kept subdirs in parallel
|
|
781
|
+
const [junkSizes, walkResults] = await Promise.all([
|
|
782
|
+
Promise.all(
|
|
783
|
+
junkPaths.map(async p => {
|
|
784
|
+
const size = opts.noSize ? 0 : await treeSize(p)
|
|
785
|
+
// No rm if --dryRun
|
|
786
|
+
if (!opts.dryRun) await fs.rm(p, { recursive: true, force: true })
|
|
787
|
+
return size
|
|
788
|
+
})
|
|
789
|
+
),
|
|
790
|
+
Promise.all(keptDirPaths.map(walkDir)),
|
|
791
|
+
])
|
|
792
|
+
|
|
793
|
+
for (const s of junkSizes) removedBlocks += s
|
|
794
|
+
|
|
795
|
+
// Subdirs that became empty after pruning their contents
|
|
796
|
+
const emptyDirs = keptDirPaths.filter((_, i) => !walkResults[i])
|
|
797
|
+
if (emptyDirs.length > 0) {
|
|
798
|
+
// No rm if --dryRun
|
|
799
|
+
if (!opts.dryRun) await Promise.all(emptyDirs.map(d => fs.rmdir(d)))
|
|
800
|
+
}
|
|
690
801
|
|
|
691
|
-
|
|
692
|
-
compact.push(path)
|
|
802
|
+
return keptFiles + keptDirPaths.length - emptyDirs.length > 0
|
|
693
803
|
}
|
|
694
804
|
|
|
695
|
-
|
|
805
|
+
await walkDir(opts.path)
|
|
806
|
+
return { removed, removedBlocks }
|
|
696
807
|
}
|
|
697
808
|
|
|
698
809
|
/**
|
|
699
|
-
*
|
|
700
|
-
* failed, which is what we want
|
|
701
|
-
* @param {unknown} err
|
|
702
|
-
* @returns {boolean}
|
|
703
|
-
*/
|
|
704
|
-
function hasContent(err) {
|
|
705
|
-
return (
|
|
706
|
-
!!err &&
|
|
707
|
-
typeof err === 'object' &&
|
|
708
|
-
'code' in err &&
|
|
709
|
-
(err.code === 'ENOTEMPTY' || err.code === 'ENOENT')
|
|
710
|
-
)
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
/**
|
|
714
|
-
* Removes a dir if it's empty
|
|
715
|
-
* @param {string[]} dirs
|
|
716
|
-
* @returns {Promise<void>[]}
|
|
717
|
-
*/
|
|
718
|
-
function rmEmptyDir(dirs) {
|
|
719
|
-
return dirs.map(dir =>
|
|
720
|
-
fs.rmdir(dir).catch(err => {
|
|
721
|
-
if (hasContent(err)) return
|
|
722
|
-
throw err
|
|
723
|
-
})
|
|
724
|
-
)
|
|
725
|
-
}
|
|
726
|
-
|
|
727
|
-
/**
|
|
728
|
-
* Removes a file and collects parent directories for later cleanup
|
|
729
|
-
* @param {string} file - the file to remove
|
|
730
|
-
* @param {Set<string>} visited - tracks directories we've already visited
|
|
731
|
-
* @param {Map<number, Set<string>>} dirDepths - cleanup dirs grouped by depth
|
|
732
|
-
* @param {string} rootDir - stop collecting once we reach this directory
|
|
733
|
-
*/
|
|
734
|
-
async function rimraf(
|
|
735
|
-
file,
|
|
736
|
-
visited = new Set(),
|
|
737
|
-
dirDepths = new Map(),
|
|
738
|
-
rootDir = dirname(file)
|
|
739
|
-
) {
|
|
740
|
-
// Remove the file/dir recursively
|
|
741
|
-
await fs.rm(file, { recursive: true, force: true })
|
|
742
|
-
|
|
743
|
-
// Walk up the tree collecting all the ancestors, we'll use them later on to
|
|
744
|
-
// delete directories which are left empty
|
|
745
|
-
let dir = dirname(file)
|
|
746
|
-
while (dir !== rootDir) {
|
|
747
|
-
if (visited.has(dir)) break
|
|
748
|
-
visited.add(dir)
|
|
749
|
-
|
|
750
|
-
const depth = dir.split('/').length
|
|
751
|
-
|
|
752
|
-
// Group the dirs by depth
|
|
753
|
-
const dirs = dirDepths.get(depth)
|
|
754
|
-
if (dirs) dirs.add(dir)
|
|
755
|
-
else dirDepths.set(depth, new Set([dir]))
|
|
756
|
-
|
|
757
|
-
dir = dirname(dir)
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
|
|
761
|
-
/**
|
|
762
|
-
* @typedef {Args & { path: string }} ArgsWithRequiredPath
|
|
810
|
+
* @typedef {Args & { path: string }} ArgsWithPath
|
|
763
811
|
*/
|
|
764
812
|
|
|
765
813
|
/**
|
|
766
814
|
* Removes unneeded files from node_modules
|
|
767
|
-
* @param {
|
|
815
|
+
* @param {ArgsWithPath} opts
|
|
768
816
|
*/
|
|
769
817
|
export async function prune(opts) {
|
|
770
818
|
const startTime = Date.now()
|
|
771
|
-
|
|
819
|
+
const dryRunMsg = opts.dryRun ? ' (--dryRun, nothing deleted)' : ''
|
|
820
|
+
log.info(`Pruning${dryRunMsg}:`, opts.path)
|
|
772
821
|
|
|
773
|
-
//
|
|
774
|
-
const
|
|
822
|
+
// Fire early so du runs concurrently with the walk
|
|
823
|
+
const sizePromise = opts.noSize ? undefined : getSize(opts.path)
|
|
775
824
|
const excludedGlobs = new Set(opts.exclude)
|
|
776
|
-
const activeGlobs = [
|
|
777
|
-
|
|
778
|
-
|
|
825
|
+
const activeGlobs = [
|
|
826
|
+
...(opts.noGlobs ? [] : defaultGlobs),
|
|
827
|
+
...opts.include,
|
|
828
|
+
].filter(glob => !excludedGlobs.has(glob))
|
|
779
829
|
const compiledGlobs = compileGlobs(activeGlobs)
|
|
780
830
|
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
recursive: true,
|
|
784
|
-
withFileTypes: true,
|
|
785
|
-
})
|
|
786
|
-
|
|
787
|
-
const junk = compactPaths(findJunkFiles(allFiles, compiledGlobs))
|
|
788
|
-
|
|
831
|
+
/** @type {WalkResult} */
|
|
832
|
+
let result
|
|
789
833
|
try {
|
|
790
|
-
|
|
791
|
-
const visited = new Set()
|
|
792
|
-
/** @type {Map<number, Set<string>>} */
|
|
793
|
-
const dirDepths = new Map()
|
|
794
|
-
// Rm & populate visited & dirDepths so dirs can be removed in parallel
|
|
795
|
-
await Promise.all(junk.map(x => rimraf(x, visited, dirDepths, opts.path)))
|
|
796
|
-
const depths = [...dirDepths.keys()].sort((a, b) => b - a)
|
|
797
|
-
|
|
798
|
-
/**
|
|
799
|
-
* Remove one depth level at a time, but parallelize within each level
|
|
800
|
-
* @param {number} i
|
|
801
|
-
* @returns {Promise<void>}
|
|
802
|
-
*/
|
|
803
|
-
async function removeDepth(i) {
|
|
804
|
-
if (i >= depths.length) return
|
|
805
|
-
const dirs = dirDepths.get(depths[i] || 0) ?? []
|
|
806
|
-
await Promise.all(rmEmptyDir([...dirs]))
|
|
807
|
-
await removeDepth(i + 1)
|
|
808
|
-
}
|
|
809
|
-
|
|
810
|
-
await removeDepth(0)
|
|
834
|
+
result = await walkAndPrune(compiledGlobs, opts)
|
|
811
835
|
} catch (err) {
|
|
812
836
|
throw bail(undefined, err)
|
|
813
837
|
}
|
|
814
838
|
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
839
|
+
if (!opts.dryRun) {
|
|
840
|
+
printDiff({
|
|
841
|
+
itemCount: result.removed.length,
|
|
842
|
+
removedBytes: opts.noSize ? undefined : result.removedBlocks,
|
|
843
|
+
originalSize: sizePromise ? await sizePromise : undefined,
|
|
844
|
+
startTime,
|
|
845
|
+
})
|
|
846
|
+
}
|
|
821
847
|
|
|
822
|
-
return
|
|
848
|
+
return result.removed
|
|
823
849
|
}
|
|
824
850
|
|
|
825
851
|
const entry = process.argv[1]
|
|
@@ -836,21 +862,24 @@ if (runAsScript) {
|
|
|
836
862
|
process.exit(0)
|
|
837
863
|
}
|
|
838
864
|
|
|
839
|
-
if (args.
|
|
865
|
+
if (args.showGlobs) {
|
|
840
866
|
log.log(JSON.stringify(defaultGlobs, null, 2))
|
|
841
867
|
process.exit(0)
|
|
842
868
|
}
|
|
843
869
|
|
|
844
|
-
//
|
|
870
|
+
// From this point forward we should have a path
|
|
845
871
|
if (!args.path) {
|
|
846
|
-
throw bail(
|
|
847
|
-
undefined,
|
|
848
|
-
'Path not defined. Usage: prod-files <path-to-node-modules>'
|
|
849
|
-
)
|
|
872
|
+
throw bail(undefined, 'Path not defined. Usage: prod-files <path>')
|
|
850
873
|
}
|
|
851
874
|
|
|
852
|
-
const argsWithPath = /** @type {
|
|
875
|
+
const argsWithPath = /** @type {ArgsWithPath} */ (args)
|
|
853
876
|
|
|
854
877
|
await validateNodeModulesPath(argsWithPath.path)
|
|
855
|
-
await prune(argsWithPath)
|
|
878
|
+
const pruned = await prune(argsWithPath)
|
|
879
|
+
|
|
880
|
+
// Print out paths if --dryRun
|
|
881
|
+
if (args.dryRun) {
|
|
882
|
+
if (pruned.length === 0) log.log('No results')
|
|
883
|
+
for (const item of pruned) log.log(item)
|
|
884
|
+
}
|
|
856
885
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "prod-files",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Keep only prod files by pruning non-prod files from node_modules before deploying",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"clean",
|
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
"test": "node --test",
|
|
43
43
|
"test:e2e": "cd test-project && bash prepare_test.sh && time node ../index.mjs node_modules/.pnpm",
|
|
44
44
|
"test:e2e:nuke": "trash ./test-project/node_modules ./test-project/pnpm-lock.yaml && pnpm store prune",
|
|
45
|
+
"test:e2e:run": "cd test-project && node ../index.mjs node_modules/.pnpm",
|
|
45
46
|
"test:e2e:weight": "du -sh ./test-project/node_modules/.pnpm | sort -h"
|
|
46
47
|
}
|
|
47
48
|
}
|