tiny-readdir 2.0.0 → 2.1.0

package/dist/index.d.ts

@@ -1,3 +1,3 @@
 import type { Options, Result } from './types';
-declare const readdir: (rootPath: string, options?: Options | undefined) => Promise<Result>;
+declare const readdir: (rootPath: string, options?: Options) => Promise<Result>;
 export default readdir;

package/dist/index.js

@@ -1,13 +1,11 @@
 /* IMPORT */
 import fs from 'node:fs';
 import path from 'node:path';
-import Limiter from 'promise-concurrency-limiter';
-/* HELPERS */
-const limiter = new Limiter({ concurrency: 500 });
 /* MAIN */
 const readdir = (rootPath, options) => {
     const followSymlinks = options?.followSymlinks ?? false;
     const maxDepth = options?.depth ?? Infinity;
+    const maxPaths = options?.limit ?? Infinity;
     const isIgnored = options?.ignore ?? (() => false);
     const signal = options?.signal ?? { aborted: false };
     const directories = [];
@@ -17,19 +15,28 @@ const readdir = (rootPath, options) => {
     const visited = new Set();
     const resultEmpty = { directories: [], files: [], symlinks: [], map: {} };
     const result = { directories, files, symlinks, map };
+    let foundPaths = 0;
     const handleDirectory = (dirmap, subPath, depth) => {
         if (visited.has(subPath))
             return;
+        if (foundPaths >= maxPaths)
+            return;
+        foundPaths += 1;
         dirmap.directories.push(subPath);
         directories.push(subPath);
         visited.add(subPath);
         if (depth >= maxDepth)
             return;
-        return limiter.add(() => populateResultFromPath(subPath, depth + 1));
+        if (foundPaths >= maxPaths)
+            return;
+        return populateResultFromPath(subPath, depth + 1);
     };
     const handleFile = (dirmap, subPath) => {
         if (visited.has(subPath))
             return;
+        if (foundPaths >= maxPaths)
+            return;
+        foundPaths += 1;
         dirmap.files.push(subPath);
         files.push(subPath);
         visited.add(subPath);
@@ -37,6 +44,9 @@ const readdir = (rootPath, options) => {
     const handleSymlink = (dirmap, subPath, depth) => {
         if (visited.has(subPath))
             return;
+        if (foundPaths >= maxPaths)
+            return;
+        foundPaths += 1;
         dirmap.symlinks.push(subPath);
         symlinks.push(subPath);
         visited.add(subPath);
@@ -44,7 +54,9 @@ const readdir = (rootPath, options) => {
             return;
         if (depth >= maxDepth)
             return;
-        return limiter.add(() => populateResultFromSymlink(subPath, depth + 1));
+        if (foundPaths >= maxPaths)
+            return;
+        return populateResultFromSymlink(subPath, depth + 1);
     };
     const handleStat = (dirmap, rootPath, stat, depth) => {
         if (signal.aborted)
@@ -64,7 +76,8 @@ const readdir = (rootPath, options) => {
     const handleDirent = (dirmap, rootPath, dirent, depth) => {
         if (signal.aborted)
             return;
-        const subPath = `${rootPath}${path.sep}${dirent.name}`;
+        const separator = (rootPath === path.sep) ? '' : path.sep;
+        const subPath = `${rootPath}${separator}${dirent.name}`;
         if (isIgnored(subPath))
             return;
         if (dirent.isDirectory()) {
@@ -87,6 +100,8 @@ const readdir = (rootPath, options) => {
             return;
         if (depth > maxDepth)
             return;
+        if (foundPaths >= maxPaths)
+            return;
         const dirents = await fs.promises.readdir(rootPath, { withFileTypes: true }).catch(() => []);
         if (signal.aborted)
             return;

package/dist/types.d.ts

@@ -1,21 +1,22 @@
-declare type Promisable<T> = Promise<T> | T;
-declare type Options = {
+type Promisable<T> = Promise<T> | T;
+type Options = {
     depth?: number;
+    limit?: number;
     followSymlinks?: boolean;
     ignore?: (targetPath: string) => boolean;
     signal?: {
         aborted: boolean;
     };
 };
-declare type ResultDirectory = {
+type ResultDirectory = {
     directories: string[];
     files: string[];
     symlinks: string[];
 };
-declare type ResultDirectories = {
+type ResultDirectories = {
     [path: string]: ResultDirectory;
 };
-declare type Result = ResultDirectory & {
+type Result = ResultDirectory & {
     map: ResultDirectories;
 };
 export type { Promisable, Options, ResultDirectory, ResultDirectories, Result };

package/package.json

@@ -2,7 +2,7 @@
   "name": "tiny-readdir",
   "repository": "github:fabiospampinato/tiny-readdir",
   "description": "A simple promisified recursive readdir function.",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "type": "module",
   "main": "dist/index.js",
   "exports": "./dist/index.js",
@@ -11,8 +11,9 @@
     "clean": "tsex clean",
     "compile": "tsex compile",
     "compile:watch": "tsex compile --watch",
-    "test": "tsex test",
-    "test:watch": "tsex test --watch",
+    "test": "npm run test:native && npm run test:yielding",
+    "test:native": "fava '**/native.js'",
+    "test:yielding": "node test/yielding.js",
     "prepublishOnly": "npm run clean && npm run compile && npm run test"
   },
   "keywords": [
@@ -22,13 +23,10 @@
     "simple",
     "tiny"
   ],
-  "dependencies": {
-    "promise-concurrency-limiter": "^2.0.0"
-  },
   "devDependencies": {
-    "@types/node": "^17.0.23",
-    "fava": "^0.0.6",
-    "tsex": "^1.1.1",
-    "typescript": "^4.6.3"
+    "@types/node": "^18.11.9",
+    "fava": "^0.0.7",
+    "tsex": "^1.1.3",
+    "typescript": "^4.9.3"
   }
 }

package/readme.md

@@ -17,6 +17,7 @@ const aborter = new AbortController ();
 
 const result = await readdir ( '/foo/bar', {
   depth: 20, // Maximum depth to look at
+  limit: 1_000_000, // Maximum number of files explored, useful as a stop gap in some edge cases
   followSymlinks: true, // Whether to follow symlinks or not
   ignore: targetPath => /node_modules/.test ( targetPath ), // Function that if returns true will ignore this particular file or a directory and its descendants
   signal: aborter.signal // Optional abort signal, useful for aborting potentially expensive operations

package/src/index.ts

@@ -3,19 +3,15 @@
 
 import fs from 'node:fs';
 import path from 'node:path';
-import Limiter from 'promise-concurrency-limiter';
 import type {Promisable, Options, ResultDirectory, ResultDirectories, Result} from './types';
 
-/* HELPERS */
-
-const limiter = new Limiter ({ concurrency: 500 });
-
 /* MAIN */
 
 const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
   const followSymlinks = options?.followSymlinks ?? false;
   const maxDepth = options?.depth ?? Infinity;
+  const maxPaths = options?.limit ?? Infinity;
   const isIgnored = options?.ignore ?? (() => false);
   const signal = options?.signal ?? { aborted: false };
   const directories: string[] = [];
@@ -26,17 +22,24 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
   const resultEmpty: Result = { directories: [], files: [], symlinks: [], map: {} };
   const result: Result = { directories, files, symlinks, map };
 
+  let foundPaths = 0;
+
   const handleDirectory = ( dirmap: ResultDirectory, subPath: string, depth: number ): Promisable<void> => {
 
     if ( visited.has ( subPath ) ) return;
 
+    if ( foundPaths >= maxPaths ) return;
+
+    foundPaths += 1;
     dirmap.directories.push ( subPath );
     directories.push ( subPath );
     visited.add ( subPath );
 
     if ( depth >= maxDepth ) return;
 
-    return limiter.add ( () => populateResultFromPath ( subPath, depth + 1 ) );
+    if ( foundPaths >= maxPaths ) return;
+
+    return populateResultFromPath ( subPath, depth + 1 );
 
   };
 
@@ -44,6 +47,9 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     if ( visited.has ( subPath ) ) return;
 
+    if ( foundPaths >= maxPaths ) return;
+
+    foundPaths += 1;
     dirmap.files.push ( subPath );
     files.push ( subPath );
     visited.add ( subPath );
@@ -54,6 +60,9 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     if ( visited.has ( subPath ) ) return;
 
+    if ( foundPaths >= maxPaths ) return;
+
+    foundPaths += 1;
     dirmap.symlinks.push ( subPath );
     symlinks.push ( subPath );
     visited.add ( subPath );
@@ -62,7 +71,9 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     if ( depth >= maxDepth ) return;
 
-    return limiter.add ( () => populateResultFromSymlink ( subPath, depth + 1 ) );
+    if ( foundPaths >= maxPaths ) return;
+
+    return populateResultFromSymlink ( subPath, depth + 1 );
 
   };
 
@@ -92,7 +103,8 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     if ( signal.aborted ) return;
 
-    const subPath = `${rootPath}${path.sep}${dirent.name}`;
+    const separator = ( rootPath === path.sep ) ? '' : path.sep;
+    const subPath = `${rootPath}${separator}${dirent.name}`;
 
     if ( isIgnored ( subPath ) ) return;
 
@@ -128,6 +140,8 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     if ( depth > maxDepth ) return;
 
+    if ( foundPaths >= maxPaths ) return;
+
     const dirents = await fs.promises.readdir ( rootPath, { withFileTypes: true } ).catch ( () => [] );
 
     if ( signal.aborted ) return;

package/src/types.ts

@@ -7,6 +7,7 @@ type Promisable<T> = Promise<T> | T;
 
 type Options = {
   depth?: number,
+  limit?: number,
   followSymlinks?: boolean,
   ignore?: ( targetPath: string ) => boolean,
   signal?: { aborted: boolean }

package/test/{index.js → native.js}

@@ -10,7 +10,7 @@ import readdir from '../dist/index.js';
 
 describe ( 'Tiny Readdir', it => {
 
-  it ( 'works', async t => {
+  it ( 'finds folders, files and symlinks', async t => {
 
     const cwdPath = process.cwd ();
     const root1Path = path.join ( cwdPath, 'test', 'root1' );
@@ -97,4 +97,71 @@ describe ( 'Tiny Readdir', it => {
 
   });
 
+  it ( 'supports a limit option', async t => {
+
+    const cwdPath = process.cwd ();
+    const root1Path = path.join ( cwdPath, 'test', 'root1' );
+    const root2Path = path.join ( cwdPath, 'test', 'root2' );
+    const folder1Path = path.join ( root1Path, 'folder1' );
+    const folder2Path = path.join ( root1Path, 'folder2' );
+    const folder1DeepPath = path.join ( folder1Path, 'deep' );
+    const file1aPath = path.join ( folder1Path, 'file1a.txt' );
+    const file1bPath = path.join ( folder1Path, 'file1b.txt' );
+    const file2Path = path.join ( folder2Path, 'file2.txt' );
+    const fileDeep1Path = path.join ( folder1DeepPath, 'file1.txt' );
+    const symlink1FromPath = path.join ( root1Path, 'symlink' );
+    const symlink1ToPath = root2Path;
+    const symlink2FromPath = path.join ( root2Path, 'symlink' );
+    const symlink2ToPath = root1Path;
+
+    fs.mkdirSync ( root1Path );
+    fs.mkdirSync ( root2Path );
+    fs.mkdirSync ( folder1Path );
+    fs.mkdirSync ( folder2Path );
+    fs.mkdirSync ( folder1DeepPath );
+    fs.writeFileSync ( file1aPath, '' );
+    fs.writeFileSync ( file1bPath, '' );
+    fs.writeFileSync ( file2Path, '' );
+    fs.writeFileSync ( fileDeep1Path, '' );
+    fs.symlinkSync ( symlink1ToPath, symlink1FromPath );
+    fs.symlinkSync ( symlink2ToPath, symlink2FromPath );
+
+    const expected = {
+      directories: [folder1Path, folder2Path],
+      files: [],
+      symlinks: [symlink1FromPath],
+      map: {
+        [root1Path]: {
+          directories: [folder1Path, folder2Path],
+          files: [],
+          symlinks: [symlink1FromPath]
+        },
+        [folder1Path]: {
+          directories: [],
+          files: [],
+          symlinks: []
+        },
+        [folder2Path]: {
+          directories: [],
+          files: [],
+          symlinks: []
+        }
+      }
+    };
+
+    try {
+
+      const result = await readdir ( root1Path, { limit: 3, followSymlinks: true } );
+
+      t.deepEqual ( result, expected );
+
+    } finally {
+
+      fs.rmSync ( root1Path, { recursive: true } );
+      fs.rmSync ( root2Path, { recursive: true } );
+
+    }
+
+  });
+
 });

package/test/yielding.js

@@ -0,0 +1,35 @@
+
+/* IMPORT */
+
+import readdir from '../dist/index.js';
+
+/* MAIN */
+
+const main = async () => {
+
+  let count = 0;
+  let start = Date.now ();
+
+  setInterval ( () => {
+    count += 1;
+    console.log ( 'tick', count );
+    if ( count < 100 ) return;
+    const end = Date.now ();
+    const elapsed = end - start;
+    console.log ( 'elapsed', elapsed );
+    if ( elapsed > 1500 ) {
+      process.exit ( 1 ); // Fail
+    } else {
+      process.exit ( 0 ); // Success
+    }
+  }, 10 );
+
+  await readdir ( '/' );
+
+  process.exit ( 1 ); // Fail
+
+};
+
+/* RUNNING */
+
+await main ();