tiny-readdir 1.5.0 → 2.1.0

package/.editorconfig

@@ -8,6 +8,3 @@ indent_size = 2
 indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
-
-[*.md]
-trim_trailing_whitespace = false

package/dist/index.d.ts

@@ -1,9 +1,3 @@
-import { Options, Result } from './types';
-declare const readdir: (rootPath: string, options?: Options | undefined) => Promise<Result>;
-declare const _default: typeof readdir & {
-  default: typeof readdir;
-}
-declare namespace _default {
-  export type type = readdir;
-}
-export = _default;
+import type { Options, Result } from './types';
+declare const readdir: (rootPath: string, options?: Options) => Promise<Result>;
+export default readdir;

package/dist/index.js

@@ -1,33 +1,62 @@
-"use strict";
 /* IMPORT */
-const fs = require("fs");
-const path = require("path");
-const promise_concurrency_limiter_1 = require("promise-concurrency-limiter");
-/* HELPERS */
-const limiter = new promise_concurrency_limiter_1.default({ concurrency: 500 });
-/* TINY READDIR */
+import fs from 'node:fs';
+import path from 'node:path';
+/* MAIN */
 const readdir = (rootPath, options) => {
-    var _a, _b, _c, _d;
-    const followSymlinks = (_a = options === null || options === void 0 ? void 0 : options.followSymlinks) !== null && _a !== void 0 ? _a : false, maxDepth = (_b = options === null || options === void 0 ? void 0 : options.depth) !== null && _b !== void 0 ? _b : Infinity, isIgnored = (_c = options === null || options === void 0 ? void 0 : options.ignore) !== null && _c !== void 0 ? _c : (() => false), signal = (_d = options === null || options === void 0 ? void 0 : options.signal) !== null && _d !== void 0 ? _d : { aborted: false }, directories = [], files = [], symlinks = [], map = {}, resultEmpty = { directories: [], files: [], symlinks: [], map: {} }, result = { directories, files, symlinks, map };
+    const followSymlinks = options?.followSymlinks ?? false;
+    const maxDepth = options?.depth ?? Infinity;
+    const maxPaths = options?.limit ?? Infinity;
+    const isIgnored = options?.ignore ?? (() => false);
+    const signal = options?.signal ?? { aborted: false };
+    const directories = [];
+    const files = [];
+    const symlinks = [];
+    const map = {};
+    const visited = new Set();
+    const resultEmpty = { directories: [], files: [], symlinks: [], map: {} };
+    const result = { directories, files, symlinks, map };
+    let foundPaths = 0;
     const handleDirectory = (dirmap, subPath, depth) => {
+        if (visited.has(subPath))
+            return;
+        if (foundPaths >= maxPaths)
+            return;
+        foundPaths += 1;
         dirmap.directories.push(subPath);
         directories.push(subPath);
+        visited.add(subPath);
         if (depth >= maxDepth)
             return;
-        return limiter.add(() => populateResultFromPath(subPath, depth + 1));
+        if (foundPaths >= maxPaths)
+            return;
+        return populateResultFromPath(subPath, depth + 1);
     };
     const handleFile = (dirmap, subPath) => {
+        if (visited.has(subPath))
+            return;
+        if (foundPaths >= maxPaths)
+            return;
+        foundPaths += 1;
         dirmap.files.push(subPath);
         files.push(subPath);
+        visited.add(subPath);
     };
     const handleSymlink = (dirmap, subPath, depth) => {
+        if (visited.has(subPath))
+            return;
+        if (foundPaths >= maxPaths)
+            return;
+        foundPaths += 1;
         dirmap.symlinks.push(subPath);
         symlinks.push(subPath);
+        visited.add(subPath);
         if (!followSymlinks)
             return;
         if (depth >= maxDepth)
             return;
-        return limiter.add(() => populateResultFromSymlink(subPath, depth + 1));
+        if (foundPaths >= maxPaths)
+            return;
+        return populateResultFromSymlink(subPath, depth + 1);
     };
     const handleStat = (dirmap, rootPath, stat, depth) => {
         if (signal.aborted)
@@ -47,7 +76,8 @@ const readdir = (rootPath, options) => {
     const handleDirent = (dirmap, rootPath, dirent, depth) => {
         if (signal.aborted)
             return;
-        const subPath = `${rootPath}${path.sep}${dirent.name}`;
+        const separator = (rootPath === path.sep) ? '' : path.sep;
+        const subPath = `${rootPath}${separator}${dirent.name}`;
         if (isIgnored(subPath))
             return;
         if (dirent.isDirectory()) {
@@ -70,6 +100,8 @@ const readdir = (rootPath, options) => {
             return;
         if (depth > maxDepth)
             return;
+        if (foundPaths >= maxPaths)
+            return;
         const dirents = await fs.promises.readdir(rootPath, { withFileTypes: true }).catch(() => []);
         if (signal.aborted)
             return;
@@ -80,13 +112,16 @@ const readdir = (rootPath, options) => {
     };
     const populateResultFromSymlink = async (rootPath, depth) => {
         try {
-            const realPath = await fs.promises.realpath(rootPath), stat = await fs.promises.stat(realPath), dirmap = map[rootPath] = { directories: [], files: [], symlinks: [] };
+            const realPath = await fs.promises.realpath(rootPath);
+            const stat = await fs.promises.stat(realPath);
+            const dirmap = map[rootPath] = { directories: [], files: [], symlinks: [] };
             await handleStat(dirmap, realPath, stat, depth);
         }
-        catch (_a) { }
+        catch { }
     };
     const getResult = async (rootPath, depth = 1) => {
         rootPath = path.normalize(rootPath);
+        visited.add(rootPath);
         await populateResultFromPath(rootPath, depth);
         if (signal.aborted)
             return resultEmpty;
@@ -95,6 +130,4 @@ const readdir = (rootPath, options) => {
     return getResult(rootPath);
 };
 /* EXPORT */
-module.exports = readdir;
-module.exports.default = readdir;
-Object.defineProperty(module.exports, "__esModule", { value: true });
+export default readdir;

package/dist/types.d.ts

@@ -1,21 +1,22 @@
-declare type Promisable<T> = T | Promise<T>;
-declare type Options = {
+type Promisable<T> = Promise<T> | T;
+type Options = {
     depth?: number;
+    limit?: number;
     followSymlinks?: boolean;
     ignore?: (targetPath: string) => boolean;
     signal?: {
         aborted: boolean;
     };
 };
-declare type ResultDirectory = {
+type ResultDirectory = {
     directories: string[];
     files: string[];
     symlinks: string[];
 };
-declare type ResultDirectories = {
+type ResultDirectories = {
     [path: string]: ResultDirectory;
 };
-declare type Result = ResultDirectory & {
+type Result = ResultDirectory & {
     map: ResultDirectories;
 };
-export { Promisable, Options, ResultDirectory, ResultDirectories, Result };
+export type { Promisable, Options, ResultDirectory, ResultDirectories, Result };

package/dist/types.js

@@ -1,3 +1,2 @@
-"use strict";
 /* HELPERS */
-Object.defineProperty(exports, "__esModule", { value: true });
+export {};

package/package.json

@@ -1,26 +1,20 @@
 {
   "name": "tiny-readdir",
+  "repository": "github:fabiospampinato/tiny-readdir",
   "description": "A simple promisified recursive readdir function.",
-  "version": "1.5.0",
+  "version": "2.1.0",
+  "type": "module",
   "main": "dist/index.js",
-  "types": "dist/index.d.ts",
+  "exports": "./dist/index.js",
+  "types": "./dist/index.d.ts",
   "scripts": {
-    "clean": "rimraf dist",
-    "compile": "tsc --skipLibCheck && tstei",
-    "compile:watch": "tsc --skipLibCheck --watch",
-    "prepublishOnly": "npm run clean && npm run compile"
-  },
-  "bugs": {
-    "url": "https://github.com/fabiospampinato/tiny-readdir/issues"
-  },
-  "license": "MIT",
-  "author": {
-    "name": "Fabio Spampinato",
-    "email": "spampinabio@gmail.com"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/fabiospampinato/tiny-readdir.git"
+    "clean": "tsex clean",
+    "compile": "tsex compile",
+    "compile:watch": "tsex compile --watch",
+    "test": "npm run test:native && npm run test:yielding",
+    "test:native": "fava '**/native.js'",
+    "test:yielding": "node test/yielding.js",
+    "prepublishOnly": "npm run clean && npm run compile && npm run test"
   },
   "keywords": [
     "readdir",
@@ -29,16 +23,10 @@
     "simple",
     "tiny"
   ],
-  "engines": {
-    "node": ">= 10.12.0"
-  },
-  "dependencies": {
-    "promise-concurrency-limiter": "^1.0.0"
-  },
   "devDependencies": {
-    "@types/node": "^14.14.9",
-    "rimraf": "^3.0.2",
-    "typescript": "^4.1.2",
-    "typescript-transform-export-interop": "^1.0.2"
+    "@types/node": "^18.11.9",
+    "fava": "^0.0.7",
+    "tsex": "^1.1.3",
+    "typescript": "^4.9.3"
   }
 }

package/{README.md → readme.md}

@@ -17,6 +17,7 @@ const aborter = new AbortController ();
 
 const result = await readdir ( '/foo/bar', {
   depth: 20, // Maximum depth to look at
+  limit: 1_000_000, // Maximum number of files explored, useful as a stop gap in some edge cases
   followSymlinks: true, // Whether to follow symlinks or not
   ignore: targetPath => /node_modules/.test ( targetPath ), // Function that if returns true will ignore this particular file or a directory and its descendants
   signal: aborter.signal // Optional abort signal, useful for aborting potentially expensive operations

package/src/index.ts

@@ -1,58 +1,79 @@
 
 /* IMPORT */
 
-import * as fs from 'fs';
-import * as path from 'path';
-import Limiter from 'promise-concurrency-limiter';
-import {Promisable, Options, ResultDirectory, ResultDirectories, Result} from './types';
+import fs from 'node:fs';
+import path from 'node:path';
+import type {Promisable, Options, ResultDirectory, ResultDirectories, Result} from './types';
 
-/* HELPERS */
-
-const limiter = new Limiter ({ concurrency: 500 });
-
-/* TINY READDIR */
+/* MAIN */
 
 const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
-  const followSymlinks = options?.followSymlinks ?? false,
-        maxDepth = options?.depth ?? Infinity,
-        isIgnored = options?.ignore ?? (() => false),
-        signal = options?.signal ?? { aborted: false },
-        directories: string[] = [],
-        files: string[] = [],
-        symlinks: string[] = [],
-        map: ResultDirectories = {},
-        resultEmpty: Result = { directories: [], files: [], symlinks: [], map: {} },
-        result: Result = { directories, files, symlinks, map };
+  const followSymlinks = options?.followSymlinks ?? false;
+  const maxDepth = options?.depth ?? Infinity;
+  const maxPaths = options?.limit ?? Infinity;
+  const isIgnored = options?.ignore ?? (() => false);
+  const signal = options?.signal ?? { aborted: false };
+  const directories: string[] = [];
+  const files: string[] = [];
+  const symlinks: string[] = [];
+  const map: ResultDirectories = {};
+  const visited = new Set<string> ();
+  const resultEmpty: Result = { directories: [], files: [], symlinks: [], map: {} };
+  const result: Result = { directories, files, symlinks, map };
+
+  let foundPaths = 0;
 
   const handleDirectory = ( dirmap: ResultDirectory, subPath: string, depth: number ): Promisable<void> => {
 
+    if ( visited.has ( subPath ) ) return;
+
+    if ( foundPaths >= maxPaths ) return;
+
+    foundPaths += 1;
     dirmap.directories.push ( subPath );
     directories.push ( subPath );
+    visited.add ( subPath );
 
     if ( depth >= maxDepth ) return;
 
-    return limiter.add ( () => populateResultFromPath ( subPath, depth + 1 ) );
+    if ( foundPaths >= maxPaths ) return;
+
+    return populateResultFromPath ( subPath, depth + 1 );
 
   };
 
   const handleFile = ( dirmap: ResultDirectory, subPath: string ): void => {
 
+    if ( visited.has ( subPath ) ) return;
+
+    if ( foundPaths >= maxPaths ) return;
+
+    foundPaths += 1;
     dirmap.files.push ( subPath );
     files.push ( subPath );
+    visited.add ( subPath );
 
   };
 
   const handleSymlink = ( dirmap: ResultDirectory, subPath: string, depth: number ): Promisable<void> => {
 
+    if ( visited.has ( subPath ) ) return;
+
+    if ( foundPaths >= maxPaths ) return;
+
+    foundPaths += 1;
     dirmap.symlinks.push ( subPath );
     symlinks.push ( subPath );
+    visited.add ( subPath );
 
     if ( !followSymlinks ) return;
 
     if ( depth >= maxDepth ) return;
 
-    return limiter.add ( () => populateResultFromSymlink ( subPath, depth + 1 ) );
+    if ( foundPaths >= maxPaths ) return;
+
+    return populateResultFromSymlink ( subPath, depth + 1 );
 
   };
 
@@ -82,7 +103,8 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     if ( signal.aborted ) return;
 
-    const subPath = `${rootPath}${path.sep}${dirent.name}`;
+    const separator = ( rootPath === path.sep ) ? '' : path.sep;
+    const subPath = `${rootPath}${separator}${dirent.name}`;
 
     if ( isIgnored ( subPath ) ) return;
 
@@ -118,6 +140,8 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     if ( depth > maxDepth ) return;
 
+    if ( foundPaths >= maxPaths ) return;
+
     const dirents = await fs.promises.readdir ( rootPath, { withFileTypes: true } ).catch ( () => [] );
 
     if ( signal.aborted ) return;
@@ -134,9 +158,9 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     try {
 
-      const realPath = await fs.promises.realpath ( rootPath ),
-            stat = await fs.promises.stat ( realPath ),
-            dirmap = map[rootPath] = { directories: [], files: [], symlinks: [] };
+      const realPath = await fs.promises.realpath ( rootPath );
+      const stat = await fs.promises.stat ( realPath );
+      const dirmap = map[rootPath] = { directories: [], files: [], symlinks: [] };
 
       await handleStat ( dirmap, realPath, stat, depth );
 
@@ -148,6 +172,8 @@ const readdir = ( rootPath: string, options?: Options ): Promise<Result> => {
 
     rootPath = path.normalize ( rootPath );
 
+    visited.add ( rootPath );
+
     await populateResultFromPath ( rootPath, depth );
 
     if ( signal.aborted ) return resultEmpty;

package/src/types.ts

@@ -1,12 +1,13 @@
 
 /* HELPERS */
 
-type Promisable<T> = T | Promise<T>;
+type Promisable<T> = Promise<T> | T;
 
-/* TYPES */
+/* MAIN */
 
 type Options = {
   depth?: number,
+  limit?: number,
   followSymlinks?: boolean,
   ignore?: ( targetPath: string ) => boolean,
   signal?: { aborted: boolean }
@@ -28,4 +29,4 @@ type Result = ResultDirectory & {
 
 /* EXPORT */
 
-export {Promisable, Options, ResultDirectory, ResultDirectories, Result};
+export type {Promisable, Options, ResultDirectory, ResultDirectories, Result};

package/test/native.js

@@ -0,0 +1,167 @@
+
+/* IMPORT */
+
+import {describe} from 'fava';
+import fs from 'node:fs';
+import path from 'node:path';
+import readdir from '../dist/index.js';
+
+/* MAIN */
+
+describe ( 'Tiny Readdir', it => {
+
+  it ( 'finds folders, files and symlinks', async t => {
+
+    const cwdPath = process.cwd ();
+    const root1Path = path.join ( cwdPath, 'test', 'root1' );
+    const root2Path = path.join ( cwdPath, 'test', 'root2' );
+    const folder1Path = path.join ( root1Path, 'folder1' );
+    const folder2Path = path.join ( root1Path, 'folder2' );
+    const folder1DeepPath = path.join ( folder1Path, 'deep' );
+    const file1aPath = path.join ( folder1Path, 'file1a.txt' );
+    const file1bPath = path.join ( folder1Path, 'file1b.txt' );
+    const file2Path = path.join ( folder2Path, 'file2.txt' );
+    const fileDeep1Path = path.join ( folder1DeepPath, 'file1.txt' );
+    const symlink1FromPath = path.join ( root1Path, 'symlink' );
+    const symlink1ToPath = root2Path;
+    const symlink2FromPath = path.join ( root2Path, 'symlink' );
+    const symlink2ToPath = root1Path;
+
+    fs.mkdirSync ( root1Path );
+    fs.mkdirSync ( root2Path );
+    fs.mkdirSync ( folder1Path );
+    fs.mkdirSync ( folder2Path );
+    fs.mkdirSync ( folder1DeepPath );
+    fs.writeFileSync ( file1aPath, '' );
+    fs.writeFileSync ( file1bPath, '' );
+    fs.writeFileSync ( file2Path, '' );
+    fs.writeFileSync ( fileDeep1Path, '' );
+    fs.symlinkSync ( symlink1ToPath, symlink1FromPath );
+    fs.symlinkSync ( symlink2ToPath, symlink2FromPath );
+
+    const expected = {
+      directories: [folder1Path, folder2Path, folder1DeepPath, root2Path],
+      files: [file1aPath, file1bPath, file2Path, fileDeep1Path],
+      symlinks: [symlink1FromPath, symlink2FromPath],
+      map: {
+        [root1Path]: {
+          directories: [folder1Path, folder2Path],
+          files: [],
+          symlinks: [symlink1FromPath]
+        },
+        [root2Path]: {
+          directories: [],
+          files: [],
+          symlinks: [symlink2FromPath]
+        },
+        [folder1Path]: {
+          directories: [folder1DeepPath],
+          files: [file1aPath, file1bPath],
+          symlinks: []
+        },
+        [folder2Path]: {
+          directories: [],
+          files: [file2Path],
+          symlinks: []
+        },
+        [folder1DeepPath]: {
+          directories: [],
+          files: [fileDeep1Path],
+          symlinks: []
+        },
+        [symlink1FromPath]: {
+          directories: [root2Path],
+          files: [],
+          symlinks: []
+        },
+        [symlink2FromPath]: {
+          directories: [],
+          files: [],
+          symlinks: []
+        }
+      }
+    };
+
+    try {
+
+      const result = await readdir ( root1Path, { followSymlinks: true } );
+
+      t.deepEqual ( result, expected );
+
+    } finally {
+
+      fs.rmSync ( root1Path, { recursive: true } );
+      fs.rmSync ( root2Path, { recursive: true } );
+
+    }
+
+  });
+
+  it ( 'supports a limit option', async t => {
+
+    const cwdPath = process.cwd ();
+    const root1Path = path.join ( cwdPath, 'test', 'root1' );
+    const root2Path = path.join ( cwdPath, 'test', 'root2' );
+    const folder1Path = path.join ( root1Path, 'folder1' );
+    const folder2Path = path.join ( root1Path, 'folder2' );
+    const folder1DeepPath = path.join ( folder1Path, 'deep' );
+    const file1aPath = path.join ( folder1Path, 'file1a.txt' );
+    const file1bPath = path.join ( folder1Path, 'file1b.txt' );
+    const file2Path = path.join ( folder2Path, 'file2.txt' );
+    const fileDeep1Path = path.join ( folder1DeepPath, 'file1.txt' );
+    const symlink1FromPath = path.join ( root1Path, 'symlink' );
+    const symlink1ToPath = root2Path;
+    const symlink2FromPath = path.join ( root2Path, 'symlink' );
+    const symlink2ToPath = root1Path;
+
+    fs.mkdirSync ( root1Path );
+    fs.mkdirSync ( root2Path );
+    fs.mkdirSync ( folder1Path );
+    fs.mkdirSync ( folder2Path );
+    fs.mkdirSync ( folder1DeepPath );
+    fs.writeFileSync ( file1aPath, '' );
+    fs.writeFileSync ( file1bPath, '' );
+    fs.writeFileSync ( file2Path, '' );
+    fs.writeFileSync ( fileDeep1Path, '' );
+    fs.symlinkSync ( symlink1ToPath, symlink1FromPath );
+    fs.symlinkSync ( symlink2ToPath, symlink2FromPath );
+
+    const expected = {
+      directories: [folder1Path, folder2Path],
+      files: [],
+      symlinks: [symlink1FromPath],
+      map: {
+        [root1Path]: {
+          directories: [folder1Path, folder2Path],
+          files: [],
+          symlinks: [symlink1FromPath]
+        },
+        [folder1Path]: {
+          directories: [],
+          files: [],
+          symlinks: []
+        },
+        [folder2Path]: {
+          directories: [],
+          files: [],
+          symlinks: []
+        }
+      }
+    };
+
+    try {
+
+      const result = await readdir ( root1Path, { limit: 3, followSymlinks: true } );
+
+      t.deepEqual ( result, expected );
+
+    } finally {
+
+      fs.rmSync ( root1Path, { recursive: true } );
+      fs.rmSync ( root2Path, { recursive: true } );
+
+    }
+
+  });
+
+});

package/test/yielding.js

@@ -0,0 +1,35 @@
+
+/* IMPORT */
+
+import readdir from '../dist/index.js';
+
+/* MAIN */
+
+const main = async () => {
+
+  let count = 0;
+  let start = Date.now ();
+
+  setInterval ( () => {
+    count += 1;
+    console.log ( 'tick', count );
+    if ( count < 100 ) return;
+    const end = Date.now ();
+    const elapsed = end - start;
+    console.log ( 'elapsed', elapsed );
+    if ( elapsed > 1500 ) {
+      process.exit ( 1 ); // Fail
+    } else {
+      process.exit ( 0 ); // Success
+    }
+  }, 10 );
+
+  await readdir ( '/' );
+
+  process.exit ( 1 ); // Fail
+
+};
+
+/* RUNNING */
+
+await main ();

package/tsconfig.json

@@ -1,28 +1,3 @@
 {
-  "compilerOptions": {
-    "alwaysStrict": true,
-    "declaration": true,
-    "emitDecoratorMetadata": true,
-    "experimentalDecorators": true,
-    "forceConsistentCasingInFileNames": true,
-    "inlineSourceMap": false,
-    "jsx": "react",
-    "lib": ["dom", "scripthost", "es2015", "es2016", "es2017", "es2018", "es2019", "es2020"],
-    "module": "commonjs",
-    "moduleResolution": "node",
-    "newLine": "LF",
-    "noFallthroughCasesInSwitch": true,
-    "noUnusedLocals": true,
-    "noUnusedParameters": false,
-    "outDir": "dist",
-    "pretty": true,
-    "strictNullChecks": true,
-    "target": "es2018"
-  },
-  "include": [
-    "src"
-  ],
-  "exclude": [
-    "node_modules"
-  ]
+  "extends": "tsex/tsconfig.json"
 }