thuban 0.3.2 → 0.3.3

package/packages/scanner/file-collector.js

@@ -0,0 +1,64 @@
+const fs = require('fs');
+const path = require('path');
+
+function collectFiles(dir, ignorePatterns = []) {
+  const defaultIgnore = ['node_modules', '.git', 'dist', 'build', '.next', '__pycache__', '.venv', 'venv', 'coverage', '.cache', '.turbo'];
+  const allIgnore = [...defaultIgnore, ...ignorePatterns];
+  const extensions = ['.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs', '.py', '.pyw', '.go', '.rs', '.java', '.kt', '.cs', '.php', '.rb', '.json'];
+  const maxFileSize = 1024 * 1024;
+  const maxFiles = 50000;
+  const files = [];
+  const visitedPaths = new Set(); // Track real paths to handle symlinks
+
+  function walk(currentDir) {
+    if (files.length >= maxFiles) return;
+    let entries;
+    try {
+      entries = fs.readdirSync(currentDir, { withFileTypes: true });
+    } catch (error) {
+      return;
+    }
+
+    for (const entry of entries) {
+      if (files.length >= maxFiles) return;
+      if (allIgnore.some(pattern => entry.name === pattern || entry.name.match(pattern))) continue;
+      const fullPath = path.join(currentDir, entry.name);
+
+      // Handle symlinks — resolve and skip if already visited or points outside
+      if (entry.isSymbolicLink()) {
+        try {
+          const realPath = fs.realpathSync(fullPath);
+          if (visitedPaths.has(realPath)) continue;
+          visitedPaths.add(realPath);
+          const stat = fs.statSync(fullPath); // follow symlink
+          if (stat.isDirectory()) {
+            walk(fullPath);
+          } else if (stat.isFile() && extensions.some(ext => entry.name.endsWith(ext))) {
+            if (stat.size > 0 && stat.size <= maxFileSize) files.push(fullPath);
+          }
+        } catch {
+          continue; // Broken symlink — skip
+        }
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        walk(fullPath);
+      } else if (entry.isFile() && extensions.some(ext => entry.name.endsWith(ext))) {
+        try {
+          const stat = fs.statSync(fullPath);
+          if (stat.size > 0 && stat.size <= maxFileSize) files.push(fullPath);
+        } catch (error) {
+          // Skip files we can't stat
+        }
+      }
+    }
+  }
+
+  walk(dir);
+  return files;
+}
+
+module.exports = {
+  collectFiles,
+};