ucn 4.2.3 → 5.0.2

package/core/discovery.js

@@ -6,6 +6,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const { execFileSync } = require('child_process');
 const { langTraits } = require('../languages');
 
 // Always ignore - unambiguous, never user code
@@ -29,11 +30,10 @@ const DEFAULT_IGNORES = [
     '.eggs',
     '*.egg-info',
 
-    // Build outputs
-    'dist',
+    // Framework-owned build outputs with unambiguous names. Generic names
+    // such as build/dist/out/coverage are intentionally NOT here: real,
+    // git-tracked source trees use them at arbitrary depths.
     '*-dist',
-    'build',
-    'out',
     '.next',
     'next.lock',
     '.nuxt',
@@ -47,8 +47,7 @@ const DEFAULT_IGNORES = [
     'storybook-static',
     '_site',
 
-    // Test/coverage
-    'coverage',
+    // Test/coverage tool metadata
     '.nyc_output',
     '.pytest_cache',
     '.mypy_cache',
@@ -85,7 +84,10 @@ const PROJECT_MARKERS = [
     'Cargo.toml',
     'pom.xml',
     'build.gradle',
-    'Makefile'
+    'Makefile',
+    'CMakeLists.txt',
+    'compile_commands.json',
+    'meson.build'
 ];
 
 // Test file patterns by language
@@ -93,13 +95,17 @@ const TEST_PATTERNS = {
     javascript: [
         /\.test\.(js|jsx|ts|tsx|mjs|cjs)$/,
         /\.spec\.(js|jsx|ts|tsx|mjs|cjs)$/,
+        /(^|\/)(test|spec)\.(js|jsx|ts|tsx|mjs|cjs)$/,
         /__tests__\//,
+        /(^|\/)tests?\//,
         /\.test$/
     ],
     typescript: [
         /\.test\.(ts|tsx)$/,
         /\.spec\.(ts|tsx)$/,
-        /__tests__\//
+        /(^|\/)(test|spec)\.(ts|tsx)$/,
+        /__tests__\//,
+        /(^|\/)tests?\//
     ],
     python: [
         /^test_.*\.py$/,
@@ -118,6 +124,21 @@ const TEST_PATTERNS = {
     rust: [
         /.*_test\.rs$/,
         /(^|\/)tests\//
+    ],
+    c: [
+        /(^|\/)tests?\//,
+        /(^|\/)test_.*\.(c|h)$/,
+        /.*(_test|\.test)\.(c|h)$/
+    ],
+    cpp: [
+        /(^|\/)tests?\//,
+        /(^|\/)test_.*\.(cc|cpp|cxx|c\+\+|hpp|hh|hxx|h\+\+)$/,
+        /.*(_test|\.test)\.(cc|cpp|cxx|c\+\+|hpp|hh|hxx|h\+\+)$/
+    ],
+    csharp: [
+        /(^|\/)tests?\//,
+        /.*Tests?\.cs$/,
+        /.*\.Tests\.cs$/
     ]
 };
 
@@ -128,25 +149,25 @@ const TEST_PATTERNS = {
  * @param {string} projectRoot - Project root directory
  * @returns {string[]} - Array of ignore patterns (directory/file names and globs)
  */
-function parseGitignore(projectRoot) {
-    const gitignorePath = path.join(projectRoot, '.gitignore');
-    if (!fs.existsSync(gitignorePath)) return [];
-
+function parseGitignoreFile(projectRoot, gitignorePath) {
     let content;
-    try {
-        content = fs.readFileSync(gitignorePath, 'utf-8');
-    } catch (e) {
-        return [];
-    }
-
+    try { content = fs.readFileSync(gitignorePath, 'utf-8'); } catch { return []; }
+    const baseRelative = path.relative(projectRoot, path.dirname(gitignorePath))
+        .replaceAll(path.sep, '/');
     const patterns = [];
     for (const rawLine of content.split('\n')) {
         const line = rawLine.trim();
-        // Skip empty lines, comments, negation patterns
-        if (!line || line.startsWith('#') || line.startsWith('!')) continue;
+        // Keep negations and ordering: gitignore is a last-match-wins rule.
+        if (!line || line.startsWith('#')) continue;
 
-        // Strip trailing slash (directory indicator) — shouldIgnore checks names, not types
-        let pattern = line.endsWith('/') ? line.slice(0, -1) : line;
+        const negated = line.startsWith('!');
+        const body = negated ? line.slice(1) : line;
+        if (!body) continue;
+
+        // Strip trailing slash (directory indicator) — shouldIgnore checks
+        // directories before descending, so an exact directory match covers
+        // its whole subtree.
+        let pattern = body.endsWith('/') ? body.slice(0, -1) : body;
 
         // Leading slash = ANCHORED to the .gitignore's directory (fix #226).
         // git semantics: `/locale` ignores only the root-level locale, never
@@ -158,23 +179,117 @@ function parseGitignore(projectRoot) {
         const anchored = pattern.startsWith('/');
         if (anchored) pattern = pattern.slice(1);
 
-        // Skip patterns with interior path separators — shouldIgnore matches
-        // single name segments, not full paths. Patterns like "foo/bar" would
-        // need walkDir-level support.
-        if (pattern.includes('/')) continue;
-
         // Skip empty after stripping
         if (!pattern) continue;
 
         // Avoid duplicating built-in ignores
-        if (DEFAULT_IGNORES.includes(pattern)) continue;
+        if (!negated && DEFAULT_IGNORES.includes(pattern)) continue;
 
-        patterns.push(anchored ? '/' + pattern : pattern);
+        if (!baseRelative) {
+            const normalized = anchored ? '/' + pattern : pattern;
+            patterns.push((negated ? '!' : '') + normalized);
+            continue;
+        }
+
+        // Nested .gitignore rules are scoped to their own directory. Flatten
+        // that scope into root-relative patterns for shouldIgnore(). A rule
+        // without a slash matches at every depth below its .gitignore; emit
+        // both the direct and descendant forms so the glob stays exact.
+        const scoped = `${baseRelative}/${pattern}`;
+        patterns.push((negated ? '!' : '') + scoped);
+        if (!anchored && !pattern.includes('/')) {
+            patterns.push((negated ? '!' : '') + `${baseRelative}/**/${pattern}`);
+        }
     }
 
     return patterns;
 }
 
+function fallbackGitignoreFiles(projectRoot) {
+    const found = [];
+    const visit = dir => {
+        let entries;
+        try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+        for (const entry of entries) {
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isFile() && entry.name === '.gitignore') found.push(fullPath);
+            if (!entry.isDirectory() || shouldIgnore(entry.name, DEFAULT_IGNORES, dir)) continue;
+            visit(fullPath);
+        }
+    };
+    visit(projectRoot);
+    return found;
+}
+
+function hasGitMetadata(projectRoot) {
+    let dir = path.resolve(projectRoot);
+    const filesystemRoot = path.parse(dir).root;
+    while (true) {
+        if (fs.existsSync(path.join(dir, '.git'))) return true;
+        if (dir === filesystemRoot) return false;
+        dir = path.dirname(dir);
+    }
+}
+
+function gitignoreFiles(projectRoot) {
+    if (!hasGitMetadata(projectRoot)) return fallbackGitignoreFiles(projectRoot);
+    try {
+        const output = execFileSync('git', [
+            '-C', projectRoot, 'ls-files', '-z', '--cached', '--others',
+            '--exclude-standard', '--', '.gitignore', '**/.gitignore',
+        ], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
+        return output.split('\0').filter(Boolean)
+            .map(relative => path.resolve(projectRoot, relative))
+            .filter(file => file === projectRoot || file.startsWith(projectRoot + path.sep));
+    } catch {
+        return fallbackGitignoreFiles(projectRoot);
+    }
+}
+
+/**
+ * Parse the root and every applicable nested .gitignore, preserving Git's
+ * parent-before-child override order.
+ */
+function parseGitignore(projectRoot) {
+    const root = path.resolve(projectRoot);
+    const files = gitignoreFiles(root).sort((a, b) => {
+        const depthA = path.relative(root, a).split(path.sep).length;
+        const depthB = path.relative(root, b).split(path.sep).length;
+        return depthA - depthB || compareNames(a, b);
+    });
+    const patterns = [];
+    for (const file of files) patterns.push(...parseGitignoreFile(root, file));
+    return patterns;
+}
+
+/** Return tracked files and their ancestor directories, relative to root. */
+function gitTrackedPaths(projectRoot) {
+    if (!hasGitMetadata(projectRoot)) {
+        return { files: new Set(), directories: new Set() };
+    }
+    try {
+        const output = execFileSync('git', [
+            '-C', projectRoot, 'ls-files', '-z', '--cached',
+        ], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
+        const files = new Set();
+        const directories = new Set();
+        for (const raw of output.split('\0')) {
+            if (!raw) continue;
+            const relative = path.normalize(raw).replaceAll(path.sep, '/');
+            if (relative === '..' || relative.startsWith('../') || path.isAbsolute(relative)) continue;
+            files.add(relative);
+            let parent = path.posix.dirname(relative);
+            while (parent && parent !== '.') {
+                directories.add(parent);
+                parent = path.posix.dirname(parent);
+            }
+        }
+        return { files, directories };
+    } catch {
+        return { files: new Set(), directories: new Set() };
+    }
+}
+
 function compareNames(a, b) {
     const aLower = a.toLowerCase();
     const bLower = b.toLowerCase();
@@ -199,9 +314,12 @@ function compareNames(a, b) {
 function expandGlob(pattern, options = {}) {
     const root = path.resolve(options.root || process.cwd());
     const ignores = options.ignores || DEFAULT_IGNORES;
-    const maxDepth = options.maxDepth || 20;
-    const maxFiles = options.maxFiles || 50000;
+    const maxDepth = options.maxDepth ?? 1000;
+    const maxFiles = options.maxFiles ?? 50000;
+    const maxFileSize = options.maxFileSize ?? (8 * 1024 * 1024);
     const followSymlinks = options.followSymlinks !== false; // default true
+    const gitignorePatterns = options.gitignorePatterns || [];
+    const trackedPaths = options.trackedPaths || { files: new Set(), directories: new Set() };
 
     // Handle home directory expansion
     if (pattern.startsWith('~/')) {
@@ -217,14 +335,27 @@ function expandGlob(pattern, options = {}) {
         filePattern,
         recursive,
         ignores,
+        gitignorePatterns,
+        trackedPaths,
         maxDepth,
+        maxFileSize,
         followSymlinks,
         // Anchored gitignore patterns ('/name') apply only to entries directly
         // under the project root — the .gitignore's own directory (fix #226).
         anchorRoot: root,
+        onSkippedFile: options.onSkippedFile,
+        onDiscoveryIssue: options.onDiscoveryIssue,
+        disclosureIgnores: options.disclosureIgnores,
         onFile: (filePath) => {
             if (files.length < maxFiles) {
                 files.push(filePath);
+            } else if (options.onDiscoveryIssue) {
+                options.onDiscoveryIssue({
+                    path: filePath,
+                    kind: 'file',
+                    reason: 'max-files',
+                    detail: `discovery exceeded maxFiles=${maxFiles}`,
+                });
             }
         }
     });
@@ -293,7 +424,13 @@ function globToRegex(glob) {
  * Walk a directory tree, calling onFile for each matching file
  */
 function walkDir(dir, options, depth = 0, visited = new Set()) {
-    if (depth > options.maxDepth) return;
+    if (depth > options.maxDepth) {
+        options.onDiscoveryIssue?.({
+            path: dir, kind: 'directory', reason: 'max-depth',
+            detail: `discovery exceeded maxDepth=${options.maxDepth}`,
+        });
+        return;
+    }
     if (!fs.existsSync(dir)) return;
 
     // Track visited directories to avoid circular symlinks
@@ -301,6 +438,10 @@ function walkDir(dir, options, depth = 0, visited = new Set()) {
     try {
         realDir = fs.realpathSync(dir);
     } catch (e) {
+        options.onDiscoveryIssue?.({
+            path: dir, kind: 'directory', reason: 'unreadable-directory',
+            detail: e.message,
+        });
         return; // broken symlink
     }
     if (visited.has(realDir)) return;
@@ -310,6 +451,10 @@ function walkDir(dir, options, depth = 0, visited = new Set()) {
     try {
         entries = fs.readdirSync(dir, { withFileTypes: true });
     } catch (e) {
+        options.onDiscoveryIssue?.({
+            path: dir, kind: 'directory', reason: 'unreadable-directory',
+            detail: e.message,
+        });
         return;
     }
 
@@ -320,7 +465,51 @@ function walkDir(dir, options, depth = 0, visited = new Set()) {
     for (const entry of entries) {
         const fullPath = path.join(dir, entry.name);
 
-        if (shouldIgnore(entry.name, options.ignores, dir, dir === options.anchorRoot)) continue;
+        const entryIsDirectory = entry.isDirectory();
+        const policyIgnored = shouldIgnore(
+            entry.name,
+            options.ignores,
+            dir,
+            dir === options.anchorRoot,
+            options.anchorRoot,
+            fullPath,
+            entryIsDirectory,
+        );
+        const relative = options.anchorRoot
+            ? path.relative(options.anchorRoot, fullPath).replaceAll(path.sep, '/')
+            : '';
+        const tracked = entryIsDirectory
+            ? options.trackedPaths?.directories?.has(relative)
+            : options.trackedPaths?.files?.has(relative);
+        const gitIgnored = !tracked && options.gitignorePatterns?.length > 0 && shouldIgnore(
+            entry.name,
+            options.gitignorePatterns,
+            dir,
+            dir === options.anchorRoot,
+            options.anchorRoot,
+            fullPath,
+            entryIsDirectory,
+        );
+        if (policyIgnored || gitIgnored) {
+            if (options.disclosureIgnores && shouldIgnore(
+                entry.name,
+                options.disclosureIgnores,
+                dir,
+                dir === options.anchorRoot,
+                options.anchorRoot,
+                fullPath,
+                entryIsDirectory,
+                false,
+            )) {
+                options.onDiscoveryIssue?.({
+                    path: fullPath,
+                    kind: entryIsDirectory ? 'directory' : 'file',
+                    reason: 'config-exclude',
+                    detail: 'excluded by .ucn.json',
+                });
+            }
+            continue;
+        }
 
         let isDir = entry.isDirectory();
         let isFile = entry.isFile();
@@ -342,7 +531,25 @@ function walkDir(dir, options, depth = 0, visited = new Set()) {
             }
         } else if (isFile) {
             if (options.filePattern.test(entry.name)) {
+                let size;
+                try { size = fs.statSync(fullPath).size; } catch (e) {
+                    options.onDiscoveryIssue?.({
+                        path: fullPath, kind: 'file', reason: 'unreadable-file',
+                        detail: e.message,
+                    });
+                    continue;
+                }
+                if (size > options.maxFileSize) {
+                    options.onDiscoveryIssue?.({
+                        path: fullPath, kind: 'file', reason: 'max-file-size',
+                        detail: `${size} bytes exceeds maxFileSize=${options.maxFileSize}`,
+                        size,
+                    });
+                    continue;
+                }
                 options.onFile(fullPath);
+            } else if (options.onSkippedFile) {
+                options.onSkippedFile(fullPath);
             }
         }
     }
@@ -357,31 +564,86 @@ function walkDir(dir, options, depth = 0, visited = new Set()) {
  *   the .gitignore belongs to. Default false errs toward KEEPING files.
  * @param {string} [parentDir] - Parent directory path (for conditional checks)
  * @param {boolean} [atAnchorRoot] - Whether parentDir IS the anchor root
+ * @param {string} [anchorRoot] - Discovery root for path-aware config ignores
+ * @param {string} [fullPath] - Candidate path for root-relative matching
  */
 const _globRegexCache = new Map();
 
-function shouldIgnore(name, ignores, parentDir, atAnchorRoot = false) {
-    // Check unconditional ignores
-    for (let pattern of ignores) {
-        if (pattern.charCodeAt(0) === 47 /* '/' */) {
+function shouldIgnore(
+    name,
+    ignores,
+    parentDir = null,
+    atAnchorRoot = false,
+    anchorRoot = null,
+    fullPath = null,
+    isDirectory = false,
+    includeConditional = true,
+) {
+    let ignored = false;
+    for (const rawPattern of ignores) {
+        const negated = rawPattern.charCodeAt(0) === 33 /* ! */;
+        let pattern = negated ? rawPattern.slice(1) : rawPattern;
+        let matched = false;
+        if (pattern.includes('/') && anchorRoot && fullPath) {
+            const normalizedPattern = pattern.replace(/^\/+/, '')
+                .replaceAll('\\', '/');
+            const relative = path.relative(anchorRoot, fullPath)
+                .replaceAll(path.sep, '/');
+            let regex = _globRegexCache.get(`path:${normalizedPattern}`);
+            if (!regex) {
+                regex = globToRegex(normalizedPattern);
+                _globRegexCache.set(`path:${normalizedPattern}`, regex);
+            }
+            const directoryPrefix = normalizedPattern.endsWith('/**')
+                ? normalizedPattern.slice(0, -3).replace(/\/+$/, '')
+                : null;
+            if (regex.test(relative) ||
+                (directoryPrefix && relative === directoryPrefix)) {
+                matched = true;
+            }
+        } else if (pattern.charCodeAt(0) === 47 /* '/' */) {
             if (!atAnchorRoot) continue;
             pattern = pattern.slice(1);
-        }
-        if (pattern.includes('*')) {
+            matched = pattern.includes('*')
+                ? globToRegex(pattern).test(name)
+                : name === pattern;
+        } else if (pattern.includes('*')) {
             let regex = _globRegexCache.get(pattern);
             if (!regex) {
                 regex = globToRegex(pattern);
                 _globRegexCache.set(pattern, regex);
             }
-            if (regex.test(name)) return true;
+            if (regex.test(name)) matched = true;
         } else if (name === pattern) {
-            return true;
+            matched = true;
         }
+        if (matched) ignored = !negated;
+    }
+
+    // A later negation can re-include a descendant. Keep walking a currently
+    // ignored directory whenever such a rule might apply below it.
+    if (ignored && isDirectory && anchorRoot && fullPath) {
+        const relative = path.relative(anchorRoot, fullPath).replaceAll(path.sep, '/');
+        const mayReincludeDescendant = ignores.some(raw => {
+            if (!raw.startsWith('!')) return false;
+            const pattern = raw.slice(1).replace(/^\/+/, '');
+            return pattern.includes('/') &&
+                (pattern.startsWith(relative + '/') || pattern.includes('**'));
+        });
+        if (mayReincludeDescendant) ignored = false;
+    }
+    if (ignored) {
+        return true;
+    }
+
+    // A negated rule is only meaningful after a positive rule matched.
+    if (ignores.some(pattern => pattern.startsWith('!'))) {
+        // Fall through to conditional ignores.
     }
 
     // Check conditional ignores (only if parentDir provided)
     // Use Array.isArray to avoid matching Object.prototype properties (e.g. dir named "constructor")
-    if (parentDir && Array.isArray(CONDITIONAL_IGNORES[name])) {
+    if (includeConditional && parentDir && Array.isArray(CONDITIONAL_IGNORES[name])) {
         const markers = CONDITIONAL_IGNORES[name];
         for (const marker of markers) {
             if (fs.existsSync(path.join(parentDir, marker))) {
@@ -406,16 +668,60 @@ function findProjectRoot(startDir) {
                 return dir;
             }
         }
+        try {
+            const entries = fs.readdirSync(dir);
+            if (entries.some(name => /\.(csproj|sln|vcxproj)$/i.test(name))) {
+                return dir;
+            }
+        } catch (_) {
+            // Keep walking toward the filesystem root.
+        }
         dir = path.dirname(dir);
     }
 
     return path.resolve(startDir);
 }
 
-// All file extensions for languages UCN supports as code analysis (excludes .rb/.php/.c/.cpp etc.
-// which are extensions UCN scans but doesn't analyze). When build manifests can't tell us
-// what's in a project, we scan all of these — the file extension alone determines language.
-const ALL_SUPPORTED_EXTENSIONS = ['js', 'jsx', 'ts', 'tsx', 'mjs', 'cjs', 'py', 'go', 'java', 'rs', 'html', 'htm'];
+// All file extensions UCN analyzes. When manifests cannot identify the
+// project, extension-based discovery remains the source of truth.
+const ALL_SUPPORTED_EXTENSIONS = [
+    'js', 'jsx', 'ts', 'tsx', 'mjs', 'cjs', 'py', 'pyi', 'go', 'java', 'rs',
+    'c', 'h', 'cc', 'cpp', 'cxx', 'c++', 'hpp', 'hh', 'hxx', 'h++',
+    'cs', 'csx', 'html', 'htm',
+];
+
+// Source-like extensions that UCN deliberately does not parse. This is not an
+// attempt to recognize every file in existence: it is the honest handoff set
+// used by project-wide discovery so `repo`/`doctor` cannot report a clean,
+// complete index while silently omitting a common programming language.
+const UNSUPPORTED_SOURCE_EXTENSIONS = {
+    rb: 'Ruby', rake: 'Ruby',
+    php: 'PHP',
+    swift: 'Swift',
+    kt: 'Kotlin', kts: 'Kotlin',
+    scala: 'Scala', sc: 'Scala',
+    lua: 'Lua',
+    dart: 'Dart',
+    ex: 'Elixir', exs: 'Elixir',
+    erl: 'Erlang', hrl: 'Erlang',
+    clj: 'Clojure', cljs: 'ClojureScript', cljc: 'Clojure',
+    groovy: 'Groovy',
+    vb: 'Visual Basic', vbs: 'Visual Basic',
+    fs: 'F#', fsx: 'F#', fsi: 'F#',
+    hs: 'Haskell', lhs: 'Haskell',
+    zig: 'Zig',
+    sol: 'Solidity',
+    vue: 'Vue', svelte: 'Svelte',
+    sh: 'Shell', bash: 'Shell', zsh: 'Shell', fish: 'Shell',
+    ps1: 'PowerShell',
+    sql: 'SQL',
+};
+
+function classifyUnsupportedSourceFile(filePath) {
+    const extension = path.extname(filePath).slice(1).toLowerCase();
+    const language = UNSUPPORTED_SOURCE_EXTENSIONS[extension];
+    return language ? { extension, language } : null;
+}
 
 // Build-manifest hints: when present, we know the project has files of that language
 // regardless of whether sources are visible at the time of scan. Used as hints, not gates —
@@ -430,6 +736,10 @@ const MANIFEST_HINTS = {
     'pom.xml':           ['java'],
     'build.gradle':      ['java'],
     'build.gradle.kts':  ['java'],
+    'Makefile':          ['c', 'h', 'cc', 'cpp', 'cxx', 'hpp'],
+    'CMakeLists.txt':    ['c', 'h', 'cc', 'cpp', 'cxx', 'hpp'],
+    'compile_commands.json': ['c', 'h', 'cc', 'cpp', 'cxx', 'hpp'],
+    'meson.build':       ['c', 'h', 'cc', 'cpp', 'cxx', 'hpp'],
 };
 
 /**
@@ -585,10 +895,13 @@ module.exports = {
     isTestFile,
     findTestFileFor,
     parseGitignore,
+    gitTrackedPaths,
     DEFAULT_IGNORES,
     PROJECT_MARKERS,
     TEST_PATTERNS,
     ALL_SUPPORTED_EXTENSIONS,
+    UNSUPPORTED_SOURCE_EXTENSIONS,
+    classifyUnsupportedSourceFile,
     MANIFEST_HINTS,
     compareNames
 };