thailint 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

src/orchestrator/core.py

@@ -10,34 +10,162 @@ Overview: Provides the main entry point for linting operations by coordinating e
     appropriate file information and language metadata, executes applicable rules against contexts,
     and collects violations across all processed files. Supports recursive and non-recursive
     directory traversal, respects .thailintignore patterns at repository level, and provides
-    configurable linting through .thailint.yaml configuration files. Serves as the primary
-    interface between the linter framework and user-facing CLI/library APIs.
+    configurable linting through .thailint.yaml configuration files. Includes parallel processing
+    support for improved performance on multi-core systems. Serves as the primary interface between
+    the linter framework and user-facing CLI/library APIs.
 
 Dependencies: pathlib for file operations, BaseLintRule and BaseLintContext from core.base,
     Violation from core.types, RuleRegistry from core.registry, LinterConfigLoader from
     linter_config.loader, IgnoreDirectiveParser from linter_config.ignore, detect_language
-    from language_detector
+    from language_detector, concurrent.futures for parallel processing
 
 Exports: Orchestrator class, FileLintContext implementation class
 
 Interfaces: Orchestrator(project_root: Path | None), lint_file(file_path: Path) -> list[Violation],
-    lint_directory(dir_path: Path, recursive: bool) -> list[Violation]
+    lint_directory(dir_path: Path, recursive: bool) -> list[Violation],
+    lint_files_parallel(file_paths, max_workers) -> list[Violation]
 
 Implementation: Directory glob pattern matching for traversal (** for recursive, * for shallow),
     ignore pattern checking before file processing, dynamic context creation per file,
-    rule filtering by applicability, violation collection and aggregation across files
+    rule filtering by applicability, violation collection and aggregation across files,
+    ProcessPoolExecutor for parallel file processing
 """
 
+from __future__ import annotations
+
+import multiprocessing
+import os
+from concurrent.futures import Future, ProcessPoolExecutor, as_completed
 from pathlib import Path
 
 from src.core.base import BaseLintContext, BaseLintRule
 from src.core.registry import RuleRegistry
 from src.core.types import Violation
-from src.linter_config.ignore import IgnoreDirectiveParser
+from src.linter_config.ignore import get_ignore_parser
 from src.linter_config.loader import LinterConfigLoader
 
 from .language_detector import detect_language
 
+# Default max workers for parallel processing (capped to avoid resource contention)
+DEFAULT_MAX_WORKERS = 8
+
+# Hardcoded exclusions for files/directories that should never be linted
+# These are always skipped regardless of configuration to improve performance
+_HARDCODED_EXCLUDE_EXTENSIONS: frozenset[str] = frozenset(
+    {
+        ".pyc",
+        ".pyo",
+        ".pyd",  # Python bytecode
+        ".so",
+        ".dll",
+        ".dylib",  # Compiled libraries
+        ".class",  # Java bytecode
+        ".o",
+        ".obj",  # Object files
+    }
+)
+_HARDCODED_EXCLUDE_DIRS: frozenset[str] = frozenset(
+    {
+        "__pycache__",
+        "node_modules",
+        ".git",
+        ".svn",
+        ".hg",
+        ".venv",
+        "venv",
+        ".tox",
+        ".eggs",
+        "*.egg-info",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".ruff_cache",
+        "dist",
+        "build",
+        "htmlcov",
+    }
+)
+
+
+def _is_hardcoded_excluded(file_path: Path) -> bool:
+    """Check if file should be excluded based on hardcoded patterns.
+
+    Args:
+        file_path: Path to check
+
+    Returns:
+        True if file should be skipped (compiled file, cache directory, etc.)
+    """
+    # Check file extension
+    if file_path.suffix in _HARDCODED_EXCLUDE_EXTENSIONS:
+        return True
+
+    # Check if any parent directory is in the exclude list
+    for part in file_path.parts:
+        if part in _HARDCODED_EXCLUDE_DIRS:
+            return True
+        # Handle wildcard patterns like *.egg-info
+        if part.endswith(".egg-info"):
+            return True
+
+    return False
+
+
+def _should_include_dir(dirname: str) -> bool:
+    """Check if directory should be traversed (not excluded)."""
+    return dirname not in _HARDCODED_EXCLUDE_DIRS and not dirname.endswith(".egg-info")
+
+
+def _collect_files_from_walk(root: str, filenames: list[str]) -> list[Path]:
+    """Collect non-excluded files from a single directory."""
+    root_path = Path(root)
+    return [root_path / f for f in filenames if Path(f).suffix not in _HARDCODED_EXCLUDE_EXTENSIONS]
+
+
+def _collect_files_fast(dir_path: Path, recursive: bool = True) -> list[Path]:
+    """Collect files, skipping excluded directories entirely.
+
+    Uses os.walk() instead of glob to avoid traversing into excluded
+    directories like .venv, node_modules, __pycache__, etc.
+
+    Args:
+        dir_path: Directory to collect files from.
+        recursive: Whether to traverse subdirectories.
+
+    Returns:
+        List of file paths, excluding hardcoded exclusions.
+    """
+    files: list[Path] = []
+    for root, dirs, filenames in os.walk(dir_path):
+        dirs[:] = [d for d in dirs if _should_include_dir(d)]
+        files.extend(_collect_files_from_walk(root, filenames))
+        if not recursive:
+            break
+    return files
+
+
+def _lint_file_worker(args: tuple[Path, Path, dict]) -> list[dict]:
+    """Worker function for parallel file linting.
+
+    This function runs in a separate process and creates its own Orchestrator
+    instance to lint a single file. Results are returned as dicts to avoid
+    pickling issues with Violation dataclass.
+
+    Args:
+        args: Tuple of (file_path, project_root, config)
+
+    Returns:
+        List of violation dicts (serializable for cross-process transfer)
+    """
+    file_path, project_root, config = args
+    try:
+        # Create isolated orchestrator for this worker process
+        orchestrator = Orchestrator(project_root=project_root, config=config)
+        violations = orchestrator.lint_file(file_path)
+        # Convert to dicts for pickling
+        return [v.to_dict() for v in violations]
+    except Exception:  # nosec B112 - defensive; don't crash on worker errors
+        return []
+
 
 class FileLintContext(BaseLintContext):
     """Concrete implementation of lint context for file analysis."""
@@ -56,6 +184,7 @@ class FileLintContext(BaseLintContext):
         self._path = path
         self._language = lang
         self._content = content
+        self._lines: list[str] | None = None  # Cached line split
         self.metadata = metadata or {}
 
     @property
@@ -81,12 +210,31 @@ class FileLintContext(BaseLintContext):
         """Get programming language of file."""
         return self._language
 
+    @property
+    def file_lines(self) -> list[str]:
+        """Get file content as list of lines (cached).
+
+        Returns:
+            List of lines from file content, empty list if no content.
+        """
+        if self._lines is None:
+            content = self.file_content
+            self._lines = content.split("\n") if content else []
+        return self._lines
+
 
-class Orchestrator:
+class Orchestrator:  # thailint: ignore[srp]
     """Main linter orchestrator coordinating rule execution.
 
     Integrates rule registry, configuration loading, ignore patterns, and language
     detection to provide comprehensive linting of files and directories.
+
+    SRP Exception: Method count (12) exceeds guideline (8) because this is the
+    central orchestration point. Methods are organized into logical groups:
+    - Core linting: lint_file, lint_files, lint_directory
+    - Parallel linting: lint_files_parallel, lint_directory_parallel
+    - Helper methods: _execute_rules, _safe_check_rule, _ensure_rules_discovered, etc.
+    All methods support the single responsibility of coordinating lint operations.
     """
 
     def __init__(self, project_root: Path | None = None, config: dict | None = None):
@@ -99,7 +247,7 @@ class Orchestrator:
         self.project_root = project_root or Path.cwd()
         self.registry = RuleRegistry()
         self.config_loader = LinterConfigLoader()
-        self.ignore_parser = IgnoreDirectiveParser(self.project_root)
+        self.ignore_parser = get_ignore_parser(self.project_root)
 
         # Performance optimization: Defer rule discovery until first file is linted
         # This eliminates ~0.077s overhead for commands that don't need rules (--help, config, etc.)
@@ -125,6 +273,10 @@ class Orchestrator:
         Returns:
             List of violations found in the file.
         """
+        # Fast path: skip compiled files and common excluded directories
+        if _is_hardcoded_excluded(file_path):
+            return []
+
         if self.ignore_parser.is_ignored(file_path):
             return []
 
@@ -197,11 +349,11 @@ class Orchestrator:
             List of all violations found across all files.
         """
         violations = []
-        pattern = "**/*" if recursive else "*"
+        # Use fast file collection that skips excluded directories entirely
+        file_paths = _collect_files_fast(dir_path, recursive)
 
-        for file_path in dir_path.glob(pattern):
-            if file_path.is_file():
-                violations.extend(self.lint_file(file_path))
+        for file_path in file_paths:
+            violations.extend(self.lint_file(file_path))
 
         # Call finalize() on all rules after processing all files
         for rule in self.registry.list_all():
@@ -209,6 +361,83 @@ class Orchestrator:
 
         return violations
 
+    def lint_files_parallel(
+        self, file_paths: list[Path], max_workers: int | None = None
+    ) -> list[Violation]:
+        """Lint multiple files in parallel using process pool.
+
+        Uses ProcessPoolExecutor to distribute file linting across multiple
+        CPU cores. Each worker process creates its own Orchestrator instance.
+
+        Args:
+            file_paths: List of file paths to lint.
+            max_workers: Maximum worker processes. Defaults to min(DEFAULT_MAX_WORKERS, cpu_count).
+
+        Returns:
+            List of violations found across all files.
+        """
+        if not file_paths:
+            return []
+
+        effective_workers = max_workers or min(DEFAULT_MAX_WORKERS, multiprocessing.cpu_count())
+
+        # For small file counts, sequential is faster due to process overhead
+        if len(file_paths) < effective_workers * 2:
+            return self.lint_files(file_paths)
+
+        violations = self._execute_parallel_linting(file_paths, effective_workers)
+        violations.extend(self._finalize_rules())
+        return violations
+
+    def _execute_parallel_linting(
+        self, file_paths: list[Path], max_workers: int
+    ) -> list[Violation]:
+        """Execute parallel linting using process pool."""
+        work_items = [(fp, self.project_root, self.config) for fp in file_paths]
+
+        with ProcessPoolExecutor(max_workers=max_workers) as executor:
+            futures = [executor.submit(_lint_file_worker, item) for item in work_items]
+            return self._collect_parallel_results(futures)
+
+    def _collect_parallel_results(self, futures: list[Future[list[dict]]]) -> list[Violation]:
+        """Collect results from parallel futures."""
+        violations: list[Violation] = []
+        for future in as_completed(futures):
+            violations.extend(self._extract_violations_from_future(future))
+        return violations
+
+    def _extract_violations_from_future(self, future: Future[list[dict]]) -> list[Violation]:
+        """Extract violations from a completed future, handling errors."""
+        try:
+            return [Violation.from_dict(d) for d in future.result()]
+        except Exception:  # nosec B112 - continue on worker errors
+            return []
+
+    def _finalize_rules(self) -> list[Violation]:
+        """Call finalize() on all rules for cross-file analysis."""
+        self._ensure_rules_discovered()
+        violations: list[Violation] = []
+        for rule in self.registry.list_all():
+            violations.extend(rule.finalize())
+        return violations
+
+    def lint_directory_parallel(
+        self, dir_path: Path, recursive: bool = True, max_workers: int | None = None
+    ) -> list[Violation]:
+        """Lint all files in a directory using parallel processing.
+
+        Args:
+            dir_path: Path to directory to lint.
+            recursive: Whether to traverse subdirectories recursively.
+            max_workers: Maximum worker processes. Defaults to min(DEFAULT_MAX_WORKERS, cpu_count).
+
+        Returns:
+            List of all violations found across all files.
+        """
+        # Use fast file collection that skips excluded directories entirely
+        file_paths = _collect_files_fast(dir_path, recursive)
+        return self.lint_files_parallel(file_paths, max_workers=max_workers)
+
     def _ensure_rules_discovered(self) -> None:
         """Ensure rules have been discovered and registered (lazy initialization)."""
         if not self._rules_discovered:

{thailint-0.9.0.dist-info → thailint-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thailint
-Version: 0.9.0
+Version: 0.11.0
 Summary: The AI Linter - Enterprise-grade linting and governance for AI-generated code across multiple languages
 License: MIT
 License-File: LICENSE
@@ -37,8 +37,8 @@ Description-Content-Type: text/markdown
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
-[![Tests](https://img.shields.io/badge/tests-728%2F728%20passing-brightgreen.svg)](tests/)
-[![Coverage](https://img.shields.io/badge/coverage-87%25-brightgreen.svg)](htmlcov/)
+[![Tests](https://img.shields.io/badge/tests-884%2F884%20passing-brightgreen.svg)](tests/)
+[![Coverage](https://img.shields.io/badge/coverage-88%25-brightgreen.svg)](htmlcov/)
 [![Documentation Status](https://readthedocs.org/projects/thai-lint/badge/?version=latest)](https://thai-lint.readthedocs.io/en/latest/?badge=latest)
 [![SARIF 2.1.0](https://img.shields.io/badge/SARIF-2.1.0-orange.svg)](docs/sarif-output.md)
 
@@ -98,6 +98,12 @@ thailint complements your existing linting stack by catching the patterns AI too
   - Configurable thresholds (lines, tokens, occurrences)
   - Language-specific detection (Python, TypeScript, JavaScript)
   - False positive filtering (keyword args, imports)
+- **Collection Pipeline Linting** - Detect for loops with embedded filtering
+  - Based on Martin Fowler's "Replace Loop with Pipeline" refactoring
+  - Detects if/continue patterns that should use generator expressions
+  - Generates refactoring suggestions with generator syntax
+  - Configurable threshold (min_continues)
+  - Python support with AST analysis
 - **Method Property Linting** - Detect methods that should be @property decorators
   - Python AST-based detection
   - get_* prefix detection (Java-style getters)
@@ -741,6 +747,109 @@ Built-in filters automatically exclude common non-duplication patterns:
 
 See [DRY Linter Guide](https://thai-lint.readthedocs.io/en/latest/dry-linter/) for comprehensive documentation, storage modes, and refactoring patterns.
 
+## Collection Pipeline Linter
+
+### Overview
+
+The collection-pipeline linter detects for loops with embedded filtering (if/continue patterns) that should be refactored to use generator expressions or other collection pipelines. Based on Martin Fowler's "Replace Loop with Pipeline" refactoring pattern.
+
+### The Anti-Pattern
+
+```python
+# Anti-pattern: Embedded filtering in loop body
+for file_path in dir_path.glob(pattern):
+    if not file_path.is_file():
+        continue
+    if ignore_parser.is_ignored(file_path):
+        continue
+    violations.extend(lint_file(file_path))
+```
+
+### The Solution
+
+```python
+# Collection pipeline: Filtering separated from processing
+valid_files = (
+    f for f in dir_path.glob(pattern)
+    if f.is_file() and not ignore_parser.is_ignored(f)
+)
+for file_path in valid_files:
+    violations.extend(lint_file(file_path))
+```
+
+### Quick Start
+
+```bash
+# Check current directory
+thailint pipeline .
+
+# Check specific directory
+thailint pipeline src/
+
+# Only flag patterns with 2+ filter conditions
+thailint pipeline --min-continues 2 src/
+
+# JSON output
+thailint pipeline --format json src/
+```
+
+### Configuration
+
+```yaml
+# .thailint.yaml
+collection-pipeline:
+  enabled: true
+  min_continues: 1  # Minimum if/continue patterns to flag
+  ignore:
+    - "tests/**"
+    - "**/legacy/**"
+```
+
+### Example Violation
+
+**Detected Pattern:**
+```python
+def process_files(paths):
+    for path in paths:
+        if not path.is_file():
+            continue
+        analyze(path)
+```
+
+**Violation Message:**
+```
+src/processor.py:3 - For loop over 'paths' has embedded filtering.
+  Consider using a generator expression:
+  for path in (path for path in paths if path.is_file()):
+```
+
+**Refactored Code:**
+```python
+def process_files(paths):
+    valid_paths = (p for p in paths if p.is_file())
+    for path in valid_paths:
+        analyze(path)
+```
+
+### Why This Matters
+
+- **Separation of concerns**: Filtering logic is separate from processing logic
+- **Readability**: Intent is clear at a glance
+- **Testability**: Filtering can be tested independently
+- **Based on**: Martin Fowler's "Replace Loop with Pipeline" refactoring
+
+### Ignoring Violations
+
+```python
+# Line-level ignore
+for item in items:  # thailint: ignore[collection-pipeline]
+    if not item.valid:
+        continue
+    process(item)
+```
+
+See [Collection Pipeline Linter Guide](docs/collection-pipeline-linter.md) for comprehensive documentation and refactoring patterns.
+
 ## Magic Numbers Linter
 
 ### Overview
@@ -1351,6 +1460,9 @@ docker run --rm -v $(pwd):/data washad/thailint:latest nesting /data/src/file1.p
 # Lint specific subdirectory
 docker run --rm -v $(pwd):/data washad/thailint:latest nesting /data/src
 
+# Collection pipeline linter
+docker run --rm -v $(pwd):/data washad/thailint:latest pipeline /data/src
+
 # With custom config
 docker run --rm -v $(pwd):/data \
     washad/thailint:latest nesting --config /data/.thailint.yaml /data
@@ -1414,6 +1526,7 @@ docker run --rm -v /path/to/workspace:/workspace \
 - **[Nesting Depth Linter](https://thai-lint.readthedocs.io/en/latest/nesting-linter/)** - Nesting depth analysis guide
 - **[SRP Linter](https://thai-lint.readthedocs.io/en/latest/srp-linter/)** - Single Responsibility Principle guide
 - **[DRY Linter](https://thai-lint.readthedocs.io/en/latest/dry-linter/)** - Duplicate code detection guide
+- **[Collection Pipeline Linter](https://thai-lint.readthedocs.io/en/latest/collection-pipeline-linter/)** - Loop filtering refactoring guide
 - **[Method Property Linter](https://thai-lint.readthedocs.io/en/latest/method-property-linter/)** - Method-to-property conversion guide
 - **[Stateless Class Linter](https://thai-lint.readthedocs.io/en/latest/stateless-class-linter/)** - Stateless class detection guide
 - **[Pre-commit Hooks](https://thai-lint.readthedocs.io/en/latest/pre-commit-hooks/)** - Automated quality checks