thothctl 0.13.0__tar.gz → 0.13.2__tar.gz

{thothctl-0.13.0 → thothctl-0.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thothctl
-Version: 0.13.0
+Version: 0.13.2
 Summary: A CLI for Developer Control Plane. Accelerate your cloud IaC deployments.
 Project-URL: Homepage, https://github.com/thothforge/thothctl
 Author-email: Alejandro Velez <velez94@pm.me>

{thothctl-0.13.0 → thothctl-0.13.2}/src/thothctl/commands/scan/commands/iac.py

@@ -88,6 +88,8 @@ class RestoredIaCScanCommand(ClickCommand):
                 options=self._parse_options(options) if options else {},
                 tftool=tftool,
                 html_reports_format=html_reports_format,
+                max_workers=max_workers,
+                compact=compact,
             )
             
             # Enhanced data extraction and validation
@@ -640,4 +642,16 @@ cli = RestoredIaCScanCommand.as_click_command(
         default="soft",
         help="Enforcement mode: 'soft' reports violations (exit 0), 'hard' fails the pipeline (exit 1) when any tool finds violations",
     ),
+    click.option(
+        "--max-workers",
+        type=int,
+        default=2,
+        help="Max parallel checkov scans (default: 2, reduce to 1 on low-memory agents)",
+    ),
+    click.option(
+        "--compact",
+        is_flag=True,
+        default=False,
+        help="Use checkov --compact mode to reduce memory usage on constrained CI agents",
+    ),
 )

{thothctl-0.13.0 → thothctl-0.13.2}/src/thothctl/services/inventory/inventory_service.py

@@ -663,6 +663,15 @@ class InventoryService:
             logger.warning(f"Stack path does not exist or is not a directory: {stack_path}")
             return providers
         
+        # Pre-check: skip if stack has module references but hasn't been initialized
+        if self._has_uninitialized_modules(abs_stack_path):
+            logger.warning(
+                f"Skipping provider check for {stack_path}: "
+                "module references found but modules not initialized. "
+                "Run 'tofu init' or 'terraform init' to install modules first."
+            )
+            return providers
+        
         # Determine the command to use based on project type
         # Check if this specific stack has a terragrunt.hcl (not just the global project flag)
         stack_has_terragrunt = (abs_stack_path / "terragrunt.hcl").exists()
@@ -795,6 +804,31 @@ class InventoryService:
         # Fallback: assume hashicorp
         return ('hashicorp', source)
 
+    def _has_uninitialized_modules(self, stack_path: Path) -> bool:
+        """Check if a stack references external modules but hasn't been initialized."""
+        # Terragrunt manages its own init, skip this check
+        if (stack_path / "terragrunt.hcl").exists():
+            return False
+
+        # Look for module blocks in .tf files
+        has_module_refs = False
+        for tf_file in stack_path.glob("*.tf"):
+            try:
+                with open(tf_file, "r") as f:
+                    content = f.read()
+                # Quick check for module blocks with non-local sources
+                if re.search(r'\bmodule\s+"[^"]+"\s*\{', content):
+                    has_module_refs = True
+                    break
+            except (IOError, UnicodeDecodeError):
+                continue
+
+        if not has_module_refs:
+            return False
+
+        # Modules referenced but .terraform/modules not present = not initialized
+        return not (stack_path / ".terraform" / "modules").exists()
+
     def _parse_providers_output(self, output: str, stack_path: str = "") -> List[Provider]:
         """
         Parse the output of terraform/tofu providers command.

{thothctl-0.13.0 → thothctl-0.13.2}/src/thothctl/services/scan/scan_service.py

@@ -58,6 +58,8 @@ class ScanService:
         options: Dict[str, Dict],
         tftool: str = "tofu",
         html_reports_format: Literal["simple", "xunit"] = "simple",
+        max_workers: int = 2,
+        compact: bool = False,
     ) -> Dict[str, Dict]:
         """Execute selected security scans."""
         try:
@@ -110,6 +112,8 @@ class ScanService:
                     reports_dir=reports_dir,
                     options=options.get("checkov", {}),
                     tftool=tftool,
+                    max_workers=max_workers,
+                    compact=compact,
                 )
                 
                 # Process the directory to generate reports
@@ -607,59 +611,76 @@ class ScanService:
         return results
 
     def _recursive_terraform_scan(
-        self, directory: str, reports_dir: str, options: Dict, tftool: str
+        self, directory: str, reports_dir: str, options: Dict, tftool: str,
+        max_workers: int = 2, compact: bool = False,
     ) -> Dict[str, Dict]:
         """
         Recursively scan directories for Terraform files and run Checkov.
+
+        Optimized for memory-constrained CI agents (4 CPU / 8 GB):
+        - Parallel execution with controlled concurrency (max_workers)
+        - GC runs after each completed scan to free memory
+        - compact flag reduces checkov output memory usage
         """
+        import gc
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+
         results = {}
-        try:
-            scanner = self.available_scanners.get("checkov")
-            if not scanner:
-                raise ValueError("Checkov scanner not available")
+        scanner = self.available_scanners.get("checkov")
+        if not scanner:
+            raise ValueError("Checkov scanner not available")
 
-            # Scan current directory if it contains terraform files
-            if (Path(directory) / "main.tf").exists() or (
-                Path(directory) / "tfplan.json"
-            ).exists():
-                print(f"{Fore.MAGENTA} \n 🔎 {directory}")
-                self.logger.debug(f"Found terraform files in {directory}")
+        # Collect all scannable stack directories
+        stacks = self._find_terraform_stacks(directory)
 
-                try:
-                    result = scanner.execute_scan(
-                        directory=str(directory),
-                        reports_dir=str(reports_dir),
-                        options=options,
-                        tftool=tftool,
-                    )
-                    results[directory] = result
-                except Exception as e:
-                    self.logger.error(f"Error scanning {directory}: {e}")
-                    results[directory] = {"status": "FAIL", "error": str(e)}
-
-            # Recursively scan subdirectories
-            subdirs = [
-                d
-                for d in Path(directory).iterdir()
-                if d.is_dir() and not d.name.startswith(".")
-            ]
+        if not stacks:
+            self.logger.info(f"No terraform stacks found in {directory}")
+            return results
 
-            for subdir in subdirs:
-                subdir_results = self._recursive_terraform_scan(
-                    directory=str(subdir),
-                    reports_dir=reports_dir,
-                    options=options,
+        self.logger.info(f"Found {len(stacks)} terraform stacks to scan (workers={max_workers})")
+        print(f"{Fore.MAGENTA}\n \U0001f50e Found {len(stacks)} stacks to scan (parallel={max_workers})")
+
+        scan_options = dict(options)
+        if compact:
+            scan_options["compact"] = True
+
+        def _scan_stack(stack_dir: str) -> Tuple[str, Dict]:
+            print(f"{Fore.MAGENTA} \n \U0001f50e {stack_dir}")
+            try:
+                result = scanner.execute_scan(
+                    directory=str(stack_dir),
+                    reports_dir=str(reports_dir),
+                    options=scan_options,
                     tftool=tftool,
                 )
-                results.update(subdir_results)
+                return (stack_dir, result)
+            except Exception as e:
+                self.logger.error(f"Error scanning {stack_dir}: {e}")
+                return (stack_dir, {"status": "FAIL", "error": str(e)})
 
-        except Exception as e:
-            self.logger.error(f"Error during recursive scan: {e}")
-            results[directory] = {"status": "FAIL", "error": str(e)}
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {executor.submit(_scan_stack, s): s for s in stacks}
+            for future in as_completed(futures):
+                stack_dir, result = future.result()
+                results[stack_dir] = result
+                gc.collect()
 
         return results
 
-
+    def _find_terraform_stacks(self, directory: str) -> List[str]:
+        """Find all directories containing main.tf or tfplan.json."""
+        stacks = []
+        root = Path(directory)
+        if (root / "main.tf").exists() or (root / "tfplan.json").exists():
+            stacks.append(str(root))
+        for d in sorted(root.rglob("*")):
+            if not d.is_dir():
+                continue
+            if any(part.startswith(".") for part in d.relative_to(root).parts):
+                continue
+            if (d / "main.tf").exists() or (d / "tfplan.json").exists():
+                stacks.append(str(d))
+        return stacks
 def recursive_scan(
     directory: str,
     tool: str,

{thothctl-0.13.0 → thothctl-0.13.2}/src/thothctl/services/scan/scanners/checkov.py

@@ -262,11 +262,11 @@ class CheckovScanner(ScannerPort):
         Returns:
             List of command components
         """
-        # Remove the timeout command
-        cmd = ["checkov"]  # No timeout
+        cmd = ["checkov"]
 
         if options:
-            # Add any additional arguments
+            if options.get("compact"):
+                cmd.append("--compact")
             if "additional_args" in options:
                 if isinstance(options["additional_args"], list):
                     cmd.extend(options["additional_args"])

thothctl-0.13.2/src/thothctl/version.py

@@ -0,0 +1,2 @@
+"""Version module."""
+__version__ = "0.13.2"

thothctl-0.13.2/tests/test_parallel_scan.py

@@ -0,0 +1,193 @@
+"""Unit tests for parallel checkov scan and memory optimization."""
+
+import gc
+import tempfile
+from pathlib import Path
+from unittest.mock import Mock, patch, MagicMock
+
+import pytest
+
+from thothctl.services.scan.scan_service import ScanService
+from thothctl.services.scan.scanners.checkov import CheckovScanner
+
+
+class TestFindTerraformStacks:
+    """Test _find_terraform_stacks discovery."""
+
+    def test_finds_root_stack(self, tmp_path):
+        svc = ScanService()
+        (tmp_path / "main.tf").write_text("")
+        stacks = svc._find_terraform_stacks(str(tmp_path))
+        assert str(tmp_path) in stacks
+
+    def test_finds_nested_stacks(self, tmp_path):
+        for name in ["stack-a", "stack-b", "modules/vpc"]:
+            d = tmp_path / name
+            d.mkdir(parents=True)
+            (d / "main.tf").write_text("")
+        stacks = svc = ScanService()
+        stacks = svc._find_terraform_stacks(str(tmp_path))
+        assert len(stacks) == 3
+
+    def test_finds_tfplan_json_stacks(self, tmp_path):
+        d = tmp_path / "plan-stack"
+        d.mkdir()
+        (d / "tfplan.json").write_text("{}")
+        svc = ScanService()
+        stacks = svc._find_terraform_stacks(str(tmp_path))
+        assert len(stacks) == 1
+        assert "plan-stack" in stacks[0]
+
+    def test_skips_hidden_dirs(self, tmp_path):
+        hidden = tmp_path / ".terraform" / "modules" / "inner"
+        hidden.mkdir(parents=True)
+        (hidden / "main.tf").write_text("")
+        (tmp_path / "main.tf").write_text("")
+        svc = ScanService()
+        stacks = svc._find_terraform_stacks(str(tmp_path))
+        assert len(stacks) == 1
+
+    def test_empty_dir_returns_empty(self, tmp_path):
+        svc = ScanService()
+        assert svc._find_terraform_stacks(str(tmp_path)) == []
+
+
+class TestRecursiveTerraformScanParallel:
+    """Test _recursive_terraform_scan with parallel execution."""
+
+    def _make_stacks(self, tmp_path, count):
+        for i in range(count):
+            d = tmp_path / f"stack-{i}"
+            d.mkdir()
+            (d / "main.tf").write_text(f'resource "null_resource" "r{i}" {{}}')
+        return tmp_path
+
+    @patch.object(ScanService, '_find_terraform_stacks')
+    def test_parallel_scans_all_stacks(self, mock_find):
+        """All discovered stacks are scanned and results collected."""
+        stacks = ["/a/stack-0", "/a/stack-1", "/a/stack-2"]
+        mock_find.return_value = stacks
+
+        svc = ScanService()
+        mock_scanner = Mock()
+        mock_scanner.execute_scan.return_value = {"status": "COMPLETE"}
+        svc.available_scanners["checkov"] = Mock()
+        svc.available_scanners["checkov"].execute_scan = mock_scanner.execute_scan
+
+        results = svc._recursive_terraform_scan(
+            directory="/a", reports_dir="/r", options={}, tftool="tofu",
+            max_workers=2,
+        )
+
+        assert len(results) == 3
+        assert mock_scanner.execute_scan.call_count == 3
+        for s in stacks:
+            assert s in results
+            assert results[s]["status"] == "COMPLETE"
+
+    @patch.object(ScanService, '_find_terraform_stacks')
+    def test_single_worker_sequential(self, mock_find):
+        """max_workers=1 still scans all stacks (sequential fallback)."""
+        mock_find.return_value = ["/a/s1", "/a/s2"]
+
+        svc = ScanService()
+        mock_scanner = Mock()
+        mock_scanner.execute_scan.return_value = {"status": "COMPLETE"}
+        svc.available_scanners["checkov"] = Mock()
+        svc.available_scanners["checkov"].execute_scan = mock_scanner.execute_scan
+
+        results = svc._recursive_terraform_scan(
+            directory="/a", reports_dir="/r", options={}, tftool="tofu",
+            max_workers=1,
+        )
+        assert len(results) == 2
+
+    @patch.object(ScanService, '_find_terraform_stacks')
+    def test_failed_stack_does_not_block_others(self, mock_find):
+        """A failing stack returns FAIL but other stacks still complete."""
+        mock_find.return_value = ["/a/good", "/a/bad"]
+
+        svc = ScanService()
+        mock_scanner = Mock()
+
+        def side_effect(directory, **kw):
+            if "bad" in directory:
+                raise RuntimeError("boom")
+            return {"status": "COMPLETE"}
+
+        mock_scanner.execute_scan.side_effect = side_effect
+        svc.available_scanners["checkov"] = Mock()
+        svc.available_scanners["checkov"].execute_scan = mock_scanner.execute_scan
+
+        results = svc._recursive_terraform_scan(
+            directory="/a", reports_dir="/r", options={}, tftool="tofu",
+        )
+        assert results["/a/good"]["status"] == "COMPLETE"
+        assert results["/a/bad"]["status"] == "FAIL"
+        assert "boom" in results["/a/bad"]["error"]
+
+    @patch.object(ScanService, '_find_terraform_stacks')
+    def test_no_stacks_returns_empty(self, mock_find):
+        mock_find.return_value = []
+        svc = ScanService()
+        results = svc._recursive_terraform_scan(
+            directory="/a", reports_dir="/r", options={}, tftool="tofu",
+        )
+        assert results == {}
+
+    @patch.object(ScanService, '_find_terraform_stacks')
+    def test_compact_flag_passed_to_scanner(self, mock_find):
+        """compact=True adds compact key to options passed to scanner."""
+        mock_find.return_value = ["/a/s1"]
+
+        svc = ScanService()
+        mock_scanner = Mock()
+        mock_scanner.execute_scan.return_value = {"status": "COMPLETE"}
+        svc.available_scanners["checkov"] = Mock()
+        svc.available_scanners["checkov"].execute_scan = mock_scanner.execute_scan
+
+        svc._recursive_terraform_scan(
+            directory="/a", reports_dir="/r", options={}, tftool="tofu",
+            compact=True,
+        )
+
+        call_opts = mock_scanner.execute_scan.call_args
+        assert call_opts.kwargs.get("options", {}).get("compact") is True or \
+               call_opts[1].get("options", {}).get("compact") is True
+
+
+class TestCheckovBuildCommand:
+    """Test checkov _build_command handles compact flag."""
+
+    def test_compact_adds_flag(self):
+        scanner = CheckovScanner()
+        cmd = scanner._build_command("/dir", {"compact": True})
+        assert "--compact" in cmd
+
+    def test_no_compact_by_default(self):
+        scanner = CheckovScanner()
+        cmd = scanner._build_command("/dir", {})
+        assert "--compact" not in cmd
+
+    def test_compact_false_no_flag(self):
+        scanner = CheckovScanner()
+        cmd = scanner._build_command("/dir", {"compact": False})
+        assert "--compact" not in cmd
+
+    def test_additional_args_string(self):
+        scanner = CheckovScanner()
+        cmd = scanner._build_command("/dir", {"additional_args": "--skip-check CKV_AWS_1"})
+        assert "--skip-check" in cmd
+        assert "CKV_AWS_1" in cmd
+
+    def test_additional_args_list(self):
+        scanner = CheckovScanner()
+        cmd = scanner._build_command("/dir", {"additional_args": ["--framework", "terraform"]})
+        assert "--framework" in cmd
+        assert "terraform" in cmd
+
+    def test_compact_with_additional_args(self):
+        scanner = CheckovScanner()
+        cmd = scanner._build_command("/dir", {"compact": True, "additional_args": "--quiet"})
+        assert "--compact" in cmd
+        assert "--quiet" in cmd

thothctl-0.13.0/src/thothctl/version.py

@@ -1,2 +0,0 @@
-"""Version module."""
-__version__ = "0.13.0"