thothctl 0.20.0__tar.gz → 0.20.2__tar.gz

{thothctl-0.20.0 → thothctl-0.20.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thothctl
-Version: 0.20.0
+Version: 0.20.2
 Summary: A CLI for Developer Control Plane. Accelerate your cloud IaC deployments.
 Project-URL: SourceCode, https://github.com/thothforge/thothctl
 Project-URL: HomePage, https://github.com/thothforge/thothctl

{thothctl-0.20.0 → thothctl-0.20.2}/src/thothctl/services/document/architecture_renderer.py

@@ -179,13 +179,17 @@ def render_architecture_diagram(topology, output_path: str, fmt: str = "png") ->
                             if not cls:
                                 continue
 
-                            # Build label
+                            # Build label — use friendly AWS service name
                             short_name = node.name.split("[")[0]
+                            # Use label (e.g. "Aurora", "VPC") unless name is more descriptive
+                            display_name = node.label
+                            if short_name and short_name not in ("this", "default") and short_name != node.label.lower().replace(" ", "_"):
+                                display_name = f"{node.label}\n({short_name})"
                             action_badge = {
                                 "create": " 🆕", "update": " ✏️",
                                 "delete": " 🗑️", "replace": " ♻️"
                             }.get(node.action.value, "")
-                            label = f"{short_name}{action_badge}"
+                            label = f"{display_name}{action_badge}"
 
                             obj = cls(label)
                             node_objects[node.address] = obj

{thothctl-0.20.0 → thothctl-0.20.2}/src/thothctl/services/scan/scanners/opa.py

@@ -5,6 +5,10 @@ Supports two modes:
 - opa: Plan-based evaluation using `opa exec` against tfplan.json files
 
 Both modes use the same Rego policy language and can share policies.
+
+Data files (params.yaml / params.json) in the policy directory are automatically
+loaded into OPA's data namespace, enabling externalized policy parameters.
+YAML files are converted to JSON at scan time for OPA compatibility.
 """
 
 import json
@@ -53,6 +57,58 @@ class OPAScanner(ScannerPort):
 
     # ── Conftest mode ──────────────────────────────────────────────────
 
+    def _prepare_data_files(self, policy_dir: str) -> None:
+        """Convert YAML data files to JSON in the policy directory.
+
+        OPA/Conftest natively load .json files into the data namespace but not YAML.
+        This method finds *.yaml/*.yml data files (excluding .rego test data) in the
+        policy directory and generates corresponding .json files so that Rego policies
+        can reference them via data.<filename>.<key>.
+
+        This enables teams to manage policy parameters in human-readable YAML while
+        keeping OPA compatibility transparent.
+        """
+        policy_path = Path(policy_dir)
+        if not policy_path.is_dir():
+            return
+
+        yaml_files = list(policy_path.glob("*.yaml")) + list(policy_path.glob("*.yml"))
+
+        # Filter: only convert files that look like data/param files
+        # (skip files that are test fixtures, conftest config, etc.)
+        skip_prefixes = ("conftest", "opa", ".")
+        data_files = [
+            f for f in yaml_files
+            if not f.name.startswith(skip_prefixes)
+        ]
+
+        if not data_files:
+            return
+
+        try:
+            import yaml
+        except ImportError:
+            self.logger.warning(
+                "PyYAML not installed — cannot convert YAML data files to JSON. "
+                "Install with: pip install pyyaml"
+            )
+            return
+
+        for yaml_file in data_files:
+            json_file = yaml_file.with_suffix(".json")
+            # Only regenerate if YAML is newer than JSON (or JSON doesn't exist)
+            if json_file.exists() and json_file.stat().st_mtime >= yaml_file.stat().st_mtime:
+                continue
+            try:
+                with open(yaml_file, "r", encoding="utf-8") as yf:
+                    data = yaml.safe_load(yf)
+                with open(json_file, "w", encoding="utf-8") as jf:
+                    json.dump(data, jf, indent=2, ensure_ascii=False)
+                self.logger.info(f"Converted {yaml_file.name} → {json_file.name}")
+                self.ui.show_info(f"📄 Data file: {yaml_file.name} → {json_file.name}")
+            except Exception as e:
+                self.logger.warning(f"Failed to convert {yaml_file}: {e}")
+
     def _scan_with_conftest(
         self, directory: str, reports_dir: str, policy_dir: str, options: Dict
     ) -> Dict[str, str]:
@@ -74,6 +130,9 @@ class OPAScanner(ScannerPort):
             )
             return {"status": "SKIPPED", "error": "No policy directory found"}
 
+        # Convert YAML data files to JSON for OPA data namespace
+        self._prepare_data_files(abs_policy)
+
         # Collect scannable files
         scan_files = self._find_scannable_files(abs_dir)
         if not scan_files:
@@ -196,6 +255,9 @@ class OPAScanner(ScannerPort):
             )
             return {"status": "SKIPPED", "error": "No policy directory found"}
 
+        # Convert YAML data files to JSON for OPA data namespace
+        self._prepare_data_files(abs_policy)
+
         # Find tfplan.json files
         plan_files = self._find_plan_files(abs_dir)
         if not plan_files:

thothctl-0.20.2/src/thothctl/version.py

@@ -0,0 +1,2 @@
+"""Version module."""
+__version__ = "0.20.2"

thothctl-0.20.2/tests/test_opa_data_files.py

@@ -0,0 +1,169 @@
+"""Unit tests for OPA scanner YAML→JSON data file conversion."""
+import json
+import os
+import tempfile
+from pathlib import Path
+
+import pytest
+import yaml
+
+from thothctl.services.scan.scanners.opa import OPAScanner
+
+
+class TestPrepareDataFiles:
+    """Test the _prepare_data_files YAML→JSON conversion."""
+
+    def setup_method(self):
+        self.scanner = OPAScanner()
+
+    def test_converts_yaml_to_json(self, tmp_path):
+        """params.yaml should be converted to params.json."""
+        params = {
+            "tags": {
+                "required_tags": ["Project", "Environment"],
+                "allowed_values": {
+                    "Project": ["sbsb001-bank4us", "sbsb002-kora"],
+                    "Environment": ["dev", "qa", "prod"],
+                },
+            },
+            "naming": {"pattern": "^[a-z0-9-]+$"},
+        }
+        yaml_file = tmp_path / "params.yaml"
+        yaml_file.write_text(yaml.dump(params))
+
+        self.scanner._prepare_data_files(str(tmp_path))
+
+        json_file = tmp_path / "params.json"
+        assert json_file.exists(), "params.json should be created"
+        loaded = json.loads(json_file.read_text())
+        assert loaded == params
+
+    def test_skips_conftest_yaml(self, tmp_path):
+        """conftest.yaml should NOT be converted."""
+        conftest_yaml = tmp_path / "conftest.yaml"
+        conftest_yaml.write_text("parser: yaml\n")
+
+        self.scanner._prepare_data_files(str(tmp_path))
+
+        assert not (tmp_path / "conftest.json").exists()
+
+    def test_skips_opa_yaml(self, tmp_path):
+        """opa.yaml should NOT be converted."""
+        opa_yaml = tmp_path / "opa.yaml"
+        opa_yaml.write_text("services:\n  - name: default\n")
+
+        self.scanner._prepare_data_files(str(tmp_path))
+
+        assert not (tmp_path / "opa.json").exists()
+
+    def test_skips_hidden_files(self, tmp_path):
+        """Hidden .yaml files should NOT be converted."""
+        hidden = tmp_path / ".secrets.yaml"
+        hidden.write_text("api_key: abc123\n")
+
+        self.scanner._prepare_data_files(str(tmp_path))
+
+        assert not (tmp_path / ".secrets.json").exists()
+
+    def test_multiple_yaml_files(self, tmp_path):
+        """Multiple YAML data files should all be converted."""
+        (tmp_path / "params.yaml").write_text(yaml.dump({"key": "val1"}))
+        (tmp_path / "overrides.yaml").write_text(yaml.dump({"key": "val2"}))
+
+        self.scanner._prepare_data_files(str(tmp_path))
+
+        assert (tmp_path / "params.json").exists()
+        assert (tmp_path / "overrides.json").exists()
+        assert json.loads((tmp_path / "params.json").read_text()) == {"key": "val1"}
+        assert json.loads((tmp_path / "overrides.json").read_text()) == {"key": "val2"}
+
+    def test_caching_skips_if_json_is_newer(self, tmp_path):
+        """Should NOT regenerate JSON if it's already up-to-date."""
+        params = {"tags": {"required_tags": ["Project"]}}
+        yaml_file = tmp_path / "params.yaml"
+        yaml_file.write_text(yaml.dump(params))
+
+        json_file = tmp_path / "params.json"
+        json_file.write_text(json.dumps({"old": "data"}))
+
+        # Make JSON newer than YAML
+        os.utime(json_file, (9999999999, 9999999999))
+
+        self.scanner._prepare_data_files(str(tmp_path))
+
+        # JSON should keep old content since it's newer
+        loaded = json.loads(json_file.read_text())
+        assert loaded == {"old": "data"}
+
+    def test_regenerates_if_yaml_is_newer(self, tmp_path):
+        """Should regenerate JSON if YAML is newer."""
+        params = {"tags": {"required_tags": ["Project", "Product"]}}
+        yaml_file = tmp_path / "params.yaml"
+        yaml_file.write_text(yaml.dump(params))
+
+        json_file = tmp_path / "params.json"
+        json_file.write_text(json.dumps({"stale": "data"}))
+
+        # Make YAML newer than JSON
+        os.utime(json_file, (1000000000, 1000000000))
+        os.utime(yaml_file, (9999999999, 9999999999))
+
+        self.scanner._prepare_data_files(str(tmp_path))
+
+        loaded = json.loads(json_file.read_text())
+        assert loaded == params
+
+    def test_handles_empty_directory(self, tmp_path):
+        """Should do nothing for empty directories."""
+        self.scanner._prepare_data_files(str(tmp_path))
+        # No exception, no files created
+        assert list(tmp_path.iterdir()) == []
+
+    def test_handles_nonexistent_directory(self):
+        """Should do nothing for non-existent directories."""
+        self.scanner._prepare_data_files("/nonexistent/path")
+        # Should not raise
+
+    def test_yml_extension_also_converted(self, tmp_path):
+        """*.yml files should also be converted."""
+        (tmp_path / "sizing.yml").write_text(
+            yaml.dump({"blocked_types": ["m5.4xlarge"]})
+        )
+
+        self.scanner._prepare_data_files(str(tmp_path))
+
+        json_file = tmp_path / "sizing.json"
+        assert json_file.exists()
+        assert json.loads(json_file.read_text()) == {"blocked_types": ["m5.4xlarge"]}
+
+
+class TestOPAScannerIntegration:
+    """Integration tests for OPA scanner with data files and Rego policies."""
+
+    def test_conftest_loads_params_from_json(self, tmp_path):
+        """Conftest should correctly resolve data.params references from params.json."""
+        # Create a minimal policy that uses data.params
+        policy_dir = tmp_path / "policy"
+        policy_dir.mkdir()
+
+        rego = """package tagging
+required_tags := data.params.tags.required_tags
+deny contains msg if {
+    resource := input.resource_changes[_]
+    required := required_tags[_]
+    not resource.change.after.tags_all[required]
+    msg := sprintf("Missing tag: %s", [required])
+}
+"""
+        (policy_dir / "tagging.rego").write_text(rego)
+        (policy_dir / "params.yaml").write_text(
+            yaml.dump({"tags": {"required_tags": ["Project", "Environment"]}})
+        )
+
+        # Prepare data files (convert YAML→JSON)
+        scanner = OPAScanner()
+        scanner._prepare_data_files(str(policy_dir))
+
+        # Verify JSON was generated correctly
+        params_json = json.loads((policy_dir / "params.json").read_text())
+        assert params_json["tags"]["required_tags"] == ["Project", "Environment"]

thothctl-0.20.0/src/thothctl/version.py

@@ -1,2 +0,0 @@
-"""Version module."""
-__version__ = "0.20.0"