kapitan 0.36.3.dev1__tar.gz → 0.36.3.dev2__tar.gz

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kapitan
-Version: 0.36.3.dev1
+Version: 0.36.3.dev2
 Summary: Generic templated configuration management for Kubernetes, Terraform and other things
 Project-URL: Homepage, https://kapitan.dev
 Project-URL: Repository, https://github.com/kapicorp/kapitan

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/kapitan/cached.py

@@ -38,6 +38,12 @@ inv_sources: set[str] = set()
 # Kadet caches
 kapitan_input_kadet = None
 
+# Helm cache: rendered manifests keyed on (chart_dir content, helm_values,
+# helm_params, helm_path). Lets HelmChart() callers skip the helm subprocess
+# even when their kadet cache misses (e.g. when an unrelated inventory key
+# changed but helm-relevant inputs did not).
+kapitan_input_helm = None
+
 # Shared cache metrics for the compile pool, keyed by input_type_name.
 # Populated by compile_targets() only when caching is enabled, then propagated
 # to workers via the pool initializer so every InputCache(input_type_name=N)

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/kapitan/dependency_manager/base.py

@@ -353,6 +353,62 @@ def fetch_helm_archive(helm_path, repo, chart_name, version, save_path):
     )
 
 
+def helm_dependencies_digest(target_name: str) -> bytes | None:
+    """Stable digest over the on-disk helm charts declared as dependencies of ``target_name``.
+
+    Returns ``None`` when the target declares no helm dependencies — callers
+    should skip mixing into their cache key in that case, so non-helm users
+    keep their existing keys (mirrors :func:`kapitan.topics.consumed_topics_digest`).
+
+    Each helm dep's ``output_path`` is walked recursively via the same
+    ``walk_and_hash`` helper kadet uses for its component dir; the digest
+    therefore captures every chart-file change including subchart content
+    and ``Chart.yaml`` version bumps. ``cached.inv`` holds the model-dumped
+    inventory captured before ``fetch_dependencies`` normalized
+    ``output_path`` on the live pydantic objects, so the path is resolved
+    here the same way ``fetch_dependencies`` does.
+    """
+    # Local imports to avoid cycles: kapitan.inputs.cache → cache helpers,
+    # and kapitan.cached → runtime state. dependency_manager is imported
+    # eagerly at startup; deferring keeps that path lean.
+    from kapitan import cached
+    from kapitan.inputs.cache import InputCache, walk_and_hash
+    from kapitan.utils import normalise_join_path
+
+    try:
+        target_params = cached.inv[target_name]["parameters"]["kapitan"]
+    except (KeyError, TypeError):
+        return None
+
+    deps = target_params.get("dependencies") or []
+    helm_deps = [d for d in deps if d.get("type") == KapitanDependencyTypes.HELM]
+    if not helm_deps:
+        return None
+
+    helm_deps.sort(
+        key=lambda d: (
+            d.get("output_path", ""),
+            d.get("chart_name", ""),
+            d.get("version") or "",
+        )
+    )
+
+    base = getattr(cached.args, "output_path", ".") or "."
+    input_cache = getattr(cached, "kapitan_input_kadet", None)
+
+    h = InputCache.hash_object()
+    for dep in helm_deps:
+        rel_output_path = dep.get("output_path", "")
+        out_path = normalise_join_path(base, rel_output_path)
+        # Mix the declared path itself so a rename of the dep (with identical
+        # file content) still moves the digest.
+        h.update(rel_output_path.encode("utf-8"))
+        h.update(b"\x00")
+        walk_and_hash(Path(out_path), input_cache, h)
+        h.update(b"\x00")
+    return h.digest()
+
+
 def exists_in_cache(item_path):
     return os.path.exists(item_path)
 

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/kapitan/helm_cli.py

@@ -1,3 +1,4 @@
+import functools
 import logging
 import os
 import subprocess
@@ -7,6 +8,37 @@ from subprocess import DEVNULL, PIPE
 logger = logging.getLogger(__name__)
 
 
+@functools.cache
+def helm_version(helm_path: str | None = None) -> str:
+    """Return ``helm version --short`` output for cache-key purposes.
+
+    Resolves the binary the same way :func:`helm_cli` does (explicit
+    ``helm_path``, then ``KAPITAN_HELM_PATH`` env var, then ``"helm"`` via
+    PATH). Memoized per resolved binary so we incur at most one
+    ``helm version`` subprocess per distinct binary per worker process.
+
+    On failure (binary missing, timeout, non-zero exit), returns a sentinel
+    string derived from the path so callers that include it in a cache key
+    still differentiate user-supplied paths. The render itself will fail
+    loudly downstream if the binary is truly broken.
+    """
+    binary = helm_path or os.getenv("KAPITAN_HELM_PATH", "helm")
+    try:
+        res = subprocess.run(
+            [binary, "version", "--short"],
+            stdout=PIPE,
+            stderr=DEVNULL,
+            check=False,
+            timeout=10,
+        )
+        if res.returncode == 0:
+            return res.stdout.decode().strip()
+        logger.debug("helm version probe for %s returned %d", binary, res.returncode)
+    except (FileNotFoundError, subprocess.TimeoutExpired) as e:
+        logger.debug("helm version probe failed for %s: %s", binary, e)
+    return f"unresolved:{binary}"
+
+
 def helm_cli(helm_path, args, stdout=None, verbose=False, timeout=None):
     # if helm is not specified, try to get it from env var, and defaults to looking up helm in the path.
     if not helm_path:

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/kapitan/inputs/__init__.py

@@ -14,7 +14,7 @@ from .base import InputType
 # compile_targets() pre-allocates one CacheMetrics per entry so all workers
 # share counters across the multiprocessing pool. Add an entry here when a
 # new input type adopts InputCache.
-CACHEABLE_INPUT_TYPES: tuple[str, ...] = ("kadet",)
+CACHEABLE_INPUT_TYPES: tuple[str, ...] = ("kadet", "helm")
 
 
 @functools.cache  # Use lru_cache for caching

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/kapitan/inputs/cache.py

@@ -163,3 +163,47 @@ class InputCache:
                 pass
             else:
                 raise
+
+
+def walk_and_hash(path: Path, input_cache: InputCache | None, path_hash):
+    """
+    Recursively walk a path and update a hash object with the contents of all files.
+    This implementation is deterministic.
+
+    ``input_cache`` may be ``None`` or a falsy ``InputCache``; when truthy its
+    ``kv_cache`` dict is used to memoize per-file digests across calls so the
+    same file isn't re-read for every walk within a worker process.
+    """
+    if not path.exists() or str(path).endswith("__pycache__"):
+        return
+
+    if path.is_file():
+        if cached_hash_digest := get_path_hash_from_input_kv(path, input_cache):
+            path_hash.update(cached_hash_digest)
+            logger.debug(
+                "KV Memory hit for path: %s, digest: %s", path, path_hash.hexdigest()
+            )
+            return
+
+        with open(path, "rb") as fp:
+            file_hash = InputCache.hash_file_digest(fp)
+            digest = file_hash.digest()
+            set_path_hash_input_kv(path, digest, input_cache)
+            path_hash.update(digest)
+
+    elif path.is_dir():
+        for item in sorted(path.iterdir(), key=lambda p: p.name):
+            walk_and_hash(item, input_cache, path_hash)
+
+
+def get_path_hash_from_input_kv(path: Path, input_cache: InputCache | None):
+    try:
+        if input_cache:
+            return input_cache.kv_cache[str(path)]
+    except KeyError:
+        return None
+
+
+def set_path_hash_input_kv(path: Path, h_file, input_cache: InputCache | None):
+    if input_cache:
+        input_cache.kv_cache[str(path)] = h_file

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/kapitan/inputs/helm.py

@@ -5,15 +5,19 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import json
 import logging
 import os
 import tempfile
+from pathlib import Path
 
 import yaml
 
+from kapitan import cached
 from kapitan.errors import HelmTemplateError
-from kapitan.helm_cli import helm_cli
+from kapitan.helm_cli import helm_cli, helm_version
 from kapitan.inputs.base import InputType
+from kapitan.inputs.cache import InputCache, walk_and_hash
 from kapitan.inputs.kadet import BaseModel, BaseObj
 from kapitan.inventory.model.input_types import KapitanInputTypeHelmConfig
 
@@ -240,6 +244,45 @@ def write_helm_values_file(helm_values: dict):
     return helm_values_file
 
 
+def _helmchart_cache_obj():
+    """Return the helm InputCache when caching is enabled, else ``None``.
+
+    Lazy-initialised per worker process. Shares the multiprocessing
+    CacheMetrics counter pre-allocated by ``compile_targets`` so hits/misses
+    show up in the end-of-compile summary alongside kadet's.
+    """
+    if not getattr(cached.args, "cache", False):
+        return None
+    if cached.kapitan_input_helm is None:
+        metrics = (cached.input_cache_metrics or {}).get("helm")
+        cached.kapitan_input_helm = InputCache("helm", metrics=metrics)
+    return cached.kapitan_input_helm
+
+
+def _helmchart_cache_key(chart_dir, helm_values, helm_params, helm_path, cache_obj):
+    """Hash everything that determines ``helm template`` output for HelmChart().
+
+    The chart_dir contents are walked recursively, so file edits to templates,
+    Chart.yaml, values.yaml or subcharts all move the key. ``helm_values`` and
+    ``helm_params`` are JSON-serialised with sorted keys for determinism.
+    """
+    h = InputCache.hash_object()
+    h.update(b"chart\x00")
+    walk_and_hash(Path(chart_dir), cache_obj, h)
+    h.update(b"\x00values\x00")
+    h.update(json.dumps(helm_values, sort_keys=True, default=str).encode("utf-8"))
+    h.update(b"\x00params\x00")
+    h.update(json.dumps(helm_params, sort_keys=True, default=str).encode("utf-8"))
+    h.update(b"\x00helm_path\x00")
+    h.update(str(helm_path or "").encode("utf-8"))
+    # Mix in the actual binary's version so a helm upgrade (which can change
+    # CRD handling, manifest field order, schema validation) busts the cache
+    # even when ``helm_path`` itself is unchanged or None.
+    h.update(b"\x00helm_version\x00")
+    h.update(helm_version(helm_path).encode("utf-8"))
+    return h.hexdigest()
+
+
 class HelmChart(BaseModel):
     """
     Represents a Helm chart. Renders the chart and stores the rendered objects in self.root.
@@ -264,6 +307,23 @@ class HelmChart(BaseModel):
             ] = BaseObj.from_dict(obj)
 
     def load_chart(self):
+        # Cache layer is independent of kadet's: an unrelated inventory key
+        # change invalidates kadet but leaves this hit, so the helm subprocess
+        # is only re-run when (chart_dir + helm_values + helm_params +
+        # helm_path) actually changed.
+        cache_obj = _helmchart_cache_obj()
+        cache_key = None
+        if cache_obj:
+            cache_key = _helmchart_cache_key(
+                self.chart_dir,
+                self.helm_values,
+                self.helm_params,
+                self.helm_path,
+                cache_obj,
+            )
+            if (cached_docs := cache_obj.get(cache_key)) is not None:
+                return cached_docs
+
         helm_values_file = None
         if self.helm_values != {}:
             helm_values_file = write_helm_values_file(self.helm_values)
@@ -278,4 +338,7 @@ class HelmChart(BaseModel):
         if error_message:
             raise HelmTemplateError(error_message)
 
-        return [doc for doc in yaml.safe_load_all(output) if doc is not None]
+        docs = [doc for doc in yaml.safe_load_all(output) if doc is not None]
+        if cache_obj is not None:
+            cache_obj.set(cache_key, docs)
+        return docs

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/kapitan/inputs/kadet.py

@@ -20,9 +20,10 @@ from kadet import BaseModel, BaseObj, Dict
 
 from kapitan import cached
 from kapitan.defaults import KADET_COMPONENT_MODULE_PREFIX
+from kapitan.dependency_manager.base import helm_dependencies_digest
 from kapitan.errors import CompileError
 from kapitan.inputs.base import InputType
-from kapitan.inputs.cache import InputCache
+from kapitan.inputs.cache import InputCache, walk_and_hash
 from kapitan.inventory.model.input_types import KapitanInputTypeKadetConfig
 from kapitan.topics import consumed_topics_digest, current_target
 
@@ -153,6 +154,16 @@ class Kadet(InputType):
                 if topics_digest := consumed_topics_digest(target_name):
                     extra_inputs.append(topics_digest)
 
+                # Mix in a digest of every helm chart this target declared
+                # under ``parameters.kapitan.dependencies``. Without this,
+                # editing files under a fetched chart_dir (which lives
+                # outside the kadet component path that walk_and_hash sees)
+                # would not invalidate the kadet cache, and HelmChart()
+                # callers would get stale renders. ``None`` for targets
+                # with no helm deps so non-helm users keep current keys.
+                if helm_deps_digest := helm_dependencies_digest(target_name):
+                    extra_inputs.append(helm_deps_digest)
+
                 inputs_hash = self.inputs_hash(
                     inventory_digest(current_target.get()),
                     target_name,
@@ -303,48 +314,6 @@ class Kadet(InputType):
         return False
 
 
-def walk_and_hash(path: Path, input_cache: InputCache, path_hash):
-    """
-    Recursively walk a path and update a hash object with the contents of all files.
-    This implementation is deterministic.
-    """
-    if not path.exists() or str(path).endswith("__pycache__"):
-        return
-
-    if path.is_file():
-        if cached_hash_digest := get_path_hash_from_input_kv(path, input_cache):
-            path_hash.update(cached_hash_digest)
-            logger.debug(
-                "KV Memory hit for path: %s, digest: %s", path, path_hash.hexdigest()
-            )
-            return
-
-        with open(path, "rb") as fp:
-            file_hash = InputCache.hash_file_digest(fp)
-            digest = file_hash.digest()
-            set_path_hash_input_kv(path, digest, input_cache)
-            path_hash.update(digest)
-
-    elif path.is_dir():
-        for item in sorted(path.iterdir(), key=lambda p: p.name):
-            walk_and_hash(item, input_cache, path_hash)
-
-
-def get_path_hash_from_input_kv(path: Path, input_cache: InputCache):
-    try:
-        # TODO temp hack to avoid input_cache being False
-        if input_cache:
-            return input_cache.kv_cache[str(path)]
-    except KeyError:
-        return None
-
-
-def set_path_hash_input_kv(path: Path, h_file, input_cache: InputCache):
-    # TODO temp hack to avoid input_cache being False
-    if input_cache:
-        input_cache.kv_cache[str(path)] = h_file
-
-
 def _to_dict(obj):
     """
     recursively update obj should it contain other

{kapitan-0.36.3.dev1 → kapitan-0.36.3.dev2}/tests/test_dependency_manager.py

@@ -26,6 +26,7 @@ from kapitan.dependency_manager.base import (
     fetch_helm_chart,
     fetch_http_source,
     fetch_oci_dependency,
+    helm_dependencies_digest,
 )
 from kapitan.errors import HelmFetchingError, OCIFetchingError
 from kapitan.inventory.model import KapitanInventorySettings
@@ -667,3 +668,219 @@ class OciFetchDependencyTest(unittest.TestCase):
             self.assertIn("only subdirectories", warning_text)
             self.assertIn("subpath", warning_text)
             self.assertIn("system", warning_text)
+
+
+@pytest.mark.usefixtures("reset_cached_args")
+class HelmDependenciesDigestTest(unittest.TestCase):
+    """Cover the cache-key digest derived from ``parameters.kapitan.dependencies``.
+
+    The digest is what makes HelmChart() inside kadet cache-correct: chart files
+    live outside the kadet component path, so without this their edits do not
+    perturb the kadet cache key. See PR description for the full motivation.
+    """
+
+    def _set_inv(self, deps, target="t1"):
+        from kapitan import cached
+
+        cached.inv = {target: {"parameters": {"kapitan": {"dependencies": deps}}}}
+
+    def _set_output_base(self, base):
+        from argparse import Namespace
+
+        from kapitan import cached
+
+        cached.args = Namespace(output_path=base, cache=False)
+
+    def _write_chart(self, root: Path, name: str, body: str = "v1"):
+        chart_dir = root / name
+        (chart_dir / "templates").mkdir(parents=True, exist_ok=True)
+        (chart_dir / "Chart.yaml").write_text(f"name: {name}\nversion: 1.0.0\n")
+        (chart_dir / "templates" / "deploy.yaml").write_text(body)
+        return chart_dir
+
+    def test_returns_none_when_target_has_no_deps(self):
+        self._set_inv([])
+        self._set_output_base(".")
+        self.assertIsNone(helm_dependencies_digest("t1"))
+
+    def test_returns_none_when_only_non_helm_deps(self):
+        self._set_inv(
+            [
+                {
+                    "type": "git",
+                    "source": "https://example.com/repo.git",
+                    "output_path": "vendor/repo",
+                }
+            ]
+        )
+        self._set_output_base(".")
+        self.assertIsNone(helm_dependencies_digest("t1"))
+
+    def test_returns_none_when_target_missing(self):
+        self._set_inv([], target="other")
+        self._set_output_base(".")
+        self.assertIsNone(helm_dependencies_digest("missing"))
+
+    def test_digest_changes_when_chart_file_changes(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            self._write_chart(tmp_path / "charts", "foo")
+            self._set_output_base(tmp)
+            self._set_inv(
+                [
+                    {
+                        "type": "helm",
+                        "source": "https://example.com",
+                        "output_path": "charts/foo",
+                        "chart_name": "foo",
+                        "version": "1.0.0",
+                    }
+                ]
+            )
+
+            before = helm_dependencies_digest("t1")
+            self.assertIsNotNone(before)
+
+            # Edit a template file under the declared chart_dir.
+            (tmp_path / "charts" / "foo" / "templates" / "deploy.yaml").write_text("v2")
+            after = helm_dependencies_digest("t1")
+
+            self.assertNotEqual(before, after)
+
+    def test_digest_stable_when_unrelated_files_change(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            self._write_chart(tmp_path / "charts", "foo")
+            # Sibling files outside the chart_dir must not influence the digest.
+            (tmp_path / "unrelated.yaml").write_text("a")
+            self._set_output_base(tmp)
+            self._set_inv(
+                [
+                    {
+                        "type": "helm",
+                        "source": "https://example.com",
+                        "output_path": "charts/foo",
+                        "chart_name": "foo",
+                        "version": "1.0.0",
+                    }
+                ]
+            )
+
+            before = helm_dependencies_digest("t1")
+            (tmp_path / "unrelated.yaml").write_text("b")
+            after = helm_dependencies_digest("t1")
+
+            self.assertEqual(before, after)
+
+    def test_digest_order_invariant(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            self._write_chart(tmp_path / "charts", "foo")
+            self._write_chart(tmp_path / "charts", "bar")
+            self._set_output_base(tmp)
+
+            dep_foo = {
+                "type": "helm",
+                "source": "https://example.com",
+                "output_path": "charts/foo",
+                "chart_name": "foo",
+                "version": "1.0.0",
+            }
+            dep_bar = {
+                "type": "helm",
+                "source": "https://example.com",
+                "output_path": "charts/bar",
+                "chart_name": "bar",
+                "version": "1.0.0",
+            }
+
+            self._set_inv([dep_foo, dep_bar])
+            d1 = helm_dependencies_digest("t1")
+
+            self._set_inv([dep_bar, dep_foo])
+            d2 = helm_dependencies_digest("t1")
+
+            self.assertEqual(d1, d2)
+
+    def test_digest_ignores_non_helm_deps_alongside(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            self._write_chart(tmp_path / "charts", "foo")
+            self._set_output_base(tmp)
+
+            helm_dep = {
+                "type": "helm",
+                "source": "https://example.com",
+                "output_path": "charts/foo",
+                "chart_name": "foo",
+                "version": "1.0.0",
+            }
+            git_dep = {
+                "type": "git",
+                "source": "https://example.com/repo.git",
+                "output_path": "vendor/repo",
+            }
+
+            self._set_inv([helm_dep])
+            with_only_helm = helm_dependencies_digest("t1")
+
+            self._set_inv([helm_dep, git_dep])
+            with_extras = helm_dependencies_digest("t1")
+
+            self.assertEqual(with_only_helm, with_extras)
+
+    def test_digest_changes_when_output_path_renamed_same_content(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            self._write_chart(tmp_path / "charts", "foo")
+            self._write_chart(tmp_path / "charts", "foo-alias")
+            # foo-alias has the same files as foo because of identical _write_chart;
+            # only the declared output_path label differs.
+            self._set_output_base(tmp)
+
+            self._set_inv(
+                [
+                    {
+                        "type": "helm",
+                        "source": "https://example.com",
+                        "output_path": "charts/foo",
+                        "chart_name": "foo",
+                        "version": "1.0.0",
+                    }
+                ]
+            )
+            before = helm_dependencies_digest("t1")
+
+            self._set_inv(
+                [
+                    {
+                        "type": "helm",
+                        "source": "https://example.com",
+                        "output_path": "charts/foo-alias",
+                        "chart_name": "foo",
+                        "version": "1.0.0",
+                    }
+                ]
+            )
+            after = helm_dependencies_digest("t1")
+
+            self.assertNotEqual(before, after)
+
+    def test_walks_missing_path_gracefully(self):
+        # Path does not exist on disk yet (e.g. user ran --cache without --fetch).
+        # walk_and_hash silently returns; we still fold the declared output_path
+        # in, so the digest is deterministic.
+        self._set_inv(
+            [
+                {
+                    "type": "helm",
+                    "source": "https://example.com",
+                    "output_path": "charts/missing",
+                    "chart_name": "missing",
+                    "version": "1.0.0",
+                }
+            ]
+        )
+        self._set_output_base("/nonexistent-base")
+        digest = helm_dependencies_digest("t1")
+        self.assertIsNotNone(digest)