kubernator 1.0.14__py3-none-any.whl → 1.0.24.dev20251109010128__py3-none-any.whl

kubernator/plugins/helm.py

@@ -27,7 +27,7 @@ from shutil import which, copy
 from typing import Sequence
 
 import yaml
-from jsonschema import Draft7Validator, draft7_format_checker
+from jsonschema import Draft7Validator
 
 from kubernator.api import (KubernatorPlugin, Globs, StripNL,
                             scan_dir,
@@ -37,7 +37,7 @@ from kubernator.api import (KubernatorPlugin, Globs, StripNL,
                             validator_with_defaults,
                             get_golang_os,
                             get_golang_machine,
-                            prepend_os_path
+                            prepend_os_path, TemplateEngine, get_cache_dir
                             )
 from kubernator.plugins.k8s_api import K8SResource
 from kubernator.proc import DEVNULL
@@ -85,12 +85,12 @@ HELM_SCHEMA = {
         }
     },
     "type": "object",
-    "required": ["repository", "chart", "version", "name", "namespace"]
+    "required": ["chart", "name", "namespace"]
 }
 
 Draft7Validator.check_schema(HELM_SCHEMA)
 HELM_VALIDATOR_CLS = validator_with_defaults(Draft7Validator)
-HELM_VALIDATOR = HELM_VALIDATOR_CLS(HELM_SCHEMA, format_checker=draft7_format_checker)
+HELM_VALIDATOR = HELM_VALIDATOR_CLS(HELM_SCHEMA, format_checker=Draft7Validator.FORMAT_CHECKER)
 
 
 class HelmPlugin(KubernatorPlugin):
@@ -102,18 +102,25 @@ class HelmPlugin(KubernatorPlugin):
         self.context = None
         self.repositories = set()
         self.helm_dir = None
+        self.template_engine = TemplateEngine(logger)
+
+        self._repositories_not_populated = True
+        self._helm_repositories_file = None
+        self._charts_used: dict[str, list[tuple[str, str]]] = {}
 
     def set_context(self, context):
         self.context = context
 
     def stanza(self):
         context = self.context
-        stanza = [context.helm.helm_file, f"--kubeconfig={context.kubeconfig.kubeconfig}"]
+        stanza = [context.helm.helm_file,
+                  f"--kubeconfig={context.kubeconfig.kubeconfig}",
+                  f"--repository-config={self._helm_repositories_file}"]
         if logger.getEffectiveLevel() < logging.INFO:
             stanza.append("--debug")
         return stanza
 
-    def register(self, version=None):
+    def register(self, version=None, check_chart_versions=False):
         context = self.context
         context.app.register_plugin("kubeconfig")
         context.app.register_plugin("k8s")
@@ -130,7 +137,7 @@ class HelmPlugin(KubernatorPlugin):
             helm_tar = tarfile.open(helm_file_dl)
             helm_tar.extractall(self.helm_dir.name)
 
-            copy(Path(self.helm_dir.name)/f"{get_golang_os()}-{get_golang_machine()}"/"helm", helm_file)
+            copy(Path(self.helm_dir.name) / f"{get_golang_os()}-{get_golang_machine()}" / "helm", helm_file)
 
             os.chmod(helm_file, 0o500)
             prepend_os_path(self.helm_dir.name)
@@ -142,6 +149,8 @@ class HelmPlugin(KubernatorPlugin):
 
             logger.debug("Found Helm in %r", helm_file)
 
+        helm_dir = get_cache_dir("helm")
+        self._helm_repositories_file = Path(helm_dir) / "repositories.yaml"
         context.globals.helm = dict(default_includes=Globs(["*.helm.yaml", "*.helm.yml"], True),
                                     default_excludes=Globs([".*"], True),
                                     namespace_transformer=True,
@@ -149,6 +158,7 @@ class HelmPlugin(KubernatorPlugin):
                                     stanza=self.stanza,
                                     add_helm_template=self.add_helm_template,
                                     add_helm=self.add_helm,
+                                    check_chart_versions=check_chart_versions,
                                     )
 
     def handle_init(self):
@@ -180,11 +190,46 @@ class HelmPlugin(KubernatorPlugin):
             display_p = context.app.display_path(p)
             logger.debug("Adding Helm template from %s", display_p)
 
-            helm_templates = load_file(logger, p, FileType.YAML, display_p)
+            helm_templates = load_file(logger, p, FileType.YAML, display_p,
+                                       self.template_engine,
+                                       {"ktor": context})
 
             for helm_template in helm_templates:
                 self._add_helm(helm_template, display_p)
 
+    def handle_summary(self):
+        context = self.context
+        if context.helm.check_chart_versions:
+            logger.info("Checking Helm chart versions")
+            all_chart_versions = json.loads(self.context.app.run_capturing_out(self.stanza() +
+                                                                               ["search", "repo",
+                                                                                "-l", "-o", "json",
+                                                                                ],
+                                                                               stderr_logger,
+                                                                               ))
+
+            def chart_version_part_normalizer(x):
+                return int(x) if x.isnumeric() else x
+
+            def chart_version_normalizer(x):
+                return tuple(map(chart_version_part_normalizer, x.split(".")))
+
+            charts_versions = {}
+            for chart_version in all_chart_versions:
+                charts_versions.setdefault(chart_version["name"], []).append(
+                    chart_version_normalizer(chart_version["version"]))
+
+            for chart, version_source in self._charts_used.items():
+                chart_versions = sorted(charts_versions[chart])
+                for version, source in version_source:
+                    if chart_versions[-1] > chart_version_normalizer(version):
+                        logger.warning("Chart %s is version %s while the latest is %s (defined in %s)",
+                                       chart.split("/")[1],
+                                       version,
+                                       ".".join(map(str, chart_versions[-1])),
+                                       source,
+                                       )
+
     def add_helm_template(self, template):
         return self._add_helm(template, calling_frame_source())
 
@@ -201,6 +246,23 @@ class HelmPlugin(KubernatorPlugin):
         return self._internal_add_helm(source, **{k.replace("-", "_"): v for k, v in template.items()})
 
     def _add_repository(self, repository: str):
+        def _update_repositories():
+            if self.repositories:
+                self.context.app.run(self.stanza() + ["repo", "update"],
+                                     stdout_logger,
+                                     stderr_logger).wait()
+
+        if self._repositories_not_populated:
+            preexisting_repositories = json.loads(
+                self.context.app.run_capturing_out(self.stanza() + ["repo", "list", "-o", "json"], stderr_logger))
+            for pr in preexisting_repositories:
+                repo_name = pr["name"]
+                repo_url = pr["url"]
+                logger.debug("Recording pre-existing repository %s mapping %s", repo_name, repo_url)
+                self.repositories.add(repo_name)
+            _update_repositories()
+            self._repositories_not_populated = False
+
         repository_hash = sha256(repository.encode("UTF-8")).hexdigest()
         logger.debug("Repository %s mapping to %s", repository, repository_hash)
         if repository_hash not in self.repositories:
@@ -208,24 +270,52 @@ class HelmPlugin(KubernatorPlugin):
             self.context.app.run(self.stanza() + ["repo", "add", repository_hash, repository],
                                  stdout_logger,
                                  stderr_logger).wait()
-            self.context.app.run(self.stanza() + ["repo", "update"],
-                                 stdout_logger,
-                                 stderr_logger).wait()
+            _update_repositories()
             self.repositories.add(repository_hash)
 
         return repository_hash
 
-    def _internal_add_helm(self, source, *, repository, chart, version, name, namespace, include_crds,
-                           values=None, values_file=None):
+    def _internal_add_helm(self, source, *, chart, name, namespace, include_crds,
+                           values=None, values_file=None, repository=None, version=None):
         if values and values_file:
             raise RuntimeError(f"In {source} either values or values file may be specified, but not both")
 
+        if (repository and chart and chart.startswith("oci://") or
+                not repository and chart and not chart.startswith("oci://")):
+            raise RuntimeError(
+                f"In {source} either repository must be specified or OCI-chart must be used, but not both")
+
+        if not version and repository:
+            raise RuntimeError(f"In {source} version must be specified unless OCI-chart is used")
+
         if values_file:
             values_file = Path(values_file)
             if not values_file.is_absolute():
                 values_file = self.context.app.cwd / values_file
+            values = list(load_file(logger, values_file, FileType.YAML,
+                                    template_engine=self.template_engine,
+                                    template_context={"ktor": self.context,
+                                                      "helm": {"chart": chart,
+                                                               "name": name,
+                                                               "namespace": namespace,
+                                                               "include_crds": include_crds,
+                                                               "repository": repository,
+                                                               "version": version,
+                                                               }}))
+            values = values[0] if values else {}
+
+        version_spec = []
+        if repository:
+            repository_hash = self._add_repository(repository)
+            chart_name = f"{repository_hash}/{chart}"
+            if version:
+                self._charts_used.setdefault(chart_name, []).append((version, source))
+        else:
+            chart_name = chart
+
+        if version:
+            version_spec = ["--version", version]
 
-        repository_hash = self._add_repository(repository)
         stdin = DEVNULL
 
         if values:
@@ -237,14 +327,13 @@ class HelmPlugin(KubernatorPlugin):
         resources = self.context.app.run_capturing_out(self.stanza() +
                                                        ["template",
                                                         name,
-                                                        f"{repository_hash}/{chart}",
-                                                        "--version", version,
+                                                        chart_name,
                                                         "-n", namespace,
                                                         "-a", ",".join(self.context.k8s.get_api_versions())
                                                         ] +
+                                                       version_spec +
                                                        (["--include-crds"] if include_crds else []) +
-                                                       (["-f", values_file] if values_file else []) +
-                                                       (["-f", "-"] if values else []),
+                                                       ["-f", "-"],
                                                        stderr_logger,
                                                        stdin=stdin,
                                                        )

kubernator/plugins/istio.py

@@ -25,6 +25,7 @@ from pathlib import Path
 from shutil import which
 
 import yaml
+
 from kubernator.api import (KubernatorPlugin, scan_dir,
                             TemplateEngine,
                             load_remote_file,
@@ -33,7 +34,8 @@ from kubernator.api import (KubernatorPlugin, scan_dir,
                             Globs,
                             get_golang_os,
                             get_golang_machine,
-                            prepend_os_path, jp)
+                            prepend_os_path, jp, load_file)
+from kubernator.plugins.k8s import api_exc_normalize_body, api_exc_format_body
 from kubernator.plugins.k8s_api import K8SResourcePluginMixin
 
 logger = logging.getLogger("kubernator.istio")
@@ -53,6 +55,9 @@ class IstioPlugin(KubernatorPlugin, K8SResourcePluginMixin):
         self.client_version = None
         self.server_version = None
         self.provision_operator = False
+        self.install = False
+        self.upgrade = False
+        self.upgrade_from_operator = False
         self.template_engine = TemplateEngine(logger)
 
         self.istioctl_dir = None
@@ -133,25 +138,41 @@ class IstioPlugin(KubernatorPlugin, K8SResourcePluginMixin):
         context = self.context
 
         version, version_out_js = self.test_istioctl()
-        self.client_version = tuple(version.split("."))
-        mesh_versions = set(tuple(m.value.split(".")) for m in MESH_PILOT_JP.find(version_out_js))
+        self.client_version = tuple(map(int, version.split(".")))
+        mesh_versions = set(tuple(map(int, m.value.split("."))) for m in MESH_PILOT_JP.find(version_out_js))
 
         if mesh_versions:
             self.server_version = max(mesh_versions)
 
         if not self.server_version:
             logger.info("No Istio mesh has been found and it'll be created")
+            self.install = True
             self.provision_operator = True
-        elif self.server_version < self.client_version:
-            logger.info("Istio client is version %s while server is up to %s - operator will be upgraded",
-                        ".".join(self.client_version),
-                        ".".join(self.server_version))
+        elif self.server_version != self.client_version:
+            logger.info("Istio client is version %s while server is up to %s - up/downgrade will be performed",
+                        ".".join(map(str, self.client_version)),
+                        ".".join(map(str, self.server_version)))
+            self.upgrade = True
             self.provision_operator = True
 
+        if self.client_version >= (1, 24, 0):
+            # No more operator in 1.24.0+
+            self.provision_operator = False
+
+        if self.upgrade and (self.client_version >= (1, 24, 0) > self.server_version):
+            self.upgrade_from_operator = True
+
+        if self.upgrade and (self.client_version < (1, 24, 0) <= self.server_version):
+            raise ValueError(f"Unable to downgrade Istio from {self.server_version} to {self.client_version}")
+
         # Register Istio-related CRDs with K8S
+        if self.client_version >= (1, 24, 0):
+            crd_path = "manifests/charts/base/files/crd-all.gen.yaml"
+        else:
+            crd_path = "manifests/charts/base/crds/crd-all.gen.yaml"
         self.context.k8s.load_remote_crds(
-            f"https://raw.githubusercontent.com/istio/istio/{'.'.join(self.client_version)}/"
-            "manifests/charts/base/crds/crd-all.gen.yaml", "yaml")
+            f"https://raw.githubusercontent.com/istio/istio/{'.'.join(map(str, self.client_version))}/{crd_path}",
+            "yaml")
 
         # This plugin only deals with Istio Operator, so only load that stuff
         self.resource_definitions_schema = load_remote_file(logger,
@@ -161,8 +182,10 @@ class IstioPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                                                             FileType.JSON)
         self._populate_resource_definitions()
 
-        self.add_remote_crds(f"https://raw.githubusercontent.com/istio/istio/{'.'.join(self.client_version)}/"
-                             f"manifests/charts/istio-operator/crds/crd-operator.yaml", FileType.YAML)
+        crd_operator_version = (1, 23, 4) if self.client_version >= (1, 24, 0) else self.client_version
+        self.add_remote_crds(
+            f"https://raw.githubusercontent.com/istio/istio/{'.'.join(map(str, crd_operator_version))}/"
+            f"manifests/charts/istio-operator/crds/crd-operator.yaml", FileType.YAML)
 
         # Exclude Istio YAMLs from K8S resource loading
         context.k8s.default_excludes.add("*.istio.yaml")
@@ -189,10 +212,11 @@ class IstioPlugin(KubernatorPlugin, K8SResourcePluginMixin):
             display_p = context.app.display_path(p)
             logger.info("Adding Istio Operator from %s", display_p)
 
-            with open(p, "rt") as file:
-                template = self.template_engine.from_string(file.read())
+            manifests = load_file(logger, p, FileType.YAML, display_p,
+                                  self.template_engine,
+                                  {"ktor": context})
 
-            self.add_resources(template.render({"ktor": context}), display_p)
+            self.add_resources(manifests, display_p)
 
     def handle_apply(self):
         context = self.context
@@ -211,41 +235,130 @@ class IstioPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                 context.app.run(context.istio.stanza() + ["validate", "-f", operators_file.name],
                                 stdout_logger, stderr_logger).wait()
 
-                self._operator_init(operators_file, True)
-
-                if not context.app.args.dry_run:
-                    self._operator_init(operators_file, False)
-
-    def _operator_init(self, operators_file, dry_run):
+                dry_run = context.app.args.dry_run
+
+                if self.provision_operator:
+                    self._create_istio_system_ns(True)
+                    self._operator_init(operators_file, True)
+
+                    if not dry_run:
+                        self._create_istio_system_ns(False)
+                        self._operator_init(operators_file, False)
+                elif self.install:
+                    self._install(operators_file, True)
+
+                    if not dry_run:
+                        self._install(operators_file, False)
+                elif self.upgrade:
+                    def _upgrade(dry_run):
+                        if self.upgrade_from_operator:
+                            # delete deployment -n istio-system istio-operator
+                            self._delete_resource_internal({"apiVersion": "apps/v1",
+                                                            "kind": "Deployment",
+                                                            "metadata":
+                                                                {
+                                                                    "namespace": "istio-system",
+                                                                    "name": "istio-operator"
+                                                                }
+                                                            }, dry_run, True)
+                        self._upgrade(operators_file, dry_run)
+
+                    _upgrade(True)
+                    if not dry_run:
+                        _upgrade(False)
+
+    def _delete_resource_internal(self, manifest, dry_run=True, missing_ok=False):
         from kubernetes import client
         from kubernetes.client.rest import ApiException
 
         context = self.context
+        k8s_client = context.k8s.client
 
-        status_details = " (dry run)" if dry_run else ""
+        res = self._create_resource(manifest)
+        res.rdef.populate_api(client, k8s_client)
+        try:
+            res.delete(dry_run=dry_run)
+        except ApiException as e:
+            api_exc_normalize_body(e)
+            try:
+                skip = False
+                if e.status == 404 and missing_ok:
+                    skip = True
+                if not skip:
+                    raise
+            except ApiException as e:
+                api_exc_format_body(e)
+                raise
+
+        return res
+
+    def _create_resource_internal(self, manifest, dry_run=True, exists_ok=False):
+        from kubernetes import client
+        from kubernetes.client.rest import ApiException
 
+        context = self.context
         k8s_client = context.k8s.client
-        logger.info("Creating istio-system namespace%s", status_details)
-        istio_system = self.add_resource({"apiVersion": "v1",
-                                          "kind": "Namespace",
-                                          "metadata": {
-                                              "labels": {
-                                                  "istio-injection": "disabled"
-                                              },
-                                              "name": "istio-system"
-                                          }})
-        istio_system.rdef.populate_api(client, k8s_client)
+
+        res = self._create_resource(manifest)
+        res.rdef.populate_api(client, k8s_client)
         try:
-            istio_system.create(dry_run=dry_run)
+            res.create(dry_run=dry_run)
         except ApiException as e:
             skip = False
-            if e.status == 409:
-                status = json.loads(e.body)
-                if status["reason"] == "AlreadyExists":
-                    skip = True
-            if not skip:
+            api_exc_normalize_body(e)
+            try:
+                if e.status == 409:
+                    status = e.body
+                    if status["reason"] == "AlreadyExists" and exists_ok:
+                        skip = True
+                if not skip:
+                    raise
+            except ApiException as e:
+                api_exc_format_body(e)
                 raise
 
+        return res
+
+    def _install(self, operators_file, dry_run):
+        context = self.context
+        status_details = " (dry run)" if dry_run else ""
+
+        logger.info("Running Istio install%s", status_details)
+        istio_install_cmd = context.istio.stanza() + ["install", "-f", operators_file.name, "-y", "--verify"]
+        context.app.run(istio_install_cmd + (["--dry-run"] if dry_run else []),
+                        stdout_logger,
+                        stderr_logger).wait()
+
+    def _upgrade(self, operators_file, dry_run):
+        context = self.context
+        status_details = " (dry run)" if dry_run else ""
+
+        logger.info("Running Istio upgrade%s", status_details)
+        istio_upgrade_cmd = context.istio.stanza() + ["upgrade", "-f", operators_file.name, "-y", "--verify"]
+        context.app.run(istio_upgrade_cmd + (["--dry-run"] if dry_run else []),
+                        stdout_logger,
+                        stderr_logger).wait()
+
+    def _create_istio_system_ns(self, dry_run):
+        status_details = " (dry run)" if dry_run else ""
+        logger.info("Creating istio-system namespace%s", status_details)
+        self._create_resource_internal({"apiVersion": "v1",
+                                        "kind": "Namespace",
+                                        "metadata": {
+                                            "labels": {
+                                                "istio-injection": "disabled"
+                                            },
+                                            "name": "istio-system"
+                                        }
+                                        },
+                                       dry_run=dry_run,
+                                       exists_ok=True
+                                       )
+
+    def _operator_init(self, operators_file, dry_run):
+        context = self.context
+        status_details = " (dry run)" if dry_run else ""
+
         logger.info("Running Istio operator init%s", status_details)
         istio_operator_init = context.istio.stanza() + ["operator", "init", "-f", operators_file.name]
         context.app.run(istio_operator_init + (["--dry-run"] if dry_run else []),