kubernator 1.0.14__py3-none-any.whl → 1.0.24.dev20251109010128__py3-none-any.whl

kubernator/plugins/k8s.py

@@ -30,6 +30,7 @@ from typing import Iterable, Callable, Sequence
 
 import jsonpatch
 import yaml
+from kubernetes.client import ApiException
 
 from kubernator.api import (KubernatorPlugin,
                             Globs,
@@ -38,7 +39,9 @@ from kubernator.api import (KubernatorPlugin,
                             FileType,
                             load_remote_file,
                             StripNL,
-                            install_python_k8s_client)
+                            install_python_k8s_client,
+                            TemplateEngine,
+                            sleep)
 from kubernator.merge import extract_merge_instructions, apply_merge_instructions
 from kubernator.plugins.k8s_api import (K8SResourcePluginMixin,
                                         K8SResource,
@@ -68,6 +71,34 @@ def final_resource_validator(resources: Sequence[K8SResource],
                     resource, resource.source)
 
 
+def normalize_pkg_version(v: str):
+    v_split = v.split(".")
+    rev = v_split[-1]
+    if not rev.isdigit():
+        new_rev = ""
+        for c in rev:
+            if not c.isdigit():
+                break
+            new_rev += c
+        v_split[-1] = new_rev
+    return tuple(map(int, v_split))
+
+
+def api_exc_normalize_body(e: "ApiException"):
+    if e.headers and "content-type" in e.headers:
+        content_type = e.headers["content-type"]
+        if content_type == "application/json" or content_type.endswith("+json"):
+            e.body = json.loads(e.body)
+        elif (content_type in ("application/yaml", "application/x-yaml", "text/yaml",
+                               "text/x-yaml") or content_type.endswith("+yaml")):
+            e.body = yaml.safe_load(e.body)
+
+
+def api_exc_format_body(e: ApiException):
+    if not isinstance(e.body, (str, bytes)):
+        e.body = json.dumps(e.body, indent=4)
+
+
 class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
     logger = logger
 
@@ -81,6 +112,9 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
 
         self._transformers = []
         self._validators = []
+        self._manifest_patchers = []
+        self._summary = 0, 0, 0
+        self._template_engine = TemplateEngine(logger)
 
     def set_context(self, context):
         self.context = context
@@ -104,6 +138,8 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                                                       ("apps", "StatefulSet"): K8SPropagationPolicy.ORPHAN,
                                                       ("apps", "Deployment"): K8SPropagationPolicy.ORPHAN,
                                                       ("storage.k8s.io", "StorageClass"): K8SPropagationPolicy.ORPHAN,
+                                                      (None, "Pod"): K8SPropagationPolicy.BACKGROUND,
+                                                      ("batch", "Job"): K8SPropagationPolicy.ORPHAN,
                                                       },
                                    default_includes=Globs(["*.yaml", "*.yml"], True),
                                    default_excludes=Globs([".*"], True),
@@ -115,12 +151,14 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                                    add_transformer=self.api_add_transformer,
                                    remove_transformer=self.api_remove_transformer,
                                    add_validator=self.api_remove_validator,
+                                   add_manifest_patcher=self.api_add_manifest_patcher,
                                    get_api_versions=self.get_api_versions,
                                    create_resource=self.create_resource,
                                    disable_client_patches=disable_client_patches,
                                    field_validation=field_validation,
                                    field_validation_warn_fatal=field_validation_warn_fatal,
                                    field_validation_warnings=0,
+                                   conflict_retry_delay=0.3,
                                    _k8s=self,
                                    )
         context.k8s = dict(default_includes=Globs(context.globals.k8s.default_includes),
@@ -150,7 +188,7 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
 
         logger.info("Using Kubernetes client version =~%s.0 for server version %s",
                     server_minor, ".".join(k8s.server_version))
-        pkg_dir = install_python_k8s_client(self.context.app.run, server_minor, logger,
+        pkg_dir = install_python_k8s_client(self.context.app.run_passthrough_capturing, server_minor, logger,
                                             stdout_logger, stderr_logger, k8s.disable_client_patches)
 
         modules_to_delete = []
@@ -183,7 +221,7 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
 
         k8s.client = self._setup_k8s_client()
         version = client.VersionApi(k8s.client).get_code()
-        if "-eks-" in version.git_version:
+        if "-eks-" or "-gke" in version.git_version:
             git_version = version.git_version.split("-")[0]
         else:
             git_version = version.git_version
@@ -192,7 +230,8 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
         k8s.server_git_version = git_version
 
         logger.info("Found Kubernetes %s on %s", k8s.server_git_version, k8s.client.configuration.host)
-        K8SResource._k8s_client_version = tuple(map(int, pkg_version("kubernetes").split(".")))
+
+        K8SResource._k8s_client_version = normalize_pkg_version(pkg_version("kubernetes"))
         K8SResource._k8s_field_validation = k8s.field_validation
         K8SResource._k8s_field_validation_patched = not k8s.disable_client_patches
         K8SResource._logger = self.logger
@@ -224,7 +263,10 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
             display_p = context.app.display_path(p)
             logger.debug("Adding Kubernetes manifest from %s", display_p)
 
-            manifests = load_file(logger, p, FileType.YAML, display_p)
+            manifests = load_file(logger, p, FileType.YAML, display_p,
+                                  self._template_engine,
+                                  {"ktor": context}
+                                  )
 
             for manifest in manifests:
                 if manifest:
@@ -248,6 +290,7 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
 
         patch_field_excludes = [re.compile(e) for e in context.globals.k8s.patch_field_excludes]
         dump_results = []
+        total_created, total_patched, total_deleted = 0, 0, 0
         for resource in self.resources.values():
             if dump:
                 resource_id = {"apiVersion": resource.api_version,
@@ -280,13 +323,17 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                 create_func = partial(resource.create, dry_run=dry_run)
                 delete_func = partial(resource.delete, dry_run=dry_run)
 
-            self._apply_resource(dry_run,
-                                 patch_field_excludes,
-                                 resource,
-                                 patch_func,
-                                 create_func,
-                                 delete_func,
-                                 status_msg)
+            created, patched, deleted = self._apply_resource(dry_run,
+                                                             patch_field_excludes,
+                                                             resource,
+                                                             patch_func,
+                                                             create_func,
+                                                             delete_func,
+                                                             status_msg)
+
+            total_created += created
+            total_patched += patched
+            total_deleted += deleted
 
         if ((dump or dry_run) and
                 k8s.field_validation_warn_fatal and self.context.globals.k8s.field_validation_warnings):
@@ -301,6 +348,12 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                           indent=4 if file_format == "json-pretty" else None)
             else:
                 yaml.safe_dump(dump_results, file)
+        else:
+            self._summary = total_created, total_patched, total_deleted
+
+    def handle_summary(self):
+        total_created, total_patched, total_deleted = self._summary
+        logger.info("Created %d, patched %d, deleted %d resources", total_created, total_patched, total_deleted)
 
     def api_load_resources(self, path: Path, file_type: str):
         return self.add_local_resources(path, FileType[file_type.upper()])
@@ -322,6 +375,10 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
         if validator not in self._validators:
             self._validators.append(validator)
 
+    def api_add_manifest_patcher(self, patcher):
+        if patcher not in self._manifest_patchers:
+            self._manifest_patchers.append(patcher)
+
     def api_remove_transformer(self, transformer):
         if transformer in self._transformers:
             self._transformers.remove(transformer)
@@ -340,6 +397,17 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
             frame = frame.f_back
         return ValueError((msg % args) if args else msg).with_traceback(tb)
 
+    def _patch_manifest(self,
+                        manifest: dict,
+                        resource_description: str):
+        for patcher in reversed(self._manifest_patchers):
+            logger.debug("Applying patcher %s to %s",
+                         getattr(patcher, "__name__", patcher),
+                         resource_description)
+            manifest = patcher(manifest, resource_description) or manifest
+
+        return manifest
+
     def _transform_resource(self, resources: Sequence[K8SResource], resource: K8SResource) -> K8SResource:
         for transformer in reversed(self._transformers):
             logger.debug("Applying transformer %s to %s from %s",
@@ -378,8 +446,8 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
 
         def handle_400_strict_validation_error(e: ApiException):
             if e.status == 400:
-                status = json.loads(e.body)
-
+                # Assumes the body has been parsed
+                status = e.body
                 if status["status"] == "Failure":
                     if FIELD_VALIDATION_STRICT_MARKER in status["message"]:
                         message = status["message"]
@@ -394,18 +462,31 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                                      resource, resource.source, status["message"])
                         raise e from None
 
-        def create(exists_ok=False):
+        def create(exists_ok=False, wait_for_delete=False):
             logger.info("Creating resource %s%s%s", resource, status_msg,
                         " (ignoring existing)" if exists_ok else "")
-            try:
-                create_func()
-            except ApiException as e:
-                if exists_ok:
-                    if e.status == 409:
-                        status = json.loads(e.body)
-                        if status["reason"] == "AlreadyExists":
-                            return
-                raise
+            while True:
+                try:
+                    create_func()
+                    return
+                except ApiException as __e:
+                    api_exc_normalize_body(__e)
+                    try:
+                        if exists_ok or wait_for_delete:
+                            if __e.status == 409:
+                                status = __e.body
+                                if status["reason"] == "AlreadyExists":
+                                    if wait_for_delete:
+                                        sleep(self.context.k8s.conflict_retry_delay)
+                                        logger.info("Retry creating resource %s%s%s", resource, status_msg,
+                                                    " (ignoring existing)" if exists_ok else "")
+                                        continue
+                                    else:
+                                        return
+                        raise
+                    except ApiException as ___e:
+                        api_exc_format_body(___e)
+                        raise
 
         merge_instrs, normalized_manifest = extract_merge_instructions(resource.manifest, resource)
         if merge_instrs:
@@ -419,14 +500,20 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
             remote_resource = resource.get()
             logger.trace("Current resource %s: %s", resource, remote_resource)
         except ApiException as e:
-            if e.status == 404:
-                try:
-                    create()
-                except ApiException as e:
-                    if not handle_400_strict_validation_error(e):
-                        raise
-
-            else:
+            api_exc_normalize_body(e)
+            try:
+                if e.status == 404:
+                    try:
+                        create()
+                        return 1, 0, 0
+                    except ApiException as e:
+                        api_exc_normalize_body(e)
+                        if not handle_400_strict_validation_error(e):
+                            raise
+                else:
+                    raise
+            except ApiException as _e:
+                api_exc_format_body(_e)
                 raise
         else:
             logger.trace("Attempting to retrieve a normalized patch for resource %s: %s", resource, normalized_manifest)
@@ -436,34 +523,44 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                                                  dry_run=True,
                                                  force=True)
             except ApiException as e:
-                if e.status == 422:
-                    status = json.loads(e.body)
-                    details = status["details"]
-                    immutable_key = details["group"], details["kind"]
-
-                    try:
-                        propagation_policy = self.context.k8s.immutable_changes[immutable_key]
-                    except KeyError:
-                        raise e from None
+                try:
+                    api_exc_normalize_body(e)
+
+                    if e.status == 422:
+                        status = e.body
+                        # Assumes the body has been unmarshalled
+                        details = status["details"]
+                        immutable_key = details.get("group"), details["kind"]
+
+                        try:
+                            propagation_policy = self.context.k8s.immutable_changes[immutable_key]
+                        except KeyError:
+                            raise e from None
+                        else:
+                            for cause in details["causes"]:
+                                if (
+                                        cause["reason"] == "FieldValueInvalid" and
+                                        "field is immutable" in cause["message"]
+                                        or
+                                        cause["reason"] == "FieldValueForbidden" and
+                                        ("Forbidden: updates to" in cause["message"]
+                                         or
+                                         "Forbidden: pod updates" in cause["message"])
+                                ):
+                                    logger.info("Deleting resource %s (cascade %s)%s", resource,
+                                                propagation_policy.policy,
+                                                status_msg)
+                                    delete_func(propagation_policy=propagation_policy)
+                                    create(exists_ok=dry_run, wait_for_delete=not dry_run)
+                                    return 1, 0, 1
+                            raise
                     else:
-                        for cause in details["causes"]:
-                            if (
-                                    cause["reason"] == "FieldValueInvalid" and
-                                    "field is immutable" in cause["message"]
-                                    or
-                                    cause["reason"] == "FieldValueForbidden" and
-                                    "Forbidden: updates to" in cause["message"]
-                            ):
-                                logger.info("Deleting resource %s (cascade %s)%s", resource,
-                                            propagation_policy.policy,
-                                            status_msg)
-                                delete_func(propagation_policy=propagation_policy)
-                                create(exists_ok=dry_run)
-                                return
-                        raise
-                else:
-                    if not handle_400_strict_validation_error(e):
-                        raise
+                        if not handle_400_strict_validation_error(e):
+                            raise
+                except ApiException as _e:
+                    api_exc_format_body(_e)
+                    raise
+
             else:
                 logger.trace("Merged resource %s: %s", resource, merged_resource)
                 if merge_instrs:
@@ -476,8 +573,10 @@ class KubernetesPlugin(KubernatorPlugin, K8SResourcePluginMixin):
                 if patch:
                     logger.info("Patching resource %s%s", resource, status_msg)
                     patch_func(patch)
+                    return 0, 1, 0
                 else:
                     logger.info("Nothing to patch for resource %s", resource)
+                    return 0, 0, 0
 
     def _filter_resource_patch(self, patch: Iterable[Mapping], excludes: Iterable[re.compile]):
         result = []

kubernator/plugins/k8s_api.py

@@ -23,19 +23,17 @@ from collections import namedtuple
 from collections.abc import Callable, Mapping, MutableMapping, Sequence, Iterable
 from enum import Enum, auto
 from functools import partial
-from io import StringIO
 from pathlib import Path
 from typing import Union, Optional
 
 import yaml
 from jsonschema._format import FormatChecker
-from jsonschema._types import int_types, str_types
-from jsonschema._validators import required
+from jsonschema._keywords import required
 from jsonschema.exceptions import ValidationError
-from jsonschema.validators import extend, Draft7Validator, RefResolver
-from openapi_schema_validator import OAS30Validator
+from jsonschema.validators import extend, Draft7Validator
+from openapi_schema_validator import OAS31Validator
 
-from kubernator.api import load_file, FileType, load_remote_file, calling_frame_source
+from kubernator.api import load_file, FileType, load_remote_file, calling_frame_source, parse_yaml_docs
 
 K8S_WARNING_HEADER = re.compile(r'(?:,\s*)?(\d{3})\s+(\S+)\s+"(.+?)(?<!\\)"(?:\s+\"(.+?)(?<!\\)\")?\s*')
 UPPER_FOLLOWED_BY_LOWER_RE = re.compile(r"(.)([A-Z][a-z]+)")
@@ -89,11 +87,11 @@ def is_integer(instance):
     # bool inherits from int, so ensure bools aren't reported as ints
     if isinstance(instance, bool):
         return False
-    return isinstance(instance, int_types)
+    return isinstance(instance, int)
 
 
 def is_string(instance):
-    return isinstance(instance, str_types)
+    return isinstance(instance, str)
 
 
 def type_validator(validator, data_type, instance, schema):
@@ -107,7 +105,7 @@ def type_validator(validator, data_type, instance, schema):
         yield ValidationError("%r is not of type %s" % (instance, data_type))
 
 
-K8SValidator = extend(OAS30Validator, validators={
+K8SValidator = extend(OAS31Validator, validators={
     "type": type_validator,
     "required": required
 })
@@ -117,22 +115,22 @@ k8s_format_checker = FormatChecker()
 
 @k8s_format_checker.checks("int32")
 def check_int32(value):
-    return -2147483648 < value < 2147483647
+    return value is not None and (-2147483648 < value < 2147483647)
 
 
 @k8s_format_checker.checks("int64")
 def check_int64(value):
-    return -9223372036854775808 < value < 9223372036854775807
+    return value is not None and (-9223372036854775808 < value < 9223372036854775807)
 
 
 @k8s_format_checker.checks("float")
 def check_float(value):
-    return -3.4E+38 < value < +3.4E+38
+    return value is not None and (-3.4E+38 < value < +3.4E+38)
 
 
 @k8s_format_checker.checks("double")
 def check_double(value):
-    return -1.7E+308 < value < +1.7E+308
+    return value is not None and (-1.7E+308 < value < +1.7E+308)
 
 
 @k8s_format_checker.checks("byte", ValueError)
@@ -148,10 +146,6 @@ def check_int_or_string(value):
     return check_int32(value) if is_integer(value) else is_string(value)
 
 
-# def make_api_version(group, version):
-#    return f"{group}/{version}" if group else version
-
-
 def to_group_and_version(api_version):
     group, _, version = api_version.partition("/")
     if not version:
@@ -529,7 +523,7 @@ class K8SResourcePluginMixin:
             source = calling_frame_source()
 
         if isinstance(manifests, str):
-            manifests = list(yaml.safe_load_all(StringIO(manifests)))
+            manifests = list(parse_yaml_docs(manifests, source))
 
         if isinstance(manifests, (Mapping, dict)):
             return self.add_resource(manifests, source)
@@ -562,7 +556,7 @@ class K8SResourcePluginMixin:
             source = calling_frame_source()
 
         if isinstance(manifests, str):
-            manifests = list(yaml.safe_load_all(StringIO(manifests)))
+            manifests = list(parse_yaml_docs(manifests, source))
 
         if isinstance(manifests, (Mapping, dict)):
             return self.add_crd(manifests, source)
@@ -619,14 +613,19 @@ class K8SResourcePluginMixin:
 
     def _create_resource(self, manifest: dict, source: Union[str, Path] = None):
         resource_description = K8SResource.get_manifest_description(manifest, source)
-        self.logger.debug("Validating K8S manifest for %s", resource_description)
 
+        new_manifest = self._patch_manifest(manifest, resource_description)
+        if new_manifest != manifest:
+            manifest = new_manifest
+            resource_description = K8SResource.get_manifest_description(manifest, source)
+
+        self.logger.debug("Validating K8S manifest for %s", resource_description)
         errors = list(self._validate_resource(manifest, source))
         if errors:
             for error in errors:
-                if source:
-                    self.logger.error("Error detected in K8S manifest %s from %s",
-                                      resource_description, source, exc_info=error)
+                self.logger.error("Error detected in K8S manifest %s from %s: \n%s",
+                                  resource_description, source or "<unknown>", yaml.safe_dump(manifest, None),
+                                  exc_info=error)
             raise errors[0]
 
         rdef = self._get_manifest_rdef(manifest)
@@ -649,6 +648,11 @@ class K8SResourcePluginMixin:
 
         return resource
 
+    def _patch_manifest(self,
+                        manifest: dict,
+                        resource_description: str):
+        return manifest
+
     def _transform_resource(self,
                             resources: Sequence[K8SResource],
                             resource: K8SResource) -> K8SResource:
@@ -663,10 +667,8 @@ class K8SResourcePluginMixin:
                 yield error
             else:
                 rdef = error
-                # schema = ChainMap(manifest, self.resource_definitions_schema)
                 k8s_validator = K8SValidator(rdef.schema,
-                                             format_checker=k8s_format_checker,
-                                             resolver=RefResolver.from_schema(self.resource_definitions_schema))
+                                             format_checker=k8s_format_checker)
                 yield from k8s_validator.iter_errors(manifest)
 
     def _get_manifest_rdef(self, manifest):
@@ -744,6 +746,8 @@ class K8SResourcePluginMixin:
                 rdef_paths[path] = actions
 
         for k, schema in k8s_def["definitions"].items():
+            # This short-circuits the resolution of the references to the top of the document
+            schema["definitions"] = k8s_def["definitions"]
             for key in k8s_resource_def_key(schema):
                 for rdef in K8SResourceDef.from_manifest(key, schema, self.resource_paths):
                     self.resource_definitions[key] = rdef

kubernator/plugins/kops.py

@@ -30,7 +30,7 @@ from kubernator.api import (KubernatorPlugin, scan_dir,
                             io_StringIO,
                             TemplateEngine,
                             StripNL,
-                            Globs)
+                            Globs, load_file)
 from kubernator.plugins.k8s_api import K8SResourcePluginMixin
 from kubernator.proc import CalledProcessError
 
@@ -136,10 +136,11 @@ class KopsPlugin(KubernatorPlugin, K8SResourcePluginMixin):
             display_p = context.app.display_path(p)
             logger.debug("Adding Kops resources from %s", display_p)
 
-            with open(p, "rt") as file:
-                template = self.template_engine.from_string(file.read())
+            manifests = load_file(logger, p, FileType.YAML, display_p,
+                                  self.template_engine,
+                                  {"ktor": context})
 
-            self.add_resources(template.render({"ktor": context}), display_p)
+            self.add_resources(manifests, display_p)
 
     def update(self):
         context = self.context

kubernator/plugins/kubectl.py

@@ -23,6 +23,8 @@ import tempfile
 from pathlib import Path
 from shutil import which, copy
 
+import yaml
+
 from kubernator.api import (KubernatorPlugin,
                             prepend_os_path,
                             StripNL,
@@ -89,15 +91,44 @@ class KubectlPlugin(KubernatorPlugin):
         context.globals.kubectl = dict(version=version,
                                        kubectl_file=kubectl_file,
                                        stanza=self.stanza,
-                                       test=self.test_kubectl
+                                       test=self.test_kubectl,
+                                       run=self.run,
+                                       run_capturing=self.run_capturing,
+                                       get=self.get,
                                        )
 
         context.globals.kubectl.version = context.kubectl.test()
 
+    def run_capturing(self, *args, **kwargs):
+        return self.context.app.run_capturing_out(self.stanza() +
+                                                  list(args),
+                                                  stderr_logger,
+                                                  **kwargs
+                                                  )
+
+    def run(self, *args, **kwargs):
+        self.context.app.run(self.stanza() +
+                             list(args),
+                             stdout_logger,
+                             stderr_logger,
+                             **kwargs
+                             ).wait()
+
+    def get(self, resource_type, resource_name, namespace=None):
+        args = ["get", resource_type, resource_name]
+        if namespace:
+            args += ["-n", namespace]
+        args += ["-o", "yaml"]
+
+        res = list(yaml.safe_load_all(self.context.kubectl.run_capturing(*args)))
+        if len(res):
+            if len(res) > 1:
+                return res
+            return res[0]
+        return None
+
     def test_kubectl(self):
-        version_out: str = self.context.app.run_capturing_out(self.stanza() +
-                                                              ["version", "--client=true", "-o", "json"],
-                                                              stderr_logger)
+        version_out: str = self.run_capturing("version", "--client=true", "-o", "json")
 
         version_out_js = json.loads(version_out)
         kubectl_version = version_out_js["clientVersion"]["gitVersion"][1:]