kubernator 1.0.14__py3-none-any.whl → 1.0.24.dev20251109010128__py3-none-any.whl

kubernator/__init__.py

@@ -16,7 +16,7 @@
 #   limitations under the License.
 #
 
-__version__ = "1.0.14"
+__version__ = "1.0.24.dev20251109010128"
 
 
 def _main():

kubernator/api.py

@@ -23,6 +23,7 @@ import os
 import platform
 import re
 import sys
+import textwrap
 import traceback
 import urllib.parse
 from collections.abc import Callable
@@ -38,14 +39,16 @@ from typing import Optional, Union, MutableSequence
 
 import requests
 import yaml
-from appdirs import user_config_dir
 from diff_match_patch import diff_match_patch
+from gevent import sleep
 from jinja2 import (Environment,
                     ChainableUndefined,
                     make_logging_undefined,
                     Template as JinjaTemplate,
                     pass_context)
 from jsonschema import validators
+from platformdirs import user_cache_dir
+from yaml import MarkedYAMLError
 
 from kubernator._json_path import jp  # noqa: F401
 from kubernator._k8s_client_patches import (URLLIB_HEADERS_PATCH,
@@ -57,6 +60,98 @@ _CACHE_HEADER_TRANSLATION = {"etag": "if-none-match",
 _CACHE_HEADERS = ("etag", "last-modified")
 
 
+def to_json(obj: Union[dict, list]):
+    return json.dumps(obj)
+
+
+def to_yaml_str(s: str):
+    return yaml.safe_dump(s)
+
+
+def to_json_yaml_str(obj: Union[dict, list]):
+    """
+    Takes `obj`, dumps as json representation, converts json representation to YAML string literal.
+    """
+    return to_yaml_str(to_json(obj))
+
+
+def to_yaml_str_block(s: str, indent: int = 4, pretty_indent: int = 2):
+    """
+    Takes a multiline string, dedents it then indents it `indent` spaces for in-yaml alignment and
+    `pretty-indent` spaces for in-block alignment.
+    """
+    return (f"|+{pretty_indent}\n" +
+            textwrap.indent(textwrap.dedent(s), " " * (indent + pretty_indent)))
+
+
+def to_json_yaml_str_block(obj: Union[str, dict, list], indent: int = 4, pretty_indent=2):
+    """
+    Takes an `obj`, serializes it as pretty JSON with `pretty_indent` in-json indentation and then
+    passes it to `to_yaml_str_block`.
+    """
+    return to_yaml_str_block(json.dumps(obj, indent=pretty_indent), indent=indent, pretty_indent=pretty_indent)
+
+
+def to_yaml(obj: Union[dict, list], level_indent: int, indent: int):
+    s = yaml.safe_dump(obj, indent=indent)
+    return "\n" + textwrap.indent(s, " " * level_indent)
+
+
+class TemplateEngine:
+    VARIABLE_START_STRING = "{${"
+    VARIABLE_END_STRING = "}$}"
+
+    def __init__(self, logger):
+        self.template_failures = 0
+        self.templates = {}
+
+        class CollectingUndefined(ChainableUndefined):
+            __slots__ = ()
+
+            def __str__(self):
+                self.template_failures += 1
+                return super().__str__()
+
+        logging_undefined = make_logging_undefined(
+            logger=logger,
+            base=CollectingUndefined
+        )
+
+        @pass_context
+        def variable_finalizer(ctx, value):
+            normalized_value = str(value)
+            if self.VARIABLE_START_STRING in normalized_value and self.VARIABLE_END_STRING in normalized_value:
+                value_template_content = sys.intern(normalized_value)
+                env: Environment = ctx.environment
+                value_template = self.templates.get(value_template_content)
+                if not value_template:
+                    value_template = env.from_string(value_template_content, env.globals)
+                    self.templates[value_template_content] = value_template
+                return value_template.render(ctx.parent)
+
+            return normalized_value
+
+        self.env = Environment(variable_start_string=self.VARIABLE_START_STRING,
+                               variable_end_string=self.VARIABLE_END_STRING,
+                               autoescape=False,
+                               finalize=variable_finalizer,
+                               undefined=logging_undefined,
+                               )
+
+        self.env.filters["to_json"] = to_json
+        self.env.filters["to_yaml_str"] = to_yaml_str
+        self.env.filters["to_yaml"] = to_yaml
+        self.env.filters["to_yaml_str_block"] = to_yaml_str_block
+        self.env.filters["to_json_yaml_str_block"] = to_json_yaml_str_block
+        self.env.filters["to_json_yaml_str"] = to_json_yaml_str
+
+    def from_string(self, template):
+        return self.env.from_string(template)
+
+    def failures(self):
+        return self.template_failures
+
+
 def calling_frame_source(depth=2):
     f = traceback.extract_stack(limit=depth + 1)[0]
     return f"file {f.filename}, line {f.lineno} in {f.name}"
@@ -87,18 +182,33 @@ def scan_dir(logger, path: Path, path_filter: Callable[[os.DirEntry], bool], exc
                 yield path / f
 
 
+def parse_yaml_docs(document: str, source=None):
+    try:
+        return list(d for d in yaml.safe_load_all(document) if d)
+    except MarkedYAMLError:
+        raise
+
+
 class FileType(Enum):
-    JSON = (json.load,)
-    YAML = (yaml.safe_load_all,)
+    TEXT = (lambda x: x,)
+    BINARY = (lambda x: x,)
+    JSON = (json.loads,)
+    YAML = (parse_yaml_docs,)
 
     def __init__(self, func):
         self.func = func
 
 
-def _load_file(logger, path: Path, file_type: FileType, source=None) -> Iterable[dict]:
-    with open(path, "rb") as f:
+def _load_file(logger, path: Path, file_type: FileType, source=None,
+               template_engine: Optional[TemplateEngine] = None,
+               template_context: Optional[dict] = None) -> Iterable[dict]:
+    with open(path, "rb" if file_type == FileType.BINARY else "rt") as f:
         try:
-            data = file_type.func(f)
+            if template_engine and not file_type == FileType.BINARY:
+                raw_data = template_engine.from_string(f.read()).render(template_context)
+            else:
+                raw_data = f.read()
+            data = file_type.func(raw_data)
             if isinstance(data, GeneratorType):
                 data = list(data)
             return data
@@ -108,27 +218,43 @@ def _load_file(logger, path: Path, file_type: FileType, source=None) -> Iterable
 
 
 def _download_remote_file(url, file_name, cache: dict):
-    with requests.get(url, headers=cache, stream=True) as r:
-        r.raise_for_status()
-        if r.status_code != 304:
-            with open(file_name, "wb") as out:
-                for chunk in r.iter_content(chunk_size=65535):
-                    out.write(chunk)
-            return dict(r.headers)
+    retry_delay = 0
+    while True:
+        if retry_delay:
+            sleep(retry_delay)
+
+        with requests.get(url, headers=cache, stream=True) as r:
+            if r.status_code == 429:
+                if not retry_delay:
+                    retry_delay = 0.2
+                else:
+                    retry_delay *= 2.0
+                if retry_delay > 2.5:
+                    retry_delay = 2.5
+                continue
+
+            r.raise_for_status()
+            if r.status_code != 304:
+                with open(file_name, "wb") as out:
+                    for chunk in r.iter_content(chunk_size=65535):
+                        out.write(chunk)
+                return dict(r.headers)
+            else:
+                return None
 
 
 def get_app_cache_dir():
-    return Path(user_config_dir("kubernator"))
+    return Path(user_cache_dir("kubernator", "karellen"))
 
 
 def get_cache_dir(category: str, sub_category: str = None):
-    config_dir = get_app_cache_dir() / category
+    cache_dir = get_app_cache_dir() / category
     if sub_category:
-        config_dir = config_dir / sub_category
-    if not config_dir.exists():
-        config_dir.mkdir(parents=True)
+        cache_dir = cache_dir / sub_category
+    if not cache_dir.exists():
+        cache_dir.mkdir(parents=True)
 
-    return config_dir
+    return cache_dir
 
 
 def download_remote_file(logger, url: str, category: str = "k8s", sub_category: str = None,
@@ -174,9 +300,12 @@ def load_remote_file(logger, url, file_type: FileType, category: str = "k8s", su
     return _load_file(logger, file_name, file_type, url)
 
 
-def load_file(logger, path: Path, file_type: FileType, source=None) -> Iterable[dict]:
+def load_file(logger, path: Path, file_type: FileType, source=None,
+              template_engine: Optional[TemplateEngine] = None,
+              template_context: Optional[dict] = None) -> Iterable[dict]:
     logger.debug("Loading %s using %s", source or path, file_type.name)
-    return _load_file(logger, path, file_type)
+    return _load_file(logger, path, file_type,
+                      source, template_engine, template_context)
 
 
 def validator_with_defaults(validator_class):
@@ -496,53 +625,6 @@ class Globs(MutableSet[Union[str, re.Pattern]]):
         return f"Globs[{self._list}]"
 
 
-class TemplateEngine:
-    VARIABLE_START_STRING = "{${"
-    VARIABLE_END_STRING = "}$}"
-
-    def __init__(self, logger):
-        self.template_failures = 0
-        self.templates = {}
-
-        class CollectingUndefined(ChainableUndefined):
-            __slots__ = ()
-
-            def __str__(self):
-                self.template_failures += 1
-                return super().__str__()
-
-        logging_undefined = make_logging_undefined(
-            logger=logger,
-            base=CollectingUndefined
-        )
-
-        @pass_context
-        def variable_finalizer(ctx, value):
-            normalized_value = str(value)
-            if self.VARIABLE_START_STRING in normalized_value and self.VARIABLE_END_STRING in normalized_value:
-                value_template_content = sys.intern(normalized_value)
-                env: Environment = ctx.environment
-                value_template = self.templates.get(value_template_content)
-                if not value_template:
-                    value_template = env.from_string(value_template_content, env.globals)
-                    self.templates[value_template_content] = value_template
-                return value_template.render(ctx.parent)
-
-            return normalized_value
-
-        self.env = Environment(variable_start_string=self.VARIABLE_START_STRING,
-                               variable_end_string=self.VARIABLE_END_STRING,
-                               autoescape=False,
-                               finalize=variable_finalizer,
-                               undefined=logging_undefined)
-
-    def from_string(self, template):
-        return self.env.from_string(template)
-
-    def failures(self):
-        return self.template_failures
-
-
 class Template:
     def __init__(self, name: str, template: JinjaTemplate, defaults: dict = None, path=None, source=None):
         self.name = name
@@ -747,8 +829,12 @@ class KubernatorPlugin:
     def handle_shutdown(self):
         pass
 
+    def handle_summary(self):
+        pass
+
 
-def install_python_k8s_client(run, package_major, logger, logger_stdout, logger_stderr, disable_patching):
+def install_python_k8s_client(run, package_major, logger, logger_stdout, logger_stderr, disable_patching,
+                              fallback=False):
     cache_dir = get_cache_dir("python")
     package_major_dir = cache_dir / str(package_major)
     package_major_dir_str = str(package_major_dir)
@@ -760,13 +846,43 @@ def install_python_k8s_client(run, package_major, logger, logger_stdout, logger_
                     str(package_major), package_major_dir)
         rmtree(package_major_dir)
 
-    if not package_major_dir.exists():
+    if not package_major_dir.exists() or not len(os.listdir(package_major_dir)):
         package_major_dir.mkdir(parents=True, exist_ok=True)
-        run([sys.executable, "-m", "pip", "install", "--no-deps", "--no-input", "--pre",
-             "--root-user-action=ignore", "--break-system-packages", "--disable-pip-version-check",
-             "--target", package_major_dir_str, f"kubernetes~={package_major}.0"], logger_stdout, logger_stderr).wait()
+        try:
+            run([sys.executable, "-m", "pip", "install", "--no-deps", "--no-input",
+                 "--root-user-action=ignore", "--break-system-packages", "--disable-pip-version-check",
+                 "--target", package_major_dir_str, f"kubernetes>={package_major!s}dev0,<{int(package_major) + 1!s}"],
+                logger_stdout, logger_stderr)
+        except CalledProcessError as e:
+            if not fallback and "No matching distribution found for" in e.stderr:
+                logger.warning("Kubernetes Client %s (%s) failed to install because the version wasn't found. "
+                               "Falling back to a client of the previous version - %s",
+                               str(package_major), package_major_dir, int(package_major) - 1)
+                return install_python_k8s_client(run,
+                                                 int(package_major) - 1,
+                                                 logger,
+                                                 logger_stdout,
+                                                 logger_stderr,
+                                                 disable_patching,
+                                                 fallback=True)
+            else:
+                raise
 
     if not patch_indicator.exists() and not disable_patching:
+        if not fallback and not len(os.listdir(package_major_dir)):
+            # Directory is empty
+            logger.warning("Kubernetes Client %s (%s) directory is empty - the client was not installed. "
+                           "Falling back to a client of the previous version - %s",
+                           str(package_major), package_major_dir, int(package_major) - 1)
+
+            return install_python_k8s_client(run,
+                                             int(package_major) - 1,
+                                             logger,
+                                             logger_stdout,
+                                             logger_stderr,
+                                             disable_patching,
+                                             fallback=True)
+
         for patch_text, target_file, skip_if_found, min_version, max_version, name in (
                 URLLIB_HEADERS_PATCH, CUSTOM_OBJECT_PATCH_23, CUSTOM_OBJECT_PATCH_25):
             patch_target = package_major_dir / target_file

kubernator/app.py

@@ -29,11 +29,13 @@ from pathlib import Path
 from shutil import rmtree
 from typing import Optional, Union
 
+import yaml
+
 import kubernator
 from kubernator.api import (KubernatorPlugin, Globs, scan_dir, PropertyDict, config_as_dict, config_parent,
                             download_remote_file, load_remote_file, Repository, StripNL, jp, get_app_cache_dir,
                             get_cache_dir, install_python_k8s_client)
-from kubernator.proc import run, run_capturing_out
+from kubernator.proc import run, run_capturing_out, run_pass_through_capturing
 
 TRACE = 5
 
@@ -55,6 +57,11 @@ logging.addLevelName(5, "TRACE")
 logging.Logger.trace = trace
 logger = logging.getLogger("kubernator")
 
+try:
+    del (yaml.resolver.Resolver.yaml_implicit_resolvers["="])
+except KeyError:
+    pass
+
 
 def define_arg_parse():
     parser = argparse.ArgumentParser(prog="kubernator",
@@ -171,42 +178,49 @@ class App(KubernatorPlugin):
         self.register_plugin(self)
 
         try:
-            while True:
-                cwd = self.next()
-                if not cwd:
-                    logger.debug("No paths left to traverse")
-                    break
+            try:
+                while True:
+                    cwd = self.next()
+                    if not cwd:
+                        logger.debug("No paths left to traverse")
+                        break
 
-                context = self.context
+                    context = self.context
 
-                logger.debug("Inspecting directory %s", self._display_path(cwd))
-                self._run_handlers(KubernatorPlugin.handle_before_dir, False, context, None, cwd)
+                    logger.debug("Inspecting directory %s", self._display_path(cwd))
+                    self._run_handlers(KubernatorPlugin.handle_before_dir, False, context, None, cwd)
 
-                if (ktor_py := (cwd / ".kubernator.py")).exists():
-                    self._run_handlers(KubernatorPlugin.handle_before_script, False, context, None, cwd)
+                    if (ktor_py := (cwd / ".kubernator.py")).exists():
+                        self._run_handlers(KubernatorPlugin.handle_before_script, False, context, None, cwd)
 
-                    for h in self.context._plugins:
-                        h.set_context(context)
+                        for h in self.context._plugins:
+                            h.set_context(context)
 
-                    self._exec_ktor(ktor_py)
+                        self._exec_ktor(ktor_py)
 
-                    for h in self.context._plugins:
-                        h.set_context(None)
+                        for h in self.context._plugins:
+                            h.set_context(None)
 
-                    self._run_handlers(KubernatorPlugin.handle_after_script, True, context, None, cwd)
+                        self._run_handlers(KubernatorPlugin.handle_after_script, True, context, None, cwd)
 
-                self._run_handlers(KubernatorPlugin.handle_after_dir, True, context, None, cwd)
+                    self._run_handlers(KubernatorPlugin.handle_after_dir, True, context, None, cwd)
 
-            self.context = self._top_dir_context
-            context = self.context
+                self.context = self._top_dir_context
+                context = self.context
 
-            self._run_handlers(KubernatorPlugin.handle_apply, True, context, None)
+                self._run_handlers(KubernatorPlugin.handle_apply, True, context, None)
 
-            self._run_handlers(KubernatorPlugin.handle_verify, True, context, None)
-        finally:
+                self._run_handlers(KubernatorPlugin.handle_verify, True, context, None)
+            finally:
+                self.context = self._top_dir_context
+                context = self.context
+                self._run_handlers(KubernatorPlugin.handle_shutdown, True, context, None)
+        except:  # noqa E722
+            raise
+        else:
             self.context = self._top_dir_context
             context = self.context
-            self._run_handlers(KubernatorPlugin.handle_shutdown, True, context, None)
+            self._run_handlers(KubernatorPlugin.handle_summary, True, context, None)
 
     def discover_plugins(self):
         importlib.invalidate_caches()
@@ -334,6 +348,7 @@ class App(KubernatorPlugin):
                                    jp=jp,
                                    run=self._run,
                                    run_capturing_out=self._run_capturing_out,
+                                   run_passthrough_capturing=self._run_passthrough_capturing,
                                    repository=self.repository,
                                    StripNL=StripNL,
                                    default_includes=Globs(["*"], True),
@@ -451,6 +466,9 @@ class App(KubernatorPlugin):
     def _run_capturing_out(self, *args, **kwargs):
         return run_capturing_out(*args, **kwargs)
 
+    def _run_passthrough_capturing(self, *args, **kwargs):
+        return run_pass_through_capturing(*args, **kwargs)
+
     def __repr__(self):
         return "Kubernator"
 
@@ -479,7 +497,7 @@ def pre_cache_k8s_clients(*versions, disable_patching=False):
     for v in versions:
         logger.info("Caching K8S client library ~=v%s.0%s...", v,
                     " (no patches)" if disable_patching else "")
-        install_python_k8s_client(run, v, logger, stdout_logger, stderr_logger, disable_patching)
+        install_python_k8s_client(run_pass_through_capturing, v, logger, stdout_logger, stderr_logger, disable_patching)
 
 
 def main():

kubernator/plugins/gke.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+#
+#   Copyright 2020 Express Systems USA, Inc
+#   Copyright 2025 Karellen, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import logging
+import os
+import tempfile
+from pathlib import Path
+from shutil import which
+
+from kubernator.api import (KubernatorPlugin,
+                            StripNL
+                            )
+
+logger = logging.getLogger("kubernator.gke")
+proc_logger = logger.getChild("proc")
+stdout_logger = StripNL(proc_logger.info)
+stderr_logger = StripNL(proc_logger.warning)
+
+
+class GkePlugin(KubernatorPlugin):
+    logger = logger
+
+    _name = "gke"
+
+    def __init__(self):
+        self.context = None
+        self.kubeconfig_dir = tempfile.TemporaryDirectory()
+        self.name = None
+        self.region = None
+        self.project = None
+        self.gcloud_file = None
+
+        super().__init__()
+
+    def set_context(self, context):
+        self.context = context
+
+    def register(self, *, name=None, region=None, project=None):
+        context = self.context
+
+        if not name:
+            raise ValueError("`name` is required")
+        if not region:
+            raise ValueError("`region` is required")
+        if not project:
+            raise ValueError("`project` is required")
+
+        self.name = name
+        self.region = region
+        self.project = project
+
+        # Use current version
+        gcloud_file = which("gcloud")
+        if not gcloud_file:
+            raise RuntimeError("`gcloud` cannot be found")
+        logger.debug("Found gcloud in %r", gcloud_file)
+        self.gcloud_file = gcloud_file
+
+        context.app.register_plugin("kubeconfig")
+
+        context.globals.gke = dict(kubeconfig=str(Path(self.kubeconfig_dir.name) / "config"),
+                                   name=name,
+                                   region=region,
+                                   project=project,
+                                   gcloud_file=gcloud_file
+                                   )
+
+    def handle_init(self):
+        context = self.context
+
+        env = dict(os.environ)
+        env["KUBECONFIG"] = str(context.gke.kubeconfig)
+        self.context.app.run([context.gke.gcloud_file, "components", "install", "gke-gcloud-auth-plugin"],
+                             stdout_logger,
+                             stderr_logger).wait()
+        self.context.app.run([context.gke.gcloud_file, "container", "clusters", "get-credentials",
+                              context.gke.name,
+                              "--region", context.gke.region,
+                              "--project", context.gke.project],
+                             stdout_logger,
+                             stderr_logger,
+                             env=env).wait()
+
+        context.kubeconfig.set(context.gke.kubeconfig)
+
+    def handle_shutdown(self):
+        self.kubeconfig_dir.cleanup()
+
+    def __repr__(self):
+        return "GKE Plugin"