nyl 0.0.2__tar.gz → 0.0.4__tar.gz

nyl-0.0.4/PKG-INFO

@@ -0,0 +1,31 @@
+Metadata-Version: 2.1
+Name: nyl
+Version: 0.0.4
+Summary: Default template for PDM package
+Author-Email: Niklas Rosenstein <rosensteinniklas@gmail.com>
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: bcrypt>=4.2.0
+Requires-Dist: databind>=4.5.2
+Requires-Dist: filelock>=3.15.4
+Requires-Dist: jinja2>=3.1.4
+Requires-Dist: kubernetes>=30.1.0
+Requires-Dist: loguru>=0.7.2
+Requires-Dist: nr-stream>=1.1.5
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: requests>=2.32.3
+Requires-Dist: stablehash>=0.2.0
+Requires-Dist: structured-templates<0.1.0,>=0.0.5
+Requires-Dist: typer>=0.12.3
+Requires-Dist: typing-extensions>=4.12.2
+Description-Content-Type: text/markdown
+
+# nyl
+
+Nyl is a versatile tool for generating Kubernetes manifests from a simple YAML configuration, encouraging
+consistent and reusable deployment configurations, project layouts and operational workflows.
+
+## Local development
+
+This project uses the [PDM](https://pdm-project.org/latest/) package manager for Python. Start with
+`$ pipx install pdm` followed by `$ pdm install --frozen-lockfile`.

nyl-0.0.4/README.md

@@ -0,0 +1,9 @@
+# nyl
+
+Nyl is a versatile tool for generating Kubernetes manifests from a simple YAML configuration, encouraging
+consistent and reusable deployment configurations, project layouts and operational workflows.
+
+## Local development
+
+This project uses the [PDM](https://pdm-project.org/latest/) package manager for Python. Start with
+`$ pipx install pdm` followed by `$ pdm install --frozen-lockfile`.

{nyl-0.0.2 → nyl-0.0.4}/pyproject.toml

@@ -1,21 +1,24 @@
 [project]
 name = "nyl"
-version = "0.0.2"
+version = "0.0.4"
 description = "Default template for PDM package"
 authors = [
     { name = "Niklas Rosenstein", email = "rosensteinniklas@gmail.com" },
 ]
 dependencies = [
+    "bcrypt>=4.2.0",
     "databind>=4.5.2",
-    "pyyaml>=6.0.1",
+    "filelock>=3.15.4",
     "jinja2>=3.1.4",
-    "loguru>=0.7.2",
     "kubernetes>=30.1.0",
-    "typer>=0.12.3",
-    "filelock>=3.15.4",
-    "stablehash>=0.2.0",
+    "loguru>=0.7.2",
     "nr-stream>=1.1.5",
+    "pyyaml>=6.0.1",
     "requests>=2.32.3",
+    "stablehash>=0.2.0",
+    "structured-templates>=0.0.5,<0.1.0",
+    "typer>=0.12.3",
+    "typing-extensions>=4.12.2",
 ]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -34,17 +37,24 @@ build-backend = "pdm.backend"
 
 [tool.pdm]
 distribution = true
+source = [
+    { name = "pypi", url = "https://pypi.org/simple" },
+]
+
+[tool.pdm.scripts]
+lint = "bash -c 'ruff format && ruff check --fix && mypy src' && pytest"
+docs = "bash -c 'cd docs && mkdocs serve'"
 
 [tool.pdm.dev-dependencies]
 dev = [
-    "ruff>=0.4.8",
-    "mypy>=1.10.0",
-    "types-pyyaml>=6.0.12.20240311",
-    "mkdocs>=1.6.0",
+    "kubernetes-stubs>=22.6.0.post1",
     "mkdocs-material>=9.5.28",
+    "mkdocs>=1.6.0",
     "mkdocstrings[python]>=0.25.1",
+    "mypy>=1.10.0",
     "pytest>=8.2.2",
-    "kubernetes-stubs>=22.6.0.post1",
+    "ruff>=0.4.8",
+    "types-pyyaml>=6.0.12.20240311",
     "types-requests>=2.32.0.20240712",
 ]
 

nyl-0.0.4/src/nyl/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+__version__ = version("nyl")

nyl-0.0.4/src/nyl/commands/__init__.py

@@ -0,0 +1,68 @@
+"""
+Nyl is a flexible configuration management tool for Kubernetes resources that can be used to generate and deploy
+applications directly or integrate as an ArgoCD ConfigManagementPlugin.
+"""
+
+from pathlib import Path
+from nyl import __version__
+from enum import Enum
+import sys
+from loguru import logger
+from typer import Option
+from nyl.tools.typer import new_typer
+
+
+app = new_typer(help=__doc__)
+
+
+from . import crds  # noqa: F401,E402
+from . import new  # noqa: E402
+from . import profile  # noqa: E402
+from . import secrets  # noqa: E402
+from . import template  # noqa: F401,E402
+from . import tun  # noqa: E402
+
+app.add_typer(new.app)
+app.add_typer(profile.app)
+app.add_typer(secrets.app)
+app.add_typer(tun.app)
+
+
+LOG_TIME_FORMAT = "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green>"
+LOG_LEVEL_FORAMT = "<level>{level: <8}</level>"
+LOG_DETAILS_FORMAT = "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan>"
+LOG_MESSAGE_FORMAT = "<level>{message}</level>"
+
+
+class LogLevel(str, Enum):
+    TRACE = "trace"
+    DEBUG = "debug"
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+    CRITICAL = "critical"
+
+
+@app.callback()
+def _callback(
+    log_level: LogLevel = Option(LogLevel.INFO, "--log-level", "-l", help="The log level to use."),
+    log_details: bool = Option(False, help="Include logger- and function names in the log message format."),
+) -> None:
+    if log_details:
+        fmt = f"{LOG_TIME_FORMAT} | {LOG_LEVEL_FORAMT} | {LOG_DETAILS_FORMAT} | {LOG_MESSAGE_FORMAT}"
+    else:
+        fmt = f"{LOG_TIME_FORMAT} | {LOG_LEVEL_FORAMT} | {LOG_MESSAGE_FORMAT}"
+
+    logger.remove()
+    logger.add(sys.stderr, level=log_level.name, format=fmt)
+    logger.opt(ansi=True).debug("Nyl v{} run from <yellow>{}</>.", __version__, Path.cwd())
+
+
+@app.command()
+def version() -> None:
+    """
+    Print the version of Nyl.
+    """
+
+    print(f"Nyl v{__version__}")
+    sys.exit(0)

nyl-0.0.4/src/nyl/commands/crds.py

@@ -0,0 +1,14 @@
+import yaml
+
+from nyl.resources.applyset import ApplySet
+from . import app
+
+
+@app.command()
+def crds() -> None:
+    """
+    Print out the CRDs that need to be installed on a Kubernetes cluster before you can use all of Nyl's features.
+    """
+
+    print("---")
+    print(yaml.safe_dump(ApplySet.CRD))

nyl-0.0.4/src/nyl/commands/new.py

@@ -0,0 +1,91 @@
+"""
+Bootstrap files for a new Nyl project or Helm chart.
+"""
+
+from pathlib import Path
+from textwrap import dedent
+
+from loguru import logger
+from nyl.tools.typer import new_typer
+
+app = new_typer(name="new", help=__doc__)
+
+
+def _write_file_dedent(dir: Path, name: str, content: str) -> None:
+    path = dir.joinpath(name)
+    if path.exists():
+        logger.warning("File already exists: {}", path)
+        return
+    logger.info("Writing {}", path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(dedent(content).lstrip())
+
+
+@app.command()
+def chart(dir: Path) -> None:
+    """
+    Similar to `helm create`, but generates a much simpler template.
+    """
+
+    dir.mkdir(parents=True, exist_ok=True)
+
+    _write_file_dedent(
+        dir,
+        "Chart.yaml",
+        f"""
+        apiVersion: v2
+        name: {dir.name}
+        version: '0.0.0'
+        """,
+    )
+
+    _write_file_dedent(
+        dir,
+        "values.yaml",
+        """
+        image: my/image:tag
+        """,
+    )
+
+    _write_file_dedent(
+        dir,
+        "values.schema.json",
+        """
+        {
+            "$schema": "https://json-schema.org/draft/2020-12/output/schema",
+            "type": "object",
+            "required": [
+                "image"
+            ],
+            "properties": {
+                "image": {
+                    "type": "string"
+                }
+            }
+        }
+        """,
+    )
+
+    _write_file_dedent(
+        dir,
+        "templates/deployment.yaml",
+        """
+        apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+            name: {{ .Release.Name }}
+        spec:
+            replicas: 1
+            selector:
+            matchLabels:
+                app: {{ .Release.Name }}
+            template:
+            metadata:
+                labels:
+                app: {{ .Release.Name }}
+            spec:
+                containers:
+                - name: {{ .Release.Name }}
+                image: {{ .Values.image }}
+        """,
+    )

{nyl-0.0.2 → nyl-0.0.4}/src/nyl/commands/profile.py

@@ -23,8 +23,5 @@ def activate(profile_name: str = Argument("default", envvar="NYL_PROFILE")) -> N
     with ProfileManager.load() as manager:
         profile = manager.activate_profile(profile_name)
 
-    # For standard tooling, like kubectl.
-    print(f"export KUBECONFIG={shlex.quote(str(profile.kubeconfig.absolute()))}")
-
-    # Used by Terraform providers.
-    print(f"export KUBE_CONFIG_PATH={shlex.quote(str(profile.kubeconfig.absolute()))}")
+    for key, value in profile.env.items():
+        print(f"export {key}={shlex.quote(value)}")

{nyl-0.0.2 → nyl-0.0.4}/src/nyl/commands/secrets.py

@@ -16,7 +16,7 @@ def list() -> None:
     List the keys for all secrets in the provider.
     """
 
-    secrets = SecretsConfig.load(SecretsConfig.find_config_file())
+    secrets = SecretsConfig.load()
     for key in secrets.provider.keys():
         print(key)
 
@@ -27,5 +27,5 @@ def get(key: str, pretty: bool = False) -> None:
     Get the value of a secret as JSON.
     """
 
-    secrets = SecretsConfig.load(SecretsConfig.find_config_file())
+    secrets = SecretsConfig.load()
     print(json.dumps(secrets.provider.get(key), indent=4 if pretty else None))

nyl-0.0.4/src/nyl/commands/template.py

@@ -0,0 +1,349 @@
+import atexit
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, cast
+from loguru import logger
+from structured_templates import TemplateEngine
+from typer import Argument, Option
+import yaml
+from nyl.generator import reconcile_generator
+from nyl.generator.dispatch import DispatchingGenerator
+from nyl.profiles import ProfileManager
+from kubernetes.config.incluster_config import load_incluster_config
+from kubernetes.config.kube_config import load_kube_config
+from kubernetes.client.api_client import ApiClient
+from nyl.project.config import ProjectConfig
+from nyl.resources.applyset import APPLYSET_LABEL_PART_OF, ApplySet
+from nyl.secrets.config import SecretsConfig
+from nyl.tools.kubectl import Kubectl
+from nyl.tools.types import Manifest, Manifests
+
+from . import app
+
+DEFAULT_PROFILE = "default"
+
+
+@dataclass
+class ManifestsWithSource:
+    """
+    Represents a list of manifests loaded from a particular source file.
+    """
+
+    manifests: Manifests
+    file: Path
+
+
+@app.command()
+def template(
+    paths: list[Path] = Argument(..., help="The YAML file(s) to render. Can be a directory."),
+    profile: Optional[str] = Option(None, envvar="NYL_PROFILE", help="The Nyl profile to use."),
+    in_cluster: bool = Option(
+        False, help="Use the in-cluster Kubernetes configuration. The --profile option is ignored."
+    ),
+    apply: bool = Option(
+        False,
+        help="Run `kubectl apply` on the rendered manifests, once for each source file. "
+        "Implies `--no-applyset-part-of`. When an ApplySet is defined in the source file, it will be applied "
+        "separately. Note that this option implies `kubectl --prune`.",
+    ),
+    applyset_part_of: bool = Option(
+        True,
+        help="Add the 'applyset.kubernetes.io/part-of' label to all resources belonging to an ApplySet (if declared). "
+        "This option must be disabled when passing the generated manifests to `kubectl apply --applyset=...`, as it "
+        "would otherwise cause an error due to the label being present on the input data.",
+    ),
+) -> None:
+    """
+    Render a package template into full Kubernetes resources.
+    """
+
+    if apply:
+        # When running with --apply, we must ensure that the --applyset-part-of option is disabled, as it would cause
+        # an error when passing the generated manifests to `kubectl apply --applyset=...`.
+        applyset_part_of = False
+
+    kubectl = Kubectl()
+    kubectl.env["KUBECTL_APPLYSET"] = "true"
+    atexit.register(kubectl.cleanup)
+
+    if in_cluster:
+        logger.info("Using in-cluster configuration.")
+        load_incluster_config()
+    else:
+        with ProfileManager.load(required=False) as profiles:
+            # If no profile to activate is specified, and there are no profiles defined, we're not activating a
+            # a profile. It should be valid to use Nyl without a `nyl-profiles.yaml` file.
+            if profile is not None or profiles.config.profiles:
+                profile = profile or DEFAULT_PROFILE
+                active = profiles.activate_profile(profile)
+                load_kube_config(str(active.kubeconfig))
+            else:
+                logger.opt(ansi=True).info(
+                    "No <yellow>nyl-profiles.yaml</> file found, using default kubeconfig and context."
+                )
+                load_kube_config()
+
+    project = ProjectConfig.load()
+    if project.file:
+        state_dir = project.file.parent / ".nyl"
+    else:
+        state_dir = Path(".nyl")
+
+    secrets = SecretsConfig.load()
+
+    template_engine = TemplateEngine(
+        globals_={
+            "secrets": secrets.provider,
+            "random_password": _random_password,
+            "bcrypt": _bcrypt,
+        }
+    )
+
+    generator = DispatchingGenerator.default(
+        git_repo_cache_dir=state_dir / "repo-cache",
+        search_path=project.config.search_path,
+        working_dir=Path.cwd(),
+        client=ApiClient(),
+    )
+
+    for source in load_manifests(paths):
+        logger.opt(ansi=True).info("Rendering manifests from <blue>{}</>.", source.file)
+
+        source.manifests = cast(Manifests, template_engine.evaluate(source.manifests))
+        source.manifests = reconcile_generator(
+            generator,
+            source.manifests,
+            on_generated=lambda m: cast(Manifest, template_engine.evaluate(m)),
+        )
+
+        # Find the namespaces that are defined in the file. If we find any manifests without a namespace, we will
+        # inject that namespace name into them.
+        # Also find the applyset defined in the file.
+        namespaces: set[str] = set()
+        applyset: ApplySet | None = None
+        for manifest in list(source.manifests):
+            if is_namespace_resource(manifest):
+                namespaces.add(manifest["metadata"]["name"])
+            elif ApplySet.matches(manifest):
+                if applyset is not None:
+                    logger.opt(ansi=True).error(
+                        "Multiple ApplySet resources defined in <yellow>{}</>, there can only be one per source.",
+                        source.file,
+                    )
+                    exit(1)
+                applyset = ApplySet.load(manifest)
+                source.manifests.remove(manifest)
+
+        if not applyset and project.config.automatic_applyset:
+            if len(namespaces) > 1:
+                logger.opt(ansi=True).error(
+                    "Multiple namespaces defined in <yellow>{}</>, but automatic ApplySet generation is enabled. "
+                    "There can only be one namespace per source in this case.",
+                    source.file,
+                )
+                exit(1)
+            elif len(namespaces) == 1:
+                applyset_name = next(iter(namespaces))
+            else:
+                applyset_name = source.file.stem
+
+            logger.opt(ansi=True).info(
+                "Automatically creating ApplySet for <blue>{}</> (name: <magenta>{}</>).", source.file, applyset_name
+            )
+            applyset = ApplySet.new(applyset_name)
+
+        if applyset is not None:
+            applyset.set_group_kinds(source.manifests)
+            # HACK: Kubectl 1.30 can't create the custom resource, so we need to create it. But it will also reject
+            #       using the custom resource unless it has the tooling label set appropriately. For more details, see
+            #       https://github.com/NiklasRosenstein/nyl/issues/5.
+            applyset.tooling = "kubectl/1.30"
+            applyset.validate()
+
+            if apply:
+                # We need to ensure that ApplySet parent object exists before invoking `kubectl apply --applyset=...`.
+                logger.opt(ansi=True).info(
+                    "Kubectl-apply ApplySet resource <yellow>{}</> from <cyan>{}</>.",
+                    applyset.reference,
+                    source.file,
+                )
+                kubectl.apply(Manifests([applyset.dump()]), force_conflicts=True)
+            else:
+                print("---")
+                print(yaml.safe_dump(applyset.dump()))
+
+        # Find all manifests without a namespace and inject the namespace name into them.
+        # If there is an applyset, ensure they are marked as part of the applyset.
+        for manifest in source.manifests:
+            if applyset is not None and applyset_part_of:
+                if APPLYSET_LABEL_PART_OF not in (labels := manifest["metadata"].setdefault("labels", {})):
+                    labels[APPLYSET_LABEL_PART_OF] = applyset.id
+
+            if not is_cluster_scoped_resource(manifest) and "namespace" not in manifest["metadata"]:
+                if len(namespaces) > 1:
+                    logger.error(
+                        "Multiple namespaces defined in '{}', but manifest {}/{} does not have a namespace.",
+                        source.file,
+                        manifest["kind"],
+                        manifest["metadata"]["name"],
+                    )
+                    exit(1)
+                elif len(namespaces) == 0:
+                    logger.warning(
+                        "No namespaces defined in '{}', injecting 'default' into manifest {}/{}.",
+                        source.file,
+                        manifest["kind"],
+                        manifest["metadata"]["name"],
+                    )
+                    manifest["metadata"]["namespace"] = "default"
+                else:
+                    logger.trace(
+                        "Injecting namespace '{}' into manifest {}/{}.",
+                        next(iter(namespaces)),
+                        manifest["kind"],
+                        manifest["metadata"]["name"],
+                    )
+                    manifest["metadata"]["namespace"] = next(iter(namespaces))
+            elif not is_cluster_scoped_resource(manifest) and manifest["metadata"]["namespace"] not in namespaces:
+                logger.warning(
+                    "Resource '{}/{}' in '{}' references namespace '{}', which is not defined in the file.",
+                    manifest["kind"],
+                    manifest["metadata"]["name"],
+                    source.file,
+                    manifest["metadata"]["namespace"],
+                )
+
+            if not apply:
+                print("---")
+                print(yaml.safe_dump(manifest))
+
+        if apply:
+            logger.info("Kubectl-apply {} manifest(s) from '{}'", len(source.manifests), source.file)
+            kubectl.apply(
+                manifests=source.manifests,
+                applyset=applyset.reference if applyset else None,
+                prune=True if applyset else False,
+                force_conflicts=True,
+            )
+
+
+def load_manifests(paths: list[Path]) -> list[ManifestsWithSource]:
+    """
+    Load all manifests from a directory.
+    """
+
+    logger.trace("Loading manifests from paths: {}", paths)
+
+    files = []
+    for path in paths:
+        if path.is_dir():
+            for item in path.iterdir():
+                if (
+                    item.name.startswith("nyl-")
+                    or item.name.startswith(".")
+                    or item.name.startswith("_")
+                    or item.suffix != ".yaml"
+                    or not item.is_file()
+                ):
+                    continue
+                files.append(item)
+        else:
+            files.append(path)
+
+    logger.trace("Files to load: {}", files)
+
+    result = []
+    for file in files:
+        manifests = Manifests(list(map(Manifest, yaml.safe_load_all(file.read_text()))))
+        result.append(ManifestsWithSource(manifests, file))
+
+    return result
+
+    #     # Check if the resource has any references and try to resolve them. If a reference cannot be resolved, the
+    #     # manifest must be skipped. We emit a warning and continue with the next manifest.
+    #     refs = list(Reference.collect(manifest))
+    #     if refs:
+    #         skip_resource = False
+    #         resolves = {}
+    #         for ref in refs:
+    #             # TODO: Determine the namespace to fall back to.
+    #             try:
+    #                 result = k8s.get(
+    #                     resource=k8s.resources.get(api_version="v1", kind=ref.kind, group=""),
+    #                     name=ref.name,
+    #                     namespace=ref.namespace,  # TODO: Determine the namespace to backfill to.
+    #                 )
+
+    #                 value = result["data"][ref.key]
+    #                 if value is None:
+    #                     raise KeyError
+    #                 assert isinstance(value, str)
+    #                 resolves[str(ref)] = base64.b64decode(value.encode("ascii")).decode("utf-8")
+    #             except NotFoundError:
+    #                 logger.warning(
+    #                     "Skipping resource {}/{} because its reference to {} could not be resolved.",
+    #                     manifest["apiVersion"],
+    #                     manifest["kind"],
+    #                     ref,
+    #                 )
+    #                 skip_resource = True
+    #                 break
+    #             except KeyError:
+    #                 logger.warning(
+    #                     "Skipping resource {}/{} because its reference to {} could not be resolved (does not contain key {}).",
+    #                     manifest["apiVersion"],
+    #                     manifest["kind"],
+    #                     ref,
+    #                     ref.key,
+    #                 )
+    #                 skip_resource = True
+    #                 break
+    #         if skip_resource:
+    #             continue
+    #         manifest = Reference.sub(manifest, lambda ref: resolves[str(ref)])
+
+
+def _random_password(length: int = 32) -> str:
+    """
+    Generate a random password.
+    """
+
+    import secrets
+
+    return secrets.token_urlsafe(length)
+
+
+def _bcrypt(password: str) -> str:
+    """
+    Hash a password using bcrypt.
+    """
+
+    import bcrypt
+
+    return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8")
+
+
+def is_namespace_resource(manifest: Manifest) -> bool:
+    """
+    Check if a manifest is a namespace resource.
+    """
+
+    return manifest.get("apiVersion") == "v1" and manifest.get("kind") == "Namespace"
+
+
+def is_cluster_scoped_resource(manifest: Manifest) -> bool:
+    """
+    Check if a manifest is a cluster scoped resource.
+    """
+
+    # HACK: We should probably just list the resources via the Kubectl API?
+    fqn = manifest.get("kind", "") + "." + manifest.get("apiVersion", "").split("/")[0]
+    return fqn in {
+        "ClusterRole.rbac.authorization.k8s.io",
+        "ClusterRoleBinding.rbac.authorization.k8s.io",
+        "CustomResourceDefinition.apiextensions.k8s.io",
+        "IngressClass.networking.k8s.io",
+        "Namespace.v1",
+        "StorageClass.storage.k8s.io",
+        "ValidatingWebhookConfiguration.admissionregistration.k8s.io",
+    }