nyl 0.1.0__tar.gz → 0.2.0__tar.gz

{nyl-0.1.0 → nyl-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: nyl
-Version: 0.1.0
+Version: 0.2.0
 Summary: Advanced templating engine for Kubernetes and GitOps.
 Author-email: Niklas Rosenstein <rosensteinniklas@gmail.com>
 License: MIT

{nyl-0.1.0 → nyl-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "nyl"
-version = "0.1.0"  # sed'ed on publish in CI
+version = "0.2.0"  # sed'ed on publish in CI
 description = "Advanced templating engine for Kubernetes and GitOps."
 authors = [
     {name = "Niklas Rosenstein", email = "rosensteinniklas@gmail.com"},

{nyl-0.1.0 → nyl-0.2.0}/src/nyl/commands/__init__.py

@@ -29,12 +29,14 @@ from . import profile  # noqa: E402
 from . import run  # noqa: F401, E402
 from . import secrets  # noqa: E402
 from . import template  # noqa: F401,E402
+from . import tools  # noqa: E402
 from . import tun  # noqa: E402
 
 app.add_typer(argocd.app)
 app.add_typer(new.app)
 app.add_typer(profile.app)
 app.add_typer(secrets.app)
+app.add_typer(tools.app)
 app.add_typer(tun.app)
 
 

{nyl-0.1.0 → nyl-0.2.0}/src/nyl/commands/crds.py

@@ -1,6 +1,7 @@
 import yaml
 
 from nyl.resources.applyset import ApplySet
+from nyl.resources.placeholder import Placeholder
 from . import app
 
 
@@ -12,3 +13,5 @@ def crds() -> None:
 
     print("---")
     print(yaml.safe_dump(ApplySet.CRD))
+    print("---")
+    print(yaml.safe_dump(Placeholder.CRD))

nyl-0.2.0/src/nyl/commands/secrets.py

@@ -0,0 +1,61 @@
+"""
+Interact with the secrets providers configured in `nyl-secrets.yaml`.
+"""
+
+import json
+
+from typer import Option
+from nyl.secrets.config import SecretsConfig
+from nyl.tools.typer import new_typer
+
+
+app = new_typer(name="secrets", help=__doc__)
+
+# Initialized from callback for access by subcommands.
+provider: str
+
+
+@app.callback()
+def callback(
+    _provider: str = Option(
+        "default",
+        "--provider",
+        help="The name of the configured secrets provider to use.",
+        envvar="NYL_SECRETS",
+    ),
+) -> None:
+    """
+    Interact with the secrets providers configured in `nyl-secrets.yaml`.
+    """
+
+    global provider
+    provider = _provider
+
+
+@app.command()
+def list(
+    providers: bool = Option(
+        False, help="List the configured secrets providers instead of the current provider's available keys."
+    ),
+) -> None:
+    """
+    List the keys for all secrets in the provider.
+    """
+
+    secrets = SecretsConfig.load()
+    if providers:
+        for alias, impl in secrets.providers.items():
+            print(alias, impl)
+    else:
+        for key in secrets.providers[provider].keys():
+            print(key)
+
+
+@app.command()
+def get(key: str, pretty: bool = False) -> None:
+    """
+    Get the value of a secret as JSON.
+    """
+
+    secrets = SecretsConfig.load()
+    print(json.dumps(secrets.providers[provider].get(key), indent=4 if pretty else None))

{nyl-0.1.0 → nyl-0.2.0}/src/nyl/commands/template.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass
 import os
 from pathlib import Path
 from textwrap import indent
-from typing import Optional, cast
+from typing import Optional
 from loguru import logger
 from typer import Argument, Option
 import yaml
@@ -66,6 +66,18 @@ def template(
         "This option must be disabled when passing the generated manifests to `kubectl apply --applyset=...`, as it "
         "would otherwise cause an error due to the label being present on the input data.",
     ),
+    default_namespace: Optional[str] = Option(
+        None,
+        help="The name of the Kubernetes namespace to fill in to Kubernetes resource that don't have a namespace set. "
+        "If this is not specified as an argument or via environment variables, it will default to the stem of the "
+        "manifest source filename. Note that if a manifest defines a Namespace resource, that namespace is used "
+        "instead, regardless of the value of this option. If a manifest source file defines multiple namespaces and a "
+        "resource without namespace is encountered, this option is considered if it matches one of the namespaces "
+        "defined in the file. Note that this option is usually problematic when rendering multiple files, as often "
+        "a single file is intended to deploy into a single namespace.\n\n"
+        "Note that Nyl's detection for cluster-scoped vs. namespace-scoped resources is not very good, yet.",
+        envvar="ARGOCD_APP_NAMESPACE",
+    ),
     inline: bool = Option(True, help="Evaluate Nyl inlined resources."),
     state_dir: Optional[Path] = Option(
         None, help="The directory to store state in (such as kubeconfig files).", envvar="NYL_STATE_DIR"
@@ -142,7 +154,9 @@ def template(
     secrets = SecretsConfig.load()
     client = ApiClient()
 
-    template_engine = NylTemplateEngine(secrets.providers[secrets_provider], client)
+    template_engine = NylTemplateEngine(
+        secrets.providers[secrets_provider], client, create_placeholders=project.config.settings.generate_placeholders
+    )
 
     generator = DispatchingGenerator.default(
         cache_dir=cache_dir,
@@ -157,19 +171,19 @@ def template(
     for source in load_manifests(paths):
         logger.opt(ansi=True).info("Rendering manifests from <blue>{}</>.", source.file)
 
-        source.manifests = cast(Manifests, template_engine.evaluate(source.manifests))
+        source.manifests = template_engine.evaluate(source.manifests)
         if inline:
             source.manifests = reconcile_generator(
                 generator,
                 source.manifests,
-                on_generated=lambda m: cast(Manifest, template_engine.evaluate(m)),
+                on_generated=lambda m: template_engine.evaluate(Manifests([m])),
             )
 
         # Find the namespaces that are defined in the file. If we find any manifests without a namespace, we will
-        # inject that namespace name into them.
-        # Also find the applyset defined in the file.
+        # inject that namespace name into them. Also find the applyset defined in the file.
         namespaces: set[str] = set()
         applyset: ApplySet | None = None
+
         for manifest in list(source.manifests):
             if is_namespace_resource(manifest):
                 namespaces.add(manifest["metadata"]["name"])
@@ -183,23 +197,40 @@ def template(
                 applyset = ApplySet.load(manifest)
                 source.manifests.remove(manifest)
 
+        # Determine what namespace to fill in if we encounter resources without it.
+        current_default_namespace = default_namespace
+        if current_default_namespace is None:
+            if len(namespaces) == 1:
+                current_default_namespace = next(iter(namespaces))
+            elif len(namespaces) == 0:
+                current_default_namespace = source.file.stem
+        elif len(namespaces) > 2 and current_default_namespace not in namespaces:
+            logger.error(
+                "The specified --default-namespace='{}' is not defined in manifest source file '{}', despite it "
+                "defining {} namespaces. If your manifest source file defines multiple namespaces, the specified "
+                "--default-namespace must be one of the namespaces defined in it (the file current defines "
+                "the following namespaces: {})",
+                default_namespace,
+                source.file,
+                len(namespaces),
+                ", ".join(namespaces),
+            )
+            exit(1)
+
         if not applyset and project.config.settings.generate_applysets:
-            if len(namespaces) > 1:
+            if not current_default_namespace:
                 logger.opt(ansi=True).error(
-                    "Multiple namespaces defined in <yellow>{}</>, but automatic ApplySet generation is enabled. "
-                    "There can only be one namespace per source in this case.",
+                    "No default namespace defined for <yellow>{}</>, but it is required for the automatically "
+                    "generated nyl.io/v1/ApplySet resource (the ApplySet is named after the default namespace).",
                     source.file,
                 )
                 exit(1)
-            elif len(namespaces) == 1:
-                applyset_name = next(iter(namespaces))
-            else:
-                applyset_name = source.file.stem
 
+            applyset_name = current_default_namespace
+            applyset = ApplySet.new(applyset_name)
             logger.opt(ansi=True).info(
                 "Automatically creating ApplySet for <blue>{}</> (name: <magenta>{}</>).", source.file, applyset_name
             )
-            applyset = ApplySet.new(applyset_name)
 
         if applyset is not None:
             applyset.set_group_kinds(source.manifests)
@@ -234,7 +265,7 @@ def template(
 
             if "metadata" not in manifest:
                 logger.opt(ansi=True).error(
-                    "Manifest in <yellow>'{}'</> has no <cyan>metadata</> key:\n\n{}",
+                    "A manifest in <yellow>'{}'</> has no <cyan>metadata</> key:\n\n{}",
                     source.file,
                     indent(yaml.safe_dump(manifest), "  "),
                 )
@@ -245,30 +276,23 @@ def template(
                     labels[APPLYSET_LABEL_PART_OF] = applyset.id
 
             if not is_cluster_scoped_resource(manifest) and "namespace" not in manifest["metadata"]:
-                if len(namespaces) > 1:
+                if not current_default_namespace:
                     logger.error(
-                        "Multiple namespaces defined in '{}', but manifest {}/{} does not have a namespace.",
+                        "A namespace-scoped manifest in '{}' has no namespace, and no default namespace is defined. "
+                        "Try defining a single namespace in the manifest source file, naming the manifest source file "
+                        "the same as that default namespace or passing the --default-namespace option.",
                         source.file,
-                        manifest["kind"],
-                        manifest["metadata"]["name"],
                     )
                     exit(1)
-                elif len(namespaces) == 0:
-                    logger.warning(
-                        "No namespaces defined in '{}', injecting 'default' into manifest {}/{}.",
-                        source.file,
-                        manifest["kind"],
-                        manifest["metadata"]["name"],
-                    )
-                    manifest["metadata"]["namespace"] = "default"
-                else:
-                    logger.trace(
-                        "Injecting namespace '{}' into manifest {}/{}.",
-                        next(iter(namespaces)),
-                        manifest["kind"],
-                        manifest["metadata"]["name"],
-                    )
-                    manifest["metadata"]["namespace"] = next(iter(namespaces))
+
+                logger.trace(
+                    "Injecting namespace '{}' into manifest {}/{}.",
+                    current_default_namespace,
+                    manifest["kind"],
+                    manifest["metadata"]["name"],
+                )
+                manifest["metadata"]["namespace"] = current_default_namespace
+
             elif not is_cluster_scoped_resource(manifest) and manifest["metadata"]["namespace"] not in namespaces:
                 logger.warning(
                     "Resource '{}/{}' in '{}' references namespace '{}', which is not defined in the file.",

nyl-0.2.0/src/nyl/commands/tools/__init__.py

@@ -0,0 +1,13 @@
+"""
+Swiss-army-knife for various common operations in conjunction with Nyl (Kubernetes, secrets management, etc).
+"""
+
+from nyl.tools.typer import new_typer
+
+
+app = new_typer(name="tools", help=__doc__)
+
+from . import sops  # noqa: E402
+from . import bcrypt  # noqa: F401,E402
+
+app.add_typer(sops.app)

nyl-0.2.0/src/nyl/commands/tools/bcrypt.py

@@ -0,0 +1,17 @@
+import sys
+from typing import Optional
+
+from bcrypt import gensalt, hashpw
+from . import app
+
+
+@app.command()
+def bcrypt(text: Optional[str] = None) -> None:
+    """
+    Bcrypt-hash the specified text. If no argument is specified, read from stdin.
+    """
+
+    if text is None:
+        text = sys.stdin.read()
+
+    print(hashpw(text.encode("utf-8"), gensalt()).decode("utf-8"))

nyl-0.1.0/src/nyl/commands/secrets.py → nyl-0.2.0/src/nyl/commands/tools/sops.py

@@ -1,75 +1,32 @@
 """
-Interact with the secrets providers configured in `nyl-secrets.yaml`.
+Utilities for SOPS files.
 """
 
-import json
 from pathlib import Path
+from shlex import quote
 import sys
 from typing import Optional
 
 from loguru import logger
 from typer import Option
 from nyl.secrets.config import SecretsConfig
-from nyl.secrets.sops import SopsFile
+from nyl.secrets.sops import SopsFile, detect_sops_format
 from nyl.tools.fs import shorter_path
 from nyl.tools.logging import lazy_str
 from nyl.tools.typer import new_typer
 
 
-app = new_typer(name="secrets", help=__doc__)
-
-sops = new_typer(name="sops", help=__doc__)
-app.add_typer(sops)
-
-# Initialized from callback for access by subcommands.
-provider: str
-
-
-@app.callback()
-def callback(
-    _provider: str = Option(
-        "default", "--provider", help="The name of the configured secrets provider to use.", envvar="NYL_SECRETS"
-    ),
-) -> None:
-    """
-    Interact with the secrets providers configured in `nyl-secrets.yaml`.
-    """
-
-    global provider
-    provider = _provider
-
-
-@app.command()
-def list(
-    providers: bool = Option(
-        False, help="List the configured secrets providers instead of the current provider's available keys."
-    ),
-) -> None:
-    """
-    List the keys for all secrets in the provider.
-    """
-
-    secrets = SecretsConfig.load()
-    if providers:
-        for alias, impl in secrets.providers.items():
-            print(alias, impl)
-    else:
-        for key in secrets.providers[provider].keys():
-            print(key)
+app = new_typer(name="sops", help=__doc__)
 
 
 @app.command()
-def get(key: str, pretty: bool = False) -> None:
-    """
-    Get the value of a secret as JSON.
-    """
-
-    secrets = SecretsConfig.load()
-    print(json.dumps(secrets.providers[provider].get(key), indent=4 if pretty else None))
-
-
-@sops.command()
 def re_encrypt(
+    provider: str = Option(
+        "default",
+        "--provider",
+        help="The name of the configured secrets provider to use.",
+        envvar="NYL_SECRETS",
+    ),
     file: Optional[Path] = Option(
         None,
         help="The SOPS-file to re-encrypt. Defaults to the SOPS file mentioned in `nyl-secrets.yaml` "
@@ -99,16 +56,43 @@ def re_encrypt(
     logger.opt(ansi=True).info("re-encrypting file '<blue>{}</>'", lazy_str(lambda f: str(shorter_path(f)), file))
 
     if file_type is None:
-        if file.suffix in (".yml", ".yaml"):
-            file_type = "yaml"
-        elif file.suffix in (".json", ".json5"):
-            file_type = "json"
-        elif file.suffix in (".sh", ".bash", ".env"):
-            file_type = "dotenv"
-        else:
+        file_type = detect_sops_format(file.suffix)
+        if not file_type:
             logger.error("could not determine SOPS input/output type from filename, specify with the --type option")
             sys.exit(1)
 
     sops = SopsFile(file)
     sops.load(file_type)
     sops.save(file_type)
+
+
+@app.command()
+def export_dotenv(
+    file: Path,
+    prefix: str = Option("", help="Only export keys with the given prefix, and strip the prefix."),
+    file_type: Optional[str] = Option(
+        None, "--type", help="The SOPS input type if it cannot be determined from the file name."
+    ),
+) -> None:
+    """
+    A utility function to export key-value pairs from a SOPS file in dotenv format. This is useful for exporting
+    environment variables from a SOPS file, e.g. using Direnv.
+
+    Note that only string values are exported.
+    """
+
+    if file_type is None:
+        file_type = detect_sops_format(file.suffix)
+        if not file_type:
+            logger.error("could not determine SOPS input type from filename, specify with the --type option")
+            sys.exit(1)
+
+    sops = SopsFile(file)
+    sops.load(file_type)
+
+    for key in sops.keys():
+        if not key.startswith(prefix):
+            continue
+        value = sops.get(key)
+        if isinstance(value, str):
+            print(f"export {key[len(prefix):]}={quote(value)}")

{nyl-0.1.0 → nyl-0.2.0}/src/nyl/generator/__init__.py

@@ -36,7 +36,7 @@ class Generator(ABC, Generic[T]):
 def reconcile_generator(
     generator: Generator[Manifest],
     manifests: Manifests,
-    on_generated: Callable[[Manifest], Manifest],
+    on_generated: Callable[[Manifest], Manifests],
 ) -> Manifests:
     """
     Recursively reconcile all Nyl resources in the manifests using the given generator.
@@ -64,7 +64,7 @@ def reconcile_generator(
         else:
             seen.add(resource_hash)
             for manifest in generator.generate(resource):
-                queue.append(on_generated(manifest))
+                queue.extend(on_generated(manifest))
             loops += 1
 
     return result

{nyl-0.1.0 → nyl-0.2.0}/src/nyl/profiles/kubeconfig.py

@@ -109,7 +109,7 @@ class KubeconfigManager:
         self, *, profile_name: str, path: Path, context: str, api_host: str, api_port: int
     ) -> Path:
         kubeconfig_data = yaml.safe_load(path.read_text())
-        kubeconfig_data = _trim_to_context(kubeconfig_data, context)
+        kubeconfig_data = _trim_to_context(kubeconfig_data, context, rename_context=profile_name)
 
         # TODO: Do we need to support the Kubernetes API hosted on a subpath?
         kubeconfig_data["clusters"][0]["cluster"]["server"] = f"https://{api_host}:{api_port}"
@@ -122,7 +122,9 @@ class KubeconfigManager:
         return final_kubeconfig
 
 
-def _trim_to_context(kubeconfig: dict[str, Any], context: str | None) -> dict[str, Any]:
+def _trim_to_context(
+    kubeconfig: dict[str, Any], context: str | None, rename_context: str | None = None
+) -> dict[str, Any]:
     """
     Trim a Kubeconfig down to a single context. If *context* is `None`, it will be trinmed to the current context.
     """
@@ -130,12 +132,14 @@ def _trim_to_context(kubeconfig: dict[str, Any], context: str | None) -> dict[st
     if context is None:
         context = kubeconfig["current-context"]
     else:
-        kubeconfig["current-context"] = context
+        kubeconfig["current-context"] = rename_context or context
 
     kubeconfig["contexts"] = [c for c in kubeconfig["contexts"] if c["name"] == context]
     if not kubeconfig["contexts"]:
         raise ValueError(f"Context '{context}' not found in Kubeconfig file.")
     context_data = kubeconfig["contexts"][0]
+    if rename_context:
+        context_data["name"] = rename_context
 
     kubeconfig["clusters"] = [c for c in kubeconfig["clusters"] if c["name"] == context_data["context"]["cluster"]]
     if not kubeconfig["clusters"]:

{nyl-0.1.0 → nyl-0.2.0}/src/nyl/project/config.py

@@ -24,6 +24,12 @@ class ProjectSettings:
     after the namespace.
     """
 
+    generate_placeholders: bool = False
+    """
+    If enabled, resources with templates that fail to evaluate due to unsatisfied conditions (e.g. lookup errors) will
+    be replaced with `nyl.io/v1/Placeholder` resources during templating. Otherwise, the error will be propagated.
+    """
+
     components_path: Path | None = None
     """
     Path to the directory that contains Nyl components.

nyl-0.2.0/src/nyl/resources/placeholder.py

@@ -0,0 +1,110 @@
+import base64
+from dataclasses import dataclass
+import hashlib
+from typing import Annotated, ClassVar
+from nyl.resources import API_VERSION_K8S, NylResource, ObjectMetadata
+from databind.core import SerializeDefaults
+
+
+@dataclass(kw_only=True)
+class Placeholder(NylResource, api_version=API_VERSION_K8S):
+    """
+    A Placeholder is a dummy custom resource that is used to represent a resource that failed to evaluate during
+    templating. This is useful when you want to define a manifest that has dependencies on other manifests based on
+    `lookup()` calls, but you want to avoid the evaluation of the manifest to fail if the lookup fails.
+
+    The placeholder can inform you that a component of your manifest is missing, and you can then decide how to handle
+    the situation.
+
+    Note that removal of Placeholder resources is not handled by Nyl: It will either be pruned as part of a
+    reconciliation by ArgoCD (if you're using that) or pruned with ApplySets. If you're not using either of those, you
+    will have to manually remove the Placeholder resources.
+
+    A placeholder can contain arbitrary `spec` data and usually contains fields to help you identify the missing
+    resource.
+    """
+
+    # HACK: Can't set it on the class level, see https://github.com/NiklasRosenstein/python-databind/issues/73.
+    metadata: Annotated[ObjectMetadata, SerializeDefaults(False)]
+
+    CRD: ClassVar = {
+        "apiVersion": "apiextensions.k8s.io/v1",
+        "kind": "CustomResourceDefinition",
+        "metadata": {
+            "name": f"placeholders.{API_VERSION_K8S.split('/')[0]}",
+        },
+        "spec": {
+            "group": API_VERSION_K8S.split("/")[0],
+            "names": {
+                "kind": "Placeholder",
+                "plural": "placeholders",
+            },
+            "scope": "Namespaced",
+            "versions": [
+                {
+                    "name": "v1",
+                    "served": True,
+                    "storage": True,
+                    "schema": {
+                        "openAPIV3Schema": {
+                            "type": "object",
+                            "properties": {
+                                "spec": {
+                                    "type": "object",
+                                    "properties": {
+                                        "message": {
+                                            "type": "string",
+                                            "description": "A message that describes the missing resource.",
+                                        },
+                                        "reason": {
+                                            "type": "string",
+                                            "description": "The reason why the resource is missing (e.g. 'NotFound').",
+                                        },
+                                    },
+                                },
+                            },
+                        }
+                    },
+                }
+            ],
+        },
+    }
+
+    @staticmethod
+    def new(name: str, namespace: str) -> "Placeholder":
+        return Placeholder(
+            metadata=ObjectMetadata(
+                name=name,
+                namespace=namespace,
+            )
+        )
+
+
+def calculate_applyset_id(*, name: str, namespace: str = "", group: str) -> str:
+    """
+    Calculate the ID of a Kubernetes ApplySet with the specified name.
+    """
+
+    # reference: https://kubernetes.io/docs/reference/labels-annotations-taints/#applyset-kubernetes-io-id
+    hash = hashlib.sha256(f"{name}.{namespace}.ApplySet.{group}".encode()).digest()
+    uid = base64.b64encode(hash).decode().rstrip("=").replace("/", "_").replace("+", "-")
+    return f"applyset-{uid}-v1"
+
+
+def get_canonical_resource_kind_name(api_version: str, kind: str) -> str:
+    """
+    Given the apiVersion and kind of a Kubernetes resource, return the canonical name of the resource. This name can
+    be used to identify the resource in an ApplySet's `applyset.kubernetes.io/contains-group-kinds` annotation.
+
+    Note that according to the [reference][1], the resource name should use the plural form, but it appears that the
+    resource kind name is also accepted. Deriving the plural form will be difficult without querying the Kubernetes
+    API.
+
+    [1]: https://kubernetes.io/docs/reference/labels-annotations-taints/#applyset-kubernetes-io-contains-group-kinds
+
+    Args:
+        api_version: The apiVersion of the resource.
+        kind: The kind of the resource.
+    """
+
+    return (f"{kind}." + (api_version.split("/")[0] if "/" in api_version else "")).rstrip(".")

{nyl-0.1.0 → nyl-0.2.0}/src/nyl/secrets/sops.py

@@ -108,3 +108,20 @@ class SopsFile(SecretProvider):
                 raise KeyError(".".join(parts[: idx + 1]))
             value = value[part]
         return value
+
+
+def detect_sops_format(suffix: str) -> str | None:
+    """
+    Tells the SOPS file format based on the file suffix. Never returns "binary".
+    Returns `None` if the format cannot be determined.
+    """
+
+    suffix = suffix.removeprefix(".")
+    if suffix in ("yml", "yaml"):
+        return "yaml"
+    elif suffix in ("json", "json5"):
+        return "json"
+    elif suffix in ("sh", "bash", "env"):
+        return "dotenv"
+    else:
+        return None