nyl 0.4.0__tar.gz → 0.5.0__tar.gz

{nyl-0.4.0 → nyl-0.5.0}/.kraken.py

@@ -1,5 +1,7 @@
 from kraken.common import buildscript
+
 buildscript(requirements=["kraken-build @ git+https://github.com/kraken-build/kraken.git@nr/python-project#egg=kraken-build&subdirectory=kraken-build"])
 
 from kraken.build.python.v1alpha1 import python_project
+
 python_project()

{nyl-0.4.0 → nyl-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: nyl
-Version: 0.4.0
+Version: 0.5.0
 Summary: Advanced templating engine for Kubernetes and GitOps.
 Author-email: Niklas Rosenstein <rosensteinniklas@gmail.com>
 License: MIT

nyl-0.5.0/docs/content/reference/configuration/secrets.md

@@ -0,0 +1,152 @@
+# Secrets
+
+You can connect Nyl with various secret providers to retrieve external or encrypted data that can be used in your
+templates. This is useful for keeping sensitive data out of your configuration files and ensuring that they are not
+accidentally committed to a version control system.
+
+## Configuration
+
+Secret providers are configured in a `nyl-secrets.<ext>` file that is located in the current working directory or any of
+its parent directories. Secret providers may also be defined in a [Project configuration file](./projects.md), though
+the file closer to the working directory will take precedence.
+
+There is no "global" way to define a secrets provider, as secrets are considered project-specific.
+
+As with other configuration file types, the file extension can be `.toml`, `.yaml`, or `.json`.
+
+The configuration contains any number of named secret providers. When not specified otherwise, Nyl will assume that the
+provider to use is named `default`. The provider to use can be overriden by passing the corresponding CLI option to
+respective Nyl commands or by setting the `NYL_SECRETS` environment variable.
+
+## Inspecting secret providers
+
+You can inspect secret providers using the `nyl secrets` command.
+
+```
+nyl secrets list            List the keys for all secrets in the provider.
+nyl secrets get <key>       Get the value of a secret as JSON.
+```
+
+## Templating
+
+Secrets are made available to templates using the `secrets.get()` function. The function takes a single argument, the
+key of the secret to retrieve.
+
+__Example__
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: my-secret
+type: Opaque
+stringData:
+  password: ${{ secrets.get("my-password") }}
+```
+
+## CLI
+
+The `nyl secrets` command allows you to interact with the configured secret provider(s).
+
+```
+ Usage: nyl secrets [OPTIONS] COMMAND [ARGS]...                                 
+                                                                                
+ Interact with the secrets providers configured in `nyl-secrets.yaml`.          
+                                                                                
+╭─ Options ────────────────────────────────────────────────────────────────────╮
+│ --provider        TEXT  The name of the configured secrets provider to use.  │
+│                         [env var: NYL_SECRETS]                               │
+│                         [default: default]                                   │
+│ --help                  Show this message and exit.                          │
+╰──────────────────────────────────────────────────────────────────────────────╯
+╭─ Commands ───────────────────────────────────────────────────────────────────╮
+│ get     Get the value of a secret as JSON.                                   │
+│ list    List the keys for all secrets in the provider.                       │
+│ set     Set the value of a secret.                                           │
+│ unset   Unset the value of a secret.                                         │
+╰──────────────────────────────────────────────────────────────────────────────╯
+```
+
+---
+
+## Provider: [Sops]
+
+  [Sops]: https://github.com/getsops/sops
+
+Allows you to retrieve secrets from a [Sops] encrypted file. For a GitOps workflow, the file must be commited to the
+same repository to ensure that Nyl has access to it when it is invoked as an ArgoCD Config Management plugin. You also
+must have the `sops` program installed.
+
+__Example__
+
+=== "TOML"
+
+    ```toml title="nyl-secrets.toml"
+    [default]
+    type = "sops"
+    path = "../secrets.yaml"
+    ```
+
+=== "YAML"
+
+    ```yaml title="nyl-secrets.yaml"
+    default:
+      type: sops
+      path: ../secrets.yaml
+    ```
+
+=== "JSON"
+
+    ```toml title="nyl-secrets.json"
+    {
+      "default": {
+        "type": "sops",
+        "path": "../secrets.yaml"
+      }
+    }
+    ```
+
+The secrets will be decoded using the `sops` program, hence all the typical ways to configure Sops and how it decrypts
+files apply. The `path` field is relative to the location of the `nyl-secrets.yaml` file.
+
+---
+
+## Provider: [KubernetesSecret](https://kubernetes.io/docs/concepts/configuration/secret/)
+
+Allows you to point to a Kubernetes Secret as a source for secrets. Since Kubernetes secret values must be strings,
+this provider does not currently support nested keys, although it could be made possible by assuming TOML/YAML/JSON
+format for the Kubernetes `Secret` keys in the `data` field.
+
+The secret provider will use the same Kubernetes context that is configured in Nyl via [Profiles](./profiles.md).
+
+__Example__
+
+=== "TOML"
+
+    ```toml title="nyl-secrets.toml"
+    [default]
+    type = "KubernetesSecret"
+    name = "nyl-secrets"
+    namespace = "default"
+    ```
+
+=== "YAML"
+
+    ```yaml title="nyl-secrets.yaml"
+    default:
+      type: KubernetesSecret
+      name: nyl-secrets
+      namespace: default
+    ```
+
+=== "JSON"
+
+    ```toml title="nyl-secrets.json"
+    {
+      "default": {
+        "type": "KubernetesSecret",
+        "name": "nyl-secrets",
+        "namespace": "default"
+      }
+    }
+    ```

{nyl-0.4.0 → nyl-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "nyl"
-version = "0.4.0"  # sed'ed on publish in CI
+version = "0.5.0"  # sed'ed on publish in CI
 description = "Advanced templating engine for Kubernetes and GitOps."
 authors = [
     {name = "Niklas Rosenstein", email = "rosensteinniklas@gmail.com"},

{nyl-0.4.0 → nyl-0.5.0}/src/kubernetes/dynamic/client.pyi

@@ -1,4 +1,5 @@
 from typing import Any, Iterable, Protocol
+
 from kubernetes.client.api_client import ApiClient
 from kubernetes.dynamic.discovery import Discoverer
 from kubernetes.dynamic.resource import Resource, ResourceInstance

{nyl-0.4.0 → nyl-0.5.0}/src/kubernetes/dynamic/discovery.pyi

@@ -1,5 +1,6 @@
 from collections.abc import Sequence
 from typing import Any
+
 from kubernetes.dynamic.resource import Resource
 
 class Discoverer:

{nyl-0.4.0 → nyl-0.5.0}/src/nyl/commands/__init__.py

@@ -6,39 +6,30 @@ applications directly or integrate as an ArgoCD ConfigManagementPlugin.
 import atexit
 import json
 import os
-from pathlib import Path
+import sys
 import time
-from typing import Optional
-from nyl import __version__
+from dataclasses import dataclass
 from enum import Enum
-import sys
+from pathlib import Path
+from typing import Optional
+
 from loguru import logger
-from typer import Option
+from typer import Option, Typer
+
+from kubernetes.client.api_client import ApiClient
+from nyl import __version__
+from nyl.profiles import ProfileManager
+from nyl.project.config import ProjectConfig
+from nyl.secrets.config import SecretsConfig
+from nyl.tools.di import DependenciesProvider
 from nyl.tools.logging import lazy_str
 from nyl.tools.shell import pretty_cmd
 from nyl.tools.typer import new_typer
 
+app: Typer = new_typer(help=__doc__)
 
-app = new_typer(help=__doc__)
-
-
-from . import argocd  # noqa: E402
-from . import crds  # noqa: F401,E402
-from . import new  # noqa: E402
-from . import profile  # noqa: E402
-from . import run  # noqa: F401, E402
-from . import secrets  # noqa: E402
-from . import template  # noqa: F401,E402
-from . import tools  # noqa: E402
-from . import tun  # noqa: E402
-
-app.add_typer(argocd.app)
-app.add_typer(new.app)
-app.add_typer(profile.app)
-app.add_typer(secrets.app)
-app.add_typer(tools.app)
-app.add_typer(tun.app)
-
+# A global instance that we use for dependency injection.
+PROVIDER = DependenciesProvider.default()
 
 LOG_TIME_FORMAT = "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green>"
 LOG_LEVEL_FORAMT = "<level>{level: <8}</level>"
@@ -55,6 +46,15 @@ class LogLevel(str, Enum):
     CRITICAL = "critical"
 
 
+# Retrieving the Kubernetes API client depends on whether in-cluster configuration should be used or not.
+@dataclass(kw_only=True)
+class ApiClientConfig:
+    in_cluster: bool
+    " Load the in-cluster configuration if enabled; forego any Nyl profile configuration. "
+    profile: str | None
+    " If not loading the in-cluster configuration, use the given Nyl profile. Otherwise, use the default kubeconfig. "
+
+
 @app.callback()
 def _callback(
     quiet: bool = Option(False, "--quiet", "-q", help="Shortcut for --log-level=error."),
@@ -98,6 +98,18 @@ def _callback(
         lazy_str(pretty_cmd, sys.argv),
     )
 
+    PROVIDER.set_lazy(ProfileManager, lambda: ProfileManager.load(required=False))
+    PROVIDER.set_lazy(SecretsConfig, lambda: SecretsConfig.load(dependencies=PROVIDER))
+    PROVIDER.set_lazy(ProjectConfig, lambda: ProjectConfig.load(dependencies=PROVIDER))
+    PROVIDER.set_lazy(
+        ApiClient,
+        lambda: template.get_incluster_kubernetes_client()
+        if PROVIDER.get(ApiClientConfig).in_cluster
+        else template.get_profile_kubernetes_client(
+            PROVIDER.get(ProfileManager), PROVIDER.get(ApiClientConfig).profile
+        ),
+    )
+
 
 @app.command()
 def version() -> None:
@@ -107,3 +119,21 @@ def version() -> None:
 
     print(f"Nyl v{__version__}")
     sys.exit(0)
+
+
+from . import argocd  # noqa: E402
+from . import crds  # noqa: F401,E402
+from . import new  # noqa: E402
+from . import profile  # noqa: E402
+from . import run  # noqa: F401, E402
+from . import secrets  # noqa: E402
+from . import template  # noqa: F401,E402
+from . import tools  # noqa: E402
+from . import tun  # noqa: E402
+
+app.add_typer(argocd.app)
+app.add_typer(new.app)
+app.add_typer(profile.app)
+app.add_typer(secrets.app)
+app.add_typer(tools.app)
+app.add_typer(tun.app)

{nyl-0.4.0 → nyl-0.5.0}/src/nyl/commands/argocd.py

@@ -5,12 +5,14 @@ Helper commands for the ArgoCD integration.
 import sys
 
 from loguru import logger
+from typer import Typer
+
+from nyl.commands import PROVIDER
 from nyl.project.config import ProjectConfig
 from nyl.secrets.config import SecretsConfig
 from nyl.tools.typer import new_typer
 
-
-app = new_typer(name="argocd", help=__doc__)
+app: Typer = new_typer(name="argocd", help=__doc__)
 
 
 @app.command()
@@ -21,10 +23,10 @@ def discovery() -> None:
     """
 
     found = False
-    if (file := ProjectConfig.load().file) is not None:
+    if (file := PROVIDER.get(ProjectConfig).file) is not None:
         print(file.absolute())
         found = True
-    if (file := SecretsConfig.load().file) is not None:
+    if (file := PROVIDER.get(SecretsConfig).file) is not None:
         print(file.absolute())
         found = True
 

{nyl-0.4.0 → nyl-0.5.0}/src/nyl/commands/crds.py

@@ -2,6 +2,7 @@ import yaml
 
 from nyl.resources.applyset import ApplySet
 from nyl.resources.placeholder import Placeholder
+
 from . import app
 
 

{nyl-0.4.0 → nyl-0.5.0}/src/nyl/commands/new.py

@@ -7,11 +7,13 @@ from pathlib import Path
 from textwrap import dedent
 
 from loguru import logger
-from typer import Option
+from typer import Option, Typer
+
+from nyl.commands import PROVIDER
 from nyl.project.config import ProjectConfig
 from nyl.tools.typer import new_typer
 
-app = new_typer(name="new", help=__doc__)
+app: Typer = new_typer(name="new", help=__doc__)
 
 
 class ComponentType(Enum):
@@ -106,5 +108,5 @@ def component(
 ) -> None:
     """Create the boilerplate for a Nyl component in the components directory."""
 
-    components_path = ProjectConfig.load().get_components_path()
+    components_path = PROVIDER.get(ProjectConfig).get_components_path()
     chart(components_path / api_version / kind)

{nyl-0.4.0 → nyl-0.5.0}/src/nyl/commands/profile.py

@@ -4,12 +4,13 @@ Interact with your Nyl profile configuration.
 
 import shlex
 
-from typer import Argument
+from typer import Argument, Typer
+
+from nyl.commands import PROVIDER
 from nyl.profiles import ProfileManager
 from nyl.tools.typer import new_typer
 
-
-app = new_typer(name="profile", help=__doc__)
+app: Typer = new_typer(name="profile", help=__doc__)
 
 
 @app.command()
@@ -20,7 +21,7 @@ def activate(profile_name: str = Argument("default", envvar="NYL_PROFILE")) -> N
     Evaluate the stdout of this command to export the KUBECONFIG into your environment.
     """
 
-    with ProfileManager.load() as manager:
+    with PROVIDER.get(ProfileManager) as manager:
         profile = manager.activate_profile(profile_name)
 
     for key, value in profile.env.items():
@@ -33,7 +34,7 @@ def get_kubeconfig(profile_name: str = Argument("default", envvar="NYL_PROFILE")
     Similar to `nyl profile activate`, but prints only the path to the `KUBECONFIG` file.
     """
 
-    with ProfileManager.load() as manager:
+    with PROVIDER.get(ProfileManager) as manager:
         profile = manager.activate_profile(profile_name)
 
     print(profile.kubeconfig)

{nyl-0.4.0 → nyl-0.5.0}/src/nyl/commands/run.py

@@ -1,18 +1,20 @@
 import atexit
 import os
-from pathlib import Path
 import subprocess
 import sys
+from pathlib import Path
 from tempfile import TemporaryDirectory
+
 from loguru import logger
 from typer import Argument, Option
-from nyl.tools import yaml
+
+from nyl.commands import PROVIDER
 from nyl.profiles import ActivatedProfile, ProfileManager
 from nyl.profiles.kubeconfig import _trim_to_context
+from nyl.tools import yaml
 from nyl.tools.logging import lazy_str
 from nyl.tools.shell import pretty_cmd
 
-
 from . import app
 
 # app = new_typer(name="run", help=__doc__)
@@ -42,7 +44,7 @@ def run(
     `nyl-profiles.yaml` configuration or from the same-named context in the global kubeconfig.
     """
 
-    manager = ProfileManager.load(required=False)
+    manager = PROVIDER.get(ProfileManager)
     if manager and profile_name in manager.config.profiles:
         with manager:
             profile = manager.activate_profile(profile_name)

nyl-0.5.0/src/nyl/commands/secrets.py

@@ -0,0 +1,93 @@
+"""
+Interact with the secrets providers configured in `nyl-secrets.yaml`.
+"""
+
+import json
+import json as _json
+
+from loguru import logger
+from typer import Option, Typer
+
+from nyl.commands import PROVIDER, ApiClientConfig
+from nyl.secrets import SecretProvider
+from nyl.secrets.config import SecretsConfig
+from nyl.tools.typer import new_typer
+
+app: Typer = new_typer(name="secrets", help=__doc__)
+
+
+@app.callback()
+def callback(
+    provider: str = Option(
+        "default",
+        "--provider",
+        help="The name of the configured secrets provider to use.",
+        envvar="NYL_SECRETS",
+    ),
+    profile: str | None = Option(
+        None,
+        "--profile",
+        help="The Nyl profile to assume.",
+        envvar="NYL_PROFILE",
+    ),
+) -> None:
+    """
+    Interact with the secrets providers configured in `nyl-secrets.yaml`.
+    """
+
+    PROVIDER.set(ApiClientConfig, ApiClientConfig(in_cluster=False, profile=profile))
+    PROVIDER.set_lazy(tuple[str, SecretProvider], lambda: (provider, PROVIDER.get(SecretsConfig).providers[provider]))
+    PROVIDER.set_lazy(SecretProvider, lambda: PROVIDER.get(tuple[str, SecretProvider])[1])  # type: ignore[type-abstract]
+
+
+@app.command()
+def list(
+    providers: bool = Option(
+        False, help="List the configured secrets providers instead of the current provider's available keys."
+    ),
+) -> None:
+    """
+    List the keys for all secrets in the provider.
+    """
+
+    if providers:
+        for alias, impl in PROVIDER.get(SecretsConfig).providers.items():
+            print(alias, impl)
+    else:
+        for key in PROVIDER.get(SecretProvider).keys():  # type: ignore[type-abstract]
+            print(key)
+
+
+@app.command()
+def get(key: str, pretty: bool = False, raw: bool = False) -> None:
+    """
+    Get the value of a secret as JSON.
+    """
+
+    value = PROVIDER.get(SecretProvider).get(key)  # type: ignore[type-abstract]
+    if raw and isinstance(value, str):
+        print(value)
+    else:
+        print(json.dumps(value, indent=4 if pretty else None))
+
+
+@app.command()
+def set(key: str, value: str, json: bool = False) -> None:
+    """
+    Set the value of a secret.
+    """
+
+    provider_name, secrets = PROVIDER.get(tuple[str, SecretProvider])
+    logger.info("Setting key '{}' in provider '{}'", key, provider_name)
+    secrets.set(key, _json.loads(value) if json else value)
+
+
+@app.command()
+def unset(key: str) -> None:
+    """
+    Unset the value of a secret.
+    """
+
+    provider_name, secrets = PROVIDER.get(tuple[str, SecretProvider])
+    logger.info("Unsetting key '{}' in provider '{}'", key, provider_name)
+    secrets.unset(key)

{nyl-0.4.0 → nyl-0.5.0}/src/nyl/commands/template.py

@@ -1,33 +1,45 @@
 import atexit
-from dataclasses import dataclass
 import os
+from dataclasses import dataclass
+from enum import Enum
 from pathlib import Path
 from textwrap import indent
-from typing import Optional
+from typing import Any, Literal, Optional, cast
+
 from loguru import logger
-from nyl.resources.postprocessor import PostProcessor
 from typer import Argument, Option
-from nyl.tools import yaml
+
+from kubernetes.client.api_client import ApiClient
+from kubernetes.config.incluster_config import load_incluster_config
+from kubernetes.config.kube_config import load_kube_config
+from nyl.commands import PROVIDER, ApiClientConfig, app
 from nyl.generator import reconcile_generator
 from nyl.generator.dispatch import DispatchingGenerator
 from nyl.profiles import ProfileManager
-from kubernetes.config.incluster_config import load_incluster_config
-from kubernetes.config.kube_config import load_kube_config
-from kubernetes.client.api_client import ApiClient
 from nyl.project.config import ProjectConfig
 from nyl.resources import API_VERSION_INLINE, NylResource
 from nyl.resources.applyset import APPLYSET_LABEL_PART_OF, ApplySet
+from nyl.resources.postprocessor import PostProcessor
 from nyl.secrets.config import SecretsConfig
 from nyl.templating import NylTemplateEngine
+from nyl.tools import yaml
 from nyl.tools.kubectl import Kubectl
 from nyl.tools.logging import lazy_str
 from nyl.tools.types import Manifest, Manifests
 
-from . import app
-
 DEFAULT_PROFILE = "default"
 
 
+# Need an enum for typer
+class OnLookupFailure(str, Enum):
+    Error = "Error"
+    CreatePlaceholder = "CreatePlaceholder"
+    SkipResource = "SkipResource"
+
+    def to_literal(self) -> Literal["Error", "CreatePlaceholder", "SkipResource"]:
+        return cast(Any, self.name)  # type: ignore[no-any-return]
+
+
 @dataclass
 class ManifestsWithSource:
     """
@@ -38,6 +50,37 @@ class ManifestsWithSource:
     file: Path
 
 
+def get_incluster_kubernetes_client() -> ApiClient:
+    logger.info("Using in-cluster configuration.")
+    load_incluster_config()
+    return ApiClient()
+
+
+def get_profile_kubernetes_client(profiles: ProfileManager, profile: str | None) -> ApiClient:
+    """
+    Create a Kubernetes :class:`ApiClient` from the selected *profile*.
+
+    If no *profile* is specified, but the profile manager contains at least one profile, the *profile* argument will
+    default to the value of :data:`DEFAULT_PROFILE` (which is `"default"`). Otherwise, if no profile is selected and
+    none is configured, the standard Kubernetes config load takes place (i.e. try `KUBECONFIG` and then
+    `~/.kube/config`).
+    """
+
+    with profiles:
+        # If no profile to activate is specified, and there are no profiles defined, we're not activating a
+        # a profile. It should be valid to use Nyl without a `nyl-profiles.yaml` file.
+        if profile is not None or profiles.config.profiles:
+            profile = profile or DEFAULT_PROFILE
+            active = profiles.activate_profile(profile)
+            load_kube_config(str(active.kubeconfig))
+        else:
+            logger.opt(ansi=True).info(
+                "No <yellow>nyl-profiles.yaml</> file found, using default kubeconfig and context."
+            )
+            load_kube_config()
+    return ApiClient()
+
+
 @app.command()
 def template(
     paths: list[Path] = Argument(..., help="The YAML file(s) to render. Can be a directory."),
@@ -88,6 +131,10 @@ def template(
         help="The directory to store cache data in. If not set, a directory in the --state-dir is used.",
         envvar="NYL_CACHE_DIR",
     ),
+    on_lookup_failure: OnLookupFailure | None = Option(
+        None,
+        help="Specify what to do when a lookup() call in a Nyl templated manifest fails. This overrides the nyl-project.toml setting if specified.",
+    ),
 ) -> None:
     """
     Render a package template into full Kubernetes resources.
@@ -125,24 +172,10 @@ def template(
     #       plugin without granting it access to the Kubernetes API. Most relevant bits of information that Nyl requires
     #       about the cluster are passed via the environment variables.
     #       See https://argo-cd.readthedocs.io/en/stable/user-guide/build-environment/
-    if in_cluster:
-        logger.info("Using in-cluster configuration.")
-        load_incluster_config()
-    else:
-        with ProfileManager.load(required=False) as profiles:
-            # If no profile to activate is specified, and there are no profiles defined, we're not activating a
-            # a profile. It should be valid to use Nyl without a `nyl-profiles.yaml` file.
-            if profile is not None or profiles.config.profiles:
-                profile = profile or DEFAULT_PROFILE
-                active = profiles.activate_profile(profile)
-                load_kube_config(str(active.kubeconfig))
-            else:
-                logger.opt(ansi=True).info(
-                    "No <yellow>nyl-profiles.yaml</> file found, using default kubeconfig and context."
-                )
-                load_kube_config()
+    PROVIDER.set(ApiClientConfig, ApiClientConfig(in_cluster=in_cluster, profile=profile))
+    client = PROVIDER.get(ApiClient)
 
-    project = ProjectConfig.load()
+    project = PROVIDER.get(ProjectConfig)
     if generate_applysets is not None:
         project.config.settings.generate_applysets = generate_applysets
 
@@ -152,11 +185,14 @@ def template(
     if cache_dir is None:
         cache_dir = state_dir / "cache"
 
-    secrets = SecretsConfig.load()
-    client = ApiClient()
+    secrets = PROVIDER.get(SecretsConfig)
 
     template_engine = NylTemplateEngine(
-        secrets.providers[secrets_provider], client, create_placeholders=project.config.settings.generate_placeholders
+        secrets.providers[secrets_provider],
+        client,
+        on_lookup_failure=on_lookup_failure.to_literal()
+        if on_lookup_failure
+        else project.config.settings.on_lookup_failure,
     )
 
     generator = DispatchingGenerator.default(