nyl 0.0.0__tar.gz → 0.0.2__tar.gz

nyl-0.0.2/PKG-INFO

@@ -0,0 +1,155 @@
+Metadata-Version: 2.1
+Name: nyl
+Version: 0.0.2
+Summary: Default template for PDM package
+Author-Email: Niklas Rosenstein <rosensteinniklas@gmail.com>
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: databind>=4.5.2
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: jinja2>=3.1.4
+Requires-Dist: loguru>=0.7.2
+Requires-Dist: kubernetes>=30.1.0
+Requires-Dist: typer>=0.12.3
+Requires-Dist: filelock>=3.15.4
+Requires-Dist: stablehash>=0.2.0
+Requires-Dist: nr-stream>=1.1.5
+Requires-Dist: requests>=2.32.3
+Description-Content-Type: text/markdown
+
+# nyl
+
+Nyl is a versatile Kubernetes resource management tool that fulfills various functions to enable reusable
+deployment configurations.
+
+__Features__
+
+* Generate Kubernetes manifests from a simple YAML configuration with templating support.
+* Inject secrets into Kubernetes manifests as time of generation.
+* Bootstrap a cluster and transfer control to ArgoCD or use it to manage a cluster from CI/CD or locally.
+* Connect to a Kubernetes cluster via an SSH tunnel.
+
+__Roadmap__
+
+* Integrate as an ArgoCD ConfigManagementPlugin.
+* Diffing
+* How to work around fields owned by a different field manager (e.g. MetalLB CRD `conversion..caBundle`) or `Job` resources?
+
+## Concepts
+
+### Packages
+
+A package is similar to a Helm chart in that it is a source for Kubernetes resources that can make use of
+templating to conditionally render resources and inject values into them. Resources generated by packages
+may make use of Nyl-specific resource kinds that are only available time of resource generation (see the
+[Templating > Resource Kinds](#resource-kinds) section below).
+
+Nyl packages can be checked into a source repository in a DRY form, but can be compiled to a bundle (e.g.
+including other referenced packages or Helm charts) for deployment when needed (e.g. for shipping to an
+air-gapped environment).
+
+Packages may have a `nyl-package.yaml` file that defines a schema for the package parameters and additional
+metadata. A package without this file may still accept and use parameters in its templates.
+
+### Applications
+
+An application is an instance of one package that is deployed to a Kubernetes cluster and are in turn defined
+also in a Nyl package. Packages instantiated as applications may not produce other applications. Applications
+are usually accompanied by a `nyl-deployment.yaml` file that defines the top-level templating context for the
+package(s), such as the secret store.
+
+When deploying a package as an application, the package must not generated resources other than applications,
+as all deployed resources must be owned by an application.
+
+### Deployment
+
+A deployment is where things get together: This is the place where you define the applications that should be
+deployed to a Kubernetes cluster, which cluster they are being deployed to, as well as sourcing secrets from
+a secret store. The configuration for a deployment is defined in a `nyl-deployment.yaml` file.
+
+```yaml
+apiVersion: nyl/v1
+kind: Deployment
+spec:
+  secretStores:
+    default:
+      type: Sops
+      path: secrets.yaml
+```
+
+__Spec__
+
+* `secrets` (array): The secret stores to make available in the templating context of the deployment. Any secrets
+  need to be injected from these stores into the application values at templating time.
+
+## Templating
+
+  [Jinja2]: https://jinja.palletsprojects.com/en/3.0.x/
+
+Nyl uses [Jinja2] as a templating engine, and as such is slightly different from Helm.
+
+### Injecting secrets
+
+Secrets are only available at the deployment level and need to be propagated further down.
+
+```yaml
+apiVersion: nyl/v1
+kind: Application
+metadata:
+  name: my-app
+spec:
+  package: ./path/to/package
+  values:
+    theSecret: {{ Secrets.default.get("my-secret") }}
+```
+
+### Resource kinds
+
+At templating time, Nyl supports special resource kinds that will be expanded to more Kubernetes resources.
+
+#### `templating.nyl/v1/HelmChart`
+
+This resource can be used to instantiate Helm charts as part of a package.
+
+```yaml
+apiVersion: templating.nyl/v1
+kind: HelmChart
+spec:
+  repository: https://kubernetes.github.io/ingress-nginx
+  chart: ingress-nginx
+  version: 4.10.1
+  releaseName: ingress-nginx
+  releaseNamespace: ingress-nginx
+  values: {}
+```
+
+__Spec__
+
+* `repository` (string): The Helm repository URL, OCI registry URL (sans the chart name), a local path or a
+  Git repository clone URL.
+* `chart` (string): The name of the chart in the repository, the chart name to append to the OCI registry URL,
+  the name of the chart in the local path or the path to the chart in the Git repository.
+* `version` (string): The version of the chart to use. Only relevant for Helm/OCI repositories.
+* `releaseName` (string): The name of the Helm release.
+* `releaseNamespace` (string): The namespace to install the Helm release into. If any resources generated by the
+  chart have no `metadata.namespace` field, it will be set to this value.
+* `values` (object): The values to pass to the Helm chart.
+
+#### `templating.nyl/v1/Package`
+
+Use this resource to instantiate another package.
+
+```yaml
+apiVersion: templating.nyl/v1
+kind: Package
+spec:
+  package: ./path/to/package
+  values: {}
+```
+
+__Spec__
+
+* `package` (string): The path to the package to instantiate. Must be prefixed with `./` to be resolved relative
+  to the package that references it. Otherwise, it will be resolved in a search path that is defined in the
+  `nyl-deployment.yaml` file.
+* `values` (object): The values to pass to the package.

nyl-0.0.2/README.md

@@ -0,0 +1,136 @@
+# nyl
+
+Nyl is a versatile Kubernetes resource management tool that fulfills various functions to enable reusable
+deployment configurations.
+
+__Features__
+
+* Generate Kubernetes manifests from a simple YAML configuration with templating support.
+* Inject secrets into Kubernetes manifests as time of generation.
+* Bootstrap a cluster and transfer control to ArgoCD or use it to manage a cluster from CI/CD or locally.
+* Connect to a Kubernetes cluster via an SSH tunnel.
+
+__Roadmap__
+
+* Integrate as an ArgoCD ConfigManagementPlugin.
+* Diffing
+* How to work around fields owned by a different field manager (e.g. MetalLB CRD `conversion..caBundle`) or `Job` resources?
+
+## Concepts
+
+### Packages
+
+A package is similar to a Helm chart in that it is a source for Kubernetes resources that can make use of
+templating to conditionally render resources and inject values into them. Resources generated by packages
+may make use of Nyl-specific resource kinds that are only available time of resource generation (see the
+[Templating > Resource Kinds](#resource-kinds) section below).
+
+Nyl packages can be checked into a source repository in a DRY form, but can be compiled to a bundle (e.g.
+including other referenced packages or Helm charts) for deployment when needed (e.g. for shipping to an
+air-gapped environment).
+
+Packages may have a `nyl-package.yaml` file that defines a schema for the package parameters and additional
+metadata. A package without this file may still accept and use parameters in its templates.
+
+### Applications
+
+An application is an instance of one package that is deployed to a Kubernetes cluster and are in turn defined
+also in a Nyl package. Packages instantiated as applications may not produce other applications. Applications
+are usually accompanied by a `nyl-deployment.yaml` file that defines the top-level templating context for the
+package(s), such as the secret store.
+
+When deploying a package as an application, the package must not generated resources other than applications,
+as all deployed resources must be owned by an application.
+
+### Deployment
+
+A deployment is where things get together: This is the place where you define the applications that should be
+deployed to a Kubernetes cluster, which cluster they are being deployed to, as well as sourcing secrets from
+a secret store. The configuration for a deployment is defined in a `nyl-deployment.yaml` file.
+
+```yaml
+apiVersion: nyl/v1
+kind: Deployment
+spec:
+  secretStores:
+    default:
+      type: Sops
+      path: secrets.yaml
+```
+
+__Spec__
+
+* `secrets` (array): The secret stores to make available in the templating context of the deployment. Any secrets
+  need to be injected from these stores into the application values at templating time.
+
+## Templating
+
+  [Jinja2]: https://jinja.palletsprojects.com/en/3.0.x/
+
+Nyl uses [Jinja2] as a templating engine, and as such is slightly different from Helm.
+
+### Injecting secrets
+
+Secrets are only available at the deployment level and need to be propagated further down.
+
+```yaml
+apiVersion: nyl/v1
+kind: Application
+metadata:
+  name: my-app
+spec:
+  package: ./path/to/package
+  values:
+    theSecret: {{ Secrets.default.get("my-secret") }}
+```
+
+### Resource kinds
+
+At templating time, Nyl supports special resource kinds that will be expanded to more Kubernetes resources.
+
+#### `templating.nyl/v1/HelmChart`
+
+This resource can be used to instantiate Helm charts as part of a package.
+
+```yaml
+apiVersion: templating.nyl/v1
+kind: HelmChart
+spec:
+  repository: https://kubernetes.github.io/ingress-nginx
+  chart: ingress-nginx
+  version: 4.10.1
+  releaseName: ingress-nginx
+  releaseNamespace: ingress-nginx
+  values: {}
+```
+
+__Spec__
+
+* `repository` (string): The Helm repository URL, OCI registry URL (sans the chart name), a local path or a
+  Git repository clone URL.
+* `chart` (string): The name of the chart in the repository, the chart name to append to the OCI registry URL,
+  the name of the chart in the local path or the path to the chart in the Git repository.
+* `version` (string): The version of the chart to use. Only relevant for Helm/OCI repositories.
+* `releaseName` (string): The name of the Helm release.
+* `releaseNamespace` (string): The namespace to install the Helm release into. If any resources generated by the
+  chart have no `metadata.namespace` field, it will be set to this value.
+* `values` (object): The values to pass to the Helm chart.
+
+#### `templating.nyl/v1/Package`
+
+Use this resource to instantiate another package.
+
+```yaml
+apiVersion: templating.nyl/v1
+kind: Package
+spec:
+  package: ./path/to/package
+  values: {}
+```
+
+__Spec__
+
+* `package` (string): The path to the package to instantiate. Must be prefixed with `./` to be resolved relative
+  to the package that references it. Otherwise, it will be resolved in a search path that is defined in the
+  `nyl-deployment.yaml` file.
+* `values` (object): The values to pass to the package.

nyl-0.0.2/pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "nyl"
+version = "0.0.2"
+description = "Default template for PDM package"
+authors = [
+    { name = "Niklas Rosenstein", email = "rosensteinniklas@gmail.com" },
+]
+dependencies = [
+    "databind>=4.5.2",
+    "pyyaml>=6.0.1",
+    "jinja2>=3.1.4",
+    "loguru>=0.7.2",
+    "kubernetes>=30.1.0",
+    "typer>=0.12.3",
+    "filelock>=3.15.4",
+    "stablehash>=0.2.0",
+    "nr-stream>=1.1.5",
+    "requests>=2.32.3",
+]
+requires-python = ">=3.10"
+readme = "README.md"
+
+[project.license]
+text = "MIT"
+
+[project.scripts]
+nyl = "nyl.commands:app"
+
+[build-system]
+requires = [
+    "pdm-backend",
+]
+build-backend = "pdm.backend"
+
+[tool.pdm]
+distribution = true
+
+[tool.pdm.dev-dependencies]
+dev = [
+    "ruff>=0.4.8",
+    "mypy>=1.10.0",
+    "types-pyyaml>=6.0.12.20240311",
+    "mkdocs>=1.6.0",
+    "mkdocs-material>=9.5.28",
+    "mkdocstrings[python]>=0.25.1",
+    "pytest>=8.2.2",
+    "kubernetes-stubs>=22.6.0.post1",
+    "types-requests>=2.32.0.20240712",
+]
+
+[tool.ruff]
+line-length = 120

nyl-0.0.2/src/nyl/appmanager/__init__.py

@@ -0,0 +1,3 @@
+from .appmanager import Application, ApplicationManager
+
+__all__ = ["Application", "ApplicationManager"]

nyl-0.0.2/src/nyl/appmanager/appmanager.py

@@ -0,0 +1,150 @@
+from dataclasses import dataclass
+from typing import Any
+from kubernetes.client import CustomObjectsApi
+from kubernetes.client.api_client import ApiClient
+from . import crd
+from nyl.tools.kubectl import Kubectl
+
+
+@dataclass
+class Application:
+    """
+    Represents an application deployed in the system.
+    """
+
+    name: str
+    contains_group_kinds: list[str]
+
+
+class ApplicationManager:
+    """
+    Manage applications deployed in the system.
+    """
+
+    def __init__(self, client: ApiClient, kubectl: Kubectl) -> None:
+        """
+        Args:
+            client: Kubernetes API client.
+            kubectl: A corresponding `kubectl` client to interact with the cluster. We re-use kubectl's applset
+                implementation to create and manage applications in the system.
+        """
+
+        self._client = client
+        self._kubectl = kubectl
+        self._kubectl_version_cached: str | None = None
+
+    @property
+    def _kubectl_version(self) -> str:
+        """
+        Get the version of the kubectl client.
+        """
+
+        if self._kubectl_version_cached is None:
+            self._kubectl_version_cached = self._kubectl.version()["gitVersion"]
+        return self._kubectl_version_cached
+
+    def _get_applyset_ref(self, name: str) -> str:
+        """
+        Get the reference to the ApplySet resource for the application.
+        """
+
+        return f"application.{crd.GROUP}/{name}"
+
+    def list_applications(self) -> Application:
+        """
+        List applications deployed in the system.
+        """
+
+        api = CustomObjectsApi(self._client)
+        objects = api.list_cluster_custom_object(  # noqa
+            group=crd.GROUP,
+            version=crd.VERSION,
+            plural=crd.PLURAL,
+        )
+
+        # TODO
+        return  # type: ignore[return-value]
+
+    def get_application(self, name: str) -> Application:
+        """
+        Get an application deployed in the system.
+        """
+
+        api = CustomObjectsApi(self._client)
+        obj = api.get_cluster_custom_object(group=crd.GROUP, version=crd.VERSION, plural=crd.PLURAL, name=name)
+        print(obj)
+        # TODO
+        return  # type: ignore[return-value]
+
+    def delete_application(self, name: str, cascade: bool = True) -> None:
+        """
+        Delete an application from the system.
+
+        Args:
+            name: The name of the application to delete.
+            cascade: If True, delete all resources associated with the application.
+        """
+
+        if cascade:
+            self._kubectl.apply(
+                manifests=[],
+                force_conflicts=True,
+                server_side=True,
+                applyset=self._get_applyset_ref(name),
+            )
+
+        api = CustomObjectsApi(self._client)
+        api.delete_cluster_custom_object(group=crd.GROUP, version=crd.VERSION, plural=crd.PLURAL, name=name)
+
+    # def upsert_application(self, app: Application) -> None:
+    #     """
+    #     Create or update an application in the system.
+    #     """
+
+    #     api = CustomObjectsApi(self._client)
+    #     api.create_cluster_custom_object(
+    #         group=crd.GROUP,
+    #         version=crd.VERSION,
+    #         plural=crd.PLURAL,
+    #         body=crd.generate_application_resource(
+    #             app.name,
+    #             contains_resources=app.contains_group_kinds,
+    #             kubectl_version=self._kubectl_version,
+    #         ),
+    #     )
+
+    def update_application(self, name: str, manifests: list[dict[str, Any]]) -> None:
+        """
+        Update an application in the system or create it.
+
+        Args:
+            name: The name of the application to update.
+            manifests: The updated manifests of the application.
+        """
+
+        resource_kinds = {crd.get_canonical_resource_kind_name(m["apiVersion"], m["kind"]) for m in manifests}
+
+        # Ensure that the application object exists.
+        # TODO Check if it exists instead of always calling the create endpoint?
+        # TODO Do we maybe need to update the resource kinds only after the apply?
+        #       Somehow kubectl must understand the previous resource kinds that were involved in the
+        #       application prior to the update.
+
+        api = CustomObjectsApi(self._client)
+        api.create_cluster_custom_object(
+            group=crd.GROUP,
+            version=crd.VERSION,
+            plural=crd.PLURAL,
+            body=crd.generate_application_resource(
+                name,
+                contains_resources=resource_kinds,
+                kubectl_version=self._kubectl_version,
+            ),
+        )
+
+        self._kubectl.apply(
+            manifests=manifests,
+            force_conflicts=True,
+            server_side=True,
+            applyset=self._get_applyset_ref(name),
+        )

nyl-0.0.2/src/nyl/appmanager/crd.py

@@ -0,0 +1,107 @@
+"""
+This module implements a Kubernetes CRD that represents Nyl's own understanding of an application, which also
+acts as a Kubernetes ApplySet. To read more about ApplySets, check out
+
+    https://kubernetes.io/blog/2023/05/09/introducing-kubectl-applyset-pruning/.
+"""
+
+import base64
+import hashlib
+from typing import Any, Collection
+
+
+GROUP = "nyl.io"
+KIND = "Application"
+PLURAL = "applications"
+VERSION = "v1"
+
+# note: the only purpose of this CRD is to create resources that act as a parent for ApplySets.
+#       check out this GitHub issue, and specifically this comment for more information:
+#       https://github.com/kubernetes/enhancements/issues/3659#issuecomment-1753091733
+APPLICATION_CRD = {
+    "apiVersion": "apiextensions.k8s.io/v1",
+    "kind": "CustomResourceDefinition",
+    "metadata": {
+        "name": f"{PLURAL}.{GROUP}",
+        "labels": {
+            "applyset.kubernetes.io/is-parent-type": "true",
+        },
+    },
+    "spec": {
+        "group": GROUP,
+        "names": {
+            "kind": KIND,
+            "plural": PLURAL,
+        },
+        "scope": "Cluster",
+        "versions": [
+            {
+                "name": VERSION,
+                "served": True,
+                "storage": True,
+                "schema": {
+                    "openAPIV3Schema": {
+                        "type": "object",
+                    }
+                },
+            }
+        ],
+    },
+}
+
+
+def calculate_applyset_id(*, name: str, namespace: str = "", group: str) -> str:
+    """
+    Calculate the ID of a Kubernetes ApplySet with the specified name.
+    """
+
+    # reference: https://kubernetes.io/docs/reference/labels-annotations-taints/#applyset-kubernetes-io-id
+    hash = hashlib.sha256(f"{name}.{namespace}.ApplySet.{group}".encode()).digest()
+    uid = base64.b64encode(hash).decode().rstrip("=").replace("/", "_").replace("+", "-")
+    return f"applyset-{uid}-v1"
+
+
+def get_canonical_resource_kind_name(api_version: str, kind: str) -> str:
+    """
+    Given the apiVersion and kind of a Kubernetes resource, return the canonical name of the resource. This name can
+    be used to identify the resource in an ApplySet's `applyset.kubernetes.io/contains-group-kinds` annotation.
+
+    Args:
+        api_version: The apiVersion of the resource.
+        kind: The kind of the resource.
+    """
+
+    return (f"{kind}." + (api_version.split("/")[0] if "/" in api_version else "")).rstrip(".")
+
+
+def generate_application_resource(
+    name: str,
+    contains_resources: Collection[str],
+    kubectl_version: str,
+) -> dict[str, Any]:
+    """
+    Generate a Kubernetes resource that represents an application in Nyl's system.
+
+    Args:
+        name: The name of the application.
+        contains_resources: A list of Kubernetes resources names that are contained in this application. This is a
+            required information for Kubernetes ApplySets. You can use `get_canonical_resource_kind_name` to get the
+            names of the resource kinds.
+        kubectl_version: The version of kubectl that was used to create this resource. This is a required information
+            for the `applyset.kubernetes.io/tooling` annotation in the resource if `kubectl` is used to apply it.
+    """
+
+    return {
+        "apiVersion": f"{GROUP}/{VERSION}",
+        "kind": KIND,
+        "metadata": {
+            "name": name,
+            "annotations": {
+                "applyset.kubernetes.io/tooling": f"kubectl/{kubectl_version}",
+                "applyset.kubernetes.io/contains-group-kinds": ",".join(sorted(contains_resources)),
+            },
+            "labels": {
+                "applyset.kubernetes.io/id": calculate_applyset_id(name=name, group=GROUP),
+            },
+        },
+    }

nyl-0.0.2/src/nyl/commands/__init__.py

@@ -0,0 +1,19 @@
+"""
+Nyl is a flexible configuration management tool for Kubernetes resources that can be used to generate and deploy
+applications directly or integrate as an ArgoCD ConfigManagementPlugin.
+"""
+
+from nyl.tools.typer import new_typer
+
+
+app = new_typer(help=__doc__)
+
+
+from . import profile  # noqa: E402
+from . import secrets  # noqa: E402
+from . import template  # noqa: F401,E402
+from . import tun  # noqa: E402
+
+app.add_typer(profile.app)
+app.add_typer(secrets.app)
+app.add_typer(tun.app)

nyl-0.0.2/src/nyl/commands/profile.py

@@ -0,0 +1,30 @@
+"""
+Interact with your Nyl profile configuration.
+"""
+
+import shlex
+
+from typer import Argument
+from nyl.profiles import ProfileManager
+from nyl.tools.typer import new_typer
+
+
+app = new_typer(name="profile", help=__doc__)
+
+
+@app.command()
+def activate(profile_name: str = Argument("default", envvar="NYL_PROFILE")) -> None:
+    """
+    Activate a Nyl profile.
+
+    Evaluate the stdout of this command to export the KUBECONFIG into your environment.
+    """
+
+    with ProfileManager.load() as manager:
+        profile = manager.activate_profile(profile_name)
+
+    # For standard tooling, like kubectl.
+    print(f"export KUBECONFIG={shlex.quote(str(profile.kubeconfig.absolute()))}")
+
+    # Used by Terraform providers.
+    print(f"export KUBE_CONFIG_PATH={shlex.quote(str(profile.kubeconfig.absolute()))}")

nyl-0.0.2/src/nyl/commands/secrets.py

@@ -0,0 +1,31 @@
+"""
+Interact with the secrets providers configured in `nyl-secrets.yaml`.
+"""
+
+import json
+from nyl.secrets.config import SecretsConfig
+from nyl.tools.typer import new_typer
+
+
+app = new_typer(name="secrets", help=__doc__)
+
+
+@app.command()
+def list() -> None:
+    """
+    List the keys for all secrets in the provider.
+    """
+
+    secrets = SecretsConfig.load(SecretsConfig.find_config_file())
+    for key in secrets.provider.keys():
+        print(key)
+
+
+@app.command()
+def get(key: str, pretty: bool = False) -> None:
+    """
+    Get the value of a secret as JSON.
+    """
+
+    secrets = SecretsConfig.load(SecretsConfig.find_config_file())
+    print(json.dumps(secrets.provider.get(key), indent=4 if pretty else None))