nyl 0.0.2__tar.gz → 0.0.3__tar.gz

{nyl-0.0.2 → nyl-0.0.3}/PKG-INFO

@@ -1,20 +1,23 @@
 Metadata-Version: 2.1
 Name: nyl
-Version: 0.0.2
+Version: 0.0.3
 Summary: Default template for PDM package
 Author-Email: Niklas Rosenstein <rosensteinniklas@gmail.com>
 License: MIT
 Requires-Python: >=3.10
+Requires-Dist: bcrypt>=4.2.0
 Requires-Dist: databind>=4.5.2
-Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: filelock>=3.15.4
 Requires-Dist: jinja2>=3.1.4
-Requires-Dist: loguru>=0.7.2
 Requires-Dist: kubernetes>=30.1.0
-Requires-Dist: typer>=0.12.3
-Requires-Dist: filelock>=3.15.4
-Requires-Dist: stablehash>=0.2.0
+Requires-Dist: loguru>=0.7.2
 Requires-Dist: nr-stream>=1.1.5
+Requires-Dist: pyyaml>=6.0.1
 Requires-Dist: requests>=2.32.3
+Requires-Dist: stablehash>=0.2.0
+Requires-Dist: structured-templates<0.1.0,>=0.0.5
+Requires-Dist: typer>=0.12.3
+Requires-Dist: typing-extensions>=4.12.2
 Description-Content-Type: text/markdown
 
 # nyl

{nyl-0.0.2 → nyl-0.0.3}/pyproject.toml

@@ -1,21 +1,24 @@
 [project]
 name = "nyl"
-version = "0.0.2"
+version = "0.0.3"
 description = "Default template for PDM package"
 authors = [
     { name = "Niklas Rosenstein", email = "rosensteinniklas@gmail.com" },
 ]
 dependencies = [
+    "bcrypt>=4.2.0",
     "databind>=4.5.2",
-    "pyyaml>=6.0.1",
+    "filelock>=3.15.4",
     "jinja2>=3.1.4",
-    "loguru>=0.7.2",
     "kubernetes>=30.1.0",
-    "typer>=0.12.3",
-    "filelock>=3.15.4",
-    "stablehash>=0.2.0",
+    "loguru>=0.7.2",
     "nr-stream>=1.1.5",
+    "pyyaml>=6.0.1",
     "requests>=2.32.3",
+    "stablehash>=0.2.0",
+    "structured-templates>=0.0.5,<0.1.0",
+    "typer>=0.12.3",
+    "typing-extensions>=4.12.2",
 ]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -35,16 +38,19 @@ build-backend = "pdm.backend"
 [tool.pdm]
 distribution = true
 
+[tool.pdm.scripts]
+lint = "bash -c 'ruff format && ruff check --fix && mypy src' && pytest"
+
 [tool.pdm.dev-dependencies]
 dev = [
-    "ruff>=0.4.8",
-    "mypy>=1.10.0",
-    "types-pyyaml>=6.0.12.20240311",
-    "mkdocs>=1.6.0",
+    "kubernetes-stubs>=22.6.0.post1",
     "mkdocs-material>=9.5.28",
+    "mkdocs>=1.6.0",
     "mkdocstrings[python]>=0.25.1",
+    "mypy>=1.10.0",
     "pytest>=8.2.2",
-    "kubernetes-stubs>=22.6.0.post1",
+    "ruff>=0.4.8",
+    "types-pyyaml>=6.0.12.20240311",
     "types-requests>=2.32.0.20240712",
 ]
 

nyl-0.0.3/src/nyl/commands/__init__.py

@@ -0,0 +1,43 @@
+"""
+Nyl is a flexible configuration management tool for Kubernetes resources that can be used to generate and deploy
+applications directly or integrate as an ArgoCD ConfigManagementPlugin.
+"""
+
+from enum import Enum
+import sys
+from loguru import logger
+from typer import Option
+from nyl.tools.typer import new_typer
+
+
+app = new_typer(help=__doc__)
+
+
+from . import crds  # noqa: F401,E402
+from . import new  # noqa: E402
+from . import profile  # noqa: E402
+from . import secrets  # noqa: E402
+from . import template  # noqa: F401,E402
+from . import tun  # noqa: E402
+
+app.add_typer(new.app)
+app.add_typer(profile.app)
+app.add_typer(secrets.app)
+app.add_typer(tun.app)
+
+
+class LogLevel(str, Enum):
+    TRACE = "trace"
+    DEBUG = "debug"
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+    CRITICAL = "critical"
+
+
+@app.callback()
+def _callback(
+    log_level: LogLevel = Option(LogLevel.INFO, "--log-level", "-l", help="The log level to use."),
+) -> None:
+    logger.remove()
+    logger.add(sys.stderr, level=log_level.name)

nyl-0.0.3/src/nyl/commands/crds.py

@@ -0,0 +1,14 @@
+import yaml
+
+from nyl.resources.applyset import ApplySet
+from . import app
+
+
+@app.command()
+def crds() -> None:
+    """
+    Print out the CRDs that need to be installed on a Kubernetes cluster before you can use all of Nyl's features.
+    """
+
+    print("---")
+    print(yaml.safe_dump(ApplySet.CRD))

nyl-0.0.3/src/nyl/commands/new.py

@@ -0,0 +1,91 @@
+"""
+Bootstrap files for a new Nyl project or Helm chart.
+"""
+
+from pathlib import Path
+from textwrap import dedent
+
+from loguru import logger
+from nyl.tools.typer import new_typer
+
+app = new_typer(name="new", help=__doc__)
+
+
+def _write_file_dedent(dir: Path, name: str, content: str) -> None:
+    path = dir.joinpath(name)
+    if path.exists():
+        logger.warning("File already exists: {}", path)
+        return
+    logger.info("Writing {}", path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(dedent(content).lstrip())
+
+
+@app.command()
+def chart(dir: Path) -> None:
+    """
+    Similar to `helm create`, but generates a much simpler template.
+    """
+
+    dir.mkdir(parents=True, exist_ok=True)
+
+    _write_file_dedent(
+        dir,
+        "Chart.yaml",
+        f"""
+        apiVersion: v2
+        name: {dir.name}
+        version: '0.0.0'
+        """,
+    )
+
+    _write_file_dedent(
+        dir,
+        "values.yaml",
+        """
+        image: my/image:tag
+        """,
+    )
+
+    _write_file_dedent(
+        dir,
+        "values.schema.json",
+        """
+        {
+            "$schema": "https://json-schema.org/draft/2020-12/output/schema",
+            "type": "object",
+            "required": [
+                "image"
+            ],
+            "properties": {
+                "image": {
+                    "type": "string"
+                }
+            }
+        }
+        """,
+    )
+
+    _write_file_dedent(
+        dir,
+        "templates/deployment.yaml",
+        """
+        apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+            name: {{ .Release.Name }}
+        spec:
+            replicas: 1
+            selector:
+            matchLabels:
+                app: {{ .Release.Name }}
+            template:
+            metadata:
+                labels:
+                app: {{ .Release.Name }}
+            spec:
+                containers:
+                - name: {{ .Release.Name }}
+                image: {{ .Values.image }}
+        """,
+    )

{nyl-0.0.2 → nyl-0.0.3}/src/nyl/commands/profile.py

@@ -23,8 +23,5 @@ def activate(profile_name: str = Argument("default", envvar="NYL_PROFILE")) -> N
     with ProfileManager.load() as manager:
         profile = manager.activate_profile(profile_name)
 
-    # For standard tooling, like kubectl.
-    print(f"export KUBECONFIG={shlex.quote(str(profile.kubeconfig.absolute()))}")
-
-    # Used by Terraform providers.
-    print(f"export KUBE_CONFIG_PATH={shlex.quote(str(profile.kubeconfig.absolute()))}")
+    for key, value in profile.env.items():
+        print(f"export {key}={shlex.quote(value)}")

{nyl-0.0.2 → nyl-0.0.3}/src/nyl/commands/secrets.py

@@ -16,7 +16,7 @@ def list() -> None:
     List the keys for all secrets in the provider.
     """
 
-    secrets = SecretsConfig.load(SecretsConfig.find_config_file())
+    secrets = SecretsConfig.load()
     for key in secrets.provider.keys():
         print(key)
 
@@ -27,5 +27,5 @@ def get(key: str, pretty: bool = False) -> None:
     Get the value of a secret as JSON.
     """
 
-    secrets = SecretsConfig.load(SecretsConfig.find_config_file())
+    secrets = SecretsConfig.load()
     print(json.dumps(secrets.provider.get(key), indent=4 if pretty else None))

nyl-0.0.3/src/nyl/commands/template.py

@@ -0,0 +1,311 @@
+import atexit
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+from loguru import logger
+from structured_templates import TemplateEngine
+from typer import Argument, Option
+import yaml
+from nyl.generator import reconcile_generator
+from nyl.generator.dispatch import DispatchingGenerator
+from nyl.profiles import ProfileManager
+from kubernetes.config.incluster_config import load_incluster_config
+from kubernetes.config.kube_config import load_kube_config
+from kubernetes.client.api_client import ApiClient
+from nyl.project.config import ProjectConfig
+from nyl.resources.applyset import APPLYSET_LABEL_PART_OF, ApplySet
+from nyl.secrets.config import SecretsConfig
+from nyl.tools.kubectl import Kubectl
+from nyl.tools.types import Manifest, Manifests
+
+from . import app
+
+
+@dataclass
+class ManifestsWithSource:
+    """
+    Represents a list of manifests loaded from a particular source file.
+    """
+
+    manifests: Manifests
+    file: Path
+
+
+@app.command()
+def template(
+    paths: list[Path] = Argument(..., help="The YAML file(s) to render. Can be a directory."),
+    profile: str = Option("default", envvar="NYL_PROFILE", help="The Nyl profile to use."),
+    in_cluster: bool = Option(
+        False, help="Use the in-cluster Kubernetes configuration. The --profile option is ignored."
+    ),
+    apply: bool = Option(
+        False,
+        help="Run `kubectl apply` on the rendered manifests, once for each source file. "
+        "Implies `--no-applyset-part-of`. When an ApplySet is defined in the source file, it will be applied "
+        "separately. Note that this option implies `kubectl --prune`.",
+    ),
+    applyset_part_of: bool = Option(
+        True,
+        help="Add the 'applyset.kubernetes.io/part-of' label to all resources belonging to an ApplySet (if declared). "
+        "This option must be disabled when passing the generated manifests to `kubectl apply --applyset=...`, as it "
+        "would otherwise cause an error due to the label being present on the input data.",
+    ),
+) -> None:
+    """
+    Render a package template into full Kubernetes resources.
+    """
+
+    if apply:
+        # When running with --apply, we must ensure that the --applyset-part-of option is disabled, as it would cause
+        # an error when passing the generated manifests to `kubectl apply --applyset=...`.
+        applyset_part_of = False
+
+    kubectl = Kubectl()
+    kubectl.env["KUBECTL_APPLYSET"] = "true"
+    atexit.register(kubectl.cleanup)
+
+    if in_cluster:
+        logger.info("Using in-cluster configuration.")
+        load_incluster_config()
+    else:
+        with ProfileManager.load() as profiles:
+            active = profiles.activate_profile(profile)
+            logger.info(f"Using profile '{profile}' with kubeconfig '{active.kubeconfig}'.")
+            load_kube_config(str(active.kubeconfig))
+
+    project = ProjectConfig.load()
+    if project.file:
+        state_dir = project.file.parent / ".nyl"
+    else:
+        state_dir = Path(".nyl")
+
+    secrets = SecretsConfig.load()
+
+    template_engine = TemplateEngine(
+        globals_={
+            "secrets": secrets.provider,
+            "random_password": _random_password,
+            "bcrypt": _bcrypt,
+        }
+    )
+
+    generator = DispatchingGenerator.default(
+        git_repo_cache_dir=state_dir / "repo-cache",
+        search_path=project.config.search_path,
+        working_dir=Path.cwd(),
+        client=ApiClient(),
+    )
+
+    for source in load_manifests(paths):
+        logger.info("Rendering manifests from {}", source.file)
+
+        source.manifests = cast(Manifests, template_engine.evaluate(source.manifests))
+        source.manifests = reconcile_generator(
+            generator,
+            source.manifests,
+            on_generated=lambda m: cast(Manifest, template_engine.evaluate(m)),
+        )
+
+        # Find the namespaces that are defined in the file. If we find any manifests without a namespace, we will
+        # inject that namespace name into them.
+        # Also find the applyset defined in the file.
+        namespaces: set[str] = set()
+        applyset: ApplySet | None = None
+        for manifest in list(source.manifests):
+            if is_namespace_resource(manifest):
+                namespaces.add(manifest["metadata"]["name"])
+            elif ApplySet.matches(manifest):
+                if applyset is not None:
+                    logger.error(
+                        "Multiple ApplySet resources defined in '{}', there can only be one per source.",
+                        source.file,
+                    )
+                    exit(1)
+                applyset = ApplySet.load(manifest)
+                source.manifests.remove(manifest)
+
+        if applyset is not None:
+            applyset.set_group_kinds(source.manifests)
+            # HACK: Kubectl 1.30 can't create the custom resource, so we need to create it. But it will also reject
+            #       using the custom resource unless it has the tooling label set appropriately. For more details, see
+            #       https://github.com/NiklasRosenstein/nyl/issues/5.
+            applyset.tooling = "kubectl/1.30"
+            applyset.validate()
+
+            if apply:
+                # We need to ensure that ApplySet parent object exists before invoking `kubectl apply --applyset=...`.
+                logger.info("Kubectl-apply ApplySet resource '{}' from '{}'", applyset.reference, source.file)
+                kubectl.apply(Manifests([applyset.dump()]), force_conflicts=True)
+            else:
+                print("---")
+                print(yaml.safe_dump(applyset.dump()))
+
+        # Find all manifests without a namespace and inject the namespace name into them.
+        # If there is an applyset, ensure they are marked as part of the applyset.
+        for manifest in source.manifests:
+            if applyset is not None and applyset_part_of:
+                if APPLYSET_LABEL_PART_OF not in (labels := manifest["metadata"].setdefault("labels", {})):
+                    labels[APPLYSET_LABEL_PART_OF] = applyset.id
+
+            if not is_cluster_scoped_resource(manifest) and "namespace" not in manifest["metadata"]:
+                if len(namespaces) > 1:
+                    logger.error(
+                        "Multiple namespaces defined in '{}', but manifest {}/{} does not have a namespace.",
+                        source.file,
+                        manifest["kind"],
+                        manifest["metadata"]["name"],
+                    )
+                    exit(1)
+                elif len(namespaces) == 0:
+                    logger.warning(
+                        "No namespaces defined in '{}', injecting 'default' into manifest {}/{}.",
+                        source.file,
+                        manifest["kind"],
+                        manifest["metadata"]["name"],
+                    )
+                    manifest["metadata"]["namespace"] = "default"
+                else:
+                    logger.trace(
+                        "Injecting namespace '{}' into manifest {}/{}.",
+                        next(iter(namespaces)),
+                        manifest["kind"],
+                        manifest["metadata"]["name"],
+                    )
+                    manifest["metadata"]["namespace"] = next(iter(namespaces))
+            elif not is_cluster_scoped_resource(manifest) and manifest["metadata"]["namespace"] not in namespaces:
+                logger.warning(
+                    "Resource '{}/{}' in '{}' references namespace '{}', which is not defined in the file.",
+                    manifest["kind"],
+                    manifest["metadata"]["name"],
+                    source.file,
+                    manifest["metadata"]["namespace"],
+                )
+
+            if not apply:
+                print("---")
+                print(yaml.safe_dump(manifest))
+
+        if apply:
+            logger.info("Kubectl-apply {} manifest(s) from '{}'", len(source.manifests), source.file)
+            kubectl.apply(
+                manifests=source.manifests,
+                applyset=applyset.reference if applyset else None,
+                prune=True if applyset else False,
+                force_conflicts=True,
+            )
+
+
+def load_manifests(paths: list[Path]) -> list[ManifestsWithSource]:
+    """
+    Load all manifests from a directory.
+    """
+
+    logger.trace("Loading manifests from paths: {}", paths)
+
+    files = []
+    for path in paths:
+        if path.is_dir():
+            for item in path.iterdir():
+                if item.name.startswith("nyl-") or item.suffix != ".yaml" or not item.is_file():
+                    continue
+                files.append(item)
+        else:
+            files.append(path)
+
+    logger.trace("Files to load: {}", files)
+
+    result = []
+    for file in files:
+        manifests = Manifests(list(map(Manifest, yaml.safe_load_all(file.read_text()))))
+        result.append(ManifestsWithSource(manifests, file))
+
+    return result
+
+    #     # Check if the resource has any references and try to resolve them. If a reference cannot be resolved, the
+    #     # manifest must be skipped. We emit a warning and continue with the next manifest.
+    #     refs = list(Reference.collect(manifest))
+    #     if refs:
+    #         skip_resource = False
+    #         resolves = {}
+    #         for ref in refs:
+    #             # TODO: Determine the namespace to fall back to.
+    #             try:
+    #                 result = k8s.get(
+    #                     resource=k8s.resources.get(api_version="v1", kind=ref.kind, group=""),
+    #                     name=ref.name,
+    #                     namespace=ref.namespace,  # TODO: Determine the namespace to backfill to.
+    #                 )
+
+    #                 value = result["data"][ref.key]
+    #                 if value is None:
+    #                     raise KeyError
+    #                 assert isinstance(value, str)
+    #                 resolves[str(ref)] = base64.b64decode(value.encode("ascii")).decode("utf-8")
+    #             except NotFoundError:
+    #                 logger.warning(
+    #                     "Skipping resource {}/{} because its reference to {} could not be resolved.",
+    #                     manifest["apiVersion"],
+    #                     manifest["kind"],
+    #                     ref,
+    #                 )
+    #                 skip_resource = True
+    #                 break
+    #             except KeyError:
+    #                 logger.warning(
+    #                     "Skipping resource {}/{} because its reference to {} could not be resolved (does not contain key {}).",
+    #                     manifest["apiVersion"],
+    #                     manifest["kind"],
+    #                     ref,
+    #                     ref.key,
+    #                 )
+    #                 skip_resource = True
+    #                 break
+    #         if skip_resource:
+    #             continue
+    #         manifest = Reference.sub(manifest, lambda ref: resolves[str(ref)])
+
+
+def _random_password(length: int = 32) -> str:
+    """
+    Generate a random password.
+    """
+
+    import secrets
+
+    return secrets.token_urlsafe(length)
+
+
+def _bcrypt(password: str) -> str:
+    """
+    Hash a password using bcrypt.
+    """
+
+    import bcrypt
+
+    return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8")
+
+
+def is_namespace_resource(manifest: Manifest) -> bool:
+    """
+    Check if a manifest is a namespace resource.
+    """
+
+    return manifest.get("apiVersion") == "v1" and manifest.get("kind") == "Namespace"
+
+
+def is_cluster_scoped_resource(manifest: Manifest) -> bool:
+    """
+    Check if a manifest is a cluster scoped resource.
+    """
+
+    # HACK: We should probably just list the resources via the Kubectl API?
+    fqn = manifest.get("kind", "") + "." + manifest.get("apiVersion", "").split("/")[0]
+    return fqn in {
+        "ClusterRole.rbac.authorization.k8s.io",
+        "ClusterRoleBinding.rbac.authorization.k8s.io",
+        "CustomResourceDefinition.apiextensions.k8s.io",
+        "IngressClass.networking.k8s.io",
+        "Namespace.v1",
+        "StorageClass.storage.k8s.io",
+        "ValidatingWebhookConfiguration.admissionregistration.k8s.io",
+    }

{nyl-0.0.2 → nyl-0.0.3}/src/nyl/commands/tun.py

@@ -2,8 +2,6 @@
 Access SSH tunnels globally managed by Nyl.
 """
 
-from typing import Optional
-
 from loguru import logger
 from typer import Argument
 from rich.console import Console
@@ -24,7 +22,7 @@ def status(all: bool = False) -> None:
     Show the status of all tunnels.
     """
 
-    config = ProfileConfig.load(ProfileConfig.find_config_file(not_found_ok=True))
+    config = ProfileConfig.load(required=False)
 
     table = Table()
     table.add_column("Profile", justify="right", style="cyan")
@@ -48,6 +46,7 @@ def status(all: bool = False) -> None:
         # on the system, while there may be tunnel information available for them from previous runs.
         for profile_name, profile in config.profiles.items():
             if profile_name not in tunnel_by_profile and profile.tunnel:
+                assert config.file is not None, "Profile configuration file must be set."
                 tunnels.append((get_tunnel_spec(config.file, profile_name, profile.tunnel), TunnelStatus.empty()))
 
         tunnels.sort(key=lambda x: (x[0].locator.profile, x[0].locator.config_file))
@@ -76,12 +75,12 @@ def status(all: bool = False) -> None:
 
 
 @app.command()
-def start(profile_name: str) -> None:
+def start(profile_name: str = Argument("default", envvar="NYL_PROFILE")) -> None:
     """
     Open a tunnel to the cluster targeted by the profile.
     """
 
-    config = ProfileConfig.load(ProfileConfig.find_config_file())
+    config = ProfileConfig.load()
 
     try:
         profile = config.profiles[profile_name]
@@ -106,17 +105,17 @@ def start(profile_name: str) -> None:
 
 
 @app.command()
-def stop(profile_name: Optional[str] = Argument(None)) -> None:
+def stop(profile_name: str = Argument("default", envvar="NYL_PROFILE"), all: bool = False) -> None:
     """
     Close all tunnels or the tunnel for a specific profile.
     """
 
-    if profile_name is None:
+    if all:
         with TunnelManager() as manager:
             for spec, _status in manager.get_tunnels():
                 manager.close_tunnel(spec.locator)
         return
 
-    config = ProfileConfig.load(ProfileConfig.find_config_file())
+    config = ProfileConfig.load()
     with TunnelManager() as manager:
         manager.close_tunnel(TunnelSpec.Locator(str(config.file), profile_name))