gitops 0.11.5__tar.gz → 0.12.0__tar.gz

{gitops-0.11.5 → gitops-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: gitops
-Version: 0.11.5
+Version: 0.12.0
 Summary: Manage multiple apps across one or more k8s clusters.
 Author-email: Jarek Głowacki <jarekwg@gmail.com>
 License-File: LICENSE

{gitops-0.11.5 → gitops-0.12.0}/charts/gitops/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v1
 appVersion: "1.0"
 description: GitOps Server Helm chart.
 name: gitops
-version: 0.11.5
+version: 0.12.0

{gitops-0.11.5 → gitops-0.12.0}/gitops/__init__.py

@@ -2,9 +2,11 @@ import os
 import sys
 from pathlib import Path
 
+from gitops.utils.apps import App, get_app_details, get_apps
+
 from .utils.cli import success, warning
 
-__version__ = "0.11.5"
+__version__ = "0.12.0"
 
 
 # Checking gitops version matches cluster repo version.
@@ -25,3 +27,5 @@ if versions_path.exists():
                 f" requirement {success(min_gitops_version)}.",
                 file=sys.stderr,
             )
+
+__all__ = ["App", "get_app_details", "get_apps", "__version__"]

{gitops-0.11.5 → gitops-0.12.0}/gitops/common/app.py

@@ -24,8 +24,19 @@ class App:
         deployments: dict | None = None,
         secrets: dict | None = None,
         load_secrets: bool = True,
+        encode_secrets: bool = True,
+        # deprecated
         account_id: str = "",
     ) -> None:
+        """
+        :params name: The name of the app
+        :params path: The path to the app directory
+        :params deployments: The deployments dictionary (injecting for testing)
+        :params secrets: The secrets dictionary (injecting for testing)
+        :params load_secrets: Whether to load the secrets file
+        :params encode_secrets: Whether to base64 encode the secrets
+        """
+        self.encode_secrets = encode_secrets
         self.name = name
         self.path = path
         self.account_id = account_id
@@ -47,6 +58,9 @@ class App:
             and json.dumps(self.values, sort_keys=True) == json.dumps(other.values, sort_keys=True)
         )
 
+    def __repr__(self) -> str:
+        return f"App(name={self.name}, cluster={self.cluster}, tags={self.tags})"
+
     def is_inactive(self) -> bool:
         return "inactive" in self.values.get("tags", [])
 
@@ -61,11 +75,11 @@ class App:
             current_dict = current_dict.setdefault(key, {})
         current_dict[keys[-1]] = value
 
-    def _make_values(self, deployments: dict, secrets: dict) -> dict:
-        values = {
-            **deployments,
-            "secrets": {**{k: b64encode(v.encode()).decode() for k, v in secrets.items()}},
-        }
+    def _make_values(self, deployments: dict, secrets: dict[str, str]) -> dict:
+        def encode(value: str) -> str:
+            return b64encode(str(value).encode()).decode() if self.encode_secrets else value
+
+        values = {**deployments, "secrets": {**{k: encode(v) for k, v in secrets.items()}}}
 
         image = self._make_image(deployments)
         if image:
@@ -99,6 +113,11 @@ class App:
         """305686791668.dkr.ecr.ap-southeast-2.amazonaws.com/[uptick]:yoink-9f03ac80f3"""
         return self.image.split(":")[0].split("/")[-1]
 
+    @property
+    def image_repository(self) -> str:
+        """[305686791668.dkr.ecr.ap-southeast-2.amazonaws.com]/uptick:yoink-9f03ac80f3"""
+        return self.image.split("/")[0]
+
     @property
     def image_tag(self) -> str:
         """305686791668.dkr.ecr.ap-southeast-2.amazonaws.com/uptick:[yoink-9f03ac80f3]"""
@@ -127,6 +146,11 @@ class App:
     def service_account_name(self) -> str:
         return self.values.get("serviceAccount", {}).get("name") or self.values.get("serviceAccountName") or "default"
 
+    @property
+    def secrets(self) -> dict[str, str]:
+        # TODO: This should be a first class property
+        return self.values.get("secrets", {})
+
 
 class Chart:
     """Represents a Helm chart

{gitops-0.11.5 → gitops-0.12.0}/gitops/main.py

@@ -1,6 +1,10 @@
+import logging
+
 from invoke import Collection, Program, Task
 
-from . import __version__, core, db, shorthands
+from . import __version__, core, shorthands
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
 
 version = __version__
 
@@ -12,7 +16,4 @@ for core_ns in [core, shorthands]:
     for task in tasks:
         namespace.add_task(task)
 
-# Namespace the rarer ones.
-namespace.add_collection(db)  # type: ignore
-
 program = Program(namespace=namespace, version=version)

{gitops-0.11.5 → gitops-0.12.0}/gitops/utils/apps.py

@@ -1,5 +1,6 @@
 import sys
 from pathlib import Path
+from typing import Literal
 
 from colorama import Fore
 from tabulate import tabulate
@@ -21,13 +22,16 @@ def is_valid_app_directory(directory: Path) -> bool:
     return all(file_paths)
 
 
-def get_app_details(app_name: str, load_secrets: bool = True, exit_if_not_found: bool = True) -> App:
+def get_app_details(
+    app_name: str, load_secrets: bool = True, encode_secrets: bool = True, exit_if_not_found: bool = True
+) -> App:
     account_id = get_account_id() if load_secrets else "UNKNOWN"
     try:
         app = App(
             app_name,
             path=str(get_apps_directory() / app_name),
             load_secrets=load_secrets,
+            encode_secrets=encode_secrets,
             account_id=account_id,
         )
     except FileNotFoundError as e:
@@ -63,10 +67,11 @@ def update_app(app_name: str, **kwargs: object) -> None:
 def get_apps(  # noqa: C901
     filter: set[str] | list[str] | str = "",
     exclude: set[str] | list[str] | str = "",
-    mode: str = "PROMPT",
+    mode: Literal["PROMPT", "PREVIEW", "SILENT"] | None = "PROMPT",
     autoexclude_inactive: bool = True,
     message: str | None = None,
     load_secrets: bool = True,
+    encode_secrets: bool = True,
 ) -> list[App]:
     """Return apps that contain ALL of the tags listed in `filter` and NONE of the tags listed in
     `exclude`. The incoming filter and exclude params may come in as a list or commastring.
@@ -101,7 +106,7 @@ def get_apps(  # noqa: C901
             continue
         elif not is_valid_app_directory(entry):
             continue
-        app = get_app_details(entry.name, load_secrets=load_secrets)
+        app = get_app_details(entry.name, load_secrets=load_secrets, encode_secrets=encode_secrets)
 
         pseudotags = [app.name, app.cluster]
         if app.image and app.image_prefix:

gitops-0.12.0/gitops/utils/images.py

@@ -0,0 +1,113 @@
+import datetime
+import logging
+from functools import lru_cache
+from hashlib import md5
+from typing import TYPE_CHECKING
+
+import boto3
+import botocore
+import botocore.exceptions
+from colorama import Fore
+
+from .cli import colourise
+
+if TYPE_CHECKING:
+    from mypy_boto3_ecr.type_defs import ImageDetailTypeDef  # type: ignore
+BATCH_SIZE = 100
+
+logger = logging.getLogger(__name__)
+
+
+def get_image(tag: str) -> str:
+    """Finds a specific image in ECR."""
+    # TODO
+    raise NotImplementedError
+
+
+@lru_cache
+def get_latest_image(repository_name: str, prefix: str, ecr_repository: str | None = None) -> str | None:  # noqa:C901
+    """Finds latest image in ECR with the given prefix and returns the image tag
+
+    param ecr_repository is expected in this format: 305686791668.dkr.ecr.ap-southeast-2.amazonaws.com
+    """
+    describe_image_args = {}
+    region_name = None
+    if ecr_repository:
+        account_id = ecr_repository.split(".")[0]
+        describe_image_args["registryId"] = account_id
+        region_name = ecr_repository.split(".")[3]
+
+    ecr_client = boto3.client("ecr", region_name=region_name)
+    client_paginator = ecr_client.get_paginator("describe_images")
+
+    results = []
+
+    # First we try to find the image with `*latest`
+    image_tag = f"{prefix}-latest" if prefix else "latest"
+
+    def add_image_to_results(image: "ImageDetailTypeDef") -> None:
+        if prefix != "":
+            if prefix_tags := [
+                tag for tag in image["imageTags"] if tag.startswith(prefix + "-") and "latest" not in tag
+            ]:
+                results.append((prefix_tags[0], image["imagePushedAt"]))
+        else:
+            if prefix_tags := [tag for tag in image["imageTags"] if "-" not in tag and "latest" not in tag]:
+                results.append((prefix_tags[0], image["imagePushedAt"]))
+
+    try:
+        image = ecr_client.describe_images(
+            repositoryName=repository_name,
+            imageIds=[{"imageTag": image_tag}],
+            **describe_image_args,  # type: ignore
+        )["imageDetails"][0]
+        # TODO: Remove this check after we've fully migrated to the new image tagging scheme
+        if image["imagePushedAt"] < datetime.datetime(2024, 9, 1, 0, 0, 0, tzinfo=datetime.timezone.utc):
+            logger.warning(f"Image {image_tag} is too old to be considered latest.")
+            raise ValueError("Image is too old")
+
+        add_image_to_results(image)
+    except (botocore.exceptions.ClientError, ValueError):
+        # Ok we couldn't find the -latest image; lets scan images manually
+        for ecr_response in client_paginator.paginate(
+            repositoryName=repository_name,
+            filter={
+                "tagStatus": "TAGGED",
+            },
+            **describe_image_args,  # type: ignore
+        ):
+            for image in ecr_response["imageDetails"]:
+                add_image_to_results(image)
+    if not results:
+        if prefix:
+            logger.info(f'No images found in repository: {repository_name} with tag "{prefix}-*".')
+        else:
+            logger.info(f"No images found in repository: {repository_name}")
+        return None
+
+    latest_image_tag = sorted(results, key=lambda image: image[1], reverse=True)[0][0]
+    return latest_image_tag
+
+
+def colour_image(image_tag: str) -> str:
+    if not image_tag:
+        return image_tag
+
+    bits = image_tag.split("-")
+    if len(bits) > 1:
+        bits[0] = colourise(bits[0], color_hash(bits[1]))
+        return "-".join(bits)
+    else:
+        return colourise(bits[0], color_hash(bits[0]))
+
+
+def color_hash(bit: str) -> str:
+    return [
+        Fore.RED,
+        Fore.GREEN,
+        Fore.YELLOW,
+        Fore.BLUE,
+        Fore.MAGENTA,
+        Fore.CYAN,
+        Fore.WHITE,
+    ][int.from_bytes(md5(bit.encode()).digest(), "big") % 7]  # noqa: S324

gitops-0.12.0/gitops_server/types.py

@@ -0,0 +1,40 @@
+import logging
+import os
+from typing import TypedDict
+
+from gitops.common.app import App
+from gitops_server import settings
+
+logger = logging.getLogger("gitops")
+
+
+class RunOutput(TypedDict):
+    exit_code: int
+    output: str
+
+
+class UpdateAppResult(RunOutput):
+    app_name: str
+    slack_message: str
+
+
+class AppDefinitions:
+    def __init__(self, name, apps: dict[str, App] | None = None, path: str | None = None):
+        self.name = name
+        self.apps = apps or {}
+
+        if path:
+            path = os.path.join(path, "apps")
+
+            for entry in os.listdir(path):
+                entry_path = os.path.join(path, entry)
+                if entry[0] != "." and not os.path.isfile(entry_path):
+                    app = App(entry, entry_path, account_id=settings.ACCOUNT_ID)
+                    self.apps[entry] = app
+
+        # Removing apps that are suspended or not part of this cluster
+        for app in list(self.apps.values()):
+            # We only care for apps pertaining to our current cluster.
+            if app.values["cluster"] != settings.CLUSTER_NAME or "suspended" in app.tags:
+                # and suspended apps are considered removed from the cluster.
+                self.apps.pop(app.name)

{gitops-0.11.5 → gitops-0.12.0}/gitops_server/workers/deployer/deploy.py

@@ -67,8 +67,7 @@ async def post_result_summary(source: str, results: list[UpdateAppResult]):
 async def load_app_definitions(url: str, sha: str) -> AppDefinitions:
     logger.info(f'Loading app definitions at "{sha}".')
     async with temp_repo(url, ref=sha) as repo:
-        app_definitions = AppDefinitions(name=get_repo_name_from_url(url))
-        app_definitions.from_path(repo)
+        app_definitions = AppDefinitions(name=get_repo_name_from_url(url), path=repo)
         return app_definitions
 
 

{gitops-0.11.5 → gitops-0.12.0}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "gitops"
-version = "0.11.5"
+version = "0.12.0"
 description = "Manage multiple apps across one or more k8s clusters."
 requires-python = ">=3.12"
 readme = "README.md"

gitops-0.12.0/tests/conftest.py

@@ -0,0 +1,3 @@
+import gitops_server.settings
+
+gitops_server.settings.CLUSTER_NAME = "test-cluster"

{gitops-0.11.5 → gitops-0.12.0}/tests/test_app.py

@@ -105,3 +105,18 @@ class TestChart:
 
         assert chart.type == "local"
         assert chart.path == "."
+
+    def test_app_namespace_property(self):
+        path = create_test_yaml()
+        app = App("test", path, encode_secrets=True)
+
+        assert app.namespace
+
+    def test_app_encode_secrets_works(self):
+        path = create_test_yaml()
+        app = App("test", path, encode_secrets=True)
+
+        assert app.secrets["SNAPE"] != "KILLS_DUMBLEDORE"
+
+        app_decoded = App("test", path, encode_secrets=False)
+        assert app_decoded.secrets["SNAPE"] == "KILLS_DUMBLEDORE"

{gitops-0.11.5 → gitops-0.12.0}/tests/test_deploy.py

@@ -4,10 +4,11 @@ from unittest.mock import patch
 import pytest
 
 from gitops.common.app import App
+from gitops_server.types import AppDefinitions
 from gitops_server.workers.deployer import Deployer
 
 from .sample_data import SAMPLE_GITHUB_PAYLOAD, SAMPLE_GITHUB_PAYLOAD_SKIP_MIGRATIONS
-from .utils import mock_load_app_definitions
+from .utils import create_test_yaml, mock_load_app_definitions
 
 # Patch gitops_server.git.run & check correct commands + order
 # Patch command that reads yaml from cluster repo +
@@ -120,3 +121,16 @@ class TestDeploy:
         ]
         for where, check in check_in_run_mock:
             assert check in post_mock.call_args_list[where][0][0]
+
+
+class TestLoadAppDefinitions:
+    def test_load_app_definitions_ignores_suspended_apps(self):
+        app_definitions = AppDefinitions(
+            "mock-repo",
+            apps={
+                "in-cluster": App("in-cluster", path=create_test_yaml(tags=[])),
+                "suspended": App("suspended", path=create_test_yaml(tags=["suspended"])),
+                "not-in-cluster": App("not-in-cluster", path=create_test_yaml(tags=[], cluster="other-cluster")),
+            },
+        )
+        assert len(app_definitions.apps) == 1

{gitops-0.11.5 → gitops-0.12.0}/tests/utils.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 import yaml
 
 from gitops.common.app import App
@@ -20,14 +22,14 @@ async def mock_load_app_definitions(url, sha):
     return app_definitions
 
 
-def create_test_yaml(fg=4, bg=2):
+def create_test_yaml(fg=4, bg=2, **kwargs: Any):
     data = {
         "chart": "https://github.com/some/chart",
         "images": {"template": "template-tag-{tag}"},
         "namespace": "mynamespace",
         "tags": ["tag1", "tag2"],
         "image-tag": "myimagetag",
-        "cluster": "UNKNOWN",
+        "cluster": "test-cluster",
         "containers": {"fg": {"replicas": fg}, "bg": {"replicas": bg}},
         "environment": {
             "DJANGO_SETTINGS_MODULE": "my.settings.module",
@@ -36,6 +38,10 @@ def create_test_yaml(fg=4, bg=2):
             "MEDIA_CLOUDINARY_PREFIX": "cloudinaryprefix",
         },
     }
+
+    for k, v in kwargs.items():
+        data[k] = v
+
     with open("/tmp/deployment.yml", "w+") as fh:
         fh.write(yaml.dump(data))