gitops 0.11.6__tar.gz → 0.12.0__tar.gz

{gitops-0.11.6 → gitops-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: gitops
-Version: 0.11.6
+Version: 0.12.0
 Summary: Manage multiple apps across one or more k8s clusters.
 Author-email: Jarek Głowacki <jarekwg@gmail.com>
 License-File: LICENSE

{gitops-0.11.6 → gitops-0.12.0}/charts/gitops/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v1
 appVersion: "1.0"
 description: GitOps Server Helm chart.
 name: gitops
-version: 0.11.6
+version: 0.12.0

{gitops-0.11.6 → gitops-0.12.0}/gitops/__init__.py

@@ -6,7 +6,7 @@ from gitops.utils.apps import App, get_app_details, get_apps
 
 from .utils.cli import success, warning
 
-__version__ = "0.11.6"
+__version__ = "0.12.0"
 
 
 # Checking gitops version matches cluster repo version.

{gitops-0.11.6 → gitops-0.12.0}/gitops/common/app.py

@@ -113,6 +113,11 @@ class App:
         """305686791668.dkr.ecr.ap-southeast-2.amazonaws.com/[uptick]:yoink-9f03ac80f3"""
         return self.image.split(":")[0].split("/")[-1]
 
+    @property
+    def image_repository(self) -> str:
+        """[305686791668.dkr.ecr.ap-southeast-2.amazonaws.com]/uptick:yoink-9f03ac80f3"""
+        return self.image.split("/")[0]
+
     @property
     def image_tag(self) -> str:
         """305686791668.dkr.ecr.ap-southeast-2.amazonaws.com/uptick:[yoink-9f03ac80f3]"""

{gitops-0.11.6 → gitops-0.12.0}/gitops/main.py

@@ -1,6 +1,10 @@
+import logging
+
 from invoke import Collection, Program, Task
 
-from . import __version__, core, db, shorthands
+from . import __version__, core, shorthands
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
 
 version = __version__
 
@@ -12,7 +16,4 @@ for core_ns in [core, shorthands]:
     for task in tasks:
         namespace.add_task(task)
 
-# Namespace the rarer ones.
-namespace.add_collection(db)  # type: ignore
-
 program = Program(namespace=namespace, version=version)

gitops-0.12.0/gitops/utils/images.py

@@ -0,0 +1,113 @@
+import datetime
+import logging
+from functools import lru_cache
+from hashlib import md5
+from typing import TYPE_CHECKING
+
+import boto3
+import botocore
+import botocore.exceptions
+from colorama import Fore
+
+from .cli import colourise
+
+if TYPE_CHECKING:
+    from mypy_boto3_ecr.type_defs import ImageDetailTypeDef  # type: ignore
+BATCH_SIZE = 100
+
+logger = logging.getLogger(__name__)
+
+
+def get_image(tag: str) -> str:
+    """Finds a specific image in ECR."""
+    # TODO
+    raise NotImplementedError
+
+
+@lru_cache
+def get_latest_image(repository_name: str, prefix: str, ecr_repository: str | None = None) -> str | None:  # noqa:C901
+    """Finds latest image in ECR with the given prefix and returns the image tag
+
+    param ecr_repository is expected in this format: 305686791668.dkr.ecr.ap-southeast-2.amazonaws.com
+    """
+    describe_image_args = {}
+    region_name = None
+    if ecr_repository:
+        account_id = ecr_repository.split(".")[0]
+        describe_image_args["registryId"] = account_id
+        region_name = ecr_repository.split(".")[3]
+
+    ecr_client = boto3.client("ecr", region_name=region_name)
+    client_paginator = ecr_client.get_paginator("describe_images")
+
+    results = []
+
+    # First we try to find the image with `*latest`
+    image_tag = f"{prefix}-latest" if prefix else "latest"
+
+    def add_image_to_results(image: "ImageDetailTypeDef") -> None:
+        if prefix != "":
+            if prefix_tags := [
+                tag for tag in image["imageTags"] if tag.startswith(prefix + "-") and "latest" not in tag
+            ]:
+                results.append((prefix_tags[0], image["imagePushedAt"]))
+        else:
+            if prefix_tags := [tag for tag in image["imageTags"] if "-" not in tag and "latest" not in tag]:
+                results.append((prefix_tags[0], image["imagePushedAt"]))
+
+    try:
+        image = ecr_client.describe_images(
+            repositoryName=repository_name,
+            imageIds=[{"imageTag": image_tag}],
+            **describe_image_args,  # type: ignore
+        )["imageDetails"][0]
+        # TODO: Remove this check after we've fully migrated to the new image tagging scheme
+        if image["imagePushedAt"] < datetime.datetime(2024, 9, 1, 0, 0, 0, tzinfo=datetime.timezone.utc):
+            logger.warning(f"Image {image_tag} is too old to be considered latest.")
+            raise ValueError("Image is too old")
+
+        add_image_to_results(image)
+    except (botocore.exceptions.ClientError, ValueError):
+        # Ok we couldn't find the -latest image; lets scan images manually
+        for ecr_response in client_paginator.paginate(
+            repositoryName=repository_name,
+            filter={
+                "tagStatus": "TAGGED",
+            },
+            **describe_image_args,  # type: ignore
+        ):
+            for image in ecr_response["imageDetails"]:
+                add_image_to_results(image)
+    if not results:
+        if prefix:
+            logger.info(f'No images found in repository: {repository_name} with tag "{prefix}-*".')
+        else:
+            logger.info(f"No images found in repository: {repository_name}")
+        return None
+
+    latest_image_tag = sorted(results, key=lambda image: image[1], reverse=True)[0][0]
+    return latest_image_tag
+
+
+def colour_image(image_tag: str) -> str:
+    if not image_tag:
+        return image_tag
+
+    bits = image_tag.split("-")
+    if len(bits) > 1:
+        bits[0] = colourise(bits[0], color_hash(bits[1]))
+        return "-".join(bits)
+    else:
+        return colourise(bits[0], color_hash(bits[0]))
+
+
+def color_hash(bit: str) -> str:
+    return [
+        Fore.RED,
+        Fore.GREEN,
+        Fore.YELLOW,
+        Fore.BLUE,
+        Fore.MAGENTA,
+        Fore.CYAN,
+        Fore.WHITE,
+    ][int.from_bytes(md5(bit.encode()).digest(), "big") % 7]  # noqa: S324

{gitops-0.11.6 → gitops-0.12.0}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "gitops"
-version = "0.11.6"
+version = "0.12.0"
 description = "Manage multiple apps across one or more k8s clusters."
 requires-python = ">=3.12"
 readme = "README.md"

gitops-0.11.6/gitops/db.py

@@ -1,254 +0,0 @@
-import asyncio
-import base64
-import datetime
-import os
-import random
-import time
-
-import boto3
-import dsnparse  # type: ignore[import]
-from invoke import run, task
-
-from .utils import kube
-from .utils.apps import get_app_details
-from .utils.cli import progress, success, warning
-
-# TODO: Abstract this code out of gitops.
-
-
-#####################
-# DATABASE COMMANDS #
-#####################
-@task
-def backup(ctx, app_name):
-    """Backup production or staging database."""
-    values = {"name": f"{app_name}-backup", "app": app_name}
-    app = get_app_details(app_name, load_secrets=False)
-    asyncio.run(
-        kube._run_job(
-            "jobs/backup-job.yml",
-            values,
-            context=app.cluster,
-            namespace=app.namespace,
-            sequential=True,
-        )
-    )
-
-
-@task
-def list_backups(ctx, app):
-    kube.list_backups("workforce", app)
-
-
-@task
-def restore_backup(ctx, app_name, index, cleanup=True):
-    """Restore backed up database."""
-    kube.confirm_database(app_name)
-    backups = kube.get_backups("workforce", app_name)
-    backup = backups[int(index) - 1]
-    values = {
-        "name": f"{app_name}-restore",
-        "timestamp": backup[0],
-        "app": app_name,
-    }
-    app = get_app_details(app_name, load_secrets=False)
-    asyncio.run(
-        kube._run_job(
-            "jobs/restore-job.yml",
-            values,
-            context=app.cluster,
-            namespace=app.namespace,
-            cleanup=cleanup,
-        )
-    )
-
-
-@task
-def copy_db(ctx, source, destination, skip_backup=False, cleanup=True):
-    """Copy database between apps."""
-    source_app = get_app_details(source, load_secrets=True)
-    destination_app = get_app_details(destination, load_secrets=True)
-    kube.confirm_database(destination)
-    values = {
-        "name": f"copy-db-{source}-{destination}",
-        "source": source,
-        "destination": destination,
-        "SOURCE_DATABASE_URL_ENCODED": source_app.values["secrets"]["DATABASE_URL"],
-        "DESTINATION_DATABASE_URL_ENCODED": destination_app.values["secrets"]["DATABASE_URL"],
-        "skip_backup": "skip" if skip_backup else "",
-    }
-    asyncio.run(
-        kube._run_job(
-            "jobs/copy-db-job.yml",
-            values,
-            context=source_app.cluster,
-            namespace=source_app.namespace,
-            cleanup=cleanup,
-        )
-    )
-    print(progress("You may want to clear the redis cache now!"))
-    print(progress(f"\t- gitops mcommand {destination} clear_cache"))
-
-    print(progress(":: Running migrations"))
-    run(f"gitops migrate {destination}", echo=True)
-
-
-@task
-def download_backup(ctx, app, index=None, path=None, datestamp=False):
-    """Download production or staging backup."""
-    if not index:
-        backups = kube.get_backups("workforce", app)
-        if not backups:
-            print(warning(f"No backups found for {app}"))
-            return
-        index = len(backups)
-        print(progress("No index specified. Downloading latest backup: " + backups[-1][3]))
-
-    kube.download_backup("workforce", app, index, path, datestamp=datestamp)
-
-
-@task
-def proxy(
-    ctx,
-    app_name,
-    local_port=None,
-    bastion_instance_id=None,
-    aws_availability_zone=None,
-    background=False,
-):
-    """Creates a proxy to RDS. Can supply either the app name or a DSN
-
-    Usage: gitops db.proxy app_name
-    or     gitops db.proxy postgres://...:...@5432/db
-    """
-
-    try:
-        database_url = app_name
-        database_dsn = dsnparse.parse(database_url)
-        app_name = database_dsn.user
-    except ValueError:
-        # app_name was not a valid dsn
-        app = get_app_details(app_name, load_secrets=True)
-        database_url = base64.b64decode(app.values["secrets"]["DATABASE_URL"].encode("ascii")).decode("ascii")
-        database_dsn = dsnparse.parse(database_url)
-
-    if not local_port:
-        local_port = random.randint(1000, 9999)  # noqa
-
-    # Maybe we need to connect via RDS IAM instead!
-    modified_dsn = dsnparse.parse(database_url)
-    modified_dsn.hostname = "localhost"
-    modified_dsn.port = local_port
-    rds_iam = False
-
-    if not database_dsn.password:
-        result = run(
-            f"aws rds generate-db-auth-token --hostname {database_dsn.host} --port"
-            f" {database_dsn.port} --username {database_dsn.user}",
-            hide=True,
-        )
-        modified_dsn.password = result.stdout.strip()
-        rds_iam = True
-
-    bastion_instance_id = bastion_instance_id or os.environ.get("GITOPS_BASTION_INSTANCE_ID")
-    if not bastion_instance_id:
-        raise Exception("Please set GITOPS_BASTION_INSTANCE_ID environment variable for db proxy to work.")
-
-    aws_availability_zone = aws_availability_zone or os.environ.get("GITOPS_AWS_AVAILABILITY_ZONE")
-    if not aws_availability_zone:
-        raise Exception("Please set GITOPS_AWS_AVAILABILITY_ZONE environment variable for db proxy to work.")
-
-    print(progress(f"Creating a proxy to the RDS instance of: {app_name} "))
-
-    if rds_iam:
-        proxy_dsn = (
-            f"'user={modified_dsn.user} password={modified_dsn.password} host={modified_dsn.hostname} dbname={modified_dsn.dbname}"
-            f" port={modified_dsn.port}'"
-        )
-    else:
-        proxy_dsn = modified_dsn.geturl()
-    print(
-        progress(
-            "Proxy established to :"
-            f"postgres://{modified_dsn.user}@{modified_dsn.hostname}:{modified_dsn.port}/{modified_dsn.dbname}\n"
-        )
-    )
-    print(progress(f"Connect to the db using: {proxy_dsn}\n"))
-    cmd = f"""aws ssm start-session  \
-        --target {bastion_instance_id}  \
-        --document-name AWS-StartPortForwardingSessionToRemoteHost \
-        --region {aws_availability_zone} \
-        --parameters='{{"host": ["{database_dsn.host}"], "portNumber":["{database_dsn.port}"],"localPortNumber":["{local_port}"]}}'
-    """
-    return proxy_dsn, run(cmd, hide=True, asynchronous=background)
-
-
-@task
-def pgcli(
-    ctx,
-    app_name,
-):
-    """Opens pgcli to a remote DB"""
-    print("making proxy")
-    proxy_dsn, ctx = proxy(ctx, app_name, background=True)
-    with ctx:
-        print("Waiting for proxy to open")
-        time.sleep(4)
-        run(f"pgcli {proxy_dsn}", pty=True)
-
-
-@task
-def logs(ctx, app_name, last=24):
-    """
-    Fetches RDS logs since the last N hours.
-
-    Usage:
-        gitops db.logs APP --last 5
-    """
-    rds = boto3.client("rds")
-    app = get_app_details(app_name, load_secrets=True)
-    db_name = app.name.replace("_", "")
-    logs = rds.describe_db_log_files(DBInstanceIdentifier=db_name)
-    for log in logs["DescribeDBLogFiles"][-last:]:
-        log_file = rds.download_db_log_file_portion(
-            DBInstanceIdentifier=db_name,
-            LogFileName=log["LogFileName"],
-        )
-        for line in log_file["LogFileData"].split("\n"):
-            if line:
-                print(
-                    str(datetime.datetime.fromisoformat(line[:19]).replace(tzinfo=datetime.timezone.utc).astimezone())
-                    + line[23:]
-                )
-
-
-@task
-def wipe_db(ctx, destination, skip_backup=False, cleanup=True):
-    """Wipes a customers database."""
-    source_app = get_app_details(destination, load_secrets=True)
-    kube.confirm_database(destination)
-    values = {
-        "name": f"wipe-db-{destination}",
-        "destination": destination,
-        "SOURCE_DATABASE_URL_ENCODED": source_app.values["secrets"]["DATABASE_URL"],
-        "DESTINATION_DATABASE_URL_ENCODED": source_app.values["secrets"]["DATABASE_URL"],
-        "skip_backup": "skip" if skip_backup else "",
-    }
-    print(progress(f":: Wiping database {destination}"))
-    asyncio.run(
-        kube._run_job(
-            "jobs/wipe-db-job.yml",
-            values,
-            context=source_app.cluster,
-            namespace=source_app.namespace,
-            cleanup=cleanup,
-        )
-    )
-    print(success("Deleted!"))
-    print(progress(":: Running migrations"))
-    run(f"gitops migrate {destination}", echo=True)
-
-    print(progress(":: Clearing cache"))
-    run(f"gitops mcommand {destination} clear_cache", echo=True)
-
-    print(progress("Success!"))

gitops-0.11.6/gitops/utils/images.py

@@ -1,70 +0,0 @@
-from functools import lru_cache
-from hashlib import md5
-
-import boto3
-from colorama import Fore
-
-from .cli import colourise
-
-BATCH_SIZE = 100
-
-
-def get_image(tag: str) -> str:
-    """Finds a specific image in ECR."""
-    # TODO
-    raise NotImplementedError
-
-
-@lru_cache
-def get_latest_image(repository_name: str, prefix: str) -> str | None:
-    """Finds latest image in ECR with the given prefix and returns the image tag"""
-    ecr_client = boto3.client("ecr")
-    client_paginator = ecr_client.get_paginator("describe_images")
-
-    results = []
-    for ecr_response in client_paginator.paginate(
-        repositoryName=repository_name,
-        filter={"tagStatus": "TAGGED"},
-        maxResults=BATCH_SIZE,
-    ):
-        for image in ecr_response["imageDetails"]:
-            if prefix != "":
-                if prefix_tags := [tag for tag in image["imageTags"] if tag.startswith(prefix + "-")]:
-                    results.append((prefix_tags[0], image["imagePushedAt"]))
-            else:
-                if prefix_tags := [tag for tag in image["imageTags"] if "-" not in tag]:
-                    results.append((prefix_tags[0], image["imagePushedAt"]))
-
-    if not results:
-        if prefix:
-            print(f'No images found in repository: {repository_name} with tag "{prefix}-*".')
-        else:
-            print(f"No images found in repository: {repository_name}")
-        return None
-
-    latest_image_tag = sorted(results, key=lambda image: image[1], reverse=True)[0][0]
-    return latest_image_tag
-
-
-def colour_image(image_tag: str) -> str:
-    if not image_tag:
-        return image_tag
-
-    bits = image_tag.split("-")
-    if len(bits) > 1:
-        bits[0] = colourise(bits[0], color_hash(bits[1]))
-        return "-".join(bits)
-    else:
-        return colourise(bits[0], color_hash(bits[0]))
-
-
-def color_hash(bit: str) -> str:
-    return [
-        Fore.RED,
-        Fore.GREEN,
-        Fore.YELLOW,
-        Fore.BLUE,
-        Fore.MAGENTA,
-        Fore.CYAN,
-        Fore.WHITE,
-    ][int.from_bytes(md5(bit.encode()).digest(), "big") % 7]  # noqa: S324