gitops 0.11.1__tar.gz → 0.11.4__tar.gz

{gitops-0.11.1 → gitops-0.11.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: gitops
-Version: 0.11.1
+Version: 0.11.4
 Summary: Manage multiple apps across one or more k8s clusters.
 Author-email: Jarek Głowacki <jarekwg@gmail.com>
 License-File: LICENSE

{gitops-0.11.1 → gitops-0.11.4}/charts/gitops/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v1
 appVersion: "1.0"
 description: GitOps Server Helm chart.
 name: gitops
-version: 0.11.1
+version: 0.11.4

{gitops-0.11.1 → gitops-0.11.4}/gitops/__init__.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 from .utils.cli import success, warning
 
-__version__ = "0.11.1"
+__version__ = "0.11.4"
 
 
 # Checking gitops version matches cluster repo version.

{gitops-0.11.1 → gitops-0.11.4}/gitops_server/utils/__init__.py

@@ -1,12 +1,25 @@
 import asyncio
 import logging
+from pathlib import Path
+from collections.abc import Callable
 
 from ..types import RunOutput
 
-logger = logging.getLogger("gitops")
+logger = logging.getLogger("gitops.run")
 
 
-async def run(command, suppress_errors=False) -> RunOutput:
+async def _read_stream(stream: asyncio.StreamReader | None, cb: Callable[[bytes], None]) -> None:
+    if stream is None:
+        return
+    while True:
+        line = await stream.readline()
+        if line:
+            cb(line)
+        else:
+            break
+
+
+async def run(command: str, suppress_errors: bool = False, cwd: str | Path | None = None) -> RunOutput:
     """Run a shell command.
 
     Runs the command in an asyncio executor to keep things async. Will
@@ -15,10 +28,26 @@ async def run(command, suppress_errors=False) -> RunOutput:
     exit_code = 0
     logger.info(f'Running "{command}".')
     proc = await asyncio.create_subprocess_shell(
-        command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+        command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=cwd
+    )
+    stdout, stderr = b"", b""
+
+    def log_stdout(line: bytes) -> None:
+        nonlocal stdout
+        stdout += line
+        logger.info(f"{line.decode()}")
+
+    def log_stderr(line: bytes) -> None:
+        nonlocal stderr
+        stderr += line
+        logger.info(f"{line.decode()}")
+
+    await asyncio.gather(
+        _read_stream(proc.stdout, log_stdout),
+        _read_stream(proc.stderr, log_stderr),
     )
 
-    stdout, stderr = await proc.communicate()
+    await proc.communicate()
     exit_code = proc.returncode if proc.returncode not in (None, 128) else 1  # type: ignore
     if exit_code == 0:
         return RunOutput(exit_code=exit_code, output=stdout.decode())

gitops-0.11.4/gitops_server/utils/git.py

@@ -0,0 +1,88 @@
+import asyncio
+import logging
+import os
+import re
+import tempfile
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from opentelemetry.trace import get_tracer
+
+from . import run
+
+tracer = get_tracer(__name__)
+logger = logging.getLogger("gitops")
+
+REPO_CACHE_DIR = Path("/tmp/gitops/repocache")
+REPO_CACHE: dict[str, Path] = {}
+repo_lock = asyncio.Semaphore(1)
+
+
+async def clone_repo(git_repo_url: str, path: str, sha: str | None = None, branch: str | None = None):
+    """Shallow Clones a git repo url to path and git-crypt unlocks all encrypted files"""
+    logger.info(f'Cloning "{git_repo_url}".')
+
+    url_with_oauth_token = git_repo_url.replace("://", f"://{os.environ['GITHUB_OAUTH_TOKEN'].strip()}@")
+
+    with tracer.start_as_current_span("tempo_repo.clone_repo"):
+        if sha:
+            await run(f"git clone --depth 100 {url_with_oauth_token} {path}; cd {path}; git checkout {sha}")
+        elif branch:
+            await run(f"git clone --depth 1 {url_with_oauth_token} {path} --branch {branch}")
+        else:
+            await run(f"git clone --depth 100 {url_with_oauth_token} {path};")
+
+    if GIT_CRYPT_KEY_FILE := os.environ.get("GIT_CRYPT_KEY_FILE"):
+        with tracer.start_as_current_span("temp_repo.git_crypt_unlock"):
+            await run(f"cd {path}; git-crypt unlock {GIT_CRYPT_KEY_FILE}")
+
+
+def is_sha(sha_or_ref: str | None) -> bool:
+    """Check if the given string is a valid SHA-1 hash or a short SHA."""
+    if not sha_or_ref:
+        return False
+    return bool(re.fullmatch(r"[a-fA-F0-9]{4,40}", sha_or_ref))
+
+
+@asynccontextmanager
+async def temp_repo(git_repo_url: str, ref: str | None) -> AsyncGenerator[str, None]:
+    """Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
+    # Someone passed us a branch name, not a sha
+
+    if not is_sha(ref):
+        async with temp_repo_branch(git_repo_url, branch=ref) as temp_repo_path:
+            yield temp_repo_path
+            return
+
+    with tracer.start_as_current_span("checkout_temp_repo"):
+        cache_path = REPO_CACHE_DIR / git_repo_url.split("/")[-1].split(".")[0]
+
+        # Prep the repo cache
+        if not (cache_path / ".git").exists():
+            logger.info("Repo %s not in cache, cloning", git_repo_url)
+            async with repo_lock:
+                if cache_path.exists():
+                    await run(f"rm -rf {cache_path}", suppress_errors=True)
+                if not cache_path.exists():
+                    cache_path.mkdir(parents=True)
+                REPO_CACHE[git_repo_url] = cache_path
+                await clone_repo(git_repo_url, path=str(cache_path))
+
+        # Copy the repo cache to a temporary folder
+        with tempfile.TemporaryDirectory() as temporary_folder_path:
+            await run(f"rm -rf {temporary_folder_path}", suppress_errors=True)
+            await run(f"cp -r {cache_path} {temporary_folder_path}")
+            await run(f"cd {temporary_folder_path};git fetch; git checkout {ref}")
+
+            yield temporary_folder_path
+
+
+@asynccontextmanager
+async def temp_repo_branch(git_repo_url: str, branch: str | None) -> AsyncGenerator[str, None]:
+    """Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
+    with tracer.start_as_current_span("checkout_temp_repo_branch"):
+        # Copy the repo cache to a temporary folder
+        with tempfile.TemporaryDirectory() as temporary_folder_path:
+            await clone_repo(git_repo_url, path=str(temporary_folder_path), branch=branch)
+            yield temporary_folder_path

{gitops-0.11.1 → gitops-0.11.4}/gitops_server/workers/deployer/deploy.py

@@ -65,7 +65,7 @@ async def post_result_summary(source: str, results: list[UpdateAppResult]):
 @tracer.start_as_current_span("load_app_definitions")
 async def load_app_definitions(url: str, sha: str) -> AppDefinitions:
     logger.info(f'Loading app definitions at "{sha}".')
-    async with temp_repo(url, sha=sha) as repo:
+    async with temp_repo(url, ref=sha) as repo:
         app_definitions = AppDefinitions(name=get_repo_name_from_url(url))
         app_definitions.from_path(repo)
         return app_definitions
@@ -115,33 +115,37 @@ class Deployer:
         )
 
     async def deploy(self):
-        with tracer.start_as_current_span("deploy"):
-            added_apps, updated_apps, removed_apps = self.calculate_app_deltas()
-            if not (added_apps | updated_apps | removed_apps):
-                logger.info("No deltas; aborting.")
-                return
-            logger.info(
-                f"Running deployment for these deltas: A{list(added_apps)}, U{list(updated_apps)},"
-                f" R{list(removed_apps)}"
-            )
-            await post_init_summary(
-                source=self.current_app_definitions.name,
-                username=self.author_name,
-                added_apps=added_apps,
-                updated_apps=updated_apps,
-                removed_apps=removed_apps,
-                commit_message=self.commit_message,
-            )
-            update_results = await asyncio.gather(
-                *[
-                    self.update_app_deployment(self.current_app_definitions.apps[app_name])
-                    for app_name in (added_apps | updated_apps)
-                ]
-            )
-            uninstall_results = await asyncio.gather(
-                *[self.uninstall_app(self.previous_app_definitions.apps[app_name]) for app_name in removed_apps]
-            )
-            await post_result_summary(self.current_app_definitions.name, update_results + uninstall_results)
+        added_apps, updated_apps, removed_apps = self.calculate_app_deltas()
+        current_span = trace.get_current_span()
+        if current_span:
+            current_span.set_attribute("gitops.added_apps", len(added_apps))
+            current_span.set_attribute("gitops.updated_aps", len(updated_apps))
+            current_span.set_attribute("gitops.removed_app", len(removed_apps))
+        if not (added_apps | updated_apps | removed_apps):
+            logger.info("No deltas; aborting.")
+            return
+        logger.info(
+            f"Running deployment for these deltas: A{list(added_apps)}, U{list(updated_apps)},"
+            f" R{list(removed_apps)}"
+        )
+        await post_init_summary(
+            source=self.current_app_definitions.name,
+            username=self.author_name,
+            added_apps=added_apps,
+            updated_apps=updated_apps,
+            removed_apps=removed_apps,
+            commit_message=self.commit_message,
+        )
+        update_results = await asyncio.gather(
+            *[
+                self.update_app_deployment(self.current_app_definitions.apps[app_name])
+                for app_name in (added_apps | updated_apps)
+            ]
+        )
+        uninstall_results = await asyncio.gather(
+            *[self.uninstall_app(self.previous_app_definitions.apps[app_name]) for app_name in removed_apps]
+        )
+        await post_result_summary(self.current_app_definitions.name, update_results + uninstall_results)
 
     async def uninstall_app(self, app: App) -> UpdateAppResult:
         with tracer.start_as_current_span("uninstall_app", attributes={"app": app.name}):
@@ -169,7 +173,7 @@ class Deployer:
                 if app.chart.type == "git":
                     span.set_attribute("gitops.chart.type", "git")
                     assert app.chart.git_repo_url
-                    async with temp_repo(app.chart.git_repo_url, sha=app.chart.git_sha) as chart_folder_path:
+                    async with temp_repo(app.chart.git_repo_url, ref=app.chart.git_sha) as chart_folder_path:
                         with tracer.start_as_current_span("helm_dependency_build"):
                             await run(f"cd {chart_folder_path}; helm dependency build")
 

{gitops-0.11.1 → gitops-0.11.4}/gitops_server/workers/deployer/worker.py

@@ -1,10 +1,14 @@
 import asyncio
 import logging
 
+from opentelemetry import trace
+
 from .deploy import Deployer
 
 logger = logging.getLogger("gitops_worker")
 
+tracer = trace.get_tracer(__name__)
+
 
 class DeployQueueWorker:
     """Simple synchronous background work queue.
@@ -52,5 +56,12 @@ class DeployQueueWorker:
         ref = work.get("ref")
         logger.info(f'Have a push to "{ref}".')
         if ref == "refs/heads/master":
-            deployer = await Deployer.from_push_event(work)
-            await deployer.deploy()
+            with tracer.start_as_current_span("gitops_process_webhook") as current_span:
+                deployer = await Deployer.from_push_event(work)
+                current_span.set_attribute("gitops.ref", ref)
+                current_span.set_attribute("gitops.after", work.get("after"))
+                current_span.set_attribute("gitops.before", work.get("before"))
+                current_span.set_attribute("gitops.author_email", deployer.author_email)
+                current_span.set_attribute("gitops.author_name", deployer.author_name)
+                current_span.set_attribute("gitops.commit_message", deployer.commit_message)
+                await deployer.deploy()

{gitops-0.11.1 → gitops-0.11.4}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "gitops"
-version = "0.11.1"
+version = "0.11.4"
 description = "Manage multiple apps across one or more k8s clusters."
 requires-python = ">=3.12"
 readme = "README.md"
@@ -44,6 +44,7 @@ dev-dependencies = [
     "types-pyyaml>=6.0.12.20240808",
     "types-tabulate>=0.9.0.20240106",
     "boto3-stubs>=1.35.11",
+    "pytest-asyncio>=0.24.0",
 ]
 
 

{gitops-0.11.1 → gitops-0.11.4}/release.py

@@ -4,8 +4,9 @@ import re
 pyproject = pathlib.Path("pyproject.toml")
 init = pathlib.Path("gitops/__init__.py")
 chart = pathlib.Path("charts/gitops/Chart.yaml")
+current = re.search(r'version = "(.*)"', pyproject.read_text()).group(1)  # type: ignore
 
-version = input("Enter a new version eg: (0.9.1): ").strip()
+version = input(f"Enter a new version. Currently at ({current}): ").strip()
 
 pyproject_text = pyproject.read_text()
 pyproject.write_text(re.sub(r'version = ".*"', f'version = "{version}"', pyproject_text, count=1))

gitops-0.11.4/tests/test_git.py

@@ -0,0 +1,50 @@
+import os
+import tempfile
+from contextlib import asynccontextmanager
+
+import pytest
+
+from gitops_server.utils import git, run
+
+os.environ["GITHUB_OAUTH_TOKEN"] = ""
+
+
+@asynccontextmanager
+async def make_dummy_repo():
+    with tempfile.TemporaryDirectory() as temporary_folder_path:
+        if os.environ.get("CI"):
+            await run("git config --global user.email 'you@example.com'")
+            await run("git config --global user.name 'Your Name'")
+            await run("git config --global init.defaultBranch main")
+        await run("git init", cwd=temporary_folder_path)
+        for _ in range(10):
+            await run("git commit -m 'temporary commit' --allow-empty", cwd=temporary_folder_path)
+        await run("git checkout -b 'test'", cwd=temporary_folder_path)
+        for _ in range(10):
+            await run("git commit -m 'temporary commit' --allow-empty", cwd=temporary_folder_path)
+        await run("git checkout main", cwd=temporary_folder_path)
+        yield temporary_folder_path
+
+
+@pytest.mark.asyncio
+class TestGit:
+    async def test_temp_repo_with_branch(self):
+        async with make_dummy_repo() as test_repo:
+            async with git.temp_repo(test_repo, ref="test"):
+                pass
+
+            async with git.temp_repo(test_repo, ref="main"):
+                pass
+
+    async def test_temp_repo_with_no_ref(self):
+        async with make_dummy_repo() as test_repo:
+            async with git.temp_repo(test_repo, ref=None):
+                pass
+
+    async def test_temp_repo_with_sha(self):
+        async with make_dummy_repo() as test_repo:
+            sha = (await run("git rev-parse HEAD", cwd=test_repo))["output"].strip()
+            assert git.is_sha(sha)
+
+            async with git.temp_repo(test_repo, ref=sha):
+                pass

{gitops-0.11.1 → gitops-0.11.4}/uv.lock

@@ -277,7 +277,7 @@ wheels = [
 
 [[package]]
 name = "gitops"
-version = "0.11.0"
+version = "0.11.3"
 source = { editable = "." }
 dependencies = [
     { name = "boto3" },
@@ -305,6 +305,7 @@ dev = [
     { name = "boto3-stubs" },
     { name = "mypy" },
     { name = "pytest" },
+    { name = "pytest-asyncio" },
     { name = "ruff" },
     { name = "types-colorama" },
     { name = "types-pyyaml" },
@@ -334,6 +335,7 @@ dev = [
     { name = "boto3-stubs", specifier = ">=1.35.11" },
     { name = "mypy", specifier = ">=1.11.2" },
     { name = "pytest", specifier = ">=8.3.2" },
+    { name = "pytest-asyncio", specifier = ">=0.24.0" },
     { name = "ruff", specifier = ">=0.6.3" },
     { name = "types-colorama", specifier = ">=0.4.15.20240311" },
     { name = "types-pyyaml", specifier = ">=6.0.12.20240808" },
@@ -941,6 +943,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0f/f9/cf155cf32ca7d6fa3601bc4c5dd19086af4b320b706919d48a4c79081cf9/pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5", size = 341802 },
 ]
 
+[[package]]
+name = "pytest-asyncio"
+version = "0.24.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/52/6d/c6cf50ce320cf8611df7a1254d86233b3df7cc07f9b5f5cbcb82e08aa534/pytest_asyncio-0.24.0.tar.gz", hash = "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276", size = 49855 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/96/31/6607dab48616902f76885dfcf62c08d929796fc3b2d2318faf9fd54dbed9/pytest_asyncio-0.24.0-py3-none-any.whl", hash = "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b", size = 18024 },
+]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"

gitops-0.11.1/gitops_server/utils/git.py

@@ -1,37 +0,0 @@
-import logging
-import os
-import tempfile
-from collections.abc import AsyncGenerator
-from contextlib import asynccontextmanager
-
-from opentelemetry.trace import get_tracer
-
-from . import run
-
-tracer = get_tracer(__name__)
-
-BASE_REPO_DIR = "/var/gitops/repos"
-
-logger = logging.getLogger("gitops")
-
-
-async def clone_repo(git_repo_url: str, path: str, sha: str | None = None):
-    """Shallow Clones a git repo url to path and git-crypt unlocks all encrypted files"""
-    logger.info(f'Cloning "{git_repo_url}".')
-
-    url_with_oauth_token = git_repo_url.replace("://", f"://{os.environ['GITHUB_OAUTH_TOKEN'].strip()}@")
-
-    with tracer.start_as_current_span("tempo_repo.clone_repo"):
-        await run(f"git clone {url_with_oauth_token} {path}; cd {path}; git checkout {sha}")
-
-    with tracer.start_as_current_span("temp_repo.git_crypt_unlock"):
-        await run(f'cd {path}; git-crypt unlock {os.environ["GIT_CRYPT_KEY_FILE"]}')
-
-
-@asynccontextmanager
-async def temp_repo(git_repo_url: str, sha: str | None = None) -> AsyncGenerator[str, None]:
-    """Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
-    with tracer.start_as_current_span("checkout_temp_repo"):
-        with tempfile.TemporaryDirectory() as temporary_folder_path:
-            await clone_repo(git_repo_url, path=temporary_folder_path, sha=sha)
-            yield temporary_folder_path