edge-containers-cli 3.0.4__tar.gz → 3.1.1__tar.gz

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/.github/workflows/_release.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Create GitHub Release
         # We pin to the SHA, not the tag, for security reasons.
         # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions
-        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15
+        uses: softprops/action-gh-release@9d7c94cfd0a1f3ed45544c887983e9fa900f0564 # v0.1.15
         with:
           prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
           files: "*"

edge-containers-cli-3.1.1/.github/workflows/_sys_test.yml

@@ -0,0 +1,36 @@
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        type: string
+        description: The version of python to install
+        required: true
+      runs-on:
+        type: string
+        description: The runner to run this job on
+        required: true
+
+jobs:
+  run:
+    runs-on: ${{ inputs.runs-on }}
+    env:
+      EC_INTERACTIVE_TESTING: "true"
+      EC_K8S_NAMESPACE: "default"
+      EC_SERVICES_REPO: "https://github.com/marcelldls/bl01t.git"
+    steps:
+      - name: start minikube
+        id: minikube
+        uses: medyagh/setup-minikube@master
+      - run: kubectl version
+      - run: helm version
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install latest versions of python packages
+        uses: ./.github/actions/install_requirements
+        with:
+          python-version: ${{ inputs.python-version }}
+          pip-install: ".[dev]"
+      - name: Set up namespace
+        run: ec deploy epics-pvcs 2024.2.1
+      - name: system tests
+        run: tox -e tests

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/.github/workflows/ci.yml

@@ -34,6 +34,19 @@ jobs:
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  system:
+    needs: check
+    if: needs.check.outputs.branch-pr == ''
+    strategy:
+      matrix:
+        runs-on: ["ubuntu-latest"]
+        python-version: ["3.11"]
+      fail-fast: false
+    uses: ./.github/workflows/_sys_test.yml
+    with:
+      runs-on: ${{ matrix.runs-on }}
+      python-version: ${{ matrix.python-version }}
+
   dist:
     needs: check
     if: needs.check.outputs.branch-pr == ''

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edge-containers-cli
-Version: 3.0.4
+Version: 3.1.1
 Summary: CLI for deploying and managing epics containers IOCs and services
 Author-email: Giles Knap <giles.knap@gmail.com>
 License:                                  Apache License
@@ -219,8 +219,7 @@ Requires-Dist: typer[all]
 Requires-Dist: requests
 Requires-Dist: ruamel.yaml
 Requires-Dist: jinja2
-Requires-Dist: pandas
-Requires-Dist: pyarrow
+Requires-Dist: polars
 Provides-Extra: dev
 Requires-Dist: copier; extra == "dev"
 Requires-Dist: pipdeptree; extra == "dev"

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/pyproject.toml

@@ -19,8 +19,7 @@ dependencies = [
     "requests",
     "ruamel.yaml",
     "jinja2",
-    "pandas",
-    "pyarrow",
+    "polars",
 ]
 
 dynamic = ["version"]

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/src/edge_containers_cli/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.0.4'
-__version_tuple__ = version_tuple = (3, 0, 4)
+__version__ = version = '3.1.1'
+__version_tuple__ = version_tuple = (3, 1, 1)

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/src/edge_containers_cli/cmds/cli.py

@@ -1,6 +1,7 @@
 import tempfile
 from pathlib import Path
 
+import polars
 import typer
 from natsort import natsorted
 
@@ -159,18 +160,14 @@ def list(
     ctx: typer.Context,
 ):
     """List all IOCs/services available in the helm registry"""
-    typer.echo(typer.style(f"{'Available services:':35}Latest instance:", bold=True))
     tmp_dir = Path(tempfile.mkdtemp())
     svc_graph = create_svc_graph(ctx.obj.beamline_repo, tmp_dir)
     svc_list = natsorted(svc_graph.keys())
     log.debug(f"svc_graph = {svc_graph}")
 
-    for svc in svc_list:
-        if len(svc_graph[svc]) == 0:
-            latest_instance = "None found."
-        else:
-            latest_instance = natsorted(svc_graph[svc])[-1]
-        typer.echo(f"{svc:35}{latest_instance}")
+    versions = [natsorted(svc_graph[svc])[-1] for svc in svc_list]
+    services_df = polars.from_dict({"name": svc_list, "version": versions})
+    print(services_df)
 
     cleanup_temp(tmp_dir)
 
@@ -183,7 +180,6 @@ def instances(
     ),
 ):
     """List all versions of the IOC/service available in the helm registry"""
-    typer.echo(f"Available instances for {service_name}:")
     tmp_dir = Path(tempfile.mkdtemp())
     svc_graph = create_svc_graph(ctx.obj.beamline_repo, tmp_dir)
     try:
@@ -192,7 +188,8 @@ def instances(
         svc_list = []
 
     sorted_list = natsorted(svc_list)[::-1]
-    typer.echo("  ".join(sorted_list))
+    services_df = polars.from_dict({"version": sorted_list})
+    print(services_df)
 
     cleanup_temp(tmp_dir)
 

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/src/edge_containers_cli/cmds/helm.py

@@ -1,5 +1,4 @@
 import tempfile
-from datetime import datetime
 from pathlib import Path
 from typing import Optional
 
@@ -7,7 +6,13 @@ import typer
 
 import edge_containers_cli.globals as globals
 import edge_containers_cli.shell as shell
-from edge_containers_cli.utils import chdir, check_instance_path, cleanup_temp, log
+from edge_containers_cli.utils import (
+    chdir,
+    check_instance_path,
+    cleanup_temp,
+    local_version,
+    log,
+)
 
 
 class Helm:
@@ -31,9 +36,7 @@ class Helm:
         self.beamline_repo = repo
         self.namespace = namespace
         self.args = args
-        self.version = version or datetime.strftime(
-            datetime.now(), "%Y.%-m.%-d-b%-H.%-M"
-        )
+        self.version = version or local_version()
         self.template = template
 
         self.tmp = Path(tempfile.mkdtemp())

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/src/edge_containers_cli/cmds/k8s_commands.py

@@ -10,13 +10,13 @@ from io import StringIO
 from pathlib import Path
 from typing import Optional
 
-import pandas as pd
+import polars
 import typer
 
 import edge_containers_cli.globals as globals
 import edge_containers_cli.shell as shell
 from edge_containers_cli.cmds.helm import Helm
-from edge_containers_cli.cmds.kubectl import json_service_info, json_service_types
+from edge_containers_cli.cmds.kubectl import jsonpath_deploy_info, jsonpath_pod_info
 from edge_containers_cli.logging import log
 
 
@@ -72,7 +72,6 @@ class K8sCommands:
     ):
         self.namespace: str = ""
         self.beamline_repo: str = ""
-        self.service_type: str = ""
         # TODO isnt ctx always set??
         if ctx is not None:
             namespace = ctx.namespace
@@ -170,27 +169,70 @@ class K8sCommands:
     def ps(self, all: bool, wide: bool):
         """List all IOCs and Services in the current namespace"""
 
-        helm_json = shell.run_command(
-            f"helm list -n {self.namespace} -o json", interactive=False
-        )
-        df = pd.read_json(StringIO(str(helm_json)))
-        log.debug(df)
+        services_df = polars.DataFrame()
 
-        if not all:
-            # only show running services - get the pods list and merge with helm list
-            services_csv = shell.run_command(
-                f"kubectl get pods -n {self.namespace} {json_service_info}",
+        # Gives all services (running & not running) and their image
+        for resource in ["deployment", "statefulset"]:
+            kubectl_res = shell.run_command(
+                f"kubectl get {resource} -n {self.namespace} {jsonpath_deploy_info}",
                 interactive=False,
             )
-            pods_df = pd.read_csv(  # type: ignore
-                StringIO(str(services_csv)),
-                names=json_service_types.keys(),  # type: ignore
-                dtype=json_service_types,  # type: ignore
+            if kubectl_res:
+                res_df = polars.read_csv(
+                    StringIO(str(kubectl_res)),
+                    separator=",",
+                    has_header=False,
+                    new_columns=["name", "image"],
+                )
+                log.debug(res_df)
+                services_df = polars.concat([services_df, res_df], how="diagonal")
+        if services_df.is_empty():
+            print("No deployed services found")
+            raise typer.Exit()
+
+        # Gives the status, restarts for running services
+        kubectl_gtpo = shell.run_command(
+            f"kubectl get pods -n {self.namespace} {jsonpath_pod_info}",
+            interactive=False,
+        )
+        if kubectl_gtpo:
+            gtpo_df = polars.read_csv(
+                StringIO(str(kubectl_gtpo)),
+                separator=",",
+                has_header=False,
+                new_columns=["name", "running", "restarts"],
             )
-            log.debug(pods_df)
-
-            df = pd.merge(pods_df, df, left_on="name", right_on="name")
-
-        df.drop(columns=["revision", "updated", "status", "chart"], inplace=True)
+            services_df = services_df.join(gtpo_df, on="name", how="left")
+            services_df = services_df.with_columns(
+                polars.col("running").replace({"Running": True}, default=False),
+                polars.col("restarts").fill_null(0),
+            )
+        elif all:
+            services_df = services_df.with_columns(
+                running=polars.lit(False), restarts=polars.lit(0)
+            )
+        else:
+            print("No running services found")
+            raise typer.Exit()
 
-        print(df.to_string(index=False))
+        # Adds the version, deployment time for all services
+        helm_out = shell.run_command(
+            f"helm list -n {self.namespace} -o json", interactive=False
+        )
+        helm_df = polars.read_json(StringIO(str(helm_out)))
+        helm_df = helm_df.rename({"app_version": "version", "updated": "deployed"})
+        helm_df = helm_df.with_columns(polars.col("deployed").str.slice(0, 19))
+        services_df = services_df.join(helm_df, on="name", how="left")
+        log.debug(services_df)
+
+        # Arrange columns
+        services_df = services_df.select(
+            ["name", "version", "running", "restarts", "deployed", "image"]
+        )
+        if not all:
+            services_df = services_df.filter(polars.col("running").eq(True))
+            log.debug(services_df)
+        if not wide:
+            services_df.drop_in_place("image")
+            log.debug(services_df)
+        print(services_df)

edge-containers-cli-3.1.1/src/edge_containers_cli/cmds/kubectl.py

@@ -0,0 +1,15 @@
+jsonpath_pod_info = (
+    "-o jsonpath='"
+    r'{range .items[*]}{..labels.app}{","}{.status.phase}'
+    r'{","}{..containerStatuses[0].restartCount}'
+    r'{"\n"}{end}'
+    "'"
+)
+
+jsonpath_deploy_info = (
+    "-o jsonpath='"
+    r'{range .items[*]}{.metadata.name}{","}'
+    r"{range .spec.template.spec.containers[*]}{.image}"
+    r'{"\n"}{end}{end}'
+    "'"
+)

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/src/edge_containers_cli/cmds/local_commands.py

@@ -9,6 +9,7 @@ However, for the moment, Using this by connecting to each server and running
 tool like Portainer is a decent workflow.
 """
 
+import json
 import re
 import tempfile
 from datetime import datetime
@@ -16,7 +17,7 @@ from io import StringIO
 from pathlib import Path
 from typing import Optional
 
-import pandas as pd
+import polars
 import requests
 import typer
 
@@ -31,6 +32,7 @@ from edge_containers_cli.utils import (
     cleanup_temp,
     generic_ioc_from_image,
     get_instance_image_name,
+    local_version,
 )
 
 
@@ -73,7 +75,7 @@ class LocalCommands:
         self.docker.remove(self.service_name)
 
     def _do_deploy(self, ioc_instance: Path, version: str, args: str):
-        service_name, ioc_path = check_instance_path(ioc_instance)
+        service_name, _ = check_instance_path(ioc_instance)
 
         image = get_instance_image_name(ioc_instance)
         log.debug(f"deploying {ioc_instance} with image {image}")
@@ -120,7 +122,7 @@ class LocalCommands:
         Use a local copy of an ioc instance definition to deploy a temporary
         version of the IOC to the local docker instance
         """
-        version = datetime.strftime(datetime.now(), "%Y.%-m.%-d-b%-H.%-M")
+        version = local_version()
         if not yes:
             typer.echo(
                 f"Deploy TEMPORARY version {version} "
@@ -163,35 +165,54 @@ class LocalCommands:
     def ps(self, all: bool, wide: bool):
         all_arg = " --all" if all else ""
 
-        # We have to build the table ourselves because docker is unable to
-        # format a table with labels.
-        format = "{{.Names}}%{{.Labels}}%{{.State}}%{{.Image}}"
-
-        result = shell.run_command(
-            f"{self.docker.docker} ps{all_arg} --filter label=is_IOC=true "
-            f'--format "{format}"',
+        # List services
+        avail_services = shell.run_command(
+            f"{self.docker.docker} ps{all_arg} -q --filter label=is_IOC=true",
             interactive=False,
         )
 
-        # this regex extracts just the version from the set of all labels,
-        # docker and podman have different output formats
-        if self.docker.is_docker:
-            result = re.sub(r"%.*?[,%]version=([^,%]*).*?%", r"%\1%", str(result))
-        else:
-            result = re.sub(r"%.*? version:([^\],%]*).*?%", r"%\1%", str(result))
-
-        result = result.replace("%", ",")
-
-        df = pd.read_csv(  # type: ignore
-            StringIO(result),
-            names=["name", "version", "state", "image"],  # type: ignore
+        if not avail_services:
+            if all_arg:
+                print("No deployed services found")
+            else:
+                print("No running services found")
+            raise typer.Exit()
+
+        # Retrieve data
+        result_json = shell.run_command(
+            f"{self.docker.docker} inspect "
+            f"$({self.docker.docker} ps{all_arg} -q --filter label=is_IOC=true)",
+            interactive=False,
         )
-        log.debug(df)
+        log.debug(result_json)
+        services_dicts = json.load(StringIO(str(result_json)))
+
+        select_data = []
+        for service in services_dicts:
+            # Make docker output look like podman
+            if self.docker.is_docker:
+                service["Name"] = service["Name"][1:]  # inspect leads name with /
+
+            # Note if adding more keys that there are differences
+            # between what is available between docker and podman
+            select_data.append(
+                {
+                    "name": service["Name"],
+                    "version": service["Config"]["Labels"]["version"],
+                    "running": "true" if service["State"]["Running"] else "false",
+                    "restarts": service["RestartCount"],
+                    "deployed": datetime.strptime(
+                        service["Created"].split(".")[0], "%Y-%m-%dT%H:%M:%S"
+                    ),
+                    "image": service["Config"]["Image"],
+                }
+            )
+        log.debug(select_data)
 
-        if len(df.index) == 0:
-            typer.echo("No running services found")
-        else:
-            print(df.to_string(index=False))
+        services_df = polars.DataFrame(select_data)
+        if not wide:
+            services_df.drop_in_place("image")
+        print(services_df)
 
     def validate_instance(self, ioc_instance: Path):
         check_instance_path(ioc_instance)

{edge-containers-cli-3.0.4 → edge-containers-cli-3.1.1}/src/edge_containers_cli/docker.py

@@ -3,10 +3,8 @@ Utility functions for working interacting with docker / podman CLI
 """
 
 import re
-import sys
 from pathlib import Path
 from time import sleep
-from typing import Optional
 
 import typer
 
@@ -14,12 +12,6 @@ import edge_containers_cli.globals as globals
 import edge_containers_cli.shell as shell
 from edge_containers_cli.logging import log
 
-IMAGE_TAG = "local"
-MOUNTED_FILES = ["/.bashrc", "/.inputrc", "/.bash_eternal_history"]
-
-# podman needs this security option to allow containers to mount tmp etc.
-PODMAN_OPT = " --security-opt=label=type:container_runtime_t"
-
 
 class Docker:
     """
@@ -67,78 +59,6 @@ class Docker:
 
         log.debug(f"buildx={self.is_buildx} ({result})")
 
-    def _all_params(
-        self, args: str, mounts: Optional[list[Path]] = None, exec: bool = False
-    ):
-        """
-        set up parameters for call to docker/podman
-        """
-        opts = PODMAN_OPT if not self.is_docker and not exec else ""
-
-        if self.devcontainer:
-            if sys.stdin.isatty():
-                # interactive
-                env = "-e DISPLAY -e SHELL -e TERM -it"
-            else:
-                env = "-e DISPLAY -e SHELL"
-
-            volumes = ""
-            for file in MOUNTED_FILES:
-                file_path = Path(file)
-                if file_path.exists():
-                    volumes += f" -v {file}:/root/{file_path.name}"
-            if mounts is not None:
-                for mount in mounts:
-                    volumes += f" -v {mount}"
-
-            log.debug(f"env={env} volumes={volumes} opts={opts}")
-
-            params = f"{env}{opts}{volumes}" + f" {args}" if args else ""
-        else:
-            params = f"{opts}" + (f"{args}" if args else "")
-
-        return params
-
-    def run(self, name: str, args: str = "", mounts: Optional[list[Path]] = None):
-        """
-        run a command in a local container
-        """
-        params = self._all_params(args, mounts=mounts)
-        shell.run_command(
-            f"{self.docker} run --rm --name {name} {params}", interactive=True
-        )
-
-    def build(
-        self,
-        context: str,
-        name: str,
-        target: str,
-        args: str = "",
-        cache_from: str = "",
-        cache_to: str = "",
-        push: bool = False,
-        arch: globals.Architecture = globals.Architecture.linux,
-    ):
-        """
-        build a container
-        """
-        if self.is_buildx:
-            cmd = f"{self.docker} buildx"
-            shell.run_command(
-                f"{cmd} create --driver docker-container --use", interactive=False
-            )
-            args += f" --cache-from={cache_from}" if cache_from else ""
-            args += f" --cache-to={cache_to},mode=max" if cache_to else ""
-            args += " --push" if push else " --load "
-        else:
-            cmd = f"{self.docker}"
-
-        t_arch = f" --build-arg TARGET_ARCHITECTURE={arch}" if self.devcontainer else ""
-
-        shell.run_command(
-            f"{cmd} build --target {target}{t_arch} {args} -t {name} {context}"
-        )
-
     def exec(
         self,
         container: str,

edge-containers-cli-3.1.1/src/edge_containers_cli/git.py

@@ -0,0 +1,71 @@
+"""
+Utility functions for working with git
+"""
+
+import os
+from pathlib import Path
+
+import edge_containers_cli.shell as shell
+from edge_containers_cli.logging import log
+from edge_containers_cli.shell import check_services_repo
+from edge_containers_cli.utils import chdir
+
+
+def create_svc_graph(repo: str, folder: Path) -> dict:
+    """
+    return a dictionary of the available IOCs (by discovering the children
+    to the services/ folder in the beamline repo) as well as a list of the corresponding
+    available versions for each IOC (by discovering the tags in the beamline repo at
+    which changes to the instance were made since the last tag) and the respective
+    list of available versions
+    """
+    svc_graph = {}
+
+    check_services_repo(repo)
+    shell.run_command(f"git clone {repo} {folder}", interactive=False)
+    path_list = os.listdir(os.path.join(folder, "services"))
+    service_list = [
+        path
+        for path in path_list
+        if os.path.isdir(os.path.join(folder, "services", path))
+    ]
+    log.debug(f"service_list = {service_list}")
+
+    with chdir(folder):  # From python 3.11 can use contextlib.chdir(folder)
+        for service_name in service_list:
+            service_name = Path(service_name).name
+            result = str(
+                shell.run_command("git tag --sort=committerdate", interactive=False)
+            )
+            log.debug(f"checking these tags for changes in the instance: {result}")
+
+            version_list = []
+            tags = result.split("\n")
+            tags.remove("")
+
+            for tag_no, _ in enumerate(tags):
+                # Check initial configuration
+                if not tag_no:
+                    cmd = f"git ls-tree -r {tags[tag_no]} --name-only"
+                    result = str(
+                        shell.run_command(cmd, interactive=False, error_OK=True)
+                    )
+                    if service_name in result:
+                        version_list.append(tags[tag_no])
+
+                # Check repo changes
+                else:
+                    cmd = f"git diff --name-only {tags[tag_no-1]} {tags[tag_no]}"
+                    result = str(
+                        shell.run_command(cmd, interactive=False, error_OK=True)
+                    )
+                    if service_name in result:
+                        version_list.append(tags[tag_no])
+
+            # Capture services committed since the most recent tag
+            if not version_list:
+                version_list.append("")
+
+            svc_graph[service_name] = version_list
+
+    return svc_graph