paasta-tools 1.4.25__tar.gz → 1.4.27__tar.gz

{paasta-tools-1.4.25/paasta_tools.egg-info → paasta-tools-1.4.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.25
+Version: 1.4.27
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.25 → paasta-tools-1.4.27}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.25"
+__version__ = "1.4.27"

{paasta-tools-1.4.25 → paasta-tools-1.4.27}/paasta_tools/oom_logger.py

@@ -33,10 +33,16 @@ log {
   destination(paasta_oom_logger);
 };
 """
+import argparse
+import json
 import re
 import sys
 from collections import namedtuple
+from typing import Dict
 
+import grpc
+from containerd.services.containers.v1 import containers_pb2
+from containerd.services.containers.v1 import containers_pb2_grpc
 from docker.errors import APIError
 
 from paasta_tools.cli.utils import get_instance_config
@@ -76,6 +82,16 @@ LogLine = namedtuple(
 )
 
 
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="paasta_oom_logger")
+    parser.add_argument(
+        "--containerd",
+        action="store_true",
+        help="Use containerd to inspect containers, otherwise use docker",
+    )
+    return parser.parse_args()
+
+
 def capture_oom_events_from_stdin():
     process_name_regex = re.compile(
         r"^\d+\s[a-zA-Z0-9\-]+\s.*\]\s(.+)\sinvoked\soom-killer:"
@@ -93,6 +109,25 @@ def capture_oom_events_from_stdin():
         """,
         re.VERBOSE,
     )
+    oom_regex_kubernetes_containerd_systemd_cgroup = re.compile(
+        r"""
+        ^(\d+)\s # timestamp
+        ([a-zA-Z0-9\-]+) # hostname
+        \s.*oom-kill:.*task_memcg=/.*\.slice/.* # loosely match systemd slice and containerid
+        cri-containerd:(\w{64}).*$ # containerid
+        """,
+        re.VERBOSE,
+    )
+
+    oom_regex_kubernetes_containerd_systemd_cgroup_structured = re.compile(
+        r"""
+        ^(\d+)\s # timestamp
+        ([a-zA-Z0-9\-]+) # hostname
+        \s.*oom-kill:.*task_memcg=/kubepods\.slice/.* # match systemd slice and containerid
+        cri-containerd-(\w{64}).*$ # containerid
+        """,
+        re.VERBOSE,
+    )
     oom_regex_kubernetes_structured = re.compile(
         r"""
         ^(\d+)\s # timestamp
@@ -115,6 +150,8 @@ def capture_oom_events_from_stdin():
         oom_regex_kubernetes,
         oom_regex_kubernetes_structured,
         oom_regex_kubernetes_systemd_cgroup,
+        oom_regex_kubernetes_containerd_systemd_cgroup,
+        oom_regex_kubernetes_containerd_systemd_cgroup_structured,
     ]
 
     process_name = ""
@@ -136,11 +173,18 @@ def capture_oom_events_from_stdin():
                 break
 
 
-def get_container_env_as_dict(docker_inspect):
+def get_container_env_as_dict(
+    is_cri_containerd: bool, container_inspect: dict
+) -> Dict[str, str]:
     env_vars = {}
-    config = docker_inspect.get("Config")
+    if is_cri_containerd:
+        config = container_inspect.get("process")
+        env_key = "env"
+    else:
+        config = container_inspect.get("Config")
+        env_key = "Env"
     if config is not None:
-        env = config.get("Env", [])
+        env = config.get(env_key, [])
         for i in env:
             name, _, value = i.partition("=")
             env_vars[name] = value
@@ -209,18 +253,26 @@ def send_sfx_event(service, instance, cluster):
         counter.count()
 
 
+def get_containerd_container(container_id: str) -> containers_pb2.Container:
+    with grpc.insecure_channel("unix:///run/containerd/containerd.sock") as channel:
+        containersv1 = containers_pb2_grpc.ContainersStub(channel)
+        return containersv1.Get(
+            containers_pb2.GetContainerRequest(id=container_id),
+            metadata=(("containerd-namespace", "k8s.io"),),
+        ).container
+
+
 def main():
     if clog is None:
         print("CLog logger unavailable, exiting.", file=sys.stderr)
         sys.exit(1)
-
+    args = parse_args()
     clog.config.configure(
         scribe_host="169.254.255.254",
         scribe_port=1463,
         monk_disable=False,
         scribe_disable=False,
     )
-
     cluster = load_system_paasta_config().get_cluster()
     client = get_docker_client()
     for (
@@ -229,11 +281,22 @@ def main():
         container_id,
         process_name,
     ) in capture_oom_events_from_stdin():
-        try:
-            docker_inspect = client.inspect_container(resource_id=container_id)
-        except (APIError):
-            continue
-        env_vars = get_container_env_as_dict(docker_inspect)
+        if args.containerd:
+            # then we're using containerd to inspect containers
+            try:
+                container_info = get_containerd_container(container_id)
+            except grpc.RpcError as e:
+                print("An error occurred while getting the container:", e)
+                continue
+            container_spec_raw = container_info.spec.value.decode("utf-8")
+            container_inspect = json.loads(container_spec_raw)
+        else:
+            # we're using docker to inspect containers
+            try:
+                container_inspect = client.inspect_container(resource_id=container_id)
+            except (APIError):
+                continue
+        env_vars = get_container_env_as_dict(args.containerd, container_inspect)
         service = env_vars.get("PAASTA_SERVICE", "unknown")
         instance = env_vars.get("PAASTA_INSTANCE", "unknown")
         mesos_container_id = env_vars.get("MESOS_CONTAINER_NAME", "mesos-null")

{paasta-tools-1.4.25 → paasta-tools-1.4.27/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.25
+Version: 1.4.27
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.25 → paasta-tools-1.4.27}/paasta_tools.egg-info/SOURCES.txt

@@ -227,7 +227,6 @@ paasta_tools/metrics/metastatus_lib.py
 paasta_tools/metrics/metrics_lib.py
 paasta_tools/monitoring/__init__.py
 paasta_tools/monitoring/check_k8s_api_performance.py
-paasta_tools/monitoring/kill_orphaned_docker_containers.py
 paasta_tools/paastaapi/__init__.py
 paasta_tools/paastaapi/api_client.py
 paasta_tools/paastaapi/configuration.py

{paasta-tools-1.4.25 → paasta-tools-1.4.27}/paasta_tools.egg-info/requires.txt

@@ -8,12 +8,14 @@ botocore
 bravado>=10.2.0
 certifi
 choice>=0.1
+containerd
 cookiecutter>=1.4.0
 croniter
 docker
 dulwich>=0.17.3
 ephemeral-port-reserve>=1.0.1
 graphviz
+grpcio
 gunicorn
 humanfriendly
 humanize>=0.5.1
@@ -47,7 +49,7 @@ requests-cache>=0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib>=2.18.20
+service-configuration-lib>=2.18.21
 signalfx
 slackclient>=1.2.1
 sticht>=1.1.0

{paasta-tools-1.4.25 → paasta-tools-1.4.27}/requirements-minimal.txt

@@ -8,12 +8,14 @@ botocore
 bravado >= 10.2.0
 certifi
 choice >= 0.1
+containerd
 cookiecutter >= 1.4.0
 croniter
 docker
 dulwich >= 0.17.3
 ephemeral-port-reserve >= 1.0.1
 graphviz
+grpcio
 gunicorn
 humanfriendly
 humanize >= 0.5.1
@@ -54,7 +56,7 @@ requests-cache >= 0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib >= 2.18.20
+service-configuration-lib >= 2.18.21
 signalfx
 slackclient >= 1.2.1
 sticht >= 1.1.0

{paasta-tools-1.4.25 → paasta-tools-1.4.27}/setup.py

@@ -62,7 +62,6 @@ setup(
         "paasta_tools/kubernetes/bin/paasta_cleanup_stale_nodes.py",
         "paasta_tools/kubernetes/bin/paasta_secrets_sync.py",
         "paasta_tools/log_task_lifecycle_events.py",
-        "paasta_tools/monitoring/kill_orphaned_docker_containers.py",
         "paasta_tools/paasta_deploy_tron_jobs",
         "paasta_tools/paasta_execute_docker_command.py",
         "paasta_tools/paasta_remote_run.py",

paasta-tools-1.4.25/paasta_tools/monitoring/kill_orphaned_docker_containers.py

@@ -1,72 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import sys
-
-import a_sync
-
-from paasta_tools import mesos_tools
-from paasta_tools.utils import get_docker_client
-from paasta_tools.utils import get_running_mesos_docker_containers
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(
-        description=(
-            "Cross references running containers with task ids from the mesos slave"
-            " and optionally kills them."
-        )
-    )
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="Actually kill the containers. (defaults to dry-run)",
-    )
-    args = parser.parse_args()
-    return args
-
-
-@a_sync.to_blocking
-async def main():
-    args = parse_args()
-    docker_client = get_docker_client()
-
-    running_mesos_task_ids = [
-        task["id"]
-        for task in mesos_tools.filter_running_tasks(
-            await mesos_tools.get_running_tasks_from_frameworks("")
-        )
-    ]
-    running_mesos_docker_containers = get_running_mesos_docker_containers()
-
-    orphaned_containers = []
-    for container in running_mesos_docker_containers:
-        mesos_task_id = mesos_tools.get_mesos_id_from_container(
-            container=container, client=docker_client
-        )
-        if mesos_task_id not in running_mesos_task_ids:
-            orphaned_containers.append(
-                (container["Names"][0].strip("/"), mesos_task_id)
-            )
-
-    if orphaned_containers:
-        print(
-            "CRIT: Docker containers are orphaned: {}{}".format(
-                ", ".join(
-                    f"{container_name} ({mesos_task_id})"
-                    for container_name, mesos_task_id in orphaned_containers
-                ),
-                " and will be killed" if args.force else "",
-            )
-        )
-        if args.force:
-            for container_name, mesos_task_id in orphaned_containers:
-                docker_client.kill(container_name)
-        sys.exit(1)
-    else:
-        print("OK: All mesos task IDs accounted for")
-        sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()