paasta-tools 1.24.1__tar.gz → 1.25.1__tar.gz

{paasta_tools-1.24.1/paasta_tools.egg-info → paasta_tools-1.25.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.24.1
+Version: 1.25.1
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -59,7 +59,7 @@ Requires-Dist: requests-cache>=0.4.10
 Requires-Dist: retry
 Requires-Dist: ruamel.yaml
 Requires-Dist: sensu-plugin
-Requires-Dist: service-configuration-lib>=3.3.3
+Requires-Dist: service-configuration-lib>=3.3.5
 Requires-Dist: signalfx
 Requires-Dist: slackclient>=1.2.1
 Requires-Dist: sticht>=1.1.0

{paasta_tools-1.24.1 → paasta_tools-1.25.1}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.24.1"
+__version__ = "1.25.1"

paasta_tools-1.25.1/paasta_tools/cleanup_expired_autoscaling_overrides.py

@@ -0,0 +1,155 @@
+import argparse
+import json
+import logging
+import sys
+import time
+from datetime import datetime
+from datetime import timezone
+
+from kubernetes.client.rest import ApiException
+
+from paasta_tools.kubernetes_tools import AUTOSCALING_OVERRIDES_CONFIGMAP_NAME
+from paasta_tools.kubernetes_tools import AUTOSCALING_OVERRIDES_CONFIGMAP_NAMESPACE
+from paasta_tools.kubernetes_tools import get_namespaced_configmap
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import replace_namespaced_configmap
+
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=f"Clean up expired entries from the {AUTOSCALING_OVERRIDES_CONFIGMAP_NAME} configmap"
+    )
+    parser.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Print entries to be removed instead of removing them",
+    )
+    parser.add_argument(
+        "-v", "--verbose", dest="verbose", action="store_true", default=False
+    )
+    args = parser.parse_args()
+    return args
+
+
+def setup_logging(verbose):
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(level=level)
+    # silence kubernetes client metadata logs since debug ones are pretty noisy
+    logging.getLogger("kubernetes.client.rest").setLevel(logging.ERROR)
+
+
+def main():
+    args = parse_args()
+    setup_logging(args.verbose)
+
+    kube_client = KubeClient()
+
+    try:
+        configmap = get_namespaced_configmap(
+            name=AUTOSCALING_OVERRIDES_CONFIGMAP_NAME,
+            namespace=AUTOSCALING_OVERRIDES_CONFIGMAP_NAMESPACE,
+            kube_client=kube_client,
+        )
+    except ApiException as e:
+        if e.status == 404:
+            log.debug(
+                f"ConfigMap {AUTOSCALING_OVERRIDES_CONFIGMAP_NAME} not found, nothing to clean up"
+            )
+            sys.exit(0)
+        else:
+            log.error(f"Error retrieving ConfigMap: {e}")
+            sys.exit(1)
+
+    if configmap is None:
+        log.debug(
+            f"ConfigMap {AUTOSCALING_OVERRIDES_CONFIGMAP_NAME} not found, nothing to clean up"
+        )
+        sys.exit(0)
+
+    if not configmap.data:
+        log.debug("ConfigMap has no data, nothing to clean up")
+        sys.exit(0)
+
+    current_timestamp = (
+        time.time()
+    )  # we'll fix the comparison time to the start of the script execution
+    expired_entries = []  # used just for logging
+    unexpired_data = {}  # used to replace the existing configmap data
+
+    log.debug(f"Current timestamp: {current_timestamp}")
+    for service_instance, override_json in configmap.data.items():
+        log.debug(f"Processing entry for {service_instance}")
+
+        try:
+            override_data = json.loads(override_json)
+        except json.JSONDecodeError:
+            log.exception(
+                f"Failed to parse JSON for {service_instance}, removing entry"
+            )
+            expired_entries.append(service_instance)
+            continue
+
+        if "expire_after" not in override_data:
+            log.warning(
+                f"Entry for {service_instance} missing expire_after field, removing entry"
+            )
+            expired_entries.append(service_instance)
+            continue
+
+        try:
+            expire_after = float(override_data["expire_after"])
+        except (ValueError, TypeError):
+            log.exception(
+                f"Failed to parse expire_after for {service_instance}, removing entry"
+            )
+            expired_entries.append(service_instance)
+            continue
+
+        if current_timestamp > expire_after:
+            expire_datetime = datetime.fromtimestamp(expire_after, tz=timezone.utc)
+            log.debug(
+                f"Entry for {service_instance} expired at {expire_after} ({expire_datetime})"
+            )
+            expired_entries.append(service_instance)
+        else:
+            expire_datetime = datetime.fromtimestamp(expire_after, tz=timezone.utc)
+            log.debug(
+                f"Entry for {service_instance} expires at {expire_after} ({expire_datetime}), keeping"
+            )
+            unexpired_data[service_instance] = override_json
+
+    if not expired_entries:
+        log.info("No expired entries found")
+        sys.exit(0)
+
+    if args.dry_run:
+        log.info(
+            "Would have removed the following expired entries:\n "
+            + "\n ".join(expired_entries)
+        )
+        sys.exit(0)
+
+    try:
+        configmap.data = (
+            unexpired_data  # naughtily re-use the existing configmap object
+        )
+        replace_namespaced_configmap(  # NOTE: this is not a patch since you cannot currently remove entries in a patch
+            name=AUTOSCALING_OVERRIDES_CONFIGMAP_NAME,
+            namespace=AUTOSCALING_OVERRIDES_CONFIGMAP_NAMESPACE,
+            body=configmap,
+            kube_client=kube_client,
+        )
+        log.info(
+            f"Successfully removed {len(expired_entries)} expired entries:\n "
+            + "\n ".join(expired_entries)
+        )
+    except ApiException as e:
+        log.error(f"Failed to update ConfigMap: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

{paasta_tools-1.24.1 → paasta_tools-1.25.1}/paasta_tools/kubernetes_tools.py

@@ -4709,6 +4709,49 @@ def patch_namespaced_configmap(
             raise
 
 
+def replace_namespaced_configmap(
+    name: str,
+    body: V1ConfigMap,
+    *,
+    namespace: str,
+    kube_client: KubeClient,
+) -> V1ConfigMap:
+    """
+    Replaces an existing configmap with the given configmap.
+    """
+    try:
+        return kube_client.core.replace_namespaced_config_map(
+            name=name, namespace=namespace, body=body
+        )
+    except ApiException as e:
+        if e.status == 404:
+            raise ValueError(f"ConfigMap {name} not found in namespace {namespace}")
+        else:
+            raise
+
+
+def delete_namespaced_configmap(
+    name: str,
+    body: V1ConfigMap,
+    *,
+    namespace: str,
+    kube_client: KubeClient,
+) -> V1ConfigMap:
+    """
+    Delete a configmap with the given name in the given namespace.
+    """
+    try:
+        return kube_client.core.delete_namespaced_config_map(
+            name=name,
+            namespace=namespace,
+        )
+    except ApiException as e:
+        if e.status == 404:
+            raise ValueError(f"ConfigMap {name} not found in namespace {namespace}")
+        else:
+            raise
+
+
 def get_or_create_namespaced_configmap(
     configmap: str,
     *,

{paasta_tools-1.24.1 → paasta_tools-1.25.1/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.24.1
+Version: 1.25.1
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -59,7 +59,7 @@ Requires-Dist: requests-cache>=0.4.10
 Requires-Dist: retry
 Requires-Dist: ruamel.yaml
 Requires-Dist: sensu-plugin
-Requires-Dist: service-configuration-lib>=3.3.3
+Requires-Dist: service-configuration-lib>=3.3.5
 Requires-Dist: signalfx
 Requires-Dist: slackclient>=1.2.1
 Requires-Dist: sticht>=1.1.0

{paasta_tools-1.24.1 → paasta_tools-1.25.1}/paasta_tools.egg-info/SOURCES.txt

@@ -22,6 +22,7 @@ paasta_tools/check_kubernetes_services_replication.py
 paasta_tools/check_oom_events.py
 paasta_tools/check_services_replication_tools.py
 paasta_tools/check_spark_jobs.py
+paasta_tools/cleanup_expired_autoscaling_overrides.py
 paasta_tools/cleanup_kubernetes_cr.py
 paasta_tools/cleanup_kubernetes_crd.py
 paasta_tools/cleanup_kubernetes_jobs.py

{paasta_tools-1.24.1 → paasta_tools-1.25.1}/paasta_tools.egg-info/entry_points.txt

@@ -4,6 +4,7 @@ paasta-api = paasta_tools.api.api:main
 paasta-deployd = paasta_tools.deployd.master:main
 paasta-fsm = paasta_tools.cli.fsm_cmd:main
 paasta_broadcast_log = paasta_tools.broadcast_log_to_services:main
+paasta_cleanup_expired_autoscaling_overrides = paasta_tools.cleanup_expired_autoscaling_overrides:main
 paasta_cleanup_tron_namespaces = paasta_tools.cleanup_tron_namespaces:main
 paasta_docker_wrapper = paasta_tools.docker_wrapper:main
 paasta_dump_locally_running_services = paasta_tools.dump_locally_running_services:main

{paasta_tools-1.24.1 → paasta_tools-1.25.1}/paasta_tools.egg-info/requires.txt

@@ -50,7 +50,7 @@ requests-cache>=0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib>=3.3.3
+service-configuration-lib>=3.3.5
 signalfx
 slackclient>=1.2.1
 sticht>=1.1.0

{paasta_tools-1.24.1 → paasta_tools-1.25.1}/requirements-minimal.txt

@@ -57,7 +57,7 @@ requests-cache >= 0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib >= 3.3.3
+service-configuration-lib >= 3.3.5
 signalfx
 slackclient >= 1.2.1
 sticht >= 1.1.0

{paasta_tools-1.24.1 → paasta_tools-1.25.1}/setup.py

@@ -82,6 +82,7 @@ setup(
             "paasta-fsm=paasta_tools.cli.fsm_cmd:main",
             "paasta_prune_completed_pods=paasta_tools.prune_completed_pods:main",
             "paasta_cleanup_tron_namespaces=paasta_tools.cleanup_tron_namespaces:main",
+            "paasta_cleanup_expired_autoscaling_overrides=paasta_tools.cleanup_expired_autoscaling_overrides:main",
             "paasta_list_kubernetes_service_instances=paasta_tools.list_kubernetes_service_instances:main",
             "paasta_list_tron_namespaces=paasta_tools.list_tron_namespaces:main",
             "paasta_setup_tron_namespace=paasta_tools.setup_tron_namespace:main",