paasta-tools 1.24.1__tar.gz → 1.26.0__tar.gz

{paasta_tools-1.24.1/paasta_tools.egg-info → paasta_tools-1.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.24.1
+Version: 1.26.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -59,7 +59,7 @@ Requires-Dist: requests-cache>=0.4.10
 Requires-Dist: retry
 Requires-Dist: ruamel.yaml
 Requires-Dist: sensu-plugin
-Requires-Dist: service-configuration-lib>=3.3.3
+Requires-Dist: service-configuration-lib>=3.3.5
 Requires-Dist: signalfx
 Requires-Dist: slackclient>=1.2.1
 Requires-Dist: sticht>=1.1.0

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.24.1"
+__version__ = "1.26.0"

paasta_tools-1.26.0/paasta_tools/cleanup_expired_autoscaling_overrides.py

@@ -0,0 +1,155 @@
+import argparse
+import json
+import logging
+import sys
+import time
+from datetime import datetime
+from datetime import timezone
+
+from kubernetes.client.rest import ApiException
+
+from paasta_tools.kubernetes_tools import AUTOSCALING_OVERRIDES_CONFIGMAP_NAME
+from paasta_tools.kubernetes_tools import AUTOSCALING_OVERRIDES_CONFIGMAP_NAMESPACE
+from paasta_tools.kubernetes_tools import get_namespaced_configmap
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import replace_namespaced_configmap
+
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=f"Clean up expired entries from the {AUTOSCALING_OVERRIDES_CONFIGMAP_NAME} configmap"
+    )
+    parser.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Print entries to be removed instead of removing them",
+    )
+    parser.add_argument(
+        "-v", "--verbose", dest="verbose", action="store_true", default=False
+    )
+    args = parser.parse_args()
+    return args
+
+
+def setup_logging(verbose):
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(level=level)
+    # silence kubernetes client metadata logs since debug ones are pretty noisy
+    logging.getLogger("kubernetes.client.rest").setLevel(logging.ERROR)
+
+
+def main():
+    args = parse_args()
+    setup_logging(args.verbose)
+
+    kube_client = KubeClient()
+
+    try:
+        configmap = get_namespaced_configmap(
+            name=AUTOSCALING_OVERRIDES_CONFIGMAP_NAME,
+            namespace=AUTOSCALING_OVERRIDES_CONFIGMAP_NAMESPACE,
+            kube_client=kube_client,
+        )
+    except ApiException as e:
+        if e.status == 404:
+            log.debug(
+                f"ConfigMap {AUTOSCALING_OVERRIDES_CONFIGMAP_NAME} not found, nothing to clean up"
+            )
+            sys.exit(0)
+        else:
+            log.error(f"Error retrieving ConfigMap: {e}")
+            sys.exit(1)
+
+    if configmap is None:
+        log.debug(
+            f"ConfigMap {AUTOSCALING_OVERRIDES_CONFIGMAP_NAME} not found, nothing to clean up"
+        )
+        sys.exit(0)
+
+    if not configmap.data:
+        log.debug("ConfigMap has no data, nothing to clean up")
+        sys.exit(0)
+
+    current_timestamp = (
+        time.time()
+    )  # we'll fix the comparison time to the start of the script execution
+    expired_entries = []  # used just for logging
+    unexpired_data = {}  # used to replace the existing configmap data
+
+    log.debug(f"Current timestamp: {current_timestamp}")
+    for service_instance, override_json in configmap.data.items():
+        log.debug(f"Processing entry for {service_instance}")
+
+        try:
+            override_data = json.loads(override_json)
+        except json.JSONDecodeError:
+            log.exception(
+                f"Failed to parse JSON for {service_instance}, removing entry"
+            )
+            expired_entries.append(service_instance)
+            continue
+
+        if "expire_after" not in override_data:
+            log.warning(
+                f"Entry for {service_instance} missing expire_after field, removing entry"
+            )
+            expired_entries.append(service_instance)
+            continue
+
+        try:
+            expire_after = float(override_data["expire_after"])
+        except (ValueError, TypeError):
+            log.exception(
+                f"Failed to parse expire_after for {service_instance}, removing entry"
+            )
+            expired_entries.append(service_instance)
+            continue
+
+        if current_timestamp > expire_after:
+            expire_datetime = datetime.fromtimestamp(expire_after, tz=timezone.utc)
+            log.debug(
+                f"Entry for {service_instance} expired at {expire_after} ({expire_datetime})"
+            )
+            expired_entries.append(service_instance)
+        else:
+            expire_datetime = datetime.fromtimestamp(expire_after, tz=timezone.utc)
+            log.debug(
+                f"Entry for {service_instance} expires at {expire_after} ({expire_datetime}), keeping"
+            )
+            unexpired_data[service_instance] = override_json
+
+    if not expired_entries:
+        log.info("No expired entries found")
+        sys.exit(0)
+
+    if args.dry_run:
+        log.info(
+            "Would have removed the following expired entries:\n "
+            + "\n ".join(expired_entries)
+        )
+        sys.exit(0)
+
+    try:
+        configmap.data = (
+            unexpired_data  # naughtily re-use the existing configmap object
+        )
+        replace_namespaced_configmap(  # NOTE: this is not a patch since you cannot currently remove entries in a patch
+            name=AUTOSCALING_OVERRIDES_CONFIGMAP_NAME,
+            namespace=AUTOSCALING_OVERRIDES_CONFIGMAP_NAMESPACE,
+            body=configmap,
+            kube_client=kube_client,
+        )
+        log.info(
+            f"Successfully removed {len(expired_entries)} expired entries:\n "
+            + "\n ".join(expired_entries)
+        )
+    except ApiException as e:
+        log.error(f"Failed to update ConfigMap: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/paasta_tools/cli/schemas/tron_schema.json

@@ -91,6 +91,11 @@
                     },
                     "uniqueItems": true
                 },
+                "idempotent": {
+                    "type": "boolean",
+                    "default": false,
+                    "$comment": "This will be used to determine whether the action can be retried without side effects."
+                },
                 "extra_constraints": {
                     "type": "array",
                     "items": {

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/paasta_tools/kubernetes_tools.py

@@ -4709,6 +4709,49 @@ def patch_namespaced_configmap(
             raise
 
 
+def replace_namespaced_configmap(
+    name: str,
+    body: V1ConfigMap,
+    *,
+    namespace: str,
+    kube_client: KubeClient,
+) -> V1ConfigMap:
+    """
+    Replaces an existing configmap with the given configmap.
+    """
+    try:
+        return kube_client.core.replace_namespaced_config_map(
+            name=name, namespace=namespace, body=body
+        )
+    except ApiException as e:
+        if e.status == 404:
+            raise ValueError(f"ConfigMap {name} not found in namespace {namespace}")
+        else:
+            raise
+
+
+def delete_namespaced_configmap(
+    name: str,
+    body: V1ConfigMap,
+    *,
+    namespace: str,
+    kube_client: KubeClient,
+) -> V1ConfigMap:
+    """
+    Delete a configmap with the given name in the given namespace.
+    """
+    try:
+        return kube_client.core.delete_namespaced_config_map(
+            name=name,
+            namespace=namespace,
+        )
+    except ApiException as e:
+        if e.status == 404:
+            raise ValueError(f"ConfigMap {name} not found in namespace {namespace}")
+        else:
+            raise
+
+
 def get_or_create_namespaced_configmap(
     configmap: str,
     *,

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/paasta_tools/tron_tools.py

@@ -260,6 +260,7 @@ class TronActionConfigDict(InstanceConfigDict, total=False):
     # TODO: TRON-2145: use this to implement timeout for non-spark actions in tron
     max_runtime: str
     mrjob: bool
+    idempotent: bool
 
 
 class TronActionConfig(InstanceConfig):
@@ -422,6 +423,9 @@ class TronActionConfig(InstanceConfig):
     def get_job_name(self):
         return self.job
 
+    def get_idempotent(self) -> bool:
+        return self.config_dict.get("idempotent", False)
+
     def get_action_name(self):
         return self.action
 
@@ -977,6 +981,7 @@ def format_tron_action_dict(action_config: TronActionConfig):
         "secret_volumes": action_config.get_secret_volumes(),
         "expected_runtime": action_config.get_expected_runtime(),
         "trigger_downstreams": action_config.get_trigger_downstreams(),
+        "idempotent": action_config.get_idempotent(),
         "triggered_by": action_config.get_triggered_by(),
         "on_upstream_rerun": action_config.get_on_upstream_rerun(),
         "trigger_timeout": action_config.get_trigger_timeout(),
@@ -1046,6 +1051,9 @@ def format_tron_action_dict(action_config: TronActionConfig):
                 limit=63,
                 suffix=4,
             ),
+            "tron.yelp.com/idempotent-action": str(
+                action_config.get_idempotent()
+            ).lower(),
             # XXX: should this be different for Spark drivers launched by Tron?
             "app.kubernetes.io/managed-by": "tron",
         }

{paasta_tools-1.24.1 → paasta_tools-1.26.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.24.1
+Version: 1.26.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -59,7 +59,7 @@ Requires-Dist: requests-cache>=0.4.10
 Requires-Dist: retry
 Requires-Dist: ruamel.yaml
 Requires-Dist: sensu-plugin
-Requires-Dist: service-configuration-lib>=3.3.3
+Requires-Dist: service-configuration-lib>=3.3.5
 Requires-Dist: signalfx
 Requires-Dist: slackclient>=1.2.1
 Requires-Dist: sticht>=1.1.0

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/paasta_tools.egg-info/SOURCES.txt

@@ -22,6 +22,7 @@ paasta_tools/check_kubernetes_services_replication.py
 paasta_tools/check_oom_events.py
 paasta_tools/check_services_replication_tools.py
 paasta_tools/check_spark_jobs.py
+paasta_tools/cleanup_expired_autoscaling_overrides.py
 paasta_tools/cleanup_kubernetes_cr.py
 paasta_tools/cleanup_kubernetes_crd.py
 paasta_tools/cleanup_kubernetes_jobs.py

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/paasta_tools.egg-info/entry_points.txt

@@ -4,6 +4,7 @@ paasta-api = paasta_tools.api.api:main
 paasta-deployd = paasta_tools.deployd.master:main
 paasta-fsm = paasta_tools.cli.fsm_cmd:main
 paasta_broadcast_log = paasta_tools.broadcast_log_to_services:main
+paasta_cleanup_expired_autoscaling_overrides = paasta_tools.cleanup_expired_autoscaling_overrides:main
 paasta_cleanup_tron_namespaces = paasta_tools.cleanup_tron_namespaces:main
 paasta_docker_wrapper = paasta_tools.docker_wrapper:main
 paasta_dump_locally_running_services = paasta_tools.dump_locally_running_services:main

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/paasta_tools.egg-info/requires.txt

@@ -50,7 +50,7 @@ requests-cache>=0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib>=3.3.3
+service-configuration-lib>=3.3.5
 signalfx
 slackclient>=1.2.1
 sticht>=1.1.0

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/requirements-minimal.txt

@@ -57,7 +57,7 @@ requests-cache >= 0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib >= 3.3.3
+service-configuration-lib >= 3.3.5
 signalfx
 slackclient >= 1.2.1
 sticht >= 1.1.0

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/setup.py

@@ -82,6 +82,7 @@ setup(
             "paasta-fsm=paasta_tools.cli.fsm_cmd:main",
             "paasta_prune_completed_pods=paasta_tools.prune_completed_pods:main",
             "paasta_cleanup_tron_namespaces=paasta_tools.cleanup_tron_namespaces:main",
+            "paasta_cleanup_expired_autoscaling_overrides=paasta_tools.cleanup_expired_autoscaling_overrides:main",
             "paasta_list_kubernetes_service_instances=paasta_tools.list_kubernetes_service_instances:main",
             "paasta_list_tron_namespaces=paasta_tools.list_tron_namespaces:main",
             "paasta_setup_tron_namespace=paasta_tools.setup_tron_namespace:main",

{paasta_tools-1.24.1 → paasta_tools-1.26.0}/tests/test_tron_tools.py

@@ -1043,6 +1043,7 @@ class TestTronTools:
             "mem": 1200,
             "disk": 42,
             "env": mock.ANY,
+            "idempotent": False,
             "topology_spread_constraints": [
                 {
                     "label_selector": {
@@ -1083,6 +1084,7 @@ class TestTronTools:
                 "paasta.yelp.com/service": "my_service",
                 "yelp.com/owner": "compute_infra_platform_experience",
                 "app.kubernetes.io/managed-by": "tron",
+                "tron.yelp.com/idempotent-action": "false",
             },
             "annotations": {
                 "paasta.yelp.com/routable_ip": "false",
@@ -1150,6 +1152,7 @@ class TestTronTools:
             "deploy_group": "prod",
             "executor": "spark",
             "disk": 42,
+            "idempotent": True,
             "pool": "special_pool",
             "env": {"SHELL": "/bin/bash"},
             "secret_volumes": [
@@ -1449,7 +1452,9 @@ class TestTronTools:
                 "paasta.yelp.com/prometheus_shard": "ml-compute",
                 "spark.yelp.com/user": "TRON",
                 "spark.yelp.com/driver_ui_port": "39091",
+                "tron.yelp.com/idempotent-action": "true",
             },
+            "idempotent": True,
             "annotations": {
                 "paasta.yelp.com/routable_ip": "true",
                 "paasta.yelp.com/service": "my_service",
@@ -1493,6 +1498,7 @@ class TestTronTools:
             "deploy_group": "prod",
             "pool": "default",
             "executor": "paasta",
+            "idempotent": True,
         }
         branch_dict = {
             "docker_image": "my_service:paasta-123abcde",
@@ -1534,6 +1540,7 @@ class TestTronTools:
             "disk": 1024,
             "cap_add": [],
             "cap_drop": CAPS_DROP,
+            "idempotent": True,
             "labels": {
                 "paasta.yelp.com/cluster": "test-cluster",
                 "paasta.yelp.com/instance": "job_name.instance_name",
@@ -1541,6 +1548,7 @@ class TestTronTools:
                 "paasta.yelp.com/service": "my_service",
                 "yelp.com/owner": "compute_infra_platform_experience",
                 "app.kubernetes.io/managed-by": "tron",
+                "tron.yelp.com/idempotent-action": "true",
             },
             "annotations": {
                 "paasta.yelp.com/routable_ip": "false",
@@ -1678,6 +1686,7 @@ class TestTronTools:
             "cpus": 2,
             "mem": 1200,
             "disk": 42,
+            "idempotent": False,
             "cap_add": [],
             "cap_drop": CAPS_DROP,
             "labels": {
@@ -1687,6 +1696,7 @@ class TestTronTools:
                 "paasta.yelp.com/service": "my_service",
                 "yelp.com/owner": "compute_infra_platform_experience",
                 "app.kubernetes.io/managed-by": "tron",
+                "tron.yelp.com/idempotent-action": "false",
             },
             "annotations": {
                 "paasta.yelp.com/routable_ip": "false",
@@ -1811,6 +1821,7 @@ class TestTronTools:
             "cpus": 2,
             "mem": 1200,
             "disk": 42,
+            "idempotent": False,
             "env": mock.ANY,
             "topology_spread_constraints": [
                 {
@@ -1849,6 +1860,7 @@ class TestTronTools:
                 "paasta.yelp.com/service": "my_service",
                 "yelp.com/owner": "compute_infra_platform_experience",
                 "app.kubernetes.io/managed-by": "tron",
+                "tron.yelp.com/idempotent-action": "false",
             },
             "annotations": {
                 "paasta.yelp.com/routable_ip": "false",
@@ -2004,7 +2016,7 @@ fake_job:
         # that are not static, this will cause continuous reconfiguration, which
         # will add significant load to the Tron API, which happened in DAR-1461.
         # but if this is intended, just change the hash.
-        assert hasher.hexdigest() == "b4c7ee4d47787f081b58b44df10a07f1"
+        assert hasher.hexdigest() == "5cf9985b6227ce18dc87376b687046f7"
 
     def test_override_default_pool_override(self, tmpdir):
         soa_dir = tmpdir.mkdir("test_create_complete_config_soa")