paasta-tools 1.13.5__tar.gz → 1.14.0__tar.gz

{paasta-tools-1.13.5/paasta_tools.egg-info → paasta-tools-1.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.13.5
+Version: 1.14.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.13.5"
+__version__ = "1.14.0"

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/api/api.py

@@ -189,6 +189,26 @@ def make_app(global_config=None):
         "/v1/service_autoscaler/pause",
         request_method="GET",
     )
+    config.add_route(
+        "remote_run.start",
+        "/v1/remote_run/{service}/{instance}/start",
+        request_method="POST",
+    )
+    config.add_route(
+        "remote_run.stop",
+        "/v1/remote_run/{service}/{instance}/stop",
+        request_method="POST",
+    )
+    config.add_route(
+        "remote_run.poll",
+        "/v1/remote_run/{service}/{instance}/poll",
+        request_method="GET",
+    )
+    config.add_route(
+        "remote_run.token",
+        "/v1/remote_run/{service}/{instance}/token",
+        request_method="GET",
+    )
     config.add_route("version", "/v1/version")
     config.add_route("deploy_queue.list", "/v1/deploy_queue")
     config.scan()

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/api/api_docs/swagger.json

@@ -2,7 +2,7 @@
     "swagger": "2.0",
     "info": {
         "title": "Paasta API",
-        "version": "1.0.0"
+        "version": "1.1.0"
     },
     "basePath": "/v1",
     "schemes": [
@@ -846,6 +846,194 @@
                     }
                 ]
             }
+        },
+        "/remote_run/{service}/{instance}/start": {
+            "post": {
+                "responses": {
+                    "200": {
+                        "description": "Successfully started remote-run sandbox",
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunOutcome"
+                        }
+                    },
+                    "404": {
+                        "description": "Deployment key not found"
+                    },
+                    "500": {
+                        "description": "Failure"
+                    }
+                },
+                "summary": "Launch a remote-run pod",
+                "operationId": "remote_run_start",
+                "tags": [
+                    "remote_run"
+                ],
+                "parameters": [
+                    {
+                        "in": "path",
+                        "description": "Service name",
+                        "name": "service",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "path",
+                        "description": "Instance name",
+                        "name": "instance",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "body",
+                        "description": "Job bootstrap settings",
+                        "name": "json_body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunStart"
+                        }
+                    }
+                ]
+            }
+        },
+        "/remote_run/{service}/{instance}/stop": {
+            "post": {
+                "responses": {
+                    "200": {
+                        "description": "Remote run pod stopped",
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunOutcome"
+                        }
+                    },
+                    "404": {
+                        "description": "Service instance not found"
+                    },
+                    "500": {
+                        "description": "Failure"
+                    }
+                },
+                "summary": "Stop a remote run",
+                "operationId": "remote_run_stop",
+                "tags": [
+                    "service"
+                ],
+                "parameters": [
+                    {
+                        "in": "path",
+                        "description": "Service name",
+                        "name": "service",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "path",
+                        "description": "Instance name",
+                        "name": "instance",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "body",
+                        "description": "Details about the job to be stopped",
+                        "name": "json_body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunStop"
+                        }
+                    }
+                ]
+            }
+        },
+        "/remote_run/{service}/{instance}/poll": {
+            "get": {
+                "responses": {
+                    "200": {
+                        "description": "Pod status information",
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunOutcome"
+                        }
+                    },
+                    "404": {
+                        "description": "Service instance not found"
+                    },
+                    "500": {
+                        "description": "Failure"
+                    }
+                },
+                "summary": "Check if remote run pod is ready",
+                "operationId": "remote_run_poll",
+                "tags": [
+                    "remote_run"
+                ],
+                "parameters": [
+                    {
+                        "in": "path",
+                        "description": "Service name",
+                        "name": "service",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "path",
+                        "description": "Instance name",
+                        "name": "instance",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "query",
+                        "description": "Kubernetes job name (returned from the /start endpoint)",
+                        "name": "job_name",
+                        "required": true,
+                        "type": "string"
+                    }
+                ]
+            }
+        },
+        "/remote_run/{service}/{instance}/token": {
+            "get": {
+                "responses": {
+                    "200": {
+                        "description": "Token generated successfully",
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunToken"
+                        }
+                    },
+                    "404": {
+                        "description": "Service instance not found"
+                    },
+                    "500": {
+                        "description": "Failure"
+                    }
+                },
+                "summary": "Get a remote run token",
+                "operationId": "remote_run_token",
+                "tags": [
+                    "remote_run"
+                ],
+                "parameters": [
+                    {
+                        "in": "path",
+                        "description": "Service name",
+                        "name": "service",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "path",
+                        "description": "Instance name",
+                        "name": "instance",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "query",
+                        "description": "User requesting the token",
+                        "name": "user",
+                        "required": true,
+                        "type": "string"
+                    }
+                ]
+            }
         }
     },
     "definitions": {
@@ -1970,6 +2158,72 @@
                 }
             }
         },
+        "RemoteRunStart": {
+            "type": "object",
+            "properties": {
+                "interactive": {
+                    "type": "boolean"
+                },
+                "recreate": {
+                    "type": "boolean"
+                },
+                "user": {
+                    "type": "string"
+                },
+                "max_duration": {
+                    "type": "integer"
+                }
+            },
+            "required": [
+                "user"
+            ]
+        },
+        "RemoteRunStop": {
+            "type": "object",
+            "properties": {
+                "user": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "user"
+            ]
+        },
+        "RemoteRunOutcome": {
+            "type": "object",
+            "properties": {
+                "status": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "pod_name": {
+                    "type": "string"
+                },
+                "job_name": {
+                    "type": "string"
+                },
+                "namespace": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "status",
+                "message"
+            ]
+        },
+        "RemoteRunToken": {
+            "type": "object",
+            "properties": {
+                "token": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "token"
+            ]
+        },
         "Resource": {
             "type": "array",
             "items": {

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/api/client.py

@@ -37,6 +37,7 @@ class PaastaOApiClient:
     default: paastaapis.DefaultApi
     resources: paastaapis.ResourcesApi
     service: paastaapis.ServiceApi
+    remote_run: paastaapis.RemoteRunApi
     api_error: Type[paastaapi.ApiException]
     connection_error: Type[paastaapi.ApiException]
     timeout_error: Type[paastaapi.ApiException]
@@ -72,6 +73,7 @@ def get_paasta_oapi_client_by_url(
         default=paastaapis.DefaultApi(client),
         resources=paastaapis.ResourcesApi(client),
         service=paastaapis.ServiceApi(client),
+        remote_run=paastaapis.RemoteRunApi(client),
         api_error=paastaapi.ApiException,
         connection_error=paastaapi.ApiException,
         timeout_error=paastaapi.ApiException,

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/api/views/instance.py

@@ -30,12 +30,12 @@ from pyramid.response import Response
 from pyramid.view import view_config
 
 import paasta_tools.mesos.exceptions as mesos_exceptions
-from paasta_tools import paasta_remote_run
 from paasta_tools import tron_tools
 from paasta_tools.api import settings
 from paasta_tools.api.views.exception import ApiFailure
 from paasta_tools.cli.cmds.status import get_actual_deployments
 from paasta_tools.instance import kubernetes as pik
+from paasta_tools.mesos_tools import get_all_frameworks as get_all_mesos_frameworks
 from paasta_tools.utils import compose_job_id
 from paasta_tools.utils import DeploymentVersion
 from paasta_tools.utils import NoConfigurationForServiceError
@@ -94,11 +94,19 @@ def tron_instance_status(
     return status
 
 
+def legacy_remote_run_filter_frameworks(service, instance, frameworks=None):
+    if frameworks is None:
+        frameworks = get_all_mesos_frameworks(active_only=True)
+
+    prefix = f"paasta-remote {service}.{instance}"
+    return [f for f in frameworks if f.name.startswith(prefix)]
+
+
 def adhoc_instance_status(
     instance_status: Mapping[str, Any], service: str, instance: str, verbose: int
 ) -> List[Dict[str, Any]]:
     status = []
-    filtered = paasta_remote_run.remote_run_filter_frameworks(service, instance)
+    filtered = legacy_remote_run_filter_frameworks(service, instance)
     filtered.sort(key=lambda x: x.name)
     for f in filtered:
         launch_time, run_id = re.match(

paasta-tools-1.14.0/paasta_tools/api/views/remote_run.py

@@ -0,0 +1,107 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import traceback
+
+from pyramid.view import view_config
+
+from paasta_tools.api import settings
+from paasta_tools.api.views.exception import ApiFailure
+from paasta_tools.kubernetes.remote_run import remote_run_ready
+from paasta_tools.kubernetes.remote_run import remote_run_start
+from paasta_tools.kubernetes.remote_run import remote_run_stop
+from paasta_tools.kubernetes.remote_run import remote_run_token
+from paasta_tools.utils import load_system_paasta_config
+
+
+DEFAULT_MAX_DURATION = 60 * 60  # 1 hour
+DEFAULT_MAX_DURATION_LIMIT = 8 * 60 * 60  # 8 hours
+
+
+@view_config(route_name="remote_run.start", request_method="POST", renderer="json")
+def view_remote_run_start(request):
+    system_config = load_system_paasta_config()
+    service = request.swagger_data["service"]
+    instance = request.swagger_data["instance"]
+    user = request.swagger_data["json_body"]["user"]
+    interactive = request.swagger_data["json_body"].get("interactive", True)
+    recreate = request.swagger_data["json_body"].get("recreate", False)
+    max_duration = min(
+        request.swagger_data["json_body"].get("max_duration", DEFAULT_MAX_DURATION),
+        system_config.get_remote_run_duration_limit(DEFAULT_MAX_DURATION_LIMIT),
+    )
+    try:
+        return remote_run_start(
+            service=service,
+            instance=instance,
+            cluster=settings.cluster,
+            user=user,
+            interactive=interactive,
+            recreate=recreate,
+            max_duration=max_duration,
+        )
+    except Exception:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)
+
+
+@view_config(route_name="remote_run.poll", request_method="GET", renderer="json")
+def view_remote_run_poll(request):
+    service = request.swagger_data["service"]
+    instance = request.swagger_data["instance"]
+    job_name = request.swagger_data["job_name"]
+    try:
+        return remote_run_ready(
+            service=service,
+            instance=instance,
+            cluster=settings.cluster,
+            job_name=job_name,
+        )
+    except Exception:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)
+
+
+@view_config(route_name="remote_run.stop", request_method="POST", renderer="json")
+def view_remote_run_stop(request):
+    service = request.swagger_data["service"]
+    instance = request.swagger_data["instance"]
+    user = request.swagger_data["json_body"]["user"]
+    try:
+        return remote_run_stop(
+            service=service,
+            instance=instance,
+            cluster=settings.cluster,
+            user=user,
+        )
+    except Exception:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)
+
+
+@view_config(route_name="remote_run.token", request_method="GET", renderer="json")
+def view_remote_run_token(request):
+    service = request.swagger_data["service"]
+    instance = request.swagger_data["instance"]
+    user = request.swagger_data["user"]
+    try:
+        token = remote_run_token(
+            service=service,
+            instance=instance,
+            cluster=settings.cluster,
+            user=user,
+        )
+        return {"token": token}
+    except Exception:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/cli/cmds/local_run.py

@@ -1227,7 +1227,6 @@ def configure_and_run_docker_container(
 
     for volume in instance_config.get_volumes(
         system_paasta_config.get_volumes(),
-        system_paasta_config.get_uses_bulkdata_default(),
     ):
         if os.path.exists(volume["hostPath"]):
             volumes.append(

paasta-tools-1.14.0/paasta_tools/cli/cmds/remote_run.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# Copyright 2015-2017 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+
+from paasta_tools.utils import PaastaColors
+
+
+def add_subparser(subparsers: argparse._SubParsersAction):
+    subparsers.add_parser(
+        "remote-run",
+        help="Schedule adhoc service sandbox on PaaSTA cluster",
+        description=(
+            "`paasta remote-run` is useful for running adhoc commands in "
+            "context of a service's Docker image."
+        ),
+    )
+
+
+def paasta_remote_run(args: argparse.Namespace):
+    print(PaastaColors.red("Error: functionality under construction"))
+    return 1

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/cli/cmds/spark_run.py

@@ -1219,7 +1219,6 @@ def paasta_spark_run(args: argparse.Namespace) -> int:
 
     volumes = instance_config.get_volumes(
         system_paasta_config.get_volumes(),
-        system_paasta_config.get_uses_bulkdata_default(),
     )
     app_base_name = get_spark_app_name(args.cmd or instance_config.get_cmd())
 

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/contrib/service_shard_update.py

@@ -171,6 +171,13 @@ def parse_args():
         type=str,
         dest="iam_role",
     )
+    parser.add_argument(
+        "--environment",
+        help="Environment to deploy in, defaults to all environments if not specified",
+        required=False,
+        choices=DEPLOY_MAPPINGS.keys(),
+        dest="environment",
+    )
     return parser.parse_args()
 
 
@@ -201,11 +208,17 @@ def main(args):
         working_dir=args.local_dir or "/nail/tmp",
         do_clone=args.local_dir is None,
     )
+    deploy_environments = (
+        {args.environment: DEPLOY_MAPPINGS[args.environment]}
+        if args.environment
+        else DEPLOY_MAPPINGS
+    )
+
     with updater:
         deploy_file = updater.get_existing_configs(args.service, "deploy")
         smartstack_file = updater.get_existing_configs(args.service, "smartstack")
         shard_deploy_groups = {
-            f"{prefix}.{args.shard_name}" for prefix in DEPLOY_MAPPINGS.keys()
+            f"{prefix}.{args.shard_name}" for prefix in deploy_environments.keys()
         }
         pipeline_steps = {step["step"] for step in deploy_file["pipeline"]}
 
@@ -226,7 +239,7 @@ def main(args):
                 log.info(f"{step} added to deploy config")
             updater.write_configs(args.service, "deploy", deploy_file)
 
-            for deploy_prefix, config_paths in DEPLOY_MAPPINGS.items():
+            for deploy_prefix, config_paths in deploy_environments.items():
                 for config_path in config_paths:
                     # Determine configuration suffix (PAASTA-18216)
                     eks_config = updater.get_existing_configs(

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/frameworks/native_service_config.py

@@ -172,7 +172,6 @@ class NativeServiceConfig(LongRunningServiceConfig):
         """
         docker_volumes = self.get_volumes(
             system_volumes=system_paasta_config.get_volumes(),
-            uses_bulkdata_default=system_paasta_config.get_uses_bulkdata_default(),
         )
         task: TaskInfo = {
             "name": "",

{paasta-tools-1.13.5 → paasta-tools-1.14.0}/paasta_tools/kubernetes/application/controller_wrappers.py

@@ -6,6 +6,7 @@ from typing import Union
 
 from kubernetes.client import V1DeleteOptions
 from kubernetes.client import V1Deployment
+from kubernetes.client import V1Job
 from kubernetes.client import V1PodDisruptionBudget
 from kubernetes.client import V1StatefulSet
 from kubernetes.client.rest import ApiException
@@ -13,6 +14,7 @@ from kubernetes.client.rest import ApiException
 from paasta_tools.autoscaling.autoscaling_service_lib import autoscaling_is_paused
 from paasta_tools.eks_tools import load_eks_service_config_no_cache
 from paasta_tools.kubernetes_tools import create_deployment
+from paasta_tools.kubernetes_tools import create_job
 from paasta_tools.kubernetes_tools import create_pod_disruption_budget
 from paasta_tools.kubernetes_tools import create_stateful_set
 from paasta_tools.kubernetes_tools import ensure_service_account
@@ -411,14 +413,63 @@ class StatefulSetWrapper(Application):
         )
 
 
+class JobWrapper(Application):
+    def __init__(
+        self,
+        item: V1Job,
+        logging=logging.getLogger(__name__),
+    ):
+        item.spec.replicas = None  # avoid causing errors in parent class
+        super().__init__(item, logging)
+
+    def deep_delete(self, kube_client: KubeClient) -> None:
+        """Remove resources related to the job"""
+        delete_options = V1DeleteOptions(propagation_policy="Foreground")
+        try:
+            kube_client.batches.delete_namespaced_job(
+                self.item.metadata.name,
+                self.item.metadata.namespace,
+                body=delete_options,
+            )
+        except ApiException as e:
+            if e.status == 404:
+                # Job does not exist, nothing to delete but
+                # we can consider this a success.
+                self.logging.debug(
+                    "not deleting nonexistent job/{} from namespace/{}".format(
+                        self.item.metadata.name,
+                        self.item.metadata.namespace,
+                    )
+                )
+            else:
+                raise
+        else:
+            self.logging.info(
+                "deleted job/{} from namespace/{}".format(
+                    self.item.metadata.name,
+                    self.item.metadata.namespace,
+                )
+            )
+
+    def create(self, kube_client: KubeClient):
+        """Create and start Kubernetes Job"""
+        create_job(
+            kube_client=kube_client,
+            formatted_job=self.item,
+            namespace=self.soa_config.get_namespace(),
+        )
+
+
 def get_application_wrapper(
-    formatted_application: Union[V1Deployment, V1StatefulSet]
+    formatted_application: Union[V1Deployment, V1StatefulSet, V1Job]
 ) -> Application:
     app: Application
     if isinstance(formatted_application, V1Deployment):
         app = DeploymentWrapper(formatted_application)
     elif isinstance(formatted_application, V1StatefulSet):
         app = StatefulSetWrapper(formatted_application)
+    elif isinstance(formatted_application, V1Job):
+        app = JobWrapper(formatted_application)
     else:
         raise Exception("Unknown kubernetes object to update")