paasta-tools 1.13.6__tar.gz → 1.14.1__tar.gz

{paasta-tools-1.13.6/paasta_tools.egg-info → paasta-tools-1.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.13.6
+Version: 1.14.1
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.13.6 → paasta-tools-1.14.1}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.13.6"
+__version__ = "1.14.1"

{paasta-tools-1.13.6 → paasta-tools-1.14.1}/paasta_tools/api/api.py

@@ -189,6 +189,26 @@ def make_app(global_config=None):
         "/v1/service_autoscaler/pause",
         request_method="GET",
     )
+    config.add_route(
+        "remote_run.start",
+        "/v1/remote_run/{service}/{instance}/start",
+        request_method="POST",
+    )
+    config.add_route(
+        "remote_run.stop",
+        "/v1/remote_run/{service}/{instance}/stop",
+        request_method="POST",
+    )
+    config.add_route(
+        "remote_run.poll",
+        "/v1/remote_run/{service}/{instance}/poll",
+        request_method="GET",
+    )
+    config.add_route(
+        "remote_run.token",
+        "/v1/remote_run/{service}/{instance}/token",
+        request_method="GET",
+    )
     config.add_route("version", "/v1/version")
     config.add_route("deploy_queue.list", "/v1/deploy_queue")
     config.scan()

{paasta-tools-1.13.6 → paasta-tools-1.14.1}/paasta_tools/api/api_docs/swagger.json

@@ -2,7 +2,7 @@
     "swagger": "2.0",
     "info": {
         "title": "Paasta API",
-        "version": "1.0.0"
+        "version": "1.1.0"
     },
     "basePath": "/v1",
     "schemes": [
@@ -846,6 +846,194 @@
                     }
                 ]
             }
+        },
+        "/remote_run/{service}/{instance}/start": {
+            "post": {
+                "responses": {
+                    "200": {
+                        "description": "Successfully started remote-run sandbox",
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunOutcome"
+                        }
+                    },
+                    "404": {
+                        "description": "Deployment key not found"
+                    },
+                    "500": {
+                        "description": "Failure"
+                    }
+                },
+                "summary": "Launch a remote-run pod",
+                "operationId": "remote_run_start",
+                "tags": [
+                    "remote_run"
+                ],
+                "parameters": [
+                    {
+                        "in": "path",
+                        "description": "Service name",
+                        "name": "service",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "path",
+                        "description": "Instance name",
+                        "name": "instance",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "body",
+                        "description": "Job bootstrap settings",
+                        "name": "json_body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunStart"
+                        }
+                    }
+                ]
+            }
+        },
+        "/remote_run/{service}/{instance}/stop": {
+            "post": {
+                "responses": {
+                    "200": {
+                        "description": "Remote run pod stopped",
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunOutcome"
+                        }
+                    },
+                    "404": {
+                        "description": "Service instance not found"
+                    },
+                    "500": {
+                        "description": "Failure"
+                    }
+                },
+                "summary": "Stop a remote run",
+                "operationId": "remote_run_stop",
+                "tags": [
+                    "service"
+                ],
+                "parameters": [
+                    {
+                        "in": "path",
+                        "description": "Service name",
+                        "name": "service",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "path",
+                        "description": "Instance name",
+                        "name": "instance",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "body",
+                        "description": "Details about the job to be stopped",
+                        "name": "json_body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunStop"
+                        }
+                    }
+                ]
+            }
+        },
+        "/remote_run/{service}/{instance}/poll": {
+            "get": {
+                "responses": {
+                    "200": {
+                        "description": "Pod status information",
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunOutcome"
+                        }
+                    },
+                    "404": {
+                        "description": "Service instance not found"
+                    },
+                    "500": {
+                        "description": "Failure"
+                    }
+                },
+                "summary": "Check if remote run pod is ready",
+                "operationId": "remote_run_poll",
+                "tags": [
+                    "remote_run"
+                ],
+                "parameters": [
+                    {
+                        "in": "path",
+                        "description": "Service name",
+                        "name": "service",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "path",
+                        "description": "Instance name",
+                        "name": "instance",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "query",
+                        "description": "Kubernetes job name (returned from the /start endpoint)",
+                        "name": "job_name",
+                        "required": true,
+                        "type": "string"
+                    }
+                ]
+            }
+        },
+        "/remote_run/{service}/{instance}/token": {
+            "get": {
+                "responses": {
+                    "200": {
+                        "description": "Token generated successfully",
+                        "schema": {
+                            "$ref": "#/definitions/RemoteRunToken"
+                        }
+                    },
+                    "404": {
+                        "description": "Service instance not found"
+                    },
+                    "500": {
+                        "description": "Failure"
+                    }
+                },
+                "summary": "Get a remote run token",
+                "operationId": "remote_run_token",
+                "tags": [
+                    "remote_run"
+                ],
+                "parameters": [
+                    {
+                        "in": "path",
+                        "description": "Service name",
+                        "name": "service",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "path",
+                        "description": "Instance name",
+                        "name": "instance",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "in": "query",
+                        "description": "User requesting the token",
+                        "name": "user",
+                        "required": true,
+                        "type": "string"
+                    }
+                ]
+            }
         }
     },
     "definitions": {
@@ -1970,6 +2158,72 @@
                 }
             }
         },
+        "RemoteRunStart": {
+            "type": "object",
+            "properties": {
+                "interactive": {
+                    "type": "boolean"
+                },
+                "recreate": {
+                    "type": "boolean"
+                },
+                "user": {
+                    "type": "string"
+                },
+                "max_duration": {
+                    "type": "integer"
+                }
+            },
+            "required": [
+                "user"
+            ]
+        },
+        "RemoteRunStop": {
+            "type": "object",
+            "properties": {
+                "user": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "user"
+            ]
+        },
+        "RemoteRunOutcome": {
+            "type": "object",
+            "properties": {
+                "status": {
+                    "type": "integer"
+                },
+                "message": {
+                    "type": "string"
+                },
+                "pod_name": {
+                    "type": "string"
+                },
+                "job_name": {
+                    "type": "string"
+                },
+                "namespace": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "status",
+                "message"
+            ]
+        },
+        "RemoteRunToken": {
+            "type": "object",
+            "properties": {
+                "token": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "token"
+            ]
+        },
         "Resource": {
             "type": "array",
             "items": {

{paasta-tools-1.13.6 → paasta-tools-1.14.1}/paasta_tools/api/client.py

@@ -37,6 +37,7 @@ class PaastaOApiClient:
     default: paastaapis.DefaultApi
     resources: paastaapis.ResourcesApi
     service: paastaapis.ServiceApi
+    remote_run: paastaapis.RemoteRunApi
     api_error: Type[paastaapi.ApiException]
     connection_error: Type[paastaapi.ApiException]
     timeout_error: Type[paastaapi.ApiException]
@@ -72,6 +73,7 @@ def get_paasta_oapi_client_by_url(
         default=paastaapis.DefaultApi(client),
         resources=paastaapis.ResourcesApi(client),
         service=paastaapis.ServiceApi(client),
+        remote_run=paastaapis.RemoteRunApi(client),
         api_error=paastaapi.ApiException,
         connection_error=paastaapi.ApiException,
         timeout_error=paastaapi.ApiException,

{paasta-tools-1.13.6 → paasta-tools-1.14.1}/paasta_tools/api/views/instance.py

@@ -30,12 +30,12 @@ from pyramid.response import Response
 from pyramid.view import view_config
 
 import paasta_tools.mesos.exceptions as mesos_exceptions
-from paasta_tools import paasta_remote_run
 from paasta_tools import tron_tools
 from paasta_tools.api import settings
 from paasta_tools.api.views.exception import ApiFailure
 from paasta_tools.cli.cmds.status import get_actual_deployments
 from paasta_tools.instance import kubernetes as pik
+from paasta_tools.mesos_tools import get_all_frameworks as get_all_mesos_frameworks
 from paasta_tools.utils import compose_job_id
 from paasta_tools.utils import DeploymentVersion
 from paasta_tools.utils import NoConfigurationForServiceError
@@ -94,11 +94,19 @@ def tron_instance_status(
     return status
 
 
+def legacy_remote_run_filter_frameworks(service, instance, frameworks=None):
+    if frameworks is None:
+        frameworks = get_all_mesos_frameworks(active_only=True)
+
+    prefix = f"paasta-remote {service}.{instance}"
+    return [f for f in frameworks if f.name.startswith(prefix)]
+
+
 def adhoc_instance_status(
     instance_status: Mapping[str, Any], service: str, instance: str, verbose: int
 ) -> List[Dict[str, Any]]:
     status = []
-    filtered = paasta_remote_run.remote_run_filter_frameworks(service, instance)
+    filtered = legacy_remote_run_filter_frameworks(service, instance)
     filtered.sort(key=lambda x: x.name)
     for f in filtered:
         launch_time, run_id = re.match(

paasta-tools-1.14.1/paasta_tools/api/views/remote_run.py

@@ -0,0 +1,107 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import traceback
+
+from pyramid.view import view_config
+
+from paasta_tools.api import settings
+from paasta_tools.api.views.exception import ApiFailure
+from paasta_tools.kubernetes.remote_run import remote_run_ready
+from paasta_tools.kubernetes.remote_run import remote_run_start
+from paasta_tools.kubernetes.remote_run import remote_run_stop
+from paasta_tools.kubernetes.remote_run import remote_run_token
+from paasta_tools.utils import load_system_paasta_config
+
+
+DEFAULT_MAX_DURATION = 60 * 60  # 1 hour
+DEFAULT_MAX_DURATION_LIMIT = 8 * 60 * 60  # 8 hours
+
+
+@view_config(route_name="remote_run.start", request_method="POST", renderer="json")
+def view_remote_run_start(request):
+    system_config = load_system_paasta_config()
+    service = request.swagger_data["service"]
+    instance = request.swagger_data["instance"]
+    user = request.swagger_data["json_body"]["user"]
+    interactive = request.swagger_data["json_body"].get("interactive", True)
+    recreate = request.swagger_data["json_body"].get("recreate", False)
+    max_duration = min(
+        request.swagger_data["json_body"].get("max_duration", DEFAULT_MAX_DURATION),
+        system_config.get_remote_run_duration_limit(DEFAULT_MAX_DURATION_LIMIT),
+    )
+    try:
+        return remote_run_start(
+            service=service,
+            instance=instance,
+            cluster=settings.cluster,
+            user=user,
+            interactive=interactive,
+            recreate=recreate,
+            max_duration=max_duration,
+        )
+    except Exception:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)
+
+
+@view_config(route_name="remote_run.poll", request_method="GET", renderer="json")
+def view_remote_run_poll(request):
+    service = request.swagger_data["service"]
+    instance = request.swagger_data["instance"]
+    job_name = request.swagger_data["job_name"]
+    try:
+        return remote_run_ready(
+            service=service,
+            instance=instance,
+            cluster=settings.cluster,
+            job_name=job_name,
+        )
+    except Exception:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)
+
+
+@view_config(route_name="remote_run.stop", request_method="POST", renderer="json")
+def view_remote_run_stop(request):
+    service = request.swagger_data["service"]
+    instance = request.swagger_data["instance"]
+    user = request.swagger_data["json_body"]["user"]
+    try:
+        return remote_run_stop(
+            service=service,
+            instance=instance,
+            cluster=settings.cluster,
+            user=user,
+        )
+    except Exception:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)
+
+
+@view_config(route_name="remote_run.token", request_method="GET", renderer="json")
+def view_remote_run_token(request):
+    service = request.swagger_data["service"]
+    instance = request.swagger_data["instance"]
+    user = request.swagger_data["user"]
+    try:
+        token = remote_run_token(
+            service=service,
+            instance=instance,
+            cluster=settings.cluster,
+            user=user,
+        )
+        return {"token": token}
+    except Exception:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)

{paasta-tools-1.13.6 → paasta-tools-1.14.1}/paasta_tools/cli/cmds/local_run.py

@@ -1227,7 +1227,6 @@ def configure_and_run_docker_container(
 
     for volume in instance_config.get_volumes(
         system_paasta_config.get_volumes(),
-        system_paasta_config.get_uses_bulkdata_default(),
     ):
         if os.path.exists(volume["hostPath"]):
             volumes.append(

paasta-tools-1.14.1/paasta_tools/cli/cmds/remote_run.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# Copyright 2015-2017 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import pty
+import shlex
+import time
+
+from paasta_tools.cli.utils import get_paasta_oapi_api_clustername
+from paasta_tools.cli.utils import get_paasta_oapi_client_with_auth
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.paastaapi.model.remote_run_start import RemoteRunStart
+from paasta_tools.paastaapi.model.remote_run_stop import RemoteRunStop
+from paasta_tools.utils import get_username
+from paasta_tools.utils import list_clusters
+from paasta_tools.utils import list_services
+from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import SystemPaastaConfig
+
+
+KUBECTL_CMD_TEMPLATE = (
+    "kubectl-eks-{cluster} --token {token} exec -it -n {namespace} {pod} -- /bin/bash"
+)
+
+
+def paasta_remote_run_start(
+    args: argparse.Namespace,
+    system_paasta_config: SystemPaastaConfig,
+) -> int:
+    client = get_paasta_oapi_client_with_auth(
+        cluster=get_paasta_oapi_api_clustername(cluster=args.cluster, is_eks=True),
+        system_paasta_config=system_paasta_config,
+    )
+    if not client:
+        print("Cannot get a paasta-api client")
+        return 1
+
+    user = get_username()
+    start_response = client.remote_run.remote_run_start(
+        args.service,
+        args.instance,
+        RemoteRunStart(
+            user=user,
+            interactive=args.interactive,
+            recreate=args.recreate,
+            max_duration=args.max_duration,
+        ),
+    )
+    if start_response.status >= 300:
+        print(f"Error from PaaSTA APIs while starting job: {start_response.message}")
+        return 1
+
+    start_time = time.time()
+    while time.time() - start_time < args.timeout:
+        poll_response = client.remote_run.remote_run_poll(
+            args.service,
+            args.instance,
+            start_response.job_name,
+        )
+        if poll_response.status == 200:
+            break
+        time.sleep(10)
+    else:
+        print("Timed out while waiting for job to start")
+        return 1
+
+    if not args.interactive:
+        print("Successfully started remote-run job")
+        return 0
+
+    token_response = client.remote_run.remote_run_token(
+        args.service, args.instance, user
+    )
+
+    exec_command = KUBECTL_CMD_TEMPLATE.format(
+        cluster=args.cluster,
+        namespace=poll_response.namespace,
+        pod=poll_response.pod_name,
+        token=token_response.token,
+    )
+    pty.spawn(shlex.split(exec_command))
+    return 0
+
+
+def paasta_remote_run_stop(
+    args: argparse.Namespace,
+    system_paasta_config: SystemPaastaConfig,
+) -> int:
+    client = get_paasta_oapi_client_with_auth(
+        cluster=get_paasta_oapi_api_clustername(cluster=args.cluster, is_eks=True),
+        system_paasta_config=system_paasta_config,
+    )
+    if not client:
+        print("Cannot get a paasta-api client")
+        return 1
+    response = client.remote_run.remote_run_stop(
+        args.service, args.instance, RemoteRunStop(user=get_username())
+    )
+    print(response.message)
+    return 0 if response.status < 300 else 1
+
+
+def add_common_args_to_parser(parser: argparse.ArgumentParser):
+    service_arg = parser.add_argument(
+        "-s",
+        "--service",
+        help="The name of the service you wish to inspect. Required.",
+        required=True,
+    )
+    service_arg.completer = lazy_choices_completer(list_services)  # type: ignore
+    parser.add_argument(
+        "-i",
+        "--instance",
+        help=(
+            "Simulate a docker run for a particular instance of the "
+            "service, like 'main' or 'canary'. Required."
+        ),
+        required=True,
+    )
+    cluster_arg = parser.add_argument(
+        "-c",
+        "--cluster",
+        help="The name of the cluster you wish to run your task on. Required.",
+        required=True,
+    )
+    cluster_arg.completer = lazy_choices_completer(list_clusters)  # type: ignore
+
+
+def add_subparser(subparsers: argparse._SubParsersAction) -> None:
+    remote_run_parser = subparsers.add_parser(
+        "remote-run",
+        help="Run services / jobs remotely",
+        description="'paasta remote-run' runs services / jobs remotely",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    subparsers = remote_run_parser.add_subparsers(dest="remote_run_command")
+    start_parser = subparsers.add_parser(
+        "start",
+        help="Start or connect to a remote-run job",
+        description="Starts or connects to a remote-run-job",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    start_parser.add_argument(
+        "-I",
+        "--interactive",
+        help=(
+            "Run container in interactive mode. If interactive is set the "
+            'default command will be "bash" unless otherwise set by the "--cmd" flag'
+        ),
+        action="store_true",
+        default=False,
+    )
+    start_parser.add_argument(
+        "-m",
+        "--max-duration",
+        help=(
+            "Amount of time in seconds after which the job is "
+            "automatically stopped (capped by the API backend)"
+        ),
+        type=int,
+        default=1800,
+    )
+    start_parser.add_argument(
+        "-r",
+        "--recreate",
+        help="Recreate remote-run job if already existing",
+        action="store_true",
+        default=False,
+    )
+    start_parser.add_argument(
+        "-t",
+        "--timeout",
+        help="Maximum time to wait for a job to start, in seconds",
+        type=int,
+        default=600,
+    )
+    stop_parser = subparsers.add_parser(
+        "stop",
+        help="Stop your remote-run job if it exists",
+        description="Stop your remote-run job if it exists",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    add_common_args_to_parser(start_parser)
+    add_common_args_to_parser(stop_parser)
+    remote_run_parser.set_defaults(command=paasta_remote_run)
+
+
+def paasta_remote_run(args: argparse.Namespace) -> int:
+    system_paasta_config = load_system_paasta_config()
+    if args.remote_run_command == "start":
+        return paasta_remote_run_start(args, system_paasta_config)
+    elif args.remote_run_command == "stop":
+        return paasta_remote_run_stop(args, system_paasta_config)
+    raise ValueError(f"Unsupported subcommand: {args.remote_run_command}")