paasta-tools 1.22.0__tar.gz → 1.23.0__tar.gz

{paasta_tools-1.22.0/paasta_tools.egg-info → paasta_tools-1.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.22.0
+Version: 1.23.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -58,7 +58,7 @@ Requires-Dist: requests-cache>=0.4.10
 Requires-Dist: retry
 Requires-Dist: ruamel.yaml
 Requires-Dist: sensu-plugin
-Requires-Dist: service-configuration-lib>=3.3.2
+Requires-Dist: service-configuration-lib>=3.3.3
 Requires-Dist: signalfx
 Requires-Dist: slackclient>=1.2.1
 Requires-Dist: sticht>=1.1.0

{paasta_tools-1.22.0 → paasta_tools-1.23.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.22.0"
+__version__ = "1.23.0"

{paasta_tools-1.22.0 → paasta_tools-1.23.0}/paasta_tools/api/api_docs/swagger.json

@@ -1037,6 +1037,9 @@
                     "404": {
                         "description": "Service instance not found"
                     },
+                    "409": {
+                        "description": "A pod was found but is currently being terminated"
+                    },
                     "500": {
                         "description": "Failure"
                     }

{paasta_tools-1.22.0 → paasta_tools-1.23.0}/paasta_tools/cli/cmds/remote_run.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 import argparse
 import shutil
+import subprocess
+import sys
 import time
 from typing import List
 
@@ -32,9 +34,12 @@ from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import SystemPaastaConfig
 
 
-KUBECTL_CMD_TEMPLATE = (
+KUBECTL_EXEC_CMD_TEMPLATE = (
     "{kubectl_wrapper} --token {token} exec -it -n {namespace} {pod} -- /bin/bash"
 )
+KUBECTL_CP_CMD_TEMPLATE = (
+    "{kubectl_wrapper} --token {token} -n {namespace} cp {filename} {pod}:/tmp/"
+)
 
 
 def _list_services_and_toolboxes() -> List[str]:
@@ -55,7 +60,9 @@ def _list_services_and_toolboxes() -> List[str]:
 def paasta_remote_run_start(
     args: argparse.Namespace,
     system_paasta_config: SystemPaastaConfig,
+    recursed: bool = False,
 ) -> int:
+    status_prefix = "\x1b[2K\r"  # Clear line, carriage return
     client = get_paasta_oapi_client_with_auth(
         cluster=get_paasta_oapi_api_clustername(cluster=args.cluster, is_eks=True),
         system_paasta_config=system_paasta_config,
@@ -95,10 +102,18 @@ def paasta_remote_run_start(
         if poll_response.status == 200:
             print("")
             break
-        print(f"\rStatus: {poll_response.message}", end="")
+        print(f"{status_prefix}Status: {poll_response.message}", end="")
+        if poll_response.status == 404:
+            # Probably indicates a pod was terminating. Now that its gone, retry the whole process
+            if not recursed:
+                print("\nPod finished terminating. Rerunning")
+                return paasta_remote_run_start(args, system_paasta_config, True)
+            else:
+                print("\nSomething went wrong. Pod still not found.")
+                return 1
         time.sleep(10)
     else:
-        print("Timed out while waiting for job to start")
+        print(f"{status_prefix}Timed out while waiting for job to start")
         return 1
 
     if not args.interactive and not args.toolbox:
@@ -120,13 +135,28 @@ def paasta_remote_run_start(
         kubectl_wrapper = f"kubectl-eks-{args.cluster}"
         if not shutil.which(kubectl_wrapper):
             kubectl_wrapper = f"kubectl-{args.cluster}"
-        exec_command = KUBECTL_CMD_TEMPLATE.format(
+        exec_command = KUBECTL_EXEC_CMD_TEMPLATE.format(
             kubectl_wrapper=kubectl_wrapper,
             namespace=poll_response.namespace,
             pod=poll_response.pod_name,
             token=token_response.token,
         )
 
+    if args.copy_file:
+        for filename in args.copy_file:
+            cp_command = KUBECTL_CP_CMD_TEMPLATE.format(
+                kubectl_wrapper=kubectl_wrapper,
+                namespace=poll_response.namespace,
+                pod=poll_response.pod_name,
+                filename=filename,
+                token=token_response.token,
+            ).split(" ")
+            call = subprocess.run(cp_command, capture_output=True)
+            if call.returncode != 0:
+                print("Error copying file to remote-run pod: ", file=sys.stderr)
+                print(call.stderr.decode("utf-8"), file=sys.stderr)
+                return 1
+
     run_interactive_cli(exec_command)
     return 0
 
@@ -232,6 +262,12 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         type=int,
         default=600,
     )
+    start_parser.add_argument(
+        "--copy-file",
+        help="Adds a local file to /tmp inside the pod",
+        type=str,
+        action="append",
+    )
     stop_parser = subparsers.add_parser(
         "stop",
         help="Stop your remote-run job if it exists",

{paasta_tools-1.22.0 → paasta_tools-1.23.0}/paasta_tools/cli/cmds/spark_run.py

@@ -375,6 +375,20 @@ def add_subparser(subparsers):
         default=False,
     )
 
+    list_parser.add_argument(
+        "--jira-ticket",
+        help=(
+            "The top level jira ticket used to track the project that this spark-job is related to. "
+            "eg: --jira-ticket=PROJ-123. "
+            "Must be passed for all adhoc jobs. "
+            "See https://yelpwiki.yelpcorp.com/spaces/AML/pages/402885641. "
+        ),
+        type=str,
+        required=False,
+        dest="jira_ticket",
+        default=None,
+    )
+
     aws_group = list_parser.add_argument_group(
         title="AWS credentials options",
         description="If --aws-credentials-yaml is specified, it overrides all "
@@ -1383,6 +1397,7 @@ def paasta_spark_run(args: argparse.Namespace) -> int:
         use_eks=True,
         k8s_server_address=k8s_server_address,
         service_account_name=service_account_name,
+        jira_ticket=args.jira_ticket,
     )
 
     return configure_and_run_docker_container(

{paasta_tools-1.22.0 → paasta_tools-1.23.0}/paasta_tools/kubernetes/remote_run.py

@@ -189,6 +189,8 @@ def remote_run_ready(
     if not pod:
         return {"status": 404, "message": "No pod found"}
     if pod.status.phase == "Running":
+        if pod.metadata.deletion_timestamp:
+            return {"status": 409, "message": "Pod is terminating"}
         result: RemoteRunOutcome = {
             "status": 200,
             "message": "Pod ready",

{paasta_tools-1.22.0 → paasta_tools-1.23.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.22.0
+Version: 1.23.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -58,7 +58,7 @@ Requires-Dist: requests-cache>=0.4.10
 Requires-Dist: retry
 Requires-Dist: ruamel.yaml
 Requires-Dist: sensu-plugin
-Requires-Dist: service-configuration-lib>=3.3.2
+Requires-Dist: service-configuration-lib>=3.3.3
 Requires-Dist: signalfx
 Requires-Dist: slackclient>=1.2.1
 Requires-Dist: sticht>=1.1.0

{paasta_tools-1.22.0 → paasta_tools-1.23.0}/paasta_tools.egg-info/requires.txt

@@ -49,7 +49,7 @@ requests-cache>=0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib>=3.3.2
+service-configuration-lib>=3.3.3
 signalfx
 slackclient>=1.2.1
 sticht>=1.1.0

{paasta_tools-1.22.0 → paasta_tools-1.23.0}/requirements-minimal.txt

@@ -56,7 +56,7 @@ requests-cache >= 0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib >= 3.3.2
+service-configuration-lib >= 3.3.3
 signalfx
 slackclient >= 1.2.1
 sticht >= 1.1.0