paasta-tools 1.30.9__py3-none-any.whl → 1.35.8__py3-none-any.whl

paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.30.9"
+__version__ = "1.35.8"

paasta_tools/api/api_docs/swagger.json

@@ -1788,6 +1788,11 @@
         "KubernetesVersion": {
             "type": "object",
             "properties": {
+                "container_port": {
+                    "description": "Port the container is expecting to receive traffic on",
+                    "type": "integer",
+                    "format": "int32"
+                },
                 "type": {
                     "description": "Type of version (ReplicaSet or ControllerRevision)",
                     "type": "string"

paasta_tools/cli/cmds/autoscale.py

@@ -48,6 +48,8 @@ def add_subparser(subparsers):
         "-s",
         "--service",
         help="Service that you want to autoscale. Like 'example_service'.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     autoscale_parser.add_argument(
         "-i",

paasta_tools/cli/cmds/check.py

@@ -53,6 +53,8 @@ def add_subparser(subparsers):
         "-s",
         "--service",
         help="The name of the service you wish to inspect. Defaults to autodetect.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     check_parser.add_argument(
         "-y",

paasta_tools/cli/cmds/cook_image.py

@@ -48,6 +48,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
             '"services-", as included in a Jenkins job name, '
             "will be stripped."
         ),
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
         required=True,
     )
     list_parser.add_argument(

paasta_tools/cli/cmds/get_docker_image.py

@@ -34,6 +34,8 @@ def add_subparser(subparsers):
         "--service",
         help="Name of the service which you want to get the docker image for.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-i",

paasta_tools/cli/cmds/get_image_version.py

@@ -54,6 +54,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         "--service",
         help="Name of the service which you want to get the image version for.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     arg_service.completer = lazy_choices_completer(list_services)  # type: ignore
     parser.add_argument(

paasta_tools/cli/cmds/get_latest_deployment.py

@@ -33,6 +33,8 @@ def add_subparser(subparsers):
         "--service",
         help="Name of the service which you want to get the latest deployment for.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-i",

paasta_tools/cli/cmds/info.py

@@ -51,7 +51,11 @@ def add_subparser(subparsers):
         ),
     )
     list_parser.add_argument(
-        "-s", "--service", help="The name of the service you wish to inspect"
+        "-s",
+        "--service",
+        help="The name of the service you wish to inspect",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-d",

paasta_tools/cli/cmds/itest.py

@@ -41,6 +41,8 @@ def add_subparser(subparsers):
         help="Test and build docker image for this service. Leading "
         '"services-", as included in a Jenkins job name, '
         "will be stripped.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
         required=True,
     )
     list_parser.add_argument(

paasta_tools/cli/cmds/list_namespaces.py

@@ -31,6 +31,8 @@ def add_subparser(subparsers) -> None:
         "--service",
         help="Name of the service which you want to list the namespaces for.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     # Most services likely don't need to filter by cluster/instance, and can add namespaces from all instances
     list_parser.add_argument(

paasta_tools/cli/cmds/local_run.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import base64
 import datetime
 import json
 import os
@@ -31,7 +32,9 @@ from urllib.parse import urlparse
 import boto3
 import requests
 from docker import errors
+from docker.api.client import APIClient
 from mypy_extensions import TypedDict
+from service_configuration_lib import read_service_configuration
 
 from paasta_tools.adhoc_tools import get_default_interactive_config
 from paasta_tools.cli.authentication import get_service_auth_token
@@ -325,7 +328,11 @@ def add_subparser(subparsers):
         ),
     )
     list_parser.add_argument(
-        "-s", "--service", help="The name of the service you wish to inspect"
+        "-s",
+        "--service",
+        help="The name of the service you wish to inspect",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-c",
@@ -612,26 +619,83 @@ class LostContainerException(Exception):
     pass
 
 
-def docker_pull_image(docker_url):
-    """Pull an image via ``docker pull``. Uses the actual pull command instead of the python
-    bindings due to the docker auth/registry transition. Once we are past Docker 1.6
-    we can use better credential management, but for now this function assumes the
-    user running the command has already been authorized for the registry"""
+class DockerAuthConfig(TypedDict):
+    username: str
+    password: str
+
+
+def get_readonly_docker_registry_auth_config(
+    docker_url: str,
+) -> DockerAuthConfig | None:
+    system_paasta_config = load_system_paasta_config()
+    config_path = system_paasta_config.get_readonly_docker_registry_auth_file()
+
+    try:
+        with open(config_path) as f:
+            docker_config = json.load(f)
+    except Exception:
+        print(
+            PaastaColors.yellow(
+                "Warning: unable to load read-only docker registry credentials."
+            ),
+            file=sys.stderr,
+        )
+        # the best we can do is try to pull with whatever auth the user has configured locally
+        # i.e., root-owned docker config in /root/.docker/config.json
+        return None
+    registry = docker_url.split("/")[0]
+
+    # find matching auth config - our usual ro config will have at least two entries
+    # at the time this comment was written
+    auths = docker_config
+    for auth_url, auth_data in auths.items():
+        if registry in auth_url:
+            # Decode the base64 auth string if present
+            if "auth" in auth_data:
+                auth_string = base64.b64decode(auth_data["auth"]).decode("utf-8")
+                username, password = auth_string.split(":", 1)
+                return {"username": username, "password": password}
+
+    # we'll hit this for registries like docker-dev or extra-private internal registries
+    return None
+
+
+def docker_pull_image(docker_client: APIClient, docker_url: str) -> None:
+    """Pull an image using the docker-py library with read-only registry credentials"""
     print(
-        "Please wait while the image (%s) is pulled (times out after 30m)..."
-        % docker_url,
+        f"Please wait while the image ({docker_url}) is pulled (times out after 30m)...",
         file=sys.stderr,
     )
-    with Timeout(
-        seconds=1800, error_message=f"Timed out pulling docker image from {docker_url}"
-    ), open(os.devnull, mode="wb") as DEVNULL:
-        ret, _ = _run("docker pull %s" % docker_url, stream=True, stdin=DEVNULL)
-        if ret != 0:
-            print(
-                "\nPull failed. Are you authorized to run docker commands?",
-                file=sys.stderr,
-            )
-            sys.exit(ret)
+
+    auth_config = get_readonly_docker_registry_auth_config(docker_url)
+    if not auth_config:
+        print(
+            PaastaColors.yellow(
+                "Warning: No read-only docker registry credentials found, attempting to pull without them."
+            ),
+            file=sys.stderr,
+        )
+
+    try:
+        with Timeout(
+            seconds=1800,
+            error_message=f"Timed out pulling docker image from {docker_url}",
+        ):
+            # this is slightly funky since pull() returns the output line-by-line, but as a dict
+            # ...that we then need to format back to the usual `docker pull` output
+            # :p
+            for line in docker_client.pull(
+                docker_url, auth_config=auth_config, stream=True, decode=True
+            ):
+                # not all lines have an 'id' key :(
+                id_prefix = f"{line['id']}: " if "id" in line else ""
+                print(f"{id_prefix}{line['status']}", file=sys.stderr)
+    except Exception as e:
+        print(
+            f"\nPull failed. Error: {e}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
 
 
 def get_container_id(docker_client, container_name):
@@ -1226,7 +1290,7 @@ def configure_and_run_docker_container(
             return 1
 
     if pull_image:
-        docker_pull_image(docker_url)
+        docker_pull_image(docker_client, docker_url)
 
     for volume in instance_config.get_volumes(
         system_paasta_config.get_volumes(),
@@ -1302,10 +1366,40 @@ def docker_config_available():
     )
 
 
+def should_reexec_as_root(
+    service: str, skip_secrets: bool, action: str, soa_dir: str = DEFAULT_SOA_DIR
+) -> bool:
+    # local-run can't pull secrets from Vault in prod without a root-owned token
+    need_vault_token = not skip_secrets and action == "pull"
+
+    # there are some special teams with their own private docker registries and no ro creds
+    # however, we don't know what registry is to be used without loading the service config
+    service_info = read_service_configuration(service, soa_dir)
+    # technically folks can set the standard registry as a value here, but atm no one is doing that :p
+    registry_override = service_info.get("docker_registry")
+    # note: we could also have a list of registries that have ro creds, but this seems fine for now
+    uses_private_registry = (
+        registry_override
+        and registry_override
+        in load_system_paasta_config().get_private_docker_registries()
+    )
+    need_docker_config = uses_private_registry and action == "pull"
+
+    return (need_vault_token or need_docker_config) and os.geteuid() != 0
+
+
 def paasta_local_run(args):
-    if args.action == "pull" and os.geteuid() != 0 and not docker_config_available():
-        print("Re-executing paasta local-run --pull with sudo..")
-        os.execvp("sudo", ["sudo", "-H"] + sys.argv)
+    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
+    if should_reexec_as_root(
+        service, args.skip_secrets, args.action, args.yelpsoa_config_root
+    ):
+        # XXX: we should re-architect this to not need sudo, but for now,
+        # re-exec ourselves with sudo to get access to the paasta vault token
+        # NOTE: once we do that, we can also remove the venv check above :)
+        print(
+            "Re-executing paasta local-run --pull with sudo for Vault/Docker registry access..."
+        )
+        os.execvp("sudo", ["sudo", "-H", "/usr/bin/paasta"] + sys.argv[1:])
     if args.action == "build" and not makefile_responds_to("cook-image"):
         print(
             "A local Makefile with a 'cook-image' target is required for --build",
@@ -1332,8 +1426,6 @@ def paasta_local_run(args):
 
     local_run_config = system_paasta_config.get_local_run_config()
 
-    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
-
     if args.cluster:
         cluster = args.cluster
     else:

paasta_tools/cli/cmds/logs.py

@@ -117,6 +117,8 @@ def add_subparser(subparsers) -> None:
         "-s",
         "--service",
         help="The name of the service you wish to inspect. Defaults to autodetect.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     status_parser.add_argument(
         "-c",

paasta_tools/cli/cmds/mark_for_deployment.py

@@ -157,6 +157,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         help="Name of the service which you wish to mark for deployment. Leading "
         '"services-" will be stripped.',
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     arg_service.completer = lazy_choices_completer(list_services)  # type: ignore
     list_parser.add_argument(
@@ -1264,7 +1266,15 @@ class MarkForDeploymentProcess(RollbackSlackDeploymentProcess):
             self.ping_authors()
 
     def send_manual_rollback_instructions(self) -> None:
-        if self.deployment_version != self.old_deployment_version:
+        # NOTE: new deploy groups are not exactly a particularly frequent occurrence, but
+        # we want to prevent sending messages that look like
+        # `If you need to roll back manually, run: paasta rollback --service $S --deploy-group $G --commit None`
+        # since that's not actually valid/actionable.
+        # thus the seemingly out-of-nowhere old_git_sha check: new deploy groups won't have value set there.
+        if (
+            self.deployment_version != self.old_deployment_version
+            and self.old_git_sha is not None
+        ):
             extra_rollback_args = ""
             if self.old_deployment_version.image_version:
                 extra_rollback_args = (
@@ -1843,7 +1853,7 @@ async def wait_for_deployment(
             system_paasta_config.get_mark_for_deployment_default_time_before_first_diagnosis()
         )
 
-    with progressbar.ProgressBar(maxval=total_instances) as bar:
+    with progressbar.ProgressBar(max_value=total_instances) as bar:
         instance_done_futures = []
         with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
             for cluster, instance_configs in instance_configs_per_cluster.items():

paasta_tools/cli/cmds/mesh_status.py

@@ -44,9 +44,10 @@ def add_subparser(subparsers) -> None:
     mesh_status_parser.add_argument(
         "-s",
         "--service",
-        type=str,
         help="The name of the service you wish to inspect",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     mesh_status_parser.add_argument(
         "-i",

paasta_tools/cli/cmds/push_to_registry.py

@@ -62,6 +62,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         help='Name of service for which you wish to upload a docker image. Leading "services-", '
         "as included in a Jenkins job name, will be stripped.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     list_parser.add_argument(
         "-c",

paasta_tools/cli/cmds/remote_run.py

@@ -295,11 +295,20 @@ def paasta_remote_run_stop(
 
 
 def add_common_args_to_parser(parser: argparse.ArgumentParser):
+    def _validate_service_name(name: str) -> str:
+        # remove trailing slash if present for folks tab-completing directories
+        if name.rstrip("/") not in _list_services_and_toolboxes():
+            raise ValueError(f"{name} is not a known service name")
+        return name
+
     service_arg = parser.add_argument(
         "-s",
         "--service",
         help="The name of the service you wish to inspect. Required.",
         required=True,
+        # not using `choices` for validation to avoid the help text
+        # for the command being incredibly large
+        type=_validate_service_name,
     )
     service_arg.completer = lazy_choices_completer(_list_services_and_toolboxes)  # type: ignore
     instance_or_toolbox = parser.add_mutually_exclusive_group()
@@ -323,6 +332,7 @@ def add_common_args_to_parser(parser: argparse.ArgumentParser):
         "--cluster",
         help="The name of the cluster you wish to run your task on. Required.",
         required=True,
+        choices=list_clusters(),
     )
     cluster_arg.completer = lazy_choices_completer(list_clusters)  # type: ignore
 

paasta_tools/cli/cmds/rollback.py

@@ -107,7 +107,11 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
     )
 
     arg_service = list_parser.add_argument(
-        "-s", "--service", help='Name of the service to rollback (e.g. "service1")'
+        "-s",
+        "--service",
+        help='Name of the service to rollback (e.g. "service1")',
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     arg_service.completer = lazy_choices_completer(list_services)  # type: ignore
     list_parser.add_argument(

paasta_tools/cli/cmds/secret.py

@@ -212,7 +212,7 @@ def _add_vault_auth_args(parser: argparse.ArgumentParser):
         dest="vault_auth_method",
         required=False,
         default="token",
-        choices=["token", "ldap"],
+        choices=["token", "okta"],
     )
     parser.add_argument(
         "--vault-token-file",
@@ -246,6 +246,8 @@ def _add_common_args(parser: argparse.ArgumentParser, allow_shared: bool = True)
         "--service",
         required=not allow_shared,
         help="The name of the service on which you wish to act",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
 
     if allow_shared:
@@ -456,7 +458,7 @@ def paasta_secret(args):
                 "vault_token_file": args.vault_token_file,
                 # best solution so far is to change the below string to "token",
                 # so that token file is picked up from argparse
-                "vault_auth_method": "ldap",  # must have LDAP to get 2FA push for prod
+                "vault_auth_method": "okta",  # must use Okta to get 2FA push
             },
         )
         secret_provider.write_secret(

paasta_tools/cli/cmds/security_check.py

@@ -28,6 +28,8 @@ def add_subparser(subparsers):
         help='Name of service for which you wish to check. Leading "services-", as included in a '
         "Jenkins job name, will be stripped.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     list_parser.add_argument(
         "-c", "--commit", help="Git sha of the image to check", required=True

paasta_tools/cli/cmds/spark_run.py

@@ -238,6 +238,8 @@ def add_subparser(subparsers):
         "--service",
         help="The name of the service from which the Spark image is built.",
         default=DEFAULT_SPARK_SERVICE,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
 
     list_parser.add_argument(
@@ -897,6 +899,8 @@ def configure_and_run_docker_container(
         )
     )  # type:ignore
     environment.update(extra_driver_envs)
+    if "jupyter-lab" == args.cmd:
+        environment["SPARK_DRIVER_TYPE"] = "jupyter"
 
     if args.use_service_auth_token:
         environment["YELP_SVC_AUTHZ_TOKEN"] = get_service_auth_token()

paasta_tools/cli/cmds/status.py

@@ -203,7 +203,11 @@ def add_subparser(
 
 def add_instance_filter_arguments(status_parser, verb: str = "inspect") -> None:
     status_parser.add_argument(
-        "-s", "--service", help=f"The name of the service you wish to {verb}"
+        "-s",
+        "--service",
+        help=f"The name of the service you wish to {verb}",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     status_parser.add_argument(
         "-c",
@@ -891,7 +895,7 @@ def _print_flink_status_from_job_manager(
 
         # Print Flink Costs Link
         output.append(
-            f"      Flink Cost: https://splunk.yelpcorp.com/en-US/app/yelp_computeinfra/paasta_service_utilization?form.service={service}&form.field1.earliest=-30d%40d&form.field1.latest=now&form.instance={instance}&form.cluster={cluster}"
+            f"      Flink Cost: https://app.cloudzero.com/explorer?activeCostType=invoiced_amortized_cost&partitions=costcontext%3AResource%20Summary&dateRange=Last%2030%20Days&costcontext%3AKube%20Paasta%20Cluster={cluster}&costcontext%3APaasta%20Instance={instance}&costcontext%3APaasta%20Service={service}&showRightFlyout=filters"
         )
 
         output.append(f"{OUTPUT_HORIZONTAL_RULE}")
@@ -917,11 +921,13 @@ def _print_flink_status_from_job_manager(
             else f"{pod_evicted_count}"
         )
 
+    pods_total_count = pod_running_count + pod_evicted_count + pod_other_count
     output.append(
         "    Pods:"
         f" {pod_running_count} running,"
         f" {evicted} evicted,"
-        f" {pod_other_count} other"
+        f" {pod_other_count} other,"
+        f" {pods_total_count} total"
     )
 
     if not should_job_info_be_shown(status["state"]):
@@ -944,12 +950,19 @@ def _print_flink_status_from_job_manager(
             output.append(str(e))
             return 1
 
+        jobs_total_count = (
+            overview.jobs_running
+            + overview.jobs_finished
+            + overview.jobs_failed
+            + overview.jobs_cancelled
+        )
         output.append(
             "    Jobs:"
             f" {overview.jobs_running} running,"
             f" {overview.jobs_finished} finished,"
             f" {overview.jobs_failed} failed,"
-            f" {overview.jobs_cancelled} cancelled"
+            f" {overview.jobs_cancelled} cancelled,"
+            f" {jobs_total_count} total"
         )
         output.append(
             "   "
@@ -1313,7 +1326,9 @@ def get_version_table_entry(
             for state in ReplicaState
             if state in replica_state_counts
         ]
-        entry.append(f"  Replica States: {' / '.join(replica_state_display)}")
+        entry.append(
+            f"  Replica States: {' / '.join(replica_state_display)} / {replica_state_counts.total()} total"
+        )
         if not verbose:
             unhealthy_replicas = [
                 (state, pod) for state, pod in replica_states if state.is_unhealthy()
@@ -1321,13 +1336,23 @@ def get_version_table_entry(
             if unhealthy_replicas:
                 entry.append("    Unhealthy Replicas:")
                 replica_table = create_replica_table(
-                    unhealthy_replicas, service, instance, cluster, verbose
+                    unhealthy_replicas,
+                    service,
+                    instance,
+                    cluster,
+                    version.container_port,
+                    verbose,
                 )
                 for line in replica_table:
                     entry.append(f"      {line}")
         else:
             replica_table = create_replica_table(
-                replica_states, service, instance, cluster, verbose
+                replica_states,
+                service,
+                instance,
+                cluster,
+                version.container_port,
+                verbose,
             )
             for line in replica_table:
                 entry.append(f"    {line}")
@@ -1467,6 +1492,7 @@ def create_replica_table(
     service: str,
     instance: str,
     cluster: str,
+    container_port: int,
     verbose: int = 0,
 ) -> List[str]:
     header = ["ID", "IP/Port", "Host deployed to", "Started at what localtime", "State"]
@@ -1474,9 +1500,10 @@ def create_replica_table(
     for state, pod in pods:
         start_datetime = datetime.fromtimestamp(pod.create_timestamp)
         humanized_start_time = humanize.naturaltime(start_datetime)
+
         row = [
             pod.name,
-            f"{pod.ip}:8888" if pod.ip else "None",
+            f"{pod.ip}:{container_port}" if pod.ip else "None",
             pod.host or "None",
             humanized_start_time,
             state.formatted_message,