paasta-tools 1.27.0__py3-none-any.whl → 1.35.8__py3-none-any.whl

paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.27.0"
+__version__ = "1.35.8"

paasta_tools/api/api_docs/swagger.json

@@ -2,7 +2,7 @@
     "swagger": "2.0",
     "info": {
         "title": "Paasta API",
-        "version": "1.2.0"
+        "version": "1.3.0"
     },
     "basePath": "/v1",
     "schemes": [
@@ -1788,6 +1788,11 @@
         "KubernetesVersion": {
             "type": "object",
             "properties": {
+                "container_port": {
+                    "description": "Port the container is expecting to receive traffic on",
+                    "type": "integer",
+                    "format": "int32"
+                },
                 "type": {
                     "description": "Type of version (ReplicaSet or ControllerRevision)",
                     "type": "string"
@@ -2324,6 +2329,9 @@
                 },
                 "toolbox": {
                     "type": "boolean"
+                },
+                "command": {
+                    "type": "string"
                 }
             },
             "required": [

paasta_tools/api/tweens/auth.py

@@ -77,7 +77,8 @@ class AuthTweenFactory:
         method: str,
         service: Optional[str],
     ) -> AuthorizationOutcome:
-        """Check if API request is authorized
+        """
+        Check if API request is authorized
 
         :param str path: API path
         :param str token: authentication token

paasta_tools/api/views/instance.py

@@ -26,6 +26,7 @@ from typing import Mapping
 from typing import Optional
 
 import a_sync
+from pyramid.request import Request
 from pyramid.response import Response
 from pyramid.view import view_config
 
@@ -139,7 +140,10 @@ def no_configuration_for_service_message(cluster, service, instance):
 @view_config(
     route_name="service.instance.status", request_method="GET", renderer="json"
 )
-def instance_status(request):
+def instance_status(
+    request: Request,
+) -> dict[str, Any]:  # godspeed to anyone typing the retval here
+    # NOTE: swagger_data is populated by pyramid_swagger
     service = request.swagger_data.get("service")
     instance = request.swagger_data.get("instance")
     verbose = request.swagger_data.get("verbose") or 0
@@ -353,7 +357,10 @@ def get_deployment_version(
     request_method="GET",
     renderer="json",
 )
-def instance_mesh_status(request):
+def instance_mesh_status(
+    request: Request,
+) -> dict[str, Any]:  # godspeed to anyone typing the retval here
+    # NOTE: swagger_data is populated by pyramid_swagger
     service = request.swagger_data.get("service")
     instance = request.swagger_data.get("instance")
     include_envoy = request.swagger_data.get("include_envoy")

paasta_tools/api/views/remote_run.py

@@ -35,6 +35,7 @@ def view_remote_run_start(request):
     interactive = request.swagger_data["json_body"].get("interactive", True)
     recreate = request.swagger_data["json_body"].get("recreate", False)
     is_toolbox = request.swagger_data["json_body"].get("toolbox", False)
+    command = request.swagger_data["json_body"].get("command", None)
     max_duration = min(
         request.swagger_data["json_body"].get("max_duration", DEFAULT_MAX_DURATION),
         get_max_job_duration_limit(),
@@ -49,6 +50,7 @@ def view_remote_run_start(request):
             recreate=recreate,
             max_duration=max_duration,
             is_toolbox=is_toolbox,
+            command=command,
         )
     except Exception:
         error_message = traceback.format_exc()

paasta_tools/async_utils.py

@@ -3,9 +3,11 @@ import functools
 import time
 import weakref
 from collections import defaultdict
+from typing import Any
 from typing import AsyncIterable
 from typing import Awaitable
 from typing import Callable
+from typing import Coroutine
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -97,7 +99,8 @@ async def aiter_to_list(
 def async_timeout(
     seconds: int = 10,
 ) -> Callable[
-    [Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]  # wrapped  # inner
+    [Callable[..., Coroutine[Any, Any, T]]],
+    Callable[..., Coroutine[Any, Any, T]],  # wrapped  # inner
 ]:
     def outer(wrapped):
         @functools.wraps(wrapped)

paasta_tools/bounce_lib.py

@@ -42,11 +42,14 @@ BounceMethodResult = TypedDict(
 
 BounceMethod = Callable[
     [
-        Arg(BounceMethodConfigDict, "new_config"),
-        Arg(bool, "new_app_running"),
-        Arg(Collection, "happy_new_tasks"),
-        Arg(Sequence, "old_non_draining_tasks"),
-        DefaultArg(float, "margin_factor"),
+        Arg(
+            BounceMethodConfigDict,
+            "new_config",  # noqa: F821  # flake8 false-positive, these are not var references
+        ),
+        Arg(bool, "new_app_running"),  # noqa: F821  # flake8 false-positive
+        Arg(Collection, "happy_new_tasks"),  # noqa: F821  # flake8 false-positive
+        Arg(Sequence, "old_non_draining_tasks"),  # noqa: F821  # flake8 false-positive
+        DefaultArg(float, "margin_factor"),  # noqa: F821  # flake8 false-positive
     ],
     BounceMethodResult,
 ]

paasta_tools/check_services_replication_tools.py

@@ -56,10 +56,16 @@ log = logging.getLogger(__name__)
 
 CheckServiceReplication = Callable[
     [
-        Arg(InstanceConfig_T, "instance_config"),
-        Arg(Dict[str, Dict[str, List[V1Pod]]], "pods_by_service_instance"),
-        Arg(Any, "replication_checker"),
-        NamedArg(bool, "dry_run"),
+        Arg(
+            InstanceConfig_T,
+            "instance_config",  # noqa: F821  # flake8 false-positive, these are not var references
+        ),
+        Arg(
+            Dict[str, Dict[str, List[V1Pod]]],
+            "pods_by_service_instance",  # noqa: F821  # flake8 false-positive
+        ),
+        Arg(Any, "replication_checker"),  # noqa: F821  # flake8 false-positive
+        NamedArg(bool, "dry_run"),  # noqa: F821  # flake8 false-positive
     ],
     Optional[bool],
 ]

paasta_tools/check_spark_jobs.py

@@ -124,7 +124,7 @@ def format_framework(info):
 def format_message_for_service(service, frameworks):
     output = f"Found the following long-running Spark frameworks associated with service {service}.\n"
     output += (
-        f"Please check why they are still running and terminate if appropriate.\n\n"
+        "Please check why they are still running and terminate if appropriate.\n\n"
     )
     output += "\n".join(format_framework(f) for f in frameworks)
     return output

paasta_tools/cli/cli.py

@@ -68,7 +68,7 @@ class PrintsHelpOnErrorArgumentParser(argparse.ArgumentParser):
 def list_external_commands():
     p = subprocess.check_output(["/bin/bash", "-p", "-c", "compgen -A command paasta-"])
     lines = p.decode("utf-8").strip().split("\n")
-    return {l.replace("paasta-", "", 1) for l in lines}
+    return {line.replace("paasta-", "", 1) for line in lines}
 
 
 def calling_external_command():
@@ -132,10 +132,10 @@ PAASTA_SUBCOMMANDS = {
 }
 
 
-def get_argparser(commands=None):
+def get_argparser(commands: list[str] | None = None) -> argparse.ArgumentParser:
     """Create and return argument parser for a set of subcommands.
 
-    :param commands: Union[None, List[str]] If `commands` argument is `None`,
+    :param commands: list[str] | None: If `commands` argument is `None`,
     add full parsers for all subcommands, if `commands` is empty list -
     add thin parsers for all subcommands, otherwise - add full parsers for
     subcommands in the argument.
@@ -170,7 +170,7 @@ def get_argparser(commands=None):
 
     # Adding a separate help subparser allows us to respond to "help" without --help
     help_parser = subparsers.add_parser(
-        "help", help=f"run `paasta <subcommand> -h` for help"
+        "help", help="run `paasta <subcommand> -h` for help"
     )
     help_parser.set_defaults(command=None)
 

paasta_tools/cli/cmds/autoscale.py

@@ -48,6 +48,8 @@ def add_subparser(subparsers):
         "-s",
         "--service",
         help="Service that you want to autoscale. Like 'example_service'.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     autoscale_parser.add_argument(
         "-i",

paasta_tools/cli/cmds/check.py

@@ -53,6 +53,8 @@ def add_subparser(subparsers):
         "-s",
         "--service",
         help="The name of the service you wish to inspect. Defaults to autodetect.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     check_parser.add_argument(
         "-y",

paasta_tools/cli/cmds/cook_image.py

@@ -48,6 +48,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
             '"services-", as included in a Jenkins job name, '
             "will be stripped."
         ),
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
         required=True,
     )
     list_parser.add_argument(

paasta_tools/cli/cmds/get_docker_image.py

@@ -34,6 +34,8 @@ def add_subparser(subparsers):
         "--service",
         help="Name of the service which you want to get the docker image for.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-i",

paasta_tools/cli/cmds/get_image_version.py

@@ -54,6 +54,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         "--service",
         help="Name of the service which you want to get the image version for.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     arg_service.completer = lazy_choices_completer(list_services)  # type: ignore
     parser.add_argument(

paasta_tools/cli/cmds/get_latest_deployment.py

@@ -33,6 +33,8 @@ def add_subparser(subparsers):
         "--service",
         help="Name of the service which you want to get the latest deployment for.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-i",

paasta_tools/cli/cmds/info.py

@@ -51,7 +51,11 @@ def add_subparser(subparsers):
         ),
     )
     list_parser.add_argument(
-        "-s", "--service", help="The name of the service you wish to inspect"
+        "-s",
+        "--service",
+        help="The name of the service you wish to inspect",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-d",
@@ -78,8 +82,9 @@ def get_smartstack_endpoints(service, soa_dir):
         service, full_name=False, soa_dir=soa_dir
     ):
         mode = config.get("mode", "http")
+        url_scheme = "http" if mode == "http2" else mode
         port = config.get("proxy_port")
-        endpoints.append(f"{mode}://169.254.255.254:{port} ({name})")
+        endpoints.append(f"{url_scheme}://169.254.255.254:{port} ({name})")
     return endpoints
 
 
@@ -96,10 +101,12 @@ def get_deployments_strings(service: str, soa_dir: str) -> List[str]:
             service=service, namespace="main", soa_dir=soa_dir
         )
         service_mode = service_config.get_mode()
+        url_scheme = "http" if service_mode == "http2" else service_mode
+
         for cluster in deployments_to_clusters(deployments):
             if service_mode in TESTABLE_SERVICE_MODES:
                 link = PaastaColors.cyan(
-                    f"{service_mode}://{service}.proxy.{cluster}.paasta/"
+                    f"{url_scheme}://{service}.proxy.{cluster}.paasta/"
                 )
             else:
                 link = "N/A"

paasta_tools/cli/cmds/itest.py

@@ -41,6 +41,8 @@ def add_subparser(subparsers):
         help="Test and build docker image for this service. Leading "
         '"services-", as included in a Jenkins job name, '
         "will be stripped.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
         required=True,
     )
     list_parser.add_argument(

paasta_tools/cli/cmds/list_namespaces.py

@@ -31,6 +31,8 @@ def add_subparser(subparsers) -> None:
         "--service",
         help="Name of the service which you want to list the namespaces for.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     # Most services likely don't need to filter by cluster/instance, and can add namespaces from all instances
     list_parser.add_argument(

paasta_tools/cli/cmds/local_run.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import base64
 import datetime
 import json
 import os
@@ -31,7 +32,9 @@ from urllib.parse import urlparse
 import boto3
 import requests
 from docker import errors
+from docker.api.client import APIClient
 from mypy_extensions import TypedDict
+from service_configuration_lib import read_service_configuration
 
 from paasta_tools.adhoc_tools import get_default_interactive_config
 from paasta_tools.cli.authentication import get_service_auth_token
@@ -314,7 +317,10 @@ def add_subparser(subparsers):
             "test, ensuring that a service will work inside the docker container as expected. "
             "Additionally, 'local-run' can healthcheck a service per the configured healthcheck.\n\n"
             "Alternatively, 'local-run' can be used with --pull, which will pull the currently "
-            "deployed docker image and use it, instead of building one."
+            "deployed docker image and use it, instead of building one.\n\n"
+            "While we've tried to make 'local-run' match the real PaaSTA environment, "
+            "there are some differences/limitations: e.g., certain features like `boto_keys` "
+            "are not supported."
         ),
         epilog=(
             "Note: 'paasta local-run' uses docker commands, which may require elevated privileges "
@@ -322,7 +328,11 @@ def add_subparser(subparsers):
         ),
     )
     list_parser.add_argument(
-        "-s", "--service", help="The name of the service you wish to inspect"
+        "-s",
+        "--service",
+        help="The name of the service you wish to inspect",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-c",
@@ -609,26 +619,83 @@ class LostContainerException(Exception):
     pass
 
 
-def docker_pull_image(docker_url):
-    """Pull an image via ``docker pull``. Uses the actual pull command instead of the python
-    bindings due to the docker auth/registry transition. Once we are past Docker 1.6
-    we can use better credential management, but for now this function assumes the
-    user running the command has already been authorized for the registry"""
+class DockerAuthConfig(TypedDict):
+    username: str
+    password: str
+
+
+def get_readonly_docker_registry_auth_config(
+    docker_url: str,
+) -> DockerAuthConfig | None:
+    system_paasta_config = load_system_paasta_config()
+    config_path = system_paasta_config.get_readonly_docker_registry_auth_file()
+
+    try:
+        with open(config_path) as f:
+            docker_config = json.load(f)
+    except Exception:
+        print(
+            PaastaColors.yellow(
+                "Warning: unable to load read-only docker registry credentials."
+            ),
+            file=sys.stderr,
+        )
+        # the best we can do is try to pull with whatever auth the user has configured locally
+        # i.e., root-owned docker config in /root/.docker/config.json
+        return None
+    registry = docker_url.split("/")[0]
+
+    # find matching auth config - our usual ro config will have at least two entries
+    # at the time this comment was written
+    auths = docker_config
+    for auth_url, auth_data in auths.items():
+        if registry in auth_url:
+            # Decode the base64 auth string if present
+            if "auth" in auth_data:
+                auth_string = base64.b64decode(auth_data["auth"]).decode("utf-8")
+                username, password = auth_string.split(":", 1)
+                return {"username": username, "password": password}
+
+    # we'll hit this for registries like docker-dev or extra-private internal registries
+    return None
+
+
+def docker_pull_image(docker_client: APIClient, docker_url: str) -> None:
+    """Pull an image using the docker-py library with read-only registry credentials"""
     print(
-        "Please wait while the image (%s) is pulled (times out after 30m)..."
-        % docker_url,
+        f"Please wait while the image ({docker_url}) is pulled (times out after 30m)...",
         file=sys.stderr,
     )
-    with Timeout(
-        seconds=1800, error_message=f"Timed out pulling docker image from {docker_url}"
-    ), open(os.devnull, mode="wb") as DEVNULL:
-        ret, _ = _run("docker pull %s" % docker_url, stream=True, stdin=DEVNULL)
-        if ret != 0:
-            print(
-                "\nPull failed. Are you authorized to run docker commands?",
-                file=sys.stderr,
-            )
-            sys.exit(ret)
+
+    auth_config = get_readonly_docker_registry_auth_config(docker_url)
+    if not auth_config:
+        print(
+            PaastaColors.yellow(
+                "Warning: No read-only docker registry credentials found, attempting to pull without them."
+            ),
+            file=sys.stderr,
+        )
+
+    try:
+        with Timeout(
+            seconds=1800,
+            error_message=f"Timed out pulling docker image from {docker_url}",
+        ):
+            # this is slightly funky since pull() returns the output line-by-line, but as a dict
+            # ...that we then need to format back to the usual `docker pull` output
+            # :p
+            for line in docker_client.pull(
+                docker_url, auth_config=auth_config, stream=True, decode=True
+            ):
+                # not all lines have an 'id' key :(
+                id_prefix = f"{line['id']}: " if "id" in line else ""
+                print(f"{id_prefix}{line['status']}", file=sys.stderr)
+    except Exception as e:
+        print(
+            f"\nPull failed. Error: {e}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
 
 
 def get_container_id(docker_client, container_name):
@@ -977,12 +1044,12 @@ def run_docker_container(
             # First try to write the file as a string
             # This is for text like config files
             with open(temp_secret_filename, "w") as f:
-                f.write(secret_content)
+                f.write(secret_content)  # type: ignore  # TODO: make this type-safe rather than rely on exceptions
         except TypeError:
             # If that fails, try to write it as bytes
             # This is for binary files like TLS keys
             with open(temp_secret_filename, "wb") as fb:
-                fb.write(secret_content)
+                fb.write(secret_content)  # type: ignore  # TODO: make this type-safe rather than rely on exceptions
 
         # Append this to the list of volumes passed to docker run
         volumes.append(f"{temp_secret_filename}:{container_mount_path}:ro")
@@ -1223,7 +1290,7 @@ def configure_and_run_docker_container(
             return 1
 
     if pull_image:
-        docker_pull_image(docker_url)
+        docker_pull_image(docker_client, docker_url)
 
     for volume in instance_config.get_volumes(
         system_paasta_config.get_volumes(),
@@ -1299,10 +1366,40 @@ def docker_config_available():
     )
 
 
+def should_reexec_as_root(
+    service: str, skip_secrets: bool, action: str, soa_dir: str = DEFAULT_SOA_DIR
+) -> bool:
+    # local-run can't pull secrets from Vault in prod without a root-owned token
+    need_vault_token = not skip_secrets and action == "pull"
+
+    # there are some special teams with their own private docker registries and no ro creds
+    # however, we don't know what registry is to be used without loading the service config
+    service_info = read_service_configuration(service, soa_dir)
+    # technically folks can set the standard registry as a value here, but atm no one is doing that :p
+    registry_override = service_info.get("docker_registry")
+    # note: we could also have a list of registries that have ro creds, but this seems fine for now
+    uses_private_registry = (
+        registry_override
+        and registry_override
+        in load_system_paasta_config().get_private_docker_registries()
+    )
+    need_docker_config = uses_private_registry and action == "pull"
+
+    return (need_vault_token or need_docker_config) and os.geteuid() != 0
+
+
 def paasta_local_run(args):
-    if args.action == "pull" and os.geteuid() != 0 and not docker_config_available():
-        print("Re-executing paasta local-run --pull with sudo..")
-        os.execvp("sudo", ["sudo", "-H"] + sys.argv)
+    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
+    if should_reexec_as_root(
+        service, args.skip_secrets, args.action, args.yelpsoa_config_root
+    ):
+        # XXX: we should re-architect this to not need sudo, but for now,
+        # re-exec ourselves with sudo to get access to the paasta vault token
+        # NOTE: once we do that, we can also remove the venv check above :)
+        print(
+            "Re-executing paasta local-run --pull with sudo for Vault/Docker registry access..."
+        )
+        os.execvp("sudo", ["sudo", "-H", "/usr/bin/paasta"] + sys.argv[1:])
     if args.action == "build" and not makefile_responds_to("cook-image"):
         print(
             "A local Makefile with a 'cook-image' target is required for --build",
@@ -1329,8 +1426,6 @@ def paasta_local_run(args):
 
     local_run_config = system_paasta_config.get_local_run_config()
 
-    service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
-
     if args.cluster:
         cluster = args.cluster
     else:

paasta_tools/cli/cmds/logs.py

@@ -30,6 +30,7 @@ from typing import Callable
 from typing import ContextManager
 from typing import Dict
 from typing import Iterable
+from typing import Iterator
 from typing import List
 from typing import Mapping
 from typing import MutableSequence
@@ -52,10 +53,31 @@ try:
 except ImportError:
     scribereader = None
 
+# NOTE: this is an internal-only package, so we won't be able to typecheck against it with mypy
+# without these hacky inlined stubs
 try:
     from logreader.readers import S3LogsReader
+
+    s3reader_available = True
 except ImportError:
-    S3LogsReader = None
+    s3reader_available = False
+
+    class S3LogsReader:  # type: ignore[no-redef]  # stub class for internal-only package
+        def __init__(self, superregion: str) -> None:
+            raise ImportError(
+                "logreader (internal Yelp package) is not available - unable to display logs."
+            )
+
+        def get_log_reader(
+            self,
+            log_name: str,
+            start_datetime: datetime.datetime,
+            end_datetime: datetime.datetime,
+        ) -> Iterator[str]:
+            raise NotImplementedError(
+                "logreader (internal Yelp package) is not available - unable to display logs."
+            )
+
 
 from pytimeparse.timeparse import timeparse
 
@@ -95,6 +117,8 @@ def add_subparser(subparsers) -> None:
         "-s",
         "--service",
         help="The name of the service you wish to inspect. Defaults to autodetect.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     status_parser.add_argument(
         "-c",
@@ -156,7 +180,7 @@ def add_subparser(subparsers) -> None:
         dest="soa_dir",
         metavar="SOA_DIR",
         default=DEFAULT_SOA_DIR,
-        help=f"Define a different soa config directory. Defaults to %(default)s.",
+        help="Define a different soa config directory. Defaults to %(default)s.",
     )
 
     status_parser.add_argument(
@@ -1174,7 +1198,7 @@ class VectorLogsReader(LogReader):
     ) -> None:
         super().__init__()
 
-        if S3LogsReader is None:
+        if not s3reader_available:
             raise Exception("yelp_clog package must be available to use S3LogsReader")
 
         self.cluster_map = cluster_map
@@ -1226,16 +1250,16 @@ class VectorLogsReader(LogReader):
                 except ValueError:
                     timestamp = pytz.utc.localize(datetime.datetime.min)
 
-                line = {"raw_line": line, "sort_key": timestamp}
-                aggregated_logs.append(line)
+                formatted_line = {"raw_line": line, "sort_key": timestamp}
+                aggregated_logs.append(formatted_line)
 
         aggregated_logs = list(
             {line["raw_line"]: line for line in aggregated_logs}.values()
         )
         aggregated_logs.sort(key=lambda log_line: log_line["sort_key"])
 
-        for line in aggregated_logs:
-            print_log(line["raw_line"], levels, raw_mode, strip_headers)
+        for formatted_line in aggregated_logs:
+            print_log(formatted_line["raw_line"], levels, raw_mode, strip_headers)
 
     def tail_logs(
         self,