paasta-tools 1.4.16__tar.gz → 1.4.18__tar.gz

{paasta-tools-1.4.16/paasta_tools.egg-info → paasta-tools-1.4.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.16
+Version: 1.4.18
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.16 → paasta-tools-1.4.18}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.16"
+__version__ = "1.4.18"

{paasta-tools-1.4.16 → paasta-tools-1.4.18}/paasta_tools/cli/cmds/local_run.py

@@ -38,6 +38,7 @@ from paasta_tools.cli.cmds.check import makefile_responds_to
 from paasta_tools.cli.cmds.cook_image import paasta_cook_image
 from paasta_tools.cli.utils import figure_out_service_name
 from paasta_tools.cli.utils import get_instance_config
+from paasta_tools.cli.utils import get_service_auth_token
 from paasta_tools.cli.utils import lazy_choices_completer
 from paasta_tools.cli.utils import list_instances
 from paasta_tools.cli.utils import pick_random_port
@@ -506,6 +507,17 @@ def add_subparser(subparsers):
         required=False,
         default=False,
     )
+    list_parser.add_argument(
+        "--use-service-auth-token",
+        help=(
+            "Acquire service authentication token for the underlying instance,"
+            " and set it in the container environment"
+        ),
+        action="store_true",
+        dest="use_service_auth_token",
+        required=False,
+        default=False,
+    )
     list_parser.add_argument(
         "--sha",
         help=(
@@ -817,6 +829,7 @@ def run_docker_container(
     assume_role_arn="",
     use_okta_role=False,
     assume_role_aws_account: Optional[str] = None,
+    use_service_auth_token: bool = False,
 ):
     """docker-py has issues running a container with a TTY attached, so for
     consistency we execute 'docker run' directly in both interactive and
@@ -906,6 +919,9 @@ def run_docker_container(
         )
         environment.update(aws_creds)
 
+    if use_service_auth_token:
+        environment["YELP_SVC_AUTHZ_TOKEN"] = get_service_auth_token()
+
     local_run_environment = get_local_run_environment_vars(
         instance_config=instance_config, port0=chosen_port, framework=framework
     )
@@ -1251,6 +1267,7 @@ def configure_and_run_docker_container(
         assume_role_arn=args.assume_role_arn,
         assume_role_aws_account=assume_role_aws_account,
         use_okta_role=args.use_okta_role,
+        use_service_auth_token=args.use_service_auth_token,
     )
 
 

{paasta-tools-1.4.16 → paasta-tools-1.4.18}/paasta_tools/cli/schemas/tron_schema.json

@@ -383,18 +383,18 @@
                         },
                         "spark.executor.memory": {
                             "type": "string",
-                            "pattern": "^[1-9]+[0-9]*[kmg]$"
+                            "pattern": "^[1-9]+[0-9]*[kmgt]$"
                         },
                         "spark.driver.memory": {
                             "type": "string",
-                            "pattern": "^[1-9]+[0-9]*[kmg]$"
+                            "pattern": "^[1-9]+[0-9]*[kmgt]$"
                         },
                         "spark.driver.memoryOverhead": {
                             "$comment": "we still need to validate this in code since there's a spark-enforced minimum of 384mb",
                             "oneOf": [
                                 {
                                     "type": "string",
-                                    "pattern": "^[1-9]+[0-9]*[kmg]$"
+                                    "pattern": "^[1-9]+[0-9]*[kmgt]$"
                                 },
                                 {
                                     "type": "number",
@@ -408,7 +408,7 @@
                             "oneOf": [
                                 {
                                     "type": "string",
-                                    "pattern": "^[1-9]+[0-9]*[kmg]$"
+                                    "pattern": "^[1-9]+[0-9]*[kmgt]$"
                                 },
                                 {
                                     "type": "number",
@@ -419,7 +419,7 @@
                         },
                         "spark.driver.maxResultSize": {
                             "type": "string",
-                            "pattern": "^[1-9]+[0-9]*[kmg]$"
+                            "pattern": "^[1-9]+[0-9]*[kmgt]$"
                         },
                         "spark.scheduler.minRegisteredResourcesRatio": {
                             "type": "number",
@@ -429,7 +429,7 @@
                         },
                         "spark.scheduler.maxRegisteredResourcesWaitingTime": {
                             "type": "string",
-                            "pattern": "^[1-9]+[0-9]*(s|min)$"
+                            "pattern": "^[1-9]+[0-9]*(ms|s|m|min|h|d)$"
                         },
                         "spark.kubernetes.allocation.batch.size": {
                             "type": "integer"

{paasta-tools-1.4.16 → paasta-tools-1.4.18}/paasta_tools/cli/utils.py

@@ -37,6 +37,8 @@ from typing import Set
 from typing import Tuple
 
 import ephemeral_port_reserve
+from botocore.credentials import InstanceMetadataFetcher
+from botocore.credentials import InstanceMetadataProvider
 from mypy_extensions import NamedArg
 
 from paasta_tools import remote_git
@@ -76,6 +78,22 @@ from paasta_tools.utils import SystemPaastaConfig
 from paasta_tools.utils import validate_service_instance
 from paasta_tools.vitesscluster_tools import load_vitess_instance_config
 
+try:
+    from vault_tools.paasta_secret import get_client as get_vault_client
+    from vault_tools.paasta_secret import get_vault_url
+    from vault_tools.paasta_secret import get_vault_ca
+except ImportError:
+
+    def get_vault_client(url: str, capath: str) -> None:
+        pass
+
+    def get_vault_url(ecosystem: str) -> str:
+        return ""
+
+    def get_vault_ca(ecosystem: str) -> str:
+        return ""
+
+
 log = logging.getLogger(__name__)
 
 
@@ -1083,3 +1101,34 @@ def get_paasta_oapi_api_clustername(cluster: str, is_eks: bool) -> str:
     "eks-" prefix
     """
     return f"eks-{cluster}" if is_eks else cluster
+
+
+def get_current_ecosystem() -> str:
+    """Get current ecosystem from host configs, defaults to dev if no config is found"""
+    try:
+        with open("/nail/etc/ecosystem") as f:
+            return f.read().strip()
+    except IOError:
+        pass
+    return "devc"
+
+
+def get_service_auth_token() -> str:
+    """Uses instance profile to authenticate with Vault and generate token for service authentication"""
+    ecosystem = get_current_ecosystem()
+    vault_client = get_vault_client(get_vault_url(ecosystem), get_vault_ca(ecosystem))
+    vault_role = load_system_paasta_config().get_service_auth_vault_role()
+    metadata_provider = InstanceMetadataProvider(
+        iam_role_fetcher=InstanceMetadataFetcher(),
+    )
+    instance_credentials = metadata_provider.load().get_frozen_credentials()
+    vault_client.auth.aws.iam_login(
+        instance_credentials.access_key,
+        instance_credentials.secret_key,
+        instance_credentials.token,
+        mount_point="aws-iam",
+        role=vault_role,
+        use_token=True,
+    )
+    response = vault_client.secrets.identity.generate_signed_id_token(name=vault_role)
+    return response["data"]["token"]

{paasta-tools-1.4.16 → paasta-tools-1.4.18}/paasta_tools/kubernetes_tools.py

@@ -531,8 +531,8 @@ def limit_size_with_hash(name: str, limit: int = 63, suffix: int = 4) -> str:
     """
     if len(name) > limit:
         digest = hashlib.md5(name.encode()).digest()
-        hash = base64.b32encode(digest).decode().replace("=", "").lower()
-        return f"{name[:(limit-suffix-1)]}-{hash[:suffix]}"
+        hashed = base64.b32encode(digest).decode().replace("=", "").lower()
+        return f"{name[:(limit-suffix-1)]}-{hashed[:suffix]}"
     else:
         return name
 

{paasta-tools-1.4.16 → paasta-tools-1.4.18}/paasta_tools/tron_tools.py

@@ -364,6 +364,18 @@ class TronActionConfig(InstanceConfig):
             ),
             user=spark_tools.SPARK_JOB_USER,
         )
+        # delete the dynamically generated spark.app.id to prevent frequent config updates in Tron.
+        # spark.app.id will be generated later by yelp spark-submit wrapper or Spark itself.
+        spark_conf.pop("spark.app.id", None)
+        # use a static spark.app.name to prevent frequent config updates in Tron.
+        # md5 and base64 will always generate the same encoding for a string.
+        # This spark.app.name might be overridden by yelp spark-submit wrapper.
+        if "spark.app.name" in spark_conf:
+            spark_conf["spark.app.name"] = limit_size_with_hash(
+                f"tron_spark_{self.get_service()}_{self.get_instance()}_{self.get_action_name()}"
+                if "spark.app.name" not in stringified_spark_args
+                else stringified_spark_args["spark.app.name"]
+            )
         # TODO: Remove this once dynamic pod template is generated inside the driver using spark-submit wrapper
         if "spark.kubernetes.executor.podTemplateFile" in spark_conf:
             print(

{paasta-tools-1.4.16 → paasta-tools-1.4.18}/paasta_tools/utils.py

@@ -2041,6 +2041,7 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     secret_sync_delay_seconds: float
     use_multiple_log_readers: Optional[List[str]]
     service_auth_token_settings: ProjectedSAVolume
+    service_auth_vault_role: str
     always_authenticating_services: List[str]
     mysql_port_mappings: Dict
     vitess_images: Dict
@@ -2756,6 +2757,9 @@ class SystemPaastaConfig:
     def get_service_auth_token_volume_config(self) -> ProjectedSAVolume:
         return self.config_dict.get("service_auth_token_settings", {})
 
+    def get_service_auth_vault_role(self) -> str:
+        return self.config_dict.get("service_auth_vault_role", "service_authz")
+
     def get_always_authenticating_services(self) -> List[str]:
         return self.config_dict.get("always_authenticating_services", [])
 

{paasta-tools-1.4.16 → paasta-tools-1.4.18/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.16
+Version: 1.4.18
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp