paasta-tools 0.230.1__tar.gz → 0.232.0__tar.gz

{paasta-tools-0.230.1/paasta_tools.egg-info → paasta-tools-0.232.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 0.230.1
+Version: 0.232.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "0.230.1"
+__version__ = "0.232.0"

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools/cli/schemas/eks_schema.json

@@ -586,6 +586,28 @@
                     },
                     "uniqueItems": true
                 },
+                "projected_sa_volumes": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "container_path": {
+                                "type": "string"
+                            },
+                            "audience": {
+                                "type": "string"
+                            },
+                            "expiration_seconds": {
+                                "type": "integer"
+                            }
+                        },
+                        "required": [
+                            "container_path",
+                            "audience"
+                        ]
+                    },
+                    "uniqueItems": true
+                },
                 "replication_threshold": {
                     "type": "integer",
                     "minimum": 0

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools/cli/schemas/kubernetes_schema.json

@@ -586,6 +586,28 @@
                     },
                     "uniqueItems": true
                 },
+                "projected_sa_volumes": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "container_path": {
+                                "type": "string"
+                            },
+                            "audience": {
+                                "type": "string"
+                            },
+                            "expiration_seconds": {
+                                "type": "integer"
+                            }
+                        },
+                        "required": [
+                            "container_path",
+                            "audience"
+                        ]
+                    },
+                    "uniqueItems": true
+                },
                 "replication_threshold": {
                     "type": "integer",
                     "minimum": 0

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools/cli/schemas/tron_schema.json

@@ -283,6 +283,28 @@
                     },
                     "uniqueItems": true
                 },
+                "projected_sa_volumes": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "container_path": {
+                                "type": "string"
+                            },
+                            "audience": {
+                                "type": "string"
+                            },
+                            "expiration_seconds": {
+                                "type": "integer"
+                            }
+                        },
+                        "required": [
+                            "container_path",
+                            "audience"
+                        ]
+                    },
+                    "uniqueItems": true
+                },
                 "cluster": {
                     "type": "string"
                 },

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools/cli/utils.py

@@ -53,6 +53,9 @@ from paasta_tools.long_running_service_tools import LongRunningServiceConfig
 from paasta_tools.marathon_tools import load_marathon_service_config
 from paasta_tools.monkrelaycluster_tools import load_monkrelaycluster_instance_config
 from paasta_tools.nrtsearchservice_tools import load_nrtsearchservice_instance_config
+from paasta_tools.nrtsearchserviceeks_tools import (
+    load_nrtsearchserviceeks_instance_config,
+)
 from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
 from paasta_tools.tron_tools import load_tron_instance_config
 from paasta_tools.utils import _log
@@ -796,6 +799,9 @@ INSTANCE_TYPE_HANDLERS: Mapping[str, InstanceTypeHandler] = defaultdict(
     nrtsearchservice=InstanceTypeHandler(
         get_service_instance_list, load_nrtsearchservice_instance_config
     ),
+    nrtsearchserviceeks=InstanceTypeHandler(
+        get_service_instance_list, load_nrtsearchserviceeks_instance_config
+    ),
     monkrelays=InstanceTypeHandler(
         get_service_instance_list, load_monkrelaycluster_instance_config
     ),
@@ -829,6 +835,9 @@ LONG_RUNNING_INSTANCE_TYPE_HANDLERS: Mapping[
     nrtsearchservice=LongRunningInstanceTypeHandler(
         get_service_instance_list, load_nrtsearchservice_instance_config
     ),
+    nrtsearchserviceeks=LongRunningInstanceTypeHandler(
+        get_service_instance_list, load_nrtsearchserviceeks_instance_config
+    ),
     monkrelays=LongRunningInstanceTypeHandler(
         get_service_instance_list, load_monkrelaycluster_instance_config
     ),

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools/instance/kubernetes.py

@@ -71,6 +71,7 @@ INSTANCE_TYPE_CR_ID = dict(
     kafkacluster=kafkacluster_tools.cr_id,
     vitesscluster=vitesscluster_tools.cr_id,
     nrtsearchservice=nrtsearchservice_tools.cr_id,
+    nrtsearchserviceeks=nrtsearchservice_tools.cr_id,
     monkrelaycluster=monkrelaycluster_tools.cr_id,
 )
 

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools/kubernetes_tools.py

@@ -98,6 +98,7 @@ from kubernetes.client import V1PodSpec
 from kubernetes.client import V1PodTemplateSpec
 from kubernetes.client import V1PreferredSchedulingTerm
 from kubernetes.client import V1Probe
+from kubernetes.client import V1ProjectedVolumeSource
 from kubernetes.client import V1ReplicaSet
 from kubernetes.client import V1ResourceRequirements
 from kubernetes.client import V1RoleBinding
@@ -108,6 +109,7 @@ from kubernetes.client import V1SecretKeySelector
 from kubernetes.client import V1SecretVolumeSource
 from kubernetes.client import V1SecurityContext
 from kubernetes.client import V1ServiceAccount
+from kubernetes.client import V1ServiceAccountTokenProjection
 from kubernetes.client import V1StatefulSet
 from kubernetes.client import V1StatefulSetSpec
 from kubernetes.client import V1Subject
@@ -115,6 +117,7 @@ from kubernetes.client import V1TCPSocketAction
 from kubernetes.client import V1TopologySpreadConstraint
 from kubernetes.client import V1Volume
 from kubernetes.client import V1VolumeMount
+from kubernetes.client import V1VolumeProjection
 from kubernetes.client import V1WeightedPodAffinityTerm
 from kubernetes.client import V2beta2CrossVersionObjectReference
 from kubernetes.client import V2beta2HorizontalPodAutoscaler
@@ -171,6 +174,7 @@ from paasta_tools.utils import load_v2_deployments_json
 from paasta_tools.utils import PaastaColors
 from paasta_tools.utils import PaastaNotConfiguredError
 from paasta_tools.utils import PersistentVolume
+from paasta_tools.utils import ProjectedSAVolume
 from paasta_tools.utils import SecretVolume
 from paasta_tools.utils import SystemPaastaConfig
 from paasta_tools.utils import time_cache
@@ -227,6 +231,9 @@ DEFAULT_SIDECAR_REQUEST: KubeContainerResourceRequest = {
     "ephemeral-storage": "256Mi",
 }
 
+DEFAULT_PROJECTED_SA_EXPIRATION_SECONDS = 3600
+PROJECTED_SA_TOKEN_PATH = "token"
+
 
 # conditions is None when creating a new HPA, but the client raises an error in that case.
 # For detail, https://github.com/kubernetes-client/python/issues/553
@@ -1002,6 +1009,14 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             "secret--{name}".format(name=secret_volume["secret_name"]), length_limit=63
         )
 
+    def get_projected_sa_volume_name(
+        self, projected_sa_volume: ProjectedSAVolume
+    ) -> str:
+        return self.get_sanitised_volume_name(
+            "projected-sa--{audience}".format(audience=projected_sa_volume["audience"]),
+            length_limit=63,
+        )
+
     def get_boto_secret_volume_name(self, service_name: str) -> str:
         return self.get_sanitised_volume_name(
             f"secret-boto-key-{service_name}", length_limit=63
@@ -1130,6 +1145,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                     aws_ebs_volumes=[],
                     persistent_volumes=[],
                     secret_volumes=[],
+                    projected_sa_volumes=[],
                 ),
             )
         return None
@@ -1440,6 +1456,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                 aws_ebs_volumes=aws_ebs_volumes,
                 persistent_volumes=self.get_persistent_volumes(),
                 secret_volumes=secret_volumes,
+                projected_sa_volumes=self.get_projected_sa_volumes(),
             ),
         )
         containers = [service_container] + self.get_sidecar_containers(  # type: ignore
@@ -1477,6 +1494,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
         docker_volumes: Sequence[DockerVolume],
         aws_ebs_volumes: Sequence[AwsEbsVolume],
         secret_volumes: Sequence[SecretVolume],
+        projected_sa_volumes: Sequence[ProjectedSAVolume],
     ) -> Sequence[V1Volume]:
         pod_volumes = []
         unique_docker_volumes = {
@@ -1534,6 +1552,27 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                     ),
                 )
             )
+        for projected_volume in projected_sa_volumes:
+            pod_volumes.append(
+                V1Volume(
+                    name=self.get_projected_sa_volume_name(projected_volume),
+                    projected=V1ProjectedVolumeSource(
+                        sources=[
+                            V1VolumeProjection(
+                                service_account_token=V1ServiceAccountTokenProjection(
+                                    audience=projected_volume["audience"],
+                                    expiration_seconds=projected_volume.get(
+                                        "expiration_seconds",
+                                        DEFAULT_PROJECTED_SA_EXPIRATION_SECONDS,
+                                    ),
+                                    path=PROJECTED_SA_TOKEN_PATH,
+                                )
+                            )
+                        ],
+                    ),
+                ),
+            )
+
         boto_volume = self.get_boto_volume()
         if boto_volume:
             pod_volumes.append(boto_volume)
@@ -1694,6 +1733,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
         aws_ebs_volumes: Sequence[AwsEbsVolume],
         persistent_volumes: Sequence[PersistentVolume],
         secret_volumes: Sequence[SecretVolume],
+        projected_sa_volumes: Sequence[ProjectedSAVolume],
     ) -> Sequence[V1VolumeMount]:
         volume_mounts = (
             [
@@ -1728,6 +1768,14 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                 )
                 for volume in secret_volumes
             ]
+            + [
+                V1VolumeMount(
+                    mount_path=volume["container_path"],
+                    name=self.get_projected_sa_volume_name(volume),
+                    read_only=True,
+                )
+                for volume in projected_sa_volumes
+            ]
         )
         if self.config_dict.get("boto_keys", []):
             secret_hash = self.get_boto_secret_hash()
@@ -2140,6 +2188,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                 docker_volumes=docker_volumes + hacheck_sidecar_volumes,
                 aws_ebs_volumes=self.get_aws_ebs_volumes(),
                 secret_volumes=self.get_secret_volumes(),
+                projected_sa_volumes=self.get_projected_sa_volumes(),
             ),
         )
         # need to check if there are node selectors/affinities. if there are none

paasta-tools-0.232.0/paasta_tools/nrtsearchserviceeks_tools.py

@@ -0,0 +1,68 @@
+from typing import Optional
+
+import service_configuration_lib
+
+from paasta_tools.nrtsearchservice_tools import NrtsearchServiceDeploymentConfig
+from paasta_tools.utils import BranchDictV2
+from paasta_tools.utils import deep_merge_dictionaries
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import load_service_instance_config
+from paasta_tools.utils import load_v2_deployments_json
+
+
+class NrtsearchServiceEksDeploymentConfig(NrtsearchServiceDeploymentConfig):
+    config_filename_prefix = "nrtsearchserviceeks"
+
+
+def load_nrtsearchserviceeks_instance_config(
+    service: str,
+    instance: str,
+    cluster: str,
+    load_deployments: bool = True,
+    soa_dir: str = DEFAULT_SOA_DIR,
+) -> NrtsearchServiceEksDeploymentConfig:
+    """Read a service instance's configuration for Nrtsearch.
+
+    If a branch isn't specified for a config, the 'branch' key defaults to
+    paasta-${cluster}.${instance}.
+
+    :param service: The service name
+    :param instance: The instance of the service to retrieve
+    :param cluster: The cluster to read the configuration for
+    :param load_deployments: A boolean indicating if the corresponding deployments.json for this service
+                             should also be loaded
+    :param soa_dir: The SOA configuration directory to read from
+    :returns: A dictionary of whatever was in the config for the service instance"""
+    general_config = service_configuration_lib.read_service_configuration(
+        service, soa_dir=soa_dir
+    )
+    instance_config = load_service_instance_config(
+        service, instance, "nrtsearchserviceeks", cluster, soa_dir=soa_dir
+    )
+    general_config = deep_merge_dictionaries(
+        overrides=instance_config, defaults=general_config
+    )
+
+    branch_dict: Optional[BranchDictV2] = None
+    if load_deployments:
+        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
+        temp_instance_config = NrtsearchServiceEksDeploymentConfig(
+            service=service,
+            cluster=cluster,
+            instance=instance,
+            config_dict=general_config,
+            branch_dict=None,
+            soa_dir=soa_dir,
+        )
+        branch = temp_instance_config.get_branch()
+        deploy_group = temp_instance_config.get_deploy_group()
+        branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group)
+
+    return NrtsearchServiceEksDeploymentConfig(
+        service=service,
+        cluster=cluster,
+        instance=instance,
+        config_dict=general_config,
+        branch_dict=branch_dict,
+        soa_dir=soa_dir,
+    )

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools/utils.py

@@ -138,6 +138,7 @@ INSTANCE_TYPES = (
     "vitesscluster",
     "monkrelays",
     "nrtsearchservice",
+    "nrtsearchserviceeks",
 )
 
 PAASTA_K8S_INSTANCE_TYPES = {
@@ -155,6 +156,7 @@ INSTANCE_TYPE_TO_K8S_NAMESPACE = {
     "kafkacluster": "paasta-kafkaclusters",
     "vitesscluster": "paasta-vitessclusters",
     "nrtsearchservice": "paasta-nrtsearchservices",
+    "nrtsearchserviceeks": "paasta-nrtsearchservices",
 }
 
 SHARED_SECRETS_K8S_NAMESPACES = {"paasta-spark", "paasta-cassandraclusters"}
@@ -285,6 +287,12 @@ class SecretVolume(TypedDict, total=False):
     items: List[SecretVolumeItem]
 
 
+class ProjectedSAVolume(TypedDict, total=False):
+    container_path: str
+    audience: str
+    expiration_seconds: int
+
+
 class TronSecretVolume(SecretVolume, total=False):
     secret_volume_name: str
 
@@ -329,6 +337,7 @@ class InstanceConfigDict(TypedDict, total=False):
     extra_volumes: List[DockerVolume]
     aws_ebs_volumes: List[AwsEbsVolume]
     secret_volumes: List[SecretVolume]
+    projected_sa_volumes: List[ProjectedSAVolume]
     security: SecurityConfigDict
     dependencies_reference: str
     dependencies: Dict[str, Dict]
@@ -958,6 +967,9 @@ class InstanceConfig:
     def get_secret_volumes(self) -> List[SecretVolume]:
         return self.config_dict.get("secret_volumes", [])
 
+    def get_projected_sa_volumes(self) -> List[ProjectedSAVolume]:
+        return self.config_dict.get("projected_sa_volumes", [])
+
     def get_iam_role(self) -> str:
         return self.config_dict.get("iam_role", "")
 

{paasta-tools-0.230.1 → paasta-tools-0.232.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 0.230.1
+Version: 0.232.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-0.230.1 → paasta-tools-0.232.0}/paasta_tools.egg-info/SOURCES.txt

@@ -66,6 +66,7 @@ paasta_tools/monitoring_tools.py
 paasta_tools/monkrelaycluster_tools.py
 paasta_tools/native_mesos_scheduler.py
 paasta_tools/nrtsearchservice_tools.py
+paasta_tools/nrtsearchserviceeks_tools.py
 paasta_tools/oom_logger.py
 paasta_tools/paasta_cluster_boost.py
 paasta_tools/paasta_deploy_tron_jobs