paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/frameworks/native_service_config.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python
+import copy
+from typing import Any
+from typing import cast
+from typing import List
+from typing import Optional
+
+from mypy_extensions import TypedDict
+
+from paasta_tools.long_running_service_tools import load_service_namespace_config
+from paasta_tools.long_running_service_tools import LongRunningServiceConfig
+from paasta_tools.long_running_service_tools import LongRunningServiceConfigDict
+from paasta_tools.long_running_service_tools import ServiceNamespaceConfig
+from paasta_tools.utils import BranchDictV2
+from paasta_tools.utils import compose_job_id
+from paasta_tools.utils import Constraint  # noqa, imported for typing.
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DockerParameter
+from paasta_tools.utils import get_code_sha_from_dockerurl
+from paasta_tools.utils import get_config_hash
+from paasta_tools.utils import load_service_instance_config
+from paasta_tools.utils import load_v2_deployments_json
+from paasta_tools.utils import SystemPaastaConfig
+
+
+MESOS_TASK_SPACER = "."
+
+
+VolumeInfo = TypedDict(
+    "VolumeInfo", {"container_path": str, "host_path": str, "mode": str}
+)
+
+_Docker_PortMapping = TypedDict(
+    "_Docker_PortMapping", {"host_port": int, "container_port": int, "protocol": str}
+)
+
+DockerInfo = TypedDict(
+    "DockerInfo",
+    {
+        "image": str,
+        "network": str,
+        "port_mappings": List[_Docker_PortMapping],
+        "parameters": List[DockerParameter],
+    },
+)
+
+ContainerInfo = TypedDict(
+    "ContainerInfo", {"type": str, "docker": DockerInfo, "volumes": List[VolumeInfo]}
+)
+
+
+_CommandInfo_URI = TypedDict(
+    "_CommandInfo_URI",
+    {
+        "value": str,
+        "executable": bool,
+        "extract": bool,
+        "cache": bool,
+        "output_file": str,
+    },
+    total=False,
+)
+
+CommandInfo = TypedDict(
+    "CommandInfo",
+    {
+        "uris": List[_CommandInfo_URI],
+        "environment": Any,
+        "shell": bool,
+        "value": str,
+        "arguments": List[str],
+        "user": str,
+    },
+    total=False,
+)
+
+Value_Scalar = TypedDict("Value_Scalar", {"value": float})
+
+Value_Range = TypedDict("Value_Range", {"begin": int, "end": int})
+
+Value_Ranges = TypedDict("Value_Ranges", {"range": List[Value_Range]})
+
+Resource = TypedDict(
+    "Resource",
+    {"name": str, "type": str, "scalar": Value_Scalar, "ranges": Value_Ranges},
+    total=False,
+)
+
+
+TaskID = TypedDict("TaskID", {"value": str})
+
+SlaveID = TypedDict("SlaveID", {"value": str})
+
+
+class TaskInfoBase(TypedDict):
+    name: str
+    task_id: TaskID
+    agent_id: SlaveID
+    resources: List[Resource]
+
+
+class TaskInfo(TaskInfoBase):
+    container: ContainerInfo
+    command: CommandInfo
+
+
+class NativeServiceConfigDict(LongRunningServiceConfigDict):
+    pass
+
+
+class NativeServiceConfig(LongRunningServiceConfig):
+    config_dict: NativeServiceConfigDict
+    config_filename_prefix = "paasta_native"
+
+    def __init__(
+        self,
+        service: str,
+        instance: str,
+        cluster: str,
+        config_dict: NativeServiceConfigDict,
+        branch_dict: Optional[BranchDictV2],
+        soa_dir: str,
+        service_namespace_config: Optional[ServiceNamespaceConfig] = None,
+    ) -> None:
+        super().__init__(
+            cluster=cluster,
+            instance=instance,
+            service=service,
+            config_dict=config_dict,
+            branch_dict=branch_dict,
+            soa_dir=soa_dir,
+        )
+        # service_namespace_config may be omitted/set to None at first, then set
+        # after initializing. e.g. we do this in load_paasta_native_job_config
+        # so we can call get_nerve_namespace() to figure out what SNC to read.
+        # It may also be set to None if this service is not in nerve.
+        if service_namespace_config is not None:
+            self.service_namespace_config = service_namespace_config
+        else:
+            self.service_namespace_config = ServiceNamespaceConfig()
+
+    def task_name(self, base_task: TaskInfo) -> str:
+        code_sha = get_code_sha_from_dockerurl(
+            base_task["container"]["docker"]["image"]
+        )
+
+        filled_in_task = copy.deepcopy(base_task)
+        filled_in_task["name"] = ""
+        filled_in_task["task_id"] = {"value": ""}
+        filled_in_task["agent_id"] = {"value": ""}
+
+        config_hash = get_config_hash(
+            filled_in_task, force_bounce=self.get_force_bounce()
+        )
+
+        return compose_job_id(
+            self.service,
+            self.instance,
+            git_hash=code_sha,
+            config_hash=config_hash,
+            spacer=MESOS_TASK_SPACER,
+        )
+
+    def base_task(
+        self, system_paasta_config: SystemPaastaConfig, portMappings=True
+    ) -> TaskInfo:
+        """Return a TaskInfo Dict with all the fields corresponding to the
+        configuration filled in.
+
+        Does not include task.agent_id or a task.id; those need to be
+        computed separately.
+        """
+        docker_volumes = self.get_volumes(
+            system_volumes=system_paasta_config.get_volumes(),
+        )
+        task: TaskInfo = {
+            "name": "",
+            "task_id": {"value": ""},
+            "agent_id": {"value": ""},
+            "container": {
+                "type": "DOCKER",
+                "docker": {
+                    "image": self.get_docker_url(),
+                    "parameters": [
+                        {"key": param["key"], "value": param["value"]}
+                        for param in self.format_docker_parameters()
+                    ],
+                    "network": self.get_mesos_network_mode(),
+                    "port_mappings": [],
+                },
+                "volumes": [
+                    {
+                        "container_path": volume["containerPath"],
+                        "host_path": volume["hostPath"],
+                        "mode": volume["mode"].upper(),
+                    }
+                    for volume in docker_volumes
+                ],
+            },
+            "command": {
+                "value": str(self.get_cmd()),
+                "uris": [
+                    {
+                        "value": system_paasta_config.get_dockercfg_location(),
+                        "extract": False,
+                    }
+                ],
+            },
+            "resources": [
+                {
+                    "name": "cpus",
+                    "type": "SCALAR",
+                    "scalar": {"value": self.get_cpus()},
+                },
+                {"name": "mem", "type": "SCALAR", "scalar": {"value": self.get_mem()}},
+            ],
+        }
+
+        if portMappings:
+            task["container"]["docker"]["port_mappings"] = [
+                {
+                    "container_port": self.get_container_port(),
+                    # filled by tasks_and_state_for_offer()
+                    "host_port": 0,
+                    "protocol": "tcp",
+                }
+            ]
+
+            task["resources"].append(
+                {
+                    "name": "ports",
+                    "type": "RANGES",
+                    "ranges": {
+                        # filled by tasks_and_state_for_offer
+                        "range": [{"begin": 0, "end": 0}]
+                    },
+                }
+            )
+
+        task["name"] = self.task_name(task)
+
+        return task
+
+    def get_mesos_network_mode(self) -> str:
+        return self.get_net().upper()
+
+
+def load_paasta_native_job_config(
+    service: str,
+    instance: str,
+    cluster: str,
+    load_deployments: bool = True,
+    soa_dir: str = DEFAULT_SOA_DIR,
+    instance_type: str = "paasta_native",
+    config_overrides: Optional[NativeServiceConfigDict] = None,
+) -> NativeServiceConfig:
+    instance_config_dict = cast(
+        NativeServiceConfigDict,
+        load_service_instance_config(
+            service=service,
+            instance=instance,
+            instance_type=instance_type,
+            cluster=cluster,
+            soa_dir=soa_dir,
+        ),
+    )
+    branch_dict: Optional[BranchDictV2] = None
+    instance_config_dict.update(config_overrides or {})
+    if load_deployments:
+        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
+        temp_instance_config = NativeServiceConfig(
+            service=service,
+            cluster=cluster,
+            instance=instance,
+            config_dict=instance_config_dict,
+            branch_dict=None,
+            soa_dir=soa_dir,
+        )
+        branch = temp_instance_config.get_branch()
+        deploy_group = temp_instance_config.get_deploy_group()
+        branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group)
+
+    service_config = NativeServiceConfig(
+        service=service,
+        cluster=cluster,
+        instance=instance,
+        config_dict=instance_config_dict,
+        branch_dict=branch_dict,
+        soa_dir=soa_dir,
+    )
+
+    service_namespace_config = load_service_namespace_config(
+        service=service, namespace=service_config.get_nerve_namespace(), soa_dir=soa_dir
+    )
+    service_config.service_namespace_config = service_namespace_config
+
+    return service_config
+
+
+class UnknownNativeServiceError(Exception):
+    pass

paasta_tools/frameworks/task_store.py

@@ -0,0 +1,245 @@
+import copy
+import json
+from typing import Any
+from typing import Dict
+from typing import Tuple
+from typing import Type
+from typing import TypeVar
+from typing import Union
+
+from kazoo.client import KazooClient
+from kazoo.exceptions import BadVersionError
+from kazoo.exceptions import NodeExistsError
+from kazoo.exceptions import NoNodeError
+from kazoo.protocol.states import ZnodeStat
+
+from paasta_tools.utils import _log
+
+
+class MesosTaskParametersIsImmutableError(Exception):
+    pass
+
+
+_SelfT = TypeVar("_SelfT", bound="MesosTaskParameters")
+
+
+class MesosTaskParameters:
+    health: Any
+    mesos_task_state: str
+    is_draining: bool
+    is_healthy: bool
+    offer: Any
+    resources: Any
+
+    def __init__(
+        self,
+        health=None,
+        mesos_task_state=None,
+        is_draining=None,
+        is_healthy=None,
+        offer=None,
+        resources=None,
+    ):
+        self.__dict__["health"] = health
+        self.__dict__["mesos_task_state"] = mesos_task_state
+        self.__dict__["is_draining"] = is_draining
+        self.__dict__["is_healthy"] = is_healthy
+        self.__dict__["offer"] = offer
+        self.__dict__["resources"] = resources
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+    def __repr__(self):
+        return "{}(\n    {})".format(
+            type(self).__name__,
+            ",\n    ".join(["%s=%r" % kv for kv in self.__dict__.items()]),
+        )
+
+    def __setattr__(self, name, value):
+        raise MesosTaskParametersIsImmutableError()
+
+    def __delattr__(self, name):
+        raise MesosTaskParametersIsImmutableError()
+
+    def merge(self: _SelfT, **kwargs) -> "MesosTaskParameters":
+        """Return a merged MesosTaskParameters object, where attributes in other take precedence over self."""
+
+        new_dict = copy.deepcopy(self.__dict__)
+        new_dict.update(kwargs)
+
+        return MesosTaskParameters(**new_dict)
+
+    @classmethod
+    def deserialize(cls: Type[_SelfT], serialized_params: Union[str, bytes]) -> _SelfT:
+        return cls(**json.loads(serialized_params))
+
+    def serialize(self):
+        return json.dumps(self.__dict__).encode("utf-8")
+
+
+class TaskStore:
+    def __init__(self, service_name, instance_name, framework_id, system_paasta_config):
+        self.service_name = service_name
+        self.instance_name = instance_name
+        self.framework_id = framework_id
+        self.system_paasta_config = system_paasta_config
+
+    def get_task(self, task_id: str) -> MesosTaskParameters:
+        """Get task data for task_id. If we don't know about task_id, return None"""
+        raise NotImplementedError()
+
+    def get_all_tasks(self) -> Dict[str, MesosTaskParameters]:
+        """Returns a dictionary of task_id -> MesosTaskParameters for all known tasks."""
+        raise NotImplementedError()
+
+    def overwrite_task(self, task_id: str, params: MesosTaskParameters) -> None:
+        raise NotImplementedError()
+
+    def add_task_if_doesnt_exist(self, task_id: str, **kwargs) -> None:
+        """Add a task if it does not already exist. If it already exists, do nothing."""
+        if self.get_task(task_id) is not None:
+            return
+        else:
+            self.overwrite_task(task_id, MesosTaskParameters(**kwargs))
+
+    def update_task(self, task_id: str, **kwargs) -> MesosTaskParameters:
+        existing_task = self.get_task(task_id)
+        if existing_task:
+            merged_params = existing_task.merge(**kwargs)
+        else:
+            merged_params = MesosTaskParameters(**kwargs)
+
+        self.overwrite_task(task_id, merged_params)
+        return merged_params
+
+    def garbage_collect_old_tasks(self, max_dead_task_age: float) -> None:
+        # TODO: call me.
+        # TODO: implement in base class.
+        raise NotImplementedError()
+
+    def close(self):
+        pass
+
+
+class DictTaskStore(TaskStore):
+    def __init__(self, service_name, instance_name, framework_id, system_paasta_config):
+        self.tasks: Dict[str, MesosTaskParameters] = {}
+        super().__init__(
+            service_name, instance_name, framework_id, system_paasta_config
+        )
+
+    def get_task(self, task_id: str) -> MesosTaskParameters:
+        return self.tasks.get(task_id)
+
+    def get_all_tasks(self) -> Dict[str, MesosTaskParameters]:
+        """Returns a dictionary of task_id -> MesosTaskParameters for all known tasks."""
+        return dict(self.tasks)
+
+    def overwrite_task(self, task_id: str, params: MesosTaskParameters) -> None:
+        # serialize/deserialize to make sure the returned values are the same format as ZKTaskStore.
+        self.tasks[task_id] = MesosTaskParameters.deserialize(params.serialize())
+
+
+class ZKTaskStore(TaskStore):
+    def __init__(self, service_name, instance_name, framework_id, system_paasta_config):
+        super().__init__(
+            service_name, instance_name, framework_id, system_paasta_config
+        )
+        self.zk_hosts = system_paasta_config.get_zk_hosts()
+
+        # For some reason, I could not get the code suggested by this SO post to work to ensure_path on the chroot.
+        # https://stackoverflow.com/a/32785625/25327
+        # Plus, it just felt dirty to modify instance attributes of a running connection, especially given that
+        # KazooClient.set_hosts() doesn't allow you to change the chroot. Must be for a good reason.
+
+        chroot = f"task_store/{service_name}/{instance_name}/{framework_id}"
+
+        temp_zk_client = KazooClient(hosts=self.zk_hosts)
+        temp_zk_client.start()
+        temp_zk_client.ensure_path(chroot)
+        temp_zk_client.stop()
+        temp_zk_client.close()
+
+        self.zk_client = KazooClient(hosts=f"{self.zk_hosts}/{chroot}")
+        self.zk_client.start()
+        self.zk_client.ensure_path("/")
+
+    def close(self):
+        self.zk_client.stop()
+        self.zk_client.close()
+
+    def get_task(self, task_id: str) -> MesosTaskParameters:
+        params, stat = self._get_task(task_id)
+        return params
+
+    def _get_task(self, task_id: str) -> Tuple[MesosTaskParameters, ZnodeStat]:
+        """Like get_task, but also returns the ZnodeStat that self.zk_client.get() returns"""
+        try:
+            data, stat = self.zk_client.get("/%s" % task_id)
+            return MesosTaskParameters.deserialize(data), stat
+        except NoNodeError:
+            return None, None
+        except json.decoder.JSONDecodeError:
+            _log(
+                service=self.service_name,
+                instance=self.instance_name,
+                level="debug",
+                component="deploy",
+                line=f"Warning: found non-json-decodable value in zookeeper for task {task_id}: {data}",
+            )
+            return None, None
+
+    def get_all_tasks(self):
+        all_tasks = {}
+
+        for child_path in self.zk_client.get_children("/"):
+            task_id = self._task_id_from_zk_path(child_path)
+            params = self.get_task(task_id)
+            # sometimes there are bogus child ZK nodes. Ignore them.
+            if params is not None:
+                all_tasks[task_id] = params
+
+        return all_tasks
+
+    def update_task(self, task_id: str, **kwargs):
+        retry = True
+        while retry:
+            retry = False
+            existing_task, stat = self._get_task(task_id)
+
+            zk_path = self._zk_path_from_task_id(task_id)
+            if existing_task:
+                merged_params = existing_task.merge(**kwargs)
+                try:
+                    self.zk_client.set(
+                        zk_path, merged_params.serialize(), version=stat.version
+                    )
+                except BadVersionError:
+                    retry = True
+            else:
+                merged_params = MesosTaskParameters(**kwargs)
+                try:
+                    self.zk_client.create(zk_path, merged_params.serialize())
+                except NodeExistsError:
+                    retry = True
+
+        return merged_params
+
+    def overwrite_task(
+        self, task_id: str, params: MesosTaskParameters, version=-1
+    ) -> None:
+        try:
+            self.zk_client.set(
+                self._zk_path_from_task_id(task_id), params.serialize(), version=version
+            )
+        except NoNodeError:
+            self.zk_client.create(
+                self._zk_path_from_task_id(task_id), params.serialize()
+            )
+
+    def _zk_path_from_task_id(self, task_id: str) -> str:
+        return "/%s" % task_id
+
+    def _task_id_from_zk_path(self, zk_path: str) -> str:
+        return zk_path.lstrip("/")

paasta_tools/generate_all_deployments

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# Generates all the per-service deployments.json files
+#
+
+set -eo pipefail
+
+# xargs will return 0 if everything went ok, but 12X if something else went wrong
+paasta list | shuf | xargs -n 1 -r -P 4 generate_deployments_for_service -s

paasta_tools/generate_authenticating_services.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+A simple script to enumerate all services participating in authenticated
+communications, and list them in a YAML/JSON file.
+"""
+import argparse
+import glob
+import logging
+import os
+from typing import Dict
+from typing import List
+from typing import Set
+
+from paasta_tools import yaml_tools as yaml
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import write_json_configuration_file
+from paasta_tools.utils import write_yaml_configuration_file
+
+
+AUTHORIZATION_CONFIG_FILE = "authorization.yaml"
+
+
+def list_services_in_authz_config(config_path: str) -> Set[str]:
+    auth_config: dict = {}
+    try:
+        with open(config_path) as f:
+            auth_config = yaml.safe_load(f)
+    except Exception as e:
+        logging.warning(f"Issue loading {config_path}: {e}")
+    return {
+        service
+        for rule in auth_config.get("authorization", {}).get("rules", [])
+        for service in rule.get("identity_groups", {}).get("services", [])
+    }
+
+
+def enumerate_authenticating_services() -> Dict[str, List[str]]:
+    result = set()
+    config_path_pattern = os.path.join(DEFAULT_SOA_DIR, "*", AUTHORIZATION_CONFIG_FILE)
+    for authz_config in glob.glob(config_path_pattern):
+        result.update(list_services_in_authz_config(authz_config))
+    result.update(load_system_paasta_config().get_always_authenticating_services())
+    return {"services": sorted(result)}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Output filepath",
+        dest="output_filename",
+        required=True,
+    )
+    parser.add_argument(
+        "-f",
+        "--format",
+        help="Output format. Defaults to %(default)s",
+        dest="output_format",
+        choices=["yaml", "json"],
+        default="yaml",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+    if args.output_format == "yaml":
+        writer = write_yaml_configuration_file
+    elif args.output_format == "json":
+        writer = write_json_configuration_file
+    else:
+        raise NotImplementedError(f"Unknown format: {args.output_format}")
+    configuration = enumerate_authenticating_services()
+    writer(filename=args.output_filename, configuration=configuration)
+
+
+if __name__ == "__main__":
+    main()