paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/remote_git.py

@@ -0,0 +1,127 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+
+import dulwich.client
+import dulwich.errors
+
+from paasta_tools.utils import _run
+from paasta_tools.utils import timeout
+
+
+def _make_determine_wants_func(ref_mutator):
+    """Returns a safer version of ref_mutator, suitable for passing as the
+    determine_wants argument to dulwich's send_pack method. The returned
+    function will not delete or modify any existing refs."""
+
+    def determine_wants(old_refs):
+        refs = {k.decode("UTF-8"): v.decode("UTF-8") for k, v in old_refs.items()}
+        new_refs = ref_mutator(refs)
+        new_refs = {k.encode("UTF-8"): v.encode("UTF-8") for k, v in new_refs.items()}
+        new_refs.update(old_refs)  # Make sure we don't delete/modify anything.
+        return new_refs
+
+    return determine_wants
+
+
+def make_force_push_mutate_refs_func(targets, sha):
+    """Create a 'force push' function that will inform send_pack that we want
+    to mark a certain list of target branches/tags to point to a particular
+    git_sha.
+
+    :param targets: List of branches/tags to point at the input sha
+    :param sha: The git sha to point the branches/tags at
+    :returns: A function to do the ref manipulation that a dulwich client can use"""
+
+    def mutate_refs(refs):
+        for target in targets:
+            refs[target.encode("UTF-8")] = sha.encode("UTF-8")
+        return refs
+
+    return mutate_refs
+
+
+def create_remote_refs(git_url, ref_mutator, force=False):
+    """Creates refs (tags, branches) on a remote git repo.
+
+    :param git_url: the URL or path to the remote git repo.
+    :param ref_mutator: A function that determines the new refs to create on
+                        the remote repo. This gets passed a dictionary of the
+                        remote server's refs in the format {name : hash, ...},
+                        and should return a dictionary of the same format.
+    :param force: Bool, defaults to false. If true we will overwrite
+                  refs even if they are already set.
+    :returns: The map of refs, with our changes applied.
+    """
+    client, path = dulwich.client.get_transport_and_path(git_url)
+
+    if force is False:
+        determine_wants = _make_determine_wants_func(ref_mutator)
+    else:
+        determine_wants = ref_mutator
+    # We know we don't need to push any objects.
+
+    def generate_pack_contents(have, want):
+        return []
+
+    return client.send_pack(path, determine_wants, generate_pack_contents)
+
+
+class LSRemoteException(Exception):
+    pass
+
+
+@timeout(
+    seconds=60,
+    error_message="Timed out connecting to git server, is it reachable from where you are?",
+    use_signals=False,
+)
+def list_remote_refs(git_url):
+    """Get the refs from a remote git repo as a dictionary of name->hash."""
+    client, path = dulwich.client.get_transport_and_path(git_url)
+    try:
+        refs = client.fetch_pack(path, lambda refs: [], None, lambda data: None)
+        return {k.decode("UTF-8"): v.decode("UTF-8") for k, v in refs.items()}
+    except dulwich.errors.HangupException as e:
+        raise LSRemoteException(f"Unable to fetch remote refs from {git_url}: {e}")
+
+
+def get_authors(git_url, from_sha, to_sha):
+    """Gets the list of authors who contributed to a git changeset.
+    Currently only supports fetching this in a very "yelpy" way by
+    executing a gitolite command"""
+    matches = re.match("(?P<git_server>.*):(?P<git_repo>.*)", git_url)
+    if matches is None:
+        return (1, f"could not understand the git url {git_url} for authors detection")
+    git_server = matches.group("git_server")
+    git_repo = matches.group("git_repo")
+    if git_server is None:
+        return (
+            1,
+            f"could not understand the git server in {git_url} for authors detection",
+        )
+    if git_repo is None:
+        return (
+            1,
+            f"could not understand the git repo in {git_url} for authors detection",
+        )
+
+    if "git.yelpcorp.com" in git_server:
+        ssh_command = (
+            f"ssh {git_server} authors-of-changeset {git_repo} {from_sha} {to_sha}"
+        )
+        return _run(command=ssh_command, timeout=5.0)
+    else:
+        # TODO: PAASTA-16927: support getting authors for services on GHE
+        return 1, f"Fetching authors not supported for {git_server}"

paasta_tools/run-paasta-api-in-dev-mode.py

@@ -0,0 +1,57 @@
+import json
+import os
+from distutils.dir_util import copy_tree
+
+from paasta_tools.cli.utils import pick_random_port
+
+
+def main():
+    print("-------------------------------------------------------")
+    print(
+        "Please run export PAASTA_SYSTEM_CONFIG_DIR=etc_paasta_for_development to continue"
+    )
+    print(
+        "Please set environment variable PAASTA_TEST_CLUSTER to the cluster you want to use."
+    )
+    print("This is necessary for tron jobs")
+    print("-------------------------------------------------------")
+    cluster = os.environ.get("PAASTA_TEST_CLUSTER", "norcal-devc")
+    config_path = "etc_paasta_for_development"
+
+    copy_tree("/etc/paasta", os.path.join(os.getcwd(), config_path))
+    # Generate tron.json
+    tron_config = {"tron": {"url": f"http://tron-{cluster}:8089"}}
+    with open(config_path + "/tron.json", "w") as f:
+        json.dump(tron_config, f)
+    # find unused port
+    port = pick_random_port("paasta-dev-api")
+    # Generate api endpoints
+    api_endpoints = {"api_endpoints": {cluster: f"http://localhost:{port}"}}
+    api_endpoints_path = os.path.join(os.getcwd(), config_path, "api_endpoints.json")
+    os.chmod(api_endpoints_path, 0o777)
+    with open(api_endpoints_path, "w") as f:
+        json.dump(api_endpoints, f)
+
+    # export config path
+    os.environ["PAASTA_SYSTEM_CONFIG_DIR"] = config_path
+
+    api_single_process = os.environ.get("PAASTA_API_SINGLE_PROCESS")
+    if api_single_process is not None and api_single_process.lower() == "true":
+        from paasta_tools.api.api import redirect_argv
+
+        with redirect_argv(["-D", "-c", cluster, str(port)]):
+            from paasta_tools.api import api
+
+            api.main("dev-mode")
+    else:
+        os.execl(
+            ".tox/py38-linux/bin/python",
+            ".tox/py38-linux/bin/python",
+            "-m",
+            "paasta_tools.api.api",
+            *["-D", "-c", cluster, str(port)],
+        )
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/run-paasta-api-playground.py

@@ -0,0 +1,51 @@
+import json
+import os
+
+from paasta_tools.cli.utils import pick_random_port
+
+
+def main():
+    print("-------------------------------------------------------")
+    print(
+        "Please run export PAASTA_SYSTEM_CONFIG_DIR=etc_paasta_playground to continue"
+    )
+    print(
+        "Please set environment variable PAASTA_TEST_CLUSTER to the cluster you want to use."
+    )
+    print("-------------------------------------------------------")
+    user = os.environ["USER"]
+    cluster = os.environ.get("PAASTA_TEST_CLUSTER", f"kind-{user}-k8s-test")
+    config_path = "etc_paasta_playground"
+
+    # find unused ports
+    port = pick_random_port("paasta-dev-api")
+
+    # Generate api endpoints
+    api_endpoints = {"api_endpoints": {cluster: f"http://localhost:{port}"}}
+    api_endpoints_path = os.path.join(config_path, "api_endpoints.json")
+    with open(api_endpoints_path, "w") as f:
+        json.dump(api_endpoints, f)
+
+    # export config path
+    os.environ["PAASTA_SYSTEM_CONFIG_DIR"] = config_path
+
+    api_single_process = os.environ.get("PAASTA_API_SINGLE_PROCESS")
+    if api_single_process is not None and api_single_process.lower() == "true":
+        from paasta_tools.api.api import redirect_argv
+
+        with redirect_argv(["-D", "-c", cluster, str(port)]):
+            from paasta_tools.api import api
+
+            api.main("dev-mode")
+    else:
+        os.execl(
+            ".tox/py38-linux/bin/python",
+            ".tox/py38-linux/bin/python",
+            "-m",
+            "paasta_tools.api.api",
+            *["-D", "-c", cluster, str(port)],
+        )
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/secret_providers/__init__.py

@@ -0,0 +1,66 @@
+import os
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Mapping
+from typing import Optional
+
+from mypy_extensions import TypedDict
+from service_configuration_lib import read_service_configuration
+
+
+class BaseSecretProvider:
+    def __init__(
+        self,
+        soa_dir: Optional[str],
+        service_name: Optional[str],
+        cluster_names: List[str],
+        **kwargs: Any,
+    ) -> None:
+        self.soa_dir = soa_dir
+        self.cluster_names = cluster_names
+        self.service_name = service_name
+        if service_name:
+            self.secret_dir = os.path.join(self.soa_dir, self.service_name, "secrets")
+            service_config = read_service_configuration(self.service_name, self.soa_dir)
+            self.encryption_key = service_config.get("encryption_key", "paasta")
+
+    def decrypt_environment(
+        self, environment: Dict[str, str], **kwargs: Any
+    ) -> Dict[str, str]:
+        raise NotImplementedError
+
+    def write_secret(
+        self,
+        action: str,
+        secret_name: str,
+        plaintext: bytes,
+        cross_environment_motivation: Optional[str] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    def decrypt_secret(self, secret_name: str) -> str:
+        raise NotImplementedError
+
+    def decrypt_secret_raw(self, secret_name: str) -> bytes:
+        raise NotImplementedError
+
+    def get_secret_signature_from_data(self, data: Mapping[str, Any]) -> Optional[str]:
+        raise NotImplementedError
+
+    def get_data_from_vault_path(self, path: str) -> Optional[Dict[str, str]]:
+        raise NotImplementedError
+
+
+class CryptoKey(TypedDict):
+    key_name: str
+    key_version: str
+    key: str
+
+
+class SecretProvider(BaseSecretProvider):
+    def get_key_versions(self, key_name: str) -> List[CryptoKey]:
+        """
+        Dummy attribute to satisfy `mypy` because the class is imported dynamically via __import__
+        """
+        raise NotImplementedError

paasta_tools/secret_providers/vault.py

@@ -0,0 +1,214 @@
+import getpass
+import logging
+import os
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Mapping
+from typing import Optional
+
+from paasta_tools.secret_providers import CryptoKey
+
+try:
+    from vault_tools.client.jsonsecret import get_plaintext
+    from vault_tools.paasta_secret import get_vault_client
+    from vault_tools.gpg import TempGpgKeyring
+    from vault_tools.paasta_secret import encrypt_secret
+    import hvac
+except ImportError:
+
+    def get_plaintext(*args: Any, **kwargs: Any) -> bytes:
+        return b"No plain text available without vault_tools"
+
+    def get_vault_client(*args: Any, **kwargs: Any) -> None:
+        return None
+
+    TempGpgKeyring = None
+
+    def encrypt_secret(*args: Any, **kwargs: Any) -> None:
+        return None
+
+
+from paasta_tools.secret_providers import BaseSecretProvider
+from paasta_tools.secret_tools import get_secret_name_from_ref
+
+
+log = logging.getLogger(__name__)
+
+
+class SecretProvider(BaseSecretProvider):
+    def __init__(
+        self,
+        soa_dir: Optional[str],
+        service_name: Optional[str],
+        cluster_names: List[str],
+        vault_cluster_config: Dict[str, str] = {},
+        vault_auth_method: str = "ldap",
+        vault_token_file: str = "/root/.vault-token",
+        vault_num_uses: int = 1,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(soa_dir, service_name, cluster_names)
+        self.vault_cluster_config = vault_cluster_config
+        self.vault_auth_method = vault_auth_method
+        self.vault_token_file = vault_token_file
+        self.ecosystems = self.get_vault_ecosystems_for_clusters()
+        self.clients: Mapping[str, hvac.Client] = {}
+        if vault_auth_method == "ldap":
+            username = getpass.getuser()
+            password = getpass.getpass(
+                "Please enter your LDAP password to auth with Vault\n"
+            )
+        else:
+            username = None
+            password = None
+        for ecosystem in self.ecosystems:
+            self.clients[ecosystem] = get_vault_client(
+                ecosystem=ecosystem,
+                num_uses=vault_num_uses,
+                vault_auth_method=self.vault_auth_method,
+                vault_token_file=self.vault_token_file,
+                username=username,
+                password=password,
+            )
+
+    def decrypt_environment(
+        self, environment: Dict[str, str], **kwargs: Any
+    ) -> Dict[str, str]:
+        client = self.clients[self.ecosystems[0]]
+        secret_environment = {}
+        for k, v in environment.items():
+            secret_name = get_secret_name_from_ref(v)
+            secret_path = os.path.join(self.secret_dir, f"{secret_name}.json")
+            secret = get_plaintext(
+                client=client,
+                env=self.ecosystems[0],
+                path=secret_path,
+                cache_enabled=False,
+                cache_dir=None,
+                cache_key=None,
+                context=self.service_name,
+                rescue_failures=False,
+            ).decode("utf-8")
+            secret_environment[k] = secret
+        return secret_environment
+
+    def get_vault_ecosystems_for_clusters(self) -> List[str]:
+        try:
+            return list(
+                {
+                    self.vault_cluster_config[cluster_name]
+                    for cluster_name in self.cluster_names
+                }
+            )
+        except KeyError as e:
+            print(
+                "Cannot find a vault cluster for the %s paasta cluster. A mapping must exist "
+                "in /etc/paasta so we contact the correct vault cluster to get/set secrets"
+                % e
+            )
+            raise
+
+    def write_secret(
+        self,
+        action: str,
+        secret_name: str,
+        plaintext: bytes,
+        cross_environment_motivation: Optional[str] = None,
+    ) -> None:
+        with TempGpgKeyring(overwrite=True):
+            for ecosystem in self.ecosystems:
+                client = self.clients[ecosystem]
+                encrypt_secret(
+                    client=client,
+                    action=action,
+                    ecosystem=ecosystem,
+                    secret_name=secret_name,
+                    soa_dir=self.soa_dir,
+                    plaintext=plaintext,
+                    service_name=self.service_name,
+                    transit_key=self.encryption_key,
+                    cross_environment_motivation=cross_environment_motivation,
+                )
+
+    def decrypt_secret(self, secret_name: str) -> str:
+        return self.decrypt_secret_raw(secret_name).decode("utf-8")
+
+    def decrypt_secret_raw(self, secret_name: str) -> bytes:
+        client = self.clients[self.ecosystems[0]]
+        secret_path = os.path.join(self.secret_dir, f"{secret_name}.json")
+        return get_plaintext(
+            client=client,
+            path=secret_path,
+            env=self.ecosystems[0],
+            cache_enabled=False,
+            cache_key=None,
+            cache_dir=None,
+            context=self.service_name,
+            rescue_failures=False,
+        )
+
+    def get_secret_signature_from_data(self, data: Mapping[str, Any]) -> Optional[str]:
+        ecosystem = self.ecosystems[0]
+        if data["environments"].get(ecosystem):
+            return data["environments"][ecosystem]["signature"]
+        else:
+            return None
+
+    def get_data_from_vault_path(self, path: str) -> Optional[Dict[str, str]]:
+        # clients.read returns None if not set
+        # if it is set, it returns an object with { **metadata, data: {} }
+        # eg lease_id, request_id, etc. we only care about 'data' here
+
+        client = self.clients[self.ecosystems[0]]
+
+        # there is a chance client is None (ie if the connection is invalid)
+        if client is None:
+            raise Exception(
+                "possible Vault misconfiguration as client is not available"
+            )
+
+        entry = client.read(path)
+
+        # returns one of 3 things:
+        #   entry -> could be None
+        #   entry["data"] -> could be an object with content
+        #   entry["data"] -> could be empty (ie no secrets)
+        # all are plausible and valid scenarios that give different information about the path
+
+        if entry is not None:
+            return entry.get("data", {})
+        return None
+
+    def get_key_versions(
+        self,
+        key_name: str,
+    ) -> List[CryptoKey]:
+        """
+        Retrieve all versions of Vault key based on its metadata
+        """
+        client = self.clients[self.ecosystems[0]]
+        crypto_keys: List[CryptoKey] = []
+        try:
+            meta_response = client.secrets.kv.read_secret_metadata(
+                path=key_name, mount_point="keystore"
+            )
+
+            for key_version in meta_response["data"]["versions"].keys():
+                key_response = client.secrets.kv.read_secret_version(
+                    path=key_name, version=key_version, mount_point="keystore"
+                )
+                crypto_keys.append(
+                    {
+                        "key_name": key_name,
+                        "key_version": key_response["data"]["metadata"]["version"],
+                        "key": key_response["data"]["data"]["key"],
+                    }
+                )
+        except hvac.exceptions.VaultError:
+            log.warning(
+                f"Could not fetch key versions for {key_name} on {self.ecosystems[0]}"
+            )
+            pass
+
+        return crypto_keys