paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/cli/cmds/wait_for_deployment.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains methods used by the paasta client to wait for deployment
+of a docker image to a cluster.instance.
+"""
+import asyncio
+import logging
+from typing import Optional
+
+from paasta_tools.cli.cmds.mark_for_deployment import NoSuchCluster
+from paasta_tools.cli.cmds.mark_for_deployment import report_waiting_aborted
+from paasta_tools.cli.cmds.mark_for_deployment import wait_for_deployment
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.cli.utils import list_deploy_groups
+from paasta_tools.cli.utils import NoSuchService
+from paasta_tools.cli.utils import validate_git_sha
+from paasta_tools.cli.utils import validate_given_deploy_groups
+from paasta_tools.cli.utils import validate_service_name
+from paasta_tools.cli.utils import validate_short_git_sha
+from paasta_tools.remote_git import list_remote_refs
+from paasta_tools.remote_git import LSRemoteException
+from paasta_tools.utils import _log
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DeploymentVersion
+from paasta_tools.utils import get_git_url
+from paasta_tools.utils import get_latest_deployment_tag
+from paasta_tools.utils import list_services
+from paasta_tools.utils import PaastaColors
+from paasta_tools.utils import TimeoutError
+
+DEFAULT_DEPLOYMENT_TIMEOUT = 3600  # seconds
+
+
+log = logging.getLogger(__name__)
+
+
+class VersionError(Exception):
+    pass
+
+
+class DeployGroupError(Exception):
+    pass
+
+
+def add_subparser(subparsers):
+    list_parser = subparsers.add_parser(
+        "wait-for-deployment",
+        help="Wait a service to be deployed to deploy_group",
+        description=(
+            "'paasta wait-for-deployment' waits for a previously marked for "
+            "deployment service to be deployed to deploy_group."
+        ),
+        epilog=(
+            "Note: Access and credentials to the Git repo of a service "
+            "are required for this command to work."
+        ),
+    )
+    list_parser.add_argument(
+        "-u",
+        "--git-url",
+        help=(
+            "Git url for service. Defaults to the normal git URL for " "the service."
+        ),
+        default=None,
+    )
+    list_parser.add_argument(
+        "-c",
+        "-k",
+        "--commit",
+        help="Git sha to wait for deployment",
+        required=True,
+        type=validate_short_git_sha,
+    )
+    list_parser.add_argument(
+        "-i",
+        "--image-version",
+        help="Extra version metadata to mark for deployment",
+        required=False,
+        default=None,
+    )
+    list_parser.add_argument(
+        "-l",
+        "--deploy-group",
+        help="deploy group (e.g. cluster1.canary, cluster2.main).",
+        required=True,
+    ).completer = lazy_choices_completer(list_deploy_groups)
+    list_parser.add_argument(
+        "-s",
+        "--service",
+        help="Name of the service which you wish to wait for deployment. "
+        'Leading "services-" will be stripped.',
+        required=True,
+    ).completer = lazy_choices_completer(list_services)
+    list_parser.add_argument(
+        "-t",
+        "--timeout",
+        dest="timeout",
+        type=int,
+        default=DEFAULT_DEPLOYMENT_TIMEOUT,
+        help=(
+            "Time in seconds to wait for paasta to deploy the service. "
+            "If the timeout is exceeded we return 1. "
+            "Default is %(default)s seconds."
+        ),
+    )
+    list_parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    list_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        dest="verbose",
+        default=0,
+        help="Print out more output.",
+    )
+    list_parser.add_argument(
+        "--polling-interval",
+        dest="polling_interval",
+        type=float,
+        default=None,
+        help="How long to wait between each time we check to see if an instance is done deploying.",
+    )
+    list_parser.add_argument(
+        "--diagnosis-interval",
+        dest="diagnosis_interval",
+        type=float,
+        default=None,
+        help="How long to wait between diagnoses of why the bounce isn't done.",
+    )
+    list_parser.add_argument(
+        "--time-before-first-diagnosis",
+        dest="time_before_first_diagnosis",
+        type=float,
+        default=None,
+        help="Wait this long before trying to diagnose why the bounce isn't done.",
+    )
+
+    list_parser.set_defaults(command=paasta_wait_for_deployment)
+
+
+def get_latest_marked_version(
+    git_url: str, deploy_group: str
+) -> Optional[DeploymentVersion]:
+    """Return the latest marked for deployment version or None"""
+    # TODO: correct this function for new tag format
+    refs = list_remote_refs(git_url)
+    _, sha, image_version = get_latest_deployment_tag(refs, deploy_group)
+    if sha:
+        return DeploymentVersion(sha=sha, image_version=image_version)
+    # We did not find a ref for this deploy group
+    return None
+
+
+def validate_version_is_latest(
+    version: DeploymentVersion, git_url: str, deploy_group: str, service: str
+):
+    """Verify if the requested version  is the latest marked for deployment.
+
+    Raise exception when the provided version is not the latest
+    marked for deployment in 'deploy_group' for 'service'.
+    """
+    try:
+        marked_version = get_latest_marked_version(git_url, deploy_group)
+    except LSRemoteException as e:
+        print(
+            "Error talking to the git server: {}\n"
+            "It is not possible to verify that {} is marked for deployment in {}, "
+            "but I assume that it is marked and will continue waiting..".format(
+                e, version, deploy_group
+            )
+        )
+        return
+    if marked_version is None:
+        raise VersionError(
+            "ERROR: Nothing is marked for deployment "
+            "in {} for {}".format(deploy_group, service)
+        )
+    if version != marked_version:
+        raise VersionError(
+            "ERROR: The latest version marked for "
+            "deployment in {} is {}".format(deploy_group, marked_version)
+        )
+
+
+def validate_deploy_group(deploy_group: str, service: str, soa_dir: str):
+    """Validate deploy_group.
+
+    Raise exception if the specified deploy group is not used anywhere.
+    """
+    in_use_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
+    _, invalid_deploy_groups = validate_given_deploy_groups(
+        in_use_deploy_groups, [deploy_group]
+    )
+
+    if len(invalid_deploy_groups) == 1:
+        raise DeployGroupError(
+            "ERROR: These deploy groups are not currently "
+            "used anywhere: {}.\n"
+            "You probably need one of these in-use deploy "
+            "groups?:\n   {}".format(
+                ",".join(invalid_deploy_groups), ",".join(in_use_deploy_groups)
+            )
+        )
+
+
+def paasta_wait_for_deployment(args):
+    """Wrapping wait_for_deployment"""
+    if args.verbose:
+        log.setLevel(level=logging.DEBUG)
+    else:
+        log.setLevel(level=logging.INFO)
+
+    service = args.service
+    if service and service.startswith("services-"):
+        service = service.split("services-", 1)[1]
+
+    if args.git_url is None:
+        args.git_url = get_git_url(service=service, soa_dir=args.soa_dir)
+
+    args.commit = validate_git_sha(sha=args.commit, git_url=args.git_url)
+
+    version = DeploymentVersion(sha=args.commit, image_version=args.image_version)
+
+    try:
+        validate_service_name(service, soa_dir=args.soa_dir)
+        validate_deploy_group(args.deploy_group, service, args.soa_dir)
+        validate_version_is_latest(version, args.git_url, args.deploy_group, service)
+    except (VersionError, DeployGroupError, NoSuchService) as e:
+        print(PaastaColors.red(f"{e}"))
+        return 1
+
+    try:
+        asyncio.run(
+            wait_for_deployment(
+                service=service,
+                deploy_group=args.deploy_group,
+                git_sha=args.commit,
+                image_version=args.image_version,
+                soa_dir=args.soa_dir,
+                timeout=args.timeout,
+                polling_interval=args.polling_interval,
+                diagnosis_interval=args.diagnosis_interval,
+                time_before_first_diagnosis=args.time_before_first_diagnosis,
+            )
+        )
+        _log(
+            service=service,
+            component="deploy",
+            line=(f"Deployment of {version} for {args.deploy_group} complete"),
+            level="event",
+        )
+
+    except (KeyboardInterrupt, TimeoutError, NoSuchCluster):
+        report_waiting_aborted(service, args.deploy_group)
+        return 1
+
+    return 0

paasta_tools/cli/fsm/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

paasta_tools/cli/fsm/autosuggest.py

@@ -0,0 +1,82 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import random
+
+from paasta_tools import yaml_tools as yaml
+
+
+def _get_smartstack_proxy_ports_from_file(root, file):
+    """Given a root and file (as from os.walk), attempt to return the highest
+    smartstack proxy port number (int) from that file. Returns 0 if there is no
+    smartstack proxy_port.
+    """
+    ports = set()
+    with open(os.path.join(root, file)) as f:
+        data = yaml.safe_load(f)
+
+    if file.endswith("service.yaml") and "smartstack" in data:
+        # Specifying this in service.yaml is old and deprecated and doesn't
+        # support multiple namespaces.
+        ports = {int(data["smartstack"].get("proxy_port", 0))}
+    elif file.endswith("smartstack.yaml"):
+        for namespace in data.keys():
+            ports.add(data[namespace].get("proxy_port", 0))
+
+    return ports
+
+
+def read_etc_services():
+    with open("/etc/services") as fd:
+        return fd.readlines()
+
+
+def get_inuse_ports_from_etc_services():
+    ports = set()
+    for line in read_etc_services():
+        if line.startswith("#"):
+            continue
+        try:
+            p = line.split()[1]
+            port = int(p.split("/")[0])
+            ports.add(port)
+        except Exception:
+            pass
+    return ports
+
+
+def suggest_smartstack_proxy_port(
+    yelpsoa_config_root, range_min=19000, range_max=21000
+):
+    """Pick a random available port in the 19000-21000 block"""
+    available_proxy_ports = set(range(range_min, range_max + 1))
+    for root, dirs, files in os.walk(yelpsoa_config_root):
+        for f in files:
+            if f.endswith("smartstack.yaml"):
+                try:
+                    used_ports = _get_smartstack_proxy_ports_from_file(root, f)
+                    for used_port in used_ports:
+                        available_proxy_ports.discard(used_port)
+                except Exception:
+                    pass
+    available_proxy_ports.difference_update(get_inuse_ports_from_etc_services())
+    try:
+        return random.choice(list(available_proxy_ports))
+    except IndexError:
+        raise Exception(
+            f"There are no more ports available in the range [{range_min}, {range_max}]"
+        )
+
+
+# vim: expandtab tabstop=4 sts=4 shiftwidth=4:

paasta_tools/cli/fsm/template/README.md

@@ -0,0 +1,8 @@
+== PaaSTA FSM Template
+
+This is a [cookiecutter](http://cookiecutter.readthedocs.io/en/latest/index.html) style
+template to make it easier for you to create new services in PaaSTA.
+
+Use the `paasta fsm` tool to read this template.
+
+Only a very basic template is provided here.

paasta_tools/cli/fsm/template/cookiecutter.json

@@ -0,0 +1,7 @@
+{
+    "service": "my-cool-service",
+    "proxy_port": 20000,
+    "team": "noop",
+    "description": "Description",
+    "external_link": "Url"
+}

paasta_tools/cli/fsm/template/{{cookiecutter.service}}/kubernetes-PROD.yaml

@@ -0,0 +1,91 @@
+---
+# Kubernetes is the PaaSTA component that takes a command and keeps copies of it running.
+# You can see all the options for this file here:
+# http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html#kubernetes-clustername-yaml
+
+# In PaaSTA you can have different "instances" of the service that can run in different
+# modes. Out of Yelp convention, the "main" instance is the primary one and serves main
+# web traffic.
+main:
+  # In Prod, it is important to have enough capacity to serve our users. Having 3 copies
+  # of the service is a good start. Longer term the service might need more. Follow up
+  # and check out the autoscaling documentation:
+  # http://paasta.readthedocs.org/en/latest/autoscaling.html
+  # to learn how to have the "right" number of instances all the time.
+  instances: 3
+
+  deploy_group: everything
+
+  # cpus is a soft limit, and services can burst to use extra cpus on the servers
+  # if they are available. `paasta status` can give a snapshot of how much
+  # cpu a service is using or the signalfx graphs can give you a view of the cpu
+  # over time. Be honest about your cpu needs to keep the scheduler informed
+  # and neighbors happy.
+  cpus: .1
+  #
+  # Mem is hard limit and at Yelp we don't allow swap. The service will OOM and die
+  # if it goes over the limit. Luckily Kubernetes will restart the service ASAP if that
+  # happens. If it happens too much you will lose replication and get an alert. Again
+  # use the Signalfx graphs to keep an eye on it and give yourself headroom.
+  mem: 1000
+
+  # All the monitoring options in monitoring.yaml can be set on a per-instance basis too.
+  # You can even change the team responsible! Uncomment as necessary.
+  #monitoring:
+  #  page: true
+
+
+# A "canary" instance is useful to most people. The difference here is that it is in different
+# deploy_group that is deployed "earlier" in the jenkins pipeline. In real life you could make
+# the canary do whatever you want. It could have a special `cmd`, or run with different amounts
+# of ram/cpu, read a different config file, etc.
+canary:
+  instances: 1
+  deploy_group: canary
+  cpus: .1
+  mem: 1000
+  # This nerve_ns line configures the instance to "show up" in the "main" pool in SmartStack.
+  # This means that live traffic hitting the "main" pool will hit the canary. Most of the time
+  # this is what services want, but you should be aware of how this "works".
+  #
+  # For example, if you had 3 instances for main, and 1 instance for this canary, if you look
+  # at a HAProxy dashboard for Smartstack, you would see 4 backends!
+  nerve_ns: main
+
+  # All the monitoring options in monitoring.yaml can be set on a per-instance basis too.
+  # You can even change the team responsible! Uncomment as necessary.
+  #monitoring:
+  #  page: false
+  #  ticket: true
+  #  slack_channels: ['mychannel']
+
+
+# Some service authors want a *different* canary that is not in the main smartstack pool
+# that they can hit directly. You can totally do this, but you have to uncomment such
+# an instance below, and also add the new smartstack.yaml section to correspond with it.
+# non_traffic_receiving_canary:
+#  instances: 1
+#  deploy_group: prod.canary
+#  cpus: .1
+#  mem: 2000
+
+
+# Running "workers" in paasta is easy. There isn't really anything special about them except
+# they probably have a special `cmd`. Of course you could change the deploy group if you wanted
+# a canary worker. You can change instances, autoscale, etc.
+#worker:
+#  instances: 1
+#  deploy_group: prod.non_canary
+#  cpus: .1
+#  mem: 1000
+#  cmd: /code/virtualenv_run/bin/python /code/my_worker.py
+# # There are also more options for specifying command healthchecks
+# # if you need a special healthcheck command, but by default
+# # Mesos will watch the process (PID) and consider it healthy if
+# # it exists.
+
+
+# Worried about lots of duplication? You can use the YAML Anchor pattern to help with that:
+# https://en.wikipedia.org/wiki/YAML#Repeated_nodes
+# It is an advanced topic. If you do try it out, be sure to ask #paasta or add the paasta
+# review board group to the review.

paasta_tools/cli/fsm/template/{{cookiecutter.service}}/monitoring.yaml

@@ -0,0 +1,20 @@
+---
+# All of these options are documented here:
+# http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html#monitoring-yaml
+
+team: {{cookiecutter.team}}
+page: true
+# If you add a runbook here, it will get included in all the
+# sensu alerts that are generated for this service
+#runbook: http://y/rb-{{cookiecutter.service}}
+
+# Alert after is a kind of grace period before alerts are sent.
+# Usually the defaults are fine, but you can up this if you need
+# to.
+#alert_after: 2m
+
+# realert_every represents how often to resend an email or irc.
+# By default it does exponential backoff, which can be a little
+# noisy at first but then will quiet down. You can override this
+# if you want it to be consistently "noisy":
+#realert_every: 3600

paasta_tools/cli/fsm/template/{{cookiecutter.service}}/service.yaml

@@ -0,0 +1,8 @@
+---
+# This shows up in `paasta info` and is helpful
+# for people who have never heard of this service before. What does it do?
+description: {{cookiecutter.description}}
+
+# This is designed to help answer the question of
+# "Why did we make this service in the first place?"
+external_link: {{cookiecutter.external_link}}

paasta_tools/cli/fsm/template/{{cookiecutter.service}}/smartstack.yaml

@@ -0,0 +1,6 @@
+---
+# See y/smartstack-config
+main:
+  proxy_port: {{cookiecutter.proxy_port}}
+  advertise: [habitat, region]
+  discover: habitat

paasta_tools/cli/fsm_cmd.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import contextlib
+import shutil
+import sys
+
+from cookiecutter.main import cookiecutter
+
+from paasta_tools.cli.fsm.autosuggest import suggest_smartstack_proxy_port
+from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import PaastaColors
+
+
+@contextlib.contextmanager
+def make_copyfile_symlink_aware():
+    """The reasoning behind this monkeypatch is that cookiecutter doesn't
+    respect symlinks at all, and at Yelp we use symlinks to reduce duplication
+    in the soa configs. Maybe cookie-cutter will accept a symlink-aware PR?
+    """
+    orig_copyfile = shutil.copyfile
+    orig_copymode = shutil.copymode
+
+    def symlink_aware_copyfile(*args, **kwargs):
+        kwargs.setdefault("follow_symlinks", False)
+        orig_copyfile(*args, **kwargs)
+
+    def symlink_aware_copymode(*args, **kwargs):
+        kwargs.setdefault("follow_symlinks", False)
+        orig_copymode(*args, **kwargs)
+
+    shutil.copyfile = symlink_aware_copyfile
+    shutil.copymode = symlink_aware_copymode
+    try:
+        yield
+    finally:
+        shutil.copyfile = orig_copyfile
+        shutil.copymode = orig_copymode
+
+
+def parse_args():
+    fsm_parser = argparse.ArgumentParser(
+        "fsm",
+        description=(
+            "'paasta fsm' is used to generate example soa-configs, which is useful during initial "
+            "service creation. Currently 'fsm' generates 'yelp-specific' configuration, but can still "
+            "be used as an example of a fully working PaaSTA service.\n\n"
+            "After 'paasta fsm' is run, the operator should inspect the generated boilerplate configuration "
+            "and adjust it to meet the particular needs of the new service."
+        ),
+    )
+    fsm_parser.add_argument(
+        "-y",
+        "--yelpsoa-config-root",
+        dest="yelpsoa_config_root",
+        default=".",
+        help=(
+            "Path to root of yelpsoa-configs checkout\n"
+            "Defaults to current working directory"
+        ),
+    )
+    fsm_parser.set_defaults(command=paasta_fsm)
+    return fsm_parser.parse_args()
+
+
+def get_paasta_config(yelpsoa_config_root):
+    variables = {"proxy_port": suggest_smartstack_proxy_port(yelpsoa_config_root)}
+    return variables
+
+
+def write_paasta_config(variables, template, destination):
+    print("Using cookiecutter template from %s" % template)
+    with make_copyfile_symlink_aware():
+        cookiecutter(
+            template=template,
+            extra_context=variables,
+            output_dir=destination,
+            overwrite_if_exists=True,
+            no_input=not sys.stdout.isatty(),
+        )
+
+
+def paasta_fsm(args):
+    variables = get_paasta_config(yelpsoa_config_root=args.yelpsoa_config_root)
+    destination = args.yelpsoa_config_root
+
+    paasta_config = load_system_paasta_config()
+    template = paasta_config.get_fsm_template()
+
+    write_paasta_config(variables=variables, template=template, destination=destination)
+
+    print(PaastaColors.yellow("               _  _(o)_(o)_  _"))
+    print(PaastaColors.red(r"             ._\`:_ F S M _:' \_,"))
+    print(PaastaColors.green(r"                 / (`---'\ `-."))
+    print(PaastaColors.cyan("              ,-`  _)    (_,"))
+    print("With My Noodly Appendage I Have Written Configs!")
+    print()
+    print("Customize Them If It Makes You Happy -- http://y/paasta For Details")
+    print("Remember To Add, Commit, And Push When You're Done:")
+    print()
+
+
+def main():
+    args = parse_args()
+    paasta_fsm(args)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/cli/paasta_tabcomplete.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -n ${ZSH_VERSION-} ]]; then
+	autoload -U +X bashcompinit && bashcompinit
+fi
+
+# This magic eval enables tab-completion for the "paasta" command
+# http://argcomplete.readthedocs.io/en/latest/index.html#synopsis
+# This comes from the paasta-tools system package
+eval "$(/opt/venvs/paasta-tools/bin/register-python-argcomplete paasta)"