paasta-tools 1.21.3__py3-none-any.whl

paasta_tools-1.21.3.data/scripts/ide_helper.py

@@ -0,0 +1,316 @@
+import json
+import os
+from typing import Any
+from typing import Dict
+
+
+def merge_vscode_settings(file: str, setting_dict: Dict[str, Any]) -> None:
+    os.makedirs("./.vscode", exist_ok=True)
+
+    json_path = os.path.join("./.vscode", file)
+    # first checks if the file exists
+    # if it does then load the file content as json into settings
+    # if it doesn't exist then set settings to empty
+    if os.path.exists(json_path):
+        with open(json_path, mode="r") as settings_file:
+            settings = json.load(settings_file)
+    else:
+        settings = {}
+
+    # update settings with the default configurations we want each file to have
+    merge_dicts(setting_dict, settings)
+
+    with open(json_path, mode="w") as settings_file:
+        json.dump(settings, settings_file, indent=2)
+
+
+def merge_dicts(merge_from: Dict[str, Any], merge_to: Dict[str, Any]) -> None:
+    for key, value in merge_from.items():
+        if key in merge_to:
+            if value not in merge_to[key]:
+                for dict_entry in merge_from[key]:
+                    if dict_entry not in merge_to[key]:
+                        if isinstance(merge_to[key], list):
+                            merge_to[key].append(dict_entry)
+                        else:
+                            temp_list = [merge_to[key]]
+                            temp_list.append(dict_entry)
+                            merge_to[key] = temp_list
+        else:
+            merge_to[key] = value
+
+
+def install_vscode_support() -> None:
+    paasta_schema_settings = {
+        "version": "0.2.0",
+        "configurations": [
+            {
+                "name": "tox test",
+                "type": "python",
+                "request": "launch",
+                "cwd": "${workspaceFolder}",
+                "console": "integratedTerminal",
+                "python": "${workspaceFolder}/.paasta/bin/python",
+                "program": "${workspaceFolder}/.paasta/bin/tox",
+                "subProcess": True,
+                "args": ["-e", "py38-linux,docs,mypy,tests"],
+            },
+            {
+                "name": "paasta cli",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.cli.cli",
+            },
+            {
+                "name": "paasta rollback",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.cli.cli",
+                "args": [
+                    "rollback",
+                    "--service",
+                    "katamari_test_service",
+                    "--deploy-group",
+                    "dev.canary",
+                    "--commit",
+                    "fa7f2023c84736bd05201ef96ebd3c3fed6ab903",  # pragma: whitelist secret
+                ],
+            },
+            {
+                "name": "paasta mark-for-deployment",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.cli.cli",
+                "args": [
+                    "mark-for-deployment",
+                    "--service",
+                    "katamari_test_service",
+                    "--deploy-group",
+                    "dev.canary",
+                    "--commit",
+                    "224173aca322207994e655c4316ddb16f00eaab7",  # pragma: whitelist secret
+                    "--wait-for-deployment",
+                ],
+            },
+            {
+                "name": "paasta status",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.cli.cli",
+                "args": [
+                    "status",
+                    "--service",
+                    "katamari_test_service",
+                    "--clusters",
+                    "norcal-devc",
+                    "--instance",
+                    "canary",
+                ],
+            },
+            {
+                "name": "paasta playground",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.cli.cli",
+                "justMyCode": False,
+                "env": {"PAASTA_SYSTEM_CONFIG_DIR": "./etc_paasta_playground/"},
+                "args": [
+                    "status",
+                    "--service",
+                    "compute-infra-test-service",
+                    "--clusters",
+                    "kind-${env:USER}-k8s-test",
+                    "-d",
+                    "./soa_config_playground/",
+                ],
+                "preLaunchTask": "Run API Playground",
+            },
+            {
+                "name": "paasta status playground",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.cli.cli",
+                "justMyCode": False,
+                "env": {"PAASTA_SYSTEM_CONFIG_DIR": "./etc_paasta_playground/"},
+                "args": [
+                    "status",
+                    "--service",
+                    "compute-infra-test-service",
+                    "--clusters",
+                    "kind-${env:USER}-k8s-test",
+                    "-d",
+                    "./soa_config_playground/",
+                ],
+            },
+            {
+                "name": "paasta logs",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.cli.cli",
+                "args": [
+                    "logs",
+                    "--service",
+                    "katamari_test_service",
+                    "--cluster",
+                    "norcal-devc",
+                    "--instance",
+                    "canary",
+                ],
+            },
+            {
+                "name": "paasta validate",
+                # This command has to be ran from inside the service repo in yelpsoa-configs
+                "cwd": "${userHome}/pg/yelpsoa-configs/",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.cli.cli",
+                "args": ["validate", "--service", "katamari_test_service"],
+            },
+            {
+                # 1) Follow step 1 in "Running the PaaSTA HTTP API Locally" wiki
+                # 2) Run this "paasta API" test to debug paasta API
+                # 3) Run client command, e.g. PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_for_development/  .tox/py38-linux/bin/python paasta_tools/cli/cli.py status --clusters norcal-devc --service katamari_test_service
+                "name": "paasta API",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.run-paasta-api-in-dev-mode",
+                "env": {
+                    "KUBECONFIG": "./etc_paasta_for_development/admin.conf",
+                    "PAASTA_SYSTEM_CONFIG_DIR": "./etc_paasta_for_development/",
+                    "PAASTA_TEST_CLUSTER": "norcal-devc",
+                    "PYDEVD_USE_CYTHON": "NO",
+                    "PYTHONUNBUFFERED": "1",
+                    "PAASTA_API_SINGLE_PROCESS": "true",
+                },
+            },
+            {
+                "name": "paasta API playground",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.run-paasta-api-playground",
+                "justMyCode": False,
+                "env": {
+                    "KUBECONFIG": "./k8s_itests/kubeconfig",
+                    "PAASTA_SYSTEM_CONFIG_DIR": "./etc_paasta_playground/",
+                    "PAASTA_TEST_CLUSTER": "kind-${env:USER}-k8s-test",
+                    "PYDEVD_USE_CYTHON": "NO",
+                    "PYTHONUNBUFFERED": "1",
+                    "PAASTA_API_SINGLE_PROCESS": "true",
+                    "PAASTA_API_SOA_DIR": "./soa_config_playground/",
+                },
+            },
+            {
+                "name": "Run setup k8s job in playground",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.setup_kubernetes_job",
+                "justMyCode": False,
+                "args": [
+                    "-d",
+                    "./soa_config_playground",
+                    "-c",
+                    "kind-${env:USER}-k8s-test",
+                    "compute-infra-test-service.autoscaling",
+                ],
+                "env": {
+                    "KUBECONFIG": "./k8s_itests/kubeconfig",
+                    "PAASTA_SYSTEM_CONFIG_DIR": "./etc_paasta_playground/",
+                    "PAASTA_TEST_CLUSTER": "kind-${env:USER}-k8s-test",
+                    "PYDEVD_USE_CYTHON": "NO",
+                    "PYTHONUNBUFFERED": "1",
+                },
+            },
+            {
+                "name": "Generate deployments.json in playground",
+                "cwd": "${workspaceFolder}",
+                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "type": "python",
+                "request": "launch",
+                "module": "paasta_tools.generate_deployments_for_service",
+                "justMyCode": False,
+                "args": [
+                    "-d",
+                    "./soa_config_playground",
+                    "-s",
+                    "compute-infra-test-service",
+                    "-v",
+                ],
+                "env": {
+                    "PAASTA_SYSTEM_CONFIG_DIR": "./etc_paasta_playground/",
+                    "PYDEVD_USE_CYTHON": "NO",
+                    "PYTHONUNBUFFERED": "1",
+                },
+            },
+        ],
+    }
+    merge_vscode_settings("launch.json", paasta_schema_settings)
+
+    recommended_extensions = {
+        "recommendations": [
+            "redhat.vscode-yaml",
+            "ms-python.python",
+            "ms-python.vscode-pylance",
+            "ms-vscode.makefile-tools",
+        ]
+    }
+    merge_vscode_settings("extensions.json", recommended_extensions)
+
+    playground_task = {
+        "version": "2.0.0",
+        "tasks": [
+            {
+                "label": "Run API Playground",
+                "type": "shell",
+                "isBackground": True,
+                "command": "make playground-api",
+                "presentation": {
+                    "reveal": "always",
+                    "panel": "new",
+                },
+                "problemMatcher": [
+                    {
+                        "pattern": [
+                            {
+                                "regexp": ".",
+                                "file": 1,
+                                "location": 2,
+                                "message": 3,
+                            }
+                        ],
+                        "background": {
+                            "activeOnStart": True,
+                            "beginsPattern": ".",
+                            "endsPattern": "^(.*?)\\[INFO\\]",
+                        },
+                    }
+                ],
+            }
+        ],
+    }
+    merge_vscode_settings("tasks.json", playground_task)
+    print("VS Code IDE Helpers installed")
+
+
+if __name__ == "__main__":
+    install_vscode_support()

paasta_tools-1.21.3.data/scripts/is_pod_healthy_in_proxy.py

@@ -0,0 +1,139 @@
+#!python
+# Copyright 2015-2018 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import os
+import sys
+
+from paasta_tools.envoy_tools import are_namespaces_up_in_eds
+from paasta_tools.envoy_tools import are_services_up_in_pod as is_envoy_ready
+from paasta_tools.smartstack_tools import (
+    are_services_up_on_ip_port as is_smartstack_ready,
+)
+from paasta_tools.utils import load_system_paasta_config
+
+
+system_paasta_config = load_system_paasta_config()
+
+synapse_port = system_paasta_config.get_synapse_port()
+synapse_host = "169.254.255.254"
+synapse_haproxy_url_format = system_paasta_config.get_synapse_haproxy_url_format()
+
+envoy_host = os.environ["PAASTA_HOST"]
+envoy_admin_port = system_paasta_config.get_envoy_admin_port()
+envoy_admin_endpoint_format = system_paasta_config.get_envoy_admin_endpoint_format()
+envoy_eds_path = "/nail/etc/envoy/endpoints"
+pod_ip = os.environ["PAASTA_POD_IP"]
+
+###############################################################
+#
+# This file is used in the hacheck sidecar, make sure to update `check_proxy_up.sh`
+# when changing this file
+#
+###############################################################
+
+
+def get_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--enable-smartstack",
+        dest="smartstack_readiness_check_enabled",
+        action="store_true",
+        help="Check smartstack readiness",
+    )
+
+    parser.add_argument(
+        "--enable-envoy",
+        action="store_true",
+        dest="envoy_readiness_check_enabled",
+        help="Check envoy readiness",
+    )
+
+    parser.add_argument(
+        "--envoy-check-mode",
+        choices=["admin-port", "eds-dir"],
+        default="admin-port",
+        help="Query Envoy backends through the admin interface (default) or the EDS directory",
+    )
+
+    parser.add_argument(
+        "pod_port",
+        help="Pod Port",
+        type=int,
+    )
+
+    parser.add_argument(
+        "services",
+        nargs="+",
+        help="List of service.instance names",
+    )
+
+    return parser
+
+
+def main() -> None:
+    args = get_parser().parse_args()
+
+    if args.smartstack_readiness_check_enabled:
+        smartstack_ready = is_smartstack_ready(
+            synapse_host=synapse_host,
+            synapse_port=synapse_port,
+            synapse_haproxy_url_format=synapse_haproxy_url_format,
+            services=args.services,
+            host_ip=pod_ip,
+            host_port=args.pod_port,
+        )
+    else:
+        smartstack_ready = True
+
+    if args.envoy_readiness_check_enabled:
+        if args.envoy_check_mode == "admin-port":
+            envoy_ready = is_envoy_ready(
+                envoy_host=envoy_host,
+                envoy_admin_port=envoy_admin_port,
+                envoy_admin_endpoint_format=envoy_admin_endpoint_format,
+                registrations=args.services,
+                pod_ip=pod_ip,
+                pod_port=args.pod_port,
+            )
+        elif args.envoy_check_mode == "eds-dir":
+            envoy_ready = are_namespaces_up_in_eds(
+                envoy_eds_path=envoy_eds_path,
+                namespaces=args.services,
+                pod_ip=pod_ip,
+                pod_port=args.pod_port,
+            )
+
+    else:
+        envoy_ready = True
+
+    if smartstack_ready and envoy_ready:
+        sys.exit(0)
+    else:
+        if not smartstack_ready:
+            print(
+                f"Could not find backend {pod_ip}:{args.pod_port} for service {args.services} "
+                f"on Haproxy at {synapse_host}:{synapse_port}"
+            )
+        if not envoy_ready:
+            print(
+                f"Could not find backend {pod_ip}:{args.pod_port} for service {args.services} "
+                f"on Envoy at {envoy_host}"
+            )
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools-1.21.3.data/scripts/is_pod_healthy_in_smartstack.py

@@ -0,0 +1,50 @@
+#!python
+# Copyright 2015-2018 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import sys
+
+from paasta_tools.smartstack_tools import are_services_up_on_ip_port
+from paasta_tools.utils import load_system_paasta_config
+
+system_paasta_config = load_system_paasta_config()
+synapse_port = system_paasta_config.get_synapse_port()
+synapse_host = "169.254.255.254"
+synapse_haproxy_url_format = system_paasta_config.get_synapse_haproxy_url_format()
+host_ip = os.environ["PAASTA_POD_IP"]
+port = sys.argv[1]
+services = sys.argv[2:]
+
+###############################################################
+#
+# This file is used in the hacheck sidecar, make sure to update `check_smartstack_up.sh`
+# when changing this file
+#
+###############################################################
+
+if are_services_up_on_ip_port(
+    synapse_host=synapse_host,
+    synapse_port=synapse_port,
+    synapse_haproxy_url_format=synapse_haproxy_url_format,
+    services=services,
+    host_ip=host_ip,
+    host_port=int(port),
+):
+    sys.exit(0)
+else:
+    print(
+        f"Could not find backend {host_ip}:{port} for service {services} "
+        f"on Synapse at {synapse_host}:{synapse_port}"
+    )
+    sys.exit(1)

paasta_tools-1.21.3.data/scripts/kill_bad_containers.py

@@ -0,0 +1,109 @@
+#!python
+import sys
+
+try:
+    import iptc
+except TypeError:
+    print(
+        "Failed to import iptc. This happens sometimes during a python upgrade or during bootstrapping"
+    )
+    sys.exit(0)
+
+from paasta_tools import iptables
+from paasta_tools.utils import get_docker_client
+
+
+def get_container_from_dport(dport, docker_client):
+    for container in docker_client.containers():
+        try:
+            ports = container["Ports"]
+            for port in ports:
+                if "PublicPort" in port:
+                    if port["PublicPort"] == int(dport):
+                        return container
+        except KeyError:
+            print(ports)
+            pass
+
+
+def target_rule_to_dport(rule):
+    try:
+        # (('tcp', (('dport', ('31493',)),)),)
+        return rule.matches[0][1][0][1][0]
+    except IndexError:
+        return None
+
+
+def kill_containers_with_duplicate_iptables_rules(docker_client):
+    chain_name = "DOCKER"
+    table = iptc.Table(iptc.Table.NAT)
+    chain = iptc.Chain(table, chain_name)
+
+    targets_seen = {}
+    raw_rules_seen = {}
+
+    for iptables_rule in chain.rules:
+        rule = iptables.Rule.from_iptc(iptables_rule)
+        target = rule.target_parameters
+        if target not in targets_seen:
+            targets_seen[target] = rule
+            raw_rules_seen[target] = iptables_rule
+        else:
+            dport = target_rule_to_dport(rule)
+            if dport is None:
+                continue
+            print(
+                "This is the second time we've seen a rule with the same target_parameters!"
+            )
+            print(rule)
+            container1 = get_container_from_dport(dport, docker_client)
+            print("The other rule with that target is:")
+            print(targets_seen[target])
+            dport2 = target_rule_to_dport(targets_seen[target])
+            container2 = get_container_from_dport(dport2, docker_client)
+            if container1 is None:
+                print(
+                    "We have a duplicate iptables rule going to a container1, but no container1!"
+                )
+                print(rule)
+                print("Deleting this rule")
+                chain.delete_rule(iptables_rule)
+            elif container2 is None:
+                print(
+                    "We have a iptables rule going to a container2, but no container2!"
+                )
+                print(targets_seen[target])
+                print("Deleting this rule")
+                chain.delete_rule(raw_rules_seen[target])
+            elif container1["Id"] == container2["Id"]:
+                print("The same container is getting traffic for both ports!")
+                print(container1)
+                print("Killing the container")
+                docker_client.kill(container1["Id"])
+                print("Deleting both iptables rules")
+                chain.delete_rule(iptables_rule)
+                chain.delete_rule(raw_rules_seen[target])
+            elif container1["Id"] != container2["Id"]:
+                print(
+                    "These are two different containers, which means we have duplicate ips:"
+                )
+                print(container1)
+                print(container2)
+                print("Not sure which to kill, killing both")
+                docker_client.kill(container1["Id"])
+                docker_client.kill(container2["Id"])
+                print("Deleting the both iptables rules for good measure")
+                chain.delete_rule(iptables_rule)
+                chain.delete_rule(raw_rules_seen[target])
+            else:
+                print("Something unexpected went wrong. Exiting 1")
+                sys.exit(1)
+
+
+def main():
+    docker_client = get_docker_client()
+    kill_containers_with_duplicate_iptables_rules(docker_client)
+
+
+if __name__ == "__main__":
+    sys.exit(main())