paasta-tools 1.0.5__tar.gz → 1.1.0__tar.gz

{paasta-tools-1.0.5/paasta_tools.egg-info → paasta-tools-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.0.5
+Version: 1.1.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.0.5 → paasta-tools-1.1.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.0.5"
+__version__ = "1.1.0"

paasta-tools-1.1.0/paasta_tools/generate_authenticating_services.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+A simple script to enumerate all services partecipating in authenticated
+communications, and list them in a YAML/JSON file.
+"""
+import argparse
+import glob
+import logging
+import os
+from typing import Dict
+from typing import List
+from typing import Set
+
+import yaml
+
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import write_json_configuration_file
+from paasta_tools.utils import write_yaml_configuration_file
+
+
+AUTHORIZATION_CONFIG_FILE = "authorization.yaml"
+
+
+def list_services_in_authz_config(config_path: str) -> Set[str]:
+    auth_config: dict = {}
+    try:
+        with open(config_path) as f:
+            auth_config = yaml.safe_load(f)
+    except Exception as e:
+        logging.warning(f"Issue loading {config_path}: {e}")
+    return {
+        service
+        for rule in auth_config.get("authorization", {}).get("rules", [])
+        for service in rule.get("identity_groups", {}).get("services", [])
+    }
+
+
+def enumerate_authenticating_services() -> Dict[str, List[str]]:
+    result = set()
+    config_path_pattern = os.path.join(DEFAULT_SOA_DIR, "*", AUTHORIZATION_CONFIG_FILE)
+    for authz_config in glob.glob(config_path_pattern):
+        result.update(list_services_in_authz_config(authz_config))
+    return {"services": sorted(result)}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Output filepath",
+        dest="output_filename",
+        required=True,
+    )
+    parser.add_argument(
+        "-f",
+        "--format",
+        help="Output format. Defaults to %(default)s",
+        dest="output_format",
+        choices=["yaml", "json"],
+        default="yaml",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+    if args.output_format == "yaml":
+        writer = write_yaml_configuration_file
+    elif args.output_format == "json":
+        writer = write_json_configuration_file
+    else:
+        raise NotImplementedError(f"Unknown format: {args.output_format}")
+    configuration = enumerate_authenticating_services()
+    writer(filename=args.output_filename, configuration=configuration)
+
+
+if __name__ == "__main__":
+    main()

{paasta-tools-1.0.5 → paasta-tools-1.1.0}/paasta_tools/generate_services_file.py

@@ -17,19 +17,17 @@ A simple script to enumerate all smartstack namespaces and output
 a /etc/services compatible file
 """
 import argparse
-import json
 import os
-import socket
-from datetime import datetime
 
 import service_configuration_lib
-import yaml
 
 from paasta_tools.long_running_service_tools import get_all_namespaces
 from paasta_tools.long_running_service_tools import get_all_namespaces_for_service
 from paasta_tools.utils import atomic_file_write
 from paasta_tools.utils import compose_job_id
 from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import write_json_configuration_file
+from paasta_tools.utils import write_yaml_configuration_file
 
 
 YOCALHOST = "169.254.255.254"
@@ -71,40 +69,6 @@ def get_service_lines_for_service(service):
     return lines
 
 
-def write_yaml_file(filename):
-    previous_config = maybe_load_previous_config(filename, yaml.safe_load)
-    configuration = generate_configuration()
-
-    if previous_config and previous_config == configuration:
-        return
-
-    with atomic_file_write(filename) as fp:
-        fp.write(
-            "# This file is automatically generated by paasta_tools.\n"
-            "# It was automatically generated at {now} on {host}.\n".format(
-                host=socket.getfqdn(), now=datetime.now().isoformat()
-            )
-        )
-        yaml.safe_dump(
-            configuration,
-            fp,
-            indent=2,
-            explicit_start=True,
-            default_flow_style=False,
-            allow_unicode=False,
-        )
-
-
-def maybe_load_previous_config(filename, config_loader):
-    try:
-        with open(filename, "r") as fp:
-            previous_config = config_loader(fp)
-            return previous_config
-    except Exception:
-        pass
-    return None
-
-
 def generate_configuration():
     service_data = get_all_namespaces()
     config = {}
@@ -116,14 +80,6 @@ def generate_configuration():
     return config
 
 
-def write_json_file(filename):
-    configuration = generate_configuration()
-    with atomic_file_write(filename) as fp:
-        json.dump(
-            obj=configuration, fp=fp, indent=2, sort_keys=True, separators=(",", ": ")
-        )
-
-
 def write_rfc1700_file(filename):
     strings = [
         "# This file is generated by generate_services_file",
@@ -140,10 +96,16 @@ def main():
     args = parse_args()
     if args.output_format == "rfc1700":
         write_rfc1700_file(filename=args.output_filename)
-    elif args.output_format == "yaml":
-        write_yaml_file(filename=args.output_filename)
-    elif args.output_format == "json":
-        write_json_file(filename=args.output_filename)
+    elif args.output_format in ("json", "yaml"):
+        writer = (
+            write_json_configuration_file
+            if args.output_format == "json"
+            else write_yaml_configuration_file
+        )
+        writer(
+            filename=args.output_filename,
+            configuration=generate_configuration(),
+        )
     else:
         raise (NotImplementedError)
 

{paasta-tools-1.0.5 → paasta-tools-1.1.0}/paasta_tools/generate_services_yaml.py

@@ -14,7 +14,8 @@
 # limitations under the License.
 import argparse
 
-from paasta_tools.generate_services_file import write_yaml_file
+from paasta_tools.generate_services_file import generate_configuration
+from paasta_tools.utils import write_yaml_configuration_file
 
 
 def main(argv=None):
@@ -22,7 +23,7 @@ def main(argv=None):
     parser.add_argument("output_path")
     args = parser.parse_args(argv)
 
-    write_yaml_file(args.output_path)
+    write_yaml_configuration_file(args.output_path, generate_configuration())
 
 
 if __name__ == "__main__":

{paasta-tools-1.0.5 → paasta-tools-1.1.0}/paasta_tools/utils.py

@@ -63,6 +63,7 @@ from typing import NamedTuple
 from typing import Optional
 from typing import Sequence
 from typing import Set
+from typing import TextIO
 from typing import Tuple
 from typing import Type
 from typing import TypeVar
@@ -73,6 +74,7 @@ import dateutil.tz
 import ldap3
 import requests_cache
 import service_configuration_lib
+import yaml
 from docker import Client
 from docker.utils import kwargs_from_env
 from kazoo.client import KazooClient
@@ -4137,3 +4139,68 @@ def get_k8s_url_for_cluster(cluster: str) -> Optional[str]:
 @lru_cache(maxsize=1)
 def is_using_unprivileged_containers() -> bool:
     return "podman" in os.getenv("DOCKER_HOST", "")
+
+
+def maybe_load_previous_config(
+    filename: str, config_loader: Callable[[TextIO], dict]
+) -> Optional[dict]:
+    """Try to load configuration file
+
+    :param str filename: path to load from
+    :param Callable[[TextIO], dict] config_loader: parser for the configuration
+    :return: configuration data, None if loading fails
+    """
+    try:
+        with open(filename, "r") as fp:
+            previous_config = config_loader(fp)
+            return previous_config
+    except Exception:
+        pass
+    return None
+
+
+def write_json_configuration_file(filename: str, configuration: dict) -> None:
+    """Atomically write configuration to JSON file
+
+    :param str filename: path to write to
+    :param dict configuration: configuration data
+    """
+    with atomic_file_write(filename) as fp:
+        json.dump(
+            obj=configuration,
+            fp=fp,
+            indent=2,
+            sort_keys=True,
+            separators=(",", ": "),
+        )
+
+
+def write_yaml_configuration_file(
+    filename: str, configuration: dict, check_existing: bool = True
+) -> None:
+    """Atomically write configuration to YAML file
+
+    :param str filename: path to write to
+    :param dict configuration: configuration data
+    :param bool check_existing: if existing file already matches config, do not overwrite
+    """
+    if check_existing:
+        previous_config = maybe_load_previous_config(filename, yaml.safe_load)
+        if previous_config and previous_config == configuration:
+            return
+
+    with atomic_file_write(filename) as fp:
+        fp.write(
+            "# This file is automatically generated by paasta_tools.\n"
+            "# It was automatically generated at {now} on {host}.\n".format(
+                host=socket.getfqdn(), now=datetime.datetime.now().isoformat()
+            )
+        )
+        yaml.safe_dump(
+            configuration,
+            fp,
+            indent=2,
+            explicit_start=True,
+            default_flow_style=False,
+            allow_unicode=False,
+        )

{paasta-tools-1.0.5 → paasta-tools-1.1.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.0.5
+Version: 1.1.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.0.5 → paasta-tools-1.1.0}/paasta_tools.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ paasta_tools/firewall_update.py
 paasta_tools/flink_tools.py
 paasta_tools/flinkeks_tools.py
 paasta_tools/generate_all_deployments
+paasta_tools/generate_authenticating_services.py
 paasta_tools/generate_deployments_for_service.py
 paasta_tools/generate_services_file.py
 paasta_tools/generate_services_yaml.py

{paasta-tools-1.0.5 → paasta-tools-1.1.0}/setup.py

@@ -57,6 +57,7 @@ setup(
         "paasta_tools/generate_deployments_for_service.py",
         "paasta_tools/generate_services_file.py",
         "paasta_tools/generate_services_yaml.py",
+        "paasta_tools/generate_authenticating_services.py",
         "paasta_tools/kubernetes/bin/kubernetes_remove_evicted_pods.py",
         "paasta_tools/kubernetes/bin/paasta_cleanup_stale_nodes.py",
         "paasta_tools/kubernetes/bin/paasta_secrets_sync.py",