paasta-tools 1.1.2__tar.gz → 1.4.0__tar.gz

{paasta-tools-1.1.2/paasta_tools.egg-info → paasta-tools-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.1.2
+Version: 1.4.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.1.2 → paasta-tools-1.4.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.1.2"
+__version__ = "1.4.0"

{paasta-tools-1.1.2 → paasta-tools-1.4.0}/paasta_tools/cli/cmds/logs.py

@@ -33,10 +33,12 @@ from typing import Iterable
 from typing import List
 from typing import Mapping
 from typing import MutableSequence
+from typing import Optional
 from typing import Sequence
 from typing import Set
 from typing import Tuple
 from typing import Type
+from typing import Union
 
 import isodate
 import pytz
@@ -48,6 +50,11 @@ try:
 except ImportError:
     scribereader = None
 
+try:
+    from clog.readers import S3LogsReader
+except ImportError:
+    S3LogsReader = None
+
 from pytimeparse.timeparse import timeparse
 
 from paasta_tools.cli.utils import figure_out_service_name
@@ -289,7 +296,7 @@ def paasta_log_line_passes_filter(
     if not check_timestamp_in_range(timestamp, start_time, end_time):
         return False
     return (
-        parsed_line.get("level") in levels
+        (parsed_line.get("level") is None or parsed_line.get("level") in levels)
         and parsed_line.get("component") in components
         and (
             parsed_line.get("cluster") in clusters
@@ -459,7 +466,9 @@ def register_log_reader(name):
     return outer
 
 
-def get_log_reader_class(name: str) -> Type["ScribeLogReader"]:
+def get_log_reader_class(
+    name: str,
+) -> Union[Type["ScribeLogReader"], Type["VectorLogsReader"]]:
     return _log_reader_classes[name]
 
 
@@ -467,12 +476,44 @@ def list_log_readers() -> Iterable[str]:
     return _log_reader_classes.keys()
 
 
-def get_log_reader() -> "ScribeLogReader":
+def get_default_log_reader() -> "LogReader":
     log_reader_config = load_system_paasta_config().get_log_reader()
     log_reader_class = get_log_reader_class(log_reader_config["driver"])
     return log_reader_class(**log_reader_config.get("options", {}))
 
 
+def get_log_reader(components: Set[str]) -> "LogReader":
+    log_readers_config = load_system_paasta_config().get_log_readers()
+    # ideally we should use a single "driver" for all components, but in cases where more than one is used for different components,
+    # we should use the first one that supports all requested components
+    # otherwise signal an error and suggest to provide a different list of components
+    components_lists = []
+    for log_reader_config in log_readers_config:
+        supported_components = log_reader_config.get("components", [])
+        components_lists.append(supported_components)
+        if components.issubset(supported_components):
+            log_reader_class = get_log_reader_class(log_reader_config["driver"])
+            print(
+                PaastaColors.cyan(
+                    "Using '{}' driver to fetch logs...".format(
+                        log_reader_config["driver"]
+                    )
+                ),
+                file=sys.stderr,
+            )
+            return log_reader_class(**log_reader_config.get("options", {}))
+    print(
+        PaastaColors.cyan(
+            "Supplied list of components '{}' is not supported by any log reader. Supported components are:\n\t{}".format(
+                ", ".join(components),
+                "\n\tor ".join([",".join(comp_list) for comp_list in components_lists]),
+            )
+        ),
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+
 class LogReader:
     # Tailing, i.e actively viewing logs as they come in
     SUPPORTS_TAILING = False
@@ -1123,7 +1164,76 @@ class ScribeLogReader(LogReader):
             return env
 
 
-def scribe_env_to_locations(scribe_env):
+@register_log_reader("vector-logs")
+class VectorLogsReader(LogReader):
+    SUPPORTS_TIME = True
+
+    def __init__(self, cluster_map: Mapping[str, Any]) -> None:
+        super().__init__()
+
+        if S3LogsReader is None:
+            raise Exception("yelp_clog package must be available to use S3LogsReader")
+
+        self.cluster_map = cluster_map
+
+    def get_superregion_for_cluster(self, cluster: str) -> Optional[str]:
+        return self.cluster_map.get(cluster, None)
+
+    def print_logs_by_time(
+        self,
+        service,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+        levels,
+        components: Iterable[str],
+        clusters,
+        instances,
+        pods,
+        raw_mode,
+        strip_headers,
+    ) -> None:
+        stream_name = get_log_name_for_service(service, prefix="app_output")
+        superregion = self.get_superregion_for_cluster(clusters[0])
+        reader = S3LogsReader(superregion)
+        start_date = start_time.date()
+        end_date = end_time.date()
+        aggregated_logs: List[Dict[str, Any]] = []
+
+        for line in reader.get_log_reader(
+            log_name=stream_name, min_date=start_date, max_date=end_date
+        ):
+            if paasta_log_line_passes_filter(
+                line,
+                levels,
+                service,
+                components,
+                clusters,
+                instances,
+                pods,
+                start_time=start_time,
+                end_time=end_time,
+            ):
+                try:
+                    parsed_line = json.loads(line)
+                    timestamp = isodate.parse_datetime(parsed_line.get("timestamp"))
+                    if not timestamp.tzinfo:
+                        timestamp = pytz.utc.localize(timestamp)
+                except ValueError:
+                    timestamp = pytz.utc.localize(datetime.datetime.min)
+
+                line = {"raw_line": line, "sort_key": timestamp}
+                aggregated_logs.append(line)
+
+        aggregated_logs = list(
+            {line["raw_line"]: line for line in aggregated_logs}.values()
+        )
+        aggregated_logs.sort(key=lambda log_line: log_line["sort_key"])
+
+        for line in aggregated_logs:
+            print_log(line["raw_line"], levels, raw_mode, strip_headers)
+
+
+def scribe_env_to_locations(scribe_env) -> Mapping[str, Any]:
     """Converts a scribe environment to a dictionary of locations. The
     return value is meant to be used as kwargs for `scribereader.get_tail_host_and_port`.
     """
@@ -1188,9 +1298,7 @@ def generate_start_end_time(
     return start_time, end_time
 
 
-def validate_filtering_args(
-    args: argparse.Namespace, log_reader: ScribeLogReader
-) -> bool:
+def validate_filtering_args(args: argparse.Namespace, log_reader: LogReader) -> bool:
     if not log_reader.SUPPORTS_LINE_OFFSET and args.line_offset is not None:
         print(
             PaastaColors.red(
@@ -1254,7 +1362,7 @@ def validate_filtering_args(
 
 def pick_default_log_mode(
     args: argparse.Namespace,
-    log_reader: ScribeLogReader,
+    log_reader: LogReader,
     service: str,
     levels: Sequence[str],
     components: Iterable[str],
@@ -1320,6 +1428,14 @@ def pick_default_log_mode(
     return 1
 
 
+def pick_log_reader(cluster: str, components: Set[str]) -> LogReader:
+    uses_log_readers = load_system_paasta_config().use_multiple_log_readers()
+    if uses_log_readers and cluster in uses_log_readers:
+        return get_log_reader(components)
+    else:
+        return get_default_log_reader()
+
+
 def paasta_logs(args: argparse.Namespace) -> int:
     """Print the logs for as Paasta service.
     :param args: argparse.Namespace obj created from sys.args by cli"""
@@ -1327,7 +1443,7 @@ def paasta_logs(args: argparse.Namespace) -> int:
 
     service = figure_out_service_name(args, soa_dir)
 
-    cluster = args.cluster
+    clusters = args.cluster
     if (
         args.cluster is None
         or args.instance is None
@@ -1339,7 +1455,7 @@ def paasta_logs(args: argparse.Namespace) -> int:
         )
         return 1
 
-    if verify_instances(args.instance, service, cluster, soa_dir):
+    if verify_instances(args.instance, service, clusters, soa_dir):
         return 1
 
     instance = args.instance
@@ -1362,16 +1478,16 @@ def paasta_logs(args: argparse.Namespace) -> int:
 
     levels = [DEFAULT_LOGLEVEL, "debug"]
 
-    log.debug(f"Going to get logs for {service} on cluster {cluster}")
+    log.debug(f"Going to get logs for {service} on cluster {clusters}")
 
-    log_reader = get_log_reader()
+    log_reader = pick_log_reader(clusters[0], components)
 
     if not validate_filtering_args(args, log_reader):
         return 1
     # They haven't specified what kind of filtering they want, decide for them
     if args.line_count is None and args.time_from is None and not args.tail:
         return pick_default_log_mode(
-            args, log_reader, service, levels, components, cluster, instance, pods
+            args, log_reader, service, levels, components, clusters, instance, pods
         )
     if args.tail:
         print(
@@ -1381,7 +1497,7 @@ def paasta_logs(args: argparse.Namespace) -> int:
             service=service,
             levels=levels,
             components=components,
-            clusters=cluster,
+            clusters=clusters,
             instances=[instance],
             pods=pods,
             raw_mode=args.raw_mode,
@@ -1402,7 +1518,7 @@ def paasta_logs(args: argparse.Namespace) -> int:
             line_count=args.line_count,
             levels=levels,
             components=components,
-            clusters=cluster,
+            clusters=clusters,
             instances=[instance],
             pods=pods,
             raw_mode=args.raw_mode,
@@ -1416,7 +1532,7 @@ def paasta_logs(args: argparse.Namespace) -> int:
             line_offset=args.line_offset,
             levels=levels,
             components=components,
-            clusters=cluster,
+            clusters=clusters,
             instances=[instance],
             pods=pods,
             raw_mode=args.raw_mode,
@@ -1437,7 +1553,7 @@ def paasta_logs(args: argparse.Namespace) -> int:
         end_time=end_time,
         levels=levels,
         components=components,
-        clusters=cluster,
+        clusters=clusters,
         instances=[instance],
         pods=pods,
         raw_mode=args.raw_mode,

{paasta-tools-1.1.2 → paasta-tools-1.4.0}/paasta_tools/generate_authenticating_services.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-A simple script to enumerate all services partecipating in authenticated
+A simple script to enumerate all services participating in authenticated
 communications, and list them in a YAML/JSON file.
 """
 import argparse
@@ -27,6 +27,7 @@ from typing import Set
 import yaml
 
 from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import write_json_configuration_file
 from paasta_tools.utils import write_yaml_configuration_file
 
@@ -53,6 +54,7 @@ def enumerate_authenticating_services() -> Dict[str, List[str]]:
     config_path_pattern = os.path.join(DEFAULT_SOA_DIR, "*", AUTHORIZATION_CONFIG_FILE)
     for authz_config in glob.glob(config_path_pattern):
         result.update(list_services_in_authz_config(authz_config))
+    result.update(load_system_paasta_config().get_always_authenticating_services())
     return {"services": sorted(result)}
 
 

{paasta-tools-1.1.2 → paasta-tools-1.4.0}/paasta_tools/kubernetes_tools.py

@@ -21,6 +21,7 @@ import os
 import re
 from datetime import datetime
 from enum import Enum
+from functools import lru_cache
 from inspect import currentframe
 from pathlib import Path
 from typing import Any
@@ -2552,6 +2553,13 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             "topology_spread_constraints", default_pod_topology_spread_constraints
         )
 
+    def get_projected_sa_volumes(self) -> List[ProjectedSAVolume]:
+        return add_volumes_for_authenticating_services(
+            service_name=self.service,
+            config_volumes=super().get_projected_sa_volumes(),
+            soa_dir=self.soa_dir,
+        )
+
 
 def get_kubernetes_secret_hashes(
     environment_variables: Mapping[str, str], service: str, namespace: str
@@ -4406,3 +4414,35 @@ def get_kubernetes_secret_volumes(
                 ] = secret_contents
 
     return secret_volumes
+
+
+@lru_cache()
+def get_authenticating_services(soa_dir: str = DEFAULT_SOA_DIR) -> Set[str]:
+    """Load list of services participating in authenticated traffic"""
+    authenticating_services_conf_path = os.path.join(soa_dir, "authenticating.yaml")
+    config = service_configuration_lib.read_yaml_file(authenticating_services_conf_path)
+    return set(config.get("services", []))
+
+
+def add_volumes_for_authenticating_services(
+    service_name: str,
+    config_volumes: List[ProjectedSAVolume],
+    soa_dir: str = DEFAULT_SOA_DIR,
+) -> List[ProjectedSAVolume]:
+    """Add projected service account volume to the list of volumes if service
+    participates in authenticated traffic. In case of changes, a new list is returned,
+    no updates in-place.
+
+    :param str service_name: name of the service
+    :param List[ProjectedSAVolume] config_volumes: existing projected volumes from service config
+    :param str soa_dir: path to SOA configurations directory
+    :return: updated list of projected service account volumes
+    """
+    token_config = load_system_paasta_config().get_service_auth_token_volume_config()
+    if (
+        token_config
+        and service_name in get_authenticating_services(soa_dir)
+        and not any(volume == token_config for volume in config_volumes)
+    ):
+        config_volumes = [token_config, *config_volumes]
+    return config_volumes

{paasta-tools-1.1.2 → paasta-tools-1.4.0}/paasta_tools/tron_tools.py

@@ -56,6 +56,7 @@ from paasta_tools.utils import get_k8s_url_for_cluster
 from paasta_tools.utils import validate_pool
 from paasta_tools.utils import PoolsNotConfiguredError
 from paasta_tools.utils import DockerVolume
+from paasta_tools.utils import ProjectedSAVolume
 
 from paasta_tools import spark_tools
 
@@ -71,6 +72,7 @@ from paasta_tools.secret_tools import is_shared_secret
 from paasta_tools.secret_tools import is_shared_secret_from_secret_name
 from paasta_tools.secret_tools import get_secret_name_from_ref
 from paasta_tools.kubernetes_tools import get_paasta_secret_name
+from paasta_tools.kubernetes_tools import add_volumes_for_authenticating_services
 from paasta_tools.secret_tools import SHARED_SECRET_SERVICE
 
 from paasta_tools import monitoring_tools
@@ -627,6 +629,14 @@ class TronActionConfig(InstanceConfig):
     def get_service_account_name(self) -> Optional[str]:
         return self.config_dict.get("service_account_name")
 
+    def get_projected_sa_volumes(self) -> Optional[List[ProjectedSAVolume]]:
+        projected_volumes = add_volumes_for_authenticating_services(
+            service_name=self.service,
+            config_volumes=super().get_projected_sa_volumes(),
+            soa_dir=self.soa_dir,
+        )
+        return projected_volumes if projected_volumes else None
+
 
 class TronJobConfig:
     """Represents a job in Tron, consisting of action(s) and job-level configuration values."""
@@ -958,6 +968,9 @@ def format_tron_action_dict(action_config: TronActionConfig):
                 dry_run=action_config.for_validation,
             )
 
+        # service account token volumes for service authentication
+        result["projected_sa_volumes"] = action_config.get_projected_sa_volumes()
+
         extra_volumes = action_config.get_extra_volumes()
         if executor == "spark":
             is_mrjob = action_config.config_dict.get("mrjob", False)

{paasta-tools-1.1.2 → paasta-tools-1.4.0}/paasta_tools/utils.py

@@ -75,7 +75,7 @@ import ldap3
 import requests_cache
 import service_configuration_lib
 import yaml
-from docker import Client
+from docker import APIClient
 from docker.utils import kwargs_from_env
 from kazoo.client import KazooClient
 from mypy_extensions import TypedDict
@@ -1359,6 +1359,7 @@ class LogWriterConfig(TypedDict):
 class LogReaderConfig(TypedDict):
     driver: str
     options: Dict
+    components: Optional[List]
 
 
 # The active log writer.
@@ -1960,6 +1961,7 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     ldap_search_ou: str
     local_run_config: LocalRunConfig
     log_reader: LogReaderConfig
+    log_readers: List[LogReaderConfig]
     log_writer: LogWriterConfig
     mark_for_deployment_max_polling_threads: int
     mark_for_deployment_default_polling_interval: float
@@ -2017,6 +2019,9 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     sidecar_requirements_config: Dict[str, KubeContainerResourceRequest]
     eks_cluster_aliases: Dict[str, str]
     secret_sync_delay_seconds: float
+    use_multiple_log_readers: Optional[List[str]]
+    service_auth_token_settings: ProjectedSAVolume
+    always_authenticating_services: List[str]
 
 
 def load_system_paasta_config(
@@ -2321,6 +2326,25 @@ class SystemPaastaConfig:
                 % self.directory
             )
 
+    def get_log_readers(self) -> List[LogReaderConfig]:
+        """Get the log_readers configuration out of global paasta config
+
+        :returns: the log_readers list of dicts.
+        """
+        try:
+            return self.config_dict["log_readers"]
+        except KeyError:
+            raise PaastaNotConfiguredError(
+                "Could not find log_readers in configuration directory: %s"
+                % self.directory
+            )
+
+    def use_multiple_log_readers(self) -> Optional[List[str]]:
+        """
+        Get the list of clusters that are using multiple log readers
+        """
+        return self.config_dict.get("use_multiple_log_readers")
+
     def get_metrics_provider(self) -> Optional[str]:
         """Get the metrics_provider configuration out of global paasta config
 
@@ -2703,6 +2727,12 @@ class SystemPaastaConfig:
     def get_kube_clusters(self) -> Dict:
         return self.config_dict.get("kube_clusters", {})
 
+    def get_service_auth_token_volume_config(self) -> ProjectedSAVolume:
+        return self.config_dict.get("service_auth_token_settings", {})
+
+    def get_always_authenticating_services(self) -> List[str]:
+        return self.config_dict.get("always_authenticating_services", [])
+
 
 def _run(
     command: Union[str, List[str]],
@@ -3315,12 +3345,12 @@ def get_docker_host() -> str:
     return os.environ.get("DOCKER_HOST", "unix://var/run/docker.sock")
 
 
-def get_docker_client() -> Client:
+def get_docker_client() -> APIClient:
     client_opts = kwargs_from_env(assert_hostname=False)
     if "base_url" in client_opts:
-        return Client(**client_opts)
+        return APIClient(**client_opts)
     else:
-        return Client(base_url=get_docker_host(), **client_opts)
+        return APIClient(base_url=get_docker_host(), **client_opts)
 
 
 def get_running_mesos_docker_containers() -> List[Dict]:

{paasta-tools-1.1.2 → paasta-tools-1.4.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.1.2
+Version: 1.4.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.1.2 → paasta-tools-1.4.0}/paasta_tools.egg-info/requires.txt

@@ -10,7 +10,7 @@ certifi
 choice>=0.1
 cookiecutter>=1.4.0
 croniter
-docker-py>=1.2.3
+docker
 dulwich>=0.17.3
 ephemeral-port-reserve>=1.0.1
 graphviz

{paasta-tools-1.1.2 → paasta-tools-1.4.0}/requirements-minimal.txt

@@ -10,9 +10,7 @@ certifi
 choice >= 0.1
 cookiecutter >= 1.4.0
 croniter
-# Don't update this unless you have confirmed the client works with
-# the Docker version deployed on PaaSTA servers
-docker-py >= 1.2.3
+docker
 dulwich >= 0.17.3
 ephemeral-port-reserve >= 1.0.1
 graphviz