paasta-tools 1.11.1__tar.gz → 1.11.3__tar.gz

{paasta-tools-1.11.1/paasta_tools.egg-info → paasta-tools-1.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.11.1
+Version: 1.11.3
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.11.1"
+__version__ = "1.11.3"

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools/cli/cmds/info.py

@@ -36,6 +36,9 @@ NO_EXTERNAL_LINK_MESSAGE = (
     "Please add one that points to a reference doc for your service"
 )
 
+# modes that depend on smartstack port cannot be tested via paasta proxies, so we exclude those
+TESTABLE_SERVICE_MODES = {"http", "https"}
+
 
 def add_subparser(subparsers):
     list_parser = subparsers.add_parser(
@@ -94,14 +97,9 @@ def get_deployments_strings(service: str, soa_dir: str) -> List[str]:
         )
         service_mode = service_config.get_mode()
         for cluster in deployments_to_clusters(deployments):
-            if service_mode == "tcp":
-                service_port = service_config.get("proxy_port")
-                link = PaastaColors.cyan(
-                    "%s://paasta-%s.yelp:%d/" % (service_mode, cluster, service_port)
-                )
-            elif service_mode == "http" or service_mode == "https":
+            if service_mode in TESTABLE_SERVICE_MODES:
                 link = PaastaColors.cyan(
-                    f"{service_mode}://{service}.paasta-{cluster}.yelp/"
+                    f"{service_mode}://{service}.proxy.{cluster}.paasta/"
                 )
             else:
                 link = "N/A"
@@ -111,8 +109,7 @@ def get_deployments_strings(service: str, soa_dir: str) -> List[str]:
 
 def get_dashboard_urls(service):
     output = [
-        " - %s (Sensu Alerts)"
-        % (PaastaColors.cyan("https://uchiwa.yelpcorp.com/#/events?q=%s" % service))
+        " - %s (service load y/sl2)" % (PaastaColors.cyan(f"http://y/{service}_load"))
     ]
     return output
 
@@ -137,7 +134,9 @@ def get_service_info(service, soa_dir):
         % PaastaColors.cyan(get_runbook(service=service, overrides={}, soa_dir=soa_dir))
     )
     output.append("Git Repo: %s" % git_url)
-    output.append("Deployed to the following clusters:")
+    output.append(
+        "Deployed to the following clusters (with test URLs, where available):"
+    )
     output.extend(get_deployments_strings(service, soa_dir))
     if smartstack_endpoints:
         output.append("Smartstack endpoint(s):")

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools/cli/cmds/logs.py

@@ -53,7 +53,7 @@ except ImportError:
     scribereader = None
 
 try:
-    from clog.readers import S3LogsReader
+    from logreader.readers import S3LogsReader
 except ImportError:
     S3LogsReader = None
 
@@ -294,18 +294,20 @@ def paasta_log_line_passes_filter(
     except ValueError:
         log.debug("Trouble parsing line as json. Skipping. Line: %r" % line)
         return False
-    timestamp = isodate.parse_datetime(parsed_line.get("timestamp"))
-    if not check_timestamp_in_range(timestamp, start_time, end_time):
-        return False
-    return (
-        (parsed_line.get("level") is None or parsed_line.get("level") in levels)
+
+    if (
+        (instances is None or parsed_line.get("instance") in instances)
+        and (parsed_line.get("level") is None or parsed_line.get("level") in levels)
         and parsed_line.get("component") in components
         and (
             parsed_line.get("cluster") in clusters
             or parsed_line.get("cluster") == ANY_CLUSTER
         )
-        and (instances is None or parsed_line.get("instance") in instances)
-    )
+    ):
+        timestamp = isodate.parse_datetime(parsed_line.get("timestamp"))
+        if check_timestamp_in_range(timestamp, start_time, end_time):
+            return True
+    return False
 
 
 def paasta_app_output_passes_filter(
@@ -324,28 +326,24 @@ def paasta_app_output_passes_filter(
     except ValueError:
         log.debug("Trouble parsing line as json. Skipping. Line: %r" % line)
         return False
-    try:
-        timestamp = isodate.parse_datetime(parsed_line.get("timestamp"))
-    # https://github.com/gweis/isodate/issues/53
-    except ValueError:
-        return True
-    except AttributeError:
-        # Timestamp might be missing. We had an issue where OTel was splitting overly long log lines
-        # and not including timestamps in the resulting log records (OBSPLAT-2216).
-        # Although this was then fixed in OTel, we should not rely on timestamps being present,
-        # as the format cannot be guaranteed.
-        return False
-    if not check_timestamp_in_range(timestamp, start_time, end_time):
-        return False
-    return (
-        parsed_line.get("component") in components
-        and (
-            parsed_line.get("cluster") in clusters
-            or parsed_line.get("cluster") == ANY_CLUSTER
-        )
-        and (instances is None or parsed_line.get("instance") in instances)
+
+    if (
+        (instances is None or parsed_line.get("instance") in instances)
+        and parsed_line.get("cluster") in clusters
+        and parsed_line.get("component") in components
         and (pods is None or parsed_line.get("pod_name") in pods)
-    )
+    ):
+        try:
+            timestamp = isodate.parse_datetime(parsed_line.get("timestamp"))
+        except AttributeError:
+            # Timestamp might be missing. We had an issue where OTel was splitting overly long log lines
+            # and not including timestamps in the resulting log records (OBSPLAT-2216).
+            # Although this was then fixed in OTel, we should not rely on timestamps being present,
+            # as the format cannot be guaranteed.
+            return False
+        if check_timestamp_in_range(timestamp, start_time, end_time):
+            return True
+    return False
 
 
 def extract_utc_timestamp_from_log_line(line: str) -> datetime.datetime:
@@ -484,12 +482,6 @@ def list_log_readers() -> Iterable[str]:
     return _log_reader_classes.keys()
 
 
-def get_default_log_reader() -> "LogReader":
-    log_reader_config = load_system_paasta_config().get_log_reader()
-    log_reader_class = get_log_reader_class(log_reader_config["driver"])
-    return log_reader_class(**log_reader_config.get("options", {}))
-
-
 def get_log_reader(components: Set[str]) -> "LogReader":
     log_readers_config = load_system_paasta_config().get_log_readers()
     # ideally we should use a single "driver" for all components, but in cases where more than one is used for different components,
@@ -1483,14 +1475,6 @@ def pick_default_log_mode(
     return 1
 
 
-def pick_log_reader(cluster: str, components: Set[str]) -> LogReader:
-    uses_log_readers = load_system_paasta_config().use_multiple_log_readers()
-    if uses_log_readers and cluster in uses_log_readers:
-        return get_log_reader(components)
-    else:
-        return get_default_log_reader()
-
-
 def paasta_logs(args: argparse.Namespace) -> int:
     """Print the logs for as Paasta service.
     :param args: argparse.Namespace obj created from sys.args by cli"""
@@ -1535,7 +1519,7 @@ def paasta_logs(args: argparse.Namespace) -> int:
 
     log.debug(f"Going to get logs for {service} on cluster {clusters}")
 
-    log_reader = pick_log_reader(clusters[0], components)
+    log_reader = get_log_reader(components)
 
     if not validate_filtering_args(args, log_reader):
         return 1

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools/cli/schemas/eks_schema.json

@@ -93,6 +93,14 @@
                             ]
                         }
                     ]
+                },
+                {
+                    "not": {
+                        "required": [
+                            "service_account_name",
+                            "iam_role"
+                        ]
+                    }
                 }
             ],
             "properties": {

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools/cli/schemas/kubernetes_schema.json

@@ -93,6 +93,14 @@
                             ]
                         }
                     ]
+                },
+                {
+                    "not": {
+                        "required": [
+                            "service_account_name",
+                            "iam_role"
+                        ]
+                    }
                 }
             ],
             "properties": {

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools/cli/schemas/tron_schema.json

@@ -17,6 +17,16 @@
             "required": [
                 "command"
             ],
+            "allOf": [
+                {
+                    "not": {
+                        "required": [
+                            "service_account_name",
+                            "iam_role"
+                        ]
+                    }
+                }
+            ],
             "properties": {
                 "name": {
                     "$ref": "#definitions/name"

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools/kubernetes_tools.py

@@ -118,16 +118,16 @@ from kubernetes.client import V1Volume
 from kubernetes.client import V1VolumeMount
 from kubernetes.client import V1VolumeProjection
 from kubernetes.client import V1WeightedPodAffinityTerm
-from kubernetes.client import V2beta2CrossVersionObjectReference
-from kubernetes.client import V2beta2HorizontalPodAutoscaler
-from kubernetes.client import V2beta2HorizontalPodAutoscalerCondition
-from kubernetes.client import V2beta2HorizontalPodAutoscalerSpec
-from kubernetes.client import V2beta2MetricIdentifier
-from kubernetes.client import V2beta2MetricSpec
-from kubernetes.client import V2beta2MetricTarget
-from kubernetes.client import V2beta2ObjectMetricSource
-from kubernetes.client import V2beta2ResourceMetricSource
-from kubernetes.client.models import V2beta2HorizontalPodAutoscalerStatus
+from kubernetes.client import V2CrossVersionObjectReference
+from kubernetes.client import V2HorizontalPodAutoscaler
+from kubernetes.client import V2HorizontalPodAutoscalerCondition
+from kubernetes.client import V2HorizontalPodAutoscalerSpec
+from kubernetes.client import V2MetricIdentifier
+from kubernetes.client import V2MetricSpec
+from kubernetes.client import V2MetricTarget
+from kubernetes.client import V2ObjectMetricSource
+from kubernetes.client import V2ResourceMetricSource
+from kubernetes.client.models import V2HorizontalPodAutoscalerStatus
 from kubernetes.client.rest import ApiException
 from mypy_extensions import TypedDict
 from service_configuration_lib import read_soa_metadata
@@ -237,19 +237,19 @@ PROJECTED_SA_TOKEN_PATH = "token"
 # For detail, https://github.com/kubernetes-client/python/issues/553
 # This hack should be removed when the issue got fixed.
 # This is no better way to work around rn.
-class MonkeyPatchAutoScalingConditions(V2beta2HorizontalPodAutoscalerStatus):
+class MonkeyPatchAutoScalingConditions(V2HorizontalPodAutoscalerStatus):
     @property
-    def conditions(self) -> Sequence[V2beta2HorizontalPodAutoscalerCondition]:
+    def conditions(self) -> Sequence[V2HorizontalPodAutoscalerCondition]:
         return super().conditions()
 
     @conditions.setter
     def conditions(
-        self, conditions: Optional[Sequence[V2beta2HorizontalPodAutoscalerCondition]]
+        self, conditions: Optional[Sequence[V2HorizontalPodAutoscalerCondition]]
     ) -> None:
         self._conditions = list() if conditions is None else conditions
 
 
-models.V2beta2HorizontalPodAutoscalerStatus = MonkeyPatchAutoScalingConditions
+models.V2HorizontalPodAutoscalerStatus = MonkeyPatchAutoScalingConditions
 
 
 class KubeKind(NamedTuple):
@@ -607,7 +607,7 @@ class KubeClient:
         self.apiextensions = kube_client.ApiextensionsV1Api(self.api_client)
 
         self.custom = kube_client.CustomObjectsApi(self.api_client)
-        self.autoscaling = kube_client.AutoscalingV2beta2Api(self.api_client)
+        self.autoscaling = kube_client.AutoscalingV2Api(self.api_client)
         self.rbac = kube_client.RbacAuthorizationV1Api(self.api_client)
 
         self.request = self.api_client.request
@@ -801,18 +801,18 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
 
     def get_autoscaling_provider_spec(
         self, name: str, namespace: str, provider: MetricsProviderDict
-    ) -> Optional[V2beta2MetricSpec]:
+    ) -> Optional[V2MetricSpec]:
         target = provider["setpoint"]
         prometheus_hpa_metric_name = (
             f"{self.namespace_external_metric_name(provider['type'])}-prom"
         )
 
         if provider["type"] == METRICS_PROVIDER_CPU:
-            return V2beta2MetricSpec(
+            return V2MetricSpec(
                 type="Resource",
-                resource=V2beta2ResourceMetricSource(
+                resource=V2ResourceMetricSource(
                     name="cpu",
-                    target=V2beta2MetricTarget(
+                    target=V2MetricTarget(
                         type="Utilization",
                         average_utilization=int(target * 100),
                     ),
@@ -824,14 +824,14 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             METRICS_PROVIDER_GUNICORN,
             METRICS_PROVIDER_ACTIVE_REQUESTS,
         }:
-            return V2beta2MetricSpec(
+            return V2MetricSpec(
                 type="Object",
-                object=V2beta2ObjectMetricSource(
-                    metric=V2beta2MetricIdentifier(name=prometheus_hpa_metric_name),
-                    described_object=V2beta2CrossVersionObjectReference(
+                object=V2ObjectMetricSource(
+                    metric=V2MetricIdentifier(name=prometheus_hpa_metric_name),
+                    described_object=V2CrossVersionObjectReference(
                         api_version="apps/v1", kind="Deployment", name=name
                     ),
-                    target=V2beta2MetricTarget(
+                    target=V2MetricTarget(
                         type="Value",
                         # we average the number of instances needed to handle the current (or
                         # averaged) load instead of the load itself as this leads to more
@@ -843,14 +843,14 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                 ),
             )
         elif provider["type"] == METRICS_PROVIDER_PROMQL:
-            return V2beta2MetricSpec(
+            return V2MetricSpec(
                 type="Object",
-                object=V2beta2ObjectMetricSource(
-                    metric=V2beta2MetricIdentifier(name=prometheus_hpa_metric_name),
-                    described_object=V2beta2CrossVersionObjectReference(
+                object=V2ObjectMetricSource(
+                    metric=V2MetricIdentifier(name=prometheus_hpa_metric_name),
+                    described_object=V2CrossVersionObjectReference(
                         api_version="apps/v1", kind="Deployment", name=name
                     ),
-                    target=V2beta2MetricTarget(
+                    target=V2MetricTarget(
                         # Use the setpoint specified by the user.
                         type="Value",
                         value=target,
@@ -870,7 +870,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
         cluster: str,
         kube_client: KubeClient,
         namespace: str,
-    ) -> Optional[V2beta2HorizontalPodAutoscaler]:
+    ) -> Optional[V2HorizontalPodAutoscaler]:
         # Returns None if an HPA should not be attached based on the config,
         # or the config is invalid.
 
@@ -909,17 +909,17 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             paasta_prefixed("managed"): "true",
         }
 
-        hpa = V2beta2HorizontalPodAutoscaler(
+        hpa = V2HorizontalPodAutoscaler(
             kind="HorizontalPodAutoscaler",
             metadata=V1ObjectMeta(
                 name=name, namespace=namespace, annotations=dict(), labels=labels
             ),
-            spec=V2beta2HorizontalPodAutoscalerSpec(
+            spec=V2HorizontalPodAutoscalerSpec(
                 behavior=scaling_policy,
                 max_replicas=max_replicas,
                 min_replicas=min_replicas,
                 metrics=metrics,
-                scale_target_ref=V2beta2CrossVersionObjectReference(
+                scale_target_ref=V2CrossVersionObjectReference(
                     api_version="apps/v1", kind="Deployment", name=name
                 ),
             ),
@@ -3690,7 +3690,7 @@ async def get_hpa(
     kube_client: KubeClient,
     name: str,
     namespace: str,
-) -> V2beta2HorizontalPodAutoscaler:
+) -> V2HorizontalPodAutoscaler:
     async_get_hpa = a_sync.to_async(
         kube_client.autoscaling.read_namespaced_horizontal_pod_autoscaler
     )

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools/utils.py

@@ -2039,7 +2039,6 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     sidecar_requirements_config: Dict[str, KubeContainerResourceRequest]
     eks_cluster_aliases: Dict[str, str]
     secret_sync_delay_seconds: float
-    use_multiple_log_readers: Optional[List[str]]
     service_auth_token_settings: ProjectedSAVolume
     service_auth_vault_role: str
     service_auth_sso_oidc_client_id: str
@@ -2364,12 +2363,6 @@ class SystemPaastaConfig:
                 % self.directory
             )
 
-    def use_multiple_log_readers(self) -> Optional[List[str]]:
-        """
-        Get the list of clusters that are using multiple log readers
-        """
-        return self.config_dict.get("use_multiple_log_readers")
-
     def get_metrics_provider(self) -> Optional[str]:
         """Get the metrics_provider configuration out of global paasta config
 
@@ -2431,11 +2424,11 @@ class SystemPaastaConfig:
 
     def get_cluster_fqdn_format(self) -> str:
         """Get a format string that constructs a DNS name pointing at the paasta masters in a cluster. This format
-        string gets one parameter: cluster. Defaults to 'paasta-{cluster:s}.yelp'.
+        string gets one parameter: cluster. Defaults to '{cluster:s}.paasta'.
 
         :returns: A format string for constructing the FQDN of the masters in a given cluster.
         """
-        return self.config_dict.get("cluster_fqdn_format", "paasta-{cluster:s}.yelp")
+        return self.config_dict.get("cluster_fqdn_format", "{cluster:s}.paasta")
 
     def get_paasta_status_version(self) -> str:
         """Get paasta status version string (new | old). Defaults to 'old'.

{paasta-tools-1.11.1 → paasta-tools-1.11.3/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.11.1
+Version: 1.11.3
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/paasta_tools.egg-info/requires.txt

@@ -21,7 +21,7 @@ humanfriendly
 humanize>=0.5.1
 inotify>=0.2.8
 ipaddress>=1.0.22
-isodate>=0.5.0
+isodate>=0.7.2
 jsonschema[format]
 kazoo>=2.0.0
 kubernetes<26.0.0,>=18.20.0

{paasta-tools-1.11.1 → paasta-tools-1.11.3}/requirements-minimal.txt

@@ -21,7 +21,7 @@ humanfriendly
 humanize >= 0.5.1
 inotify >= 0.2.8
 ipaddress >= 1.0.22
-isodate >= 0.5.0
+isodate >= 0.7.2
 jsonschema[format]
 kazoo >= 2.0.0
 # the upper-bound here is mainly for things that use paasta-tools as a library and don't benefit