paasta-tools 1.21.2__tar.gz → 1.21.4__tar.gz

paasta_tools-1.21.4/PKG-INFO

@@ -0,0 +1,73 @@
+Metadata-Version: 2.1
+Name: paasta-tools
+Version: 1.21.4
+Summary: Tools for Yelps SOA infrastructure
+Author: Compute Infrastructure @ Yelp
+Author-email: compute-infra@yelp.com
+Provides: paasta_tools
+Requires-Python: >=3.8.0
+License-File: LICENSE
+Requires-Dist: a-sync>=0.5.0
+Requires-Dist: aiohttp>=3.5.4
+Requires-Dist: argcomplete>=0.8.1
+Requires-Dist: boto
+Requires-Dist: boto3
+Requires-Dist: boto3-type-annotations
+Requires-Dist: botocore
+Requires-Dist: bravado>=10.2.0
+Requires-Dist: certifi
+Requires-Dist: choice>=0.1
+Requires-Dist: containerd
+Requires-Dist: cookiecutter>=1.4.0
+Requires-Dist: croniter
+Requires-Dist: docker
+Requires-Dist: dulwich>=0.17.3
+Requires-Dist: ephemeral-port-reserve>=1.0.1
+Requires-Dist: graphviz
+Requires-Dist: grpcio
+Requires-Dist: gunicorn
+Requires-Dist: humanfriendly
+Requires-Dist: humanize>=0.5.1
+Requires-Dist: inotify>=0.2.8
+Requires-Dist: ipaddress>=1.0.22
+Requires-Dist: isodate>=0.7.2
+Requires-Dist: jsonschema[format]
+Requires-Dist: kazoo>=2.0.0
+Requires-Dist: kubernetes<26.0.0,>=18.20.0
+Requires-Dist: ldap3
+Requires-Dist: manhole
+Requires-Dist: mypy-extensions>=0.3.0
+Requires-Dist: nats-py
+Requires-Dist: nulltype
+Requires-Dist: objgraph
+Requires-Dist: ply
+Requires-Dist: progressbar2>=3.10.0
+Requires-Dist: prometheus-client
+Requires-Dist: pymesos>=0.2.0
+Requires-Dist: pyramid>=1.8
+Requires-Dist: pyramid-swagger>=2.3.0
+Requires-Dist: pysensu-yelp>=0.3.4
+Requires-Dist: PyStaticConfiguration
+Requires-Dist: python-crontab>=2.1.1
+Requires-Dist: python-dateutil>=2.4.0
+Requires-Dist: python-iptables
+Requires-Dist: pytimeparse>=1.1.0
+Requires-Dist: pytz>=2014.10
+Requires-Dist: requests<2.32.0,>=2.18.4
+Requires-Dist: requests-cache>=0.4.10
+Requires-Dist: retry
+Requires-Dist: ruamel.yaml
+Requires-Dist: sensu-plugin
+Requires-Dist: service-configuration-lib>=3.3.2
+Requires-Dist: signalfx
+Requires-Dist: slackclient>=1.2.1
+Requires-Dist: sticht>=1.1.0
+Requires-Dist: syslogmp
+Requires-Dist: task-processing
+Requires-Dist: thriftpy2
+Requires-Dist: transitions
+Requires-Dist: typing-extensions
+Requires-Dist: tzlocal
+Requires-Dist: urllib3
+Requires-Dist: utaw>=0.2.0
+Requires-Dist: wsgicors

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/README.md

@@ -44,7 +44,7 @@ is a heavily customised version of SmartStack and Envoy, allows many systems at
 to communicate with PaaSTA services and each other.
 
 On the other hand, requiring lots of components, means lots of infrastructure to
-setup before PaaSTA can work effectively! Realistacally, running PaaSTA outside of Yelp
+setup before PaaSTA can work effectively! Realistically, running PaaSTA outside of Yelp
 would not be sensible, because in addition to the integrations mentioned above we also
 have strong opinions encoded in other tooling that you would need to replicate. Nevertheless,
 we code PaaSTA in the open because we think it is useful to share our approach and hope that

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.21.2"
+__version__ = "1.21.4"

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/cli/authentication.py

@@ -12,63 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from functools import lru_cache
-
-from botocore.credentials import InstanceMetadataFetcher
-from botocore.credentials import InstanceMetadataProvider
+from typing import Optional
 
 from paasta_tools.utils import load_system_paasta_config
 
 
 try:
-    from vault_tools.paasta_secret import get_client as get_vault_client
-    from vault_tools.paasta_secret import get_vault_url
-    from vault_tools.paasta_secret import get_vault_ca
+    from vault_tools.oidc import get_instance_oidc_identity_token
     from okta_auth import get_and_cache_jwt_default
 except ImportError:
 
-    def get_vault_client(url: str, capath: str) -> None:
-        pass
-
-    def get_vault_url(ecosystem: str) -> str:
-        return ""
-
-    def get_vault_ca(ecosystem: str) -> str:
+    def get_instance_oidc_identity_token(
+        role: str, ecosystem: Optional[str] = None
+    ) -> str:
         return ""
 
     def get_and_cache_jwt_default(client_id: str) -> str:
         return ""
 
 
-def get_current_ecosystem() -> str:
-    """Get current ecosystem from host configs, defaults to dev if no config is found"""
-    try:
-        with open("/nail/etc/ecosystem") as f:
-            return f.read().strip()
-    except IOError:
-        pass
-    return "devc"
-
-
 @lru_cache(maxsize=1)
 def get_service_auth_token() -> str:
     """Uses instance profile to authenticate with Vault and generate token for service authentication"""
-    ecosystem = get_current_ecosystem()
-    vault_client = get_vault_client(get_vault_url(ecosystem), get_vault_ca(ecosystem))
     vault_role = load_system_paasta_config().get_service_auth_vault_role()
-    metadata_provider = InstanceMetadataProvider(
-        iam_role_fetcher=InstanceMetadataFetcher(),
-    )
-    instance_credentials = metadata_provider.load().get_frozen_credentials()
-    vault_client.auth.aws.iam_login(
-        instance_credentials.access_key,
-        instance_credentials.secret_key,
-        instance_credentials.token,
-        mount_point="aws-iam",
-        role=vault_role,
-        use_token=True,
-    )
-    response = vault_client.secrets.identity.generate_signed_id_token(name=vault_role)
-    return response["data"]["token"]
+    return get_instance_oidc_identity_token(vault_role)
 
 
 def get_sso_auth_token(paasta_apis: bool = False) -> str:

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/cli/cmds/logs.py

@@ -178,7 +178,7 @@ def add_subparser(subparsers) -> None:
         help=(
             "The time to get logs up to. "
             'This can be an ISO-8601 timestamp or a human readable duration parsable by pytimeparse such as "5m", "1d3h" etc. '
-            "Incompatiable with --line-offset and --lines. "
+            "Incompatible with --line-offset and --lines. "
             "Defaults to now."
         ),
     )
@@ -190,7 +190,7 @@ def add_subparser(subparsers) -> None:
         help=(
             "The number of lines to retrieve from the specified offset. "
             'May optionally be prefixed with a "+" or "-" to specify which direction from the offset. '
-            "Incompatiable with --from and --to. "
+            "Incompatible with --from and --to. "
             'Defaults to "-100".'
         ),
         type=int,
@@ -204,7 +204,7 @@ def add_subparser(subparsers) -> None:
             "For example, --line-offset 1 would be the first line. "
             "Paired with --lines, --line-offset +100 would give you the first 100 lines of logs. "
             "Some logging backends may not support line offsetting by time or lines. "
-            "Incompatiable with --from and --to. "
+            "Incompatible with --from and --to. "
             "Defaults to the latest line's offset."
         ),
         type=int,
@@ -804,7 +804,7 @@ class ScribeLogReader(LogReader):
                     # sure all the tailers are still running.
                     running_processes = [tt.is_alive() for tt in spawned_processes]
                     if not running_processes or not all(running_processes):
-                        log.warn(
+                        log.warning(
                             "Quitting because I expected %d log tailers to be alive but only %d are alive."
                             % (len(spawned_processes), running_processes.count(True))
                         )
@@ -819,12 +819,12 @@ class ScribeLogReader(LogReader):
                     # This extra nested catch is because it's pretty easy to be in
                     # the above try block when the user hits Ctrl-C which otherwise
                     # dumps a stack trace.
-                    log.warn("Terminating.")
+                    log.warning("Terminating.")
                     break
             except KeyboardInterrupt:
                 # Die peacefully rather than printing N threads worth of stack
                 # traces.
-                log.warn("Terminating.")
+                log.warning("Terminating.")
                 break
 
     def print_logs_by_time(
@@ -1019,7 +1019,7 @@ class ScribeLogReader(LogReader):
             tzinfo=pytz.utc
         ) - datetime.timedelta(hours=4)
         if end_time > warning_end_time:
-            log.warn("Recent logs might be incomplete. Consider tailing instead.")
+            log.warning("Recent logs might be incomplete. Consider tailing instead.")
 
         # scribereader, sadly, is not based on UTC timestamps. It uses YST
         # dates instead.

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/cli/schemas/service_schema.json

@@ -10,7 +10,10 @@
             "type": "string"
         },
         "git_url": {
-            "type": "string",
+            "type": [
+                "string",
+                "null"
+            ],
             "pattern": "^git@github.yelpcorp.com:[-a-z]+/[-_a-z0-9]+(\\.git)?$",
             "$comment": "this is obviously very tied to how we name repos at Yelp"
         },

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/cli/utils.py

@@ -1097,7 +1097,9 @@ def get_paasta_oapi_client_with_auth(
     )
 
 
-def run_interactive_cli(cmd: str, shell: str = "bash", term: str = "xterm-256color"):
+def run_interactive_cli(
+    cmd: str, shell: str = "/bin/bash", term: str = "xterm-256color"
+):
     """Runs interactive command in a pseudo terminal, handling terminal size management
 
     :param str cmd: shell command
@@ -1105,6 +1107,8 @@ def run_interactive_cli(cmd: str, shell: str = "bash", term: str = "xterm-256col
     :param str term: terminal type
     """
     cols, rows = shutil.get_terminal_size()
+    if not os.path.isabs(shell):
+        shell = shutil.which(shell)
     wrapped_cmd = (
         f"export SHELL={shell};"
         f"export TERM={term};"

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/generate_deployments_for_service.py

@@ -130,6 +130,10 @@ def get_deploy_group_mappings(
     v2_mappings: V2_Mappings = {"deployments": {}, "controls": {}}
     git_url = get_git_url(service=service, soa_dir=soa_dir)
 
+    # Some pseudo-services like toolboxes explicitly have no git_url, and therefore no deployments
+    if git_url is None:
+        return mappings, v2_mappings
+
     # Most of the time of this function is in two parts:
     # 1. getting remote refs from git. (Mostly IO, just waiting for git to get back to us.)
     # 2. loading instance configs. (Mostly CPU, copy.deepcopying yaml over and over again)

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/setup_istio_mesh.py

@@ -103,7 +103,7 @@ def load_smartstack_namespaces(soa_dir: str = DEFAULT_SOA_DIR) -> Mapping:
                 for (ns, details) in svc_namespaces.items():
                     namespaces[f"{dir}.{ns}"] = details
         except Exception as err:
-            log.warn(f"Failed to load namespaces for {dir}: {err}")
+            log.warning(f"Failed to load namespaces for {dir}: {err}")
 
     return namespaces
 

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/smartstack_tools.py

@@ -473,7 +473,7 @@ def get_service_discovery_providers(
         elif name == "envoy":
             providers.append(EnvoyServiceDiscovery(system_paasta_config))
         else:
-            log.warn("unknown provider")
+            log.warning("unknown provider")
     return providers
 
 
@@ -530,7 +530,7 @@ class BaseReplicationChecker(ReplicationChecker):
                         )
                         break
                     except Exception as e:
-                        log.warn(
+                        log.warning(
                             f"Error while getting replication info for {location} from {hostname}: {e}"
                         )
                         if hostname == hostnames[-1]:

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools/spark_tools.py

@@ -133,7 +133,7 @@ def setup_volume_mounts(volumes: List[DockerVolume]) -> Dict[str, str]:
         )
 
         if host_path in seen_paths:
-            log.warn(f"Skipping {host_path} - already added a binding for it.")
+            log.warning(f"Skipping {host_path} - already added a binding for it.")
             continue
         seen_paths.add(host_path)
 
@@ -209,7 +209,7 @@ def auto_add_timeout_for_spark_job(
             f"'timeout' could not be added to spark command: '{cmd}' due to error '{e}'. "
             "Please report to #spark."
         )
-        log.warn(err_msg)
+        log.warning(err_msg)
         print(PaastaColors.red(err_msg))
     return cmd
 

paasta_tools-1.21.4/paasta_tools.egg-info/PKG-INFO

@@ -0,0 +1,73 @@
+Metadata-Version: 2.1
+Name: paasta-tools
+Version: 1.21.4
+Summary: Tools for Yelps SOA infrastructure
+Author: Compute Infrastructure @ Yelp
+Author-email: compute-infra@yelp.com
+Provides: paasta_tools
+Requires-Python: >=3.8.0
+License-File: LICENSE
+Requires-Dist: a-sync>=0.5.0
+Requires-Dist: aiohttp>=3.5.4
+Requires-Dist: argcomplete>=0.8.1
+Requires-Dist: boto
+Requires-Dist: boto3
+Requires-Dist: boto3-type-annotations
+Requires-Dist: botocore
+Requires-Dist: bravado>=10.2.0
+Requires-Dist: certifi
+Requires-Dist: choice>=0.1
+Requires-Dist: containerd
+Requires-Dist: cookiecutter>=1.4.0
+Requires-Dist: croniter
+Requires-Dist: docker
+Requires-Dist: dulwich>=0.17.3
+Requires-Dist: ephemeral-port-reserve>=1.0.1
+Requires-Dist: graphviz
+Requires-Dist: grpcio
+Requires-Dist: gunicorn
+Requires-Dist: humanfriendly
+Requires-Dist: humanize>=0.5.1
+Requires-Dist: inotify>=0.2.8
+Requires-Dist: ipaddress>=1.0.22
+Requires-Dist: isodate>=0.7.2
+Requires-Dist: jsonschema[format]
+Requires-Dist: kazoo>=2.0.0
+Requires-Dist: kubernetes<26.0.0,>=18.20.0
+Requires-Dist: ldap3
+Requires-Dist: manhole
+Requires-Dist: mypy-extensions>=0.3.0
+Requires-Dist: nats-py
+Requires-Dist: nulltype
+Requires-Dist: objgraph
+Requires-Dist: ply
+Requires-Dist: progressbar2>=3.10.0
+Requires-Dist: prometheus-client
+Requires-Dist: pymesos>=0.2.0
+Requires-Dist: pyramid>=1.8
+Requires-Dist: pyramid-swagger>=2.3.0
+Requires-Dist: pysensu-yelp>=0.3.4
+Requires-Dist: PyStaticConfiguration
+Requires-Dist: python-crontab>=2.1.1
+Requires-Dist: python-dateutil>=2.4.0
+Requires-Dist: python-iptables
+Requires-Dist: pytimeparse>=1.1.0
+Requires-Dist: pytz>=2014.10
+Requires-Dist: requests<2.32.0,>=2.18.4
+Requires-Dist: requests-cache>=0.4.10
+Requires-Dist: retry
+Requires-Dist: ruamel.yaml
+Requires-Dist: sensu-plugin
+Requires-Dist: service-configuration-lib>=3.3.2
+Requires-Dist: signalfx
+Requires-Dist: slackclient>=1.2.1
+Requires-Dist: sticht>=1.1.0
+Requires-Dist: syslogmp
+Requires-Dist: task-processing
+Requires-Dist: thriftpy2
+Requires-Dist: transitions
+Requires-Dist: typing-extensions
+Requires-Dist: tzlocal
+Requires-Dist: urllib3
+Requires-Dist: utaw>=0.2.0
+Requires-Dist: wsgicors

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools.egg-info/SOURCES.txt

@@ -299,4 +299,55 @@ paasta_tools/secret_providers/vault.py
 paasta_tools/tron/__init__.py
 paasta_tools/tron/client.py
 paasta_tools/tron/tron_command_context.py
-paasta_tools/tron/tron_timeutils.py
+paasta_tools/tron/tron_timeutils.py
+tests/test_adhoc_tools.py
+tests/test_apply_external_resources.py
+tests/test_async_utils.py
+tests/test_bounce_lib.py
+tests/test_check_flink_services_health.py
+tests/test_check_kubernetes_services_replication.py
+tests/test_check_oom_events.py
+tests/test_check_service_replication_tools.py
+tests/test_check_spark_jobs.py
+tests/test_cleanup_kubernetes_jobs.py
+tests/test_config_utils.py
+tests/test_delete_kubernetes_deployments.py
+tests/test_deployment_utils.py
+tests/test_docker_wrapper.py
+tests/test_drain_lib.py
+tests/test_eks_tools.py
+tests/test_envoy_tools.py
+tests/test_firewall.py
+tests/test_firewall_logging.py
+tests/test_firewall_update.py
+tests/test_flink_tools.py
+tests/test_flinkeks_tools.py
+tests/test_generate_authenticating_services.py
+tests/test_generate_deployments_for_service.py
+tests/test_generate_services_file.py
+tests/test_generate_services_yaml.py
+tests/test_hacheck.py
+tests/test_iptables.py
+tests/test_kubernetes_tools.py
+tests/test_list_kubernetes_service_instances.py
+tests/test_long_running_service_tools.py
+tests/test_mac_address.py
+tests/test_mesos_tools.py
+tests/test_monitoring_tools.py
+tests/test_nrtsearchserviceseks_tools.py
+tests/test_oom_logger.py
+tests/test_paasta_execute_docker_command.py
+tests/test_paasta_service_config_loader.py
+tests/test_puppet_service_tools.py
+tests/test_remote_git.py
+tests/test_secret_tools.py
+tests/test_setup_istio_mesh.py
+tests/test_setup_kubernetes_cr.py
+tests/test_setup_kubernetes_job.py
+tests/test_setup_prometheus_adapter_config.py
+tests/test_slack.py
+tests/test_smartstack_tools.py
+tests/test_spark_tools.py
+tests/test_task_processing.py
+tests/test_tron_tools.py
+tests/test_utils.py

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools.egg-info/entry_points.txt

@@ -18,4 +18,3 @@ paasta_setup_tron_namespace = paasta_tools.setup_tron_namespace:main
 
 [paste.app_factory]
 paasta-api-config = paasta_tools.api.api:make_app
-

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/paasta_tools.egg-info/requires.txt

@@ -49,7 +49,7 @@ requests-cache>=0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib>=3.2.0
+service-configuration-lib>=3.3.2
 signalfx
 slackclient>=1.2.1
 sticht>=1.1.0

{paasta-tools-1.21.2 → paasta_tools-1.21.4}/requirements-minimal.txt

@@ -56,7 +56,7 @@ requests-cache >= 0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib >= 3.2.0
+service-configuration-lib >= 3.3.2
 signalfx
 slackclient >= 1.2.1
 sticht >= 1.1.0

paasta_tools-1.21.4/tests/test_adhoc_tools.py

@@ -0,0 +1,66 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import mock
+
+from paasta_tools import adhoc_tools
+from paasta_tools.utils import DeploymentsJsonV2
+from paasta_tools.utils import NoConfigurationForServiceError
+
+
+def test_get_default_interactive_config():
+    with mock.patch(
+        "paasta_tools.adhoc_tools.load_adhoc_job_config", autospec=True
+    ) as mock_load_adhoc_job_config:
+        mock_load_adhoc_job_config.return_value = adhoc_tools.AdhocJobConfig(
+            service="fake_service",
+            instance="interactive",
+            cluster="fake_cluster",
+            config_dict={},
+            branch_dict={"deploy_group": "fake_deploy_group"},
+        )
+        result = adhoc_tools.get_default_interactive_config(
+            "fake_service", "fake_cluster", "/fake/soa/dir", load_deployments=False
+        )
+        assert result.get_cpus() == 4
+        assert result.get_mem() == 10240
+        assert result.get_disk() == 1024
+
+
+def test_get_default_interactive_config_reads_from_tty():
+    with mock.patch(
+        "paasta_tools.adhoc_tools.prompt_pick_one", autospec=True
+    ) as mock_prompt_pick_one, mock.patch(
+        "paasta_tools.adhoc_tools.load_adhoc_job_config", autospec=True
+    ) as mock_load_adhoc_job_config, mock.patch(
+        "paasta_tools.adhoc_tools.load_v2_deployments_json", autospec=True
+    ) as mock_load_deployments_json:
+        mock_prompt_pick_one.return_value = "fake_deploygroup"
+        mock_load_adhoc_job_config.side_effect = NoConfigurationForServiceError
+        mock_load_deployments_json.return_value = DeploymentsJsonV2(
+            service="fake-service",
+            config_dict={
+                "deployments": {
+                    "fake_deploygroup": {
+                        "docker_image": mock.sentinel.docker_image,
+                        "git_sha": mock.sentinel.git_sha,
+                    }
+                },
+                "controls": {},
+            },
+        )
+        result = adhoc_tools.get_default_interactive_config(
+            "fake_service", "fake_cluster", "/fake/soa/dir", load_deployments=True
+        )
+        assert result.get_deploy_group() == "fake_deploygroup"
+        assert result.get_docker_image() == mock.sentinel.docker_image