pyinfra 3.0.dev0__py2.py3-none-any.whl → 3.0.2__py2.py3-none-any.whl

tests/test_cli/test_cli_deploy.py

@@ -9,6 +9,14 @@ from .util import run_cli
 
 
 class TestCliDeployState(PatchSSHTestCase):
+    def _run_cli(self, hosts, filename):
+        return run_cli(
+            "-y",
+            ",".join(hosts),
+            path.join("tests", "test_cli", "deploy", filename),
+            f'--chdir={path.join("tests", "test_cli", "deploy")}',
+        )
+
     def _assert_op_data(self, correct_op_name_and_host_names):
         op_order = state.get_op_order()
 
@@ -25,10 +33,14 @@ class TestCliDeployState(PatchSSHTestCase):
             assert list(op_meta.names)[0] == correct_op_name
 
             for host in state.inventory:
+                executed = False
+                host_op = state.ops[host].get(op_hash)
+                if host_op:
+                    executed = host_op.operation_meta.executed
                 if correct_host_names is True or host.name in correct_host_names:
-                    self.assertIn(op_hash, host.op_hash_order)
+                    assert executed is True
                 else:
-                    self.assertNotIn(op_hash, host.op_hash_order)
+                    assert executed is False
 
     def test_deploy(self):
         task_file_path = path.join("tasks", "a_task.py")
@@ -64,6 +76,7 @@ class TestCliDeployState(PatchSSHTestCase):
             ("Nested order loop 2/1", ("somehost", "anotherhost")),
             ("Nested order loop 2/2", ("somehost", "anotherhost")),
             ("Final limited operation", ("somehost",)),
+            ("Second final limited operation", ("anotherhost", "someotherhost")),
         ]
 
         # Run 3 iterations of the test - each time shuffling the order of the
@@ -74,12 +87,7 @@ class TestCliDeployState(PatchSSHTestCase):
             hosts = ["somehost", "anotherhost", "someotherhost"]
             shuffle(hosts)
 
-            result = run_cli(
-                "-y",
-                ",".join(hosts),
-                path.join("tests", "deploy", "deploy.py"),
-                f'--chdir={path.join("tests", "deploy")}',
-            )
+            result = self._run_cli(hosts, "deploy.py")
             assert result.exit_code == 0, result.stdout
 
             self._assert_op_data(correct_op_name_and_host_names)
@@ -104,12 +112,7 @@ class TestCliDeployState(PatchSSHTestCase):
             hosts = ["somehost", "anotherhost", "someotherhost"]
             shuffle(hosts)
 
-            result = run_cli(
-                "-y",
-                ",".join(hosts),
-                path.join("tests", "deploy", "deploy_random.py"),
-                f'--chdir={path.join("tests", "deploy")}',
-            )
+            result = self._run_cli(hosts, "deploy_random.py")
             assert result.exit_code == 0, result.stdout
 
             self._assert_op_data(correct_op_name_and_host_names)

tests/test_cli/test_cli_exceptions.py

@@ -1,34 +1,27 @@
+import sys
+from os import path
 from unittest import TestCase
 
+import pytest
 from click.testing import CliRunner
 
-from pyinfra_cli.exceptions import CliError
+from pyinfra.api import OperationError
+from pyinfra.api.exceptions import ArgumentTypeError
+from pyinfra_cli.exceptions import CliError, UnexpectedExternalError, WrappedError
 from pyinfra_cli.main import cli
 
+from .util import run_cli
+
 
 class TestCliExceptions(TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.test_cli = CliRunner()
-        cls.old_cli_show = CliError.show
-
-    @classmethod
-    def tearDownClass(cls):
-        CliError.show = cls.old_cli_show
-
-    def setUp(self):
-        self.exception = None
-        CliError.show = lambda e: self.capture_cli_error(e)
-
-    def capture_cli_error(self, e):
-        self.exception = e
-        self.old_cli_show()
+        cls.runner = CliRunner()
 
     def assert_cli_exception(self, args, message):
-        self.test_cli.invoke(cli, args)
-
-        self.assertIsInstance(self.exception, CliError)
-        assert self.exception.message == message
+        result = self.runner.invoke(cli, args, standalone_mode=False)
+        self.assertIsInstance(result.exception, CliError)
+        assert getattr(result.exception, "message") == message
 
     def test_bad_deploy_file(self):
         self.assert_cli_exception(
@@ -53,3 +46,41 @@ class TestCliExceptions(TestCase):
             ["my-server.net", "fact", "server.NotAFact"],
             "No such attribute in module pyinfra.facts.server: NotAFact",
         )
+
+
+class TestCliDeployExceptions(TestCase):
+    def _run_cli(self, hosts, filename):
+        return run_cli(
+            "-y",
+            ",".join(hosts),
+            path.join("tests", "test_cli", "deploy_fails", filename),
+            f'--chdir={path.join("tests", "test_cli", "deploy_fails")}',
+        )
+
+    def test_invalid_argument_type(self):
+        result = self._run_cli(["@local"], "invalid_argument_type.py")
+        assert isinstance(result.exception, WrappedError)
+        assert isinstance(result.exception.exception, ArgumentTypeError)
+        assert (
+            result.exception.exception.args[0]
+            == "Invalid argument `_sudo`:: None is not an instance of bool"
+        )
+
+    def test_invalid_operation_arg(self):
+        result = self._run_cli(["@local"], "invalid_operation_arg.py")
+        assert isinstance(result.exception, UnexpectedExternalError)
+        assert isinstance(result.exception.exception, TypeError)
+        assert result.exception.filename == "invalid_operation_arg.py"
+        assert result.exception.exception.args[0] == "missing a required argument: 'commands'"
+
+    @pytest.mark.skipif(
+        sys.platform.startswith("win"),
+        reason="The operation is not compatible with Windows",
+    )
+    def test_operation_error(self):
+        result = self._run_cli(["@local"], "operation_error.py")
+        assert isinstance(result.exception, WrappedError)
+        assert isinstance(result.exception.exception, OperationError)
+        assert (
+            result.exception.exception.args[0] == "operation_error.py exists and is not a directory"
+        )

tests/test_cli/test_cli_inventory.py

@@ -0,0 +1,66 @@
+from os import path
+
+from pyinfra import inventory
+from pyinfra.context import ctx_inventory, ctx_state
+
+from ..paramiko_util import PatchSSHTestCase
+from .util import run_cli
+
+
+class TestCliInventory(PatchSSHTestCase):
+    def test_load_deploy_group_data(self):
+        ctx_state.reset()
+        ctx_inventory.reset()
+
+        hosts = ["somehost", "anotherhost", "someotherhost"]
+        result = run_cli(
+            "-y",
+            ",".join(hosts),
+            path.join("tests", "test_cli", "deploy", "deploy.py"),
+            f'--chdir={path.join("tests", "test_cli", "deploy")}',
+        )
+        assert result.exit_code == 0, result.stdout
+
+        assert inventory.data.get("hello") == "world"
+        assert "leftover_data" in inventory.group_data
+        assert inventory.group_data["leftover_data"].get("still_parsed") == "never_used"
+        assert inventory.group_data["leftover_data"].get("_global_arg") == "gets_parsed"
+
+    def test_load_group_data(self):
+        ctx_state.reset()
+        ctx_inventory.reset()
+
+        hosts = ["somehost", "anotherhost", "someotherhost"]
+        result = run_cli(
+            "-y",
+            ",".join(hosts),
+            f'--group-data={path.join("tests", "test_cli", "deploy", "group_data")}',
+            "exec",
+            "uptime",
+        )
+        assert result.exit_code == 0, result.stdout
+
+        assert inventory.data.get("hello") == "world"
+        assert "leftover_data" in inventory.group_data
+        assert inventory.group_data["leftover_data"].get("still_parsed") == "never_used"
+        assert inventory.group_data["leftover_data"].get("_global_arg") == "gets_parsed"
+
+    def test_load_group_data_file(self):
+        ctx_state.reset()
+        ctx_inventory.reset()
+
+        hosts = ["somehost", "anotherhost", "someotherhost"]
+        filename = path.join("tests", "test_cli", "deploy", "group_data", "leftover_data.py")
+        result = run_cli(
+            "-y",
+            ",".join(hosts),
+            f"--group-data={filename}",
+            "exec",
+            "uptime",
+        )
+        assert result.exit_code == 0, result.stdout
+
+        assert "hello" not in inventory.data
+        assert "leftover_data" in inventory.group_data
+        assert inventory.group_data["leftover_data"].get("still_parsed") == "never_used"
+        assert inventory.group_data["leftover_data"].get("_global_arg") == "gets_parsed"

tests/test_cli/util.py

@@ -10,7 +10,7 @@ def run_cli(*arguments):
     cwd = getcwd()
     pyinfra.is_cli = True
     runner = CliRunner()
-    result = runner.invoke(cli, arguments)
+    result = runner.invoke(cli, arguments, standalone_mode=False)
     pyinfra.is_cli = False
     chdir(cwd)
     return result

tests/test_connectors/test_dockerssh.py

@@ -31,6 +31,9 @@ def fake_ssh_docker_shell(
     if str(command).startswith("rm -f"):
         return (True, CommandOutput([]))
 
+    if "$TMPDIR" in str(command):
+        return (True, CommandOutput([]))
+
     # This is a bit messy. But it's easier than trying to swap out a mock
     # when it needs to be used...
     if fake_ssh_docker_shell.custom_command:
@@ -150,10 +153,10 @@ class TestDockerSSHConnector(TestCase):
         ]
 
         inventory = make_inventory(hosts=("@dockerssh/somehost:not-an-image",))
-        state = State(inventory, Config())
-        state.get_temp_filename = lambda _: "remote_tempfile"
+        State(inventory, Config())
 
         host = inventory.get_host("@dockerssh/somehost:not-an-image")
+        host.get_temp_filename = lambda _: "remote_tempfile"
         host.connect()
 
         host.put_file("not-a-file", "not-another-file", print_output=True)
@@ -167,10 +170,10 @@ class TestDockerSSHConnector(TestCase):
     @patch("pyinfra.connectors.ssh.SSHConnector.put_file")
     def test_put_file_error(self, fake_put_file):
         inventory = make_inventory(hosts=("@dockerssh/somehost:not-an-image",))
-        state = State(inventory, Config())
-        state.get_temp_filename = lambda _: "remote_tempfile"
+        State(inventory, Config())
 
         host = inventory.get_host("@dockerssh/somehost:not-an-image")
+        host.get_temp_filename = lambda _: "remote_tempfile"
         host.connect()
 
         # SSH error
@@ -199,10 +202,10 @@ class TestDockerSSHConnector(TestCase):
         ]
 
         inventory = make_inventory(hosts=("@dockerssh/somehost:not-an-image",))
-        state = State(inventory, Config())
-        state.get_temp_filename = lambda _: "remote_tempfile"
+        State(inventory, Config())
 
         host = inventory.get_host("@dockerssh/somehost:not-an-image")
+        host.get_temp_filename = lambda _: "remote_tempfile"
         host.connect()
 
         host.get_file("not-a-file", "not-another-file", print_output=True)
@@ -222,10 +225,10 @@ class TestDockerSSHConnector(TestCase):
         ]
 
         inventory = make_inventory(hosts=("@dockerssh/somehost:not-an-image",))
-        state = State(inventory, Config())
-        state.get_temp_filename = lambda _: "remote_tempfile"
+        State(inventory, Config())
 
         host = inventory.get_host("@dockerssh/somehost:not-an-image")
+        host.get_temp_filename = lambda _: "remote_tempfile"
         host.connect()
 
         fake_get_file.return_value = True

tests/test_connectors/test_ssh.py

@@ -102,10 +102,10 @@ class TestSSHConnector(TestCase):
                 pkey=fake_key,
                 timeout=10,
                 username="vagrant",
-                _pyinfra_ssh_forward_agent=None,
+                _pyinfra_ssh_forward_agent=False,
                 _pyinfra_ssh_config_file=None,
                 _pyinfra_ssh_known_hosts_file=None,
-                _pyinfra_ssh_strict_host_key_checking=None,
+                _pyinfra_ssh_strict_host_key_checking="accept-new",
                 _pyinfra_ssh_paramiko_connect_kwargs=None,
             )
 
@@ -257,10 +257,10 @@ class TestSSHConnector(TestCase):
                 pkey=fake_key,
                 timeout=10,
                 username="vagrant",
-                _pyinfra_ssh_forward_agent=None,
+                _pyinfra_ssh_forward_agent=False,
                 _pyinfra_ssh_config_file=None,
                 _pyinfra_ssh_known_hosts_file=None,
-                _pyinfra_ssh_strict_host_key_checking=None,
+                _pyinfra_ssh_strict_host_key_checking="accept-new",
                 _pyinfra_ssh_paramiko_connect_kwargs=None,
             )
 
@@ -316,10 +316,10 @@ class TestSSHConnector(TestCase):
                 pkey=fake_dss_key,
                 timeout=10,
                 username="vagrant",
-                _pyinfra_ssh_forward_agent=None,
+                _pyinfra_ssh_forward_agent=False,
                 _pyinfra_ssh_config_file=None,
                 _pyinfra_ssh_known_hosts_file=None,
-                _pyinfra_ssh_strict_host_key_checking=None,
+                _pyinfra_ssh_strict_host_key_checking="accept-new",
                 _pyinfra_ssh_paramiko_connect_kwargs=None,
             )
 
@@ -451,14 +451,24 @@ class TestSSHConnector(TestCase):
 
     @patch("pyinfra.connectors.util.getpass")
     @patch("pyinfra.connectors.ssh.SSHClient")
-    def test_run_shell_command_sudo_password_prompt(
+    def test_run_shell_command_sudo_password_automatic_prompt(
         self,
         fake_ssh_client,
         fake_getpass,
     ):
         fake_ssh = MagicMock()
-        fake_stdout = MagicMock()
-        fake_ssh.exec_command.return_value = MagicMock(), fake_stdout, MagicMock()
+        first_fake_stdout = MagicMock()
+        second_fake_stdout = MagicMock()
+        third_fake_stdout = MagicMock()
+
+        first_fake_stdout.__iter__.return_value = ["sudo: a password is required\r"]
+        second_fake_stdout.__iter__.return_value = ["/tmp/pyinfra-sudo-askpass-XXXXXXXXXXXX"]
+
+        fake_ssh.exec_command.side_effect = [
+            (MagicMock(), first_fake_stdout, MagicMock()),  # command w/o sudo password
+            (MagicMock(), second_fake_stdout, MagicMock()),  # SUDO_ASKPASS_COMMAND
+            (MagicMock(), third_fake_stdout, MagicMock()),  # command with sudo pw
+        ]
 
         fake_ssh_client.return_value = fake_ssh
         fake_getpass.return_value = "password"
@@ -469,17 +479,18 @@ class TestSSHConnector(TestCase):
         host.connect()
 
         command = "echo Šablony"
-        fake_stdout.__iter__.return_value = ["/tmp/pyinfra-sudo-askpass-XXXXXXXXXXXX"]
-        fake_stdout.channel.recv_exit_status.return_value = 0
+        first_fake_stdout.channel.recv_exit_status.return_value = 1
+        second_fake_stdout.channel.recv_exit_status.return_value = 0
+        third_fake_stdout.channel.recv_exit_status.return_value = 0
 
-        out = host.run_shell_command(
-            command, _sudo=True, _use_sudo_password=True, print_output=True
-        )
+        out = host.run_shell_command(command, _sudo=True, print_output=True)
         assert len(out) == 2
 
         status, output = out
         assert status is True
 
+        fake_ssh.exec_command.assert_any_call(("sudo -H -n sh -c 'echo Šablony'"), get_pty=False)
+
         fake_ssh.exec_command.assert_called_with(
             (
                 "env SUDO_ASKPASS=/tmp/pyinfra-sudo-askpass-XXXXXXXXXXXX "
@@ -491,7 +502,7 @@ class TestSSHConnector(TestCase):
 
     @patch("pyinfra.connectors.util.getpass")
     @patch("pyinfra.connectors.ssh.SSHClient")
-    def test_run_shell_command_sudo_password_automatic_prompt(
+    def test_run_shell_command_sudo_password_automatic_prompt_with_special_chars_in_password(
         self,
         fake_ssh_client,
         fake_getpass,
@@ -511,7 +522,7 @@ class TestSSHConnector(TestCase):
         ]
 
         fake_ssh_client.return_value = fake_ssh
-        fake_getpass.return_value = "password"
+        fake_getpass.return_value = "p@ss'word';"
 
         inventory = make_inventory(hosts=("somehost",))
         State(inventory, Config())
@@ -534,7 +545,7 @@ class TestSSHConnector(TestCase):
         fake_ssh.exec_command.assert_called_with(
             (
                 "env SUDO_ASKPASS=/tmp/pyinfra-sudo-askpass-XXXXXXXXXXXX "
-                "PYINFRA_SUDO_PASSWORD=password "
+                """PYINFRA_SUDO_PASSWORD='p@ss'"'"'word'"'"';' """
                 "sudo -H -A -k sh -c 'echo Šablony'"
             ),
             get_pty=False,
@@ -544,13 +555,13 @@ class TestSSHConnector(TestCase):
     #
 
     @patch("pyinfra.connectors.ssh.SSHClient")
-    @patch("pyinfra.connectors.util._get_sudo_password")
+    @patch("pyinfra.connectors.util.getpass")
     def test_run_shell_command_retry_for_sudo_password(
         self,
-        fake_get_sudo_password,
+        fake_getpass,
         fake_ssh_client,
     ):
-        fake_get_sudo_password.return_value = "PASSWORD"
+        fake_getpass.return_value = "PASSWORD"
 
         fake_ssh = MagicMock()
         fake_stdin = MagicMock()
@@ -570,13 +581,13 @@ class TestSSHConnector(TestCase):
         return_values = [1, 0]  # return 0 on the second call
         fake_stdout.channel.recv_exit_status.side_effect = lambda: return_values.pop(0)
 
-        out = host.run_shell_command(command)
+        out = host.run_shell_command(command, _sudo=True)
         assert len(out) == 2
         assert out[0] is True
-        assert fake_get_sudo_password.called
+        assert fake_getpass.called
         fake_ssh.exec_command.assert_called_with(
             "env SUDO_ASKPASS=/tmp/pyinfra-sudo-askpass-XXXXXXXXXXXX "
-            "PYINFRA_SUDO_PASSWORD=PASSWORD sh -c 'echo hi'",
+            "PYINFRA_SUDO_PASSWORD=PASSWORD sudo -H -A -k sh -c 'echo hi'",
             get_pty=False,
         )
 
@@ -1041,3 +1052,57 @@ class TestSSHConnector(TestCase):
                     "not-another-file",
                     print_output=True,
                 )
+
+    @patch("pyinfra.connectors.ssh.SSHClient")
+    @patch("pyinfra.connectors.ssh.sleep")
+    def test_ssh_connect_fail_retry(self, fake_sleep, fake_ssh_client):
+        for exception_class in (
+            SSHException,
+            gaierror,
+            socket_error,
+            EOFError,
+        ):
+            fake_sleep.reset_mock()
+            fake_ssh_client.reset_mock()
+
+            inventory = make_inventory(
+                hosts=("unresposivehost",), override_data={"ssh_connect_retries": 1}
+            )
+            State(inventory, Config())
+
+            unresposivehost = inventory.get_host("unresposivehost")
+            assert unresposivehost.data.ssh_connect_retries == 1
+
+            fake_ssh_client().connect.side_effect = exception_class()
+
+            with self.assertRaises(ConnectError):
+                unresposivehost.connect(show_errors=False, raise_exceptions=True)
+
+            fake_sleep.assert_called_once()
+            assert fake_ssh_client().connect.call_count == 2
+
+    @patch("pyinfra.connectors.ssh.SSHClient")
+    @patch("pyinfra.connectors.ssh.sleep")
+    def test_ssh_connect_fail_success(self, fake_sleep, fake_ssh_client):
+        for exception_class in (
+            SSHException,
+            gaierror,
+            socket_error,
+            EOFError,
+        ):
+            fake_sleep.reset_mock()
+            fake_ssh_client.reset_mock()
+
+            inventory = make_inventory(
+                hosts=("unresposivehost",), override_data={"ssh_connect_retries": 1}
+            )
+            State(inventory, Config())
+
+            unresposivehost = inventory.get_host("unresposivehost")
+            assert unresposivehost.data.ssh_connect_retries == 1
+
+            fake_ssh_client().connect.side_effect = [exception_class(), MagicMock()]
+
+            unresposivehost.connect(show_errors=False, raise_exceptions=True)
+            fake_sleep.assert_called_once()
+            assert fake_ssh_client().connect.call_count == 2

tests/test_connectors/test_sshuserclient.py

@@ -21,7 +21,7 @@ Include other_file
 SSH_CONFIG_OTHER_FILE = """
 Host 192.168.1.1
     User "otheruser"
-    ProxyCommand None
+    # ProxyCommand None # Commented to get test passing with Paramiko > 3
     ForwardAgent yes
     UserKnownHostsFile ~/.ssh/test3
 """

tests/test_connectors/test_terraform.py

@@ -7,20 +7,23 @@ from pyinfra.connectors.terraform import TerraformInventoryConnector
 
 
 class TestTerraformConnector(TestCase):
-    def test_make_names_data_no_output_key(self):
-        with self.assertRaises(InventoryError) as context:
-            list(TerraformInventoryConnector.make_names_data())
-
-        assert context.exception.args[0] == "No Terraform output key!"
-
     @patch("pyinfra.connectors.terraform.local.shell")
     def test_make_names_data_no_output(self, fake_shell):
-        fake_shell.return_value = json.dumps({})
+        fake_shell.return_value = json.dumps(
+            {
+                "hello": {
+                    "world": [],
+                },
+            },
+        )
 
         with self.assertRaises(InventoryError) as context:
             list(TerraformInventoryConnector.make_names_data("output_key"))
 
-        assert context.exception.args[0] == "No Terraform output with key: `output_key`"
+        assert (
+            context.exception.args[0]
+            == "No Terraform output with key: `output_key`, valid keys:\n   - hello.world"
+        )
 
     @patch("pyinfra.connectors.terraform.local.shell")
     def test_make_names_data_invalid_output(self, fake_shell):

tests/test_connectors/test_vagrant.py

@@ -69,13 +69,13 @@ class TestVagrantConnector(TestCase):
     )
     @patch("pyinfra.connectors.vagrant.path.exists", lambda path: True)
     def test_make_names_data_with_options(self):
-        data = VagrantInventoryConnector.make_names_data()
+        data = list(VagrantInventoryConnector.make_names_data())
 
         assert data == [
             (
                 "@vagrant/ubuntu16",
                 {
-                    "ssh_port": "2222",
+                    "ssh_port": 2222,
                     "ssh_user": "vagrant",
                     "ssh_hostname": "127.0.0.1",
                     "ssh_key": "path/to/key",
@@ -85,7 +85,7 @@ class TestVagrantConnector(TestCase):
             (
                 "@vagrant/centos7",
                 {
-                    "ssh_port": "2200",
+                    "ssh_port": 2200,
                     "ssh_user": "vagrant",
                     "ssh_hostname": "127.0.0.1",
                     "ssh_key": "path/to/key",
@@ -103,13 +103,13 @@ class TestVagrantConnector(TestCase):
         ]
 
     def test_make_names_data_with_limit(self):
-        data = VagrantInventoryConnector.make_names_data(limit=("ubuntu16",))
+        data = list(VagrantInventoryConnector.make_names_data(name=("ubuntu16",)))
 
         assert data == [
             (
                 "@vagrant/ubuntu16",
                 {
-                    "ssh_port": "2222",
+                    "ssh_port": 2222,
                     "ssh_user": "vagrant",
                     "ssh_hostname": "127.0.0.1",
                     "ssh_key": "path/to/key",
@@ -120,4 +120,4 @@ class TestVagrantConnector(TestCase):
 
     def test_make_names_data_no_matches(self):
         with self.assertRaises(InventoryError):
-            VagrantInventoryConnector.make_names_data(limit="nope")
+            list(VagrantInventoryConnector.make_names_data(name="nope"))