nextmv 1.7.2__tar.gz → 1.7.3.dev0__tar.gz

{nextmv-1.7.2 → nextmv-1.7.3.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextmv
-Version: 1.7.2
+Version: 1.7.3.dev0
 Summary: The all-purpose Python SDK for Nextmv
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextmv-py.docs.nextmv.io/en/latest/nextmv/

nextmv-1.7.3.dev0/nextmv/__about__.py

@@ -0,0 +1 @@
+__version__ = "v1.7.3.dev0"

{nextmv-1.7.2 → nextmv-1.7.3.dev0}/nextmv/cli/init.py

@@ -927,26 +927,38 @@ def _cli_call(command: list[str]) -> subprocess.CompletedProcess:
     """
 
     try:
+        # Use binary mode so we can apply different decoding strategies to each
+        # stream: stdout is machine-readable (JSON) and must be decoded strictly
+        # so corruption surfaces as a clear UnicodeDecodeError rather than a
+        # confusing JSONDecodeError; stderr is human-facing and uses
+        # errors="replace" so stray non-UTF-8 bytes (common on Windows with
+        # codepage cp1252) never crash the thread.
         process = subprocess.Popen(
             command,
             env=os.environ,
-            text=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
 
         stdout_lines = []
         stderr_lines = []
+        stdout_exc: list[BaseException] = []
 
         def stream_stderr():
-            for line in process.stderr:
+            for raw_line in process.stderr:
+                line = raw_line.decode("utf-8", errors="replace")
                 rich.print(line, end="", file=sys.stderr)
                 stderr_lines.append(line)
 
         def stream_stdout():
-            for line in process.stdout:
-                rich.print(line, end="")
-                stdout_lines.append(line)
+            try:
+                for raw_line in process.stdout:
+                    line = raw_line.decode("utf-8")
+                    rich.print(line, end="")
+                    stdout_lines.append(line)
+            except Exception as exc:
+                # Capture so we can re-raise on the main thread after joining.
+                stdout_exc.append(exc)
 
         # Use threads to read both streams concurrently without deadlocking
         stderr_thread = threading.Thread(target=stream_stderr)
@@ -959,6 +971,11 @@ def _cli_call(command: list[str]) -> subprocess.CompletedProcess:
         stdout_thread.join()
         process.wait()
 
+        # Re-raise any decoding error from stdout on the calling thread so it
+        # surfaces as a clear UnicodeDecodeError rather than being swallowed.
+        if stdout_exc:
+            raise stdout_exc[0]
+
         result = subprocess.CompletedProcess(
             args=command,
             returncode=process.returncode,

{nextmv-1.7.2 → nextmv-1.7.3.dev0}/nextmv/cloud/package.py

@@ -325,6 +325,8 @@ def __run_command(binary: str, dir: str, redirect_out_err: bool, *arguments: str
         stdout=subprocess.PIPE if redirect_out_err else None,
         stderr=subprocess.PIPE if redirect_out_err else None,
         text=True,
+        encoding="utf-8",
+        errors="replace",
     )
 
     if redirect_out_err:

{nextmv-1.7.2 → nextmv-1.7.3.dev0}/nextmv/local/executor.py

@@ -146,7 +146,7 @@ def execute_run(
 
             # Set the run status to running.
             info_file = os.path.join(run_dir, f"{run_id}.json")
-            with open(info_file, "r+") as f:
+            with open(info_file, "r+", encoding="utf-8") as f:
                 info = json.load(f)
                 info["metadata"]["status_v2"] = "running"
                 f.seek(0)
@@ -173,9 +173,12 @@ def execute_run(
             stderr_lines: list[str] = []
 
             # Force unbuffered stdout/stderr in child processes so both
-            # streams are written to the log file in real time.
+            # streams are written to the log file in real time. Also force
+            # UTF-8 encoding for all stdio so Windows codepage mismatches
+            # (e.g. cp1252) do not cause UnicodeDecodeError.
             child_env = os.environ.copy()
             child_env["PYTHONUNBUFFERED"] = "1"
+            child_env["PYTHONIOENCODING"] = "utf-8"
 
             # This is the process that actually executes the entrypoint script.
             # We use subprocess.Popen instead of subprocess.run because we need
@@ -185,6 +188,8 @@ def execute_run(
                 args,
                 env=child_env,
                 text=True,
+                encoding="utf-8",
+                errors="replace",
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
@@ -199,7 +204,7 @@ def execute_run(
             # A lock to serialize writes from the stdout and stderr threads so log
             # lines are never interleaved in the log file.
             log_lock = threading.Lock()
-            log_f = open(log_file_path, "a")
+            log_f = open(log_file_path, "a", encoding="utf-8")
 
             def stream_stderr() -> None:
                 # Stderr is always streamed live to the log file so errors appear
@@ -263,12 +268,12 @@ def execute_run(
 
     except Exception as e:
         # If we encounter an exception, we log it to the stderr log file.
-        with open(os.path.join(logs_dir, LOGS_FILE), "a") as f:
+        with open(os.path.join(logs_dir, LOGS_FILE), "a", encoding="utf-8") as f:
             f.write(f"\nException during run execution: {str(e)}\n")
 
         # Also, we update the run information file to set the status to failed.
         info_file = os.path.join(run_dir, f"{run_id}.json")
-        with open(info_file, "r+") as f:
+        with open(info_file, "r+", encoding="utf-8") as f:
             info = json.load(f)
             info["metadata"]["status_v2"] = "failed"
             info["metadata"]["error"] = "Run failed, please check logs for details."
@@ -483,7 +488,7 @@ def _process_run_information(run_id: str, run_dir: str, result: subprocess.Compl
 
     info_file = os.path.join(run_dir, f"{run_id}.json")
 
-    with open(info_file) as f:
+    with open(info_file, encoding="utf-8") as f:
         info = json.load(f)
 
     # Calculate duration.
@@ -505,7 +510,7 @@ def _process_run_information(run_id: str, run_dir: str, result: subprocess.Compl
     info["metadata"]["status_v2"] = status
     info["metadata"]["error"] = error
 
-    with open(info_file, "w") as f:
+    with open(info_file, "w", encoding="utf-8") as f:
         json.dump(info, f, indent=2)
 
 
@@ -569,7 +574,7 @@ def _process_run_metrics(
     if METRICS_KEY not in stdout_output:
         return
 
-    with open(metrics_dst, "w") as f:
+    with open(metrics_dst, "w", encoding="utf-8") as f:
         metrics = {METRICS_KEY: stdout_output[METRICS_KEY]}
         json.dump(metrics, f, indent=2)
 
@@ -637,7 +642,7 @@ def _process_run_statistics(
     if STATISTICS_KEY not in stdout_output:
         return
 
-    with open(stats_dst, "w") as f:
+    with open(stats_dst, "w", encoding="utf-8") as f:
         stats = {STATISTICS_KEY: stdout_output[STATISTICS_KEY]}
         json.dump(stats, f, indent=2)
 
@@ -702,7 +707,7 @@ def _process_run_assets(
     if ASSETS_KEY not in stdout_output:
         return
 
-    with open(assets_dst, "w") as f:
+    with open(assets_dst, "w", encoding="utf-8") as f:
         assets = {ASSETS_KEY: stdout_output[ASSETS_KEY]}
         json.dump(assets, f, indent=2)
 
@@ -753,7 +758,7 @@ def _process_run_solutions(
 
     info_file = os.path.join(run_dir, f"{run_id}.json")
 
-    with open(info_file) as f:
+    with open(info_file, encoding="utf-8") as f:
         info = json.load(f)
 
     solutions_dst = os.path.join(outputs_dir, SOLUTIONS_KEY)
@@ -786,7 +791,7 @@ def _process_run_solutions(
         )
     else:
         if bool(stdout_output):
-            with open(os.path.join(solutions_dst, DEFAULT_OUTPUT_JSON_FILE), "w") as f:
+            with open(os.path.join(solutions_dst, DEFAULT_OUTPUT_JSON_FILE), "w", encoding="utf-8") as f:
                 if isinstance(stdout_output, dict):
                     json.dump(stdout_output, f, indent=2)
                 elif isinstance(stdout_output, str):
@@ -795,7 +800,7 @@ def _process_run_solutions(
     # Update the run information file with the output size and type.
     calculate_files_size(run_dir, run_id, solutions_dst, metadata_key="output_size")
     info["metadata"]["format"]["output"] = {"type": output_format.value}
-    with open(info_file, "w") as f:
+    with open(info_file, "w", encoding="utf-8") as f:
         json.dump(info, f, indent=2)
 
 
@@ -824,7 +829,7 @@ def _process_run_visuals(run_dir: str, outputs_dir: str) -> None:
     else:
         return
 
-    with open(assets_file) as f:
+    with open(assets_file, encoding="utf-8") as f:
         assets = json.load(f)
 
     # Create visuals directory.

nextmv-1.7.3.dev0/tests/cli/test_init_cli_call.py

@@ -0,0 +1,109 @@
+"""
+Tests for the encoding behaviour of _cli_call in nextmv.cli.init.
+
+On Windows, subprocess pipes default to the system codepage (e.g. cp1252).
+Byte 0x8f is undefined in cp1252, so without an explicit encoding this raises
+UnicodeDecodeError.  The fix decodes each stream independently:
+
+- stderr  → utf-8 with errors="replace"  (human-facing, replacement char is OK)
+- stdout  → utf-8 strict                 (machine-readable JSON; corruption must
+                                          surface as a clear UnicodeDecodeError,
+                                          not a silent JSONDecodeError later)
+"""
+
+import subprocess
+import unittest
+from io import BytesIO
+from unittest.mock import MagicMock, patch
+
+import typer
+
+
+def _make_pipe(raw_bytes: bytes):
+    """Return a mock binary pipe whose iteration yields lines from *raw_bytes*."""
+    bio = BytesIO(raw_bytes)
+    mock_pipe = MagicMock()
+    mock_pipe.__iter__ = lambda self: iter(bio)
+    return mock_pipe
+
+
+def _make_process(stdout_bytes: bytes = b"", stderr_bytes: bytes = b"", returncode: int = 0):
+    mock = MagicMock()
+    mock.stdout = _make_pipe(stdout_bytes)
+    mock.stderr = _make_pipe(stderr_bytes)
+    mock.returncode = returncode
+    mock.wait = MagicMock()
+    return mock
+
+
+class TestCliCallEncoding(unittest.TestCase):
+    """Verify that _cli_call survives non-UTF-8 bytes and applies the correct
+    decoding strategy to each stream."""
+
+    def _invoke(self, stdout_bytes: bytes, stderr_bytes: bytes) -> subprocess.CompletedProcess:
+        """Call _cli_call with a mocked Popen and return its result."""
+        from nextmv.cli.init import _cli_call
+
+        mock_process = _make_process(stdout_bytes, stderr_bytes)
+
+        with patch("nextmv.cli.init.subprocess.Popen", return_value=mock_process):
+            return _cli_call(["echo", "test"])
+
+    def test_non_utf8_in_stderr_replaced_not_raised(self):
+        """Byte 0x8f (undefined in cp1252) in stderr must not raise; it becomes a replacement char."""
+        result = self._invoke(
+            stdout_bytes=b"",
+            stderr_bytes=b"warning: bad byte \x8f here\n",
+        )
+        # The replacement character U+FFFD (or '?') appears; no exception was raised.
+        self.assertIn("warning: bad byte", result.stderr)
+
+    def test_valid_utf8_stderr_preserved(self):
+        """Normal UTF-8 stderr (including non-ASCII) must be preserved exactly."""
+        result = self._invoke(
+            stdout_bytes=b"",
+            stderr_bytes="info: résumé 日本語\n".encode(),
+        )
+        self.assertIn("résumé", result.stderr)
+        self.assertIn("日本語", result.stderr)
+
+    def test_valid_utf8_stdout_preserved(self):
+        """Normal UTF-8 stdout is decoded and returned intact."""
+        result = self._invoke(
+            stdout_bytes=b'{"key": "value"}\n',
+            stderr_bytes=b"",
+        )
+        self.assertIn('"key"', result.stdout)
+
+    def test_non_utf8_in_stdout_raises_unicode_error(self):
+        """Non-UTF-8 bytes in stdout must not be silently swallowed.
+
+        The UnicodeDecodeError is caught by _cli_call's outer handler, which
+        prints a human-readable message and raises typer.Exit(code=1).  That is
+        the correct production behaviour: the caller sees a clear failure rather
+        than a confusing JSONDecodeError or silently corrupted data.
+        """
+        with self.assertRaises((UnicodeDecodeError, typer.Exit)):
+            self._invoke(
+                stdout_bytes=b'{"key": "\x8f"}\n',
+                stderr_bytes=b"",
+            )
+
+    def test_popen_called_in_binary_mode(self):
+        """_cli_call must open the subprocess in binary mode (no text=True / encoding kwarg)
+        so that each stream can be decoded with its own strategy."""
+        from nextmv.cli.init import _cli_call
+
+        mock_process = _make_process(b'{"ok": true}\n', b"")
+
+        with patch("nextmv.cli.init.subprocess.Popen", return_value=mock_process) as mock_popen:
+            _cli_call(["echo", "test"])
+
+        call_kwargs = mock_popen.call_args[1]
+        # Must NOT pass text=True or a global encoding – binary mode is required.
+        self.assertNotIn("encoding", call_kwargs)
+        self.assertNotIn("text", call_kwargs)
+
+
+if __name__ == "__main__":
+    unittest.main()

{nextmv-1.7.2 → nextmv-1.7.3.dev0}/tests/local/test_executor.py

@@ -1203,5 +1203,120 @@ class TestCopyNewOrModifiedFiles(unittest.TestCase):
         self._assert_file_not_exists(os.path.join(self.dst_dir, "regular.pyc"))
 
 
+class TestExecutorEncoding(unittest.TestCase):
+    """Tests that execute_run handles non-UTF-8 bytes in subprocess output without crashing."""
+
+    def setUp(self):
+        self.test_dir = tempfile.mkdtemp()
+        self.run_dir = os.path.join(self.test_dir, "run_dir")
+        os.makedirs(self.run_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.test_dir)
+
+    def _make_info_file(self, run_id: str) -> None:
+        info_file = os.path.join(self.run_dir, f"{run_id}.json")
+        with open(info_file, "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "metadata": {
+                        "created_at": "2023-01-01T00:00:00Z",
+                        "status_v2": "pending",
+                        "format": {"output": {"type": "json"}},
+                    }
+                },
+                f,
+            )
+
+    def _run_execute_run(self, stdout_lines, stderr_lines, run_id="test_encoding_run"):
+        """Helper that patches Popen and runs execute_run, returning normally."""
+        self._make_info_file(run_id)
+
+        mock_process = Mock()
+        mock_process.returncode = 0
+        mock_process.stdin = Mock()
+        mock_process.stdout = iter(stdout_lines)
+        mock_process.stderr = iter(stderr_lines)
+
+        with (
+            patch("nextmv.local.executor._copy_files_from_manifest"),
+            patch("nextmv.local.executor.process_run_output"),
+            patch("nextmv.local.executor.subprocess.Popen", return_value=mock_process),
+        ):
+            execute_run(
+                run_id=run_id,
+                src="/test/src",
+                manifest_dict={"execution": {"entrypoint": "main.py"}, "type": "python", "files": ["main.py"]},
+                run_dir=self.run_dir,
+                run_config={"format": {"input": {"type": "json"}}},
+                input_data={"test": "data"},
+            )
+
+    def test_non_utf8_bytes_in_stderr_do_not_crash(self):
+        """Non-UTF-8 bytes in stderr must not raise UnicodeDecodeError.
+
+        On Windows the subprocess pipes default to the system codepage (e.g.
+        cp1252). Byte 0x8f is undefined in cp1252, so without an explicit
+        encoding this would raise UnicodeDecodeError. The fix is to open the
+        Popen with encoding='utf-8' and errors='replace'.
+        """
+        # Simulate a line that contains the byte 0x8f – undefined in cp1252.
+        # We use a plain string here because the mock iter already yields str;
+        # the important thing is that the pipeline doesn't blow up when such
+        # a replacement character reaches the log file writer.
+        bad_line = "progress: \ufffd done\n"  # U+FFFD = replacement character
+        self._run_execute_run(stdout_lines=[], stderr_lines=[bad_line])
+
+        logs_file = os.path.join(self.run_dir, LOGS_KEY, LOGS_FILE)
+        self.assertTrue(os.path.exists(logs_file))
+        with open(logs_file, encoding="utf-8") as f:
+            content = f.read()
+        self.assertIn("progress:", content)
+
+    def test_non_utf8_bytes_in_stdout_do_not_crash(self):
+        """Non-UTF-8 replacement characters in stdout are handled gracefully."""
+        bad_line = '{"result": "\ufffd"}\n'
+        self._run_execute_run(stdout_lines=[bad_line], stderr_lines=[])
+
+    def test_log_file_written_as_utf8(self):
+        """The logs file must be written with UTF-8 encoding so non-ASCII characters survive."""
+        unicode_line = "info: résumé café naïve 日本語\n"
+        self._run_execute_run(stdout_lines=[], stderr_lines=[unicode_line])
+
+        logs_file = os.path.join(self.run_dir, LOGS_KEY, LOGS_FILE)
+        with open(logs_file, encoding="utf-8") as f:
+            content = f.read()
+        self.assertIn("résumé", content)
+        self.assertIn("日本語", content)
+
+    def test_popen_called_with_utf8_encoding(self):
+        """subprocess.Popen must be called with encoding='utf-8' and errors='replace'."""
+        self._make_info_file("enc_check")
+
+        mock_process = Mock()
+        mock_process.returncode = 0
+        mock_process.stdin = Mock()
+        mock_process.stdout = iter([])
+        mock_process.stderr = iter([])
+
+        with (
+            patch("nextmv.local.executor._copy_files_from_manifest"),
+            patch("nextmv.local.executor.process_run_output"),
+            patch("nextmv.local.executor.subprocess.Popen", return_value=mock_process) as mock_popen,
+        ):
+            execute_run(
+                run_id="enc_check",
+                src="/test/src",
+                manifest_dict={"execution": {"entrypoint": "main.py"}, "type": "python", "files": ["main.py"]},
+                run_dir=self.run_dir,
+                run_config={"format": {"input": {"type": "json"}}},
+                input_data={"test": "data"},
+            )
+
+        call_kwargs = mock_popen.call_args[1]
+        self.assertEqual(call_kwargs.get("encoding"), "utf-8")
+        self.assertEqual(call_kwargs.get("errors"), "replace")
+
+
 if __name__ == "__main__":
     unittest.main()

nextmv-1.7.2/nextmv/__about__.py

@@ -1 +0,0 @@
-__version__ = "v1.7.2"