nextmv 1.4.2.dev3__tar.gz → 1.5.0__tar.gz

{nextmv-1.4.2.dev3 → nextmv-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextmv
-Version: 1.4.2.dev3
+Version: 1.5.0
 Summary: The all-purpose Python SDK for Nextmv
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextmv-py.docs.nextmv.io/en/latest/nextmv/

nextmv-1.5.0/nextmv/__about__.py

@@ -0,0 +1 @@
+__version__ = "v1.5.0"

{nextmv-1.4.2.dev3 → nextmv-1.5.0}/nextmv/cli/cloud/run/logs.py

@@ -108,14 +108,13 @@ def handle_logs(
     """
     Handle retrieving and outputting logs from a run.
 
-    If neither `tail` is True nor `logs` is specified, this function
-    returns early without doing anything. Otherwise, logs are retrieved and
-    optionally written to a file.
-
     When `tail` is True, logs are streamed in real-time to stderr as the run
     executes. When `logs` is specified (without tailing), the function waits
-    for the run to complete and then fetches all logs at once. In both cases,
-    if a `logs` file path is provided, the logs are persisted to that file.
+    for the run to complete and then fetches all logs at once. When neither
+    `tail` is True nor `file_output` is set, logs are fetched immediately and
+    printed to stderr without waiting for the run to complete. In all cases
+    where `tail` is True or `logs` is specified, if a `logs` file path is
+    provided, the logs are persisted to that file.
 
     Parameters
     ----------

{nextmv-1.4.2.dev3 → nextmv-1.5.0}/nextmv/cli/local/run/create.py

@@ -11,6 +11,7 @@ import typer
 
 from nextmv.cli.configuration.config import build_local_app
 from nextmv.cli.local.run.get import handle_outputs
+from nextmv.cli.local.run.logs import handle_logs
 from nextmv.cli.message import enum_values, error, print_json, success
 from nextmv.cli.options import LocalAppIDOption, LocalAppSrcOption
 from nextmv.input import InputFormat
@@ -39,6 +40,16 @@ def create(
         ),
     ] = None,
     # Options for controlling output.
+    logs: Annotated[
+        str | None,
+        typer.Option(
+            "--logs",
+            "-l",
+            help="Waits for the run to complete and saves the logs to this location.",
+            metavar="LOGS_PATH",
+            rich_help_panel="Output control",
+        ),
+    ] = None,
     output: Annotated[
         str | None,
         typer.Option(
@@ -50,6 +61,16 @@ def create(
             rich_help_panel="Output control",
         ),
     ] = None,
+    tail: Annotated[
+        bool,
+        typer.Option(
+            "--tail",
+            "-t",
+            help="Tail the logs until the run completes. Logs are streamed to [magenta]stderr[/magenta]. "
+            "Specify log output location with --logs.",
+            rich_help_panel="Output control",
+        ),
+    ] = False,
     wait: Annotated[
         bool,
         typer.Option(
@@ -134,6 +155,10 @@ def create(
     specify a destination (file or dir) for the output, depending on the
     content type.
 
+    Use the --tail flag to stream logs to [magenta]stderr[/magenta] until the
+    run completes. Using the --logs flag will also activate waiting, and allows
+    you to specify a file to write the logs to.
+
     [bold][underline]Examples[/underline][/bold]
 
     - Read a [magenta]json[/magenta] input via [magenta]stdin[/magenta], from an [magenta]input.json[/magenta] file,
@@ -149,11 +174,28 @@ def create(
       Wait for the run to complete and print the result to [magenta]stdout[/magenta].
         $ [dim]nextmv local run create --app-src ./my-app --input input.json --wait[/dim]
 
+    - Read a [magenta]json[/magenta] input from an [magenta]input.json[/magenta] file, and
+      create a run for an app at path [magenta]./my-app[/magenta].
+      Tail the run's logs, streaming to [magenta]stderr[/magenta].
+        $ [dim]nextmv local run create --app-src ./my-app --input input.json --tail[/dim]
+
     - Read a [magenta]json[/magenta] input from an [magenta]input.json[/magenta] file, and
       create a run for an app with ID [magenta]hare-app[/magenta].
       Wait for the run to complete and write the result to an [magenta]output.json[/magenta] file.
         $ [dim]nextmv local run create --app-id hare-app --input input.json --output output.json[/dim]
 
+    - Read a [magenta]json[/magenta] input from an [magenta]input.json[/magenta] file, and
+      create a run for an app with ID [magenta]hare-app[/magenta].
+      Wait for the run to complete, and write the logs to a [magenta]logs.log[/magenta] file.
+        $ [dim]nextmv local run create --app-id hare-app --input input.json --logs logs.log[/dim]
+
+    - Read a [magenta]json[/magenta] input from an [magenta]input.json[/magenta] file, and create a run for an app at
+      path [magenta]./my-app[/magenta]. Wait for the run to complete. Tail the run's logs, streaming to
+      [magenta]stderr[/magenta]. Write the logs to a [magenta]logs.log[/magenta] file. Write the result to an
+      [magenta]output.json[/magenta] file.
+        $ [dim]nextmv local run create --app-src ./my-app --input input.json --tail --logs logs.log \\
+            --output output.json[/dim]
+
     - Read a [magenta]multi-file[/magenta] input from an [magenta]inputs[/magenta] directory, and
       create a run for an app at path [magenta]./my-app[/magenta].
         $ [dim]nextmv local run create --app-src ./my-app --input inputs --content-format multi-file[/dim]
@@ -196,7 +238,7 @@ def create(
     )
 
     # If we don't need to poll at all we are done.
-    if not wait and output is None:
+    if not wait and not tail and output is None and logs is None:
         print_json({"run_id": run_id})
 
         return
@@ -209,6 +251,14 @@ def create(
 
     # Handle what happens after the run is created for logging and result
     # retrieval.
+    handle_logs(
+        local_app=local_app,
+        run_id=run_id,
+        tail=tail,
+        logs=logs,
+        polling_options=polling_options,
+        file_output=True,
+    )
     handle_outputs(
         local_app=local_app,
         run_id=run_id,

nextmv-1.5.0/nextmv/cli/local/run/logs.py

@@ -0,0 +1,173 @@
+"""
+This module defines the local run logs command for the Nextmv CLI.
+"""
+
+import sys
+from pathlib import Path
+from typing import Annotated
+
+import rich
+import typer
+
+from nextmv.cli.configuration.config import build_local_app
+from nextmv.cli.message import in_progress, success
+from nextmv.cli.options import LocalAppIDOption, LocalAppSrcOption, RunIDOption
+from nextmv.local.application import Application
+from nextmv.polling import PollingOptions, default_polling_options
+
+# Set up subcommand application.
+app = typer.Typer()
+
+
+@app.command()
+def logs(
+    run_id: RunIDOption,
+    app_id: LocalAppIDOption = None,
+    app_src: LocalAppSrcOption = ".",
+    output: Annotated[
+        str | None,
+        typer.Option(
+            "--output",
+            "-o",
+            help="Waits for the run to complete and saves the logs to this location.",
+            metavar="OUTPUT_PATH",
+        ),
+    ] = None,
+    tail: Annotated[
+        bool,
+        typer.Option(
+            "--tail",
+            "-t",
+            help="Tail the logs until the run completes. Logs are streamed to [magenta]stderr[/magenta]. "
+            "Specify log output location with --output.",
+        ),
+    ] = False,
+    timeout: Annotated[
+        int,
+        typer.Option(
+            help="The maximum time in seconds to wait for results when polling. Poll indefinitely if not set.",
+            metavar="TIMEOUT_SECONDS",
+        ),
+    ] = -1,
+) -> None:
+    """
+    Get the logs of a local application run.
+
+    You may identify the app by using --app-src, or --app-id if it has been
+    registered. If the app is not already registered, this command will
+    register it.
+
+    By default, the logs are fetched and printed to [magenta]stderr[/magenta].
+    Use the --tail flag to stream logs to [magenta]stderr[/magenta] until the
+    run completes. Using the --output flag will also activate waiting, and
+    allows you to specify a file to write the logs to.
+
+    [bold][underline]Examples[/underline][/bold]
+
+    - Get the logs of a run with ID [magenta]burrow-123[/magenta], belonging to an app with ID
+      [magenta]hare-app[/magenta]. Logs are printed to [magenta]stderr[/magenta].
+        $ [dim]nextmv local run logs --app-id hare-app --run-id burrow-123[/dim]
+
+    - Get the logs of a run with ID [magenta]burrow-123[/magenta], belonging to an app with ID
+      [magenta]hare-app[/magenta]. Tail the logs until the run completes.
+        $ [dim]nextmv local run logs --app-id hare-app --run-id burrow-123 --tail[/dim]
+
+    - Get the logs of a run with ID [magenta]burrow-123[/magenta], belonging to an app with ID
+      [magenta]hare-app[/magenta]. Save the logs to a [magenta]logs.log[/magenta] file.
+        $ [dim]nextmv local run logs --app-id hare-app --run-id burrow-123 --output logs.log[/dim]
+
+    - Get the logs of a run with ID [magenta]burrow-123[/magenta], belonging to an app with ID
+      [magenta]hare-app[/magenta]. Tail the logs and save them to a [magenta]logs.log[/magenta] file.
+        $ [dim]nextmv local run logs --app-id hare-app --run-id burrow-123 --tail --output logs.log[/dim]
+
+    - Get the logs of a run with ID [magenta]burrow-123[/magenta], belonging to an app with source path
+      [magenta]./my-app[/magenta].
+        $ [dim]nextmv local run logs --app-src ./my-app --run-id burrow-123[/dim]
+
+    - Get the logs of a run with ID [magenta]burrow-123[/magenta], belonging to an app with source path
+      [magenta]./my-app[/magenta]. Set a timeout of [magenta]60[/magenta] seconds.
+        $ [dim]nextmv local run logs --app-src ./my-app --run-id burrow-123 --timeout 60[/dim]
+    """
+
+    local_app = build_local_app(app_src, app_id)
+
+    # Build the polling options.
+    polling_options = default_polling_options()
+    polling_options.max_duration = timeout
+
+    handle_logs(
+        local_app=local_app,
+        run_id=run_id,
+        tail=tail,
+        logs=output,
+        polling_options=polling_options,
+        file_output=output is not None,
+    )
+
+
+def handle_logs(
+    local_app: Application,
+    run_id: str,
+    tail: bool,
+    logs: str | None,
+    polling_options: PollingOptions,
+    file_output: bool,
+) -> None:
+    """
+    Handle retrieving and outputting logs from a run.
+
+    When `tail` is True, logs are streamed in real-time to stderr as the run
+    executes. When `logs` is specified (without tailing), the function waits
+    for the run to complete and then fetches all logs at once. When neither
+    `tail` is True nor `file_output` is set, logs are fetched immediately and
+    printed to stderr without waiting for the run to complete. In all cases
+    where `tail` is True or `logs` is specified, if a `logs` file path is
+    provided, the logs are persisted to that file.
+
+    Parameters
+    ----------
+    local_app : Application
+        The local application instance used to interact with the Nextmv Local
+        API.
+    run_id : str
+        The unique identifier of the run to retrieve logs for.
+    tail : bool
+        If True, streams logs in real-time to stderr as the run executes.
+    logs : str | None
+        The file path where logs should be written. If None, logs are only
+        displayed to stderr (when tailing) and not persisted to a file.
+    polling_options : PollingOptions
+        Configuration options for polling behavior, including timeout and
+        interval settings.
+    file_output : bool
+        Indicates whether logs should be written to a file. If False, logs are
+        only printed to stderr, no matter the status of the run.
+    """
+
+    if tail:
+        in_progress(msg="Tailing logs...")
+        fetched_logs = local_app.run_logs_with_polling(
+            run_id=run_id,
+            polling_options=polling_options,
+            verbose=True,
+            rich_print=True,
+        )
+        if logs is None:
+            return
+
+        log_content = "\n".join(log_entry for log_entry in fetched_logs)
+    elif logs is not None and logs != "" and file_output:
+        in_progress(msg="Getting run logs...")
+        local_app.run_result_with_polling(run_id=run_id, polling_options=polling_options)
+        run_logs = local_app.run_logs(run_id=run_id)
+        log_content = run_logs
+    elif not file_output:
+        in_progress(msg="Getting run logs...")
+        run_logs = local_app.run_logs(run_id=run_id)
+        rich.print(run_logs, file=sys.stderr)
+        return
+    else:
+        return
+
+    Path(logs).write_text(log_content)
+    success(f"Run logs written to [magenta]{logs}[/magenta].")

{nextmv-1.4.2.dev3 → nextmv-1.5.0}/nextmv/cloud/application/_run.py

@@ -824,7 +824,7 @@ class ApplicationRunMixin:
 
         Examples
         --------
-        >>> from nextmv.cloud import PollingOptions
+        >>> from nextmv import PollingOptions
         >>> # Create custom polling options
         >>> polling_opts = PollingOptions(max_tries=50, max_duration=600)
         >>> # Get run logs with polling
@@ -1058,7 +1058,7 @@ class ApplicationRunMixin:
 
         Examples
         --------
-        >>> from nextmv.cloud import PollingOptions
+        >>> from nextmv import PollingOptions
         >>> # Create custom polling options
         >>> polling_opts = PollingOptions(max_tries=50, max_duration=600)
         >>> # Get run result with polling

{nextmv-1.4.2.dev3 → nextmv-1.5.0}/nextmv/local/application.py

@@ -16,6 +16,7 @@ import shutil
 import sys
 import tempfile
 import webbrowser
+from collections.abc import Callable
 from datetime import datetime, timezone
 from typing import Any
 
@@ -853,6 +854,97 @@ class Application(BaseModel):
 
         return logs
 
+    def run_logs_with_polling(
+        self,
+        run_id: str,
+        verbose: bool = False,
+        rich_print: bool = False,
+        polling_options: PollingOptions = DEFAULT_POLLING_OPTIONS,
+        log_func: Callable[[str], None] | None = None,
+    ) -> list[str]:
+        """
+        Get the logs of a local run with polling.
+
+        Retrieves the logs of a run. This method polls for the logs until the
+        run finishes executing or the polling strategy is exhausted. It is the
+        "real-time" equivalent of the `run_logs` method. After the polling is
+        done, all the logs are returned as a list of strings (one per line).
+        You can use the `verbose` parameter to print the logs as they are
+        obtained during the polling process. You can also use the `rich_print`
+        parameter to enable rich printing for better formatting of the logs.
+
+        Parameters
+        ----------
+        run_id : str
+            ID of the run to retrieve the logs for.
+        verbose : bool, default=False
+            Whether to print the logs as they are obtained during the polling
+            process.
+        rich_print : bool, default=False
+            Whether to use rich printing for better formatting of the logs.
+        polling_options : PollingOptions, default=DEFAULT_POLLING_OPTIONS
+            Options to use when polling for the run logs.
+        log_func : Optional[Callable[[str], None]], default=None
+            Optional custom logging function callback. This function is invoked
+            independently of the `verbose` flag. It needs to take a `str` as
+            its argument and return `None`.
+
+        Returns
+        -------
+        list[str]
+            List of log lines of the run.
+
+        Raises
+        ------
+        TimeoutError
+            If the run does not complete after the polling strategy is
+            exhausted based on time duration.
+        RuntimeError
+            If the run does not complete after the polling strategy is
+            exhausted based on number of tries.
+
+        Examples
+        --------
+        >>> from nextmv import PollingOptions
+        >>> polling_opts = PollingOptions(max_tries=50, max_duration=600)
+        >>> logs = app.run_logs_with_polling("run-123", polling_opts)
+        >>> for line in logs:
+        ...     print(line)
+        Starting optimization...
+        Found initial solution
+        """
+
+        all_logs: list[str] = []
+        lines_seen = 0
+
+        def polling_func() -> tuple[Any, bool]:
+            nonlocal lines_seen
+
+            # Check if the run has finalized. If so, all logs are guaranteed
+            # to have been flushed to the file before this check returned.
+            run_information = self.run_metadata(run_id=run_id)
+            is_done = run_information.metadata.run_is_finalized()
+
+            # Read the current log file contents and emit only new lines.
+            log_contents = self.run_logs(run_id)
+            current_lines = log_contents.splitlines() if log_contents else []
+            for line in current_lines[lines_seen:]:
+                if log_func is not None:
+                    log_func(line)
+                if verbose:
+                    if rich_print:
+                        rich.print(line, file=sys.stderr)
+                    else:
+                        log(line)
+                all_logs.append(line)
+            lines_seen = len(current_lines)
+
+            return all_logs, is_done
+
+        all_logs = poll(polling_options=polling_options, polling_func=polling_func)
+
+        return all_logs
+
     def run_metadata(self, run_id: str) -> RunInformation:
         """
         Get the metadata of a local run.
@@ -1002,7 +1094,7 @@ class Application(BaseModel):
 
         Examples
         --------
-        >>> from nextmv.cloud import PollingOptions
+        >>> from nextmv import PollingOptions
         >>> # Create custom polling options
         >>> polling_opts = PollingOptions(max_tries=50, max_duration=600)
         >>> # Get run result with polling

{nextmv-1.4.2.dev3 → nextmv-1.5.0}/nextmv/local/executor.py

@@ -20,8 +20,6 @@ resolve_output_format
     Function to determine the output format from manifest or directory structure.
 process_run_information
     Function to update run metadata including duration and status.
-process_run_logs
-    Function to process and save run logs.
 process_run_metrics
     Function to process and save run metrics.
 process_run_statistics
@@ -43,6 +41,7 @@ import shutil
 import subprocess
 import sys
 import tempfile
+import threading
 from datetime import datetime, timezone
 from typing import Any
 
@@ -159,16 +158,100 @@ def execute_run(
             cwd = __determine_cwd(manifest, default=temp_src)
             args = __determine_command(manifest) + [entrypoint] + options_args(options)
 
-            result = subprocess.run(
+            # Determine whether stdout should be streamed live to the log file.
+            # For MULTI_FILE format, stdout carries log output and must be streamed
+            # immediately. For other formats (JSON, CSV_ARCHIVE), stdout carries
+            # structured output consumed after the process completes.
+            is_multi_file = (
+                manifest.configuration is not None
+                and manifest.configuration.content is not None
+                and manifest.configuration.content.format == InputFormat.MULTI_FILE
+            ) or run_config["format"]["input"]["type"] == InputFormat.MULTI_FILE.value
+
+            log_file_path = os.path.join(logs_dir, LOGS_FILE)
+            stdout_lines: list[str] = []
+            stderr_lines: list[str] = []
+
+            # Force unbuffered stdout/stderr in child processes so both
+            # streams are written to the log file in real time.
+            child_env = os.environ.copy()
+            child_env["PYTHONUNBUFFERED"] = "1"
+
+            # This is the process that actually executes the entrypoint script.
+            # We use subprocess.Popen instead of subprocess.run because we need
+            # to stream the output in real time, which is not possible with
+            # subprocess.run.
+            process = subprocess.Popen(
                 args,
-                env=os.environ,
-                check=False,
+                env=child_env,
                 text=True,
-                capture_output=True,
-                input=stdin_input,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
                 cwd=cwd,
             )
 
+            def write_stdin() -> None:
+                # Write all input at once then close so the child process receives EOF.
+                process.stdin.write(stdin_input)
+                process.stdin.close()
+
+            # A lock to serialize writes from the stdout and stderr threads so log
+            # lines are never interleaved in the log file.
+            log_lock = threading.Lock()
+            log_f = open(log_file_path, "a")
+
+            def stream_stderr() -> None:
+                # Stderr is always streamed live to the log file so errors appear
+                # immediately even if the process is still running.
+                for line in process.stderr:
+                    with log_lock:
+                        log_f.write(line)
+                        log_f.flush()
+                    stderr_lines.append(line)
+
+            def stream_stdout() -> None:
+                # For multi-file runs, stdout carries log output, so mirror it to
+                # the log file in real time just like stderr. For other formats,
+                # stdout carries the structured result and is only buffered.
+                if is_multi_file:
+                    for line in process.stdout:
+                        with log_lock:
+                            log_f.write(line)
+                            log_f.flush()
+                        stdout_lines.append(line)
+                else:
+                    for line in process.stdout:
+                        stdout_lines.append(line)
+
+            # Run stdin, stdout, and stderr on separate threads so no single pipe
+            # can fill its OS buffer and block the process while another pipe waits,
+            # which would cause a deadlock.
+            stdin_thread = threading.Thread(target=write_stdin)
+            stderr_thread = threading.Thread(target=stream_stderr)
+            stdout_thread = threading.Thread(target=stream_stdout)
+
+            stdin_thread.start()
+            stderr_thread.start()
+            stdout_thread.start()
+
+            # Wait for all streams to be fully consumed before calling process.wait()
+            # to ensure no output is lost.
+            stdin_thread.join()
+            stderr_thread.join()
+            stdout_thread.join()
+            process.wait()
+            log_f.close()
+
+            # Assemble a CompletedProcess so the rest of the pipeline can treat this
+            # the same as a synchronous subprocess.run() result.
+            result = subprocess.CompletedProcess(
+                args=args,
+                returncode=process.returncode,
+                stdout="".join(stdout_lines),
+                stderr="".join(stderr_lines),
+            )
+
             process_run_output(
                 manifest=manifest,
                 run_id=run_id,
@@ -344,12 +427,6 @@ def process_run_output(
         run_dir=run_dir,
         result=result,
     )
-    process_run_logs(
-        output_format=output_format,
-        run_dir=run_dir,
-        result=result,
-        stdout_output=stdout_output,
-    )
     process_run_metrics(
         temp_run_outputs_dir=temp_run_outputs_dir,
         outputs_dir=outputs_dir,
@@ -472,44 +549,6 @@ def process_run_information(run_id: str, run_dir: str, result: subprocess.Comple
         json.dump(info, f, indent=2)
 
 
-def process_run_logs(
-    output_format: OutputFormat,
-    run_dir: str,
-    result: subprocess.CompletedProcess[str],
-    stdout_output: str | dict[str, Any],
-) -> None:
-    """
-    Processes the logs of the run. Writes the logs to a logs directory.
-    For multi-file format, stdout is written to logs if present.
-
-    Parameters
-    ----------
-    output_format : OutputFormat
-        The output format of the run (JSON, CSV_ARCHIVE, or MULTI_FILE).
-    run_dir : str
-        The path to the run directory where logs will be stored.
-    result : subprocess.CompletedProcess[str]
-        The result of the subprocess run containing stderr output.
-    stdout_output : Union[str, dict[str, Any]]
-        The stdout output of the run, either as raw string or parsed dictionary.
-    """
-
-    logs_dir = os.path.join(run_dir, LOGS_KEY)
-    os.makedirs(logs_dir, exist_ok=True)
-    std_err = result.stderr
-    with open(os.path.join(logs_dir, LOGS_FILE), "w") as f:
-        if output_format == OutputFormat.MULTI_FILE and bool(stdout_output):
-            if isinstance(stdout_output, dict):
-                f.write(json.dumps(stdout_output))
-            elif isinstance(stdout_output, str):
-                f.write(stdout_output)
-
-            if std_err:
-                f.write("\n")
-
-        f.write(std_err)
-
-
 def process_run_metrics(
     temp_run_outputs_dir: str,
     outputs_dir: str,

{nextmv-1.4.2.dev3 → nextmv-1.5.0}/nextmv/polling.py

@@ -85,7 +85,7 @@ class PollingOptions:
 
     Examples
     --------
-    >>> from nextmv.cloud import PollingOptions
+    >>> from nextmv import PollingOptions
     >>> # Create polling options with custom settings
     >>> polling_options = PollingOptions(
     ...     max_tries=50,
@@ -218,7 +218,7 @@ def poll(  # noqa: C901
 
     Examples
     --------
-    >>> from nextmv.cloud import PollingOptions, poll
+    >>> from nextmv import PollingOptions, poll
     >>> import time
     >>>
     >>> # Define a polling function that succeeds after 3 tries