nextpipe 0.6.2.dev5__tar.gz → 0.7.0__tar.gz

{nextpipe-0.6.2.dev5 → nextpipe-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextpipe
-Version: 0.6.2.dev5
+Version: 0.7.0
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextpipe.docs.nextmv.io/en/latest/

{nextpipe-0.6.2.dev5 → nextpipe-0.7.0}/docs/examples/complex-workflow-csv.md

@@ -24,7 +24,7 @@ import csv
 import json
 from typing import Any
 
-import nextmv.cloud
+import nextmv
 import requests
 
 from nextpipe import FlowSpec, app, log, needs, repeat, step
@@ -112,9 +112,9 @@ class Workflow(FlowSpec):
     @needs(predecessors=[solve_nextroute, solve_vroom, solve_ortools])
     @step
     def pick_best(
-        nextroute_results: list[nextmv.cloud.RunResult],
-        vroom_result: nextmv.cloud.RunResult,
-        ortools_result: nextmv.cloud.RunResult,
+        nextroute_results: list[nextmv.RunResult],
+        vroom_result: nextmv.RunResult,
+        ortools_result: nextmv.RunResult,
     ):
         """Pick the best solution based on the result value."""
 

{nextpipe-0.6.2.dev5 → nextpipe-0.7.0}/docs/examples/multifile-workflow.md

@@ -19,7 +19,6 @@ import os
 import shutil
 
 import nextmv
-import nextmv.cloud
 
 from nextpipe import FlowSpec, app, log, needs, step
 
@@ -76,7 +75,7 @@ class Flow(FlowSpec):
     )
     @needs(predecessors=[transform])
     @step
-    def solve2(result: nextmv.cloud.RunResult):
+    def solve2(result: nextmv.RunResult):
         """Runs another multi-file model."""
         pass
 
@@ -85,7 +84,7 @@ class Flow(FlowSpec):
     # via 'result.output'.
     @needs(predecessors=[solve2])
     @step
-    def prepare_output(result: nextmv.cloud.RunResult):
+    def prepare_output(result: nextmv.RunResult):
         """Transforms the result for the next step."""
         # Extract the path to the output files.
         result_path = result.output

nextpipe-0.7.0/nextpipe/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.7.0"

{nextpipe-0.6.2.dev5 → nextpipe-0.7.0}/nextpipe/flow.py

@@ -896,6 +896,138 @@ class Runner:
         description = f"This run was started by the nextpipe parent: {wf_app_id} (run-id: {wf_run_id})"
         return name, description
 
+    @staticmethod
+    def __prepare_app_run_args(
+        node: FlowNode,
+        inputs: list[object],
+        app_step: decorators.App,
+    ) -> tuple[dict[str, Any], nextmv.PollingOptions, bool]:
+        """
+        Prepare arguments for running an app step.
+
+        Returns
+        -------
+        tuple[dict[str, Any], nextmv.PollingOptions, bool]
+            A tuple of run kwargs, polling options, and whether the input is dir mode.
+        """
+
+        app_run_name, app_run_description = Runner.__determine_sub_run_info(app_step.app_id, node.id)
+
+        # TODO: We only support one predecessor for app steps for now. This may
+        # change in the future. We may want to support multiple predecessors for
+        # app steps. However, we need to think about how to handle the input and
+        # how to expose control over the input to the user.
+        if len(inputs) > 1:
+            raise Exception(f"App steps cannot have more than one predecessor, but {node.id} has {len(inputs)}")
+
+        if isinstance(inputs[0], schema.AppRunConfig):
+            app_run_config: schema.AppRunConfig = inputs[0]
+            input_data = app_run_config.input
+            if app_run_config.name:
+                app_run_name = app_run_config.name
+            if app_run_config.description:
+                app_run_description = app_run_config.description
+            # AppRunConfig options take precedence over decorator options.
+            options = app_step.options | app_run_config.get_options()
+        elif isinstance(inputs[0], nextmv.RunResult):
+            run_result: nextmv.RunResult = inputs[0]
+            input_data = run_result.output
+            options = app_step.options
+        else:
+            input_data = inputs[0]
+            options = app_step.options
+
+        is_dir_mode = isinstance(input_data, str) and os.path.isdir(input_data)
+
+        polling_options = copy.deepcopy(app_step.polling_options)
+        polling_options.initial_delay = random.uniform(0, 5)
+        if polling_options.stop is None:
+            polling_options.stop = lambda: node.cancel
+
+        run_kwargs: dict[str, Any] = {
+            "options": options,
+            "name": app_run_name,
+            "description": app_run_description,
+        }
+        if is_dir_mode:
+            run_kwargs["input_dir_path"] = input_data
+        else:
+            run_kwargs["input"] = input_data
+        if app_step.run_configuration is not None:
+            run_kwargs["configuration"] = app_step.run_configuration
+
+        return run_kwargs, polling_options, is_dir_mode
+
+    @staticmethod
+    def __execute_app_run(
+        node: FlowNode,
+        app: Application,
+        app_step: decorators.App,
+        client: Client,
+        update_dag: callable,
+        run_kwargs: dict[str, Any],
+        polling_options: nextmv.PollingOptions,
+        is_dir_mode: bool,
+        temp_dir: str,
+    ) -> nextmv.RunResult:
+        """
+        Execute an app run or return bypass output wrapped as a run result.
+        """
+
+        if app_step.bypass_result is None:
+            run_id = app.new_run(**run_kwargs)
+            console_url = f"{client.console_url}/app/{app_step.app_id}/run/{run_id}?view=details"
+            utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
+            node.run_id = run_id
+            update_dag()  # Update node with run_id before polling
+
+            if app_step.emit_live_logs:
+                utils.log_internal(f"Emitting live logs for app step {node.id} ...")
+                app.run_logs_with_polling(
+                    run_id=run_id,
+                    verbose=True,
+                    log_func=lambda msg: utils.log(message=msg.log, step_name=node.id),
+                )
+
+            return app.run_result_with_polling(run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir)
+
+        utils.log_internal(f"Bypassing app step {node.id} run, using bypass result")
+
+        result = nextmv.RunResult(
+            id="bypassed-run",
+            name="bypassed-run",
+            description="This run was bypassed using a user-supplied result.",
+            user_email="unavailable",
+            metadata=nextmv.Metadata(
+                application_id=app_step.app_id or "unavailable",
+                application_instance_id=app_step.instance_id or "unavailable",
+                application_version_id="unavailable",
+                created_at=datetime.datetime.now(),
+                duration=0.0,
+                error="",
+                input_size=0.0,
+                output_size=0.0,
+                format=nextmv.Format(
+                    format_input=nextmv.FormatInput(
+                        input_type=nextmv.InputFormat.MULTI_FILE if is_dir_mode else nextmv.InputFormat.JSON,
+                    ),
+                ),
+                status_v2=nextmv.StatusV2.succeeded,
+            ),
+        )
+        result.output = app_step.bypass_result
+
+        if isinstance(app_step.bypass_result, str) and os.path.isdir(app_step.bypass_result):
+            src = pathlib.Path(app_step.bypass_result)
+            dst = pathlib.Path(temp_dir)
+            for item in src.iterdir():
+                if item.is_file():
+                    shutil.copy2(item, dst / item.name)
+                elif item.is_dir():
+                    shutil.copytree(item, dst / item.name)
+
+        return result
+
     @staticmethod
     def __run_step(
         node: FlowNode,
@@ -935,72 +1067,7 @@ class Runner:
         if node.parent.definition.is_app():
             app_step: decorators.App = node.parent.definition.app
 
-            # Try to detect platform run context.
-            app_run_name, app_run_description = Runner.__determine_sub_run_info(app_step.app_id, node.id)
-
-            # Prepare the input for the app
-            # TODO: We only support one predecessor for app steps for now. This may
-            # change in the future. We may want to support multiple predecessors for
-            # app steps. However, we need to think about how to handle the input and
-            # how to expose control over the input to the user.
-            if len(inputs) > 1:
-                raise Exception(f"App steps cannot have more than one predecessor, but {node.id} has {len(inputs)}")
-            if isinstance(inputs[0], schema.AppRunConfig):
-                # If the input is AppRunConfig, unwrap it.
-                app_run_config: schema.AppRunConfig = inputs[0]
-                input = app_run_config.input
-                if app_run_config.name:
-                    app_run_name = app_run_config.name
-                if app_run_config.description:
-                    app_run_description = app_run_config.description
-                app_run_options = app_run_config.get_options()
-                # Merge the options from the app decorator with the options from the
-                # AppRunConfig. AppRunConfig options take precedence.
-                options = app_step.options | app_run_options
-            elif isinstance(inputs[0], nextmv.RunResult):
-                # If the input is a RunResult, we use its output as input.
-                run_result: nextmv.RunResult = inputs[0]
-                input = run_result.output
-                options = app_step.options
-            else:
-                # If the input is not AppRunConfig, we use it directly.
-                input = inputs[0]
-                options = app_step.options
-
-            # Detect dir mode / multi-file direct input
-            is_dir_mode = False
-            if isinstance(input, str) and os.path.isdir(input):
-                is_dir_mode = True
-
-            # Modify the polling options set for the step (by default or by the
-            # user) so that the initial delay is randomized and the stopping
-            # callback is configured as the node being cancelled if the user
-            # doesn't want to override it.
-            polling_options = copy.deepcopy(app_step.polling_options)
-            delay = random.uniform(0, 5)  # For lack of a better idea...
-            polling_options.initial_delay = delay
-            if polling_options.stop is None:
-                polling_options.stop = lambda: node.cancel
-
-            run_args = (
-                [],  # No nameless arguments
-                {  # We use the named arguments to pass the user arguments to the run function
-                    "options": options,
-                    "name": app_run_name,
-                    "description": app_run_description,
-                },
-            )
-
-            # Prepare input argument. We need to use 'input_dir_path' when dealing with a
-            # directory input (e.g., multi-file input).
-            if is_dir_mode:
-                run_args[1]["input_dir_path"] = input
-            else:
-                run_args[1]["input"] = input
-
-            # Apply run configuration if given.
-            if app_step.run_configuration is not None:
-                run_args[1]["configuration"] = app_step.run_configuration
+            run_kwargs, polling_options, is_dir_mode = Runner.__prepare_app_run_args(node, inputs, app_step)
 
             # Prepare the application itself.
             app = Application(
@@ -1015,66 +1082,17 @@ class Runner:
 
             # Run the application
             try:
-                if app_step.bypass_result is None:
-                    # Start the run.
-                    run_id = app.new_run(*run_args[0], **run_args[1])
-                    console_url = f"{client.console_url}/app/{app_step.app_id}/run/{run_id}?view=details"
-                    utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
-                    node.run_id = run_id
-                    update_dag()  # Update the node with the run_id before we start polling for results
-
-                    # Emit live logs if requested.
-                    if app_step.emit_live_logs:
-                        utils.log_internal(f"Emitting live logs for app step {node.id} ...")
-                        app.run_logs_with_polling(
-                            run_id=run_id,
-                            verbose=True,
-                            log_func=lambda msg: utils.log(message=msg.log, step_name=node.id),
-                        )
-
-                    # Wait for run to finish and get the result.
-                    result = app.run_result_with_polling(
-                        run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir
-                    )
-                else:
-                    utils.log_internal(f"Bypassing app step {node.id} run, using bypass result")
-
-                    # We wrap the bypass result in a dummy RunResult to ensure consistent behavior.
-                    result = nextmv.RunResult(
-                        id="bypassed-run",
-                        name="bypassed-run",
-                        description="This run was bypassed using a user-supplied result.",
-                        user_email="unavailable",
-                        metadata=nextmv.Metadata(
-                            application_id="unavailable",
-                            application_instance_id="unavailable",
-                            application_version_id="unavailable",
-                            created_at=datetime.datetime.now(),
-                            duration=0.0,
-                            error="unavailable",
-                            input_size=0.0,
-                            output_size=0.0,
-                            format=nextmv.Format(
-                                format_input=nextmv.FormatInput(
-                                    input_type=nextmv.InputFormat.MULTI_FILE
-                                    if is_dir_mode
-                                    else nextmv.InputFormat.JSON,
-                                ),
-                            ),
-                            status_v2=nextmv.StatusV2.succeeded,
-                        ),
-                    )
-                    result.output = app_step.bypass_result
-
-                    # If the user supplied a bypass result, we need to write it to the temp dir if it's in dir mode
-                    if isinstance(app_step.bypass_result, str) and os.path.isdir(app_step.bypass_result):
-                        src = pathlib.Path(app_step.bypass_result)
-                        dst = pathlib.Path(temp_dir)
-                        for item in src.iterdir():
-                            if item.is_file():
-                                shutil.copy2(item, dst / item.name)
-                            elif item.is_dir():
-                                shutil.copytree(item, dst / item.name)
+                result = Runner.__execute_app_run(
+                    node=node,
+                    app=app,
+                    app_step=app_step,
+                    client=client,
+                    update_dag=update_dag,
+                    run_kwargs=run_kwargs,
+                    polling_options=polling_options,
+                    is_dir_mode=is_dir_mode,
+                    temp_dir=temp_dir,
+                )
 
             finally:  # Make sure we clean up temp dir on failure too
                 # If the temp dir is empty, remove it

nextpipe-0.6.2.dev5/nextpipe/__about__.py

@@ -1 +0,0 @@
-__version__ = "v0.6.2-dev.5"