nextpipe 0.3.5.dev0__py3-none-any.whl → 0.4.0.dev0__py3-none-any.whl

nextpipe/__about__.py

@@ -1 +1 @@
-__version__ = "v0.3.5-dev.0"
+__version__ = "v0.4.0.dev.0"

nextpipe/decorators.py

@@ -50,6 +50,7 @@ from collections.abc import Callable
 from enum import Enum
 from functools import wraps
 
+import nextmv
 from nextmv import cloud
 from nextmv.deprecated import deprecated
 
@@ -772,10 +773,10 @@ class App:
         The ID of the instance to run.
     options : dict[str, Any]
         The options to pass to the application.
-    input_type : InputType
-        The type of input to pass to the application (JSON or FILES).
     full_result : bool
         Whether to return the full result including metadata.
+    run_configuration : nextmv.RunConfiguration
+        The configuration to apply when running the app.
     polling_options : Optional[cloud.PollingOptions]
         Options for polling for the results of the app run.
     """
@@ -788,6 +789,7 @@ class App:
         parameters: dict[str, typing.Any] = None,
         options: dict[str, typing.Any] = None,
         full_result: bool = False,
+        run_configuration: nextmv.RunConfiguration = None,
         polling_options: typing.Optional[cloud.PollingOptions] = _DEFAULT_POLLING_OPTIONS,
     ):
         """
@@ -799,16 +801,22 @@ class App:
             The ID of the Nextmv Application to run.
         instance_id : str, optional
             The ID of the instance to run. Default is defined by the app on Platform.
-        input_type : InputType, optional
-            The type of input to pass to the application, by default InputType.JSON.
         options : dict[str, Any], optional
             The options to pass to the application, by default None.
         full_result : bool, optional
             Whether to return the full result including metadata, by default False.
+        run_configuration : nextmv.RunConfiguration, optional
+            The configuration to apply when running the app, by default None.
         polling_options : Optional[cloud.PollingOptions], optional
             Options for polling for the results of the app run, by default _DEFAULT_POLLING_OPTIONS.
         """
 
+        if input_type:
+            deprecated(
+                "input_type",
+                "The 'input_type' argument is deprecated and will be removed in a future release.",
+            )
+
         # Make sure only one of options or parameters is used.
         if parameters and options:
             raise ValueError("You can only use either 'parameters' or 'options', not both.")
@@ -824,6 +832,7 @@ class App:
         self.options = options if options else {}
         self.input_type = input_type
         self.full_result = full_result
+        self.run_configuration = run_configuration
         self.polling_options = polling_options
 
     def __repr__(self):
@@ -835,7 +844,7 @@ class App:
             A string representation of the app.
         """
 
-        return f"StepRun({self.app_id}, {self.instance_id}, {self.options}, {self.input_type}, {self.full_result})"
+        return f"StepRun({self.app_id}, {self.instance_id}, {self.options}, {self.full_result})"
 
 
 def app(
@@ -845,6 +854,7 @@ def app(
     options: dict[str, typing.Any] = None,
     input_type: InputType = InputType.JSON,
     full_result: bool = False,
+    run_configuration: nextmv.RunConfiguration = None,
     polling_options: typing.Optional[cloud.PollingOptions] = _DEFAULT_POLLING_OPTIONS,
 ):
     """
@@ -871,14 +881,13 @@ def app(
     options : dict[str, Any]
         The options to pass to the application. This is a dictionary of
         parameter names and values. The values must be JSON serializable.
-    input_type : InputType
-        The type of input to pass to the application. This can be either
-        JSON or FILES. Default is JSON.
     full_result : bool
         Whether to return the full result of the application run. If this is
         set to `True`, the full result (with metadata) will be returned. If
         this is set to `False`, only the output of the application will be
         returned.
+    run_configuration : nextmv.RunConfiguration
+        The configuration to apply when running the app.
     polling_options : Optional[cloud.PollingOptions]
         Options for polling for the results of the app run. This is used to
         configure the polling behavior, such as the timeout and backoff
@@ -924,6 +933,12 @@ def app(
     ```
     """
 
+    if input_type:
+        deprecated(
+            "input_type",
+            "The 'input_type' argument is deprecated and will be removed in a future release.",
+        )
+
     # Make sure only one of options or parameters is used.
     if parameters and options:
         raise ValueError("You can only use either 'parameters' or 'options', not both.")
@@ -950,6 +965,7 @@ def app(
             options=converted_options,
             input_type=input_type,
             full_result=full_result,
+            run_configuration=run_configuration,
             polling_options=polling_options,
         )
         wrapper.step.type = StepType.APP

nextpipe/flow.py

@@ -27,14 +27,16 @@ import base64
 import copy
 import inspect
 import io
+import os
 import random
+import tempfile
 import threading
 import time
 from importlib.metadata import version
 from itertools import product
 from typing import Any, Optional, Union
 
-from nextmv.cloud import Application, Client
+from nextmv.cloud import Application, Client, RunResult
 
 from . import config, decorators, graph, schema, threads, uplink, utils
 from .__about__ import __version__
@@ -907,6 +909,7 @@ class Runner:
         # Run the step
         if node.parent.definition.is_app():
             app_step: decorators.App = node.parent.definition.app
+
             # Prepare the input for the app
             # TODO: We only support one predecessor for app steps for now. This may
             # change in the future. We may want to support multiple predecessors for
@@ -923,12 +926,23 @@ class Runner:
                 # Merge the options from the app decorator with the options from the
                 # AppRunConfig. AppRunConfig options take precedence.
                 options = app_step.options | app_run_options
+            elif isinstance(inputs[0], RunResult):
+                # If the input is a RunResult, we use its output as input.
+                run_result: RunResult = inputs[0]
+                input = run_result.output
+                options = app_step.options
+                name = node.id
             else:
                 # If the input is not AppRunConfig, we use it directly.
                 input = inputs[0]
                 options = app_step.options
                 name = node.id
 
+            # Detect dir mode / multi-file direct input
+            is_dir_mode = False
+            if isinstance(input, str) and os.path.isdir(input):
+                is_dir_mode = True
+
             # Modify the polling options set for the step (by default or by the
             # user) so that the initial delay is randomized and the stopping
             # callback is configured as the node being cancelled if the user
@@ -942,13 +956,22 @@ class Runner:
             run_args = (
                 [],  # No nameless arguments
                 {  # We use the named arguments to pass the user arguments to the run function
-                    "input": input,
-                    "run_options": options,
-                    "polling_options": polling_options,
+                    "options": options,
                     "name": name,
                 },
             )
 
+            # Prepare input argument. We need to use 'input_dir_path' when dealing with a
+            # directory input (e.g., multi-file input).
+            if is_dir_mode:
+                run_args[1]["input_dir_path"] = input
+            else:
+                run_args[1]["input"] = input
+
+            # Apply run configuration if given.
+            if app_step.run_configuration is not None:
+                run_args[1]["configuration"] = app_step.run_configuration
+
             # Prepare the application itself.
             app = Application(
                 client=client,
@@ -957,18 +980,35 @@ class Runner:
             if app_step.instance_id is not None and app_step.instance_id != "":
                 app.default_instance_id = app_step.instance_id
 
-            # Run the application
-            result = app.new_run_with_result(
-                *run_args[0],
-                **run_args[1],
-            )
-            run_id = result.id
-            node.run_id = run_id
-            utils.log_internal(f"Finished app step {node.id} run, find it at {result.console_url}")
+            # We always supply an output directory path in case of implicit multi-file output
+            temp_dir = tempfile.mkdtemp(prefix="nextpipe_output_")
 
-            # Return result (do not unwrap if full result is requested)
+            # Run the application
+            try:
+                run_id = app.new_run(*run_args[0], **run_args[1])
+                console_url = f"{client.console_url}/app/{app_step.app_id}/run/{run_id}?view=details"
+                utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
+                result = app.run_result_with_polling(
+                    run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir
+                )
+                node.run_id = run_id
+            finally:  # Make sure we clean up temp dir on failure too
+                # If the temp dir is empty, remove it
+                dir_result = False
+                if not os.listdir(temp_dir):
+                    os.rmdir(temp_dir)
+                else:
+                    dir_result = True
+
+            # Return result
+            #  - Do not unwrap if full result is requested
+            #  - If the output came in a directory, return the directory path
             if app_step.full_result:
+                if dir_result:
+                    result.output = temp_dir
                 return result
+            if dir_result:
+                return temp_dir
             return result.output
 
         else:

nextpipe/schema.py

@@ -70,8 +70,10 @@ class AppRunConfig:
 
     Parameters
     ----------
-    input : dict[str, Any], optional
-        Input data for the app, by default None.
+    input : Union[dict[str, Any], str, Any]
+        Input data for the app. A JSON app can take a dictionary, multi-file apps can take
+        a directory path as a string. Other types will be passed to the underlying Python
+        SDK as-is (e.g., nextmv.Input).
     options : Union[list[AppOption], dict[str, Any]], optional
         Options for running the app, by default empty. These can be provided as a list of
         `AppOption` instances, or, simply as a dictionary of key-value pairs.
@@ -88,8 +90,10 @@ class AppRunConfig:
     ... )
     """
 
-    input: dict[str, Any] = None
-    """Input for the app."""
+    input: Union[dict[str, Any], str, Any]
+    """Input for the app. A JSON app can take a dictionary, multi-file apps can take a
+    directory path as a string. Other types will be passed to the underlying Python SDK
+    as-is (e.g., nextmv.Input)."""
     options: Union[list[AppOption], dict[str, Any]] = field(default_factory=list)
     """Options for running the app."""
     name: Optional[str] = None

{nextpipe-0.3.5.dev0.dist-info → nextpipe-0.4.0.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextpipe
-Version: 0.3.5.dev0
+Version: 0.4.0.dev0
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextpipe.docs.nextmv.io/en/latest/
@@ -108,7 +108,7 @@ Requires-Dist: dataclasses-json>=0.6.7
 Requires-Dist: nextmv>=0.29.4
 Requires-Dist: requests>=2.31.0
 Provides-Extra: dev
-Requires-Dist: goldie>=0.1.7; extra == 'dev'
+Requires-Dist: goldie>=0.1.8; extra == 'dev'
 Requires-Dist: ruff>=0.11.6; extra == 'dev'
 Description-Content-Type: text/markdown
 

nextpipe-0.4.0.dev0.dist-info/RECORD

@@ -0,0 +1,14 @@
+nextpipe/__about__.py,sha256=mdzuNb93MdHDpJbMl-whjJJUbZi4Yyfl7Nv47quBabA,29
+nextpipe/__init__.py,sha256=SzRhi2z33zmvskfgqoRHwaKvRnt8G1_JJR1M7BXlda0,694
+nextpipe/config.py,sha256=JSShEaK3fnqieGofalQysKxEJUkyrWIRbwhdyH9yYZM,1659
+nextpipe/decorators.py,sha256=-fbmwDVi0pnEZnJ8xThCd_Hmm8O1pErG0m2nTudw-2M,25568
+nextpipe/flow.py,sha256=37AnTFWJxQJGj-eGzuOvGMzPvwxSkpdT5O3s31d3uV0,38933
+nextpipe/graph.py,sha256=Oi8HQ4fgvBYxpRK6xBoLtoxftGb-gCOb09BqFwzrwWs,2867
+nextpipe/schema.py,sha256=gzWcaDwt65OMDnciU2T8--Eb4u77J8w3sUN1iXlxNgc,3224
+nextpipe/threads.py,sha256=yxyoRj3aknVMeGZzlVM1HcXhL-iXsCovaTE5OfFM4TI,9052
+nextpipe/uplink.py,sha256=j4k7Msa1Il7uxrQeXd1yoCMWWd9KmkbQb72EZy4zMSs,11646
+nextpipe/utils.py,sha256=JTirVzh3npkjikMg3-fbtuLeIeG1zI7Rxu-1xREu2LY,7916
+nextpipe-0.4.0.dev0.dist-info/METADATA,sha256=IZcLKWIOYZ_t56pCGFm2iGQCc4TgnJqSr8vz8WKSBr0,8765
+nextpipe-0.4.0.dev0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+nextpipe-0.4.0.dev0.dist-info/licenses/LICENSE.md,sha256=FpVScuRcgdbeBNVsMzoPyBbKL2eCB1vFor1vfNNFEj4,4106
+nextpipe-0.4.0.dev0.dist-info/RECORD,,

nextpipe-0.3.5.dev0.dist-info/RECORD

@@ -1,14 +0,0 @@
-nextpipe/__about__.py,sha256=LF9FBcRfZAKRmyrREOND-uQQeSAcbb_afE2DUHOhT6A,29
-nextpipe/__init__.py,sha256=SzRhi2z33zmvskfgqoRHwaKvRnt8G1_JJR1M7BXlda0,694
-nextpipe/config.py,sha256=JSShEaK3fnqieGofalQysKxEJUkyrWIRbwhdyH9yYZM,1659
-nextpipe/decorators.py,sha256=1pndK9ej5JQJjtYFugzidlEeyyqPxWi0zvCaeDQfAro,25008
-nextpipe/flow.py,sha256=wIXeCqxuWgZuDty4pScEOv1d1m-9SkPZ2qo_55u4nkg,37150
-nextpipe/graph.py,sha256=Oi8HQ4fgvBYxpRK6xBoLtoxftGb-gCOb09BqFwzrwWs,2867
-nextpipe/schema.py,sha256=jnyXimHepzWPGCEpSyyObLtza8Cb8RSVpCP9sS9vWWI,2854
-nextpipe/threads.py,sha256=yxyoRj3aknVMeGZzlVM1HcXhL-iXsCovaTE5OfFM4TI,9052
-nextpipe/uplink.py,sha256=j4k7Msa1Il7uxrQeXd1yoCMWWd9KmkbQb72EZy4zMSs,11646
-nextpipe/utils.py,sha256=JTirVzh3npkjikMg3-fbtuLeIeG1zI7Rxu-1xREu2LY,7916
-nextpipe-0.3.5.dev0.dist-info/METADATA,sha256=eQM8CjG4xcy4zQ4O_6dBU3dWTtzVflHDFDv7IPfI2LU,8765
-nextpipe-0.3.5.dev0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-nextpipe-0.3.5.dev0.dist-info/licenses/LICENSE.md,sha256=FpVScuRcgdbeBNVsMzoPyBbKL2eCB1vFor1vfNNFEj4,4106
-nextpipe-0.3.5.dev0.dist-info/RECORD,,