nextpipe 0.7.2__tar.gz → 0.7.3.dev0__tar.gz

{nextpipe-0.7.2 → nextpipe-0.7.3.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: nextpipe
-Version: 0.7.2
+Version: 0.7.3-dev.0
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://docs.nextmv.io/nextpipe

nextpipe-0.7.3.dev0/nextpipe/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.7.3-dev.0"

{nextpipe-0.7.2 → nextpipe-0.7.3.dev0}/nextpipe/decorators.py

@@ -798,12 +798,16 @@ class App:
             The ID of the Nextmv Application to run.
         instance_id : str, optional
             The ID of the instance to run. Default is defined by the app on Platform.
+            A predecessor step may override this per run by returning an `AppRunConfig`
+            carrying its own `instance_id`.
         options : dict[str, Any], optional
             The options to pass to the application, by default None.
         full_result : bool, optional
             Whether to return the full result including metadata, by default False.
         run_configuration : nextmv.RunConfiguration, optional
-            The configuration to apply when running the app, by default None.
+            The configuration to apply when running the app, by default None. A
+            predecessor step may override this per run by returning an
+            `AppRunConfig` carrying its own `run_configuration`.
         polling_options : Optional[nextmv.PollingOptions], optional
             Options for polling for the results of the app run, by default _DEFAULT_POLLING_OPTIONS.
         bypass_result : typing.Any, optional
@@ -884,7 +888,9 @@ def app(
     app_id : str
         The ID of the application to run.
     instance_id : str
-        The ID of the instance to run. Default is defined by the app on Platform.
+        The ID of the instance to run. Default is defined by the app on Platform. A
+        predecessor step may override this per run by returning an `AppRunConfig`
+        carrying its own `instance_id`.
     options : dict[str, Any]
         The options to pass to the application. This is a dictionary of
         parameter names and values. The values must be JSON serializable.
@@ -894,7 +900,10 @@ def app(
         this is set to `False`, only the output of the application will be
         returned.
     run_configuration : nextmv.RunConfiguration
-        The configuration to apply when running the app.
+        The configuration to apply when running the app. A predecessor step may
+        override this per run by returning an `AppRunConfig` carrying its own
+        `run_configuration`. In that case, the configuration of the `AppRunConfig`
+        replaces this one as a whole, i.e., no fields are inherited from it.
     polling_options : Optional[nextmv.PollingOptions]
         Options for polling for the results of the app run. This is used to
         configure the polling behavior, such as the timeout and backoff

{nextpipe-0.7.2 → nextpipe-0.7.3.dev0}/nextpipe/flow.py

@@ -901,14 +901,15 @@ class Runner:
         node: FlowNode,
         inputs: list[object],
         app_step: decorators.App,
-    ) -> tuple[dict[str, Any], nextmv.PollingOptions, bool]:
+    ) -> tuple[dict[str, Any], nextmv.PollingOptions, bool, str]:
         """
         Prepare arguments for running an app step.
 
         Returns
         -------
-        tuple[dict[str, Any], nextmv.PollingOptions, bool]
-            A tuple of run kwargs, polling options, and whether the input is dir mode.
+        tuple[dict[str, Any], nextmv.PollingOptions, bool, str]
+            A tuple of run kwargs, polling options, whether the input is dir mode, and
+            the ID of the instance to run.
         """
 
         app_run_name, app_run_description = Runner.__determine_sub_run_info(app_step.app_id, node.id)
@@ -920,6 +921,9 @@ class Runner:
         if len(inputs) > 1:
             raise Exception(f"App steps cannot have more than one predecessor, but {node.id} has {len(inputs)}")
 
+        run_configuration = app_step.run_configuration
+        instance_id = app_step.instance_id
+
         if isinstance(inputs[0], schema.AppRunConfig):
             app_run_config: schema.AppRunConfig = inputs[0]
             input_data = app_run_config.input
@@ -929,6 +933,12 @@ class Runner:
                 app_run_description = app_run_config.description
             # AppRunConfig options take precedence over decorator options.
             options = app_step.options | app_run_config.get_options()
+            # An AppRunConfig run configuration replaces the decorator one as a whole.
+            if app_run_config.run_configuration is not None:
+                run_configuration = app_run_config.run_configuration
+            # Same for the instance to run on.
+            if app_run_config.instance_id:
+                instance_id = app_run_config.instance_id
         elif isinstance(inputs[0], nextmv.RunResult):
             run_result: nextmv.RunResult = inputs[0]
             input_data = run_result.output
@@ -953,10 +963,10 @@ class Runner:
             run_kwargs["input_dir_path"] = input_data
         else:
             run_kwargs["input"] = input_data
-        if app_step.run_configuration is not None:
-            run_kwargs["configuration"] = app_step.run_configuration
+        if run_configuration is not None:
+            run_kwargs["configuration"] = run_configuration
 
-        return run_kwargs, polling_options, is_dir_mode
+        return run_kwargs, polling_options, is_dir_mode, instance_id
 
     @staticmethod
     def __execute_app_run(
@@ -969,6 +979,7 @@ class Runner:
         polling_options: nextmv.PollingOptions,
         is_dir_mode: bool,
         temp_dir: str,
+        instance_id: str,
     ) -> nextmv.RunResult:
         """
         Execute an app run or return bypass output wrapped as a run result.
@@ -1000,7 +1011,7 @@ class Runner:
             user_email="unavailable",
             metadata=nextmv.Metadata(
                 application_id=app_step.app_id or "unavailable",
-                application_instance_id=app_step.instance_id or "unavailable",
+                application_instance_id=instance_id or "unavailable",
                 application_version_id="unavailable",
                 created_at=datetime.datetime.now(),
                 duration=0.0,
@@ -1067,15 +1078,17 @@ class Runner:
         if node.parent.definition.is_app():
             app_step: decorators.App = node.parent.definition.app
 
-            run_kwargs, polling_options, is_dir_mode = Runner.__prepare_app_run_args(node, inputs, app_step)
+            run_kwargs, polling_options, is_dir_mode, instance_id = Runner.__prepare_app_run_args(
+                node, inputs, app_step
+            )
 
             # Prepare the application itself.
             app = Application(
                 client=client,
                 id=app_step.app_id,
             )
-            if app_step.instance_id is not None and app_step.instance_id != "":
-                app.default_instance_id = app_step.instance_id
+            if instance_id is not None and instance_id != "":
+                app.default_instance_id = instance_id
 
             # We always supply an output directory path in case of implicit multi-file output
             temp_dir = tempfile.mkdtemp(prefix="nextpipe_output_")
@@ -1092,6 +1105,7 @@ class Runner:
                     polling_options=polling_options,
                     is_dir_mode=is_dir_mode,
                     temp_dir=temp_dir,
+                    instance_id=instance_id,
                 )
 
             finally:  # Make sure we clean up temp dir on failure too

{nextpipe-0.7.2 → nextpipe-0.7.3.dev0}/nextpipe/schema.py

@@ -14,11 +14,31 @@ AppRunConfig
 from dataclasses import dataclass, field
 from typing import Any
 
-from dataclasses_json import dataclass_json
+import nextmv
+from dataclasses_json import config, dataclass_json
 
 from . import utils
 
 
+def _encode_run_configuration(
+    run_configuration: nextmv.RunConfiguration | None,
+) -> dict[str, Any] | None:
+    """Encode a run configuration for JSON serialization."""
+
+    return run_configuration.to_dict() if run_configuration is not None else None
+
+
+def _decode_run_configuration(
+    run_configuration: dict[str, Any] | nextmv.RunConfiguration | None,
+) -> nextmv.RunConfiguration | None:
+    """Decode a run configuration from its JSON representation."""
+
+    if run_configuration is None or isinstance(run_configuration, nextmv.RunConfiguration):
+        return run_configuration
+
+    return nextmv.RunConfiguration.from_dict(run_configuration)
+
+
 @dataclass_json
 @dataclass
 class AppOption:
@@ -79,6 +99,16 @@ class AppRunConfig:
         `AppOption` instances, or, simply as a dictionary of key-value pairs.
     name : str, optional
         Name for the run, by default None.
+    description : str, optional
+        Description for the run, by default None.
+    run_configuration : nextmv.RunConfiguration, optional
+        The configuration to apply when running the app, by default None. If given, it
+        replaces the `run_configuration` of the `app` decorator for this run. Note that
+        it replaces it as a whole, i.e., no fields are inherited from the decorator's
+        configuration.
+    instance_id : str, optional
+        The ID of the instance to run, by default None. If given, it replaces the
+        `instance_id` of the `app` decorator for this run.
 
     Examples
     --------
@@ -88,6 +118,17 @@ class AppRunConfig:
     ...     options={"threads": 4},
     ...     name="my-run"
     ... )
+
+    The run configuration and instance can be chosen per run, e.g., to pick an
+    execution class based on the size of the scenario at hand.
+
+    >>> import nextmv
+    >>> from nextpipe import AppRunConfig
+    >>> config = AppRunConfig(
+    ...     input={"data": [1, 2, 3]},
+    ...     run_configuration=nextmv.RunConfiguration(execution_class="8c16gb12h"),
+    ...     instance_id="tuned-for-large",
+    ... )
     """
 
     input: dict[str, Any] | str | Any
@@ -100,6 +141,18 @@ class AppRunConfig:
     """Name for the run."""
     description: str | None = None
     """Description for the run."""
+    run_configuration: nextmv.RunConfiguration | None = field(
+        default=None,
+        metadata=config(
+            encoder=_encode_run_configuration,
+            decoder=_decode_run_configuration,
+        ),
+    )
+    """The configuration to apply when running the app. Replaces the `run_configuration`
+    of the `app` decorator as a whole, if given."""
+    instance_id: str | None = None
+    """The ID of the instance to run. Replaces the `instance_id` of the `app` decorator,
+    if given."""
 
     def get_options(self) -> dict[str, Any]:
         """

nextpipe-0.7.3.dev0/tests/test_flow.py

@@ -0,0 +1,104 @@
+import os
+import sys
+import unittest
+
+import nextmv
+
+from nextpipe import AppRunConfig, decorators
+from nextpipe.flow import Runner
+
+# Add the parent directory to the sys.path to allow imports from the main package. This
+# is meant to help VS Code testing features.
+sys.path.append(os.path.dirname(sys.path[0]))
+
+
+class _FakeNode:
+    """Minimal stand-in for a FlowNode, providing what run arg preparation needs."""
+
+    def __init__(self, id: str = "solve_0"):
+        self.id = id
+        self.cancel = False
+
+
+def _prepare(inputs: list[object], app_step: decorators.App) -> tuple[dict, str]:
+    """Prepare the run kwargs and instance ID for the given inputs and app step."""
+
+    run_kwargs, _, _, instance_id = Runner._Runner__prepare_app_run_args(_FakeNode(), inputs, app_step)
+    return run_kwargs, instance_id
+
+
+class TestPrepareAppRunArgs(unittest.TestCase):
+    def test_no_run_configuration(self):
+        app_step = decorators.App(app_id="echo")
+        run_kwargs, _ = _prepare([{"data": [1, 2, 3]}], app_step)
+        self.assertNotIn("configuration", run_kwargs)
+
+    def test_decorator_run_configuration(self):
+        app_step = decorators.App(
+            app_id="echo",
+            run_configuration=nextmv.RunConfiguration(execution_class="4c8gb12h"),
+        )
+        run_kwargs, _ = _prepare([{"data": [1, 2, 3]}], app_step)
+        self.assertEqual(run_kwargs["configuration"].execution_class, "4c8gb12h")
+
+    def test_app_run_config_run_configuration(self):
+        app_step = decorators.App(app_id="echo")
+        app_run_config = AppRunConfig(
+            input={"data": [1, 2, 3]},
+            run_configuration=nextmv.RunConfiguration(execution_class="8c16gb12h"),
+        )
+        run_kwargs, _ = _prepare([app_run_config], app_step)
+        self.assertEqual(run_kwargs["configuration"].execution_class, "8c16gb12h")
+
+    def test_app_run_config_run_configuration_replaces_decorator(self):
+        """The run configuration of an AppRunConfig replaces the decorator one as a whole."""
+
+        app_step = decorators.App(
+            app_id="echo",
+            run_configuration=nextmv.RunConfiguration(
+                execution_class="4c8gb12h",
+                secrets_collection_id="some-secrets",
+            ),
+        )
+        app_run_config = AppRunConfig(
+            input={"data": [1, 2, 3]},
+            run_configuration=nextmv.RunConfiguration(execution_class="8c16gb12h"),
+        )
+        run_kwargs, _ = _prepare([app_run_config], app_step)
+        self.assertEqual(run_kwargs["configuration"].execution_class, "8c16gb12h")
+        self.assertIsNone(run_kwargs["configuration"].secrets_collection_id)
+
+    def test_app_run_config_without_run_configuration_keeps_decorator(self):
+        app_step = decorators.App(
+            app_id="echo",
+            run_configuration=nextmv.RunConfiguration(execution_class="4c8gb12h"),
+        )
+        app_run_config = AppRunConfig(input={"data": [1, 2, 3]})
+        run_kwargs, _ = _prepare([app_run_config], app_step)
+        self.assertEqual(run_kwargs["configuration"].execution_class, "4c8gb12h")
+
+    def test_no_instance_id(self):
+        app_step = decorators.App(app_id="echo")
+        _, instance_id = _prepare([{"data": [1, 2, 3]}], app_step)
+        self.assertEqual(instance_id, "")
+
+    def test_decorator_instance_id(self):
+        app_step = decorators.App(app_id="echo", instance_id="decorator-instance")
+        _, instance_id = _prepare([{"data": [1, 2, 3]}], app_step)
+        self.assertEqual(instance_id, "decorator-instance")
+
+    def test_app_run_config_instance_id_replaces_decorator(self):
+        app_step = decorators.App(app_id="echo", instance_id="decorator-instance")
+        app_run_config = AppRunConfig(input={"data": [1, 2, 3]}, instance_id="per-run-instance")
+        _, instance_id = _prepare([app_run_config], app_step)
+        self.assertEqual(instance_id, "per-run-instance")
+
+    def test_app_run_config_without_instance_id_keeps_decorator(self):
+        app_step = decorators.App(app_id="echo", instance_id="decorator-instance")
+        app_run_config = AppRunConfig(input={"data": [1, 2, 3]})
+        _, instance_id = _prepare([app_run_config], app_step)
+        self.assertEqual(instance_id, "decorator-instance")
+
+
+if __name__ == "__main__":
+    unittest.main()

nextpipe-0.7.3.dev0/tests/test_schema.py

@@ -0,0 +1,69 @@
+import os
+import sys
+import unittest
+
+import nextmv
+
+from nextpipe import AppOption, AppRunConfig
+
+# Add the parent directory to the sys.path to allow imports from the main package. This
+# is meant to help VS Code testing features.
+sys.path.append(os.path.dirname(sys.path[0]))
+
+
+class TestAppRunConfig(unittest.TestCase):
+    def test_options_dict(self):
+        config = AppRunConfig(input={"data": [1, 2, 3]}, options={"threads": 4, "verbose": True}, name="test-run")
+        options = config.get_options()
+        self.assertEqual(options["threads"], "4")
+        self.assertEqual(options["verbose"], "True")
+
+    def test_options_obj(self):
+        config = AppRunConfig(
+            input={"data": [1, 2, 3]},
+            options=[AppOption(name="threads", value=4), AppOption(name="verbose", value=True)],
+            name="test-run",
+        )
+        options = config.get_options()
+        self.assertEqual(options["threads"], "4")
+        self.assertTrue(options["verbose"], "True")
+
+    def test_run_configuration_default(self):
+        config = AppRunConfig(input={"data": [1, 2, 3]})
+        self.assertIsNone(config.run_configuration)
+
+    def test_run_configuration(self):
+        config = AppRunConfig(
+            input={"data": [1, 2, 3]},
+            run_configuration=nextmv.RunConfiguration(execution_class="8c16gb12h"),
+        )
+        self.assertEqual(config.run_configuration.execution_class, "8c16gb12h")
+
+    def test_run_configuration_json_round_trip(self):
+        config = AppRunConfig(
+            input={"data": [1, 2, 3]},
+            options={"threads": 4},
+            run_configuration=nextmv.RunConfiguration(
+                execution_class="8c16gb12h",
+                secrets_collection_id="some-secrets",
+            ),
+        )
+        restored = AppRunConfig.from_json(config.to_json())
+        self.assertIsInstance(restored.run_configuration, nextmv.RunConfiguration)
+        self.assertEqual(restored.run_configuration.execution_class, "8c16gb12h")
+        self.assertEqual(restored.run_configuration.secrets_collection_id, "some-secrets")
+
+    def test_run_configuration_json_round_trip_none(self):
+        config = AppRunConfig(input={"data": [1, 2, 3]})
+        restored = AppRunConfig.from_json(config.to_json())
+        self.assertIsNone(restored.run_configuration)
+
+    def test_instance_id_default(self):
+        config = AppRunConfig(input={"data": [1, 2, 3]})
+        self.assertIsNone(config.instance_id)
+
+    def test_instance_id(self):
+        config = AppRunConfig(input={"data": [1, 2, 3]}, instance_id="tuned-for-large")
+        self.assertEqual(config.instance_id, "tuned-for-large")
+        restored = AppRunConfig.from_json(config.to_json())
+        self.assertEqual(restored.instance_id, "tuned-for-large")

nextpipe-0.7.2/nextpipe/__about__.py

@@ -1 +0,0 @@
-__version__ = "v0.7.2"

nextpipe-0.7.2/tests/test_schema.py

@@ -1,27 +0,0 @@
-import os
-import sys
-import unittest
-
-from nextpipe import AppOption, AppRunConfig
-
-# Add the parent directory to the sys.path to allow imports from the main package. This
-# is meant to help VS Code testing features.
-sys.path.append(os.path.dirname(sys.path[0]))
-
-
-class TestAppRunConfig(unittest.TestCase):
-    def test_options_dict(self):
-        config = AppRunConfig(input={"data": [1, 2, 3]}, options={"threads": 4, "verbose": True}, name="test-run")
-        options = config.get_options()
-        self.assertEqual(options["threads"], "4")
-        self.assertEqual(options["verbose"], "True")
-
-    def test_options_obj(self):
-        config = AppRunConfig(
-            input={"data": [1, 2, 3]},
-            options=[AppOption(name="threads", value=4), AppOption(name="verbose", value=True)],
-            name="test-run",
-        )
-        options = config.get_options()
-        self.assertEqual(options["threads"], "4")
-        self.assertTrue(options["verbose"], "True")