nextmv 0.29.4.dev1__tar.gz → 0.29.5.dev1__tar.gz

{nextmv-0.29.4.dev1 → nextmv-0.29.5.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextmv
-Version: 0.29.4.dev1
+Version: 0.29.5.dev1
 Summary: The all-purpose Python SDK for Nextmv
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextmv-py.docs.nextmv.io/en/latest/nextmv/

nextmv-0.29.5.dev1/nextmv/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.29.5.dev.1"

{nextmv-0.29.4.dev1 → nextmv-0.29.5.dev1}/nextmv/cloud/__init__.py

@@ -46,6 +46,7 @@ from .run import ErrorLog as ErrorLog
 from .run import ExternalRunResult as ExternalRunResult
 from .run import Format as Format
 from .run import FormatInput as FormatInput
+from .run import FormatOutput as FormatOutput
 from .run import Metadata as Metadata
 from .run import RunConfiguration as RunConfiguration
 from .run import RunInformation as RunInformation

{nextmv-0.29.4.dev1 → nextmv-0.29.5.dev1}/nextmv/cloud/application.py

@@ -56,13 +56,14 @@ from nextmv.cloud.run import (
     ExternalRunResult,
     Format,
     FormatInput,
+    FormatOutput,
     RunConfiguration,
     RunInformation,
     RunLog,
     RunResult,
     TrackedRun,
 )
-from nextmv.cloud.safe import _name_and_id
+from nextmv.cloud.safe import _name_and_id, _safe_id
 from nextmv.cloud.scenario import Scenario, ScenarioInputType, _option_sets, _scenarios_by_id
 from nextmv.cloud.secrets import Secret, SecretsCollection, SecretsCollectionSummary
 from nextmv.cloud.status import StatusV2
@@ -71,7 +72,7 @@ from nextmv.input import Input, InputFormat
 from nextmv.logger import log
 from nextmv.model import Model, ModelConfiguration
 from nextmv.options import Options
-from nextmv.output import Output
+from nextmv.output import Output, OutputFormat
 
 # Maximum size of the run input/output in bytes. This constant defines the
 # maximum allowed size for run inputs and outputs. When the size exceeds this
@@ -1127,16 +1128,26 @@ class Application:
                     f"batch experiment {id} does not exist, input_set_id must be defined to create a new one"
                 ) from e
         else:
-            runs = [
-                BatchExperimentRun(
-                    instance_id=candidate_instance_id,
-                    input_set_id=input_set_id,
-                ),
-                BatchExperimentRun(
-                    instance_id=baseline_instance_id,
-                    input_set_id=input_set_id,
-                ),
-            ]
+            # Get all input IDs from the input set.
+            input_set = self.input_set(input_set_id=input_set_id)
+            if len(input_set.input_ids) == 0:
+                raise ValueError(f"input set {input_set_id} does not contain any inputs")
+            runs = []
+            for input_id in input_set.input_ids:
+                runs.append(
+                    BatchExperimentRun(
+                        instance_id=candidate_instance_id,
+                        input_set_id=input_set_id,
+                        input_id=input_id,
+                    )
+                )
+                runs.append(
+                    BatchExperimentRun(
+                        instance_id=baseline_instance_id,
+                        input_set_id=input_set_id,
+                        input_id=input_id,
+                    )
+                )
             batch_experiment_id = self.new_batch_experiment(
                 name=name,
                 description=description,
@@ -1588,7 +1599,10 @@ class Application:
         if format is not None:
             payload["format"] = format.to_dict() if isinstance(format, Format) else format
         else:
-            payload["format"] = Format(format_input=FormatInput(input_type=InputFormat.JSON)).to_dict()
+            payload["format"] = Format(
+                format_input=FormatInput(input_type=InputFormat.JSON),
+                format_output=FormatOutput(output_type=OutputFormat.JSON),
+            ).to_dict()
 
         response = self.client.request(
             method="POST",
@@ -1610,7 +1624,7 @@ class Application:
         batch_experiment_id: Optional[str] = None,
         external_result: Optional[Union[ExternalRunResult, dict[str, Any]]] = None,
         json_configurations: Optional[dict[str, Any]] = None,
-        dir_path: Optional[str] = None,
+        input_dir_path: Optional[str] = None,
     ) -> str:
         """
         Submit an input to start a new run of the application. Returns the
@@ -1627,14 +1641,14 @@ class Application:
             input data is extracted from the `.data` property.
 
             If you want to work with `nextmv.InputFormat.CSV_ARCHIVE` or
-            `nextmv.InputFormat.MULTI_FILE`, you should use the `dir_path`
+            `nextmv.InputFormat.MULTI_FILE`, you should use the `input_dir_path`
             argument instead. This argument takes precedence over the `input`.
-            If `dir_path` is specified, this function looks for files in that
+            If `input_dir_path` is specified, this function looks for files in that
             directory and tars them, to later be uploaded using the
-            `upload_large_input` method. If both the `dir_path` and `input`
+            `upload_large_input` method. If both the `input_dir_path` and `input`
             arguments are provided, the `input` is ignored.
 
-            When `dir_path` is specified, the `configuration` argument must
+            When `input_dir_path` is specified, the `configuration` argument must
             also be provided. More specifically, the
             `RunConfiguration.format.format_input.input_type` parameter
             dictates what kind of input is being submitted to the Nextmv Cloud.
@@ -1686,12 +1700,12 @@ class Application:
         json_configurations: Optional[dict[str, Any]]
             Optional configurations for JSON serialization. This is used to
             customize the serialization before data is sent.
-        dir_path: Optional[str]
+        input_dir_path: Optional[str]
             Path to a directory containing input files. If specified, the
             function will package the files in the directory into a tar file
             and upload it as a large input. This is useful for input formats
             like `nextmv.InputFormat.CSV_ARCHIVE` or `nextmv.InputFormat.MULTI_FILE`.
-            If both `input` and `dir_path` are specified, the `input` is
+            If both `input` and `input_dir_path` are specified, the `input` is
             ignored, and the files in the directory are used instead.
 
         Returns
@@ -1708,17 +1722,17 @@ class Application:
             not `JSON`. If the final `options` are not of type `dict[str,str]`.
         """
 
-        self.__validate_dir_path_and_configuration(dir_path, configuration)
+        self.__validate_dir_path_and_configuration(input_dir_path, configuration)
 
         tar_file = ""
-        if dir_path is not None and dir_path != "":
-            if not os.path.exists(dir_path):
-                raise ValueError(f"Directory {dir_path} does not exist.")
+        if input_dir_path is not None and input_dir_path != "":
+            if not os.path.exists(input_dir_path):
+                raise ValueError(f"Directory {input_dir_path} does not exist.")
 
-            if not os.path.isdir(dir_path):
-                raise ValueError(f"Path {dir_path} is not a directory.")
+            if not os.path.isdir(input_dir_path):
+                raise ValueError(f"Path {input_dir_path} is not a directory.")
 
-            tar_file = self.__package_inputs(dir_path)
+            tar_file = self.__package_inputs(input_dir_path)
 
         input_data = None
         if isinstance(input, BaseModel):
@@ -1775,7 +1789,7 @@ class Application:
             )
         else:
             configuration = RunConfiguration()
-            configuration.resolve(input=input, dir_path=dir_path)
+            configuration.resolve(input=input, dir_path=input_dir_path)
             configuration_dict = configuration.to_dict()
 
         payload["configuration"] = configuration_dict
@@ -1814,7 +1828,8 @@ class Application:
         batch_experiment_id: Optional[str] = None,
         external_result: Optional[Union[ExternalRunResult, dict[str, Any]]] = None,
         json_configurations: Optional[dict[str, Any]] = None,
-        dir_path: Optional[str] = None,
+        input_dir_path: Optional[str] = None,
+        output_dir_path: Optional[str] = ".",
     ) -> RunResult:
         """
         Submit an input to start a new run of the application and poll for the
@@ -1833,14 +1848,14 @@ class Application:
             input data is extracted from the `.data` property.
 
             If you want to work with `nextmv.InputFormat.CSV_ARCHIVE` or
-            `nextmv.InputFormat.MULTI_FILE`, you should use the `dir_path`
+            `nextmv.InputFormat.MULTI_FILE`, you should use the `input_dir_path`
             argument instead. This argument takes precedence over the `input`.
-            If `dir_path` is specified, this function looks for files in that
+            If `input_dir_path` is specified, this function looks for files in that
             directory and tars them, to later be uploaded using the
-            `upload_large_input` method. If both the `dir_path` and `input`
+            `upload_large_input` method. If both the `input_dir_path` and `input`
             arguments are provided, the `input` is ignored.
 
-            When `dir_path` is specified, the `configuration` argument must
+            When `input_dir_path` is specified, the `configuration` argument must
             also be provided. More specifically, the
             `RunConfiguration.format.format_input.input_type` parameter
             dictates what kind of input is being submitted to the Nextmv Cloud.
@@ -1897,13 +1912,17 @@ class Application:
         json_configurations: Optional[dict[str, Any]]
             Optional configurations for JSON serialization. This is used to
             customize the serialization before data is sent.
-        dir_path: Optional[str]
+        input_dir_path: Optional[str]
             Path to a directory containing input files. If specified, the
             function will package the files in the directory into a tar file
             and upload it as a large input. This is useful for input formats
             like `nextmv.InputFormat.CSV_ARCHIVE` or `nextmv.InputFormat.MULTI_FILE`.
-            If both `input` and `dir_path` are specified, the `input` is
+            If both `input` and `input_dir_path` are specified, the `input` is
             ignored, and the files in the directory are used instead.
+        output_dir_path : Optional[str], default="."
+            Path to a directory where non-JSON output files will be saved. This is
+            required if the output is non-JSON. If the directory does not exist, it
+            will be created. Uses the current directory by default.
 
         Returns
         ----------
@@ -1936,12 +1955,13 @@ class Application:
             batch_experiment_id=batch_experiment_id,
             external_result=external_result,
             json_configurations=json_configurations,
-            dir_path=dir_path,
+            input_dir_path=input_dir_path,
         )
 
         return self.run_result_with_polling(
             run_id=run_id,
             polling_options=polling_options,
+            output_dir_path=output_dir_path,
         )
 
     def new_scenario_test(
@@ -2215,10 +2235,13 @@ class Application:
         if exist_ok and self.version_exists(version_id=id):
             return self.version(version_id=id)
 
-        payload = {}
+        if id is None:
+            id = _safe_id(prefix="version")
+
+        payload = {
+            "id": id,
+        }
 
-        if id is not None:
-            payload["id"] = id
         if name is not None:
             payload["name"] = name
         if description is not None:
@@ -2501,7 +2524,7 @@ class Application:
         )
         return RunLog.from_dict(response.json())
 
-    def run_result(self, run_id: str) -> RunResult:
+    def run_result(self, run_id: str, output_dir_path: Optional[str] = ".") -> RunResult:
         """
         Get the result of a run.
 
@@ -2511,6 +2534,10 @@ class Application:
         ----------
         run_id : str
             ID of the run to get results for.
+        output_dir_path : Optional[str], default="."
+            Path to a directory where non-JSON output files will be saved. This is
+            required if the output is non-JSON. If the directory does not exist, it
+            will be created. Uses the current directory by default.
 
         Returns
         -------
@@ -2531,12 +2558,17 @@ class Application:
 
         run_information = self.run_metadata(run_id=run_id)
 
-        return self.__run_result(run_id=run_id, run_information=run_information)
+        return self.__run_result(
+            run_id=run_id,
+            run_information=run_information,
+            output_dir_path=output_dir_path,
+        )
 
     def run_result_with_polling(
         self,
         run_id: str,
         polling_options: PollingOptions = _DEFAULT_POLLING_OPTIONS,
+        output_dir_path: Optional[str] = ".",
     ) -> RunResult:
         """
         Get the result of a run with polling.
@@ -2551,6 +2583,10 @@ class Application:
             ID of the run to retrieve the result for.
         polling_options : PollingOptions, default=_DEFAULT_POLLING_OPTIONS
             Options to use when polling for the run result.
+        output_dir_path : Optional[str], default="."
+            Path to a directory where non-JSON output files will be saved. This is
+            required if the output is non-JSON. If the directory does not exist, it
+            will be created. Uses the current directory by default.
 
         Returns
         -------
@@ -2592,7 +2628,11 @@ class Application:
 
         run_information = poll(polling_options=polling_options, polling_func=polling_func)
 
-        return self.__run_result(run_id=run_id, run_information=run_information)
+        return self.__run_result(
+            run_id=run_id,
+            run_information=run_information,
+            output_dir_path=output_dir_path,
+        )
 
     def scenario_test(self, scenario_test_id: str) -> BatchExperiment:
         """
@@ -2707,6 +2747,7 @@ class Application:
         tracked_run: TrackedRun,
         polling_options: PollingOptions = _DEFAULT_POLLING_OPTIONS,
         instance_id: Optional[str] = None,
+        output_dir_path: Optional[str] = ".",
     ) -> RunResult:
         """
         Track an external run and poll for the result. This is a convenience
@@ -2723,6 +2764,10 @@ class Application:
         instance_id: Optional[str]
             Optional instance ID if you want to associate your tracked run with
             an instance.
+        output_dir_path : Optional[str], default="."
+            Path to a directory where non-JSON output files will be saved. This is
+            required if the output is non-JSON. If the directory does not exist, it
+            will be created. Uses the current directory by default.
 
         Returns
         -------
@@ -2747,12 +2792,13 @@ class Application:
         return self.run_result_with_polling(
             run_id=run_id,
             polling_options=polling_options,
+            output_dir_path=output_dir_path,
         )
 
     def update_instance(
         self,
         id: str,
-        name: str,
+        name: Optional[str] = None,
         version_id: Optional[str] = None,
         description: Optional[str] = None,
         configuration: Optional[InstanceConfiguration] = None,
@@ -2764,14 +2810,14 @@ class Application:
         ----------
         id : str
             ID of the instance to update.
-        name : str
-            Name of the instance.
+        name : Optional[str], default=None
+            Optional name of the instance.
         version_id : Optional[str], default=None
-            ID of the version to associate the instance with.
+            Optional ID of the version to associate the instance with.
         description : Optional[str], default=None
-            Description of the instance.
+            Optional description of the instance.
         configuration : Optional[InstanceConfiguration], default=None
-            Configuration to use for the instance.
+            Optional configuration to use for the instance.
 
         Returns
         -------
@@ -2784,12 +2830,21 @@ class Application:
             If the response status code is not 2xx.
         """
 
-        payload = {}
+        # Get the instance as it currently exsits.
+        instance = self.instance(id)
+        instance_dict = instance.to_dict()
+
+        payload = {
+            "name": instance_dict["name"],
+            "version_id": instance_dict["version_id"],
+            "description": instance_dict["description"],
+            "configuration": instance_dict["configuration"],
+        }
 
-        if version_id is not None:
-            payload["version_id"] = version_id
         if name is not None:
             payload["name"] = name
+        if version_id is not None:
+            payload["version_id"] = version_id
         if description is not None:
             payload["description"] = description
         if configuration is not None:
@@ -2806,8 +2861,8 @@ class Application:
     def update_batch_experiment(
         self,
         batch_experiment_id: str,
-        name: str,
-        description: str,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
     ) -> BatchExperimentInformation:
         """
         Update a batch experiment.
@@ -2816,10 +2871,10 @@ class Application:
         ----------
         batch_experiment_id : str
             ID of the batch experiment to update.
-        name : str
-            Name of the batch experiment.
-        description : str
-            Description of the batch experiment.
+        name : Optional[str], default=None
+            Optional name of the batch experiment.
+        description : Optional[str], default=None
+            Optional description of the batch experiment.
 
         Returns
         -------
@@ -2832,10 +2887,13 @@ class Application:
             If the response status code is not 2xx.
         """
 
-        payload = {
-            "name": name,
-            "description": description,
-        }
+        payload = {}
+
+        if name is not None:
+            payload["name"] = name
+        if description is not None:
+            payload["description"] = description
+
         response = self.client.request(
             method="PATCH",
             endpoint=f"{self.experiments_endpoint}/batch/{batch_experiment_id}",
@@ -2847,9 +2905,9 @@ class Application:
     def update_managed_input(
         self,
         managed_input_id: str,
-        name: str,
-        description: str,
-    ) -> None:
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> ManagedInput:
         """
         Update a managed input.
 
@@ -2857,15 +2915,15 @@ class Application:
         ----------
         managed_input_id : str
             ID of the managed input to update.
-        name : str
-            Name of the managed input.
-        description : str
-            Description of the managed input.
+        name : Optional[str], default=None
+            Optional new name for the managed input.
+        description : Optional[str], default=None
+            Optional new description for the managed input.
 
         Returns
         -------
-        None
-            No return value.
+        ManagedInput
+            The updated managed input.
 
         Raises
         ------
@@ -2873,21 +2931,32 @@ class Application:
             If the response status code is not 2xx.
         """
 
+        managed_input = self.managed_input(managed_input_id)
+        managed_input_dict = managed_input.to_dict()
+
         payload = {
-            "name": name,
-            "description": description,
+            "name": managed_input_dict["name"],
+            "description": managed_input_dict["description"],
         }
-        _ = self.client.request(
+
+        if name is not None:
+            payload["name"] = name
+        if description is not None:
+            payload["description"] = description
+
+        response = self.client.request(
             method="PUT",
             endpoint=f"{self.endpoint}/inputs/{managed_input_id}",
             payload=payload,
         )
 
+        return ManagedInput.from_dict(response.json())
+
     def update_scenario_test(
         self,
         scenario_test_id: str,
-        name: str,
-        description: str,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
     ) -> BatchExperimentInformation:
         """
         Update a scenario test.
@@ -2900,10 +2969,10 @@ class Application:
         ----------
         scenario_test_id : str
             ID of the scenario test to update.
-        name : str
-            New name for the scenario test.
-        description : str
-            New description for the scenario test.
+        name : Optional[str], default=None
+            Optional new name for the scenario test.
+        description : Optional[str], default=None
+            Optional new description for the scenario test.
 
         Returns
         -------
@@ -2935,9 +3004,9 @@ class Application:
     def update_secrets_collection(
         self,
         secrets_collection_id: str,
-        name: str,
-        description: str,
-        secrets: list[Secret],
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        secrets: Optional[list[Secret]] = None,
     ) -> SecretsCollectionSummary:
         """
         Update a secrets collection.
@@ -2950,13 +3019,13 @@ class Application:
         ----------
         secrets_collection_id : str
             ID of the secrets collection to update.
-        name : str
-            New name for the secrets collection.
-        description : str
-            New description for the secrets collection.
-        secrets : list[Secret]
-            List of secrets to update. Each secret should be an instance of the
-            Secret class containing a key and value.
+        name : Optional[str], default=None
+            Optional new name for the secrets collection.
+        description : Optional[str], default=None
+            Optional new description for the secrets collection.
+        secrets : Optional[list[Secret]], default=None
+            Optional list of secrets to update. Each secret should be an
+            instance of the Secret class containing a key and value.
 
         Returns
         -------
@@ -2988,14 +3057,22 @@ class Application:
         'api-secrets'
         """
 
-        if len(secrets) == 0:
-            raise ValueError("secrets must be provided")
+        collection = self.secrets_collection(secrets_collection_id)
+        collection_dict = collection.to_dict()
 
         payload = {
-            "name": name,
-            "description": description,
-            "secrets": [secret.to_dict() for secret in secrets],
+            "name": collection_dict["name"],
+            "description": collection_dict["description"],
+            "secrets": collection_dict["secrets"],
         }
+
+        if name is not None:
+            payload["name"] = name
+        if description is not None:
+            payload["description"] = description
+        if secrets is not None and len(secrets) > 0:
+            payload["secrets"] = [secret.to_dict() for secret in secrets]
+
         response = self.client.request(
             method="PUT",
             endpoint=f"{self.endpoint}/secrets/{secrets_collection_id}",
@@ -3229,6 +3306,7 @@ class Application:
         self,
         run_id: str,
         run_information: RunInformation,
+        output_dir_path: Optional[str] = ".",
     ) -> RunResult:
         """
         Get the result of a run.
@@ -3245,6 +3323,10 @@ class Application:
             ID of the run to retrieve the result for.
         run_information : RunInformation
             Information about the run, including metadata such as output size.
+        output_dir_path : Optional[str], default="."
+            Path to a directory where non-JSON output files will be saved. This is
+            required if the output is non-JSON. If the directory does not exist, it
+            will be created. Uses the current directory by default.
 
         Returns
         -------
@@ -3265,10 +3347,13 @@ class Application:
         a download URL and fetch the output data separately.
         """
         query_params = None
-        large_output = False
-        if run_information.metadata.output_size > _MAX_RUN_SIZE:
+        use_presigned_url = False
+        if (
+            run_information.metadata.format.format_output.output_type != OutputFormat.JSON
+            or run_information.metadata.output_size > _MAX_RUN_SIZE
+        ):
             query_params = {"format": "url"}
-            large_output = True
+            use_presigned_url = True
 
         response = self.client.request(
             method="GET",
@@ -3278,7 +3363,7 @@ class Application:
         result = RunResult.from_dict(response.json())
         result.console_url = self.__console_url(result.id)
 
-        if not large_output:
+        if not use_presigned_url or result.metadata.status_v2 != StatusV2.succeeded:
             return result
 
         download_url = DownloadURL.from_dict(response.json()["output"])
@@ -3287,7 +3372,24 @@ class Application:
             endpoint=download_url.url,
             headers={"Content-Type": "application/json"},
         )
-        result.output = download_response.json()
+
+        # See whether we can attach the output directly or need to save to the given
+        # directory
+        if run_information.metadata.format.format_output.output_type != OutputFormat.JSON:
+            if not output_dir_path or output_dir_path == "":
+                raise ValueError(
+                    "If the output format is not JSON, an output_dir_path must be provided.",
+                )
+            if not os.path.exists(output_dir_path):
+                os.makedirs(output_dir_path, exist_ok=True)
+            # Save .tar.gz file to a temp directory and extract contents to output_dir_path
+            with tempfile.TemporaryDirectory() as tmpdirname:
+                temp_tar_path = os.path.join(tmpdirname, f"{run_id}.tar.gz")
+                with open(temp_tar_path, "wb") as f:
+                    f.write(download_response.content)
+                shutil.unpack_archive(temp_tar_path, output_dir_path)
+        else:
+            result.output = download_response.json()
 
         return result
 
@@ -3325,7 +3427,14 @@ class Application:
         }
 
         if manifest.configuration is not None and manifest.configuration.options is not None:
-            activation_request["requirements"]["options"] = manifest.configuration.options.to_dict()
+            options = manifest.configuration.options.to_dict()
+            if "format" in options and isinstance(options["format"], list):
+                # the endpoint expects a dictionary with a template key having a list of strings
+                # the app.yaml however defines format as a list of strings, so we need to convert it here
+                options["format"] = {
+                    "template": options["format"],
+                }
+            activation_request["requirements"]["options"] = options
 
         response = self.client.request(
             method="PUT",