nextmv 0.33.0__tar.gz → 0.34.0.dev0__tar.gz

{nextmv-0.33.0 → nextmv-0.34.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextmv
-Version: 0.33.0
+Version: 0.34.0.dev0
 Summary: The all-purpose Python SDK for Nextmv
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextmv-py.docs.nextmv.io/en/latest/nextmv/

nextmv-0.34.0.dev0/nextmv/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.34.0.dev0"

{nextmv-0.33.0 → nextmv-0.34.0.dev0}/nextmv/__init__.py

@@ -69,6 +69,7 @@ from .run import RunResult as RunResult
 from .run import RunType as RunType
 from .run import RunTypeConfiguration as RunTypeConfiguration
 from .run import StatisticsIndicator as StatisticsIndicator
+from .run import SyncedRun as SyncedRun
 from .run import TrackedRun as TrackedRun
 from .run import TrackedRunStatus as TrackedRunStatus
 from .run import run_duration as run_duration

{nextmv-0.33.0 → nextmv-0.34.0.dev0}/nextmv/cloud/application.py

@@ -58,7 +58,7 @@ from nextmv.logger import log
 from nextmv.manifest import Manifest
 from nextmv.model import Model, ModelConfiguration
 from nextmv.options import Options
-from nextmv.output import Output, OutputFormat
+from nextmv.output import ASSETS_KEY, STATISTICS_KEY, Asset, Output, OutputFormat, Statistics
 from nextmv.polling import DEFAULT_POLLING_OPTIONS, PollingOptions, poll
 from nextmv.run import (
     ExternalRunResult,
@@ -187,17 +187,20 @@ class Application:
         >>> app = Application.new(client=client, name="My New App", id="my-app")
         """
 
+        if id is None:
+            id = safe_id("app")
+
         if exist_ok and cls.exists(client=client, id=id):
             return Application(client=client, id=id)
 
         payload = {
             "name": name,
+            "id": id,
         }
 
         if description is not None:
             payload["description"] = description
-        if id is not None:
-            payload["id"] = id
+
         if is_workflow is not None:
             payload["is_pipeline"] = is_workflow
 
@@ -1790,7 +1793,7 @@ class Application:
             when the run is part of a batch experiment.
         external_result: Optional[Union[ExternalRunResult, dict[str, Any]]]
             External result to use for the run. This can be a
-            `cloud.ExternalRunResult` object or a dict. If the object is used,
+            `nextmv.ExternalRunResult` object or a dict. If the object is used,
             then the `.to_dict()` method is applied to extract the
             configuration. This is used when the run is an external run. We
             suggest that instead of specifying this parameter, you use the
@@ -1820,8 +1823,6 @@ class Application:
             not `JSON`. If the final `options` are not of type `dict[str,str]`.
         """
 
-        self.__validate_input_dir_path_and_configuration(input_dir_path, configuration)
-
         tar_file = ""
         if input_dir_path is not None and input_dir_path != "":
             if not os.path.exists(input_dir_path):
@@ -1879,6 +1880,7 @@ class Application:
         query_params = {}
         if instance_id is not None or self.default_instance_id is not None:
             query_params["instance_id"] = instance_id if instance_id is not None else self.default_instance_id
+
         response = self.client.request(
             method="POST",
             endpoint=f"{self.endpoint}/runs",
@@ -2993,6 +2995,7 @@ class Application:
             execution_duration=tracked_run.duration,
         )
 
+        # Handle the stderr logs if provided.
         if tracked_run.logs is not None:
             url_stderr = self.upload_url()
             self.upload_large_input(input=tracked_run.logs_text(), upload_url=url_stderr)
@@ -3001,6 +3004,47 @@ class Application:
         if tracked_run.error is not None and tracked_run.error != "":
             external_result.error_message = tracked_run.error
 
+        # Handle the statistics upload if provided.
+        stats = tracked_run.statistics
+        if stats is not None:
+            if isinstance(stats, Statistics):
+                stats_dict = stats.to_dict()
+                stats_dict = {STATISTICS_KEY: stats_dict}
+            elif isinstance(stats, dict):
+                stats_dict = stats
+                if STATISTICS_KEY not in stats_dict:
+                    stats_dict = {STATISTICS_KEY: stats_dict}
+            else:
+                raise ValueError("tracked_run.statistics must be either a `Statistics` or `dict` object")
+
+            url_stats = self.upload_url()
+            self.upload_large_input(input=stats_dict, upload_url=url_stats)
+            external_result.statistics_upload_id = url_stats.upload_id
+
+        # Handle the assets upload if provided.
+        assets = tracked_run.assets
+        if assets is not None:
+            if isinstance(assets, list):
+                assets_list = []
+                for ix, asset in enumerate(assets):
+                    if isinstance(asset, Asset):
+                        assets_list.append(asset.to_dict())
+                    elif isinstance(asset, dict):
+                        assets_list.append(asset)
+                    else:
+                        raise ValueError(f"tracked_run.assets, index {ix} must be an `Asset` or `dict` object")
+                assets_dict = {ASSETS_KEY: assets_list}
+            elif isinstance(assets, dict):
+                assets_dict = assets
+                if ASSETS_KEY not in assets_dict:
+                    assets_dict = {ASSETS_KEY: assets_dict}
+            else:
+                raise ValueError("tracked_run.assets must be either a `list[Asset]`, `list[dict]`, or `dict` object")
+
+            url_assets = self.upload_url()
+            self.upload_large_input(input=assets_dict, upload_url=url_assets)
+            external_result.assets_upload_id = url_assets.upload_id
+
         return self.new_run(
             upload_id=url_input.upload_id,
             external_result=external_result,
@@ -3860,49 +3904,6 @@ class Application:
 
         raise ValueError(f"Unknown scenario input type: {scenario.scenario_input.scenario_input_type}")
 
-    def __validate_input_dir_path_and_configuration(
-        self,
-        input_dir_path: Optional[str],
-        configuration: Optional[Union[RunConfiguration, dict[str, Any]]],
-    ) -> None:
-        """
-        Auxiliary function to validate the directory path and configuration.
-        """
-        input_type = self.__get_input_type(configuration)
-
-        # If no explicit input type is defined, there is nothing to validate.
-        if input_type is None:
-            return
-
-        # Validate that the input directory path is provided when explicitly required.
-        dir_types = (InputFormat.MULTI_FILE, InputFormat.CSV_ARCHIVE)
-        if input_type in dir_types and not input_dir_path:
-            raise ValueError(
-                f"If RunConfiguration.format.format_input.input_type is set to {input_type}, "
-                "then input_dir_path must be provided.",
-            )
-
-    def __get_input_type(self, config: Union[RunConfiguration, dict[str, Any]]) -> Optional[InputFormat]:
-        """
-        Auxiliary function to extract the input type from the run configuration.
-        """
-
-        if config is None:
-            return None
-
-        if isinstance(config, dict):
-            config = RunConfiguration.from_dict(config)
-
-        if (
-            isinstance(config, RunConfiguration)
-            and config.format is not None
-            and config.format.format_input is not None
-            and config.format.format_input.input_type is not None
-        ):
-            return config.format.format_input.input_type
-
-        return None
-
     def __package_inputs(self, dir_path: str) -> str:
         """
         This is an auxiliary function for packaging the inputs found in the

{nextmv-0.33.0 → nextmv-0.34.0.dev0}/nextmv/local/application.py

@@ -35,9 +35,19 @@ from nextmv.local.runner import run
 from nextmv.logger import log
 from nextmv.manifest import Manifest
 from nextmv.options import Options
-from nextmv.output import OUTPUTS_KEY, SOLUTIONS_KEY, OutputFormat
+from nextmv.output import ASSETS_KEY, OUTPUTS_KEY, SOLUTIONS_KEY, STATISTICS_KEY, OutputFormat
 from nextmv.polling import DEFAULT_POLLING_OPTIONS, PollingOptions, poll
-from nextmv.run import ErrorLog, Format, Run, RunConfiguration, RunInformation, RunResult, TrackedRun, TrackedRunStatus
+from nextmv.run import (
+    ErrorLog,
+    Format,
+    Run,
+    RunConfiguration,
+    RunInformation,
+    RunResult,
+    SyncedRun,
+    TrackedRun,
+    TrackedRunStatus,
+)
 from nextmv.safe import safe_id
 from nextmv.status import StatusV2
 
@@ -995,25 +1005,10 @@ class Application:
         input_type = run_result.metadata.format.format_input.input_type
 
         # Skip runs that have already been synced.
-        already_synced = run_result.synced_run_id is not None and run_result.synced_at is not None
+        synced_run, already_synced = run_result.is_synced(app_id=target.id, instance_id=instance_id)
         if already_synced:
             if verbose:
-                log(f"   ⏭️  Skipping local run `{run_id}`, already synced at {run_result.synced_at.isoformat()}.")
-
-            return False
-
-        # Skip runs that don't have the supported type. TODO: delete this when
-        # external runs support CSV_ARCHIVE and MULTI_FILE. Right now,
-        # submitting an external result with a new run is limited to JSON and
-        # TEXT. After this if statement is removed, the rest of the code should
-        # work with CSV_ARCHIVE and MULTI_FILE as well, as using the input dir
-        # path is already considered.
-        if input_type not in {InputFormat.JSON, InputFormat.TEXT}:
-            if verbose:
-                log(
-                    f"   ⏭️  Skipping local run `{run_id}`, unsupported input type: {input_type.value}. "
-                    f"Supported types are: {[InputFormat.JSON.value, InputFormat.TEXT.value]}",
-                )
+                log(f"   ⏭️  Skipping local run `{run_id}`, already synced with {synced_run.to_dict()}.")
 
             return False
 
@@ -1031,7 +1026,7 @@ class Application:
         # Read the logs of the run and place each line as an element in a list
         run_dir = os.path.join(runs_dir, run_id)
         with open(os.path.join(run_dir, LOGS_KEY, LOGS_FILE)) as f:
-            stderr_logs = f.readlines()
+            stderr_logs = [line.rstrip("\n") for line in f.readlines()]
 
         # Create the tracked run object and start configuring it.
         tracked_run = TrackedRun(
@@ -1055,11 +1050,28 @@ class Application:
             tracked_run.input_dir_path = inputs_path
 
         # Resolve the output according to its type.
-        if run_result.metadata.format.format_output.output_type == OutputFormat.JSON:
+        output_type = run_result.metadata.format.format_output.output_type
+        if output_type == OutputFormat.JSON:
             tracked_run.output = run_result.output
         else:
             tracked_run.output_dir_path = os.path.join(run_dir, OUTPUTS_KEY, SOLUTIONS_KEY)
 
+        # Resolve the statistics according to their type and presence. If
+        # working with JSON, the statistics should be resolved from the output.
+        if output_type in {OutputFormat.CSV_ARCHIVE, OutputFormat.MULTI_FILE}:
+            stats_file_path = os.path.join(run_dir, OUTPUTS_KEY, STATISTICS_KEY, f"{STATISTICS_KEY}.json")
+            if os.path.exists(stats_file_path):
+                with open(stats_file_path) as f:
+                    tracked_run.statistics = json.load(f)
+
+        # Resolve the assets according to their type and presence. If working
+        # with JSON, the assets should be resolved from the output.
+        if output_type in {OutputFormat.CSV_ARCHIVE, OutputFormat.MULTI_FILE}:
+            assets_file_path = os.path.join(run_dir, OUTPUTS_KEY, ASSETS_KEY, f"{ASSETS_KEY}.json")
+            if os.path.exists(assets_file_path):
+                with open(assets_file_path) as f:
+                    tracked_run.assets = json.load(f)
+
         # Actually sync the run by tracking it remotely on Nextmv Cloud.
         configuration = RunConfiguration(
             format=Format(
@@ -1074,13 +1086,18 @@ class Application:
         )
 
         # Mark the local run as synced by updating the local run info.
-        run_result.synced_run_id = tracked_id
-        run_result.synced_at = datetime.now(timezone.utc)
+        synced_run = SyncedRun(
+            run_id=tracked_id,
+            synced_at=datetime.now(timezone.utc),
+            app_id=target.id,
+            instance_id=instance_id,
+        )
+        run_result.add_synced_run(synced_run)
         with open(os.path.join(run_dir, f"{run_id}.json"), "w") as f:
             json.dump(run_result.to_dict(), f, indent=2)
 
         if verbose:
-            log(f"✅ Synced local run `{run_id}` as remote run `{tracked_id}`.")
+            log(f"✅ Synced local run `{run_id}` as remote run `{synced_run.to_dict()}`.")
 
         return True
 
@@ -1116,7 +1133,15 @@ class Application:
             return False
 
         # Validate outputs
-        if not self.__validate_outputs(run_dir, run_result.metadata.format.format_output.output_type):
+        format_output = run_result.metadata.format.format_output
+        if format_output is None or not format_output:
+            return False
+
+        output_type = format_output.output_type
+        if output_type is None or output_type == "":
+            return False
+
+        if not self.__validate_outputs(run_dir, output_type):
             return False
 
         # Validate logs