nextmv 0.33.0__tar.gz → 0.34.0__tar.gz

{nextmv-0.33.0 → nextmv-0.34.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextmv
-Version: 0.33.0
+Version: 0.34.0
 Summary: The all-purpose Python SDK for Nextmv
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextmv-py.docs.nextmv.io/en/latest/nextmv/

nextmv-0.34.0/nextmv/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.34.0"

{nextmv-0.33.0 → nextmv-0.34.0}/nextmv/__init__.py

@@ -25,6 +25,7 @@ from .manifest import ManifestPythonArch as ManifestPythonArch
 from .manifest import ManifestPythonModel as ManifestPythonModel
 from .manifest import ManifestRuntime as ManifestRuntime
 from .manifest import ManifestType as ManifestType
+from .manifest import default_python_manifest as default_python_manifest
 from .model import Model as Model
 from .model import ModelConfiguration as ModelConfiguration
 from .options import Option as Option
@@ -69,6 +70,7 @@ from .run import RunResult as RunResult
 from .run import RunType as RunType
 from .run import RunTypeConfiguration as RunTypeConfiguration
 from .run import StatisticsIndicator as StatisticsIndicator
+from .run import SyncedRun as SyncedRun
 from .run import TrackedRun as TrackedRun
 from .run import TrackedRunStatus as TrackedRunStatus
 from .run import run_duration as run_duration

{nextmv-0.33.0 → nextmv-0.34.0}/nextmv/cloud/application.py

@@ -58,7 +58,7 @@ from nextmv.logger import log
 from nextmv.manifest import Manifest
 from nextmv.model import Model, ModelConfiguration
 from nextmv.options import Options
-from nextmv.output import Output, OutputFormat
+from nextmv.output import ASSETS_KEY, STATISTICS_KEY, Asset, Output, OutputFormat, Statistics
 from nextmv.polling import DEFAULT_POLLING_OPTIONS, PollingOptions, poll
 from nextmv.run import (
     ExternalRunResult,
@@ -187,17 +187,20 @@ class Application:
         >>> app = Application.new(client=client, name="My New App", id="my-app")
         """
 
+        if id is None:
+            id = safe_id("app")
+
         if exist_ok and cls.exists(client=client, id=id):
             return Application(client=client, id=id)
 
         payload = {
             "name": name,
+            "id": id,
         }
 
         if description is not None:
             payload["description"] = description
-        if id is not None:
-            payload["id"] = id
+
         if is_workflow is not None:
             payload["is_pipeline"] = is_workflow
 
@@ -1790,7 +1793,7 @@ class Application:
             when the run is part of a batch experiment.
         external_result: Optional[Union[ExternalRunResult, dict[str, Any]]]
             External result to use for the run. This can be a
-            `cloud.ExternalRunResult` object or a dict. If the object is used,
+            `nextmv.ExternalRunResult` object or a dict. If the object is used,
             then the `.to_dict()` method is applied to extract the
             configuration. This is used when the run is an external run. We
             suggest that instead of specifying this parameter, you use the
@@ -1820,8 +1823,6 @@ class Application:
             not `JSON`. If the final `options` are not of type `dict[str,str]`.
         """
 
-        self.__validate_input_dir_path_and_configuration(input_dir_path, configuration)
-
         tar_file = ""
         if input_dir_path is not None and input_dir_path != "":
             if not os.path.exists(input_dir_path):
@@ -1879,6 +1880,7 @@ class Application:
         query_params = {}
         if instance_id is not None or self.default_instance_id is not None:
             query_params["instance_id"] = instance_id if instance_id is not None else self.default_instance_id
+
         response = self.client.request(
             method="POST",
             endpoint=f"{self.endpoint}/runs",
@@ -2993,6 +2995,7 @@ class Application:
             execution_duration=tracked_run.duration,
         )
 
+        # Handle the stderr logs if provided.
         if tracked_run.logs is not None:
             url_stderr = self.upload_url()
             self.upload_large_input(input=tracked_run.logs_text(), upload_url=url_stderr)
@@ -3001,6 +3004,47 @@ class Application:
         if tracked_run.error is not None and tracked_run.error != "":
             external_result.error_message = tracked_run.error
 
+        # Handle the statistics upload if provided.
+        stats = tracked_run.statistics
+        if stats is not None:
+            if isinstance(stats, Statistics):
+                stats_dict = stats.to_dict()
+                stats_dict = {STATISTICS_KEY: stats_dict}
+            elif isinstance(stats, dict):
+                stats_dict = stats
+                if STATISTICS_KEY not in stats_dict:
+                    stats_dict = {STATISTICS_KEY: stats_dict}
+            else:
+                raise ValueError("tracked_run.statistics must be either a `Statistics` or `dict` object")
+
+            url_stats = self.upload_url()
+            self.upload_large_input(input=stats_dict, upload_url=url_stats)
+            external_result.statistics_upload_id = url_stats.upload_id
+
+        # Handle the assets upload if provided.
+        assets = tracked_run.assets
+        if assets is not None:
+            if isinstance(assets, list):
+                assets_list = []
+                for ix, asset in enumerate(assets):
+                    if isinstance(asset, Asset):
+                        assets_list.append(asset.to_dict())
+                    elif isinstance(asset, dict):
+                        assets_list.append(asset)
+                    else:
+                        raise ValueError(f"tracked_run.assets, index {ix} must be an `Asset` or `dict` object")
+                assets_dict = {ASSETS_KEY: assets_list}
+            elif isinstance(assets, dict):
+                assets_dict = assets
+                if ASSETS_KEY not in assets_dict:
+                    assets_dict = {ASSETS_KEY: assets_dict}
+            else:
+                raise ValueError("tracked_run.assets must be either a `list[Asset]`, `list[dict]`, or `dict` object")
+
+            url_assets = self.upload_url()
+            self.upload_large_input(input=assets_dict, upload_url=url_assets)
+            external_result.assets_upload_id = url_assets.upload_id
+
         return self.new_run(
             upload_id=url_input.upload_id,
             external_result=external_result,
@@ -3860,49 +3904,6 @@ class Application:
 
         raise ValueError(f"Unknown scenario input type: {scenario.scenario_input.scenario_input_type}")
 
-    def __validate_input_dir_path_and_configuration(
-        self,
-        input_dir_path: Optional[str],
-        configuration: Optional[Union[RunConfiguration, dict[str, Any]]],
-    ) -> None:
-        """
-        Auxiliary function to validate the directory path and configuration.
-        """
-        input_type = self.__get_input_type(configuration)
-
-        # If no explicit input type is defined, there is nothing to validate.
-        if input_type is None:
-            return
-
-        # Validate that the input directory path is provided when explicitly required.
-        dir_types = (InputFormat.MULTI_FILE, InputFormat.CSV_ARCHIVE)
-        if input_type in dir_types and not input_dir_path:
-            raise ValueError(
-                f"If RunConfiguration.format.format_input.input_type is set to {input_type}, "
-                "then input_dir_path must be provided.",
-            )
-
-    def __get_input_type(self, config: Union[RunConfiguration, dict[str, Any]]) -> Optional[InputFormat]:
-        """
-        Auxiliary function to extract the input type from the run configuration.
-        """
-
-        if config is None:
-            return None
-
-        if isinstance(config, dict):
-            config = RunConfiguration.from_dict(config)
-
-        if (
-            isinstance(config, RunConfiguration)
-            and config.format is not None
-            and config.format.format_input is not None
-            and config.format.format_input.input_type is not None
-        ):
-            return config.format.format_input.input_type
-
-        return None
-
     def __package_inputs(self, dir_path: str) -> str:
         """
         This is an auxiliary function for packaging the inputs found in the

{nextmv-0.33.0 → nextmv-0.34.0}/nextmv/local/application.py

@@ -33,11 +33,22 @@ from nextmv.local.local import (
 )
 from nextmv.local.runner import run
 from nextmv.logger import log
-from nextmv.manifest import Manifest
+from nextmv.manifest import Manifest, default_python_manifest
 from nextmv.options import Options
-from nextmv.output import OUTPUTS_KEY, SOLUTIONS_KEY, OutputFormat
+from nextmv.output import ASSETS_KEY, OUTPUTS_KEY, SOLUTIONS_KEY, STATISTICS_KEY, OutputFormat
 from nextmv.polling import DEFAULT_POLLING_OPTIONS, PollingOptions, poll
-from nextmv.run import ErrorLog, Format, Run, RunConfiguration, RunInformation, RunResult, TrackedRun, TrackedRunStatus
+from nextmv.run import (
+    ErrorLog,
+    Format,
+    FormatInput,
+    Run,
+    RunConfiguration,
+    RunInformation,
+    RunResult,
+    SyncedRun,
+    TrackedRun,
+    TrackedRunStatus,
+)
 from nextmv.safe import safe_id
 from nextmv.status import StatusV2
 
@@ -83,6 +94,39 @@ class Application:
 
     description: Optional[str] = None
     """Description of the application."""
+    manifest: Optional[Manifest] = None
+    """
+    Manifest of the application. A manifest is a file named `app.yaml` that
+    must be present at the root of the application's `src` directory. If the
+    app is initialized, and a manifest is not present, a default Python
+    manifest will be created, using the `nextmv.default_python_manifest`
+    function. If you specify this argument, and a manifest file is already
+    present in the `src` directory, the provided manifest will override the
+    existing one.
+    """
+
+    def __post_init__(self):
+        """
+        Validate the presence of the manifest in the application.
+        """
+
+        if self.manifest is not None:
+            self.manifest.to_yaml(self.src)
+
+            return
+
+        try:
+            manifest = Manifest.from_yaml(self.src)
+            self.manifest = manifest
+
+            return
+
+        except Exception:
+            manifest = default_python_manifest()
+            self.manifest = manifest
+            manifest.to_yaml(self.src)
+
+            return
 
     @classmethod
     def initialize(
@@ -283,7 +327,7 @@ class Application:
         >>> print(f"Local run completed with ID: {run_id}")
         """
 
-        self.__validate_input_dir_path_and_configuration(input_dir_path, configuration)
+        configuration = self.__validate_input_dir_path_and_configuration(input_dir_path, configuration)
 
         if self.src is None:
             raise ValueError("`src` property for the `Application` must be specified to run the application locally")
@@ -843,69 +887,54 @@ class Application:
         self,
         input_dir_path: Optional[str],
         configuration: Optional[Union[RunConfiguration, dict[str, Any]]],
-    ) -> None:
+    ) -> RunConfiguration:
         """
         Auxiliary function to validate the directory path and configuration.
         """
 
+        if configuration is None:
+            if self.manifest.configuration is not None and self.manifest.configuration.content is not None:
+                configuration = RunConfiguration(
+                    format=Format(
+                        format_input=FormatInput(
+                            input_type=self.manifest.configuration.content.format,
+                        ),
+                    ),
+                )
+        elif isinstance(configuration, dict):
+            # Forcefully turn the configuration into a RunConfiguration object to
+            # make it easier to deal with in the other functions.
+            configuration = RunConfiguration.from_dict(configuration)
+
         if input_dir_path is None or input_dir_path == "":
-            return
+            return configuration
 
         if configuration is None:
             raise ValueError(
-                "If dir_path is provided, a RunConfiguration must also be provided.",
+                "If `dir_path` is provided, either a `RunConfiguration` must also be provided or "
+                "the application's manifest (app.yaml) must include the format under "
+                "`configuration.content.format`.",
             )
 
-        config_format = self.__extract_config_format(configuration)
-
+        config_format = configuration.format
         if config_format is None:
             raise ValueError(
-                "If dir_path is provided, RunConfiguration.format must also be provided.",
+                "If `dir_path` is provided, `RunConfiguration.format` must also be provided.",
             )
 
-        input_type = self.__extract_input_type(config_format)
-
-        if input_type is None or input_type in (InputFormat.JSON, InputFormat.TEXT):
+        input_type = config_format.format_input
+        if input_type is None:
             raise ValueError(
-                "If dir_path is provided, RunConfiguration.format.format_input.input_type must be set to a valid type. "
-                f"Valid types are: {[InputFormat.CSV_ARCHIVE, InputFormat.MULTI_FILE]}",
+                "If `dir_path` is provided, `RunConfiguration.format.format_input` must also be provided.",
             )
 
-    def __extract_config_format(self, configuration: Union[RunConfiguration, dict[str, Any]]) -> Any:
-        """Extract format from configuration, handling both RunConfiguration objects and dicts."""
-        if isinstance(configuration, RunConfiguration):
-            return configuration.format
-
-        if isinstance(configuration, dict):
-            config_format = configuration.get("format")
-            if config_format is not None and isinstance(config_format, dict):
-                return Format.from_dict(config_format) if hasattr(Format, "from_dict") else config_format
-
-            return config_format
-
-        raise ValueError("Configuration must be a RunConfiguration object or a dict.")
-
-    def __extract_input_type(self, config_format: Any) -> Any:
-        """Extract input type from config format."""
-        if isinstance(config_format, dict):
-            format_input = config_format.get("format_input") or config_format.get("input")
-            if format_input is None:
-                raise ValueError(
-                    "If dir_path is provided, RunConfiguration.format.format_input must also be provided.",
-                )
-
-            if isinstance(format_input, dict):
-                return format_input.get("input_type") or format_input.get("type")
-
-            return getattr(format_input, "input_type", None)
-
-        # Handle Format object
-        if config_format.format_input is None:
+        if input_type is None or input_type in (InputFormat.JSON, InputFormat.TEXT):
             raise ValueError(
-                "If dir_path is provided, RunConfiguration.format.format_input must also be provided.",
+                "If `dir_path` is provided, `RunConfiguration.format.format_input.input_type` must be set to "
+                f"a valid type. Valid types are: {[InputFormat.CSV_ARCHIVE, InputFormat.MULTI_FILE]}",
             )
 
-        return config_format.format_input.input_type
+        return configuration
 
     def __extract_input_data(
         self,
@@ -995,25 +1024,10 @@ class Application:
         input_type = run_result.metadata.format.format_input.input_type
 
         # Skip runs that have already been synced.
-        already_synced = run_result.synced_run_id is not None and run_result.synced_at is not None
+        synced_run, already_synced = run_result.is_synced(app_id=target.id, instance_id=instance_id)
         if already_synced:
             if verbose:
-                log(f"   ⏭️  Skipping local run `{run_id}`, already synced at {run_result.synced_at.isoformat()}.")
-
-            return False
-
-        # Skip runs that don't have the supported type. TODO: delete this when
-        # external runs support CSV_ARCHIVE and MULTI_FILE. Right now,
-        # submitting an external result with a new run is limited to JSON and
-        # TEXT. After this if statement is removed, the rest of the code should
-        # work with CSV_ARCHIVE and MULTI_FILE as well, as using the input dir
-        # path is already considered.
-        if input_type not in {InputFormat.JSON, InputFormat.TEXT}:
-            if verbose:
-                log(
-                    f"   ⏭️  Skipping local run `{run_id}`, unsupported input type: {input_type.value}. "
-                    f"Supported types are: {[InputFormat.JSON.value, InputFormat.TEXT.value]}",
-                )
+                log(f"   ⏭️  Skipping local run `{run_id}`, already synced with {synced_run.to_dict()}.")
 
             return False
 
@@ -1031,7 +1045,7 @@ class Application:
         # Read the logs of the run and place each line as an element in a list
         run_dir = os.path.join(runs_dir, run_id)
         with open(os.path.join(run_dir, LOGS_KEY, LOGS_FILE)) as f:
-            stderr_logs = f.readlines()
+            stderr_logs = f.read()
 
         # Create the tracked run object and start configuring it.
         tracked_run = TrackedRun(
@@ -1055,11 +1069,28 @@ class Application:
             tracked_run.input_dir_path = inputs_path
 
         # Resolve the output according to its type.
-        if run_result.metadata.format.format_output.output_type == OutputFormat.JSON:
+        output_type = run_result.metadata.format.format_output.output_type
+        if output_type == OutputFormat.JSON:
             tracked_run.output = run_result.output
         else:
             tracked_run.output_dir_path = os.path.join(run_dir, OUTPUTS_KEY, SOLUTIONS_KEY)
 
+        # Resolve the statistics according to their type and presence. If
+        # working with JSON, the statistics should be resolved from the output.
+        if output_type in {OutputFormat.CSV_ARCHIVE, OutputFormat.MULTI_FILE}:
+            stats_file_path = os.path.join(run_dir, OUTPUTS_KEY, STATISTICS_KEY, f"{STATISTICS_KEY}.json")
+            if os.path.exists(stats_file_path):
+                with open(stats_file_path) as f:
+                    tracked_run.statistics = json.load(f)
+
+        # Resolve the assets according to their type and presence. If working
+        # with JSON, the assets should be resolved from the output.
+        if output_type in {OutputFormat.CSV_ARCHIVE, OutputFormat.MULTI_FILE}:
+            assets_file_path = os.path.join(run_dir, OUTPUTS_KEY, ASSETS_KEY, f"{ASSETS_KEY}.json")
+            if os.path.exists(assets_file_path):
+                with open(assets_file_path) as f:
+                    tracked_run.assets = json.load(f)
+
         # Actually sync the run by tracking it remotely on Nextmv Cloud.
         configuration = RunConfiguration(
             format=Format(
@@ -1074,13 +1105,18 @@ class Application:
         )
 
         # Mark the local run as synced by updating the local run info.
-        run_result.synced_run_id = tracked_id
-        run_result.synced_at = datetime.now(timezone.utc)
+        synced_run = SyncedRun(
+            run_id=tracked_id,
+            synced_at=datetime.now(timezone.utc),
+            app_id=target.id,
+            instance_id=instance_id,
+        )
+        run_result.add_synced_run(synced_run)
         with open(os.path.join(run_dir, f"{run_id}.json"), "w") as f:
             json.dump(run_result.to_dict(), f, indent=2)
 
         if verbose:
-            log(f"✅ Synced local run `{run_id}` as remote run `{tracked_id}`.")
+            log(f"✅ Synced local run `{run_id}` as remote run `{synced_run.to_dict()}`.")
 
         return True
 
@@ -1116,7 +1152,15 @@ class Application:
             return False
 
         # Validate outputs
-        if not self.__validate_outputs(run_dir, run_result.metadata.format.format_output.output_type):
+        format_output = run_result.metadata.format.format_output
+        if format_output is None or not format_output:
+            return False
+
+        output_type = format_output.output_type
+        if output_type is None or output_type == "":
+            return False
+
+        if not self.__validate_outputs(run_dir, output_type):
             return False
 
         # Validate logs