humalab 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

humalab/metrics/summary.py

@@ -1,15 +1,11 @@
 
-from humalab.metrics.metric import MetricGranularity, Metrics, MetricType
-from humalab.constants import EpisodeStatus
+from humalab.metrics.metric import Metrics
+from humalab.constants import MetricDimType, GraphType
 
 
 class Summary(Metrics):
     def __init__(self, 
-                 name: str, 
                  summary: str,
-                 episode_id: str,
-                 run_id: str,
-                 granularity: MetricGranularity = MetricGranularity.RUN,
                  ) -> None:
         """
         A summary metric that captures a single value per episode or run.
@@ -22,26 +18,33 @@ class Summary(Metrics):
                 from being generated.
             granularity (MetricGranularity): The granularity of the metric.
         """
-        if granularity == MetricGranularity.RUN:
-            raise ValueError("Summary metrics cannot have RUN granularity.")
         if summary not in {"min", "max", "mean", "last", "first", "none"}:
             raise ValueError(f"Unsupported summary type: {summary}. Supported types are 'min', 'max', 'mean', 'last', 'first', and 'none'.")
-        super().__init__(name, MetricType.SUMMARY, episode_id=episode_id, run_id=run_id, granularity=granularity)
-        self.summary = summary
+        super().__init__(metric_dim_type= MetricDimType.ZERO_D, 
+                         graph_type=GraphType.NUMERIC)
+        self._summary = summary
+    
+    @property
+    def summary(self) -> str:
+        return self._summary
 
-    def _submit(self) -> None:
+    def _finalize(self) -> dict:
         if not self._values:
-            return
+            return {
+                "value": None,
+                "summary": self.summary
+            }
+        final_val = None
         # For summary metrics, we only keep the latest value
         if self.summary == "last":
-            self._values = [self._values[-1]]
+            final_val = self._values[-1]
         elif self.summary == "first":
-            self._values = [self._values[0]]
+            final_val = self._values[0]
         elif self.summary == "none":
-            self._values = []
+            final_val = None
         elif self.summary in {"min", "max", "mean"}:
             if not self._values:
-                self._values = []
+                final_val = None
             else:
                 if self.summary == "min":
                     agg_value = min(self._values)
@@ -49,6 +52,9 @@ class Summary(Metrics):
                     agg_value = max(self._values)
                 elif self.summary == "mean":
                     agg_value = sum(self._values) / len(self._values)
-                self._values = [agg_value]
+                final_val = agg_value
 
-        super()._submit()
+        return {
+            "value": final_val,
+            "summary": self.summary
+        }

humalab/run.py

@@ -1,26 +1,37 @@
 import uuid
-from humalab.metrics.dist_metric import DistributionMetric
-from humalab.metrics.metric import MetricGranularity, MetricType, Metrics
-from humalab.constants import EpisodeStatus
+import traceback
+import pickle
+import base64
 
+from humalab.metrics.code import Code
 from humalab.metrics.summary import Summary
-from humalab.scenario import Scenario
+
+from humalab.constants import DEFAULT_PROJECT, RESERVED_NAMES, ArtifactType
+from humalab.metrics.scenario_stats import ScenarioStats
+from humalab.humalab_api_client import EpisodeStatus, HumaLabApiClient, RunStatus
+from humalab.metrics.metric import Metrics
+from humalab.episode import Episode
+from humalab.utils import is_standard_type
+
+from humalab.scenarios.scenario import Scenario
 
 class Run:
     def __init__(self,
-                 entity: str,
-                 project: str,
                  scenario: Scenario,
+                 project: str = DEFAULT_PROJECT,
                  name: str | None = None,
                  description: str | None = None,
                  id: str | None = None,
                  tags: list[str] | None = None,
+
+                 base_url: str | None = None,
+                 api_key: str | None = None,
+                 timeout: float | None = None,
                  ) -> None:
         """
         Initialize a new Run instance.
         
         Args:
-            entity (str): The entity (user or team) under which the run is created.
             project (str): The project name under which the run is created.
             scenario (Scenario): The scenario instance for the run.
             name (str | None): The name of the run.
@@ -28,28 +39,21 @@ class Run:
             id (str | None): The unique identifier for the run. If None, a UUID is generated.
             tags (list[str] | None): A list of tags associated with the run.
         """
-        self._entity = entity
         self._project = project
         self._id = id or str(uuid.uuid4())
         self._name = name or ""
         self._description = description or ""
         self._tags = tags or []
-        self._finished = False
-
-        self._episode = str(uuid.uuid4())
 
         self._scenario = scenario
+        self._logs = {}
+        self._episodes = {}
+        self._is_finished = False
+
+        self._api_client = HumaLabApiClient(base_url=base_url,
+                                            api_key=api_key,
+                                            timeout=timeout)
 
-        self._metrics = {}
-    
-    @property
-    def entity(self) -> str:
-        """The entity (user or team) under which the run is created.
-        
-        Returns:
-            str: The entity name.
-        """
-        return self._entity
     
     @property
     def project(self) -> str:
@@ -96,15 +100,6 @@ class Run:
         """
         return self._tags
     
-    @property
-    def episode(self) -> str:
-        """The episode ID for the run.
-
-        Returns:
-            str: The episode ID.
-        """
-        return self._episode
-    
     @property
     def scenario(self) -> Scenario:
         """The scenario associated with the run.
@@ -113,102 +108,172 @@ class Run:
             Scenario: The scenario instance.
         """
         return self._scenario
+    
+    def __enter__(self):
+        return self
 
-    def finish(self,
-               status: EpisodeStatus = EpisodeStatus.PASS,
-               quiet: bool | None = None) -> None:
-        """Finish the run and submit final metrics.
+    def __exit__(self, exception_type, exception_value, exception_traceback):
+        if self._is_finished:
+            return
+        if exception_type is not None:
+            err_msg = "".join(traceback.format_exception(exception_type, exception_value, exception_traceback))
+            self.finish(status=RunStatus.ERRORED, err_msg=err_msg)
+        else:
+            self.finish()
+
+    def create_episode(self, episode_id: str | None = None) -> Episode:
+        """Reset the run for a new episode.
 
         Args:
-            status (EpisodeStatus): The final status of the episode.
-            quiet (bool | None): Whether to suppress output.
+            status (EpisodeStatus): The status of the current episode before reset.
         """
-        self._finished = True
-        self._scenario.finish()
-        for metric in self._metrics.values():
-            metric.finish(status=status)
+        episode = None
+        episode_id = episode_id or str(uuid.uuid4())
+        cur_scenario, episode_vals = self._scenario.resolve()
+        episode = Episode(run_id=self._id,
+                          episode_id=episode_id,
+                          scenario_conf=cur_scenario,
+                          episode_vals=episode_vals)
+        self._handle_scenario_stats(episode, episode_vals)
+        
+        return episode
+
+    def _handle_scenario_stats(self, episode: Episode, episode_vals: dict) -> None:
+        for metric_name, value in episode_vals.items():
+            if metric_name not in self._logs:
+                stat = ScenarioStats(name=metric_name,
+                                    distribution_type=value["distribution"],
+                                    metric_dim_type=value["metric_dim_type"],
+                                    graph_type=value["graph_type"])
+                self._logs[metric_name] = stat
+            self._logs[metric_name].log(data=value["value"],
+                                        x=episode.episode_id)
+        self._episodes[episode.episode_id] = episode
     
-    def log(self,
-            data: dict,
-            step: int | None = None,
-            commit: bool = True,
-            ) -> None:
-        """Log metrics for the run.
+    def add_metric(self, name: str, metric: Metrics) -> None:
+        if name in self._logs:
+            raise ValueError(f"{name} is a reserved name and is not allowed.")
+        self._logs[name] = metric
+
+    def log_code(self, key: str, code_content: str) -> None:
+        """Log code content as an artifact.
 
         Args:
-            data (dict): A dictionary of metric names and their values.
-            step (int | None): The step number for the metrics.
-            commit (bool): Whether to commit the metrics immediately.
+            key (str): The key for the code artifact.
+            code_content (str): The code content to log.
         """
+        if key in RESERVED_NAMES:
+            raise ValueError(f"{key} is a reserved name and is not allowed.")
+        self._logs[key] = Code(
+            run_id=self._id,
+            key=key,
+            code_content=code_content,
+        )
+
+        
+    def log(self, data: dict, x: dict | None = None, replace: bool = False) -> None:
         for key, value in data.items():
-            if key in self._metrics:
-                metric = self._metrics[key]
-                metric.log(value, step=step, commit=commit)
+            if key in RESERVED_NAMES:
+                raise ValueError(f"{key} is a reserved name and is not allowed.")
+            if key not in self._logs:
+                self._logs[key] = value
             else:
-                self._metrics[key] = Metrics(key, 
-                                             metric_type=MetricType.DEFAULT,
-                                             run_id=self._id,
-                                             granularity=MetricGranularity.EPISODE,
-                                             episode_id=self._episode)
-                self._metrics[key].log(value, step=step, commit=commit)
-
-    def reset(self, status: EpisodeStatus = EpisodeStatus.PASS) -> None:
-        """Reset the run for a new episode.
-
-        Args:
-            status (EpisodeStatus): The status of the current episode before reset.
-        """
-        self._submit_episode_status(status=status, episode=self._episode)
-        self._episode = str(uuid.uuid4())
-        self._finished = False
-        self._scenario.reset(episode_id=self._episode)
-        for metric in self._metrics.values():
-            metric.reset(episode=self._episode)
-    
-    def _submit_episode_status(self, status: EpisodeStatus, episode: str) -> None:
-        # TODO: Implement submission of episode status
-        pass
-
-    def define_metric(self, 
-                      name: str, 
-                      metric_type: MetricType = MetricType.DEFAULT,
-                      granularity: MetricGranularity = MetricGranularity.RUN,
-                      distribution_type: str | None = None,
-                      summary: str | None = None,
-                      replace: bool = False) -> None:
-        """Define a new metric for the run.
+                cur_val = self._logs[key]
+                if isinstance(cur_val, Metrics):
+                    cur_x = x.get(key) if x is not None else None
+                    cur_val.log(value, x=cur_x, replace=replace)
+                else:
+                    if replace:
+                        self._logs[key] = value
+                    else:
+                        raise ValueError(f"Cannot log value for key '{key}' as there is already a value logged.")
+    def _finish_episodes(self,
+                         status: RunStatus,
+                         err_msg: str | None = None) -> None:
+        for episode in self._episodes.values():
+            if not episode.is_finished:
+                if status == RunStatus.FINISHED:
+                    episode.finish(status=EpisodeStatus.CANCELED, err_msg=err_msg)
+                elif status == RunStatus.ERRORED:
+                    episode.finish(status=EpisodeStatus.ERRORED, err_msg=err_msg)
+                elif status == RunStatus.CANCELED:
+                    episode.finish(status=EpisodeStatus.CANCELED, err_msg=err_msg)
         
+
+    def finish(self,
+               status: RunStatus = RunStatus.FINISHED,
+               err_msg: str | None = None) -> None:
+        """Finish the run and submit final metrics.
+
         Args:
-            name (str): The name of the metric.
-            metric_type (MetricType): The type of the metric.
-            granularity (MetricGranularity): The granularity of the metric.
-            distribution_type (str | None): The type of distribution if metric_type is DISTRIBUTION.
-            summary (str | None): Specify aggregate metrics added to summary.
-                Supported aggregations include "min", "max", "mean", "last",
-                "first", and "none". "none" prevents a summary
-                from being generated.
-            replace (bool): Whether to replace the metric if it already exists.
+            status (RunStatus): The final status of the run.
+            err_msg (str | None): An optional error message.
         """
-        if name not in self._metrics or replace:
-            if metric_type == MetricType.DISTRIBUTION:
-                if distribution_type is None:
-                    raise ValueError("distribution_type must be specified for distribution metrics.")
-                self._metrics[name] = DistributionMetric(name=name, 
-                                                         distribution_type=distribution_type, 
-                                                         run_id=self._id,
-                                                         episode_id=self._episode,
-                                                         granularity=granularity)  
-            elif summary is not None:
-                self._metrics[name] = Summary(name=name, 
-                                              summary=summary, 
-                                              run_id=self._id,
-                                              episode_id=self._episode,
-                                              granularity=granularity)
+        if self._is_finished:
+            return
+        self._is_finished = True
+        self._finish_episodes(status=status, err_msg=err_msg)
+
+        self._api_client.upload_code(
+            artifact_key="scenario",
+            run_id=self._id,
+            code_content=self.scenario.yaml
+        )
+        # TODO: submit final metrics
+        for key, value in self._logs.items():
+            if isinstance(value, ScenarioStats):
+                for episode_id, episode in self._episodes.items():
+                    episode_status = episode.status
+                    value.log_status(
+                        episode_id=episode_id,
+                        episode_status=episode_status
+                    )
+                metric_val = value.finalize()
+                pickled = pickle.dumps(metric_val)
+                self._api_client.upload_scenario_stats_artifact(
+                    artifact_key=key,
+                    run_id=self._id,
+                    pickled_bytes=pickled,
+                    graph_type=value.graph_type.value,
+                    metric_dim_type=value.metric_dim_type.value
+                )
+            elif isinstance(value, Summary):
+                metric_val = value.finalize()
+                pickled = pickle.dumps(metric_val["value"])
+                self._api_client.upload_python(
+                    artifact_key=key,
+                    run_id=self._id,
+                    pickled_bytes=pickled
+                )
+            elif isinstance(value, Metrics):
+                metric_val = value.finalize()
+                pickled = pickle.dumps(metric_val)
+                self._api_client.upload_metrics(
+                    artifact_key=key,
+                    run_id=self._id,
+                    pickled_bytes=pickled,
+                    graph_type=value.graph_type.value,
+                    metric_dim_type=value.metric_dim_type.value
+                )
+            elif isinstance(value, Code):
+                self._api_client.upload_code(
+                    artifact_key=value.key,
+                    run_id=value.run_id,
+                    episode_id=value.episode_id,
+                    code_content=value.code_content
+                )
             else:
-                self._metrics[name] = Metrics(name=name, 
-                                              metric_type=metric_type, 
-                                              run_id=self._id,
-                                              episode_id=self._episode,
-                                              granularity=granularity)
-        else:
-            raise ValueError(f"Metric {name} already exists.")
+                if not is_standard_type(value):
+                    raise ValueError(f"Value for key '{key}' is not a standard type.")
+                pickled = pickle.dumps(value)
+                self._api_client.upload_python(
+                    artifact_key=key,
+                    run_id=self._id,
+                    pickled_bytes=pickled
+                )
+
+        self._api_client.update_run(
+            run_id=self._id,
+            status=status,
+            err_msg=err_msg
+        )

humalab/scenarios/__init__.py

@@ -0,0 +1,4 @@
+from .scenario import Scenario
+from .scenario_operator import list_scenarios, get_scenario
+
+__all__ = ["Scenario", "list_scenarios", "get_scenario"]