humalab 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

humalab/humalab_test.py

@@ -2,12 +2,13 @@ import unittest
 from unittest.mock import patch, MagicMock, Mock
 import uuid
 
+from humalab.constants import DEFAULT_PROJECT
 from humalab import humalab
 from humalab.run import Run
-from humalab.scenario import Scenario
+from humalab.scenarios.scenario import Scenario
 from humalab.humalab_config import HumalabConfig
 from humalab.humalab_api_client import HumaLabApiClient
-from humalab.constants import EpisodeStatus
+from humalab.humalab_api_client import EpisodeStatus, RunStatus
 
 
 class HumalabTest(unittest.TestCase):
@@ -30,9 +31,10 @@ class HumalabTest(unittest.TestCase):
         # Pre-condition
         client = Mock()
         scenario = {"key": "value"}
+        project = "test_project"
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=scenario, scenario_id=None)
+        result = humalab._pull_scenario(client=client, project=project, scenario=scenario, scenario_id=None)
 
         # Post-condition
         self.assertEqual(result, scenario)
@@ -42,32 +44,34 @@ class HumalabTest(unittest.TestCase):
         """Test that _pull_scenario fetches from API when scenario_id is provided."""
         # Pre-condition
         client = Mock()
+        project = "test_project"
         scenario_id = "test-scenario-id"
         yaml_content = "scenario: test"
         client.get_scenario.return_value = {"yaml_content": yaml_content}
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=None, scenario_id=scenario_id)
+        result = humalab._pull_scenario(client=client, project=project, scenario=None, scenario_id=scenario_id)
 
         # Post-condition
         self.assertEqual(result, yaml_content)
-        client.get_scenario.assert_called_once_with(uuid=scenario_id)
+        client.get_scenario.assert_called_once_with(project_name=project, uuid=scenario_id, version=None)
 
     def test_pull_scenario_should_prefer_scenario_id_over_scenario(self):
         """Test that _pull_scenario uses scenario_id even when scenario is provided."""
         # Pre-condition
         client = Mock()
+        project = "test_project"
         scenario = {"key": "value"}
         scenario_id = "test-scenario-id"
         yaml_content = "scenario: from_api"
         client.get_scenario.return_value = {"yaml_content": yaml_content}
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=scenario, scenario_id=scenario_id)
+        result = humalab._pull_scenario(client=client, project=project, scenario=scenario, scenario_id=scenario_id)
 
         # Post-condition
         self.assertEqual(result, yaml_content)
-        client.get_scenario.assert_called_once_with(uuid=scenario_id)
+        client.get_scenario.assert_called_once_with(project_name=project, uuid=scenario_id, version=None)
 
     # Tests for init context manager
 
@@ -92,6 +96,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": project}
+        mock_api_client.get_run.return_value = {"run_id": run_id, "name": name, "description": description, "tags": tags}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -137,6 +143,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -149,7 +157,7 @@ class HumalabTest(unittest.TestCase):
         with humalab.init() as run:
             # Post-condition
             call_kwargs = mock_run_class.call_args.kwargs
-            self.assertEqual(call_kwargs['project'], "default")
+            self.assertEqual(call_kwargs['project'], DEFAULT_PROJECT)
             self.assertEqual(call_kwargs['name'], "")
             self.assertEqual(call_kwargs['description'], "")
             self.assertIsNotNone(call_kwargs['id'])  # UUID generated
@@ -170,7 +178,18 @@ class HumalabTest(unittest.TestCase):
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
+        # Mock HTTP 404 error for get_run (run doesn't exist yet)
+        import requests
+        http_error = requests.HTTPError()
+        http_error.response = Mock()
+        http_error.response.status_code = 404
+
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.side_effect = http_error
+        # Mock create_run to return a valid UUID
+        generated_uuid = str(uuid.uuid4())
+        mock_api_client.create_run.return_value = {"run_id": generated_uuid, "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -206,6 +225,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -241,6 +262,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client.get_scenario.return_value = {"yaml_content": yaml_content}
         mock_api_client_class.return_value = mock_api_client
 
@@ -253,7 +276,7 @@ class HumalabTest(unittest.TestCase):
         # In-test
         with humalab.init(scenario_id=scenario_id) as run:
             # Post-condition
-            mock_api_client.get_scenario.assert_called_once_with(uuid=scenario_id)
+            mock_api_client.get_scenario.assert_called_once_with(project_name='default', uuid=scenario_id, version=None)
             mock_scenario_inst.init.assert_called_once()
             call_kwargs = mock_scenario_inst.init.call_args.kwargs
             self.assertEqual(call_kwargs['scenario'], yaml_content)
@@ -274,6 +297,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -304,6 +329,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -338,6 +365,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -369,33 +398,33 @@ class HumalabTest(unittest.TestCase):
         humalab.finish()
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=EpisodeStatus.PASS, quiet=None)
+        mock_run.finish.assert_called_once_with(status=RunStatus.FINISHED, err_msg=None)
 
     def test_finish_should_call_finish_on_current_run_with_custom_status(self):
         """Test that finish() calls finish on the current run with custom status."""
         # Pre-condition
         mock_run = Mock()
         humalab._cur_run = mock_run
-        status = EpisodeStatus.FAILED
+        status = RunStatus.ERRORED
 
         # In-test
         humalab.finish(status=status)
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=status, quiet=None)
+        mock_run.finish.assert_called_once_with(status=status, err_msg=None)
 
-    def test_finish_should_call_finish_on_current_run_with_quiet_parameter(self):
-        """Test that finish() calls finish on the current run with quiet parameter."""
+    def test_finish_should_call_finish_on_current_run_with_err_msg_parameter(self):
+        """Test that finish() calls finish on the current run with err_msg parameter."""
         # Pre-condition
         mock_run = Mock()
         humalab._cur_run = mock_run
-        quiet = True
+        err_msg = "Test error message"
 
         # In-test
-        humalab.finish(quiet=quiet)
+        humalab.finish(err_msg=err_msg)
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=EpisodeStatus.PASS, quiet=quiet)
+        mock_run.finish.assert_called_once_with(status=RunStatus.FINISHED, err_msg=err_msg)
 
     def test_finish_should_do_nothing_when_no_current_run(self):
         """Test that finish() does nothing when _cur_run is None."""

humalab/metrics/__init__.py

@@ -1,11 +1,11 @@
-from .metric import MetricGranularity, MetricType, Metrics
-from .dist_metric import DistributionMetric
+from .metric import Metrics
+from .code import Code
+from .scenario_stats import ScenarioStats
 from .summary import Summary
 
 __all__ = [
-    "MetricGranularity",
-    "MetricType",
+    "Code",
     "Metrics",
-    "DistributionMetric",
+    "ScenarioStats",
     "Summary",
 ]

humalab/metrics/code.py

@@ -0,0 +1,28 @@
+class Code:
+    """Class for logging code artifacts."""
+    def __init__(self, 
+                 run_id: str, 
+                 key: str,
+                 code_content: str,
+                 episode_id: str | None = None) -> None:
+        super().__init__()
+        self._run_id = run_id
+        self._key = key
+        self._code_content = code_content
+        self._episode_id = episode_id
+
+    @property
+    def run_id(self) -> str:
+        return self._run_id
+    
+    @property
+    def key(self) -> str:
+        return self._key
+    
+    @property
+    def code_content(self) -> str:
+        return self._code_content
+    
+    @property
+    def episode_id(self) -> str | None:
+        return self._episode_id

humalab/metrics/metric.py

@@ -1,129 +1,62 @@
-from enum import Enum
-from typing import Any  
-from humalab.constants import EpisodeStatus
-
-
-class MetricType(Enum):
-    DEFAULT = "default"
-    STREAM = "stream"
-    DISTRIBUTION = "distribution"
-    SUMMARY = "summary"
-
-
-class MetricGranularity(Enum):
-    STEP = "step"
-    EPISODE = "episode"
-    RUN = "run"
+from typing import Any
+from humalab.constants import MetricDimType, GraphType
 
 
 class Metrics:
-    def __init__(self, 
-                 name: str, 
-                 metric_type: MetricType,
-                 episode_id: str,
-                 run_id: str,
-                 granularity: MetricGranularity = MetricGranularity.STEP) -> None:
+    def __init__(self,
+                 metric_dim_type: MetricDimType= MetricDimType.ONE_D,
+                 graph_type: GraphType=GraphType.LINE) -> None:
         """
         Base class for different types of metrics.
-        
-        Args:
-            name (str): The name of the metric.
-            metric_type (MetricType): The type of the metric.
-            episode_id (str): The ID of the episode.
-            run_id (str): The ID of the run.
-            granularity (MetricGranularity): The granularity of the metric.
         """
-        self._name = name
-        self._metric_type = metric_type
-        self._granularity = granularity
         self._values = []
         self._x_values = []
-        self._episode_id = episode_id
-        self._run_id = run_id
-        self._last_step = -1
-    
-    def reset(self, 
-              episode_id: str | None = None) -> None:
-        """Reset the metric for a new episode or run.
+        self._step = -1
+        self._metric_dim_type = metric_dim_type
+        self._graph_type = graph_type
 
-        Args:
-            episode_id (str | None): Optional new episode ID. If None, keeps the current episode ID.
-        """
-        if self._granularity != MetricGranularity.RUN:
-            self._submit()
-            self._values = []
-            self._x_values = []
-        self._last_step = -1
-        self._episode_id = episode_id
-        
     @property
-    def name(self) -> str:
-        """The name of the metric.
-        
-        Returns:
-            str: The name of the metric.
-        """
-        return self._name
+    def metric_dim_type(self) -> MetricDimType:
+        return self._metric_dim_type
     
     @property
-    def metric_type(self) -> MetricType:
-        """The type of the metric.
-
-        Returns:
-            MetricType: The type of the metric.
-        """
-        return self._metric_type
+    def graph_type(self) -> GraphType:
+        return self._graph_type
     
-    @property
-    def granularity(self) -> MetricGranularity:
-        """The granularity of the metric.
-
-        Returns:
-            MetricGranularity: The granularity of the metric.
-        """
-        return self._granularity
-
-    def log(self, data: Any, step: int | None = None, replace: bool = True) -> None:
+    def log(self, data: Any, x: Any = None, replace: bool = False) -> None:
         """Log a new data point for the metric. The behavior depends on the granularity.    
 
         Args:
             data (Any): The data point to log.
-            step (int | None): The step number for STEP granularity. Must be provided if granularity is STEP.
-            replace (bool): Whether to replace the last logged value if logging at the same step/episode/run.
+            x (Any | None): The x-axis value associated with the data point.
+                if None, the current step is used.
+            replace (bool): Whether to replace the last logged value.
         """
-        if self._granularity == MetricGranularity.STEP:
-            if step is None:
-                raise ValueError("step Must be provided!")
-            if step == self._last_step:
-                if replace:
-                    self._values[-1] = data
-                else:
-                    raise ValueError("Cannot log the data at the same step.")
+        if replace:
+            self._values[-1] = data
+            if x is not None:
+                self._x_values[-1] = x
+        else:
+            self._values.append(data)
+            if x is not None:
+                self._x_values.append(x)
             else:
-                self._values.append(data)
-                self._x_values.append(step)
-        elif self._granularity == MetricGranularity.EPISODE:
-            if len(self._x_values) > 0 and not replace:
-                raise ValueError("Cannot log the data at the same episode.")
-            self._values = [data]
-            self._x_values = [self._episode_id]
-        else: # MetricGranularity.RUN
-            if len(self._values) > 0 and not replace:
-                raise ValueError("Cannot log the data at the same run.")
-            self._values = [data]
-            self._x_values = [self._run_id]
-    
-    def _submit(self) -> None:
-        if not self._values:
-            # If there is no data to submit, then return.
-            return
-        # TODO: Implement commit logic
-
-        # Clear data after the submission.
+                self._x_values.append(self._step)
+                self._step += 1
+        
+    def finalize(self) -> dict:
+        """Finalize the logged data for processing."""
+        ret_result = self._finalize()
+
+        return ret_result
+
+    def _finalize(self) -> dict:
+        """Process the logged data before submission. To be implemented by subclasses."""
+        ret_val = {
+            "values": self._values,
+            "x_values": self._x_values
+        }
         self._values = []
         self._x_values = []
-        
-    def finish(self) -> None:
-        """Finish the metric logging and submit the final data."""
-        self.reset()
-        self._submit()
+        self._step = -1
+        return ret_val    

humalab/metrics/scenario_stats.py

@@ -0,0 +1,95 @@
+from humalab.metrics.metric import Metrics
+from humalab.constants import ArtifactType, GraphType, MetricDimType
+from humalab.humalab_api_client import EpisodeStatus
+from typing import Any
+
+
+SCENARIO_STATS_NEED_FLATTEN = {
+    "uniform_1d",
+    "bernoulli_1d",
+    "categorical_1d",
+    "discrete_1d",
+    "log_uniform_1d",
+    "gaussian_1d",
+    "truncated_gaussian_1d"
+}
+
+
+class ScenarioStats(Metrics):
+    """Metric to track scenario statistics such as total reward, length, and success.
+
+    Attributes:
+    """
+
+    def __init__(self, 
+                 name: str,
+                 distribution_type: str,
+                 metric_dim_type: MetricDimType,
+                 graph_type: GraphType,
+                 ) -> None:
+        super().__init__(
+            metric_dim_type=metric_dim_type,
+            graph_type=graph_type
+        )
+        self._name = name
+        self._distribution_type = distribution_type
+        self._artifact_type = ArtifactType.SCENARIO_STATS
+        self._values = {}
+        self._results = {}
+
+    @property
+    def name(self) -> str:
+        return self._name
+    
+    @property
+    def distribution_type(self) -> str:
+        return self._distribution_type
+
+    @property
+    def artifact_type(self) -> ArtifactType:
+        return self._artifact_type
+    
+    def log(self, data: Any, x: Any = None, replace: bool = False) -> None:
+        if x in self._values:
+            if replace:
+                if self._distribution_type in SCENARIO_STATS_NEED_FLATTEN:
+                    data = data[0]
+                self._values[x] = data
+            else:   
+                raise ValueError(f"Data for episode_id {x} already exists. Use replace=True to overwrite.")
+        else:
+            if self._distribution_type in SCENARIO_STATS_NEED_FLATTEN:
+                data = data[0]
+            self._values[x] = data
+    
+    def log_status(self, 
+                   episode_id: str,
+                   episode_status: EpisodeStatus, 
+                   replace: bool = False) -> None:
+        """Log a new data point for the metric. The behavior depends on the granularity.    
+
+        Args:
+            data (Any): The data point to log.
+            x (Any | None): The x-axis value associated with the data point.
+                if None, the current step is used.
+            replace (bool): Whether to replace the last logged value.
+        """
+        if episode_id in self._results:
+            if replace:
+                self._results[episode_id] = episode_status
+            else:   
+                raise ValueError(f"Data for episode_id {episode_id} already exists. Use replace=True to overwrite.")
+        else:
+            self._results[episode_id] = episode_status
+
+    def _finalize(self) -> dict:
+        ret_val = {
+            "values": self._values,
+            "results": self._results,
+            "distribution_type": self._distribution_type,
+        }
+        self._values = {}
+        self._results = {}
+        return ret_val
+        
+    

humalab/metrics/summary.py

@@ -1,15 +1,11 @@
 
-from humalab.metrics.metric import MetricGranularity, Metrics, MetricType
-from humalab.constants import EpisodeStatus
+from humalab.metrics.metric import Metrics
+from humalab.constants import MetricDimType, GraphType
 
 
 class Summary(Metrics):
     def __init__(self, 
-                 name: str, 
                  summary: str,
-                 episode_id: str,
-                 run_id: str,
-                 granularity: MetricGranularity = MetricGranularity.RUN,
                  ) -> None:
         """
         A summary metric that captures a single value per episode or run.
@@ -22,26 +18,33 @@ class Summary(Metrics):
                 from being generated.
             granularity (MetricGranularity): The granularity of the metric.
         """
-        if granularity == MetricGranularity.RUN:
-            raise ValueError("Summary metrics cannot have RUN granularity.")
         if summary not in {"min", "max", "mean", "last", "first", "none"}:
             raise ValueError(f"Unsupported summary type: {summary}. Supported types are 'min', 'max', 'mean', 'last', 'first', and 'none'.")
-        super().__init__(name, MetricType.SUMMARY, episode_id=episode_id, run_id=run_id, granularity=granularity)
-        self.summary = summary
+        super().__init__(metric_dim_type= MetricDimType.ZERO_D, 
+                         graph_type=GraphType.NUMERIC)
+        self._summary = summary
+    
+    @property
+    def summary(self) -> str:
+        return self._summary
 
-    def _submit(self) -> None:
+    def _finalize(self) -> dict:
         if not self._values:
-            return
+            return {
+                "value": None,
+                "summary": self.summary
+            }
+        final_val = None
         # For summary metrics, we only keep the latest value
         if self.summary == "last":
-            self._values = [self._values[-1]]
+            final_val = self._values[-1]
         elif self.summary == "first":
-            self._values = [self._values[0]]
+            final_val = self._values[0]
         elif self.summary == "none":
-            self._values = []
+            final_val = None
         elif self.summary in {"min", "max", "mean"}:
             if not self._values:
-                self._values = []
+                final_val = None
             else:
                 if self.summary == "min":
                     agg_value = min(self._values)
@@ -49,6 +52,9 @@ class Summary(Metrics):
                     agg_value = max(self._values)
                 elif self.summary == "mean":
                     agg_value = sum(self._values) / len(self._values)
-                self._values = [agg_value]
+                final_val = agg_value
 
-        super()._submit()
+        return {
+            "value": final_val,
+            "summary": self.summary
+        }