humalab 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

humalab/humalab_config.py

@@ -6,13 +6,11 @@ class HumalabConfig:
     def __init__(self):
         self._config = {
             "workspace_path": "",
-            "entity": "",
             "base_url": "",
             "api_key": "",
             "timeout": 30.0,
         }
         self._workspace_path = ""
-        self._entity = ""
         self._base_url = ""
         self._api_key = ""
         self._timeout = 30.0
@@ -27,7 +25,6 @@ class HumalabConfig:
         with open(config_path, "r") as f:
             self._config = yaml.safe_load(f)
         self._workspace_path = os.path.expanduser(self._config["workspace_path"]) if self._config and "workspace_path" in self._config else home_path
-        self._entity = self._config["entity"] if self._config and "entity" in self._config else ""
         self._base_url = self._config["base_url"] if self._config and "base_url" in self._config else ""
         self._api_key = self._config["api_key"] if self._config and "api_key" in self._config else ""
         self._timeout = self._config["timeout"] if self._config and "timeout" in self._config else 30.0
@@ -45,16 +42,6 @@ class HumalabConfig:
         self._config["workspace_path"] = path
         self._save()
 
-    @property
-    def entity(self) -> str:
-        return str(self._entity)
-
-    @entity.setter
-    def entity(self, entity: str) -> None:
-        self._entity = entity
-        self._config["entity"] = entity
-        self._save()
-
     @property
     def base_url(self) -> str:
         return str(self._base_url)

humalab/humalab_test.py

@@ -2,12 +2,13 @@ import unittest
 from unittest.mock import patch, MagicMock, Mock
 import uuid
 
+from humalab.constants import DEFAULT_PROJECT
 from humalab import humalab
 from humalab.run import Run
-from humalab.scenario import Scenario
+from humalab.scenarios.scenario import Scenario
 from humalab.humalab_config import HumalabConfig
 from humalab.humalab_api_client import HumaLabApiClient
-from humalab.constants import EpisodeStatus
+from humalab.humalab_api_client import EpisodeStatus, RunStatus
 
 
 class HumalabTest(unittest.TestCase):
@@ -30,9 +31,10 @@ class HumalabTest(unittest.TestCase):
         # Pre-condition
         client = Mock()
         scenario = {"key": "value"}
+        project = "test_project"
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=scenario, scenario_id=None)
+        result = humalab._pull_scenario(client=client, project=project, scenario=scenario, scenario_id=None)
 
         # Post-condition
         self.assertEqual(result, scenario)
@@ -42,32 +44,34 @@ class HumalabTest(unittest.TestCase):
         """Test that _pull_scenario fetches from API when scenario_id is provided."""
         # Pre-condition
         client = Mock()
+        project = "test_project"
         scenario_id = "test-scenario-id"
         yaml_content = "scenario: test"
         client.get_scenario.return_value = {"yaml_content": yaml_content}
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=None, scenario_id=scenario_id)
+        result = humalab._pull_scenario(client=client, project=project, scenario=None, scenario_id=scenario_id)
 
         # Post-condition
         self.assertEqual(result, yaml_content)
-        client.get_scenario.assert_called_once_with(uuid=scenario_id)
+        client.get_scenario.assert_called_once_with(project_name=project, uuid=scenario_id, version=None)
 
     def test_pull_scenario_should_prefer_scenario_id_over_scenario(self):
         """Test that _pull_scenario uses scenario_id even when scenario is provided."""
         # Pre-condition
         client = Mock()
+        project = "test_project"
         scenario = {"key": "value"}
         scenario_id = "test-scenario-id"
         yaml_content = "scenario: from_api"
         client.get_scenario.return_value = {"yaml_content": yaml_content}
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=scenario, scenario_id=scenario_id)
+        result = humalab._pull_scenario(client=client, project=project, scenario=scenario, scenario_id=scenario_id)
 
         # Post-condition
         self.assertEqual(result, yaml_content)
-        client.get_scenario.assert_called_once_with(uuid=scenario_id)
+        client.get_scenario.assert_called_once_with(project_name=project, uuid=scenario_id, version=None)
 
     # Tests for init context manager
 
@@ -78,7 +82,6 @@ class HumalabTest(unittest.TestCase):
     def test_init_should_create_run_with_provided_parameters(self, mock_run_class, mock_scenario_class, mock_config_class, mock_api_client_class):
         """Test that init() creates a Run with provided parameters."""
         # Pre-condition
-        entity = "test_entity"
         project = "test_project"
         name = "test_name"
         description = "test_description"
@@ -87,13 +90,14 @@ class HumalabTest(unittest.TestCase):
         scenario_data = {"key": "value"}
 
         mock_config = Mock()
-        mock_config.entity = "default_entity"
         mock_config.base_url = "http://localhost:8000"
         mock_config.api_key = "test_key"
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": project}
+        mock_api_client.get_run.return_value = {"run_id": run_id, "name": name, "description": description, "tags": tags}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -104,7 +108,6 @@ class HumalabTest(unittest.TestCase):
 
         # In-test
         with humalab.init(
-            entity=entity,
             project=project,
             name=name,
             description=description,
@@ -116,7 +119,6 @@ class HumalabTest(unittest.TestCase):
             self.assertEqual(run, mock_run_inst)
             mock_run_class.assert_called_once()
             call_kwargs = mock_run_class.call_args.kwargs
-            self.assertEqual(call_kwargs['entity'], entity)
             self.assertEqual(call_kwargs['project'], project)
             self.assertEqual(call_kwargs['name'], name)
             self.assertEqual(call_kwargs['description'], description)
@@ -135,13 +137,14 @@ class HumalabTest(unittest.TestCase):
         """Test that init() uses config defaults when parameters are not provided."""
         # Pre-condition
         mock_config = Mock()
-        mock_config.entity = "config_entity"
         mock_config.base_url = "http://config:8000"
         mock_config.api_key = "config_key"
         mock_config.timeout = 60.0
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -154,8 +157,7 @@ class HumalabTest(unittest.TestCase):
         with humalab.init() as run:
             # Post-condition
             call_kwargs = mock_run_class.call_args.kwargs
-            self.assertEqual(call_kwargs['entity'], "config_entity")
-            self.assertEqual(call_kwargs['project'], "default")
+            self.assertEqual(call_kwargs['project'], DEFAULT_PROJECT)
             self.assertEqual(call_kwargs['name'], "")
             self.assertEqual(call_kwargs['description'], "")
             self.assertIsNotNone(call_kwargs['id'])  # UUID generated
@@ -171,13 +173,23 @@ class HumalabTest(unittest.TestCase):
         """Test that init() generates a UUID when id is not provided."""
         # Pre-condition
         mock_config = Mock()
-        mock_config.entity = "test_entity"
         mock_config.base_url = "http://localhost:8000"
         mock_config.api_key = "test_key"
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
+        # Mock HTTP 404 error for get_run (run doesn't exist yet)
+        import requests
+        http_error = requests.HTTPError()
+        http_error.response = Mock()
+        http_error.response.status_code = 404
+
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.side_effect = http_error
+        # Mock create_run to return a valid UUID
+        generated_uuid = str(uuid.uuid4())
+        mock_api_client.create_run.return_value = {"run_id": generated_uuid, "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -207,13 +219,14 @@ class HumalabTest(unittest.TestCase):
         scenario_data = {"key": "value"}
 
         mock_config = Mock()
-        mock_config.entity = "test_entity"
         mock_config.base_url = "http://localhost:8000"
         mock_config.api_key = "test_key"
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -243,13 +256,14 @@ class HumalabTest(unittest.TestCase):
         yaml_content = "scenario: from_api"
 
         mock_config = Mock()
-        mock_config.entity = "test_entity"
         mock_config.base_url = "http://localhost:8000"
         mock_config.api_key = "test_key"
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client.get_scenario.return_value = {"yaml_content": yaml_content}
         mock_api_client_class.return_value = mock_api_client
 
@@ -262,7 +276,7 @@ class HumalabTest(unittest.TestCase):
         # In-test
         with humalab.init(scenario_id=scenario_id) as run:
             # Post-condition
-            mock_api_client.get_scenario.assert_called_once_with(uuid=scenario_id)
+            mock_api_client.get_scenario.assert_called_once_with(project_name='default', uuid=scenario_id, version=None)
             mock_scenario_inst.init.assert_called_once()
             call_kwargs = mock_scenario_inst.init.call_args.kwargs
             self.assertEqual(call_kwargs['scenario'], yaml_content)
@@ -277,13 +291,14 @@ class HumalabTest(unittest.TestCase):
         """Test that init() sets the global _cur_run variable."""
         # Pre-condition
         mock_config = Mock()
-        mock_config.entity = "test_entity"
         mock_config.base_url = "http://localhost:8000"
         mock_config.api_key = "test_key"
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -308,13 +323,14 @@ class HumalabTest(unittest.TestCase):
         """Test that init() calls finish even when exception occurs in context."""
         # Pre-condition
         mock_config = Mock()
-        mock_config.entity = "test_entity"
         mock_config.base_url = "http://localhost:8000"
         mock_config.api_key = "test_key"
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -343,13 +359,14 @@ class HumalabTest(unittest.TestCase):
         timeout = 120.0
 
         mock_config = Mock()
-        mock_config.entity = "test_entity"
         mock_config.base_url = "http://localhost:8000"
         mock_config.api_key = "default_key"
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -381,33 +398,33 @@ class HumalabTest(unittest.TestCase):
         humalab.finish()
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=EpisodeStatus.PASS, quiet=None)
+        mock_run.finish.assert_called_once_with(status=RunStatus.FINISHED, err_msg=None)
 
     def test_finish_should_call_finish_on_current_run_with_custom_status(self):
         """Test that finish() calls finish on the current run with custom status."""
         # Pre-condition
         mock_run = Mock()
         humalab._cur_run = mock_run
-        status = EpisodeStatus.FAILED
+        status = RunStatus.ERRORED
 
         # In-test
         humalab.finish(status=status)
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=status, quiet=None)
+        mock_run.finish.assert_called_once_with(status=status, err_msg=None)
 
-    def test_finish_should_call_finish_on_current_run_with_quiet_parameter(self):
-        """Test that finish() calls finish on the current run with quiet parameter."""
+    def test_finish_should_call_finish_on_current_run_with_err_msg_parameter(self):
+        """Test that finish() calls finish on the current run with err_msg parameter."""
         # Pre-condition
         mock_run = Mock()
         humalab._cur_run = mock_run
-        quiet = True
+        err_msg = "Test error message"
 
         # In-test
-        humalab.finish(quiet=quiet)
+        humalab.finish(err_msg=err_msg)
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=EpisodeStatus.PASS, quiet=quiet)
+        mock_run.finish.assert_called_once_with(status=RunStatus.FINISHED, err_msg=err_msg)
 
     def test_finish_should_do_nothing_when_no_current_run(self):
         """Test that finish() does nothing when _cur_run is None."""

humalab/metrics/__init__.py

@@ -1,11 +1,11 @@
-from .metric import MetricGranularity, MetricType, Metrics
-from .dist_metric import DistributionMetric
+from .metric import Metrics
+from .code import Code
+from .scenario_stats import ScenarioStats
 from .summary import Summary
 
 __all__ = [
-    "MetricGranularity",
-    "MetricType",
+    "Code",
     "Metrics",
-    "DistributionMetric",
+    "ScenarioStats",
     "Summary",
 ]

humalab/metrics/code.py

@@ -0,0 +1,28 @@
+class Code:
+    """Class for logging code artifacts."""
+    def __init__(self, 
+                 run_id: str, 
+                 key: str,
+                 code_content: str,
+                 episode_id: str | None = None) -> None:
+        super().__init__()
+        self._run_id = run_id
+        self._key = key
+        self._code_content = code_content
+        self._episode_id = episode_id
+
+    @property
+    def run_id(self) -> str:
+        return self._run_id
+    
+    @property
+    def key(self) -> str:
+        return self._key
+    
+    @property
+    def code_content(self) -> str:
+        return self._code_content
+    
+    @property
+    def episode_id(self) -> str | None:
+        return self._episode_id

humalab/metrics/metric.py

@@ -1,129 +1,62 @@
-from enum import Enum
-from typing import Any  
-from humalab.constants import EpisodeStatus
-
-
-class MetricType(Enum):
-    DEFAULT = "default"
-    STREAM = "stream"
-    DISTRIBUTION = "distribution"
-    SUMMARY = "summary"
-
-
-class MetricGranularity(Enum):
-    STEP = "step"
-    EPISODE = "episode"
-    RUN = "run"
+from typing import Any
+from humalab.constants import MetricDimType, GraphType
 
 
 class Metrics:
-    def __init__(self, 
-                 name: str, 
-                 metric_type: MetricType,
-                 episode_id: str,
-                 run_id: str,
-                 granularity: MetricGranularity = MetricGranularity.STEP) -> None:
+    def __init__(self,
+                 metric_dim_type: MetricDimType= MetricDimType.ONE_D,
+                 graph_type: GraphType=GraphType.LINE) -> None:
         """
         Base class for different types of metrics.
-        
-        Args:
-            name (str): The name of the metric.
-            metric_type (MetricType): The type of the metric.
-            episode_id (str): The ID of the episode.
-            run_id (str): The ID of the run.
-            granularity (MetricGranularity): The granularity of the metric.
         """
-        self._name = name
-        self._metric_type = metric_type
-        self._granularity = granularity
         self._values = []
         self._x_values = []
-        self._episode_id = episode_id
-        self._run_id = run_id
-        self._last_step = -1
-    
-    def reset(self, 
-              episode_id: str | None = None) -> None:
-        """Reset the metric for a new episode or run.
+        self._step = -1
+        self._metric_dim_type = metric_dim_type
+        self._graph_type = graph_type
 
-        Args:
-            episode_id (str | None): Optional new episode ID. If None, keeps the current episode ID.
-        """
-        if self._granularity != MetricGranularity.RUN:
-            self._submit()
-            self._values = []
-            self._x_values = []
-        self._last_step = -1
-        self._episode_id = episode_id
-        
     @property
-    def name(self) -> str:
-        """The name of the metric.
-        
-        Returns:
-            str: The name of the metric.
-        """
-        return self._name
+    def metric_dim_type(self) -> MetricDimType:
+        return self._metric_dim_type
     
     @property
-    def metric_type(self) -> MetricType:
-        """The type of the metric.
-
-        Returns:
-            MetricType: The type of the metric.
-        """
-        return self._metric_type
+    def graph_type(self) -> GraphType:
+        return self._graph_type
     
-    @property
-    def granularity(self) -> MetricGranularity:
-        """The granularity of the metric.
-
-        Returns:
-            MetricGranularity: The granularity of the metric.
-        """
-        return self._granularity
-
-    def log(self, data: Any, step: int | None = None, replace: bool = True) -> None:
+    def log(self, data: Any, x: Any = None, replace: bool = False) -> None:
         """Log a new data point for the metric. The behavior depends on the granularity.    
 
         Args:
             data (Any): The data point to log.
-            step (int | None): The step number for STEP granularity. Must be provided if granularity is STEP.
-            replace (bool): Whether to replace the last logged value if logging at the same step/episode/run.
+            x (Any | None): The x-axis value associated with the data point.
+                if None, the current step is used.
+            replace (bool): Whether to replace the last logged value.
         """
-        if self._granularity == MetricGranularity.STEP:
-            if step is None:
-                raise ValueError("step Must be provided!")
-            if step == self._last_step:
-                if replace:
-                    self._values[-1] = data
-                else:
-                    raise ValueError("Cannot log the data at the same step.")
+        if replace:
+            self._values[-1] = data
+            if x is not None:
+                self._x_values[-1] = x
+        else:
+            self._values.append(data)
+            if x is not None:
+                self._x_values.append(x)
             else:
-                self._values.append(data)
-                self._x_values.append(step)
-        elif self._granularity == MetricGranularity.EPISODE:
-            if len(self._x_values) > 0 and not replace:
-                raise ValueError("Cannot log the data at the same episode.")
-            self._values = [data]
-            self._x_values = [self._episode_id]
-        else: # MetricGranularity.RUN
-            if len(self._values) > 0 and not replace:
-                raise ValueError("Cannot log the data at the same run.")
-            self._values = [data]
-            self._x_values = [self._run_id]
-    
-    def _submit(self) -> None:
-        if not self._values:
-            # If there is no data to submit, then return.
-            return
-        # TODO: Implement commit logic
-
-        # Clear data after the submission.
+                self._x_values.append(self._step)
+                self._step += 1
+        
+    def finalize(self) -> dict:
+        """Finalize the logged data for processing."""
+        ret_result = self._finalize()
+
+        return ret_result
+
+    def _finalize(self) -> dict:
+        """Process the logged data before submission. To be implemented by subclasses."""
+        ret_val = {
+            "values": self._values,
+            "x_values": self._x_values
+        }
         self._values = []
         self._x_values = []
-        
-    def finish(self) -> None:
-        """Finish the metric logging and submit the final data."""
-        self.reset()
-        self._submit()
+        self._step = -1
+        return ret_val    

humalab/metrics/scenario_stats.py

@@ -0,0 +1,95 @@
+from humalab.metrics.metric import Metrics
+from humalab.constants import ArtifactType, GraphType, MetricDimType
+from humalab.humalab_api_client import EpisodeStatus
+from typing import Any
+
+
+SCENARIO_STATS_NEED_FLATTEN = {
+    "uniform_1d",
+    "bernoulli_1d",
+    "categorical_1d",
+    "discrete_1d",
+    "log_uniform_1d",
+    "gaussian_1d",
+    "truncated_gaussian_1d"
+}
+
+
+class ScenarioStats(Metrics):
+    """Metric to track scenario statistics such as total reward, length, and success.
+
+    Attributes:
+    """
+
+    def __init__(self, 
+                 name: str,
+                 distribution_type: str,
+                 metric_dim_type: MetricDimType,
+                 graph_type: GraphType,
+                 ) -> None:
+        super().__init__(
+            metric_dim_type=metric_dim_type,
+            graph_type=graph_type
+        )
+        self._name = name
+        self._distribution_type = distribution_type
+        self._artifact_type = ArtifactType.SCENARIO_STATS
+        self._values = {}
+        self._results = {}
+
+    @property
+    def name(self) -> str:
+        return self._name
+    
+    @property
+    def distribution_type(self) -> str:
+        return self._distribution_type
+
+    @property
+    def artifact_type(self) -> ArtifactType:
+        return self._artifact_type
+    
+    def log(self, data: Any, x: Any = None, replace: bool = False) -> None:
+        if x in self._values:
+            if replace:
+                if self._distribution_type in SCENARIO_STATS_NEED_FLATTEN:
+                    data = data[0]
+                self._values[x] = data
+            else:   
+                raise ValueError(f"Data for episode_id {x} already exists. Use replace=True to overwrite.")
+        else:
+            if self._distribution_type in SCENARIO_STATS_NEED_FLATTEN:
+                data = data[0]
+            self._values[x] = data
+    
+    def log_status(self, 
+                   episode_id: str,
+                   episode_status: EpisodeStatus, 
+                   replace: bool = False) -> None:
+        """Log a new data point for the metric. The behavior depends on the granularity.    
+
+        Args:
+            data (Any): The data point to log.
+            x (Any | None): The x-axis value associated with the data point.
+                if None, the current step is used.
+            replace (bool): Whether to replace the last logged value.
+        """
+        if episode_id in self._results:
+            if replace:
+                self._results[episode_id] = episode_status
+            else:   
+                raise ValueError(f"Data for episode_id {episode_id} already exists. Use replace=True to overwrite.")
+        else:
+            self._results[episode_id] = episode_status
+
+    def _finalize(self) -> dict:
+        ret_val = {
+            "values": self._values,
+            "results": self._results,
+            "distribution_type": self._distribution_type,
+        }
+        self._values = {}
+        self._results = {}
+        return ret_val
+        
+