humalab 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

humalab/humalab_config.py

@@ -3,6 +3,18 @@ import yaml
 import os
 
 class HumalabConfig:
+    """Manages HumaLab SDK configuration settings.
+
+    Configuration is stored in ~/.humalab/config.yaml and includes workspace path,
+    API credentials, and connection settings. Values are automatically loaded on
+    initialization and saved when modified through property setters.
+
+    Attributes:
+        workspace_path (str): The local workspace directory path.
+        base_url (str): The HumaLab API base URL.
+        api_key (str): The API key for authentication.
+        timeout (float): Request timeout in seconds.
+    """
     def __init__(self):
         self._config = {
             "workspace_path": "",
@@ -17,6 +29,7 @@ class HumalabConfig:
         self._load_config()
 
     def _load_config(self):
+        """Load configuration from ~/.humalab/config.yaml."""
         home_path = Path.home()
         config_path = home_path / ".humalab" / "config.yaml"
         if not config_path.exists():
@@ -30,10 +43,16 @@ class HumalabConfig:
         self._timeout = self._config["timeout"] if self._config and "timeout" in self._config else 30.0
 
     def _save(self) -> None:
+        """Save current configuration to ~/.humalab/config.yaml."""
         yaml.dump(self._config, open(Path.home() / ".humalab" / "config.yaml", "w"))
 
     @property
     def workspace_path(self) -> str:
+        """The local workspace directory path.
+
+        Returns:
+            str: The workspace path.
+        """
         return str(self._workspace_path)
     
     @workspace_path.setter
@@ -44,30 +63,60 @@ class HumalabConfig:
 
     @property
     def base_url(self) -> str:
+        """The HumaLab API base URL.
+
+        Returns:
+            str: The base URL.
+        """
         return str(self._base_url)
 
     @base_url.setter
     def base_url(self, base_url: str) -> None:
+        """Set the HumaLab API base URL and save to config.
+
+        Args:
+            base_url (str): The new base URL.
+        """
         self._base_url = base_url
         self._config["base_url"] = base_url
         self._save()
 
     @property
     def api_key(self) -> str:
+        """The API key for authentication.
+
+        Returns:
+            str: The API key.
+        """
         return str(self._api_key)
 
     @api_key.setter
     def api_key(self, api_key: str) -> None:
+        """Set the API key and save to config.
+
+        Args:
+            api_key (str): The new API key.
+        """
         self._api_key = api_key
         self._config["api_key"] = api_key
         self._save()
 
     @property
     def timeout(self) -> float:
+        """Request timeout in seconds.
+
+        Returns:
+            float: The timeout value.
+        """
         return self._timeout
 
     @timeout.setter
     def timeout(self, timeout: float) -> None:
+        """Set the request timeout and save to config.
+
+        Args:
+            timeout (float): The new timeout in seconds.
+        """
         self._timeout = timeout
         self._config["timeout"] = timeout
         self._save()

humalab/humalab_test.py

@@ -2,12 +2,13 @@ import unittest
 from unittest.mock import patch, MagicMock, Mock
 import uuid
 
+from humalab.constants import DEFAULT_PROJECT
 from humalab import humalab
 from humalab.run import Run
-from humalab.scenario import Scenario
+from humalab.scenarios.scenario import Scenario
 from humalab.humalab_config import HumalabConfig
 from humalab.humalab_api_client import HumaLabApiClient
-from humalab.constants import EpisodeStatus
+from humalab.humalab_api_client import EpisodeStatus, RunStatus
 
 
 class HumalabTest(unittest.TestCase):
@@ -30,9 +31,10 @@ class HumalabTest(unittest.TestCase):
         # Pre-condition
         client = Mock()
         scenario = {"key": "value"}
+        project = "test_project"
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=scenario, scenario_id=None)
+        result = humalab._pull_scenario(client=client, project=project, scenario=scenario, scenario_id=None)
 
         # Post-condition
         self.assertEqual(result, scenario)
@@ -42,32 +44,34 @@ class HumalabTest(unittest.TestCase):
         """Test that _pull_scenario fetches from API when scenario_id is provided."""
         # Pre-condition
         client = Mock()
+        project = "test_project"
         scenario_id = "test-scenario-id"
         yaml_content = "scenario: test"
         client.get_scenario.return_value = {"yaml_content": yaml_content}
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=None, scenario_id=scenario_id)
+        result = humalab._pull_scenario(client=client, project=project, scenario=None, scenario_id=scenario_id)
 
         # Post-condition
         self.assertEqual(result, yaml_content)
-        client.get_scenario.assert_called_once_with(uuid=scenario_id)
+        client.get_scenario.assert_called_once_with(project_name=project, uuid=scenario_id, version=None)
 
     def test_pull_scenario_should_prefer_scenario_id_over_scenario(self):
         """Test that _pull_scenario uses scenario_id even when scenario is provided."""
         # Pre-condition
         client = Mock()
+        project = "test_project"
         scenario = {"key": "value"}
         scenario_id = "test-scenario-id"
         yaml_content = "scenario: from_api"
         client.get_scenario.return_value = {"yaml_content": yaml_content}
 
         # In-test
-        result = humalab._pull_scenario(client=client, scenario=scenario, scenario_id=scenario_id)
+        result = humalab._pull_scenario(client=client, project=project, scenario=scenario, scenario_id=scenario_id)
 
         # Post-condition
         self.assertEqual(result, yaml_content)
-        client.get_scenario.assert_called_once_with(uuid=scenario_id)
+        client.get_scenario.assert_called_once_with(project_name=project, uuid=scenario_id, version=None)
 
     # Tests for init context manager
 
@@ -92,6 +96,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": project}
+        mock_api_client.get_run.return_value = {"run_id": run_id, "name": name, "description": description, "tags": tags}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -137,6 +143,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -149,7 +157,7 @@ class HumalabTest(unittest.TestCase):
         with humalab.init() as run:
             # Post-condition
             call_kwargs = mock_run_class.call_args.kwargs
-            self.assertEqual(call_kwargs['project'], "default")
+            self.assertEqual(call_kwargs['project'], DEFAULT_PROJECT)
             self.assertEqual(call_kwargs['name'], "")
             self.assertEqual(call_kwargs['description'], "")
             self.assertIsNotNone(call_kwargs['id'])  # UUID generated
@@ -170,7 +178,18 @@ class HumalabTest(unittest.TestCase):
         mock_config.timeout = 30.0
         mock_config_class.return_value = mock_config
 
+        # Mock HTTP 404 error for get_run (run doesn't exist yet)
+        import requests
+        http_error = requests.HTTPError()
+        http_error.response = Mock()
+        http_error.response.status_code = 404
+
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.side_effect = http_error
+        # Mock create_run to return a valid UUID
+        generated_uuid = str(uuid.uuid4())
+        mock_api_client.create_run.return_value = {"run_id": generated_uuid, "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -206,6 +225,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -241,6 +262,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client.get_scenario.return_value = {"yaml_content": yaml_content}
         mock_api_client_class.return_value = mock_api_client
 
@@ -253,7 +276,7 @@ class HumalabTest(unittest.TestCase):
         # In-test
         with humalab.init(scenario_id=scenario_id) as run:
             # Post-condition
-            mock_api_client.get_scenario.assert_called_once_with(uuid=scenario_id)
+            mock_api_client.get_scenario.assert_called_once_with(project_name='default', uuid=scenario_id, version=None)
             mock_scenario_inst.init.assert_called_once()
             call_kwargs = mock_scenario_inst.init.call_args.kwargs
             self.assertEqual(call_kwargs['scenario'], yaml_content)
@@ -274,6 +297,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -304,6 +329,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -338,6 +365,8 @@ class HumalabTest(unittest.TestCase):
         mock_config_class.return_value = mock_config
 
         mock_api_client = Mock()
+        mock_api_client.create_project.return_value = {"name": DEFAULT_PROJECT}
+        mock_api_client.get_run.return_value = {"run_id": "", "name": "", "description": "", "tags": None}
         mock_api_client_class.return_value = mock_api_client
 
         mock_scenario_inst = Mock()
@@ -369,33 +398,33 @@ class HumalabTest(unittest.TestCase):
         humalab.finish()
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=EpisodeStatus.PASS, quiet=None)
+        mock_run.finish.assert_called_once_with(status=RunStatus.FINISHED, err_msg=None)
 
     def test_finish_should_call_finish_on_current_run_with_custom_status(self):
         """Test that finish() calls finish on the current run with custom status."""
         # Pre-condition
         mock_run = Mock()
         humalab._cur_run = mock_run
-        status = EpisodeStatus.FAILED
+        status = RunStatus.ERRORED
 
         # In-test
         humalab.finish(status=status)
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=status, quiet=None)
+        mock_run.finish.assert_called_once_with(status=status, err_msg=None)
 
-    def test_finish_should_call_finish_on_current_run_with_quiet_parameter(self):
-        """Test that finish() calls finish on the current run with quiet parameter."""
+    def test_finish_should_call_finish_on_current_run_with_err_msg_parameter(self):
+        """Test that finish() calls finish on the current run with err_msg parameter."""
         # Pre-condition
         mock_run = Mock()
         humalab._cur_run = mock_run
-        quiet = True
+        err_msg = "Test error message"
 
         # In-test
-        humalab.finish(quiet=quiet)
+        humalab.finish(err_msg=err_msg)
 
         # Post-condition
-        mock_run.finish.assert_called_once_with(status=EpisodeStatus.PASS, quiet=quiet)
+        mock_run.finish.assert_called_once_with(status=RunStatus.FINISHED, err_msg=err_msg)
 
     def test_finish_should_do_nothing_when_no_current_run(self):
         """Test that finish() does nothing when _cur_run is None."""

humalab/metrics/__init__.py

@@ -1,11 +1,17 @@
-from .metric import MetricGranularity, MetricType, Metrics
-from .dist_metric import DistributionMetric
+"""Metrics tracking and management.
+
+This module provides classes for tracking various types of metrics during runs and episodes,
+including general metrics, summary statistics, code artifacts, and scenario statistics.
+"""
+
+from .metric import Metrics
+from .code import Code
+from .scenario_stats import ScenarioStats
 from .summary import Summary
 
 __all__ = [
-    "MetricGranularity",
-    "MetricType",
+    "Code",
     "Metrics",
-    "DistributionMetric",
+    "ScenarioStats",
     "Summary",
 ]

humalab/metrics/code.py

@@ -0,0 +1,59 @@
+class Code:
+    """Class for logging code artifacts.
+
+    Code artifacts capture source code or configuration files associated with
+    runs or episodes. They are stored as text content and can be retrieved later
+    for reproducibility and debugging purposes.
+
+    Attributes:
+        run_id (str): The unique identifier of the associated run.
+        key (str): The artifact key/name for this code.
+        code_content (str): The actual code or text content.
+        episode_id (str | None): Optional episode identifier if scoped to an episode.
+    """
+    def __init__(self,
+                 run_id: str,
+                 key: str,
+                 code_content: str,
+                 episode_id: str | None = None) -> None:
+        super().__init__()
+        self._run_id = run_id
+        self._key = key
+        self._code_content = code_content
+        self._episode_id = episode_id
+
+    @property
+    def run_id(self) -> str:
+        """The unique identifier of the associated run.
+
+        Returns:
+            str: The run ID.
+        """
+        return self._run_id
+
+    @property
+    def key(self) -> str:
+        """The artifact key/name for this code.
+
+        Returns:
+            str: The artifact key.
+        """
+        return self._key
+
+    @property
+    def code_content(self) -> str:
+        """The actual code or text content.
+
+        Returns:
+            str: The code content.
+        """
+        return self._code_content
+
+    @property
+    def episode_id(self) -> str | None:
+        """Optional episode identifier if scoped to an episode.
+
+        Returns:
+            str | None: The episode ID, or None if run-scoped.
+        """
+        return self._episode_id

humalab/metrics/metric.py

@@ -1,129 +1,96 @@
-from enum import Enum
-from typing import Any  
-from humalab.constants import EpisodeStatus
+from typing import Any
+from humalab.constants import MetricDimType, GraphType
 
+GRAPH_TO_DIM_TYPE = {
+    GraphType.NUMERIC: MetricDimType.ZERO_D,
+    GraphType.LINE: MetricDimType.ONE_D,
+    GraphType.HISTOGRAM: MetricDimType.ONE_D,
+    GraphType.BAR: MetricDimType.ONE_D,
+    GraphType.GAUSSIAN: MetricDimType.ONE_D,
+    GraphType.SCATTER: MetricDimType.TWO_D,
+    GraphType.HEATMAP: MetricDimType.TWO_D,
+    GraphType.THREE_D_MAP: MetricDimType.THREE_D,
+}
 
-class MetricType(Enum):
-    DEFAULT = "default"
-    STREAM = "stream"
-    DISTRIBUTION = "distribution"
-    SUMMARY = "summary"
 
+class Metrics:
+    """Base class for tracking and logging metrics during runs and episodes.
 
-class MetricGranularity(Enum):
-    STEP = "step"
-    EPISODE = "episode"
-    RUN = "run"
+    Metrics provide a flexible way to log time-series data or aggregated values
+    during experiments. Data points are collected with optional x-axis values
+    and can be visualized using different graph types.
 
+    Subclasses should override _finalize() to implement custom processing logic.
+
+    Attributes:
+        graph_type (GraphType): The type of graph used for visualization.
+    """
+    def __init__(self,
+                 graph_type: GraphType=GraphType.LINE) -> None:
+        """Initialize a new Metrics instance.
 
-class Metrics:
-    def __init__(self, 
-                 name: str, 
-                 metric_type: MetricType,
-                 episode_id: str,
-                 run_id: str,
-                 granularity: MetricGranularity = MetricGranularity.STEP) -> None:
-        """
-        Base class for different types of metrics.
-        
         Args:
-            name (str): The name of the metric.
-            metric_type (MetricType): The type of the metric.
-            episode_id (str): The ID of the episode.
-            run_id (str): The ID of the run.
-            granularity (MetricGranularity): The granularity of the metric.
+            graph_type (GraphType): The type of graph to use for visualization
+                (e.g., LINE, BAR, HISTOGRAM, SCATTER). Defaults to LINE.
         """
-        self._name = name
-        self._metric_type = metric_type
-        self._granularity = granularity
         self._values = []
         self._x_values = []
-        self._episode_id = episode_id
-        self._run_id = run_id
-        self._last_step = -1
-    
-    def reset(self, 
-              episode_id: str | None = None) -> None:
-        """Reset the metric for a new episode or run.
+        self._step = -1
+        self._metric_dim_type = GRAPH_TO_DIM_TYPE.get(graph_type, MetricDimType.ONE_D)
+        self._graph_type = graph_type
 
-        Args:
-            episode_id (str | None): Optional new episode ID. If None, keeps the current episode ID.
-        """
-        if self._granularity != MetricGranularity.RUN:
-            self._submit()
-            self._values = []
-            self._x_values = []
-        self._last_step = -1
-        self._episode_id = episode_id
-        
-    @property
-    def name(self) -> str:
-        """The name of the metric.
-        
-        Returns:
-            str: The name of the metric.
-        """
-        return self._name
-    
     @property
-    def metric_type(self) -> MetricType:
-        """The type of the metric.
+    def metric_dim_type(self) -> MetricDimType:
+        """The dimensionality of the metric data.
 
         Returns:
-            MetricType: The type of the metric.
+            MetricDimType: The metric dimension type.
         """
-        return self._metric_type
-    
+        return self._metric_dim_type
+
     @property
-    def granularity(self) -> MetricGranularity:
-        """The granularity of the metric.
+    def graph_type(self) -> GraphType:
+        """The type of graph used for visualization.
 
         Returns:
-            MetricGranularity: The granularity of the metric.
+            GraphType: The graph type.
         """
-        return self._granularity
-
-    def log(self, data: Any, step: int | None = None, replace: bool = True) -> None:
-        """Log a new data point for the metric. The behavior depends on the granularity.    
+        return self._graph_type
+    
+    def log(self, data: Any, x: Any = None, replace: bool = False) -> None:
+        """Log a new data point for the metric.
 
         Args:
             data (Any): The data point to log.
-            step (int | None): The step number for STEP granularity. Must be provided if granularity is STEP.
-            replace (bool): Whether to replace the last logged value if logging at the same step/episode/run.
+            x (Any | None): The x-axis value associated with the data point.
+                If None, uses an auto-incrementing step counter.
+            replace (bool): Whether to replace the last logged value. Defaults to False.
         """
-        if self._granularity == MetricGranularity.STEP:
-            if step is None:
-                raise ValueError("step Must be provided!")
-            if step == self._last_step:
-                if replace:
-                    self._values[-1] = data
-                else:
-                    raise ValueError("Cannot log the data at the same step.")
+        if replace:
+            self._values[-1] = data
+            if x is not None:
+                self._x_values[-1] = x
+        else:
+            self._values.append(data)
+            if x is not None:
+                self._x_values.append(x)
             else:
-                self._values.append(data)
-                self._x_values.append(step)
-        elif self._granularity == MetricGranularity.EPISODE:
-            if len(self._x_values) > 0 and not replace:
-                raise ValueError("Cannot log the data at the same episode.")
-            self._values = [data]
-            self._x_values = [self._episode_id]
-        else: # MetricGranularity.RUN
-            if len(self._values) > 0 and not replace:
-                raise ValueError("Cannot log the data at the same run.")
-            self._values = [data]
-            self._x_values = [self._run_id]
-    
-    def _submit(self) -> None:
-        if not self._values:
-            # If there is no data to submit, then return.
-            return
-        # TODO: Implement commit logic
+                self._x_values.append(self._step)
+                self._step += 1
+        
+    def finalize(self) -> dict:
+        """Finalize the logged data for processing."""
+        ret_result = self._finalize()
 
-        # Clear data after the submission.
+        return ret_result
+
+    def _finalize(self) -> dict:
+        """Process the logged data before submission. To be implemented by subclasses."""
+        ret_val = {
+            "values": self._values,
+            "x_values": self._x_values
+        }
         self._values = []
         self._x_values = []
-        
-    def finish(self) -> None:
-        """Finish the metric logging and submit the final data."""
-        self.reset()
-        self._submit()
+        self._step = -1
+        return ret_val