humalab 0.1.0__py3-none-any.whl

humalab/metrics/code.py

@@ -0,0 +1,59 @@
+class Code:
+    """Class for logging code artifacts.
+
+    Code artifacts capture source code or configuration files associated with
+    runs or episodes. They are stored as text content and can be retrieved later
+    for reproducibility and debugging purposes.
+
+    Attributes:
+        run_id (str): The unique identifier of the associated run.
+        key (str): The artifact key/name for this code.
+        code_content (str): The actual code or text content.
+        episode_id (str | None): Optional episode identifier if scoped to an episode.
+    """
+    def __init__(self,
+                 run_id: str,
+                 key: str,
+                 code_content: str,
+                 episode_id: str | None = None) -> None:
+        super().__init__()
+        self._run_id = run_id
+        self._key = key
+        self._code_content = code_content
+        self._episode_id = episode_id
+
+    @property
+    def run_id(self) -> str:
+        """The unique identifier of the associated run.
+
+        Returns:
+            str: The run ID.
+        """
+        return self._run_id
+
+    @property
+    def key(self) -> str:
+        """The artifact key/name for this code.
+
+        Returns:
+            str: The artifact key.
+        """
+        return self._key
+
+    @property
+    def code_content(self) -> str:
+        """The actual code or text content.
+
+        Returns:
+            str: The code content.
+        """
+        return self._code_content
+
+    @property
+    def episode_id(self) -> str | None:
+        """Optional episode identifier if scoped to an episode.
+
+        Returns:
+            str | None: The episode ID, or None if run-scoped.
+        """
+        return self._episode_id

humalab/metrics/metric.py

@@ -0,0 +1,96 @@
+from typing import Any
+from humalab.constants import MetricDimType, GraphType
+
+GRAPH_TO_DIM_TYPE = {
+    GraphType.NUMERIC: MetricDimType.ZERO_D,
+    GraphType.LINE: MetricDimType.ONE_D,
+    GraphType.HISTOGRAM: MetricDimType.ONE_D,
+    GraphType.BAR: MetricDimType.ONE_D,
+    GraphType.GAUSSIAN: MetricDimType.ONE_D,
+    GraphType.SCATTER: MetricDimType.TWO_D,
+    GraphType.HEATMAP: MetricDimType.TWO_D,
+    GraphType.THREE_D_MAP: MetricDimType.THREE_D,
+}
+
+
+class Metrics:
+    """Base class for tracking and logging metrics during runs and episodes.
+
+    Metrics provide a flexible way to log time-series data or aggregated values
+    during experiments. Data points are collected with optional x-axis values
+    and can be visualized using different graph types.
+
+    Subclasses should override _finalize() to implement custom processing logic.
+
+    Attributes:
+        graph_type (GraphType): The type of graph used for visualization.
+    """
+    def __init__(self,
+                 graph_type: GraphType=GraphType.LINE) -> None:
+        """Initialize a new Metrics instance.
+
+        Args:
+            graph_type (GraphType): The type of graph to use for visualization
+                (e.g., LINE, BAR, HISTOGRAM, SCATTER). Defaults to LINE.
+        """
+        self._values = []
+        self._x_values = []
+        self._step = -1
+        self._metric_dim_type = GRAPH_TO_DIM_TYPE.get(graph_type, MetricDimType.ONE_D)
+        self._graph_type = graph_type
+
+    @property
+    def metric_dim_type(self) -> MetricDimType:
+        """The dimensionality of the metric data.
+
+        Returns:
+            MetricDimType: The metric dimension type.
+        """
+        return self._metric_dim_type
+
+    @property
+    def graph_type(self) -> GraphType:
+        """The type of graph used for visualization.
+
+        Returns:
+            GraphType: The graph type.
+        """
+        return self._graph_type
+    
+    def log(self, data: Any, x: Any = None, replace: bool = False) -> None:
+        """Log a new data point for the metric.
+
+        Args:
+            data (Any): The data point to log.
+            x (Any | None): The x-axis value associated with the data point.
+                If None, uses an auto-incrementing step counter.
+            replace (bool): Whether to replace the last logged value. Defaults to False.
+        """
+        if replace:
+            self._values[-1] = data
+            if x is not None:
+                self._x_values[-1] = x
+        else:
+            self._values.append(data)
+            if x is not None:
+                self._x_values.append(x)
+            else:
+                self._x_values.append(self._step)
+                self._step += 1
+        
+    def finalize(self) -> dict:
+        """Finalize the logged data for processing."""
+        ret_result = self._finalize()
+
+        return ret_result
+
+    def _finalize(self) -> dict:
+        """Process the logged data before submission. To be implemented by subclasses."""
+        ret_val = {
+            "values": self._values,
+            "x_values": self._x_values
+        }
+        self._values = []
+        self._x_values = []
+        self._step = -1
+        return ret_val    

humalab/metrics/scenario_stats.py

@@ -0,0 +1,163 @@
+from humalab.metrics.metric import Metrics
+from humalab.constants import ArtifactType, GraphType, MetricDimType
+from humalab.humalab_api_client import EpisodeStatus
+from typing import Any
+
+
+SCENARIO_STATS_NEED_FLATTEN = {
+    "uniform_1d",
+    "bernoulli_1d",
+    "categorical_1d",
+    "discrete_1d",
+    "log_uniform_1d",
+    "gaussian_1d",
+    "truncated_gaussian_1d"
+}
+
+
+DISTRIBUTION_GRAPH_TYPE = {
+    # 0D distributions
+    "uniform": GraphType.HISTOGRAM,
+    "bernoulli": GraphType.HISTOGRAM,
+    "categorical": GraphType.BAR,
+    "discrete": GraphType.BAR,
+    "log_uniform": GraphType.HISTOGRAM,
+    "gaussian": GraphType.GAUSSIAN,
+    "truncated_gaussian": GraphType.GAUSSIAN,
+
+    # 1D distributions
+    "uniform_1d": GraphType.HISTOGRAM,
+    "bernoulli_1d": GraphType.HISTOGRAM,
+    "categorical_1d": GraphType.BAR,
+    "discrete_1d": GraphType.BAR,
+    "log_uniform_1d": GraphType.HISTOGRAM,
+    "gaussian_1d": GraphType.GAUSSIAN,
+    "truncated_gaussian_1d": GraphType.GAUSSIAN,
+
+    # 2D distributions
+    "uniform_2d": GraphType.SCATTER,
+    "gaussian_2d": GraphType.HEATMAP,
+    "truncated_gaussian_2d": GraphType.HEATMAP,
+
+    # 3D distributions
+    "uniform_3d": GraphType.THREE_D_MAP,
+    "gaussian_3d": GraphType.THREE_D_MAP,
+    "truncated_gaussian_3d": GraphType.THREE_D_MAP,
+}
+
+class ScenarioStats(Metrics):
+    """Metric to track scenario statistics across episodes.
+
+    This class logs sampled values from scenario distributions and tracks episode
+    statuses. It supports various distribution types and automatically handles
+    flattening for 1D distributions.
+
+    Attributes:
+        name (str): The name of the scenario statistic.
+        distribution_type (str): The type of distribution (e.g., 'uniform', 'gaussian').
+        artifact_type (ArtifactType): The artifact type, always SCENARIO_STATS.
+    """
+
+    def __init__(self, 
+                 name: str,
+                 distribution_type: str,
+                 ) -> None:
+        super().__init__(
+            graph_type=DISTRIBUTION_GRAPH_TYPE[distribution_type]
+        )
+        self._name = name
+        self._distribution_type = distribution_type
+        self._artifact_type = ArtifactType.SCENARIO_STATS
+        self._values = {}
+        self._results = {}
+
+    @property
+    def name(self) -> str:
+        """The name of the scenario statistic.
+
+        Returns:
+            str: The statistic name.
+        """
+        return self._name
+
+    @property
+    def distribution_type(self) -> str:
+        """The type of distribution used for this statistic.
+
+        Returns:
+            str: The distribution type (e.g., 'uniform', 'gaussian').
+        """
+        return self._distribution_type
+
+    @property
+    def artifact_type(self) -> ArtifactType:
+        """The artifact type, always SCENARIO_STATS.
+
+        Returns:
+            ArtifactType: The artifact type.
+        """
+        return self._artifact_type
+    
+    def log(self, data: Any, x: Any = None, replace: bool = False) -> None:
+        """Log a sampled value from the scenario distribution.
+
+        Args:
+            data (Any): The sampled value to log.
+            x (Any | None): The key/identifier for this sample (typically episode_id).
+                If None, auto-incrementing step is used.
+            replace (bool): Whether to replace an existing value. Defaults to False.
+
+        Raises:
+            ValueError: If data for the given x already exists and replace is False.
+        """
+        if x in self._values:
+            if replace:
+                if self._distribution_type in SCENARIO_STATS_NEED_FLATTEN:
+                    data = data[0]
+                self._values[x] = data
+            else:   
+                raise ValueError(f"Data for episode_id {x} already exists. Use replace=True to overwrite.")
+        else:
+            if self._distribution_type in SCENARIO_STATS_NEED_FLATTEN:
+                data = data[0]
+            self._values[x] = data
+    
+    def log_status(self,
+                   episode_id: str,
+                   episode_status: EpisodeStatus,
+                   replace: bool = False) -> None:
+        """Log the status of an episode.
+
+        Args:
+            episode_id (str): The unique identifier of the episode.
+            episode_status (EpisodeStatus): The status of the episode.
+            replace (bool): Whether to replace an existing status for this episode.
+                Defaults to False.
+
+        Raises:
+            ValueError: If status for the episode_id already exists and replace is False.
+        """
+        if episode_id in self._results:
+            if replace:
+                self._results[episode_id] = episode_status.value
+            else:   
+                raise ValueError(f"Data for episode_id {episode_id} already exists. Use replace=True to overwrite.")
+        else:
+            self._results[episode_id] = episode_status.value
+
+    def _finalize(self) -> dict:
+        """Finalize and return all collected scenario statistics.
+
+        Returns:
+            dict: Dictionary containing values, results, and distribution type.
+        """
+        ret_val = {
+            "values": self._values,
+            "results": self._results,
+            "distribution_type": self._distribution_type,
+        }
+        self._values = {}
+        self._results = {}
+        return ret_val
+        
+    

humalab/metrics/summary.py

@@ -0,0 +1,75 @@
+
+from humalab.metrics.metric import Metrics
+from humalab.constants import MetricDimType, GraphType
+
+
+class Summary(Metrics):
+    """A metric that aggregates logged values into a single summary statistic.
+
+    Summary metrics collect values throughout a run or episode and compute a single
+    aggregated result. Supported aggregation methods include min, max, mean, first,
+    last, and none (no aggregation).
+
+    Attributes:
+        summary (str): The aggregation method used.
+    """
+    def __init__(self,
+                 summary: str,
+                 ) -> None:
+        """
+        A summary metric that captures a single value per episode or run.
+
+        Args:
+            summary (str): Specify the aggregation method for the summary.
+                Supported aggregations include "min", "max", "mean", "last",
+                "first", and "none". "none" prevents a summary from being generated.
+        """
+        if summary not in {"min", "max", "mean", "last", "first", "none"}:
+            raise ValueError(f"Unsupported summary type: {summary}. Supported types are 'min', 'max', 'mean', 'last', 'first', and 'none'.")
+        super().__init__(graph_type=GraphType.NUMERIC)
+        self._summary = summary
+    
+    @property
+    def summary(self) -> str:
+        """The aggregation method for this summary metric.
+
+        Returns:
+            str: The summary type (e.g., 'min', 'max', 'mean').
+        """
+        return self._summary
+
+    def _finalize(self) -> dict:
+        """Compute the final aggregated value.
+
+        Returns:
+            dict: Dictionary containing the aggregated value and summary type.
+        """
+        if not self._values:
+            return {
+                "value": None,
+                "summary": self.summary
+            }
+        final_val = None
+        # For summary metrics, we only keep the latest value
+        if self.summary == "last":
+            final_val = self._values[-1]
+        elif self.summary == "first":
+            final_val = self._values[0]
+        elif self.summary == "none":
+            final_val = None
+        elif self.summary in {"min", "max", "mean"}:
+            if not self._values:
+                final_val = None
+            else:
+                if self.summary == "min":
+                    agg_value = min(self._values)
+                elif self.summary == "max":
+                    agg_value = max(self._values)
+                elif self.summary == "mean":
+                    agg_value = sum(self._values) / len(self._values)
+                final_val = agg_value
+
+        return {
+            "value": final_val,
+            "summary": self.summary
+        }