harmony-client 0.1.0__cp312-cp312-win_amd64.whl

harmony_client/internal/__init__.py

@@ -0,0 +1,7 @@
+from .eval_samples_html import (
+    _extract_model_key as _extract_model_key,
+)
+from .eval_samples_html import (
+    _save_detailed_eval_table as _save_detailed_eval_table,
+)
+from .utils import stringify_thread as stringify_thread

harmony_client/internal/eval_samples_html.py

@@ -0,0 +1,122 @@
+import json
+from pathlib import Path
+
+from harmony_client import EvalSample, StringThread
+from harmony_client.internal.utils import stringify_thread
+from harmony_client.logging_table import Table
+
+
+def _extract_model_key(model_path: str | None = None) -> str:
+    model_path = model_path or ""
+    if model_path.startswith("model_registry://"):
+        return model_path[len("model_registry://") :]
+    return model_path
+
+
+def _save_detailed_eval_table(eval_samples: list[EvalSample], output_dir: str | None = None) -> None:
+    """
+    Method subject to change, only for internal library use.
+    Do not use on its own, use RecipeContext.log_eval_result instead.
+    """
+    default_output_file: str = "evaluation_samples.html"
+    if not eval_samples:
+        print("No evaluation samples to save")
+        return
+
+    # Force provided path into dir
+    if output_dir is not None:
+        output_path = Path(output_dir)
+        if output_path.is_file() or output_path.suffix:
+            eval_dir = output_path.parent
+        else:
+            eval_dir = output_path
+    else:
+        eval_dir = Path.cwd()
+
+    # Create the directory structure
+    eval_dir.mkdir(parents=True, exist_ok=True)
+    html_path = eval_dir / Path(default_output_file).name
+
+    # Collect all unique grader names to determine column structure
+    all_grader_names = set()
+    for sample in eval_samples:
+        for grade in sample.grades:
+            all_grader_names.add(grade.grader_key)
+
+    all_grader_names = sorted(all_grader_names)
+
+    # Create Table
+    headers = ["Prompt", "Model", "Completion"]
+    for grader_name in all_grader_names:
+        headers.extend([f"{grader_name}_score", f"{grader_name}_reason"])
+
+    table = Table(headers)
+
+    # Group samples by prompt (stringified thread)
+    prompt_groups = {}
+    for sample in eval_samples:
+        # Extract just the prompt part (everything except the last assistant turn) for grouping
+        turns = sample.interaction.thread.get_turns()
+        # Find prompt turns (everything except the last assistant turn)
+        prompt_turns: list[tuple[str, str]] = []
+        completion = ""
+        # Get all turns except extract the completion separately
+        for i, (role, content) in enumerate(turns):
+            if role.lower() == "assistant" and i == len(turns) - 1:
+                completion = content
+            else:
+                prompt_turns.append((role, content))
+
+        # Create prompt string for grouping
+        prompt_str = stringify_thread(StringThread(prompt_turns))
+        if prompt_str not in prompt_groups:
+            prompt_groups[prompt_str] = []
+
+        prompt_groups[prompt_str].append(
+            {
+                "model": _extract_model_key(sample.interaction.source or "Unknown"),
+                "completion": completion,
+                "grades": sample.grades,
+            }
+        )
+
+    # Create table rows
+    for prompt_str, models_data in prompt_groups.items():
+        for i, model_data in enumerate(models_data):
+            row = [
+                prompt_str if i == 0 else "",  # Only show prompt for first model in group
+                model_data["model"],
+                model_data["completion"],
+            ]
+
+            # Add grader score and reasoning columns
+            grade_dict = {grade.grader_key: grade for grade in model_data["grades"]}
+
+            for grader_name in all_grader_names:
+                if grader_name in grade_dict:
+                    grade = grade_dict[grader_name]
+                    row.extend([grade.value, grade.reasoning or ""])
+                else:
+                    row.extend(["N/A", "No evaluation"])
+
+            table.add_row(row)
+
+    # Save HTML table
+    with open(html_path, "w") as f:
+        f.write(table.to_html_table())
+
+    # Save summary metadata as JSON
+    metadata = {
+        "total_samples": len(eval_samples),
+        "unique_prompts": len(prompt_groups),
+        "models": sorted(
+            set(model_data["model"] for models_data in prompt_groups.values() for model_data in models_data)
+        ),
+        "graders": all_grader_names,
+    }
+    metadata_path = eval_dir / "metadata.json"
+
+    with open(metadata_path, "w") as f:
+        json.dump(metadata, f, indent=2)
+
+    print(f"📁 Detailed evaluation samples saved to: {html_path}")

harmony_client/internal/utils.py

@@ -0,0 +1,9 @@
+"""Utility functions for internal use."""
+
+from harmony_client import StringThread
+
+
+def stringify_thread(thread: StringThread, sep: str = "\n\n") -> str:
+    """Convert StringThread to readable text format."""
+    turns = thread.get_turns()
+    return sep.join([f"[{turn.role}]\n{turn.content}" for turn in turns])

harmony_client/logging_table.py

@@ -0,0 +1,121 @@
+from typing import Any, Sequence, overload
+
+from typing_extensions import Self
+
+
+class Table:
+    Column = list[str] | list[float]
+    Row = list[str | float]
+
+    def __init__(self, initial_headers: Sequence[str] = []):
+        self.headers: list[str] = list(initial_headers)
+        self.rows: list[Table.Row] = []
+
+    @overload
+    def add_column(self, header: str, column: list[str]) -> Self: ...
+
+    @overload
+    def add_column(self, header: str, column: list[float]) -> Self: ...
+
+    def add_column(self, header: str, column: list[str] | list[float]) -> Self:
+        if self.rows:
+            # if we're not empty we just add
+            for row, new_value in zip(self.rows, column, strict=True):
+                row.append(new_value)  # type: ignore
+        else:
+            for value in column:
+                self.rows.append([value])
+        self.headers.append(header)
+
+        return self
+
+    # Union Any is me giving up because I have too much stuff to do to be stuck on thos
+    def add_row(self, row: Sequence[str | float | Any]) -> Self:
+        assert len(row) == len(self.headers)
+        self.rows.append(row)  # type: ignore
+        return self
+
+    def add_rows(self, rows: list[Sequence[str | float | Any]]) -> Self:
+        for row in rows:
+            self.add_row(row)
+        return self
+
+    def __getitem__(self, key: str) -> Column:
+        column_index = self.headers.index(key)
+        return [row[column_index] for row in self.rows]  # type: ignore
+
+    def __contains__(self, key: str):
+        return key in self.headers
+
+    def export(self) -> tuple[list[str], list[list[str]]]:
+        return self.headers, [[str(inner) for inner in row] for row in self.rows]
+
+    def _repr_html_(self) -> str:
+        return self.to_html_table()
+
+    def to_html_table(self):
+        """
+        Generates a nicely styled HTML table with CSS for better readability,
+        especially for cells containing a lot of text.
+        """
+        # CSS styles for a professional-looking table
+        # This is defined once and applied to the table via a class.
+        css_style = """
+<style>
+    .classy-table {
+        width: 100%;
+        border-collapse: collapse; /* Removes space between borders */
+        font-family: Arial, sans-serif; /* A clean, readable font */
+        font-size: 14px;
+        box-shadow: 0 2px 5px rgba(0,0,0,0.1);
+        margin: 20px 0;
+    }
+    .classy-table th, .classy-table td {
+        padding: 12px 15px; /* Adds space inside cells */
+        border: 1px solid #ddd; /* Light grey borders */
+        text-align: left;
+        vertical-align: top; /* Aligns content to the top */
+        /* This is key for handling long text */
+        word-wrap: break-word;
+        overflow-wrap: break-word;
+        white-space: pre-wrap;
+    }
+    .classy-table th {
+        background-color: #f2f2f2; /* Light grey header background */
+        font-weight: bold;
+        color: #333;
+    }
+    .classy-table tr:nth-child(even) {
+        background-color: #f9f9f9; /* Zebra-striping for even rows */
+    }
+    .classy-table tr:hover {
+        background-color: #f1f1f1; /* Highlight row on hover */
+    }
+</style>"""
+        # Create HTML table with a specific class
+        html_table = "<table class='classy-table'>\n"
+
+        # Add table header
+        html_table += "  <thead>\n"  # Using <thead> for semantic HTML
+        html_table += "    <tr>\n"
+        for col_name in self.headers:
+            # No more inline styles needed here!
+            html_table += f"      <th>{col_name}</th>\n"
+        html_table += "    </tr>\n"
+        html_table += "  </thead>\n"
+
+        # Add table rows
+        html_table += "  <tbody>\n"  # Using <tbody> for semantic HTML
+        for row in self.rows:
+            html_table += "    <tr>\n"
+            for value in row:
+                # No more inline styles needed here either
+                html_table += f"      <td>{value}</td>\n"
+            html_table += "    </tr>\n"
+        html_table += "  </tbody>\n"
+
+        # Close HTML table
+        html_table += "</table>"
+
+        # Return the styles and the table together
+        return css_style + html_table

harmony_client/parameters/__init__.py

@@ -0,0 +1,295 @@
+import json
+from typing import TYPE_CHECKING, Any
+from uuid import UUID
+
+import aiofiles
+from loguru import logger
+from pydantic import BaseModel, model_serializer, model_validator
+
+from harmony_client import StringThread
+from harmony_client.parameters import dataset_kinds, model_kinds
+from harmony_client.runtime.context import RecipeContext
+from harmony_client.runtime.dto.DatasetSampleFormats import SampleMetadata
+
+if TYPE_CHECKING:
+    from adaptive_harmony.graders.base_grader import BaseGrader
+
+
+class Dataset[T: dataset_kinds.DatasetKind](BaseModel):
+    dataset_key: str
+    feedback_key: str | None = None
+    local_file_path: str | None = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_from_json(cls, data):
+        """Handle deserialization from JSON - accepts dict with dataset_key."""
+        # If it's a string, convert it to the expected dict format
+        if isinstance(data, str):
+            return {"dataset_key": data, "feedback_key": None}
+        return data
+
+    async def load(self, ctx: RecipeContext | None = None) -> list[StringThread]:
+        """
+        Load dataset samples from the Harmony service.
+
+        Args:
+            ctx: RecipeContext with client and file_storage
+
+        Returns:
+            list of StringThread objects
+        """
+
+        def with_metadata(thread: StringThread, metadata: SampleMetadata):
+            if metadata.external_data:
+                thread.metadata = {**metadata.model_dump(exclude_none=True), **metadata.external_data}
+            else:
+                thread.metadata = metadata.model_dump()
+
+        async def parse_internal_format(line_dict: Any) -> StringThread | None:
+            thread = None
+
+            format: type[dataset_kinds.DtoBaseModel] | None = None
+            match T:
+                case dataset_kinds.Prompt:
+                    format = dataset_kinds.DatasetPromptSample
+                case dataset_kinds.Completion:
+                    format = dataset_kinds.DatasetSample
+                case dataset_kinds.Metric:
+                    format = dataset_kinds.DatasetMetricSample
+                case dataset_kinds.Preference:
+                    format = dataset_kinds.DatasetPreferenceSample
+                case _:  # Mixed dataset case
+                    # order is important here: try the most constrained formats first
+                    formats: list[type[dataset_kinds.DtoBaseModel]] = [
+                        dataset_kinds.DatasetPreferenceSample,
+                        dataset_kinds.DatasetMetricSample,
+                        dataset_kinds.DatasetSample,
+                        dataset_kinds.DatasetPromptSample,
+                    ]
+                    for f in formats:
+                        try:
+                            f.model_validate(line_dict)
+                            format = f
+                            break
+                        except Exception:
+                            pass
+
+            if format is None:
+                raise ValueError(f"Could not determine format for line in dataset. Line {line_dict}")
+
+            match format:
+                case dataset_kinds.DatasetPromptSample:
+                    sample = format.model_validate(line_dict)
+                    thread = await StringThread.from_dataset(
+                        [(turn.root[0], turn.root[1]) for turn in sample.prompt], None
+                    )
+                    with_metadata(thread, sample.metadata)
+                case dataset_kinds.DatasetSample:
+                    sample = format.model_validate(line_dict)
+                    thread = await StringThread.from_dataset(
+                        [(turn.root[0], turn.root[1]) for turn in sample.prompt], None
+                    )
+                    thread = thread.assistant(sample.completion.root[1])
+                    with_metadata(thread, sample.metadata)
+                case dataset_kinds.DatasetMetricSample:
+                    sample = dataset_kinds.DatasetMetricSample.model_validate(line_dict)
+                    thread = await StringThread.from_dataset(
+                        [(turn.root[0], turn.root[1]) for turn in sample.prompt], None
+                    )
+                    thread = thread.assistant(sample.completion.root[1])
+                    with_metadata(thread, sample.metadata)
+                    if self.feedback_key and sample.metrics:
+                        # put metric value in "res" key in the metadata
+                        thread.metadata["res"] = sample.metrics.get(self.feedback_key)
+
+                case dataset_kinds.DatasetPreferenceSample:
+                    sample = dataset_kinds.DatasetPreferenceSample.model_validate(line_dict)
+                    thread = await StringThread.from_dataset(
+                        [(turn.root[0], turn.root[1]) for turn in sample.prompt], None
+                    )
+                    with_metadata(thread, sample.metadata)
+                    thread.metadata["other_completion"] = sample.bad_completion.root[1]
+                    thread.metadata["preferred_completion"] = sample.good_completion.root[1]
+
+            return thread
+
+        async def parse_external_format(line_dict: Any) -> StringThread | None:
+            thread = None
+            if "input" in line_dict or "messages" in line_dict:
+                key = "input" if "input" in line_dict else "messages"
+                thread = StringThread(
+                    [(inner_turn_dict["role"], inner_turn_dict["content"]) for inner_turn_dict in line_dict[key]]
+                )
+                if "completion" in line_dict and line_dict["completion"]:
+                    thread = thread.assistant(line_dict["completion"])
+            else:
+                print("Did not find `input`, or `messages` key in sample, ignoring")
+            if thread is not None:
+                thread.metadata = line_dict.get("metadata", {})
+                if "other_completion" in line_dict and "preferred_completion" in line_dict:
+                    thread.metadata["other_completion"] = line_dict["other_completion"]
+                    thread.metadata["preferred_completion"] = line_dict["preferred_completion"]
+            return thread
+
+        if ctx:
+            config_response = await ctx.client.get_dataset_config(self.dataset_key)
+            lines = ctx.file_storage.read(config_response.file_path, use_raw_path=True).decode("utf-8").splitlines()
+        else:
+            assert self.local_file_path is not None, "Local file path is required when ctx is not provided"
+            lines = []
+            async with aiofiles.open(self.local_file_path, encoding="utf-8") as f:
+                async for line in f:
+                    lines.append(line.rstrip("\n"))
+
+        threads = []
+        parse_function = None
+        for line in lines:
+            if len(line.strip()) == 0:
+                continue
+            line_dict = json.loads(line)
+
+            if parse_function is None:
+                try:
+                    thread = await parse_internal_format(line_dict)
+                    parse_function = parse_internal_format
+                except Exception as e:
+                    logger.warning("Could not read dataset as internal format, falling back to external format {}", e)
+                    thread = await parse_external_format(line_dict)
+                    parse_function = parse_external_format
+            else:
+                thread = await parse_function(line_dict)
+
+            if thread is not None:
+                threads.append(thread)
+
+        if len(threads) == 0:
+            raise ValueError("Did not find any valid format samples in the dataset")
+        return threads
+
+    def __hash__(self):
+        """Make Dataset hashable based on its keys."""
+        return hash((self.dataset_key, self.feedback_key))
+
+
+class Model[T: model_kinds.ModelKind](BaseModel):
+    model_key: str
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_from_json(cls, data):
+        """Handle deserialization - accepts string or dict."""
+        if isinstance(data, str):
+            return {"model_key": data}
+        return data
+
+    @model_serializer
+    def serialize_model(self) -> str:
+        """Serialize as just the model_key string."""
+        return self.model_key
+
+    async def to_builder(
+        self,
+        ctx: RecipeContext,
+        kv_cache_len: int | None = None,
+        tokens_to_generate: int | None = None,
+        tp: int | None = None,
+    ):
+        """
+        Create a ModelBuilder instance configured with this model's parameters.
+
+        Args:
+            ctx: RecipeContext with client
+            kv_cache_len (int | None, optional): KV cache length override
+            tokens_to_generate (int | None, optional): Tokens to generate override
+            tp (int | None, optional): Tensor parallelism override
+
+        Returns:
+            ModelBuilder: A configured model builder instance ready for use
+        """
+        config_response = await ctx.client.get_model_config(self.model_key)
+
+        kwargs = {
+            k: v
+            for k, v in {
+                "kv_cache_len": kv_cache_len or config_response.kv_cache_len,
+                "tokens_to_generate": tokens_to_generate,
+            }.items()
+            if v is not None
+        }
+        builder = ctx.client.model(config_response.path, **kwargs)
+
+        if tp_to_pass := tp or config_response.tp:
+            builder = builder.tp(tp_to_pass)
+
+        return builder
+
+    def __hash__(self):
+        """Make Model hashable based on its key."""
+        return hash(self.model_key)
+
+
+class Grader(BaseModel):
+    grader_key: str
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_from_json(cls, data):
+        """Handle deserialization - accepts string or dict."""
+        if isinstance(data, str):
+            return {"grader_key": data}
+        return data
+
+    @model_serializer
+    def serialize_model(self) -> str:
+        """Serialize as just the grader_key string."""
+        return self.grader_key
+
+    async def load(
+        self,
+        ctx: RecipeContext,
+        tp: int | None = None,
+        kv_cache_len: int | None = None,
+        max_tokens: int | None = None,
+    ) -> "BaseGrader":
+        """
+        Load a grader instance configured with this grader's parameters.
+
+        Args:
+            ctx: RecipeContext with client
+            tp (int | None, optional): Tensor parallelism override
+            kv_cache_len (int | None, optional): KV cache length override
+
+        Returns:
+            BaseGrader: A configured grader instance ready for use
+        """
+        # puke: need to figure out smth better here.
+        # I don't like code duplication but maybe we should duplicate these guys?
+        # I'm not sure tbh...
+        try:
+            from adaptive_harmony.graders import BaseGrader as GraderImpl
+        except ImportError:
+            raise ImportError(
+                "To load and instantiate a grader, the adaptive_harmony package is required. Install it with `pip install adaptive-harmony`"
+            )
+
+        from harmony_client.runtime.data import AdaptiveGrader
+
+        config_response = await ctx.client.get_grader_config(self.grader_key)
+        grader_config_data = json.loads(config_response.grader_config_json)
+
+        grader_config = AdaptiveGrader(
+            grader_id=UUID(config_response.grader_id),
+            key=config_response.key,
+            metric_id=UUID("00000000-0000-0000-0000-000000000000"),  # unused in from_config
+            name=config_response.name,
+            config=grader_config_data,
+        )
+
+        return await GraderImpl.from_config(
+            grader_config=grader_config, ctx=ctx, tp=tp, kv_cache_len=kv_cache_len, max_tokens=max_tokens
+        )
+
+    def __hash__(self):
+        """Make Grader hashable based on its key."""
+        return hash(self.grader_key)

harmony_client/parameters/dataset_kinds.py

@@ -0,0 +1,49 @@
+from harmony_client.runtime.dto.DatasetSampleFormats import (
+    DatasetMetricSample,
+    DatasetPreferenceSample,
+    DatasetPromptSample,
+    DatasetSample,
+    DtoBaseModel,
+)
+
+
+class DatasetKind:
+    def __init__(
+        self,
+        kind: str,
+        formats: list[type[DtoBaseModel]] = [
+            DatasetPreferenceSample,
+            DatasetMetricSample,
+            DatasetSample,
+            DatasetPromptSample,
+        ],
+    ):
+        self.kind = kind
+        self.formats = formats
+
+    def parse(self, line: dict):
+        for f in self.formats:
+            try:
+                return f.model_validate(line)
+            except Exception:
+                pass
+
+
+class Prompt(DatasetKind):
+    def __init__(self):
+        super().__init__("prompts", [DatasetPromptSample])
+
+
+class Preference(DatasetKind):
+    def __init__(self):
+        super().__init__("preference", [DatasetPreferenceSample])
+
+
+class Completion(DatasetKind):
+    def __init__(self):
+        super().__init__("completions", [DatasetSample])
+
+
+class Metric(DatasetKind):
+    def __init__(self):
+        super().__init__("feedbacks", [DatasetMetricSample])

harmony_client/parameters/model_kinds.py

@@ -0,0 +1,13 @@
+class ModelKind:
+    def __init__(self, kind: str):
+        self.kind = kind
+
+
+class Inferable(ModelKind):
+    def __init__(self):
+        super().__init__("all")
+
+
+class Trainable(ModelKind):
+    def __init__(self):
+        super().__init__("trainable")

harmony_client/runtime/__init__.py

@@ -0,0 +1,29 @@
+from .context import RecipeContext
+from .data import (
+    AdaptiveDatasetKind,
+    InputConfig,
+)
+from .decorators import recipe_main
+from .dto.DatasetSampleFormats import (
+    DatasetMetricSample,
+    DatasetPreferenceSample,
+    DatasetPromptSample,
+    DatasetSample,
+    SampleMetadata,
+    TurnTuple,
+)
+from .simple_notifier import SimpleProgressNotifier
+
+__all__ = [
+    "RecipeContext",
+    "AdaptiveDatasetKind",
+    "InputConfig",
+    "recipe_main",
+    "DatasetMetricSample",
+    "DatasetPreferenceSample",
+    "DatasetPromptSample",
+    "DatasetSample",
+    "SampleMetadata",
+    "TurnTuple",
+    "SimpleProgressNotifier",
+]