adaptive-harmony 0.1.23__py3-none-any.whl

adaptive_harmony/graders/combined_grader.py

@@ -0,0 +1,118 @@
+from asyncio import gather
+from typing import Any, Literal, Sequence
+
+from loguru import logger
+
+from adaptive_harmony import StringThread
+from adaptive_harmony.core.structured_output import JsonParseError
+from adaptive_harmony.graders import BaseGrader, Grade
+from adaptive_harmony.graders.exceptions import IgnoreScoreException
+
+
+class CombinedGrader(BaseGrader):
+    """
+    Combines grades from multiple graders.
+    Aggregates their results using weighted sum or average.
+    Ignores failing graders and proceeds calculating the aggregate grade with the rest.
+    """
+
+    def __init__(
+        self,
+        grader_key: str,
+        graders: Sequence[BaseGrader],
+        weights: list[float] | None = None,
+        aggregation_method: Literal["sum", "mean"] = "mean",
+        failure_rate_warning_threshold: float = 0.2,
+    ):
+        super().__init__(grader_key)
+        self.graders = graders
+        if weights:
+            assert len(weights) == len(graders), "Number of weights must match number of graders"
+        self.weights = weights or [1.0] * len(graders)
+        self.agg_method = aggregation_method
+        self.failure_rate_warning_threshold = failure_rate_warning_threshold
+
+    async def grade(self, sample: StringThread) -> Grade:
+        async def separate_success_from_fail_graders(grader: BaseGrader) -> Grade | None:
+            try:
+                return await grader.grade(sample)
+            except (IgnoreScoreException, JsonParseError):
+                # return None if score is supposed to be ignored, or judge output format failure
+                return None
+            except Exception as e:
+                # fail for any other exception
+                raise e
+
+        tasks = [separate_success_from_fail_graders(grader) for grader in self.graders]
+        results: list[Grade | None] = await gather(*tasks)
+
+        weighted_scores = []
+        failed_graders = []
+
+        # Separate successful and failed results
+        successful_graders: list[str] = []
+        successful_grades: list[Grade] = []
+        successful_weights = []
+
+        for result, weight, grader in zip(results, self.weights, self.graders):
+            if result is not None:
+                # Successful grader
+                weighted_score = result.value * weight
+                successful_graders.append(grader.grader_key)
+                weighted_scores.append(weighted_score)
+                successful_grades.append(result)
+                successful_weights.append(weight)
+            else:
+                # Failed grader
+                failed_graders.append(grader.grader_key)
+
+        # Fail if no successfull graders
+        if not successful_grades:
+            raise RuntimeError("All graders failed - cannot compute aggregate grade")
+
+        # Warn if more than a set % of scorers failed
+        total_graders = len(self.graders)
+        failure_rate = len(failed_graders) / total_graders
+        if failure_rate > self.failure_rate_warning_threshold:
+            logger.warning(f"{len(failed_graders)}/{total_graders}% of graders failed for sample: {failed_graders}")
+
+        # Aggregate scores
+        if self.agg_method == "sum":
+            final_score = sum(weighted_scores)
+        elif self.agg_method == "mean":
+            # For average, we normalize by the sum of successful weights (renormalize)
+            final_score = sum(weighted_scores) / sum(successful_weights)
+        else:
+            raise ValueError(f"Unknown aggregation method: {self.agg_method}")
+
+        # Log the combined score.
+        self.add_log({"score": final_score})
+
+        reason = "\n".join(
+            [
+                f"{grader_key}: {grade.value} - {grade.reasoning}"
+                for grader_key, grade in zip(successful_graders, successful_grades)
+            ]
+        )
+
+        return Grade(
+            value=final_score,
+            grader_key=self.grader_key,
+            reasoning=reason,
+        )
+
+    def get_logs(self, clear: bool = False, log_all_samples: bool = False) -> dict[str, Any]:
+        combined_logs = super().get_logs(clear=False, log_all_samples=log_all_samples)
+        all_logs = combined_logs
+        for grader in self.graders:
+            scorer_logs = grader.get_logs(clear=clear, log_all_samples=log_all_samples)
+            all_logs = all_logs | {f"{grader.grader_key}/{key}": value for key, value in scorer_logs.items()}
+
+        if clear:
+            self.clear_logs()
+        return all_logs
+
+    def clear_logs(self) -> None:
+        super().clear_logs()
+        for grader in self.graders:
+            grader.clear_logs()

adaptive_harmony/graders/context_relevancy_judge/__init__.py

@@ -0,0 +1,3 @@
+from .context_relevancy_judge import ContextRelevancyGrader
+
+__all__ = ["ContextRelevancyGrader"]

adaptive_harmony/graders/context_relevancy_judge/context_relevancy_judge.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Literal
+
+import numpy as np
+import pysbd
+from harmony_client import Grade, InferenceModel, StringThread
+from pydantic import BaseModel, Field
+
+from adaptive_harmony.core.utils import stringify_thread
+from adaptive_harmony.graders.base_grader import BaseGrader
+from adaptive_harmony.graders.context_relevancy_judge.prompts import DEFAULT_SHOTS, SYSTEM, USER
+from adaptive_harmony.graders.faithfulness_judge.faithfulness_judge import SupportedLanguages
+from adaptive_harmony.graders.utils import sample_score_distribution
+from adaptive_harmony.logging_table import Table
+
+
+class DocumentRelevancyResult(BaseModel):
+    reason: str = Field(description="The justification for the score given to a document. Keep it short and concise.")
+    score: Literal[0, 1] = Field(
+        description="The score for the document. A score of 1 if the document contains information relevant to answering the user input, and 0 if the document does not contain information relevant to answering the user input"
+    )
+
+
+class ContextRelevancyGrader(BaseGrader):
+    def __init__(
+        self,
+        model: InferenceModel,
+        language: SupportedLanguages = "en",
+        grader_key: str = "context_relevancy_judge",
+        grader_id: str | None = None,
+    ):
+        super().__init__(grader_key)
+        self.model = model
+        self.language = language
+        self.grader_id_or_key = grader_id or grader_key
+        self.sentence_splitter = pysbd.Segmenter(language=language)
+        self.shots = DEFAULT_SHOTS
+
+    async def grade(self, sample: StringThread) -> Grade:
+        documents = sample.metadata.get("documents", []) if sample.metadata else []
+        if not documents:
+            self.add_log(
+                {
+                    "prompt": stringify_thread(sample, sep=f"\n\n{'-' * 10}\n\n"),
+                    "error": "No document turns found in thread",
+                }
+            )
+            raise ValueError("No document turns found in thread")
+
+        user_question = next((turn[1] for turn in reversed(sample.get_turns()) if turn[0] == "user"), None)
+        if not user_question:
+            self.add_log(
+                {"prompt": stringify_thread(sample, sep=f"\n\n{'-' * 10}\n\n"), "error": "No user turn found in thread"}
+            )
+            raise ValueError("No user turn found in thread")
+
+        judging_threads = [
+            (
+                StringThread()
+                .system(SYSTEM.format(json_schema=self.model.render_schema(DocumentRelevancyResult), shots=self.shots))
+                .user(USER.format(user_question=user_question, document=document))
+            )
+            for document in documents
+        ]
+
+        try:
+            judge_tasks = [
+                self.model.temperature(0.0).generate_and_validate(thread, DocumentRelevancyResult)
+                for thread in judging_threads
+            ]
+            results = await asyncio.gather(*judge_tasks)
+        except Exception as e:
+            self.add_log(
+                {
+                    "error": str(e),
+                    "number_of_documents": len(documents),
+                    "documents": documents,
+                    "prompt": stringify_thread(judging_threads[0]),
+                }
+            )
+            raise
+
+        doc_relevancy_results = [result[1] for result in results]
+
+        reason = ""
+        for i, (document, doc_result) in enumerate(zip(documents, doc_relevancy_results)):
+            emoji = "✅" if doc_result.score == 1 else "❌"
+            result = "PASS" if doc_result.score == 1 else "FAIL"
+            doc_display = document[:150] + ("..." if len(document) > 150 else "")
+            reason += f"{emoji} Document {i}: {result}\n Content: {doc_display}:\nReason: {doc_result.reason}\n\n"
+
+        score = np.mean([float(verdict.score) for verdict in doc_relevancy_results]) if doc_relevancy_results else 0.0
+        self.add_log(
+            {
+                "score": score,
+                "reasoning": reason,
+                "number_of_documents": len(documents),
+                "documents": documents,
+                "prompt": stringify_thread(judging_threads[0]),
+            }
+        )
+        return Grade(value=float(score), grader_key=self.grader_id_or_key, reasoning=reason)
+
+    def get_logs(self, clear: bool = False, log_all_samples: bool = False) -> dict[str, float | Table]:
+        # Only clear logs at the end if clear is True
+        logs = super().get_logs(clear=False)
+
+        successfully_scored_samples = [log for log in self._logs if "score" in log]
+
+        # stratified sample range of scores to see high and low
+        if not log_all_samples:
+            subset_successfully_scored_samples = sample_score_distribution(successfully_scored_samples, 15)
+        else:
+            # if we have fewer than 15 samples or we want to log all samples, take them all
+            subset_successfully_scored_samples = successfully_scored_samples
+
+        failed_scored_samples = [log for log in self._logs if "error" in log]
+
+        sample_logs = self.get_sample_tables(subset_successfully_scored_samples, failed_scored_samples)
+
+        logs.update(sample_logs)
+
+        if clear:
+            self.clear_logs()
+
+        return logs

adaptive_harmony/graders/context_relevancy_judge/prompts.py

@@ -0,0 +1,84 @@
+DEFAULT_SCORING_SHOTS = """Example:
+INPUT
+What percentage is considered a good rental yield?
+
+STATEMENTS
+0: How are you doing today?
+1: Rental yield is how much you could expect to receive in rent each year from your buy to let investment.
+2: Rental yield is expressed as a percentage - reflecting your rental income against the property's market value.
+3: Anything around the 5-6% mark could be considered a good rental yield.
+4: Anything above 6% could be considered a very good rental yield.
+
+```json
+{
+    "verdicts": [
+        {
+            "reason": "The statement is unrelated to the input.",
+            "score": 0
+        },
+        {
+            "reason": "While the statement discusses rental yields, it does not indicate what constitutes a good rental yield.",
+            "score": 0
+        },
+        {
+            "reason": "While the statement mentions that yield is expressed as a percentage, it does not address the user question.",
+            "score": 0
+        },
+        {
+            "reason": "The statement addresses the user input, specifying what a good rental yield is.",
+            "score": 1
+        },
+        {
+            "reason": "The statement addresses the user input, specifying what a very good rental yield is.",
+            "score": 1
+        },
+    ]
+}```"""
+
+
+SYSTEM = """You are an expert data reviewer.
+You will be given a document, and a user input/question.
+Your task is to classify whether the document contains information relevant to answering the user input.
+
+IMPORTANT: Please make sure to only return in JSON format and with no further preamble explanation, with the `score` key providing the score, and the `reason` key providing the reason. The score can only be 0 or 1. Keep the reason short and concise.
+
+You always output a JSON object with the following schema, and nothing else before or after:
+{json_schema}
+
+Examples:
+{shots}"""
+
+USER = """Your real task:
+INPUT
+{user_question}
+
+DOCUMENT
+{document}
+
+```json"""
+
+DEFAULT_SHOTS = """Example 1:
+INPUT
+What percentage is considered a good rental yield?
+
+DOCUMENT
+Rental yield is how much you could expect to receive in rent each year from your buy to let investment. Rental yield is expressed as a percentage - reflecting your rental income against the property's market value.
+While calculating rental yield can give you an indication of whether investing in a buy-to-let property is worth it, there's other factors to consider. You'll also need to think about whether there might be any problem finding tenants, collecting rent or void periods, for example. Then, there's capital growth - the value by which your property is set to increase over time.
+All of these can impact your decision on whether a property is worth your investment.
+
+```json
+{"score":0,"reason":"While the document explains the concept of rental yield, there is no indication as to what a good percentage yield is."}
+```
+
+Example 2:
+INPUT
+What percentage is considered a good rental yield?
+
+DOCUMENT
+As of 2024, the average rental yield in the UK is between 5% and 8%. Anything around the 5-6% mark could be considered a 'good' rental yield, while anything above 6% could be considered very good.
+Some parts of the country can deliver significantly higher or lower returns to others. It's worth bearing in mind that you may get a lower yield in areas where the house prices are highest, such as in London and the South East.
+This is because the potential for capital gains in the region pushes sale prices up, while rent levels are less affected.
+
+```json
+{"score":1,"reason":"The document indicates what can be considered a good rental yield percentage."}
+```"""

adaptive_harmony/graders/exceptions.py

@@ -0,0 +1,9 @@
+class IgnoreScoreException(Exception):
+    """
+    Exception to indicate that a score should be ignored/is not applicable.
+    Accepts a custom message explaining why the score was ignored.
+    """
+
+    def __init__(self, message: str = "Score should be ignored"):
+        super().__init__(message)
+        self.message = message

adaptive_harmony/graders/faithfulness_judge/__init__.py

@@ -0,0 +1,3 @@
+from .faithfulness_judge import FaithfulnessGrader
+
+__all__ = ["FaithfulnessGrader"]

adaptive_harmony/graders/faithfulness_judge/faithfulness_judge.py

@@ -0,0 +1,159 @@
+from typing import Literal
+
+import pysbd
+from pydantic import BaseModel, Field
+
+from adaptive_harmony import Grade, InferenceModel, StringThread
+from adaptive_harmony.core.structured_output import JsonParseError
+from adaptive_harmony.core.utils import stringify_thread
+from adaptive_harmony.graders.base_grader import BaseGrader
+from adaptive_harmony.graders.exceptions import IgnoreScoreException
+from adaptive_harmony.graders.faithfulness_judge.prompts import SYSTEM, USER
+from adaptive_harmony.graders.utils import (
+    FailedJudgeLog,
+    SuccessJudgeLog,
+    sample_score_distribution,
+    separate_context_from_last_user_turn,
+    validate_thread_last_assistant,
+)
+from adaptive_harmony.logging_table import Table
+
+
+class SingleStatementFaithfulnessJudgeOutput(BaseModel):
+    statement_idx: int = Field(description="The original index of the sentence being scored")
+    reasoning: str = Field(description="Reasoning to support the rationale behind the score")
+    score: Literal["1", "0"] = Field(
+        description="The score of the sample, 1 if the statement is fully supported by the context, 0 if it is not"
+    )
+
+
+class FaithfulnessGraderOutput(BaseModel):
+    all_statements_scoring: list[SingleStatementFaithfulnessJudgeOutput] = Field(
+        description="An array of objects, each analyzing a single statement from the original list of statements"
+    )
+
+
+SupportedLanguages = Literal[
+    "en",
+    "hi",
+    "mr",
+    "zh",
+    "es",
+    "am",
+    "ar",
+    "hy",
+    "bg",
+    "ur",
+    "ru",
+    "pl",
+    "fa",
+    "nl",
+    "da",
+    "fr",
+    "my",
+    "el",
+    "it",
+    "ja",
+    "de",
+    "kk",
+    "sk",
+]
+
+
+class FaithfulnessGrader(BaseGrader):
+    """
+    Scores each sentence in the last assistant turn as fully supported by the context or not (1 or 0).
+    The context is the rest of the thread, excluding the system prompt.
+    The final score is the average of each sentence.
+    Requires an input language code to split the sentences.
+    """
+
+    def __init__(
+        self,
+        model: InferenceModel,
+        language: SupportedLanguages = "en",
+        grader_key: str = "faithfulness_judge",
+        grader_id: str | None = None,
+    ):
+        super().__init__(grader_key)
+        self._logs: list[SuccessJudgeLog | FailedJudgeLog] = []  # already created in super, this is for typing
+        self.model = model
+        self.language = language
+        self.sentence_splitter = pysbd.Segmenter(language=language)
+        self.grader_id_or_key = grader_id or grader_key
+
+    async def grade(self, sample: StringThread) -> Grade:
+        # Split response into sentences
+        validate_thread_last_assistant(sample)
+        # Separate conversation context from last user turn
+        context_turns, user_question = separate_context_from_last_user_turn(sample)
+        completion = sample.last_content()
+        split_sentences = self.sentence_splitter.segment(completion)
+        sentences = [f"{i}: {sentence.strip()}" for i, sentence in enumerate(split_sentences) if sentence.strip()]
+        sentences_judge_str = "\n".join(sentences)
+
+        # Build prompt
+        context_str = stringify_thread(StringThread(context_turns))
+        judge_thread = (
+            StringThread()
+            .system(SYSTEM.format(json_schema=self.model.render_schema(FaithfulnessGraderOutput)))
+            .user(USER.format(context=context_str, user_question=user_question, sentences=sentences_judge_str))
+        )
+        judge_str_prompt = stringify_thread(judge_thread, sep=f"\n\n{'-' * 10}\n\n")
+        # Generate response
+        try:
+            _, parsed_response = await self.model.temperature(0.0).generate_and_validate(
+                judge_thread, FaithfulnessGraderOutput
+            )
+        except JsonParseError as e:
+            self.add_log({"prompt": judge_str_prompt, "error": f"{str(e)}\n\nCOMPLETION:\n{e.completion}"})
+            raise
+        except Exception as e:
+            self.add_log({"prompt": judge_str_prompt, "error": str(e)})
+            raise
+
+        # Raise error if judge failed to judge any sentence
+        n_judged_sentences = len(parsed_response.all_statements_scoring)
+        if n_judged_sentences != len(sentences):
+            raise IgnoreScoreException(
+                f"Number of sentences in the response ({n_judged_sentences})"
+                f"does not match the number of sentences in the input ({len(sentences)})"
+            )
+        # Calculate avg score
+        score = round(
+            sum([float(judgement.score) for judgement in parsed_response.all_statements_scoring]) / n_judged_sentences,
+            3,
+        )
+        merged_reasoning_traces = "\n-".join(
+            [judgement.reasoning for judgement in parsed_response.all_statements_scoring]
+        )
+        self.add_log({"score": score, "prompt": judge_str_prompt, "reasoning": merged_reasoning_traces})
+
+        return Grade(value=score, grader_key=self.grader_id_or_key, reasoning=merged_reasoning_traces)
+
+    def add_log(self, log: SuccessJudgeLog | FailedJudgeLog) -> None:
+        self._logs.append(log)
+
+    def get_logs(self, clear: bool = False, log_all_samples: bool = False) -> dict[str, float | Table]:
+        # Only clear logs at the end if clear is True
+        logs = super().get_logs(clear=False)
+
+        successfully_scored_samples = [log for log in self._logs if "score" in log]
+
+        # stratified sample range of scores to see high and low
+        if not log_all_samples:
+            subset_successfully_scored_samples = sample_score_distribution(successfully_scored_samples, 15)
+        else:
+            # if we have fewer than 15 samples or we want to log all samples, take them all
+            subset_successfully_scored_samples = successfully_scored_samples
+
+        failed_scored_samples = [log for log in self._logs if "error" in log]
+
+        sample_logs = self.get_sample_tables(subset_successfully_scored_samples, failed_scored_samples)
+
+        logs.update(sample_logs)
+
+        if clear:
+            self.clear_logs()
+
+        return logs

adaptive_harmony/graders/faithfulness_judge/prompts.py

@@ -0,0 +1,22 @@
+SYSTEM = """Your task is to judge the faithfulness of a series of statements to the provided context.
+For each statement_idx you must return your reasoning to support a score you attribute to the corresponding statement.
+The score should be 1 in case the statement is fully supported and can be directly inferred based on the context, or 0 in case it cannot.
+If there is no relevant context to be faithful to, the score should be 1.
+You must score every single sentence without skipping any.
+
+If it exists, you will be given the whole CONVERSATION so far leading up to the LAST USER TURN.
+You must evaluate the statements with a focus on what is being asked in the LAST USER TURN, and never on an intermediary questions that might have been asked in course of the conversation.
+
+You always output a JSON object with the following schema, and nothing else before or after:
+{json_schema}"""
+
+
+USER = """CONVERSATION
+{context}
+
+LAST USER TURN
+{user_question}
+
+STATEMENTS
+{sentences}
+"""

adaptive_harmony/graders/range_judge/__init__.py

@@ -0,0 +1,7 @@
+from .prompts import (
+    RangeJudgeShot as RangeJudgeShot,
+)
+from .prompts import (
+    SubrangeExpectations as SubrangeExpectations,
+)
+from .range_judge import RangeJudgeGrader as RangeJudgeGrader