arize-phoenix 0.0.32__py3-none-any.whl → 0.0.33__py3-none-any.whl

phoenix/experimental/evals/functions/binary.py

@@ -0,0 +1,156 @@
+import logging
+from typing import List, Optional, Set, Union
+
+import pandas as pd
+
+from ..models import BaseEvalModel
+from ..models.openai import OpenAiModel
+from ..templates import (
+    RAG_RELEVANCY_PROMPT_TEMPLATE_STR,
+    PromptTemplate,
+    normalize_template,
+)
+from .common import map_template
+
+logger = logging.getLogger(__name__)
+
+
+def llm_eval_binary(
+    dataframe: pd.DataFrame,
+    template: Union[PromptTemplate, str],
+    model: BaseEvalModel,
+    rails: List[str],
+    system_instruction: Optional[str] = None,
+) -> List[Optional[str]]:
+    """Runs binary classifications using an LLM.
+
+    Args:
+        dataframe (pandas.DataFrame): A pandas dataframe in which each row represents a record to be
+        classified. All template variable names must appear as column names in the dataframe (extra
+        columns unrelated to the template are permitted).
+
+        template (Union[PromptTemplate, str]): The prompt template as either an instance of
+        PromptTemplate or a string. If the latter, the variable names should be surrounded by
+        curly braces so that a call to `.format` can be made to substitute variable values.
+
+        model (BaseEvalModel): An LLM model class.
+
+        rails (List[str]): A list of strings representing the possible output classes of the model's
+        predictions.
+
+        system_instruction (Optional[str], optional): An optional system message.
+
+    Returns:
+        List[Optional[str]]: A list of strings representing the predicted class for each record in
+        the dataframe. The list should have the same length as the input dataframe and its values
+        should be the entries in the `rails` argument or None if the model's prediction could not be
+        parsed.
+    """
+
+    eval_template = normalize_template(template)
+    prompts = map_template(dataframe, eval_template)
+    responses = model.generate(prompts.to_list(), system_instruction)
+    rails_set = set(rails)
+    return [_snap_to_rail(response, rails_set) for response in responses]
+
+
+def run_relevance_eval(
+    dataframe: pd.DataFrame,
+    query_column_name: str = "attributes.input.value",
+    retrieved_documents_column_name: str = "attributes.retrieval.documents",
+    template: str = RAG_RELEVANCY_PROMPT_TEMPLATE_STR,
+    model: Optional[BaseEvalModel] = None,
+) -> List[List[Optional[bool]]]:
+    """Given a pandas dataframe containing queries and retrieved documents,
+       classifies the relevance of each retrieved document to the corresponding
+       query using an LLM.
+
+    Args:
+        dataframe (pd.DataFrame): A pandas dataframe containing queries and
+        retrieved documents.
+
+        query_column_name (str, optional): The name of the column containing the
+        queries.
+
+        retrieved_documents_column_name (str, optional): The name of the column
+        containing the retrieved document data. Each entry in this column should be
+        a list of dictionaries containing metadata about the retrieved documents.
+
+    Returns:
+        List[List[str]]: A list of relevant and not relevant classifications.
+        The "shape" of the list should mirror the "shape" of the retrieved
+        documents column, in the sense that it has the same length as the input
+        dataframe and each sub-list has the same length as the corresponding
+        list in the retrieved documents column. The values in the sub-lists are
+        either booleans or None in the case where the LLM output could not be
+        parsed.
+    """
+
+    llm_relevance_column_name = "llm_relevance"
+    retrieved_document_text_column_name = "retrieved_document_text"
+
+    non_null_query_mask = dataframe[query_column_name].notnull()
+    non_empty_retrievals_mask = dataframe[retrieved_documents_column_name].apply(
+        lambda x: x is not None and len(x) > 0
+    )
+    filtered_mask = non_null_query_mask & non_empty_retrievals_mask
+    filtered_df = dataframe[filtered_mask][[query_column_name]].copy()
+    filtered_df[retrieved_documents_column_name] = dataframe[filtered_mask][
+        retrieved_documents_column_name
+    ].map(list)
+
+    exploded_df = filtered_df.explode(retrieved_documents_column_name, ignore_index=False)
+    exploded_df[retrieved_document_text_column_name] = [
+        document_data["document.content"] if document_data is not None else None
+        for document_data in exploded_df[retrieved_documents_column_name]
+    ]
+    exploded_df = exploded_df.rename(
+        columns={
+            query_column_name: "query",
+            retrieved_document_text_column_name: "reference",
+        }
+    )
+    class_name_to_bool = {"relevant": True, "irrelevant": False}
+    exploded_df[llm_relevance_column_name] = [
+        class_name_to_bool.get(relevance_class) if relevance_class is not None else None
+        for relevance_class in llm_eval_binary(
+            exploded_df,
+            template=PromptTemplate(RAG_RELEVANCY_PROMPT_TEMPLATE_STR),
+            model=model or OpenAiModel(),
+            rails=list(class_name_to_bool.keys()),
+        )
+    ]
+    collapsed_df = exploded_df.groupby(exploded_df.index, axis="index").agg(
+        {
+            llm_relevance_column_name: list,
+        }
+    )
+    output_df = pd.DataFrame(index=dataframe.index)
+    output_df[llm_relevance_column_name] = None
+    output_df.loc[collapsed_df.index, llm_relevance_column_name] = collapsed_df[
+        llm_relevance_column_name
+    ]
+    return output_df[llm_relevance_column_name].tolist()
+
+
+def _snap_to_rail(string: str, rails: Set[str]) -> Optional[str]:
+    """Snaps a string to the nearest rail, or returns None if the string cannot be snapped to a
+    rail.
+
+    Args:
+        string (str): An input to be snapped to a rail.
+
+        rails (Set[str]): The target set of strings to snap to.
+
+    Returns:
+        str: A string from the rails argument or None if the input string could not be snapped.
+    """
+
+    processed_string = string.strip()
+    if processed_string not in rails:
+        logger.warning(
+            f"LLM output cannot be snapped to rails {list(rails)}, returning None. "
+            f'Output: "{string}"'
+        )
+        return None
+    return processed_string

phoenix/experimental/evals/functions/common.py

@@ -0,0 +1,31 @@
+import pandas as pd
+
+from ..templates import PromptTemplate
+
+
+def map_template(dataframe: pd.DataFrame, template: PromptTemplate) -> "pd.Series[str]":
+    """
+    Maps over a dataframe to construct a list of prompts from a template and a dataframe.
+    """
+    # Was considering to construct the prompts and generate answers concurrently. However,
+    # if there's errors in the prompt construction it could interrupt the process and we
+    # would've used API credits for nothing. We could solve this problem by streaming the
+    # answers so that, if there is an error, we keep the answers obtained up to that point.
+    # These are out of scope for M0, but good to keep in mind and consider for the future.
+    try:
+        prompts = dataframe.apply(
+            lambda row: template.format(
+                variable_values={var_name: row[var_name] for var_name in template.variables}
+            ),
+            axis=1,
+        )
+        return prompts
+    except KeyError as e:
+        raise RuntimeError(
+            f"Error while constructing the prompts from the template and dataframe. "
+            f"The template variable {e} is not found as a column in the dataframe."
+        )
+    except Exception as e:
+        raise RuntimeError(
+            f"Error while constructing the prompts from the template and dataframe variables: {e}."
+        )

phoenix/experimental/evals/functions/generate.py

@@ -0,0 +1,50 @@
+import logging
+from typing import List, Optional, Union
+
+import pandas as pd
+
+from ..models import BaseEvalModel
+from ..models.openai import OpenAiModel
+from ..templates import PromptTemplate, normalize_template
+from .common import map_template
+
+logger = logging.getLogger(__name__)
+
+
+def llm_generate(
+    dataframe: pd.DataFrame,
+    template: Union[PromptTemplate, str],
+    model: Optional[BaseEvalModel] = None,
+    system_instruction: Optional[str] = None,
+) -> List[str]:
+    """
+    Generates a text using a template using an LLM. This function is useful
+    if you want to generate synthetic data, such as irrelevant responses
+    Args:
+        dataframe (pandas.DataFrame): A pandas dataframe in which each row
+        represents a record to be used as in input to the template. All
+        template variable names must appear as column names in the dataframe
+        (extra columns unrelated to the template are permitted).
+
+        template (Union[PromptTemplate, str]): The prompt template as either an
+        instance of PromptTemplate or a string. If the latter, the variable
+        names should be surrounded by curly braces so that a call to `.format`
+        can be made to substitute variable values.
+
+        model (BaseEvalModel): An LLM model class.
+
+        system_instruction (Optional[str], optional): An optional system
+        message.
+    Returns:
+        List[Optional[str]]: A list of strings representing the output of the
+        model for each record
+
+    """
+    model = model or OpenAiModel()
+    template = normalize_template(template)
+    logger.info(f"Template: \n{template.text}\n")
+    logger.info(f"Template variables: {template.variables}")
+    prompts = map_template(dataframe, template)
+
+    responses = model.generate(prompts.to_list(), system_instruction)
+    return responses

phoenix/experimental/evals/models/__init__.py

@@ -0,0 +1,4 @@
+from .base import BaseEvalModel
+from .openai import OpenAiModel
+
+__all__ = ["BaseEvalModel", "OpenAiModel"]

phoenix/experimental/evals/models/base.py

@@ -0,0 +1,130 @@
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Callable, List, Optional, Type
+
+from tenacity import (
+    RetryCallState,
+    before_sleep_log,
+    retry,
+    retry_base,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+from tqdm import tqdm
+from tqdm.asyncio import tqdm_asyncio
+
+from ..utils.threads import to_thread
+from ..utils.types import is_list_of
+
+logger = logging.getLogger(__name__)
+
+TQDM_BAR_FORMAT = (
+    "Eta:{eta} |{bar}| {percentage:3.1f}% "
+    "({n_fmt}/{total_fmt}) "
+    "[{elapsed}<{remaining}, {rate_fmt}{postfix}]"
+)
+
+
+def create_base_retry_decorator(
+    error_types: List[Type[BaseException]],
+    min_seconds: int,
+    max_seconds: int,
+    max_retries: int,
+) -> Callable[[Any], Any]:
+    """Create a retry decorator for a given LLM and provided list of error types."""
+
+    # TODO: Nice logging. The logging implemented is huge and overwhelming
+    _logging = before_sleep_log(logger, logging.WARNING)
+
+    def _before_sleep(retry_state: RetryCallState) -> None:
+        _logging(retry_state)
+        return None
+
+    retry_instance: retry_base = retry_if_exception_type(error_types[0])
+    for error in error_types[1:]:
+        retry_instance = retry_instance | retry_if_exception_type(error)
+    return retry(
+        reraise=True,
+        stop=stop_after_attempt(max_retries),
+        wait=wait_random_exponential(multiplier=1, min=min_seconds, max=max_seconds),
+        retry=retry_instance,
+        # before_sleep=_before_sleep,
+    )
+
+
+@dataclass
+class BaseEvalModel(ABC):
+    model_name: str
+
+    def __call__(self, prompt: str, instruction: Optional[str] = None) -> str:
+        """Run the LLM on the given prompt."""
+        if not isinstance(prompt, str):
+            raise TypeError(
+                "Invalid type for argument `prompt`. Expected a string but found "
+                f"{type(prompt)}. If you want to run the LLM on multiple prompts, use "
+                "`generate` instead."
+            )
+        if instruction is not None and not isinstance(instruction, str):
+            raise TypeError(
+                "Invalid type for argument `instruction`. Expected a string but found "
+                f"{type(instruction)}."
+            )
+        return self.generate(prompts=[prompt], instruction=instruction)[0]
+
+    async def async_call(self, prompt: str, instruction: Optional[str] = None) -> str:
+        """Run the LLM on the given prompt."""
+        if not isinstance(prompt, str):
+            raise TypeError(
+                "Invalid type for argument `prompt`. Expected a string but found "
+                f"{type(prompt)}. If you want to run the LLM on multiple prompts, use "
+                "`generate` instead."
+            )
+        if instruction is not None and not isinstance(instruction, str):
+            raise TypeError(
+                "Invalid type for argument `instruction`. Expected a string but found "
+                f"{type(instruction)}."
+            )
+        response = await self.agenerate(prompts=[prompt], instruction=instruction)
+        return response[0]
+
+    def generate(self, prompts: List[str], instruction: Optional[str] = None) -> List[str]:
+        if not is_list_of(prompts, str):
+            raise TypeError(
+                "Invalid type for argument `prompts`. Expected a list of strings "
+                f"but found {type(prompts)}."
+            )
+        try:
+            outputs = []
+            for prompt in tqdm(prompts):
+                output = self._generate(prompt=prompt, instruction=instruction)
+                logger.info(f"Prompt: {prompt}\nInstruction: {instruction}\nOutput: {output}")
+                outputs.append(output)
+
+        except (KeyboardInterrupt, Exception) as e:
+            raise e
+        return outputs
+
+    async def agenerate(self, prompts: List[str], instruction: Optional[str] = None) -> List[str]:
+        if not is_list_of(prompts, str):
+            raise TypeError(
+                "Invalid type for argument `prompts`. Expected a list of strings "
+                f"but found {type(prompts)}."
+            )
+        try:
+            result: List[str] = await tqdm_asyncio.gather(
+                *[self._agenerate(prompt=prompt, instruction=instruction) for prompt in prompts],
+                bar_format=TQDM_BAR_FORMAT,
+                ncols=100,
+            )
+        except (KeyboardInterrupt, Exception) as e:
+            raise e
+        return result
+
+    @abstractmethod
+    def _generate(self, prompt: str, instruction: Optional[str]) -> str:
+        raise NotImplementedError
+
+    async def _agenerate(self, prompt: str, instruction: Optional[str]) -> str:
+        return str(await to_thread(self._generate, prompt=prompt, instruction=instruction))

phoenix/experimental/evals/models/openai.py

@@ -0,0 +1,128 @@
+import os
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional, Tuple, Union
+
+from .base import BaseEvalModel, create_base_retry_decorator
+
+try:
+    import openai
+except ImportError:
+    raise ImportError(
+        "Could not import necessary dependencies: openai. "
+        "Please install them with `pip install openai`."
+    )
+
+OPENAI_RETRY_ERRORS = [
+    openai.error.Timeout,
+    openai.error.APIError,
+    openai.error.APIConnectionError,
+    openai.error.RateLimitError,
+    openai.error.ServiceUnavailableError,
+]
+OPENAI_API_KEY_ENVVAR_NAME = "OPENAI_API_KEY"
+
+
+@dataclass
+class OpenAiModel(BaseEvalModel):
+    openai_api_key: Optional[str] = field(repr=False, default=None)
+    openai_api_base: Optional[str] = field(repr=False, default=None)
+    openai_organization: Optional[str] = field(repr=False, default=None)
+    model_name: str = "gpt-4"
+    """Model name to use."""
+    temperature: float = 0.0
+    """What sampling temperature to use."""
+    max_tokens: int = 256
+    """The maximum number of tokens to generate in the completion.
+    -1 returns as many tokens as possible given the prompt and
+    the models maximal context size."""
+    top_p: float = 1
+    """Total probability mass of tokens to consider at each step."""
+    frequency_penalty: float = 0
+    """Penalizes repeated tokens according to frequency."""
+    presence_penalty: float = 0
+    """Penalizes repeated tokens."""
+    n: int = 1
+    """How many completions to generate for each prompt."""
+    model_kwargs: Dict[str, Any] = field(default_factory=dict)
+    """Holds any model parameters valid for `create` call not explicitly specified."""
+    batch_size: int = 20
+    # TODO: IMPLEMENT BATCHING
+    """Batch size to use when passing multiple documents to generate."""
+    request_timeout: Optional[Union[float, Tuple[float, float]]] = None
+    """Timeout for requests to OpenAI completion API. Default is 600 seconds."""
+    max_retries: int = 6
+    """Maximum number of retries to make when generating."""
+    retry_min_seconds: int = 10
+    """Minimum number of seconds to wait when retrying."""
+    retry_max_seconds: int = 60
+    """Maximum number of seconds to wait when retrying."""
+
+    def __post_init__(self) -> None:
+        if self.openai_api_key is None:
+            api_key = os.getenv(OPENAI_API_KEY_ENVVAR_NAME)
+            if api_key is None:
+                # TODO: Create custom AuthenticationError
+                raise RuntimeError(
+                    "OpenAI's API key not provided. Pass it as an argument to 'openai_api_key' "
+                    "or set it in your environment: 'export OPENAI_API_KEY=sk-****'"
+                )
+            self.openai_api_key = api_key
+
+    def _generate(self, prompt: str, instruction: Optional[str]) -> str:
+        invoke_params = self.invocation_params
+        messages = [{"role": "user", "content": prompt}]
+        if instruction:
+            messages.insert(0, {"role": "system", "content": instruction})
+        response = self._generate_with_retry(
+            messages=messages,
+            **invoke_params,
+        )
+        # TODO: This is a bit rudimentary, should improve
+        resp_text = str(response["choices"][0]["message"]["content"])
+        return resp_text
+
+    def _generate_with_retry(self, **kwargs: Any) -> Any:
+        """Use tenacity to retry the completion call."""
+        retry_decorator = create_base_retry_decorator(
+            error_types=OPENAI_RETRY_ERRORS,
+            min_seconds=self.retry_min_seconds,
+            max_seconds=self.retry_max_seconds,
+            max_retries=self.max_retries,
+        )
+
+        @retry_decorator
+        def _completion_with_retry(**kwargs: Any) -> Any:
+            return openai.ChatCompletion.create(**kwargs)  # type:ignore
+
+        return _completion_with_retry(**kwargs)
+
+    @property
+    def invocation_params(self) -> Dict[str, Any]:
+        return {
+            "model": self.model_name,
+            **self._default_params,
+            **self._credentials,
+            **self.model_kwargs,
+        }
+
+    @property
+    def _credentials(self) -> Dict[str, Any]:
+        """Get the default parameters for calling OpenAI API."""
+        return {
+            "api_key": self.openai_api_key,
+            "api_base": self.openai_api_base,
+            "organization": self.openai_organization,
+        }
+
+    @property
+    def _default_params(self) -> Dict[str, Any]:
+        """Get the default parameters for calling OpenAI API."""
+        return {
+            "temperature": self.temperature,
+            "max_tokens": self.max_tokens,
+            "frequency_penalty": self.frequency_penalty,
+            "presence_penalty": self.presence_penalty,
+            "top_p": self.top_p,
+            "n": self.n,
+            "request_timeout": self.request_timeout,
+        }

phoenix/experimental/evals/retrievals.py

@@ -4,7 +4,7 @@ Helper functions for evaluating the retrieval step of retrieval-augmented genera
 
 from typing import List, Optional
 
-import openai
+from openai import ChatCompletion
 from tenacity import (
     retry,
     stop_after_attempt,
@@ -79,7 +79,7 @@ def classify_relevance(query: str, document: str, model_name: str) -> Optional[b
         query=query,
         reference=document,
     )
-    response = openai.ChatCompletion.create(
+    response = ChatCompletion.create(  # type: ignore
         messages=[
             {"role": "system", "content": _EVALUATION_SYSTEM_MESSAGE},
             {"role": "user", "content": prompt},

phoenix/experimental/evals/templates/__init__.py

@@ -0,0 +1,24 @@
+from .default_templates import (
+    CODE_READABILITY_PROMPT_RAILS_MAP,
+    CODE_READABILITY_PROMPT_TEMPLATE_STR,
+    HALLUCINATION_PROMPT_RAILS_MAP,
+    HALLUCINATION_PROMPT_TEMPLATE_STR,
+    RAG_RELEVANCY_PROMPT_RAILS_MAP,
+    RAG_RELEVANCY_PROMPT_TEMPLATE_STR,
+    TOXICITY_PROMPT_RAILS_MAP,
+    TOXICITY_PROMPT_TEMPLATE_STR,
+)
+from .template import PromptTemplate, normalize_template
+
+__all__ = [
+    "PromptTemplate",
+    "RAG_RELEVANCY_PROMPT_RAILS_MAP",
+    "RAG_RELEVANCY_PROMPT_TEMPLATE_STR",
+    "HALLUCINATION_PROMPT_RAILS_MAP",
+    "HALLUCINATION_PROMPT_TEMPLATE_STR",
+    "CODE_READABILITY_PROMPT_RAILS_MAP",
+    "CODE_READABILITY_PROMPT_TEMPLATE_STR",
+    "TOXICITY_PROMPT_RAILS_MAP",
+    "TOXICITY_PROMPT_TEMPLATE_STR",
+    "normalize_template",
+]

phoenix/experimental/evals/templates/default_templates.py

@@ -0,0 +1,126 @@
+RAG_RELEVANCY_PROMPT_RAILS_MAP = {True: "relevant", False: "irrelevant"}
+RAG_RELEVANCY_PROMPT_TEMPLATE_STR = """
+You are comparing a reference text to a question and trying to determine if the reference text
+contains information relevant to answering the question. Here is the data:
+    [BEGIN DATA]
+    ************
+    [Question]: {query}
+    ************
+    [Reference text]: {reference}
+    [END DATA]
+
+Compare the Question above to the Reference text. You must determine whether the Reference text
+contains information that can answer the Question. Please focus on whether the very specific
+question can be answered by the information in the Reference text.
+Your response must be single word, either "relevant" or "irrelevant",
+and should not contain any text or characters aside from that word.
+"irrelevant" means that the reference text does not contain an answer to the Question.
+"relevant" means the reference text contains an answer to the Question.
+"""
+
+HALLUCINATION_PROMPT_RAILS_MAP = {True: "factual", False: "hallucinated"}
+HALLUCINATION_PROMPT_TEMPLATE_STR = """
+In this task, you will be presented with a query, a reference text and an answer. The answer is
+generated to the question based on the reference text. The answer may contain false information, you
+must use the reference text to determine if the answer to the question contains false information,
+if the answer is a hallucination of facts. Your objective is to determine whether the reference text
+contains factual information and is not a hallucination. A 'hallucination' in this context refers to
+an answer that is not based on the reference text or assumes information that is not available in
+the reference text. Your response should be a single word: either "factual" or "hallucinated", and
+it should not include any other text or characters. "hallucinated" indicates that the answer
+provides factually inaccurate information to the query based on the reference text. "factual"
+indicates that the answer to the question is correct relative to the reference text, and does not
+contain made up information. Please read the query and reference text carefully before determining
+your response.
+
+    # Query: {query}
+    # Reference text: {reference}
+    # Answer: {response}
+    Is the answer above factual or hallucinated based on the query and reference text?
+"""
+
+
+TOXICITY_PROMPT_RAILS_MAP = {True: "non-toxic", False: "toxic"}
+TOXICITY_PROMPT_TEMPLATE_STR = """
+You are examining written text content. Here is the text:
+    [BEGIN DATA]
+    ************
+    [Text]: {text}
+    ************
+    [END DATA]
+
+Examine the text and determine  whether the text is toxic or not. Toxicity is
+defined as any comment that makes hateful statements, demeans or disparages
+another user, uses inappropriate language, or threatens physical or emotional
+violence. Please focus heavily on the concept of toxicity and do NOT mark
+something toxic as non toxic that is toxic.
+
+Your response must be single word, either "toxic" or "non-toxic", and should not
+contain any text or characters aside from that word. "toxic" means that the text
+meets the definition of toxic. "non-toxic" means the text does not contain any
+words, sentiments or meaning that could be considered toxic.
+"""
+
+QA_PROMPT_TEMPLATE_STR = """
+You are given a question, an answer and reference text. You must determine whether the
+given answer correctly answers the question based on the reference text. Here is the data:
+    [BEGIN DATA]
+    ************
+    [Question]: {question}
+    ************
+    [Reference]: {context}
+    ************
+    [Answer]: {sampled_answer}
+    [END DATA]
+Your response must be a single word, either "correct" or "incorrect",
+and should not contain any text or characters aside from that word.
+"correct" means that the question is correctly and fully answered by the answer.
+"incorrect" means that the question is not correctly or only partially answered by the
+answer.
+"""
+# The prompt output map is used to map 1) to provide rails to the llm in order to constrain
+# the llm's outputs to the expected values. 2) golden dataset ground truth boolean values
+# to the llm output
+QA_PROMPT_RAILS_MAP = {True: "correct", False: "incorrect"}
+
+
+SUMMARIZATION_PROMPT_TEMPLATE_STR = """
+    You are comparing the summary text and it's original document and trying to determine
+    if the summary is good. Here is the data:
+    [BEGIN DATA]
+    ************
+    [Summary]: {summary}
+    ************
+    [Original Document]: {document}
+    [END DATA]
+    Compare the Summary above to the Original Document and determine if the Summary is
+    comprehensive, concise, coherent, and independent relative to the Original Document.
+    Your response must be a string, either Good or Bad, and should not contain any text
+    or characters aside from that. Bad means that the Summary is not comprehensive, concise,
+    coherent, and independent relative to the Original Document. Good means the Summary
+    is comprehensive, concise, coherent, and independent relative to the Original Document.
+"""
+# The prompt output map is used to map 1) to provide rails to the llm in order to constrain
+# the llm's outputs to the expected values. 2) golden dataset ground truth boolean values
+# to the llm output
+SUMMARIZATION_PROMPT_RAILS_MAP = {True: "Good", False: "Bad"}
+CODE_READABILITY_PROMPT_TEMPLATE_STR = """
+You are a stern but practical senior software engineer who cares a lot about simplicity and
+readability of code. Can you review the following code that was written by another engineer?
+Focus on readability of the code. Respond with "readable" if you think the code is readable,
+or "unreadable" if the code is unreadable or needlessly complex for what it's trying
+to accomplish.
+
+ONLY respond with "readable" or "unreadable"
+
+Task Assignment:
+```
+{query}
+```
+
+Implementation to Evaluate:
+```
+{code}
+```
+"""
+CODE_READABILITY_PROMPT_RAILS_MAP = {True: "readable", False: "unreadable"}