torchrl 0.11.0__cp314-cp314-macosx_11_0_arm64.whl

torchrl/envs/llm/reward/ifeval/_instructions_main.py

@@ -0,0 +1,131 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+"""List of modifications from original script.
+
+Modifications include:
+
+- making public functions/classes private.
+
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+from ._instructions_registry import INSTRUCTION_DICT
+
+
+@dataclasses.dataclass
+class _InputExample:
+    key: int
+    instruction_id_list: list[str]
+    prompt: str
+    kwargs: list[dict[str, str | int | None]]
+
+
+@dataclasses.dataclass
+class _OutputExample:
+    instruction_id_list: list[str]
+    prompt: str
+    response: str
+    follow_all_instructions: bool
+    follow_instruction_list: list[bool]
+
+
+def _test_instruction_following_strict(
+    inp: _InputExample,
+    response: str,
+):
+    """Tests response to see if instructions are followed."""
+    instruction_list = inp.instruction_id_list
+    if not isinstance(instruction_list, list):
+        raise ValueError(
+            f"instruction_list must be a list, got {type(instruction_list)}, {instruction_list=}"
+        )
+    is_following_list = []
+
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method.
+        kwargs = {k: v for k, v in inp.kwargs[index].items() if v}
+        instruction.build_description(**kwargs)
+        args = instruction.get_instruction_args()
+        if args and "prompt" in args:
+            instruction.build_description(prompt=inp.prompt)
+
+        if response.strip() and instruction.check_following(response):
+            is_following_list.append(True)
+        else:
+            is_following_list.append(False)
+
+    return _OutputExample(
+        instruction_id_list=inp.instruction_id_list,
+        prompt=inp.prompt,
+        response=response,
+        follow_all_instructions=all(is_following_list),
+        follow_instruction_list=is_following_list,
+    )
+
+
+def _test_instruction_following_loose(
+    inp,
+    response,
+):
+    """Tests response for an upper bound for following instructions."""
+    r = response.split("\n")
+    response_remove_first = "\n".join(r[1:]).strip()
+    response_remove_last = "\n".join(r[:-1]).strip()
+    response_remove_both = "\n".join(r[1:-1]).strip()
+    revised_response = response.replace("*", "")
+    revised_response_remove_first = response_remove_first.replace("*", "")
+    revised_response_remove_last = response_remove_last.replace("*", "")
+    revised_response_remove_both = response_remove_both.replace("*", "")
+    all_responses = [
+        response,
+        revised_response,
+        response_remove_first,
+        response_remove_last,
+        response_remove_both,
+        revised_response_remove_first,
+        revised_response_remove_last,
+        revised_response_remove_both,
+    ]
+    instruction_list = inp.instruction_id_list
+    is_following_list = []
+
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method.
+        kwargs = {k: v for k, v in inp.kwargs[index].items() if v}
+        instruction.build_description(**kwargs)
+        args = instruction.get_instruction_args()
+        if args and "prompt" in args:
+            instruction.build_description(prompt=inp.prompt)
+
+        is_following = False
+        for r in all_responses:
+            if r.strip() and instruction.check_following(r):
+                is_following = True
+                break
+
+        is_following_list.append(is_following)
+
+    return _OutputExample(
+        instruction_id_list=inp.instruction_id_list,
+        prompt=inp.prompt,
+        response=response,
+        follow_all_instructions=all(is_following_list),
+        follow_instruction_list=is_following_list,
+    )
+
+
+def _agg_inst_level_acc(items):
+    flat_items = [item for sublist in items for item in sublist]
+    inst_level_acc = sum(flat_items) / len(flat_items)
+    return inst_level_acc

torchrl/envs/llm/reward/ifeval/_instructions_registry.py

@@ -0,0 +1,100 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+# Original LICENSE:
+#  Origin: https://github.com/google-research/google-research/blob/master/instruction_following_eval/instruction_following_eval.py
+# coding=utf-8
+# Copyright 2024 The Google Research Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+from ._instructions import (
+    BulletListChecker,
+    CapitalLettersEnglishChecker,
+    CapitalWordFrequencyChecker,
+    CommaChecker,
+    ConstrainedResponseChecker,
+    EndChecker,
+    ForbiddenWords,
+    HighlightSectionChecker,
+    JsonFormat,
+    KeywordChecker,
+    KeywordFrequencyChecker,
+    LetterFrequencyChecker,
+    LowercaseLettersEnglishChecker,
+    NumberOfSentences,
+    NumberOfWords,
+    ParagraphChecker,
+    ParagraphFirstWordCheck,
+    PlaceholderChecker,
+    PostscriptChecker,
+    QuotationChecker,
+    RepeatPromptThenAnswer,
+    ResponseLanguageChecker,
+    SectionChecker,
+    TitleChecker,
+    TwoResponsesChecker,
+)
+
+_KEYWORD = "keywords:"
+_LANGUAGE = "language:"
+_LENGTH = "length_constraints:"
+_CONTENT = "detectable_content:"
+_FORMAT = "detectable_format:"
+_MULTITURN = "multi-turn:"
+_COMBINATION = "combination:"
+_STARTEND = "startend:"
+_CHANGE_CASES = "change_case:"
+_PUNCTUATION = "punctuation:"
+
+INSTRUCTION_DICT = {
+    _KEYWORD + "existence": KeywordChecker,
+    _KEYWORD + "frequency": KeywordFrequencyChecker,
+    # TODO(jeffreyzhou): make a proper set of sentences to choose from
+    # _KEYWORD + "key_sentences": KeySentenceChecker,
+    _KEYWORD + "forbidden_words": ForbiddenWords,
+    _KEYWORD + "letter_frequency": LetterFrequencyChecker,
+    _LANGUAGE + "response_language": ResponseLanguageChecker,
+    _LENGTH + "number_sentences": NumberOfSentences,
+    _LENGTH + "number_paragraphs": ParagraphChecker,
+    _LENGTH + "number_words": NumberOfWords,
+    _LENGTH + "nth_paragraph_first_word": ParagraphFirstWordCheck,
+    _CONTENT + "number_placeholders": PlaceholderChecker,
+    _CONTENT + "postscript": PostscriptChecker,
+    _FORMAT + "number_bullet_lists": BulletListChecker,
+    # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace
+    # _CONTENT + "rephrase_paragraph": RephraseParagraph,
+    _FORMAT + "constrained_response": ConstrainedResponseChecker,
+    _FORMAT + "number_highlighted_sections": HighlightSectionChecker,
+    _FORMAT + "multiple_sections": SectionChecker,
+    # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message.
+    # _FORMAT + "rephrase": RephraseChecker,
+    _FORMAT + "json_format": JsonFormat,
+    _FORMAT + "title": TitleChecker,
+    # TODO(tianjianlu): Re-enable with specific prompts.
+    # _MULTITURN + "constrained_start": ConstrainedStartChecker,
+    _COMBINATION + "two_responses": TwoResponsesChecker,
+    _COMBINATION + "repeat_prompt": RepeatPromptThenAnswer,
+    _STARTEND + "end_checker": EndChecker,
+    _CHANGE_CASES + "capital_word_frequency": CapitalWordFrequencyChecker,
+    _CHANGE_CASES + "english_capital": CapitalLettersEnglishChecker,
+    _CHANGE_CASES + "english_lowercase": LowercaseLettersEnglishChecker,
+    _PUNCTUATION + "no_comma": CommaChecker,
+    _STARTEND + "quotation": QuotationChecker,
+}