sdg-hub 0.1.0a2.dev0__py3-none-any.whl

sdg_hub/blocks/rmblocks.py

@@ -0,0 +1,194 @@
+"""Module containing blocks for scoring responses using Reward Models."""
+
+# Standard
+from typing import Dict, List
+import json
+from urllib.parse import urljoin
+
+# Third Party
+from datasets import Dataset
+import requests
+
+# Local
+from .block import Block
+from ..logger_config import setup_logger
+from ..registry import BlockRegistry
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register("PRMBlock")
+class PRMBlock(Block):
+    """A block for scoring responses using a ProcessReward Model (PRM) via HTTP API.
+
+    This block sends prompts and responses to a PRM endpoint and returns reward scores
+    for each step in the response.
+    """
+
+    def __init__(
+        self,
+        block_name: str,
+        host: str,
+        port: int,
+        model_name: str,
+        prompt_col: str,
+        response_col: str,
+        output_col: str = "step_rewards",
+        system_prompt: str = None,
+        endpoint: str = "pooling",
+        step_separator: str = "\n\n",
+        step_fill_token: str = "<extra_0>",
+    ) -> None:
+        r"""Initialize the PRM (Process Reward Model) Block.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block
+        host : str
+            Hostname of the PRM service (e.g., "0.0.0.0" or "localhost")
+        port : int
+            Port number the service is running on
+        model_name : str
+            Name of the PRM model to use
+        prompt_col : str
+            Column name containing the prompt
+        response_col : str
+            Column name containing the response
+        output_col : str, optional
+            Column name to store the reward scores, by default "step_rewards"
+        system_prompt : str, optional
+            Optional system prompt to use for scoring, by default None
+        endpoint : str, optional
+            API endpoint name, by default "pooling"
+        step_separator : str, optional
+            Separator between steps in the response, by default "\n\n"
+        step_fill_token : str, optional
+            Model specific fill token for steps in the response, by default "<extra_0>" used by Qwen2.5-Math-PRM
+        """
+        super().__init__(block_name)
+        # Construct base URL from host and port
+        self.base_url = f"http://{host.strip('/')}:{port}/"
+        self.endpoint = endpoint.strip("/")
+
+        # Construct the full API URL using urljoin
+        self.api_url = urljoin(self.base_url, self.endpoint)
+        logger.info(f"Initialized PRMBlock with API URL: {self.api_url}")
+
+        self.model_name = model_name
+        self.prompt_col = prompt_col
+        self.response_col = response_col
+        self.output_col = output_col
+        self.system_prompt = system_prompt
+        self.step_separator = step_separator
+        self.step_fill_token = step_fill_token
+
+    def _post_request(self, messages: List[Dict]) -> requests.Response:
+        """Make POST request to PRM API endpoint.
+
+        Parameters
+        ----------
+        messages : List[Dict]
+            List of message dictionaries to send to the API
+
+        Returns
+        -------
+        requests.Response
+            Response from the API
+        """
+        headers = {"User-Agent": "PRMBlock Client"}
+        prompt = {"model": self.model_name, "messages": messages}
+        response = requests.post(self.api_url, headers=headers, json=prompt)
+        return response
+
+    def _format_messages(self, sample: Dict) -> List[Dict]:
+        """Format input sample into messages for the PRM API.
+
+        Parameters
+        ----------
+        sample : Dict
+            Input sample containing prompt and response
+
+        Returns
+        -------
+        List[Dict]
+            Formatted messages for the API
+        """
+        messages = []
+        if self.system_prompt:
+            messages.append({"role": "system", "content": self.system_prompt})
+
+        messages.append({"role": "user", "content": sample[self.prompt_col]})
+        messages.append(
+            {
+                "role": "assistant",
+                "content": self.step_fill_token.join(sample[self.response_col].split(self.step_separator))
+                + self.step_fill_token,
+            }
+        )
+        return messages
+
+    def _extract_rewards(self, response: requests.Response) -> List[float]:
+        """Extract reward scores from API response.
+
+        Parameters
+        ----------
+        response : requests.Response
+            Response from the API
+
+        Returns
+        -------
+        List[float]
+            List of reward scores
+        """
+        try:
+            response_data = response.json()
+            rewards = [x[1] for x in response_data["data"][0]["data"]]
+            return rewards
+        except (KeyError, IndexError, json.JSONDecodeError) as e:
+            logger.error(f"Error extracting rewards from response: {e}")
+            return []
+
+    def _generate(self, sample: dict) -> dict:
+        """Generate reward scores for the input samples.
+
+        Parameters
+        ----------
+        sample : dict
+            Input sample to score
+
+        Returns
+        -------
+        dict
+            Dictionary with added reward scores column
+        """
+        messages = self._format_messages(sample)
+        rm_response = self._post_request(messages)
+
+        if rm_response.status_code != 200:
+            logger.error(f"API request failed with status {rm_response.status_code}")
+            rewards = [0.0] * len(
+                sample[self.response_col].split(self.step_separator)
+            )  # Default to 0 scores on failure
+        else:
+            rewards = self._extract_rewards(rm_response)
+
+        sample[self.output_col] = rewards
+        return sample
+
+    def generate(self, samples: Dataset, batch_size: int = 4) -> Dataset:
+        """Generate reward scores for the input samples.
+
+        Parameters
+        ----------
+        samples : Dataset
+            Input dataset containing samples to score
+        batch_size : int, optional
+            Number of processes to use for parallel processing, by default 4
+
+        Returns
+        -------
+        Dataset
+            Dataset with added reward scores
+        """
+        return samples.map(self._generate, num_proc=batch_size)

sdg_hub/blocks/utilblocks.py

@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: Apache-2.0
+# Third Party
+from datasets import Dataset
+
+# Local
+from .block import Block
+from ..registry import BlockRegistry
+from ..logger_config import setup_logger
+
+logger = setup_logger(__name__)
+
+
+@BlockRegistry.register("SamplePopulatorBlock")
+class SamplePopulatorBlock(Block):
+    def __init__(self, config_paths, column_name, post_fix="", **batch_kwargs) -> None:
+        super().__init__(
+            block_name=self.__class__.__name__
+        )  # Call the base class's __init__
+        self.configs = {}
+        for config in config_paths:
+            if post_fix:
+                config_name = config.replace(".yaml", f"_{post_fix}.yaml")
+            else:
+                config_name = config
+            config_key = config.split("/")[-1].split(".")[0]
+            self.configs[config_key] = self._load_config(config_name)
+        self.column_name = column_name
+        self.num_procs = batch_kwargs.get("num_procs", 8)
+
+    def _generate(self, sample) -> dict:
+        sample = {**sample, **self.configs[sample[self.column_name]]}
+        return sample
+
+    def generate(self, samples) -> Dataset:
+        samples = samples.map(self._generate, num_proc=self.num_procs)
+        return samples
+
+
+@BlockRegistry.register("SelectorBlock")
+class SelectorBlock(Block):
+    def __init__(self, choice_map, choice_col, output_col, **batch_kwargs) -> None:
+        super().__init__(block_name=self.__class__.__name__)
+        self.choice_map = choice_map
+        self.choice_col = choice_col
+        self.output_col = output_col
+        self.num_procs = batch_kwargs.get("num_procs", 8)
+
+    def _generate(self, sample) -> dict:
+        sample[self.output_col] = sample[self.choice_map[sample[self.choice_col]]]
+        return sample
+
+    def generate(self, samples: Dataset) -> Dataset:
+        samples = samples.map(self._generate, num_proc=self.num_procs)
+        return samples
+
+
+@BlockRegistry.register("CombineColumnsBlock")
+class CombineColumnsBlock(Block):
+    def __init__(self, columns, output_col, separator="\n\n", **batch_kwargs) -> None:
+        super().__init__(block_name=self.__class__.__name__)
+        self.columns = columns
+        self.output_col = output_col
+        self.separator = separator
+        self.num_procs = batch_kwargs.get("num_procs", 8)
+
+    def _generate(self, sample) -> dict:
+        sample[self.output_col] = self.separator.join(
+            [sample[col] for col in self.columns]
+        )
+        return sample
+
+    def generate(self, samples: Dataset) -> Dataset:
+        samples = samples.map(self._generate, num_proc=self.num_procs)
+        return samples
+
+
+@BlockRegistry.register("FlattenColumnsBlock")
+class FlattenColumnsBlock(Block):
+    def __init__(self, block_name: str, var_cols: list, value_name: str, var_name: str) -> None:
+        super().__init__(block_name=block_name)
+        self.var_cols = var_cols
+        self.value_name = value_name
+        self.var_name = var_name
+
+    def generate(self, samples: Dataset) -> Dataset:
+        df = samples.to_pandas()
+        id_cols = [col for col in samples.column_names if col not in self.var_cols]
+        flatten_df = df.melt(id_vars=id_cols, 
+                             value_vars=self.var_cols, 
+                             value_name=self.value_name,
+                             var_name=self.var_name)
+        
+        return Dataset.from_pandas(flatten_df)
+
+
+@BlockRegistry.register("DuplicateColumns")
+class DuplicateColumns(Block):
+    def __init__(self, block_name: str, columns_map: dict) -> None:
+        """Create duplicate of columns specified in column map.
+
+        Args:
+            columns_map (dict): mapping of existing column to new column names
+        """
+        super().__init__(block_name=block_name)
+        self.columns_map = columns_map    
+    
+    
+    def generate(self, samples: Dataset):
+        for col_to_dup in self.columns_map:
+            samples = samples.add_column(self.columns_map[col_to_dup], samples[col_to_dup])
+        return samples
+
+
+@BlockRegistry.register("RenameColumns")
+class RenameColumns(Block):
+    def __init__(self, block_name: str, columns_map: dict) -> None:
+        """Rename dataset columns.
+
+        Args:
+            columns_map (dict): mapping of existing column to new column names
+        """
+        self.columns_map = columns_map
+        super().__init__(block_name=block_name)
+    
+    
+    def generate(self, samples: Dataset):
+        samples = samples.rename_columns(self.columns_map)
+        return samples
+
+
+@BlockRegistry.register("SetToMajorityValue")
+class SetToMajorityValue(Block):
+    def __init__(self, block_name: str, col_name) -> None:
+        self.col_name = col_name
+        super().__init__(block_name)
+    
+    def generate(self, samples: Dataset):
+        samples = samples.to_pandas()
+        samples[self.col_name] = samples[self.col_name].mode()[0]
+        return Dataset.from_pandas(samples)

sdg_hub/configs/annotations/cot_reflection.yaml

@@ -0,0 +1,34 @@
+system: You are an AI assistant that uses a Chain of Thought (CoT) approach with reflection to answer queries.
+introduction: |
+  Task Description: {{ task_description }}
+principles: |
+  Follow these steps:
+  1. Think through the problem step by step within the [Start of Thinking] and [End of Thinking] tags.
+  2. Reflect on your thinking to check for any errors or improvements within the [Start of Reflection] and [End of Reflection] tags.
+  3. Make any necessary adjustments based on your reflection. 
+  4. You can iterate on your thinking and reflection before providing the final answer.
+  5. Provide your final, concise answer within the [Start of Output] and [End of Output] tags.
+examples: ~
+generation: |
+  Important: The Thinking, and Reflection sections are for your internal reasoning process only.
+  Do not include any part of the final answer in these sections.
+  The actual response to the query must be entirely contained within the Output tags. Use the following format for your response:
+
+  [Start of Thinking]
+  [Your step-by-step reasoning goes here. This is your internal thought process, not the final answer.]
+  [End of Thinking]
+  [Start of Reflection]
+  [Your reflection on your reasoning, checking for errors or improvements]
+  [End of Reflection]
+  [Iterate your thinking and reflection to make any necessary adjustments based on your reflection.]
+  
+  [Start of Output]
+  [Your final, concise answer to the query. This is the only part that will be shown to the user.]
+  [End of Output]
+  [Stop generating any text after the End of Output tag.]
+
+  Here is the query for annotation:
+  {{ prompt }}
+  
+start_tags: ["[Start of Thinking]", "[Start of Output]"]
+end_tags: ["[End of Reflection]", "[End of Output]"]

sdg_hub/configs/annotations/detailed_description.yaml

@@ -0,0 +1,10 @@
+system: ~
+introduction: |
+  Task Description: {{ task_description }}
+principles: ~
+examples: ~
+generation: |
+  Here is the query for annotation:
+  {{ prompt }}
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/annotations/detailed_description_icl.yaml

@@ -0,0 +1,32 @@
+system: ~
+introduction: |
+  Task Description: {{ task_description }}
+principles: ~
+examples: |
+  To better assist you with this task, here are some examples:
+  {% if questions_and_answers is defined %}
+  {% for sample in questions_and_answers %}
+  [Start of Question]
+  {{ sample.question }}
+  [End of Question]
+
+  [Start of Output]
+  {{ sample.answer }}
+  [End of Output]
+  {% endfor %}
+  {% else %}
+  [Start of Question]
+  {{ seed_question }}
+  [End of Question]
+
+  [Start of Output]
+  {{ seed_response }}
+  [End of Output]
+  {% endif %}
+generation: |
+  Here is the query for annotation:
+  [Start of Question]
+  {{ prompt }}
+  [End of Question]
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/annotations/simple.yaml

@@ -0,0 +1,10 @@
+system: ~
+introduction: |
+  Task Description: {{ simple_task_description }}
+principles: ~
+examples: ~
+generation: |
+  Here is the query for annotation:
+  {{ prompt }}
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/knowledge/atomic_facts.yaml

@@ -0,0 +1,45 @@
+system: You are an AI assistant knowledgeable about {{domain}} domain. Be accurate but concise in response.
+
+introduction: | 
+  Please break down the following snippet from an article about {{domain}} into atomic facts.
+
+principles: |
+  1. Makesure each fact is grounded in the given text.
+  2. Include any necessary information needed to explain the fact or concept
+  3. The atomic facts should be as simple as possible, if it’s compound sentence, break down one more time
+  4. For clarity, avoid using pronouns like ’it’, ’he’, ’she’, ’this’, ’that’ etc., and instead use the full names or titles.
+  5. Focus only on key concepts and facts. Skip any question or problems mentioned in the passage.
+
+examples: |
+  To help you understand the task, here is an example:
+  [Passage]
+  The tournament was contested by ten national teams, maintaining the same format used in 2019. After six weeks of round-robin matches, India, South Africa, Australia, and New Zealand finished as the top four and qualified for the knockout stage. In the knockout stage, India and Australia beat New Zealand and South Africa, respectively, to advance to the final, played on 19 November at the Narendra Modi Stadium in Ahmedabad. Australia won the final by six wickets, winning their sixth Cricket World Cup title.
+  [Facts]
+  1. The tournament was contested by ten national teams.
+  2. The tournament maintained the same format used in 2019.
+  3. The round-robin matches lasted for six weeks.
+  4. India finished as one of the top four teams.
+  5. South Africa finished as one of the top four teams.
+  6. Australia finished as one of the top four teams.
+  7. New Zealand finished as one of the top four teams.
+  8. India, South Africa, Australia, and New Zealand qualified for the knockout stage.
+  9. In the knockout stage, India beat New Zealand.
+  10. In the knockout stage, Australia beat South Africa.
+  11. India advanced to the final.
+  12. Australia advanced to the final.
+  13. The final was played on 19 November.
+  14. The final was held at the Narendra Modi Stadium in Ahmedabad.
+  15. Australia won the final by six wickets.
+  16. Australia won their sixth Cricket World Cup title.
+  [End]
+
+
+generation: |
+  Now it's your turn breakdown following snippet from article about {{domain}} into atomic facts following similar style as above examples
+  [Passage]
+  {{document}}
+  [Facts]
+
+
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/knowledge/auxilary_instructions.yaml

@@ -0,0 +1,35 @@
+summary_detailed:
+  - Provide me with a comprehensive summary of the given document.
+  - Prepare a detailed breakdown of the contents of the document for me.
+  - Summarize the document thoroughly, covering all important points.
+  - Create a detailed executive summary of the provided document.
+  - Compose a comprehensive overview of the document's content.
+  - Deliver a detailed synopsis of the material presented in the document.
+  - Furnish me with a detailed analysis of the document's key points.
+  - Generate a thorough summary of the main ideas in the document.
+  - Offer a detailed digest of the information contained in the document.
+  - Supply me with a comprehensive rundown of the document's contents.
+
+summary_extractive:
+  - Provide me with a summary of the document using extractive methods.
+  - Create an extractive summary for the given document.
+  - Generate an extractive summary from the document that was given to you.
+  - Summarize the document using extractive techniques.
+  - Create a summary of the provided document using extractive methods.
+  - Generate an extractive summary for the document provided.
+  - Using extractive techniques, summarize the given document.
+  - Create a summary of the document using extractive summarization.
+  - Generate an extractive summary of the document that was provided.
+  - Summarize the provided document using extractive summarization techniques.
+
+summary_atomic_facts:
+  - Identify and list all atomic facts from the document.
+  - Extract all key facts from the given document.
+  - List all the important facts from the provided document.
+  - Highlight all the atomic facts present in the document.
+  - Identify and enumerate all key facts from the given text.
+  - List out all the critical information from the document.
+  - Highlight all the essential facts from the provided text.
+  - Identify and summarize all the important details from the document.
+  - Extract all the atomic facts from the given document.
+  - List all the key takeaways from the provided text.

sdg_hub/configs/knowledge/data_recipe/default_recipe.yaml

@@ -0,0 +1,3 @@
+datasets: []
+sys_prompt: |
+  I am, Red Hat® Instruct Model based on Granite 7B, an AI language model developed by Red Hat and IBM Research, based on the Granite-7b-base language model. My primary function is to be a chat assistant.

sdg_hub/configs/knowledge/detailed_summary.yaml

@@ -0,0 +1,17 @@
+system: You are an AI assistant that is expert at summarizing text.
+
+introduction: |
+  Give me detailed summary for below document, making sure all key points are covered.
+
+principles: |
+  Do not add any new information.
+  Do not miss any key points from the provided document
+
+examples: ""
+
+generation: |
+  Document:
+  {{document}}
+
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/knowledge/evaluate_faithfulness.yaml

@@ -0,0 +1,68 @@
+system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+introduction: |
+  Determine if the provided information is corroborated by the given context. Respond with YES if the context substantiates the information, even partially. Answer NO if the context does not support the information.
+
+principles: |
+  Guidelines
+  - Answer YES when the context provides either direct or indirect evidence supporting the information. Indirect evidence may include contextual implications or inferred connections that reasonably support the information.
+  - Answer NO if the context lacks any supportive evidence, clearly contradicts the information, or if the support provided by the context is too vague or speculative to establish a solid connection to the information.
+  - Avoid using "partially" in your response. If the context provides any reasonable support (direct or indirect) for the information, consider it as a YES.
+
+  Strictly answer in this format
+  [Start of Context]
+  ...
+  [End of Context]
+  [Start of Response]
+  ... 
+  [End of Response]
+  [Start of Explanation]
+  ...
+  [End of Explanation]
+  [Start of Answer]
+  ...
+  [End of Answer]
+
+examples: |
+  Example 1:
+  [Start of Context]
+  An apple pie is a fruit pie with apples as the main filling. It's often served with whipped cream, ice cream, custard, or cheddar cheese. Typically, it has a double crust, with pastry above and below the filling. The upper crust can be solid or latticed.
+  [End of Context]
+  [Start of Response]
+  Apple pie is generally double-crusted.
+  [End of Response]
+  [Start of Explanation]
+  The context directly supports the information by stating that apple pie is "generally double-crusted," which matches the information provided.
+  [End of Explanation]
+  [Start of Answer]
+  YES
+  [End of Answer]
+
+  Example 2:
+  [Start of Context]
+  An apple pie is a fruit pie with apples as the main filling. It's often served with whipped cream, ice cream, custard, or cheddar cheese. Typically, it has a double crust, with pastry above and below the filling. The upper crust can be solid or latticed.
+  [End of Context]
+  [Start of Response]
+  Apple pies taste bad.
+  [End of Response]
+  [Start of Explanation]
+  The context does not provide any information about the taste of apple pies. The statement "Apple pies taste bad" is a subjective opinion and is not supported or mentioned in the given context.
+  [Start of Explanation]
+  [Start of Answer]
+  NO
+  [End of Answer]
+
+generation: |
+  Now, based on the above examples and guidelines, determine if the following information is supported by the context provided. Answer YES or NO.
+  * Return the explanation within the [Start of Explanation] and [End of Explanation] tags.
+  * Return the answer between [Start of Answer] and [End of Answer] tags.
+
+  [Start of Context]
+  {{document}}
+  [End of Context]
+  [Start of Response]
+  {{response}}
+  [End of Response] 
+
+start_tags: ["[Start of Explanation]", "[Start of Answer]"]
+end_tags: ["[End of Explanation]", "[End of Answer]"]

sdg_hub/configs/knowledge/evaluate_question.yaml

@@ -0,0 +1,38 @@
+system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+introduction: |
+  Given below question can you verify if it meets below requirements and based on them give a rating of 1 if it meets all of them or 0 otherwise.
+
+principles: |
+  Here are the requirements:
+
+  Non-Referential Clarity and Contextual Independence: Ensure that the question is self-explanatory and does not rely on specific, unprovided external content, such as particular documents, specific tables, or detailed datasets. The question should be structured to be understandable and clear without requiring direct access to or knowledge of these specific external sources.
+
+  Subject-Aware Completeness: The question should be crafted to be answerable on its own, given a reasonable level of specialized knowledge in the relevant subject area. It is acceptable and encouraged for the question to require specialized understanding pertinent to the topic; however, it should not depend on unique, external information not provided in the question itself. This distinction allows for questions that necessitate a deep understanding of a subject while ensuring they are not tied to specific external content like a particular dataset or a line in a document.
+
+  Please give your answer as short explanation followed by rating of either 0 or 1 as below.
+
+  * Return a short explanation within the [Start of Explanation] and [End of Explanation] tags.
+  * Return the rating on a binary 0/1 scale between [Start of Rating] and [End of Rating] tags.
+
+  [Start of Question]  
+  ...
+  [End of Question]
+
+  [Start of Explanation]
+  ...  
+  [End of Explanation]
+
+  [Start of Rating]
+  ...
+  [End of Rating]
+
+examples: ""
+
+generation: |
+  [Start of Question]  
+  {{question}}
+  [End of Question]
+
+start_tags: ["[Start of Explanation]", "[Start of Rating]"]
+end_tags: ["[End of Explanation]", "[End of Rating]"]