sdg-hub 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

sdg_hub/core/utils/error_handling.py

@@ -0,0 +1,208 @@
+"""Custom exception classes for SDG Hub error handling."""
+
+# Standard
+from typing import Optional
+
+
+class SDGHubError(Exception):
+    """Base exception class for all SDG Hub errors."""
+
+    def __init__(self, message: str, details: Optional[str] = None):
+        """Initialize SDGHubError.
+
+        Parameters
+        ----------
+        message : str
+            The main error message.
+        details : str, optional
+            Additional details about the error.
+        """
+        self.message = message
+        self.details = details
+        full_message = message
+        if details:
+            full_message = f"{message}\nDetails: {details}"
+        super().__init__(full_message)
+
+
+class FlowRunnerError(SDGHubError):
+    """Base exception class for flow runner errors."""
+
+    pass
+
+
+class DatasetLoadError(FlowRunnerError):
+    """Raised when dataset loading fails."""
+
+    pass
+
+
+class FlowConfigurationError(FlowRunnerError):
+    """Raised when flow configuration is invalid."""
+
+    pass
+
+
+class APIConnectionError(FlowRunnerError):
+    """Raised when API connection fails."""
+
+    pass
+
+
+class DataGenerationError(FlowRunnerError):
+    """Raised when data generation fails."""
+
+    pass
+
+
+class DataSaveError(FlowRunnerError):
+    """Raised when saving generated data fails."""
+
+    pass
+
+
+class BlockError(SDGHubError):
+    """Base exception class for block-related errors."""
+
+    pass
+
+
+class BlockConfigurationError(BlockError):
+    """Raised when block configuration is invalid."""
+
+    pass
+
+
+class BlockExecutionError(BlockError):
+    """Raised when block execution fails."""
+
+    pass
+
+
+class BlockValidationError(BlockError):
+    """Base exception class for block validation errors."""
+
+    pass
+
+
+class MissingColumnError(BlockValidationError):
+    """Raised when required input columns are missing from dataset."""
+
+    def __init__(
+        self, block_name: str, missing_columns: list[str], available_columns: list[str]
+    ):
+        """Initialize MissingColumnError.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block that failed validation.
+        missing_columns : List[str]
+            List of missing column names.
+        available_columns : List[str]
+            List of available column names in the dataset.
+        """
+        self.block_name = block_name
+        self.missing_columns = missing_columns
+        self.available_columns = available_columns
+
+        message = (
+            f"Block '{block_name}' missing required input columns: {missing_columns}"
+        )
+        details = f"Available columns: {available_columns}"
+
+        super().__init__(message, details)
+
+
+class EmptyDatasetError(BlockValidationError):
+    """Raised when an empty dataset is provided to a block."""
+
+    def __init__(self, block_name: str):
+        """Initialize EmptyDatasetError.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block that received the empty dataset.
+        """
+        self.block_name = block_name
+
+        message = f"Block '{block_name}' received an empty dataset"
+        details = "Dataset must contain at least one sample for processing"
+
+        super().__init__(message, details)
+
+
+class OutputColumnCollisionError(BlockValidationError):
+    """Raised when output columns would overwrite existing dataset columns."""
+
+    def __init__(
+        self, block_name: str, collision_columns: list[str], existing_columns: list[str]
+    ):
+        """Initialize OutputColumnCollisionError.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block that has column collisions.
+        collision_columns : List[str]
+            List of output columns that collide with existing columns.
+        existing_columns : List[str]
+            List of existing column names in the dataset.
+        """
+        self.block_name = block_name
+        self.collision_columns = collision_columns
+        self.existing_columns = existing_columns
+
+        message = f"Block '{block_name}' output columns would overwrite existing data: {collision_columns}"
+        details = f"Existing columns: {existing_columns}"
+
+        super().__init__(message, details)
+
+
+class TemplateValidationError(BlockValidationError):
+    """Raised when template validation fails due to missing variables."""
+
+    def __init__(
+        self,
+        block_name: str,
+        missing_variables: list[str],
+        available_variables: list[str],
+    ):
+        """Initialize TemplateValidationError.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block that failed template validation.
+        missing_variables : List[str]
+            List of missing template variable names.
+        available_variables : List[str]
+            List of available template variable names.
+        """
+        self.block_name = block_name
+        self.missing_variables = missing_variables
+        self.available_variables = available_variables
+
+        message = f"Block '{block_name}' template validation failed - missing required variables: {missing_variables}"
+        details = f"Available variables: {available_variables}"
+
+        super().__init__(message, details)
+
+
+class FlowError(SDGHubError):
+    """Base exception class for flow-related errors."""
+
+    pass
+
+
+class FlowValidationError(FlowError):
+    """Raised when flow validation fails."""
+
+    pass
+
+
+class FlowExecutionError(FlowError):
+    """Raised when flow execution fails."""
+
+    pass

sdg_hub/{utils → core/utils}/path_resolution.py

@@ -7,11 +7,11 @@ search paths.
 """
 
 # Standard
-from typing import List, Union
+from typing import Union
 import os
 
 
-def resolve_path(filename: str, search_dirs: Union[str, List[str]]) -> str:
+def resolve_path(filename: str, search_dirs: Union[str, list[str]]) -> str:
     """Resolve a file path relative to one or more search directories.
 
     Files are checked in the following order:

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/atomic_facts.yaml

@@ -0,0 +1,40 @@
+- role: system
+  content: You are an AI assistant knowledgeable about {{domain}} domain. Be accurate but concise in response.
+
+- role: user
+  content: |
+    Please break down the following snippet from an article about {{domain}} into atomic facts.
+
+    1. Makesure each fact is grounded in the given text.
+    2. Include any necessary information needed to explain the fact or concept
+    3. The atomic facts should be as simple as possible, if it's compound sentence, break down one more time
+    4. For clarity, avoid using pronouns like 'it', 'he', 'she', 'this', 'that' etc., and instead use the full names or titles.
+    5. Focus only on key concepts and facts. Skip any question or problems mentioned in the passage.
+
+    To help you understand the task, here is an example:
+    [Passage]
+    The tournament was contested by ten national teams, maintaining the same format used in 2019. After six weeks of round-robin matches, India, South Africa, Australia, and New Zealand finished as the top four and qualified for the knockout stage. In the knockout stage, India and Australia beat New Zealand and South Africa, respectively, to advance to the final, played on 19 November at the Narendra Modi Stadium in Ahmedabad. Australia won the final by six wickets, winning their sixth Cricket World Cup title.
+    [Facts]
+    1. The tournament was contested by ten national teams.
+    2. The tournament maintained the same format used in 2019.
+    3. The round-robin matches lasted for six weeks.
+    4. India finished as one of the top four teams.
+    5. South Africa finished as one of the top four teams.
+    6. Australia finished as one of the top four teams.
+    7. New Zealand finished as one of the top four teams.
+    8. India, South Africa, Australia, and New Zealand qualified for the knockout stage.
+    9. In the knockout stage, India beat New Zealand.
+    10. In the knockout stage, Australia beat South Africa.
+    11. India advanced to the final.
+    12. Australia advanced to the final.
+    13. The final was played on 19 November.
+    14. The final was held at the Narendra Modi Stadium in Ahmedabad.
+    15. Australia won the final by six wickets.
+    16. Australia won their sixth Cricket World Cup title.
+    [End]
+
+    Now it's your turn breakdown following snippet from article about {{domain}} into atomic facts following similar style as above examples
+    [Passage]
+    {{document_outline}}
+    {{document}}
+    [Facts]

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/detailed_summary.yaml

@@ -0,0 +1,13 @@
+- role: system
+  content: You are an AI assistant that is expert at summarizing text.
+
+- role: user
+  content: |
+    Give me detailed summary for below document, making sure all key points are covered.
+
+    Do not add any new information.
+    Do not miss any key points from the provided document
+
+    Document:
+    {{document_outline}}
+    {{document}}

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/evaluate_faithfulness.yaml

@@ -0,0 +1,64 @@
+- role: system
+  content: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+- role: user
+  content: |
+    Determine if the provided information is corroborated by the given context. Respond with YES if the context substantiates the information, even partially. Answer NO if the context does not support the information.
+
+    Guidelines:
+    - Answer YES when the context provides either direct or indirect evidence supporting the information. Indirect evidence may include contextual implications or inferred connections that reasonably support the information.
+    - Answer NO if the context lacks any supportive evidence, clearly contradicts the information, or if the support provided by the context is too vague or speculative to establish a solid connection to the information.
+    - Avoid using "partially" in your response. If the context provides any reasonable support (direct or indirect) for the information, consider it as a YES.
+
+    Strictly answer in this format:
+    [Start of Context]
+    ...
+    [End of Context]
+    [Start of Response]
+    ... 
+    [End of Response]
+    [Start of Explanation]
+    ...
+    [End of Explanation]
+    [Start of Answer]
+    ...
+    [End of Answer]
+
+    Example 1:
+    [Start of Context]
+    An apple pie is a fruit pie with apples as the main filling. It's often served with whipped cream, ice cream, custard, or cheddar cheese. Typically, it has a double crust, with pastry above and below the filling. The upper crust can be solid or latticed.
+    [End of Context]
+    [Start of Response]
+    Apple pie is generally double-crusted.
+    [End of Response]
+    [Start of Explanation]
+    The context directly supports the information by stating that apple pie is "generally double-crusted," which matches the information provided.
+    [End of Explanation]
+    [Start of Answer]
+    YES
+    [End of Answer]
+
+    Example 2:
+    [Start of Context]
+    An apple pie is a fruit pie with apples as the main filling. It's often served with whipped cream, ice cream, custard, or cheddar cheese. Typically, it has a double crust, with pastry above and below the filling. The upper crust can be solid or latticed.
+    [End of Context]
+    [Start of Response]
+    Apple pies taste bad.
+    [End of Response]
+    [Start of Explanation]
+    The context does not provide any information about the taste of apple pies. The statement "Apple pies taste bad" is a subjective opinion and is not supported or mentioned in the given context.
+    [End of Explanation]
+    [Start of Answer]
+    NO
+    [End of Answer]
+
+    Now, based on the above examples and guidelines, determine if the following information is supported by the context provided. Answer YES or NO.
+    * Return the explanation within the [Start of Explanation] and [End of Explanation] tags.
+    * Return the answer between [Start of Answer] and [End of Answer] tags.
+
+    [Start of Context]
+    {{document}}
+    [End of Context]
+    [Start of Response]
+    {{response}}
+    [End of Response]

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/evaluate_question.yaml

@@ -0,0 +1,29 @@
+- role: system
+  content: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+- role: user
+  content: |
+    Given below question can you verify if it meets below requirements and based on them give a rating of 1 if it meets all of them or 0 otherwise.
+
+    Here are the requirements:
+
+    Non-Referential Clarity and Contextual Independence: Ensure that the question is self-explanatory and does not rely on specific, unprovided external content, such as particular documents, specific tables, or detailed datasets. The question should be structured to be understandable and clear without requiring direct access to or knowledge of these specific external sources.
+
+    Subject-Aware Completeness: The question should be crafted to be answerable on its own, given a reasonable level of specialized knowledge in the relevant subject area. It is acceptable and encouraged for the question to require specialized understanding pertinent to the topic; however, it should not depend on unique, external information not provided in the question itself. This distinction allows for questions that necessitate a deep understanding of a subject while ensuring they are not tied to specific external content like a particular dataset or a line in a document.
+
+    Please give your answer as short explanation followed by rating of either 0 or 1 as below.
+
+    * Return a short explanation within the [Start of Explanation] and [End of Explanation] tags.
+    * Return the rating on a binary 0/1 scale between [Start of Rating] and [End of Rating] tags.
+
+    [Start of Question]  
+    {{question}}
+    [End of Question]
+
+    [Start of Explanation]
+    ...  
+    [End of Explanation]
+
+    [Start of Rating]
+    ...
+    [End of Rating]

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/evaluate_relevancy.yaml

@@ -0,0 +1,81 @@
+- role: system
+  content: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+- role: user
+  content: |
+    Your task is to assess the relevance of a given response to a specific query. This evaluation should be conducted methodically by answering two key questions:
+
+    1. Subject Matter Relevance: Does the provided response accurately match the subject matter of the user's query? This question aims to determine if the response is directly related to the main topic or issue presented in the query.
+    2. Focus and Perspective Addressing: Does the provided response effectively address the focus or perspective on the subject matter as outlined in the user's query? This question seeks to evaluate whether the response not only matches the subject matter but also aligns with the specific angle or concern raised by the user.
+
+    For each question, assign a score of 1 point if the response meets the criteria, and 0 points if it does not. After evaluating each question, provide detailed feedback explaining your reasoning behind the scores awarded.
+
+    Conclude your evaluation with a total score as a final result. The total score should represent the sum of points assigned for each question, with a maximum possible score of 2 points.
+    Only evaluate the response based on the above criteria, do not create new questions.
+
+    Example 1:
+    [Start of Question]
+    What is the impact of global warming on polar bears?
+    [End of Question]
+
+    [Start of Response]
+    Global warming leads to melting ice caps, reducing the habitat of polar bears and negatively impacting their hunting grounds.
+    [End of Response]
+
+    [Start of Feedback]
+    - Subject Matter Relevance Score: 1 (The response is directly related to the impact of global warming on polar bears.)
+    - Alignment with Query's Focus Score: 1 (The response specifically addresses how global warming affects polar bears' habitat and hunting grounds.)
+    [End of Feedback]
+
+    [Start of Score]
+    2
+    [End of Score]
+
+    Example 2:
+    [Start of Question]
+    How does photosynthesis work?
+    [End of Question]
+
+    [Start of Response]
+    Plants require sunlight and water to grow.
+    [End of Response]
+
+    [Start of Feedback]
+    - Subject Matter Relevance Score: 0 (The response is related to plant growth, but does not specifically address the process of photosynthesis.)
+    - Alignment with Query's Focus Score: 0 (The response fails to detail the photosynthesis process, missing the specific focus of the query.)
+    [End of Feedback]
+
+    [Start of Score]
+    0
+    [End of Score]
+
+    Example 3:
+    [Start of Question]
+    What are the benefits of electric vehicles?
+    [End of Question]
+
+    [Start of Response]
+    Electric vehicles reduce dependency on fossil fuels and decrease greenhouse gas emissions.
+    [End of Response]
+
+    [Start of Feedback]
+    - Subject Matter Relevance Score: 1 (The response matches the query's subject on the benefits of electric vehicles.)
+    - Alignment with Query's Focus Score: 1 (The response effectively addresses the environmental benefits of electric vehicles, aligning with the query's focus.)
+    [End of Feedback]
+
+    [Start of Score]
+    2
+    [End of Score]
+
+    Begin your response by providing the feedback followed by the score. Be as objective as possible.
+
+    [Start of Question]
+    {{question}}
+    [End of Question]
+
+    [Start of Response]
+    {{response}}
+    [End of Response]
+
+    * Return the feedback within the [Start of Feedback] and [End of Feedback] tags.
+    * Return the final score between [Start of Score] and [End of Score] tags.

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/extractive_summary.yaml

@@ -0,0 +1,13 @@
+- role: system
+  content: You are an AI assistant that is expert at summarizing text.
+
+- role: user
+  content: |
+    Give me detailed extractive summary for below document, making sure all key points are covered.
+
+    Do not add any new information.
+    Do not miss any key points from the provided document
+
+    Document:
+    {{document_outline}}
+    {{document}}

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml

@@ -0,0 +1,191 @@
+metadata:
+  name: "Advanced Document Grounded Question-Answer Generation Flow for Knowledge Tuning"
+  description: "A comprehensive flow that generates high-quality question-answer pairs from input documents using multiple LLM blocks for question generation, answer synthesis, and quality evaluation."
+  version: "1.0.0"
+  author: "SDG Hub Contributors"
+  
+  recommended_models:
+    default: "meta-llama/Llama-3.3-70B-Instruct"
+    compatible: ["microsoft/phi-4", "mistralai/Mixtral-8x7B-Instruct-v0.1"]
+    experimental: []
+  
+  tags:
+    - "question-generation"
+    - "knowledge-extraction"
+    - "qa-pairs"
+    - "document-processing"
+    - "educational"
+  
+  license: "Apache-2.0"
+  min_sdg_hub_version: "0.2.0"
+  
+  dataset_requirements:
+    required_columns:
+      - "document"
+      - "document_outline"
+      - "domain"
+      - "icl_document"
+      - "icl_query_1"
+      - "icl_response_1"
+      - "icl_query_2"
+      - "icl_response_2"
+      - "icl_query_3"
+      - "icl_response_3"
+    description: "Input dataset should contain documents with text content and domain classification. Each document should be substantial enough for meaningful question generation (minimum 100 words recommended)."
+
+blocks:
+  - block_type: DuplicateColumnsBlock
+    block_config:
+      block_name: duplicate_document_col
+      input_cols: {document: base_document}
+
+  - block_type: PromptBuilderBlock
+    block_config:
+      block_name: detailed_summary_prompt
+      input_cols: [document, document_outline]
+      output_cols: summary_prompt
+      prompt_config_path: detailed_summary.yaml
+      format_as_messages: true
+
+  - block_type: LLMChatBlock
+    block_config:
+      block_name: gen_detailed_summary
+      input_cols: summary_prompt
+      output_cols: raw_summary_detailed
+      max_tokens: 2048
+      async_mode: true
+
+  - block_type: TextParserBlock
+    block_config:
+      block_name: parse_detailed_summary
+      input_cols: raw_summary_detailed
+      output_cols: summary_detailed
+      start_tags: [""]
+      end_tags: [""]
+
+  - block_type: PromptBuilderBlock
+    block_config:
+      block_name: atomic_facts_prompt
+      input_cols: [document, document_outline, domain]
+      output_cols: atomic_facts_prompt
+      prompt_config_path: atomic_facts.yaml
+      format_as_messages: true
+
+  - block_type: LLMChatBlock
+    block_config:
+      block_name: gen_atomic_facts
+      input_cols: atomic_facts_prompt
+      output_cols: raw_atomic_facts
+      max_tokens: 2048
+      async_mode: true
+
+  - block_type: TextParserBlock
+    block_config:
+      block_name: parse_atomic_facts
+      input_cols: raw_atomic_facts
+      output_cols: summary_atomic_facts
+      start_tags: [""]
+      end_tags: [""]
+
+  - block_type: PromptBuilderBlock
+    block_config:
+      block_name: extractive_summary_prompt
+      input_cols: [document, document_outline]
+      output_cols: extractive_summary_prompt
+      prompt_config_path: extractive_summary.yaml
+      format_as_messages: true
+
+  - block_type: LLMChatBlock
+    block_config:
+      block_name: gen_extractive_summary
+      input_cols: extractive_summary_prompt
+      output_cols: raw_summary_extractive
+      max_tokens: 2048
+      async_mode: true
+
+  - block_type: TextParserBlock
+    block_config:
+      block_name: parse_extractive_summary
+      input_cols: raw_summary_extractive
+      output_cols: summary_extractive
+      start_tags: [""]
+      end_tags: [""]
+
+  - block_type: MeltColumnsBlock
+    block_config:
+      block_name: melt_summary_columns
+      input_cols: [summary_detailed, summary_extractive, summary_atomic_facts, base_document]
+      output_cols: [summary, dataset_type]
+
+  - block_type: RenameColumnsBlock
+    block_config:
+      block_name: rename_to_document_column
+      input_cols: {document: raw_document, summary: document}
+
+  - block_type: PromptBuilderBlock
+    block_config:
+      block_name: knowledge_generation_prompt
+      input_cols: [domain, document, document_outline, icl_document, icl_query_1, icl_response_1, icl_query_2, icl_response_2, icl_query_3, icl_response_3]
+      output_cols: knowledge_generation_prompt
+      prompt_config_path: generate_questions_responses.yaml
+      format_as_messages: true
+
+  - block_type: LLMChatBlock
+    block_config:
+      block_name: knowledge_generation
+      input_cols: knowledge_generation_prompt
+      output_cols: raw_knowledge_generation
+      temperature: 0.0
+      max_tokens: 2048
+      async_mode: true
+
+  - block_type: TextParserBlock
+    block_config:
+      block_name: parse_knowledge_generation
+      input_cols: raw_knowledge_generation
+      output_cols: [question, response]
+      parsing_pattern: "\\[(?:Question|QUESTION)\\]\\s*(.*?)\\s*\\[(?:Answer|ANSWER)\\]\\s*(.*?)\\s*(?=\\[(?:Question|QUESTION)\\]|$)"
+      parser_cleanup_tags: ["[END]"]
+
+  - block_type: EvaluateFaithfulnessBlock
+    block_config:
+      block_name: eval_faithfulness
+      input_cols: [document, response]
+      output_cols: [faithfulness_explanation, faithfulness_judgment]
+      prompt_config_path: evaluate_faithfulness.yaml
+      filter_value: "YES"
+      operation: eq
+      async_mode: true
+      format_as_messages: true
+      start_tags: ["[Start of Explanation]", "[Start of Answer]"]
+      end_tags: ["[End of Explanation]", "[End of Answer]"]
+
+  - block_type: EvaluateRelevancyBlock
+    block_config:
+      block_name: eval_relevancy
+      input_cols: [question, response]
+      output_cols: [relevancy_explanation, relevancy_score]
+      prompt_config_path: evaluate_relevancy.yaml
+      filter_value: 2.0
+      operation: eq
+      convert_dtype: float
+      max_tokens: 2048
+      async_mode: true
+      format_as_messages: true
+      start_tags: ["[Start of Feedback]", "[Start of Score]"]
+      end_tags: ["[End of Feedback]", "[End of Score]"]
+
+  - block_type: VerifyQuestionBlock
+    block_config:
+      block_name: verify_question
+      input_cols: [question]
+      output_cols: [verification_explanation, verification_rating]
+      prompt_config_path: evaluate_question.yaml
+      filter_value: 1.0
+      operation: ge
+      convert_dtype: float
+      max_tokens: 2048
+      async_mode: true
+      format_as_messages: true
+      start_tags: ["[Start of Explanation]", "[Start of Rating]"]
+      end_tags: ["[End of Explanation]", "[End of Rating]"]