sdg-hub 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml

@@ -61,10 +61,16 @@ blocks:
     temperature: 0.7
     n: 50
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_detailed_summary
+    input_cols: raw_summary
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_detailed_summary
-    input_cols: raw_summary
+    input_cols: extract_detailed_summary_content
     output_cols: summary
     start_tags:
     - ''
@@ -99,10 +105,16 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_questions
+    input_cols: question_list
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_question_list
-    input_cols: question_list
+    input_cols: extract_questions_content
     output_cols: question
     start_tags:
     - '[QUESTION]'
@@ -127,33 +139,61 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_answers
+    input_cols: response_dict
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_response_dict
-    input_cols: response_dict
+    input_cols: extract_answers_content
     output_cols: response
     start_tags:
     - ''
     end_tags:
     - ''
     save_reasoning_content: true
-- block_type: EvaluateFaithfulnessBlock
+- block_type: PromptBuilderBlock
   block_config:
-    block_name: eval_faithfulness
+    block_name: eval_faithful_prompt
     input_cols:
     - document
     - response
-    output_cols:
-    - faithfulness_explanation
-    - faithfulness_judgment
+    output_cols: eval_faithful_prompt
     prompt_config_path: ../../multi_summary_qa/instructlab/evaluate_faithfulness.yaml
-    filter_value: 'YES'
-    operation: eq
-    async_mode: true
     format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: eval_faithful_llm_chat
+    input_cols: eval_faithful_prompt
+    output_cols: eval_faithful_response_dict
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_eval_faithful
+    input_cols: eval_faithful_response_dict
+    extract_content: true
+    
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_eval_faithful
+    input_cols: extract_eval_faithful_content
+    output_cols: 
+    - faithfulness_explanation
+    - faithfulness_judgment
     start_tags:
     - '[Start of Explanation]'
     - '[Start of Answer]'
     end_tags:
     - '[End of Explanation]'
     - '[End of Answer]'
+- block_type: ColumnValueFilterBlock
+  block_config:
+    block_name: eval_faithful_filter
+    input_cols:
+      - faithfulness_judgment
+    filter_value: 'YES'
+    operation: eq

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/doc_direct_qa/flow.yaml

@@ -0,0 +1,159 @@
+metadata:
+  name: Document Based Knowledge Tuning Dataset Generation Flow
+  description: Directly generates QA pairs from the raw document.
+  version: 2.0.0
+  author: SDG Hub Contributors
+  recommended_models:
+    default: openai/gpt-oss-120b
+    compatible:
+    - meta-llama/Llama-3.3-70B-Instruct
+    - microsoft/phi-4
+    - mistralai/Mixtral-8x7B-Instruct-v0.1
+    experimental: []
+  tags:
+  - knowledge-tuning
+  - document-internalization
+  - question-generation
+  - qa-pairs
+  - detailed-summaries
+  license: Apache-2.0
+  min_sdg_hub_version: 0.2.0
+  dataset_requirements:
+    required_columns:
+    - document
+    - document_outline
+    - domain
+    - icl_document
+    - icl_query_1
+    - icl_query_2
+    - icl_query_3
+    description: 'Input dataset should contain documents with text content and domain classification. Each document should be substantial enough for meaningful question generation (minimum 100 words recommended). The flow generates three types
+      of summaries: detailed (n=20), extractive (n=10), and key facts (n=50), each producing corresponding QA pairs designed to help LLMs internalize document knowledge for knowledge tuning.'
+  output_columns:
+  - question
+  - response
+  - raw_document
+  - faithfulness_explanation
+  - faithfulness_judgment
+  id: stellar-peak-605
+blocks:
+- block_type: DuplicateColumnsBlock
+  block_config:
+    block_name: duplicate_document_col
+    input_cols:
+      document: base_document
+- block_type: PromptBuilderBlock
+  block_config:
+    block_name: question_generation_prompt
+    input_cols:
+    - domain
+    - document
+    - document_outline
+    - icl_document
+    - icl_query_1
+    - icl_query_2
+    - icl_query_3
+    output_cols: question_generation_prompt
+    prompt_config_path: ../generate_question_list.yaml
+    format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: question_generation
+    input_cols: question_generation_prompt
+    output_cols: question_list
+    max_tokens: 256
+    temperature: 1.0
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_questions
+    input_cols: question_list
+    extract_content: true
+    expand_lists: true
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_question_list
+    input_cols: extract_questions_content
+    output_cols: question
+    start_tags:
+    - '[QUESTION]'
+    end_tags:
+    - '[END]'
+- block_type: PromptBuilderBlock
+  block_config:
+    block_name: answer_generation_prompt
+    input_cols:
+    - question
+    - document
+    - document_outline
+    output_cols: answer_generation_prompt
+    prompt_config_path: ../generate_answers.yaml
+    format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: answer_generation
+    input_cols: answer_generation_prompt
+    output_cols: response_dict
+    max_tokens: 4096
+    temperature: 1.0
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_answer
+    input_cols: response_dict
+    extract_content: true
+    expand_lists: true
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_response_dict
+    input_cols: extract_answer_content
+    output_cols: response
+    start_tags:
+    - ''
+    end_tags:
+    - ''
+    save_reasoning_content: true
+- block_type: PromptBuilderBlock
+  block_config:
+    block_name: eval_faithful_prompt
+    input_cols:
+    - document
+    - response
+    output_cols: eval_faithful_prompt
+    prompt_config_path: ../../multi_summary_qa/instructlab/evaluate_faithfulness.yaml
+    format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: eval_faithful_llm_chat
+    input_cols: eval_faithful_prompt
+    output_cols: eval_faithful_response_dict
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_eval_faithful
+    input_cols: eval_faithful_response_dict
+    extract_content: true
+    
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_eval_faithful
+    input_cols: extract_eval_faithful_content
+    output_cols: 
+    - faithfulness_explanation
+    - faithfulness_judgment
+    start_tags:
+    - '[Start of Explanation]'
+    - '[Start of Answer]'
+    end_tags:
+    - '[End of Explanation]'
+    - '[End of Answer]'
+- block_type: ColumnValueFilterBlock
+  block_config:
+    block_name: eval_faithful_filter
+    input_cols:
+      - faithfulness_judgment
+    filter_value: 'YES'
+    operation: eq

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml

@@ -63,10 +63,16 @@ blocks:
     temperature: 0.7
     n: 50
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_extractive_summary
+    input_cols: raw_summary
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_extractive_summary
-    input_cols: raw_summary
+    input_cols: extract_extractive_summary_content
     output_cols: summary
     start_tags:
     - ''
@@ -101,10 +107,16 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_questions
+    input_cols: question_list
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_question_list
-    input_cols: question_list
+    input_cols: extract_questions_content
     output_cols: question
     start_tags:
     - '[QUESTION]'
@@ -129,33 +141,61 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_answers
+    input_cols: response_dict
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_response_dict
-    input_cols: response_dict
+    input_cols: extract_answers_content
     output_cols: response
     start_tags:
     - ''
     end_tags:
     - ''
     save_reasoning_content: true
-- block_type: EvaluateFaithfulnessBlock
+- block_type: PromptBuilderBlock
   block_config:
-    block_name: eval_faithfulness
+    block_name: eval_faithful_prompt
     input_cols:
     - document
     - response
-    output_cols:
-    - faithfulness_explanation
-    - faithfulness_judgment
+    output_cols: eval_faithful_prompt
     prompt_config_path: ../../multi_summary_qa/instructlab/evaluate_faithfulness.yaml
-    filter_value: 'YES'
-    operation: eq
-    async_mode: true
     format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: eval_faithful_llm_chat
+    input_cols: eval_faithful_prompt
+    output_cols: eval_faithful_response_dict
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_eval_faithful
+    input_cols: eval_faithful_response_dict
+    extract_content: true
+    
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_eval_faithful
+    input_cols: extract_eval_faithful_content
+    output_cols: 
+    - faithfulness_explanation
+    - faithfulness_judgement
     start_tags:
     - '[Start of Explanation]'
     - '[Start of Answer]'
     end_tags:
     - '[End of Explanation]'
     - '[End of Answer]'
+- block_type: ColumnValueFilterBlock
+  block_config:
+    block_name: eval_faithful_filter
+    input_cols:
+      - faithfulness_judgement
+    filter_value: 'YES'
+    operation: eq

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml

@@ -50,10 +50,16 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_atomic_facts
+    input_cols: raw_summary
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_atomic_facts
-    input_cols: raw_summary
+    input_cols: extract_atomic_facts_content
     output_cols: atomic_facts
     start_tags:
     - '### Key Facts With Context'
@@ -89,10 +95,16 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_key_fact_qa
+    input_cols: raw_key_fact_qa
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_key_fact_qa
-    input_cols: raw_key_fact_qa
+    input_cols: extract_key_fact_qa_content
     output_cols:
     - question
     - response

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml

@@ -54,11 +54,19 @@ blocks:
       output_cols: raw_summary_detailed
       max_tokens: 2048
       async_mode: true
+      n: 2
+
+  - block_type: LLMParserBlock
+    block_config:
+      block_name: detailed_summary
+      input_cols: raw_summary_detailed
+      extract_content: true
+      extract_reasoning_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_detailed_summary
-      input_cols: raw_summary_detailed
+      input_cols: detailed_summary_content
       output_cols: summary_detailed
       start_tags: [""]
       end_tags: [""]
@@ -78,10 +86,16 @@ blocks:
       max_tokens: 2048
       async_mode: true
 
+  - block_type: LLMParserBlock
+    block_config:
+      block_name: atomic_facts
+      input_cols: raw_atomic_facts
+      extract_content: true
+
   - block_type: TextParserBlock
     block_config:
       block_name: parse_atomic_facts
-      input_cols: raw_atomic_facts
+      input_cols: atomic_facts_content
       output_cols: summary_atomic_facts
       start_tags: [""]
       end_tags: [""]
@@ -101,10 +115,16 @@ blocks:
       max_tokens: 2048
       async_mode: true
 
+  - block_type: LLMParserBlock
+    block_config:
+      block_name: extractive_summary
+      input_cols: raw_summary_extractive
+      extract_content: true
+
   - block_type: TextParserBlock
     block_config:
       block_name: parse_extractive_summary
-      input_cols: raw_summary_extractive
+      input_cols: extractive_summary_content
       output_cols: summary_extractive
       start_tags: [""]
       end_tags: [""]
@@ -136,50 +156,150 @@ blocks:
       max_tokens: 2048
       async_mode: true
 
+  - block_type: LLMParserBlock
+    block_config:
+      block_name: get_knowledge_generation
+      input_cols: raw_knowledge_generation
+      extract_content: true
+
   - block_type: TextParserBlock
     block_config:
       block_name: parse_knowledge_generation
-      input_cols: raw_knowledge_generation
+      input_cols: get_knowledge_generation_content
       output_cols: [question, response]
       parsing_pattern: "\\[(?:Question|QUESTION)\\]\\s*(.*?)\\s*\\[(?:Answer|ANSWER)\\]\\s*(.*?)\\s*(?=\\[(?:Question|QUESTION)\\]|$)"
       parser_cleanup_tags: ["[END]"]
 
-  - block_type: EvaluateFaithfulnessBlock
+  - block_type: PromptBuilderBlock
     block_config:
-      block_name: eval_faithfulness
+      block_name: eval_faithful_prompt
       input_cols: [document, response]
-      output_cols: [faithfulness_explanation, faithfulness_judgment]
+      output_cols: eval_faithful_prompt
       prompt_config_path: evaluate_faithfulness.yaml
+      format_as_messages: true
+
+  - block_type: LLMChatBlock
+    block_config:
+      block_name: eval_faithful_llm_chat
+      input_cols: eval_faithful_prompt
+      output_cols: eval_faithful_response_dict
+      max_tokens: 2048
+      n: 1
+      async_mode: true
+
+  - block_type: LLMParserBlock
+    block_config:
+      block_name: extract_eval_faithful
+      input_cols: eval_faithful_response_dict
+      extract_content: true
+      
+  - block_type: TextParserBlock
+    block_config:
+      block_name: parse_eval_faithful
+      input_cols: extract_eval_faithful_content
+      output_cols: 
+      - faithfulness_explanation
+      - faithfulness_judgment
+      start_tags:
+      - '[Start of Explanation]'
+      - '[Start of Answer]'
+      end_tags:
+      - '[End of Explanation]'
+      - '[End of Answer]'
+
+  - block_type: ColumnValueFilterBlock
+    block_config:
+      block_name: eval_faithful_filter
+      input_cols:
+        - faithfulness_judgment
       filter_value: "YES"
       operation: eq
-      async_mode: true
-      start_tags: ["[Start of Explanation]", "[Start of Answer]"]
-      end_tags: ["[End of Explanation]", "[End of Answer]"]
 
-  - block_type: EvaluateRelevancyBlock
+  - block_type: PromptBuilderBlock
     block_config:
-      block_name: eval_relevancy
-      input_cols: [question, response]
-      output_cols: [relevancy_explanation, relevancy_score]
+      block_name: eval_relevancy_prompt
+      input_cols:
+      - question
+      - response
+      output_cols: eval_relevancy_prompt
       prompt_config_path: evaluate_relevancy.yaml
+      format_as_messages: true
+  - block_type: LLMChatBlock
+    block_config:
+      block_name: eval_relevancy_llm_chat
+      input_cols: eval_relevancy_prompt
+      output_cols: eval_relevancy_response_dict
+      max_tokens: 2048
+      n: 1
+      async_mode: true
+  - block_type: LLMParserBlock
+    block_config:
+      block_name: extract_eval_relevancy
+      input_cols: eval_relevancy_response_dict
+      extract_content: true
+      
+  - block_type: TextParserBlock
+    block_config:
+      block_name: parse_eval_relevancy
+      input_cols: extract_eval_relevancy_content
+      output_cols: 
+      - relevancy_explanation
+      - relevancy_score
+      start_tags:
+      - '[Start of Feedback]'
+      - '[Start of Score]'
+      end_tags:
+      - '[End of Feedback]'
+      - '[End of Score]'
+  - block_type: ColumnValueFilterBlock
+    block_config:
+      block_name: eval_relevancy_filter
+      input_cols:
+        - relevancy_score
       filter_value: 2.0
       operation: eq
       convert_dtype: float
-      max_tokens: 2048
-      async_mode: true
-      start_tags: ["[Start of Feedback]", "[Start of Score]"]
-      end_tags: ["[End of Feedback]", "[End of Score]"]
 
-  - block_type: VerifyQuestionBlock
+  - block_type: PromptBuilderBlock
     block_config:
-      block_name: verify_question
-      input_cols: [question]
-      output_cols: [verification_explanation, verification_rating]
+      block_name: verify_question_prompt
+      input_cols:
+      - question
+      output_cols: verify_question_prompt
       prompt_config_path: evaluate_question.yaml
+      format_as_messages: true
+  - block_type: LLMChatBlock
+    block_config:
+      block_name: verify_question_llm_chat
+      input_cols: verify_question_prompt
+      output_cols: verify_question_response_dict
+      max_tokens: 2048
+      n: 1
+      async_mode: true
+  - block_type: LLMParserBlock
+    block_config:
+      block_name: extract_verify_question
+      input_cols: verify_question_response_dict
+      extract_content: true
+      
+  - block_type: TextParserBlock
+    block_config:
+      block_name: parse_verify_question
+      input_cols: extract_verify_question_content
+      output_cols: 
+      - verification_explanation
+      - verification_rating
+      start_tags:
+      - '[Start of Explanation]'
+      - '[Start of Rating]'
+      end_tags:
+      - '[End of Explanation]'
+      - '[End of Rating]'
+  - block_type: ColumnValueFilterBlock
+    block_config:
+      block_name: verify_question_filter
+      input_cols:
+        - verification_rating
       filter_value: 1.0
       operation: ge
       convert_dtype: float
-      max_tokens: 2048
-      async_mode: true
-      start_tags: ["[Start of Explanation]", "[Start of Rating]"]
-      end_tags: ["[End of Explanation]", "[End of Rating]"]

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/atomic_facts_ja.yaml

@@ -0,0 +1,41 @@
+- role: system
+  content: You are an AI assistant knowledgeable about {{domain}} domain. Be accurate but concise in response.
+
+- role: user
+  content: |
+    Please break down the following snippet from an article about {{domain}} into atomic facts.
+
+    1. Make sure each fact is grounded in the given text.
+    2. Include any necessary information needed to explain the fact or concept
+    3. The atomic facts should be as simple as possible, if it's compound sentence, break down one more time
+    4. For clarity, avoid using pronouns like 'it', 'he', 'she', 'this', 'that' etc., and instead use the full names or titles.
+    5. Focus only on key concepts and facts. Skip any question or problems mentioned in the passage.
+    6. Output the response in Japanese.
+
+    To help you understand the task, here is an example:
+    [Passage]
+    The tournament was contested by ten national teams, maintaining the same format used in 2019. After six weeks of round-robin matches, India, South Africa, Australia, and New Zealand finished as the top four and qualified for the knockout stage. In the knockout stage, India and Australia beat New Zealand and South Africa, respectively, to advance to the final, played on 19 November at the Narendra Modi Stadium in Ahmedabad. Australia won the final by six wickets, winning their sixth Cricket World Cup title.
+    [Facts]
+    1. The tournament was contested by ten national teams.
+    2. The tournament maintained the same format used in 2019.
+    3. The round-robin matches lasted for six weeks.
+    4. India finished as one of the top four teams.
+    5. South Africa finished as one of the top four teams.
+    6. Australia finished as one of the top four teams.
+    7. New Zealand finished as one of the top four teams.
+    8. India, South Africa, Australia, and New Zealand qualified for the knockout stage.
+    9. In the knockout stage, India beat New Zealand.
+    10. In the knockout stage, Australia beat South Africa.
+    11. India advanced to the final.
+    12. Australia advanced to the final.
+    13. The final was played on 19 November.
+    14. The final was held at the Narendra Modi Stadium in Ahmedabad.
+    15. Australia won the final by six wickets.
+    16. Australia won their sixth Cricket World Cup title.
+    [End]
+
+    Now it's your turn breakdown following snippet from article about {{domain}} into atomic facts following similar style as above examples
+    [Passage]
+    {{document_outline}}
+    {{document}}
+    [Facts]

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/detailed_summary_ja.yaml

@@ -0,0 +1,14 @@
+- role: system
+  content: You are an AI assistant that is expert at summarizing text.
+
+- role: user
+  content: |
+    Give me detailed summary for below document, making sure all key points are covered.
+
+    Do not add any new information.
+    Do not miss any key points from the provided document.
+    Output the response in Japanese.
+
+    Document:
+    {{document_outline}}
+    {{document}}