sdg-hub 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

sdg_hub/core/utils/datautils.py

@@ -1,5 +1,6 @@
 # Third Party
 from datasets import Dataset, concatenate_datasets
+import numpy as np
 
 # Local
 from .error_handling import FlowValidationError
@@ -39,28 +40,45 @@ def validate_no_duplicates(dataset: Dataset) -> None:
 
     df = dataset.to_pandas()
 
-    # Try pandas duplicated() first - only convert types if we hit unhashable error
-    try:
-        duplicate_count = int(df.duplicated(keep="first").sum())
-    except TypeError as e:
-        if "unhashable type" in str(e):
-            # Convert unhashable types to tuples so pandas can hash them
-            for col in df.columns:
-                if df[col].dtype == "object":  # Only check object columns
-                    df[col] = df[col].apply(
-                        lambda x: (
-                            tuple(sorted(x.items()))
-                            if isinstance(x, dict)
-                            else tuple(x)
-                            if hasattr(x, "__iter__")
-                            and not isinstance(x, (str, bytes))
-                            else x
-                        )
-                    )
-            duplicate_count = int(df.duplicated(keep="first").sum())
-        else:
-            raise  # Re-raise if it's a different TypeError
-
+    def is_hashable(x):
+        try:
+            hash(x)
+            return True
+        except TypeError:
+            return False
+
+    def make_hashable(x):
+        if is_hashable(x):
+            # int, float, str, bytes, None etc. are already hashable
+            return x
+        if isinstance(x, np.ndarray):
+            if x.ndim == 0:
+                return make_hashable(x.item())
+            return tuple(make_hashable(i) for i in x)
+        if isinstance(x, dict):
+            # sort robustly even with heterogeneous key types
+            return tuple(
+                sorted(
+                    ((k, make_hashable(v)) for k, v in x.items()),
+                    key=lambda kv: repr(kv[0]),
+                )
+            )
+        if isinstance(x, (set, frozenset)):
+            # order‑insensitive
+            return frozenset(make_hashable(i) for i in x)
+        if hasattr(x, "__iter__"):
+            # lists, tuples, custom iterables
+            return tuple(make_hashable(i) for i in x)
+        # last‑resort fallback to a stable representation
+        return repr(x)
+
+    # Apply to the whole dataframe to ensure every cell is hashable
+    if hasattr(df, "map"):
+        df = df.map(make_hashable)
+    else:
+        df = df.applymap(make_hashable)
+
+    duplicate_count = int(df.duplicated(keep="first").sum())
     if duplicate_count > 0:
         raise FlowValidationError(
             f"Input dataset contains {duplicate_count} duplicate rows. "

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml

@@ -61,10 +61,16 @@ blocks:
     temperature: 0.7
     n: 50
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_detailed_summary
+    input_cols: raw_summary
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_detailed_summary
-    input_cols: raw_summary
+    input_cols: extract_detailed_summary_content
     output_cols: summary
     start_tags:
     - ''
@@ -99,10 +105,16 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_questions
+    input_cols: question_list
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_question_list
-    input_cols: question_list
+    input_cols: extract_questions_content
     output_cols: question
     start_tags:
     - '[QUESTION]'
@@ -127,33 +139,61 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_answers
+    input_cols: response_dict
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_response_dict
-    input_cols: response_dict
+    input_cols: extract_answers_content
     output_cols: response
     start_tags:
     - ''
     end_tags:
     - ''
     save_reasoning_content: true
-- block_type: EvaluateFaithfulnessBlock
+- block_type: PromptBuilderBlock
   block_config:
-    block_name: eval_faithfulness
+    block_name: eval_faithful_prompt
     input_cols:
     - document
     - response
-    output_cols:
-    - faithfulness_explanation
-    - faithfulness_judgment
+    output_cols: eval_faithful_prompt
     prompt_config_path: ../../multi_summary_qa/instructlab/evaluate_faithfulness.yaml
-    filter_value: 'YES'
-    operation: eq
-    async_mode: true
     format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: eval_faithful_llm_chat
+    input_cols: eval_faithful_prompt
+    output_cols: eval_faithful_response_dict
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_eval_faithful
+    input_cols: eval_faithful_response_dict
+    extract_content: true
+    
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_eval_faithful
+    input_cols: extract_eval_faithful_content
+    output_cols: 
+    - faithfulness_explanation
+    - faithfulness_judgment
     start_tags:
     - '[Start of Explanation]'
     - '[Start of Answer]'
     end_tags:
     - '[End of Explanation]'
     - '[End of Answer]'
+- block_type: ColumnValueFilterBlock
+  block_config:
+    block_name: eval_faithful_filter
+    input_cols:
+      - faithfulness_judgment
+    filter_value: 'YES'
+    operation: eq

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/doc_direct_qa/flow.yaml

@@ -0,0 +1,159 @@
+metadata:
+  name: Document Based Knowledge Tuning Dataset Generation Flow
+  description: Directly generates QA pairs from the raw document.
+  version: 2.0.0
+  author: SDG Hub Contributors
+  recommended_models:
+    default: openai/gpt-oss-120b
+    compatible:
+    - meta-llama/Llama-3.3-70B-Instruct
+    - microsoft/phi-4
+    - mistralai/Mixtral-8x7B-Instruct-v0.1
+    experimental: []
+  tags:
+  - knowledge-tuning
+  - document-internalization
+  - question-generation
+  - qa-pairs
+  - detailed-summaries
+  license: Apache-2.0
+  min_sdg_hub_version: 0.2.0
+  dataset_requirements:
+    required_columns:
+    - document
+    - document_outline
+    - domain
+    - icl_document
+    - icl_query_1
+    - icl_query_2
+    - icl_query_3
+    description: 'Input dataset should contain documents with text content and domain classification. Each document should be substantial enough for meaningful question generation (minimum 100 words recommended). The flow generates three types
+      of summaries: detailed (n=20), extractive (n=10), and key facts (n=50), each producing corresponding QA pairs designed to help LLMs internalize document knowledge for knowledge tuning.'
+  output_columns:
+  - question
+  - response
+  - raw_document
+  - faithfulness_explanation
+  - faithfulness_judgment
+  id: stellar-peak-605
+blocks:
+- block_type: DuplicateColumnsBlock
+  block_config:
+    block_name: duplicate_document_col
+    input_cols:
+      document: base_document
+- block_type: PromptBuilderBlock
+  block_config:
+    block_name: question_generation_prompt
+    input_cols:
+    - domain
+    - document
+    - document_outline
+    - icl_document
+    - icl_query_1
+    - icl_query_2
+    - icl_query_3
+    output_cols: question_generation_prompt
+    prompt_config_path: ../generate_question_list.yaml
+    format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: question_generation
+    input_cols: question_generation_prompt
+    output_cols: question_list
+    max_tokens: 256
+    temperature: 1.0
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_questions
+    input_cols: question_list
+    extract_content: true
+    expand_lists: true
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_question_list
+    input_cols: extract_questions_content
+    output_cols: question
+    start_tags:
+    - '[QUESTION]'
+    end_tags:
+    - '[END]'
+- block_type: PromptBuilderBlock
+  block_config:
+    block_name: answer_generation_prompt
+    input_cols:
+    - question
+    - document
+    - document_outline
+    output_cols: answer_generation_prompt
+    prompt_config_path: ../generate_answers.yaml
+    format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: answer_generation
+    input_cols: answer_generation_prompt
+    output_cols: response_dict
+    max_tokens: 4096
+    temperature: 1.0
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_answer
+    input_cols: response_dict
+    extract_content: true
+    expand_lists: true
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_response_dict
+    input_cols: extract_answer_content
+    output_cols: response
+    start_tags:
+    - ''
+    end_tags:
+    - ''
+    save_reasoning_content: true
+- block_type: PromptBuilderBlock
+  block_config:
+    block_name: eval_faithful_prompt
+    input_cols:
+    - document
+    - response
+    output_cols: eval_faithful_prompt
+    prompt_config_path: ../../multi_summary_qa/instructlab/evaluate_faithfulness.yaml
+    format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: eval_faithful_llm_chat
+    input_cols: eval_faithful_prompt
+    output_cols: eval_faithful_response_dict
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_eval_faithful
+    input_cols: eval_faithful_response_dict
+    extract_content: true
+    
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_eval_faithful
+    input_cols: extract_eval_faithful_content
+    output_cols: 
+    - faithfulness_explanation
+    - faithfulness_judgment
+    start_tags:
+    - '[Start of Explanation]'
+    - '[Start of Answer]'
+    end_tags:
+    - '[End of Explanation]'
+    - '[End of Answer]'
+- block_type: ColumnValueFilterBlock
+  block_config:
+    block_name: eval_faithful_filter
+    input_cols:
+      - faithfulness_judgment
+    filter_value: 'YES'
+    operation: eq

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml

@@ -63,10 +63,16 @@ blocks:
     temperature: 0.7
     n: 50
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_extractive_summary
+    input_cols: raw_summary
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_extractive_summary
-    input_cols: raw_summary
+    input_cols: extract_extractive_summary_content
     output_cols: summary
     start_tags:
     - ''
@@ -101,10 +107,16 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_questions
+    input_cols: question_list
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_question_list
-    input_cols: question_list
+    input_cols: extract_questions_content
     output_cols: question
     start_tags:
     - '[QUESTION]'
@@ -129,33 +141,61 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_answers
+    input_cols: response_dict
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_response_dict
-    input_cols: response_dict
+    input_cols: extract_answers_content
     output_cols: response
     start_tags:
     - ''
     end_tags:
     - ''
     save_reasoning_content: true
-- block_type: EvaluateFaithfulnessBlock
+- block_type: PromptBuilderBlock
   block_config:
-    block_name: eval_faithfulness
+    block_name: eval_faithful_prompt
     input_cols:
     - document
     - response
-    output_cols:
-    - faithfulness_explanation
-    - faithfulness_judgment
+    output_cols: eval_faithful_prompt
     prompt_config_path: ../../multi_summary_qa/instructlab/evaluate_faithfulness.yaml
-    filter_value: 'YES'
-    operation: eq
-    async_mode: true
     format_as_messages: true
+- block_type: LLMChatBlock
+  block_config:
+    block_name: eval_faithful_llm_chat
+    input_cols: eval_faithful_prompt
+    output_cols: eval_faithful_response_dict
+    n: 1
+    async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_eval_faithful
+    input_cols: eval_faithful_response_dict
+    extract_content: true
+    
+- block_type: TextParserBlock
+  block_config:
+    block_name: parse_eval_faithful
+    input_cols: extract_eval_faithful_content
+    output_cols: 
+    - faithfulness_explanation
+    - faithfulness_judgement
     start_tags:
     - '[Start of Explanation]'
     - '[Start of Answer]'
     end_tags:
     - '[End of Explanation]'
     - '[End of Answer]'
+- block_type: ColumnValueFilterBlock
+  block_config:
+    block_name: eval_faithful_filter
+    input_cols:
+      - faithfulness_judgement
+    filter_value: 'YES'
+    operation: eq

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml

@@ -50,10 +50,16 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_atomic_facts
+    input_cols: raw_summary
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_atomic_facts
-    input_cols: raw_summary
+    input_cols: extract_atomic_facts_content
     output_cols: atomic_facts
     start_tags:
     - '### Key Facts With Context'
@@ -89,10 +95,16 @@ blocks:
     temperature: 0.7
     n: 1
     async_mode: true
+- block_type: LLMParserBlock
+  block_config:
+    block_name: extract_key_fact_qa
+    input_cols: raw_key_fact_qa
+    extract_content: true
+    expand_lists: true
 - block_type: TextParserBlock
   block_config:
     block_name: parse_key_fact_qa
-    input_cols: raw_key_fact_qa
+    input_cols: extract_key_fact_qa_content
     output_cols:
     - question
     - response