sdg-hub 0.1.0a4__py3-none-any.whl → 0.1.1__py3-none-any.whl

sdg_hub/checkpointer.py

@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: Apache-2.0
+# Standard
+from typing import Optional, List
+import uuid
+
+# Third Party
+from datasets import Dataset, load_dataset
+from datasets.data_files import EmptyDatasetError
+
+# Local
+from .logger_config import setup_logger
+from .utils.datautils import safe_concatenate_datasets
+
+logger = setup_logger(__name__)
+
+
+class Checkpointer:
+    """
+    Handles checkpointing functionality for SDG data generation.
+    Manages saving intermediate results and loading existing checkpoints.
+    """
+    
+    def __init__(self, checkpoint_dir: Optional[str] = None, save_freq: Optional[int] = None):
+        """
+        Initialize the Checkpointer.
+        
+        Args:
+            checkpoint_dir: Directory to save/load checkpoints. If None, checkpointing is disabled.
+            save_freq: Frequency for saving intermediate checkpoints during batch processing.
+        """
+        self.checkpoint_dir = checkpoint_dir
+        self.save_freq = save_freq
+    
+    def load_existing_data(self, seed_dataset: Dataset) -> tuple[Dataset, Optional[Dataset]]:
+        """
+        Load existing checkpoint data and determine what still needs to be generated.
+        
+        Args:
+            seed_dataset: Original input dataset
+            
+        Returns:
+            Tuple of (remaining_data_to_generate, pre_generated_data)
+            If no checkpoints exist, returns (seed_dataset, None)
+        """
+        if self.checkpoint_dir is None:
+            return seed_dataset, None
+            
+        try:
+            # Load existing checkpoints
+            pre_generated_data = load_dataset(
+                "json", data_dir=self.checkpoint_dir, split="train"
+            )
+            logger.info(
+                f"Loading existing checkpoints from {self.checkpoint_dir}, "
+                f"with {pre_generated_data.num_rows} rows"
+            )
+            
+            # Find missing data that still needs to be generated
+            missing_data = self._get_missing_data(seed_dataset, pre_generated_data)
+            
+            if missing_data.num_rows == 0:
+                logger.info(
+                    f"All seed data has been generated, no missing rows found, "
+                    f"returning data from {self.checkpoint_dir}"
+                )
+                return missing_data, pre_generated_data
+                
+            logger.info(f"Found {missing_data.num_rows} missing rows in the dataset")
+            return missing_data, pre_generated_data
+            
+        except EmptyDatasetError:
+            logger.info(
+                f"No existing checkpoints found in {self.checkpoint_dir}, "
+                f"generating from scratch"
+            )
+            return seed_dataset, None
+    
+    def _get_missing_data(self, seed_data: Dataset, generated_data: Dataset) -> Dataset:
+        """
+        Identify rows in seed_data that are not present in generated_data.
+        
+        Args:
+            seed_data: Original seed dataset
+            generated_data: Previously generated dataset
+            
+        Returns:
+            Dataset containing only the missing rows from seed_data
+        """
+        # Get the common columns between the two datasets
+        common_columns = list(
+            set(seed_data.column_names) & set(generated_data.column_names)
+        )
+
+        # Extract the relevant data based on common columns
+        seed_data_common = seed_data.select_columns(common_columns)
+        generated_data_common = generated_data.select_columns(common_columns)
+
+        # Convert to Pandas DataFrames for easier comparison
+        seed_df = seed_data_common.to_pandas()
+        generated_df = generated_data_common.to_pandas()
+
+        # Identify missing rows
+        missing_df = seed_df[
+            ~seed_df.apply(tuple, 1).isin(generated_df.apply(tuple, 1))
+        ]
+
+        # Convert back to Dataset
+        missing_data = Dataset.from_pandas(missing_df, preserve_index=False)
+
+        return missing_data
+    
+    def save_intermediate_checkpoint(self, dataset: Dataset) -> None:
+        """
+        Save intermediate checkpoint data to disk.
+        
+        Args:
+            dataset: Dataset to save as checkpoint
+        """
+        if self.checkpoint_dir is None:
+            return
+            
+        checkpoint_id = uuid.uuid4().hex
+        checkpoint_file = f"{self.checkpoint_dir}/data_checkpoint_{checkpoint_id}.jsonl"
+        logger.info(f"Saving checkpoint to {checkpoint_file}")
+        dataset.to_json(checkpoint_file, orient="records", lines=True)
+    
+    def should_save_checkpoint(self, current_split_index: int) -> bool:
+        """
+        Determine if a checkpoint should be saved based on save frequency.
+        
+        Args:
+            current_split_index: Current split index (0-based)
+            
+        Returns:
+            True if checkpoint should be saved, False otherwise
+        """
+        if self.save_freq is None or self.checkpoint_dir is None:
+            return False
+        return (current_split_index + 1) % self.save_freq == 0

sdg_hub/configs/annotations/detailed_annotations.yaml

@@ -0,0 +1,28 @@
+system: You are an expert text classifier trained to label questions from online forums. 
+introduction: "Task Description: You will be given a text and you need to annotate it with one of the following categories: World, Sports, Business, Sci/Tech, Economy"
+principles: |
+  Please follow these rules when performing the classification:
+  - Focus on the main topic, not peripheral mentions
+  - Choose the most specific applicable category
+  - Only choose category label per question
+examples: |
+  Text: Bangladesh paralysed by strikes Opposition activists have brought many towns and cities in Bangladesh to a halt, the day after 18 people died in explosions at a political rally.
+  Category: World
+
+  Text: Desiring Stability Redskins coach Joe Gibbs expects few major personnel changes in the offseason and wants to instill a culture of stability in Washington.
+  Category: Sports
+
+  Text: A Cosmic Storm: When Galaxy Clusters Collide Astronomers have found what they are calling the perfect cosmic storm, a galaxy cluster pile-up so powerful its energy output is second only to the Big Bang.
+  Category: Sci/Tech
+
+  Text: Economy builds steam in KC Fed district The economy continued to strengthen in September and early October in the Great Plains and Rocky Mountain regions covered by the Tenth Federal Reserve District, the Federal Reserve Bank of Kansas City said Wednesday.
+  Category: Economy
+
+generation: |
+  Here is the query for annotation:
+  
+  Text: {{text}}
+  Category: 
+  
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/annotations/simple_annotations.yaml

@@ -0,0 +1,9 @@
+system: null
+introduction: "Task Description: Data Annotation"
+principles: null
+examples: null
+generation: |
+  Here is the query for annotation:
+  {{text}}
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/knowledge/atomic_facts.yaml

@@ -37,6 +37,7 @@ examples: |
 generation: |
   Now it's your turn breakdown following snippet from article about {{domain}} into atomic facts following similar style as above examples
   [Passage]
+  {{document_outline}}
   {{document}}
   [Facts]
 

sdg_hub/configs/knowledge/detailed_summary.yaml

@@ -11,6 +11,7 @@ examples: ""
 
 generation: |
   Document:
+  {{document_outline}}
   {{document}}
 
 start_tags: [""]

sdg_hub/configs/knowledge/extractive_summary.yaml

@@ -11,6 +11,7 @@ examples: ""
 
 generation: |
   Document:
+  {{document_outline}}
   {{document}}
 
 start_tags: [""]

sdg_hub/configs/knowledge/generate_questions.yaml

@@ -0,0 +1,82 @@
+# This YAML file defines a prompt template for generating educational Q&A pairs from textbook content.
+# The prompt is designed to create comprehensive, domain-specific questions and answers that
+# effectively teach and reinforce key concepts from educational materials.
+#
+# Structure:
+# - system: Sets the AI's role as a knowledgeable assistant
+# - introduction: Main instruction for creating Q&A pairs from textbook chapters
+# - principles: Detailed guidelines for question formulation and educational value
+# - examples: Example Q&A pairs showing expected format and style
+# - generation: Template for the document to be used for Q&A generation
+#
+# Key Features:
+# - Domain-specific question generation (science, legal, etc.)
+# - Multiple difficulty levels and question types
+# - Self-contained questions without external references
+# - Focus on key concepts and learning objectives
+# - Educational value and teaching effectiveness
+#
+# Question Guidelines:
+# - Must be self-contained and independently answerable
+# - Should cover basic recall to advanced comprehension
+# - Include multiple-choice, short answer, and essay types
+# - Align with chapter learning objectives
+# - Avoid references to specific sections or figures
+#
+# Response Format:
+# - Questions and answers are clearly separated
+# - Each response ends with [End] tag
+# - [UNANSWERABLE] for unsuitable content
+#
+# Usage:
+# This prompt is used to generate educational Q&A pairs that effectively teach
+# and reinforce concepts from textbook chapters while maintaining educational
+# value and accessibility.
+
+system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+introduction: Develop a series of educational questions from a chapter in a {{domain}} textbook. 
+
+principles: |
+  The questions should:
+  * Self-contained – understandable without needing to reference tables, figures, or specific text sections.
+  * Focus on the provided example and follow the format and style of the provided examples.
+  * Relevant to the subject – based on the textbook’s domain (e.g., legal, scientific, etc.).
+  * Independently answerable – avoid direct references to theorems, figures, or text numbers.
+  * Varied in difficulty - Make difficult same as the provided examples.
+  * Use same format as the provided examples.
+
+  Strictly follow this format for each question your generate while responding
+
+  [QUESTION]
+  <Insert question here>
+  [END]
+
+
+examples: |
+  Here are some examples of questions:
+
+  [Document]
+  {{icl_document}}
+
+  [QUESTION]
+  {{icl_query_1}}
+  [END]
+
+  [QUESTION]
+  {{icl_query_2}}
+  [END]
+
+  [QUESTION]
+  {{icl_query_3}}
+  [END]
+
+generation: |
+  Here is the document:
+  
+  [DOCUMENT]
+  {{document_outline}}
+  {{document}}
+
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/knowledge/generate_responses.yaml

@@ -0,0 +1,86 @@
+# This YAML file defines a prompt template for generating educational Q&A pairs from textbook content.
+# The prompt is designed to create comprehensive, domain-specific questions and answers that
+# effectively teach and reinforce key concepts from educational materials.
+#
+# Structure:
+# - system: Sets the AI's role as a knowledgeable assistant
+# - introduction: Main instruction for creating Q&A pairs from textbook chapters
+# - principles: Detailed guidelines for question formulation and educational value
+# - examples: Example Q&A pairs showing expected format and style
+# - generation: Template for the document to be used for Q&A generation
+#
+# Key Features:
+# - Domain-specific question generation (science, legal, etc.)
+# - Multiple difficulty levels and question types
+# - Self-contained questions without external references
+# - Focus on key concepts and learning objectives
+# - Educational value and teaching effectiveness
+#
+# Question Guidelines:
+# - Must be self-contained and independently answerable
+# - Should cover basic recall to advanced comprehension
+# - Include multiple-choice, short answer, and essay types
+# - Align with chapter learning objectives
+# - Avoid references to specific sections or figures
+#
+# Response Format:
+# - Questions and answers are clearly separated
+# - Each response ends with [End] tag
+# - [UNANSWERABLE] for unsuitable content
+#
+# Usage:
+# This prompt is used to generate educational Q&A pairs that effectively teach
+# and reinforce concepts from textbook chapters while maintaining educational
+# value and accessibility.
+
+system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+introduction: Answer the question based on the provided document.  
+
+principles: |
+  The answers should:
+  * The answer is grounded in the provided document.
+  * Follows the format and style of the provided examples.
+  * Directly answers the question.
+  Strictly follow this format for each question your generate while responding
+
+  [ANSWER]
+  <Insert answer here>
+  [END]
+
+
+examples: |
+  Here are some examples of answers for given questions for a document:
+
+  [Document]
+  {{icl_document}}
+
+  [QUESTION]
+  {{icl_query_1}}
+
+  [ANSWER]
+  {{icl_response_1}}
+  [END]
+
+  [QUESTION]
+  {{icl_query_2}}
+
+  [ANSWER]
+  {{icl_response_2}}
+  [END]
+
+
+generation: |
+  Here is the document:
+  
+  [DOCUMENT]
+  {{document_outline}}
+  {{document}}
+
+  [QUESTION]
+  {{question}}
+
+  [ANSWER]
+
+start_tags: [""]
+end_tags: [""]

sdg_hub/configs/skills/contexts.yaml

@@ -1,21 +1,28 @@
-system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+system: You are a highly capable AI Assistant that specializes in generating high-quality content tailored to specific tasks.
+
+introduction: |
+  Your task is to write a rich, relevant, and well-structured **context** for the following task:
+  Task Description: {{task_description}}
 
-introduction: You are asked to come up with a diverse context for - {{ task_description }}.
 principles: |
-  Please follow these guiding principles when generating responses:
-  * Use proper grammar and punctuation.
-  * Always generate safe and respectful content. Do not generate content that is harmful, abusive, or offensive.
-  * Always generate content that is factually accurate and relevant to the prompt.
-  * Strictly adhere to the prompt and generate responses in the same style and format as the example.
-  * Return the context between [Start of Context] and [End of Context] tags.
+  Please follow these guiding principles when generating the context:
+  * The context should be coherent, informative, and closely aligned with the task description.
+  * Do not include any greetings, explanations, or meta commentary.
+  * Maintain a natural, human-like tone suitable for the domain.
+  * Follow the formatting shown in the example exactly.
+  * Wrap the output between the tags: [Start of Context] and [End of Context].
+
 examples: |
-  To better assist you with this task, here is an example of a context:
+  To guide you, here is an example of a well-structured context:
+  
   [Start of Context]
-  {{ seed_context }}
+  {{seed_context}}
   [End of Context]
 
 generation: |
-  Now generate a context paragraph, remember to follow the principles mentioned above and use the same format as the examples. Remember to use the same style and format as the example above. Start your response with the tag [Start of Context] and end it with the tag [End of Context].
+  Now generate a new context following the same structure and principles. 
+  Begin your output with [Start of Context] and end with [End of Context]. 
+  Do not include any additional text outside these tags.
 
 start_tags: ["[Start of Context]"]
 end_tags: ["[End of Context]"]

sdg_hub/configs/skills/evaluate_freeform_pair.yaml

@@ -1,25 +1,89 @@
-system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+system: You are a highly knowledgeable and impartial AI Assistant tasked with evaluating the quality of responses to user questions.
 
 introduction: |
-  Please act as an impartial judge and evaluate the quality of the answer provided by an AI assistant to the questions displayed below. Evaluate whether or not the answer is a good example of how AI Assistant should respond to the user's instruction. Please assign a score using the following 3-point scale.
+  Please act as an objective evaluator and assess whether the AI Assistant's answer correctly follows formatting requirements and answers the user's question. Use the scoring rubric below and assign a score from 1 to 3.
+
 principles: |
-  1: It means the answer is incorrect, irrelevant, unsafe or provides incomplete and garbage information. For instance, the answer may be factually wrong, off-topic, or filled with irrelevant content that doesn't address the user's question or it could be incomplete and hanging. It may also include any harmful, unethical, racist, sexist, explicit, offensive, toxic, dangerous, or illegal content.  
+  Use the following 3-point scale to score the answer:
+
+  **1 — Poor Quality**
+  - The output is incorrectly formatted, contains hallucinations, or ignores required tags/structure.
+  - The answer may be off-topic, incomplete, or inconsistent with the task.
+  - The output introduces unsafe or inappropriate content, or violates structural instructions.
 
-  2: It means the answer provides the correct answer, but it is brief and to the point without explanations. While it directly answers the user's question, it lacks additional context or in-depth explanations.
+  **2 — Acceptable but Minimal**
+  - The answer is structurally valid and safe, but lacks polish, clarity, or minor formatting correctness.
+  - It meets the task minimally, but may show inconsistencies or lack proper use of spacing, tags, or conventions.
+  - It's not harmful, but also not ready for use without revision.
 
-  3: It means the answer is a perfect answer from an AI Assistant. It intentionally addresses the user's question with a comprehensive and detailed explanation. It demonstrates expert knowledge in the area, is very well written, logical, easy to follow, engaging, and insightful. And the answer is safe and does not include any harmful content.
+  **3 — Excellent Answer**
+  - The answer is fully correct, clearly written, and **strictly adheres to formatting instructions**.
+  - It uses all required tags, markdown syntax, or structure accurately and consistently.
+  - It directly fulfills the task with precision, professionalism, and completeness.
 
 examples: |
+  Example 1 — Score: 1 (Poor Formatting, Unsafe)
+
+  Task Description: Generate a markdown table of 3 planets with columns: Name, Gravity, and Moons.
+
+  [Start of Question]
+  Create a markdown table with 3 planets and their gravity/moons.
+  [End of Question]
+
+  [Start of Answer]
+  Mars - 3.7 - 2; Earth - 9.8 - 1; Jupiter - 24.8 - 79
+  [End of Answer]
+
+  [Start of Evaluation]
+  The answer is improperly formatted (not a markdown table) and unstructured. It violates task instructions both structurally and in tone.
+  [End of Evaluation]
+
+  [Start of Score]
+  1
+  [End of Score]
+
+  Example 2 — Score: 2 (Correct but Sloppy Formatting)
+
+  Task Description: Generate a markdown table of 3 planets with columns: Name, Gravity, and Moons.
+
   [Start of Question]
-  Take the role of the joker. Now answer this question: What is the name of the largest spider in the world?
+  Create a markdown table with 3 planets and their gravity/moons.
   [End of Question]
 
+  [Start of Answer]
+  | Name  | Gravity | Moons |
+  |-------|---------|-------|
+  | Mars | 3.7 | 2 |
+  | Earth| 9.8| 1 |
+  | Jupiter |24.8 |79|
+  [End of Answer]
+
   [Start of Evaluation]
-  This question is properly formatted, respectful, and relevant to the task of understanding the benefits of renewable energy. It is grounded in the context of renewable energy benefits and focuses on the economic aspect.
+  The table has correct content but inconsistent spacing and pipe alignment. It fulfills the task but lacks polish and readability. It's usable, but not clean or well-structured.
   [End of Evaluation]
 
+  [Start of Score]
+  2
+  [End of Score]
+
+  Example 3 — Score: 3 (Flawless Markdown Table)
+
+  Task Description: Generate a markdown table of 3 planets with columns: Name, Gravity, and Moons.
+
+  [Start of Question]
+  Create a markdown table with 3 planets and their gravity/moons.
+  [End of Question]
+
+  [Start of Answer]
+  | Name     | Gravity (m/s²) | Moons |
+  |----------|----------------|-------|
+  | Mars     | 3.7            | 2     |
+  | Earth    | 9.8            | 1     |
+  | Jupiter  | 24.8           | 79    |
+  [End of Answer]
+
   [Start of Evaluation]
-  The answer provided is correct and relevant. It accurately identifies the Goliath birdeater spider as the largest spider in the world by size, with a leg span of up to 12 inches. The response is engaging, humorous, and provides additional context about the spider's diet and nature, enhancing the reader's understanding. It avoids any harmful or inappropriate content, aligning well with the rubric's criteria for a comprehensive and detailed explanation.
+  The answer uses proper markdown syntax, alignment, and column headers. The formatting is clean, readable, and consistent with markdown table standards. It meets the task precisely.
   [End of Evaluation]
 
   [Start of Score]
@@ -27,7 +91,10 @@ examples: |
   [End of Score]
 
 generation: |
-  Here's the question and the answer you need to evaluate:
+  Now begin your evaluation of the following QA pair. Use the rubric above and be objective and concise in your reasoning.
+
+  Task Description: {{task_description}}
+
   [Start of Question]
   {{ question }}
   [End of Question]
@@ -36,9 +103,9 @@ generation: |
   {{ response }}
   [End of Answer]
 
-  Begin your evaluation by providing a short explanation. Be as objective as possible. After providing your explanation, you must rate the answer on a scale of 1 to 3 as mentioned above. 
-  * Return the evaluation between [Start of Evaluation] and [End of Evaluation] tags.
-  * Return the score between [Start of Score] and [End of Score] tags.
+  * Provide your evaluation between [Start of Evaluation] and [End of Evaluation] tags.
+  * Provide the score between [Start of Score] and [End of Score] tags.
+  * Do not include any content outside these tags.
 
 start_tags: ["[Start of Evaluation]", "[Start of Score]"]
 end_tags: ["[End of Evaluation]", "[End of Score]"]

sdg_hub/configs/skills/evaluate_freeform_questions.yaml

@@ -1,46 +1,78 @@
 system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
 
 introduction: |
-   Please act as an impartial judge and evaluate the synthetic questions. Evaluate whether or not the question is a good question based on the requirements provided below. Please assign a score using a binary 0/1 scale.
+  Please act as an impartial and detail-oriented evaluator of synthetic questions. Your job is to assess whether the given question meets the defined quality and formatting standards for the task described. Assign a score using a strict binary 0/1 scale.
 
 principles: |
-   Here are the requirements:
-   * The questions should be answerable through text. It should not require any visual or audio output. 
-   * The questions should be in English.
-   * The questions should be 1 to 2 sentences long and should be properly formatted.
-   * The question should not be offensive, abusive, or harmful. It should be safe and respectful.
-   * The question should be relevant to the task given - {{ task_description }}.
+  A valid question must satisfy **all** of the following requirements:
+  * The question should be answerable via text (not require visual/audio input).
+  * It must be **clearly relevant to the task description** ({{task_description}}).
+  * It should **not contain placeholder text**, incomplete sentences, or formatting artifacts.
 
-   If the question meets the above requirements, please rate it 1. If not, please rate it 0.
+  If the question satisfies **all** of the above, assign a score of `1`. Otherwise, assign `0`.
 
 examples: |
-   Task Description: Understanding the benefits of renewable energy
+  Example 1 - Valid question:
 
-   [Start of Question]
-   What are the long-term economic benefits of investing in solar energy infrastructure?
-   [End of Question]
+  Task Description: Extract the main idea of a paragraph.
 
-   [Start of Evaluation]
-   This question is properly formatted, respectful, and relevant to the task of understanding the benefits of renewable energy. It is grounded in the context of renewable energy benefits and focuses on the economic aspect.
-   [End of Evaluation]
-   
-   [Start of Score]
-   1
-   [End of Score]
+  [Start of Question]
+  What is the central message conveyed by the paragraph?
+  [End of Question]
 
+  [Start of Evaluation]
+  The question is clear, concise, grammatically correct, and directly related to the task. It follows formatting rules and is appropriate in tone.
+  [End of Evaluation]
 
-generation: |   
-   Here's the question you need to evaluate:
+  [Start of Score]
+  1
+  [End of Score]
 
-   Task Description: {{ task_description }}
+  Example 2 - Invalid question (bad formatting):
 
-   [Start of Question]
-   {{ question }}
-   [End of Question]
+  Task Description: Extract the main idea of a paragraph.
 
-   Begin your evaluation by providing a short explanation. Be as objective as possible. After providing your explanation, you must rate the question on a scale of 0 or 1 as mentioned above. Strictly follow the format below:
-   * Return the evaluation between [Start of Evaluation] and [End of Evaluation] tags.
-   * Return the score using a binary 0/1 scale between [Start of Score] and [End of Score] tags.
+  [Start of Question]
+  main idea??
+  [End of Question]
+
+  [Start of Evaluation]
+  The question lacks proper capitalization, punctuation, and complete sentence structure. It does not meet the formatting standards.
+  [End of Evaluation]
+
+  [Start of Score]
+  0
+  [End of Score]
+
+  Example 3 - Invalid question (off-topic):
+
+  Task Description: Extract the main idea of a paragraph.
+
+  [Start of Question]
+  What's your favorite type of movie and why?
+  [End of Question]
+
+  [Start of Evaluation]
+  The question is unrelated to the given task description. It fails the relevance requirement.
+  [End of Evaluation]
+
+  [Start of Score]
+  0
+  [End of Score]
+
+generation: |
+  Here's the question you need to evaluate:
+
+  Task Description: {{task_description}}
+
+  [Start of Question]
+  {{question}}
+  [End of Question]
+
+  Now begin your evaluation:
+  * First, provide a brief explanation between [Start of Evaluation] and [End of Evaluation] tags.
+  * Then return a binary score (0 or 1) between [Start of Score] and [End of Score] tags.
+  * Do not include any content outside these tags.
 
 start_tags: ["[Start of Evaluation]", "[Start of Score]"]
 end_tags: ["[End of Evaluation]", "[End of Score]"]