bioguider 0.2.52__py3-none-any.whl

bioguider/agents/identification_task_utils.py

@@ -0,0 +1,22 @@
+
+
+from typing import Callable, TypedDict, Optional
+from langchain_openai.chat_models.base import BaseChatOpenAI
+
+class IdentificationWorkflowState(TypedDict):
+    llm: BaseChatOpenAI
+    step_output_callback: Optional[Callable]
+    goal: str
+
+    plan_actions: Optional[str]
+    plan_reasoning: Optional[str]
+    plan_instructions: Optional[str]
+    observe_instructions: Optional[str]
+    intermediate_steps: Optional[list[str]]
+    final_answer: Optional[str]
+    final_answer_example: Optional[str]
+    step_output: Optional[str]
+    step_analysis: Optional[str]
+    step_thoughts: Optional[str]
+
+    step_count: Optional[int]

bioguider/agents/peo_common_step.py

@@ -0,0 +1,64 @@
+
+
+from typing import Optional
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from pydantic import BaseModel, Field
+from bioguider.agents.common_step import CommonState, CommonStep
+
+class PEOWorkflowState(CommonState):
+    intermediate_steps: Optional[str]
+    step_output: Optional[str]
+    step_analysis: Optional[str]
+    step_thoughts: Optional[str]
+    plan_actions: Optional[list[dict]]
+
+class PEOCommonStep(CommonStep):
+    """
+    This class is a placeholder for common step functionality in the PEO agent.
+    It is currently empty and can be extended in the future.
+    """
+    def __init__(self, llm: BaseChatOpenAI):
+        super().__init__()
+        self.llm = llm
+
+    def _build_intermediate_steps(self, state: PEOWorkflowState):
+        """
+        Build intermediate steps for the PEO workflow.
+        """
+        intermediate_steps = ""
+        # previous steps
+        if "intermediate_steps" in state:
+            for i in range(len(state['intermediate_steps'])):
+                step = state['intermediate_steps'][i].replace("{", "(").replace("}", ")")
+                intermediate_steps += step + "\n"
+        # current step
+        if "step_output" in state and state["step_output"] is not None:
+            step_content = state["step_output"]
+            step_content = step_content.replace("{", "(").replace("}", ")")
+            intermediate_steps += step_content
+        return intermediate_steps
+    
+    def _build_intermediate_analysis_and_thoughts(self, state: PEOWorkflowState):
+        intermediate_analysis = "N/A" if "step_analysis" not in state or \
+            state["step_analysis"] is None \
+            else state["step_analysis"]
+        intermediate_analysis = intermediate_analysis.replace("{", "(").replace("}", ")")
+        intermediate_thoughts = "N/A" if "step_thoughts" not in state or \
+            state["step_thoughts"] is None \
+            else state["step_thoughts"]
+        intermediate_thoughts = intermediate_thoughts.replace("{", "(").replace("}", ")")
+        return intermediate_analysis, intermediate_thoughts
+
+    @staticmethod
+    def _reset_step_state(state):
+        # move step_output to intermediate steps
+        if "intermediate_steps" not in state or state["intermediate_steps"] is None:
+            state["intermediate_steps"] = []
+        intermediate_steps = state["intermediate_steps"]
+        if "step_output" in state and state["step_output"] is not None:
+            intermediate_steps.append(state["step_output"])
+        state["intermediate_steps"] = intermediate_steps
+
+        state["step_analysis"] = None
+        state["step_thoughts"] = None
+        state["step_output"] = None

bioguider/agents/prompt_utils.py

@@ -0,0 +1,253 @@
+from enum import Enum
+from langchain_core.prompts import ChatPromptTemplate
+
+USER_INSTRUCTION = """Do not give the final result immediately. First, explain your reasoning process step by step, then provide the answer."""
+
+EVALUATION_ITEMS = [
+    ("1. Clarity & Readability", 20),
+    ("2. Completeness", 20), 
+    ("3. Organization & Navigation", 10),
+    ("4. Examples & Tutorials", 10), 
+    ("5. Maintainability & Updates", 15), 
+    ("6. Accessibility & Formatting", 15), 
+]
+
+EVALUATION_SYSTEM_PROMPT = ChatPromptTemplate.from_template("""Please act as both a **biomedical researcher** and an **experienced software developer** to evaluate the documentation quality of a GitHub repository using the evaluation criteria below.
+
+### **Evaluation Criteria (Total: 100 points)**
+
+1. **Clarity & Readability (20 points)** - Is the documentation written in a clear, concise, and easy-to-understand manner?
+2. **Completeness (20 points)** - Does the documentation cover all essential information needed for understanding, usage, and further development?
+3. **Organization & Navigation (10 points)** - Is the structure logical and easy to navigate? Are key sections easy to find?
+4. **Examples & Tutorials (10 points)** - Are there sufficient examples or tutorials to help users get started and understand core functionality?
+5. **Maintainability & Updates (15 points)** - Does the documentation reflect ongoing maintenance and version history (e.g., changelogs, version tags)?
+6. **Accessibility & Formatting (15 points)** - Is the documentation well-formatted and easy to read (e.g., Markdown formatting, appropriate use of code blocks, headers, etc.)?
+### **Repository Structure Overview**  
+_(f = file, d = directory)_
+```
+{repository_structure}
+```""")
+
+EVALUATION_ITEM_PROMPT = ChatPromptTemplate.from_template("""Here are the content of files or directories in the repository that you need to take into account:
+{files_or_directories}
+
+### **Instructions**
+
+Let's begin by evaluating **Criterion {evaluation_item}*.
+
+- If the information provided is **sufficient**, please proceed with your evaluation using the following format:
+```
+{evaluation_item} ({score_point} points)  
+    a. Score: [score out of {score_point}]  
+    b. Reason: [brief explanation justifying the score]  
+```
+- If the information provided is **insufficient**, do **not** attempt to evaluate. Instead, list the specific files or directories for which you need more detail, using the format below:
+```
+[files/directories needed for evaluation]
+```""")
+
+
+## goal: identify project type
+IDENTIFICATION_GOAL_PROJECT_TYPE = """Identify the following key attribute of the repository:
+  **project type**: The primary functional type of the project.  
+    Options and their definitions:  
+    - **package**: A reusable Python or R library intended to be imported by other software.  
+    - **application**: A standalone Python or R program that can be directly executed by users.  
+    - **pipeline**: A biomedical data processing workflow that integrates multiple tools or steps.
+    - **unknown type**: Use this only if the type cannot be determined reliably from available information.
+  **Notes**:
+    1. The project can be identified as one of the above project type.
+    2. The project may server as multiple project types, like package & pipeline, standalone application & package,
+      However, you need to investigate closely to find out the primary project type.
+    3. Do **not** rely heavily on directories like 'benchmark/' or 'tests/' when determining the project type, as they are often auxiliary."""
+
+## goal: identify primary language
+IDENTIFICATION_GOAL_PRIMARY_LANGUAGE = """Identify the following key attribute of the repository:
+  **primary language**: The primary language of the project.  
+    Options and their definitions:  
+    - **python**: Python language
+    - **R**: R language
+    - **unknown type**: Use this only if the type cannot be determined reliably from available information.
+  **Notes**:
+    The project can be identified as one of the above primary language."""
+
+## goal: identify meta data: repo name, owner, description, license
+IDENTIFICATION_GOAL_META_DATA = """Identify the following meta data of the repository:
+  **name**: The repository name.
+  **owner**: The repository user or orgnization.
+  **description**: The description of the repository.
+  **license**: The license of the repository, like 'MIT', 'Apache 2.0' or 'unknown'.
+
+**Notes**: If the above meta data can't be identified, please return 'unknown' or 'N/A'.
+"""
+
+COT_USER_INSTRUCTION = "First, explain your reasoning process step by step, then provide the answer."
+EVALUATION_INSTRUCTION="Please also clearly explain your reasoning step by step. Now, let's begin the evaluation."
+
+class CollectionGoalItemEnum(Enum):
+    UserGuide = "User Guide"
+    Tutorial = "Tutorials & Vignettes"
+    DockerGeneration = "Docker Generation"
+    Installation = "Installation"
+    License = "License"
+    Contributing = "Contributing"
+    SoftwarePackageContent = "SoftwarePackageContent"
+
+
+
+COLLECTION_GOAL = """Your goal is to collect the names of all files that are relevant to **{goal_item}**.  
+**Note:**
+ - You only need to collect the **file names**, not their contents."""
+
+COLLECTION_PROMPTS = {
+    "UserGuide": {
+        "goal_item": "User Guide",
+        "related_file_description": """A document qualifies as a **User Guide** if it includes **at least one** of the following elements.
+If **any one** of these is present, the document should be classified as a User Guide — full coverage is **not required**:
+ - **Not source code or a script** (*.py, *.R) or notebook (*.ipynb, *.Rmd) that is not intended for end-user interaction.
+ - Document **functions, methods, or classes**
+ - Describe **input parameters, return values**, and **usage syntax**
+ - Include **technical guidance** for using specific APIs
+ - Are often found in folders such as
+   * `man/` (for `.Rd` files in R)
+   * `docs/reference/`, `docs/api/`, `docs/dev/` (for Python) or similar
+   * Standalone files with names like `api.md`, `reference.md`, `user_guide.md`
+**Do not** classify the document as a User Guide if it primarily serves as a Tutorial or Example. Such documents typically include:
+ - Sample Datasets: Example data used to illustrate functionality.
+ - Narrative Explanations: Story-like descriptions guiding the user through examples.
+ - Code Walkthroughs: Detailed explanations of code snippets in a tutorial format.
+**Do not** classify the document as a User Guide if it is souce code or a script (*.py, *.R) that is not intended for end-user interaction.
+ - You can include directory names if all files in the directory are relevant to the goal item.""",
+        "plan_important_instructions": """ - **Do not** try to summarize or read the content of any source code or script (*.py, *.R) or notebook (*.ipynb, *.Rmd) that is not intended for end-user interaction.
+ - **Do not** classify the document as a User Guide if it is source code or a script (*.py, *.R) that is not intended for end-user interaction.
+ - **Do not** classify the document as a User Guide if it is a notebook (*.ipynb, *.Rmd) that is not intended for end-user interaction.
+ - You plan **must not** include any source code or script (*.py, *.R) or notebook (*.ipynb, *.Rmd) that is not intended for end-user interaction.""",
+        "observe_important_instructions": """ - **Do not** classify the document as a User Guide if it is source code or a script (*.py, *.R) that is not intended for end-user interaction.
+ - **Do not** include any source code or script (*.py, *.R) or notebook (*.ipynb, *.Rmd) in the final answer that is not intended for end-user interaction."""
+    },
+    "Tutorial": {
+        "goal_item": "Tutorials & Vignettes",
+        "related_file_description": """
+**Tutorials and Vignettes** are instructional documents or interactive notebooks that provide step-by-step guidance on using a software package or library. They typically include:
+ - Code Examples: Practical code snippets demonstrating how to use the software's features and functions.
+ - Explanatory Text: Clear explanations accompanying the code examples to help users understand the concepts and techniques being presented.​
+ - Visualizations: Graphical representations of data or results to enhance understanding.
+ - Interactive Elements: Features that allow users to experiment with the code in real-time, such as Jupyter notebooks or R Markdown files.​
+ - Use Cases: Real-world applications or scenarios where the software can be applied effectively.
+ - You can include directory names if all files in the directory are relevant to the goal item.
+**Important instructions**:
+ - **Do not** use **read_file_tool, summarize_file_tool, check_file_related_tool** on the python/R notebook files **(.ipynb, .Rmd)**, as they are too big to read.
+ - **Do not** classify the document as a Tutorial if it is source code or a script (*.py, *.R) that is not intended for end-user interaction.
+""",
+        "plan_important_instructions": """ - **Do not** use **read_file_tool, summarize_file_tool, check_file_related_tool** on the python/R notebook files **(.ipynb, .Rmd)**, as they are too big to read.
+  - For python/R notebook files **(.ipynb, .Rmd)**, **only infer** if it is the tutorial/vignette from the file name and avoid reading the content of the file.
+""",
+        "observe_important_instructions": """ - **Do not** use **read_file_tool, summarize_file_tool, check_file_related_tool** on the python/R notebook files **(.ipynb, .Rmd)**, as they are too big to read.
+  - For python/R notebook files **(.ipynb, .Rmd)**, **only infer** if it is the tutorial/vignette from the file name and avoid reading the content of the file.
+  - **Do not** include any binary files (e.g., `.png`, `.jpg`, `.jpeg`, `.gif`, `.svg`) in the final answer.
+""",
+    },
+    "DockerGeneration": {
+        "goal_item": "Generating a Dockerfile for reproducibility testing",
+        
+        "related_file_description": """A document qualifies **Dockerfile Generation** related if it includes **at least one** of the following elements.
+If **any one** of these is present, the document should be classified as a Dockerfile — full coverage is **not required**:
+ - Existing Docker Configuration
+   * Files like `Dockerfile`, `docker-compose.yml`, or any Docker-related build scripts.
+ - Installation & Environment Setup
+   * Files used to define or install dependencies.
+     * Examples: `README.md` `requirements.txt`, `environment.yml`, `setup.py`, `install.R`, `DESCRIPTION`, `pyproject.toml`, etc.
+ - Build/Runtime Scripts
+   * Shell or batch scripts used for setup, building, or launching the application.
+     * Examples: `install.sh`, `build.sh`, `run.sh`, etc.
+ - Minimal Code Examples or Get-Started Files
+   * Files that demonstrate a minimal working example of the software (e.g., for testing or reproducing results).
+     * Examples: `example.py`, `main.py`, `demo.R`, `notebooks/get_started.ipynb`, etc.
+     * These should be runnable with minimal configuration.""",
+        
+        "plan_important_instructions": """- Only include minimal code examples that demonstrate basic functionality.
+If multiple example files are found, select only the simplest and most lightweight one that is sufficient to verify the repository works.
+ - Give priority to analyzing files whose names include **"install"** or **"Dockerfile"**, as these are most likely to be useful for generating our Dockerfile
+ - The total number of collected files should **not exceed 5**.
+ - Make sure to include **only one code example**, selecting the most minimal and representative one.
+""",
+        "observe_important_instructions": """- Only include minimal code examples that demonstrate basic functionality.
+If multiple example files are found, select only the simplest and most lightweight one that is sufficient to verify the repository works.
+ - Give priority to analyzing files whose names include **"install"** or **"Dockerfile"**, as these are most likely to be useful for generating our Dockerfile
+ - The total number of collected files should **not exceed 5**.
+ - Make sure to include **only one code example**, selecting the most minimal and representative one.
+""",
+    },
+    "Installation": {
+        "goal_item": "Installation Instructions",
+        "related_file_description": """A document qualifies as **Installation Instructions** if it includes **at least one** of the following elements.
+If **any one** of these is present, the document should be classified as Installation Instructions — full coverage is **not required**:
+ - Step-by-step setup procedures for the software.
+ - Prerequisites or dependencies that need to be installed before using the software.
+ - Configuration steps required to get the software running.
+ - Troubleshooting tips related to installation issues.
+ - You can include directory names if all files in the directory are relevant to the goal item.""",
+        "plan_important_instructions": """ - Give priority to analyzing README file that contain installation instructions and the files whose names include **"install"** or **"setup"**.
+- If multiple files are found, select the most comprehensive one that covers the installation process.
+- The total number of collected files should **not exceed 3**.
+- Identify and select **no more than three** installation instruction files — choose the most comprehensive and representative ones.
+""",
+        "observe_important_instructions": """ - Give priority to analyzing README file that contain installation instructions and the files whose names include **"install"** or **"setup"**.
+- If multiple files are found, select the most comprehensive one that covers the installation process.
+- The total number of collected files should **not exceed 3**.
+- Identify and select **no more than three** installation instruction files — choose the most comprehensive and representative ones.
+""",
+    },
+    "License": {
+        "goal_item": "License Information",
+        "related_file_description": """A document qualifies as **License Information** if it includes **at least one** of the following elements.
+If **any one** of these is present, the document should be classified as License Information — full coverage is **not required**:
+ - A file named `LICENSE`, `LICENSE.txt`, or similar that explicitly states the software's license.
+ - A section in the README or documentation that describes the licensing terms.
+ - Any file that contains legal information regarding the use, distribution, or modification of the software.
+ - You can include directory names if all files in the directory are relevant to the goal item.""",
+    },
+    "Contributing": {
+        "goal_item": "Contributing Guidelines",
+        "related_file_description": """A document qualifies as **Contributing Guidelines** if it includes **at least one** of the following elements.
+If **any one** of these is present, the document should be classified as Contributing Guidelines — full coverage is **not required**:
+ - A file named `CONTRIBUTING.md`, `CONTRIBUTING.rst`, or similar that provides guidelines for contributing to the project.
+ - A section in the README or documentation that outlines how to contribute, report issues, or submit pull requests.
+ - Any file that contains instructions for developers on how to contribute to the project, including coding standards, testing procedures, and submission processes.
+ - You can include directory names if all files in the directory are relevant to the goal item.""",
+    },
+    "SoftwarePackageContent": {
+        "goal_item": "Software Package Content",
+        "related_file_description": """A file qualifies as **Software Package Content** if it meets **at least one** of the following elements.
+ - A compiled binary file that may be qualified as a compiled standalone software, please carefully analyze a binary file and its file name to identify if it is a compiled standalone software
+ - A source code file, like a file whose extension is `.py`, `.R`, `.ipynb`, `.ts`, or `.js`.
+ - An example data which is used to demonstrate usage or for tutorial. Image file should not be considered as example data.
+""",
+        "plan_important_instructions": """ - A comiled standalone software file is non-textual and appears to be in an executable format (e.g., `.exe`, `.dll`, `.so`, `.bin`, `.elf`).
+ - A comiled standalone software file **is not a script or compiled library**, that is, It is not a wrapper script (e.g., shell, Python, Python notebook or Rmd) nor a dynamic/shared library meant for linking.
+   So, when you are identifying a binary file, **do not** use any tools (our tools don't work for binary file), you need to figure out if it is compiled standalone software file by the file name and extension on your own.
+ - **Source code files** are determined by their **extensions** or **file names** (e.g., `.py`, `.R`, `.ipynb`, `.ts`, `.js`). **Do not open or summarize their content.**
+ - **Example data files** are identified by typical data extensions (e.g., `.dat`, `.csv`, `.fastq`) or names like `example_*.txt`. 
+   If extension/name is ambiguous, use summarize_file_tool to summarize file content to decide, **do not** read the file content.
+ - **Note**: You **only need to detect** whether at least **one** compiled standalone software file, **one** source code file and **one** example data file exist — no need to list all such files.
+ - **Note**: When identifying **compiled standalone software** or **example data files**, **ignore** any **image files** (e.g., `.png`, `.jpg`, `.jpeg`, `.gif`, `.svg`) and **image folders** (directories containing primarily images).
+""",
+        "observe_important_instructions": """ - A comiled standalone software file is non-textual and appears to be in an executable format (e.g., `.exe`, `.dll`, `.so`, `.bin`, `.elf`).
+ - A comiled standalone software file **is not a script or compiled library**, that is, It is not a wrapper script (e.g., shell, Python, Python notebook or Rmd) nor a dynamic/shared library meant for linking.
+ - When identifying source code file, prioritize analyzing the file's **extension** and **file name** and try to avoid reading file, using check_file_related_tool or summarizing file content.
+ - When identifying example data, prioritize analyzing the file's **extension** (like .dat, .csv, .fastq, and so on) and **file name** (like example_data.txt, example.dat, and so on). If extension/name is ambiguous, use summarizing file content to decide.
+ - **Note**: You **only need to detect** whether at least **one** compiled standalone software file, **one** source code file and **one** example data file exist — no need to list all such files.
+ - **Final answer format**: If you believe **all relevant files** have been collected:
+   Your final answer **must exactly match** the following format: 
+   **FinalAnswer:** {{"final_answer": [<N/A or a compiled filename>, <N/A or a source file name>, <N/A or a example data file name>]}} 
+   For each category, return a single file name or `"N/A"` if none found. And the return array must exactly follow this order: [<A comiled standalone software file name>, <A source code file name>, <A example data file name>]
+   For example, **FinalAnswer:** {{"final_answer": ["N/A", "app.py", "example.csv"]}} indicates:
+   * No compiled standalone software found
+   * `app.py` found as source code
+   * `example.csv` found as example data
+""",
+    },
+}
+
+
+

bioguider/agents/python_ast_repl_tool.py

@@ -0,0 +1,69 @@
+
+from pydantic import PrivateAttr
+import re
+import io
+import contextlib
+import logging
+from langchain_experimental.tools.python.tool import PythonAstREPLTool
+
+class CustomPythonAstREPLTool(PythonAstREPLTool):
+    """
+    Custom Python REPL tool that executes Python code and captures output.
+    This tool is designed to be used in a LangChain agent for executing Python code
+    and capturing the output, including any print statements.
+    """
+    __name__ = "Custom_Python_AST_REPL"
+    _exec_globals: dict = PrivateAttr()
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._exec_globals = {}
+        self._exec_globals.update(__builtins__)
+
+    def _set_globals(self, table_dict=None):
+        self._exec_globals = {}
+        self._exec_globals.update(__builtins__)
+
+        if table_dict is not None:
+            self._exec_globals.update(table_dict)
+
+    def _run(self, query: str, run_manager=None):  
+        print("================================== code here ==============================")
+        print(query)
+        print("===========================================================================")
+        code_match = re.search(r"```(.*?)```", query, re.DOTALL)
+        if code_match:
+            # Extract code within backticks
+            code = code_match.group(1)
+        else:
+            code = query
+        code = code.strip()
+        if code.startswith("python"):
+            code = code[len("python"):].lstrip()
+        
+        if code.endswith("Observation"):
+            code = code[:-len("Observation")].rstrip()
+        
+        code_lines = code.strip().split('\n')
+        code = '\n'.join(code_lines[:-1])   # avoid printing the last line twice
+        last_line = code_lines[-1]
+        
+        output_capture = io.StringIO()
+        with contextlib.redirect_stdout(output_capture), contextlib.redirect_stderr(output_capture):
+            logging.getLogger().handlers[0].stream = output_capture
+            try:
+                exec(code, self._exec_globals)
+                try:
+                    result = eval(last_line, self._exec_globals)
+                    if result is not None:
+                        print(result, file=output_capture)
+                except:
+                    pass
+            except Exception as e:
+                return str(e)
+        
+        # Retrieve the output and return it
+        output = output_capture.getvalue()
+        return output if output else "Execution completed without output."
+
+
+

bioguider/agents/rag_collection_task.py

@@ -0,0 +1,130 @@
+
+import os
+from adalflow import Document
+from langchain_core.prompts import ChatPromptTemplate
+from pydantic import BaseModel, Field
+
+from .common_agent_2step import CommonAgentTwoSteps
+from ..rag.rag import RAG
+
+RAG_COLLECT_SYSTEM_PROMPT = ChatPromptTemplate.from_template("""
+You are an expert in repository documents retrieval and collection.
+Your task is to collect relevant documents based on the user's query using the RAG system.
+Here is the user's query:
+{query}
+The following are the documents extracted from the RAG system:
+{documents}
+Please analyze the documents one by one and determine which ones are relevant to the user's query.
+Return a list of boolean values indicating the relevance of each document. Output example:
+[True, False, True, ...]  # True if the document is relevant, False otherwise
+""")
+
+class RAGCollectResult(BaseModel):
+    """
+    Represents the result of a RAG collection task.
+    
+    Attributes:
+        query (str): The user's query.
+        documents (list): List of documents retrieved from the RAG system.
+        relevance (list): List of boolean values indicating the relevance of each document.
+    """
+    query: str = Field(..., description="The user's query")
+    documents: list[str] = Field(..., description="List of documents retrieved from the RAG system")
+    relevance: list[bool] = Field(..., description="List of boolean values indicating the relevance of each document")
+
+RAGCollectResultSchema = {
+  'description': "Represents the result of a RAG collection task.\n\nAttributes:\n    query (str): The user's query.\n    documents (list): List of documents retrieved from the RAG system.\n    relevance (list): List of boolean values indicating the relevance of each document.", 
+  'properties': {
+    'query': {'description': "The user's query", 'title': 'Query', 'type': 'string'}, 
+    'documents': {'description': 'List of documents retrieved from the RAG system', 'items': {'type': 'string'}, 'title': 'Documents', 'type': 'array'}, 
+    'relevance': {'description': 'List of boolean values indicating the relevance of each document', 'items': {'type': 'boolean'}, 'title': 'Relevance', 'type': 'array'}
+  }, 
+  'required': [
+    'query', 'documents', 'relevance'
+  ], 
+  'title': 'RAGCollectResult', 
+  'type': 'object'
+}
+
+class RAGCollectionTaskItem:
+    def __init__(self, llm, rag: RAG, step_callback, batch_size: int = 5):
+        """
+        Initialize the RAGCollectionTaskItem with a repository URL or local path.
+
+        Args:
+            rag: An instance of the RAG class
+        """
+        self.llm = llm
+        self.rag = rag
+        self.batch_size = batch_size
+        self.step_callback = step_callback
+
+    def collect(self, query: str, rag_documents: list[Document]) -> list[Document]:
+        relevant_documents = []
+        for i in range(0, len(rag_documents), self.batch_size):
+            contents = [' - ' + doc.text for doc in rag_documents[i:i + self.batch_size]]
+            documents_text = "\n".join(contents)
+            prompt = RAG_COLLECT_SYSTEM_PROMPT.format(query=query, documents=documents_text)
+            prompt = prompt.replace("{", "{{").replace("}", "}}")  # Escape curly braces for LangChain
+            agent = CommonAgentTwoSteps(llm=self.llm)
+            res, _, token_usage, reasoning = agent.go(
+                system_prompt=prompt,
+                instruction_prompt="Please analyze the documents and determine their relevance to the query.",
+                schema=RAGCollectResultSchema,
+            )
+            self.step_callback(
+                step_output=f"**Reasoning Process**: {reasoning}\n",
+            )
+            self.step_callback(
+                step_output=f"**RAG Collection Result**: {res}",
+            )
+            self.step_callback(
+                token_usage=token_usage,
+            )
+            res = RAGCollectResult(**res)
+            relevants = self._collect_documents(
+                rag_documents[i:i + self.batch_size],
+                res.relevance
+            )
+            relevant_documents.extend(relevants)
+        return relevant_documents
+
+    def _collect_documents(self, docs: list[Document], relevants: list[bool]) -> list[Document]:
+        """
+        Collect documents based on relevance.
+
+        Args:
+            docs: List of documents to filter
+            relevants: List of boolean values indicating relevance
+
+        Returns:
+            List of relevant documents
+        """
+        return [doc for doc, relevant in zip(docs, relevants) if relevant]
+        
+            
+
+class RAGCollectionTask:
+    def __init__(self, rag: RAG):
+        """
+        Initialize the RAGCollectionTask with a repository URL or local path.
+
+        Args:
+            repo_url_or_path: URL or local path to the repository
+            access_token: Optional access token for private repositories
+        """
+        self.rag = rag
+        
+    
+    def query(self, query: str) -> list:
+        """
+        Process a query using RAG.
+
+        Args:
+            query: The user's query
+
+        Returns:
+            retrieved_documents: List of documents retrieved based on the query
+        """
+        return self.rag.query_doc(query)
+

bioguider/conversation.py

@@ -0,0 +1,67 @@
+
+from abc import ABC, abstractmethod
+from langchain_core.messages import BaseMessage
+from langchain_deepseek import ChatDeepSeek
+from openai import AuthenticationError
+from pydantic import PositiveFloat, PositiveInt
+
+class Conversation(ABC):
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def chat(
+        question: str,
+        messages: list[BaseMessage] = None
+    ):
+        """ chat with LLM """
+
+class DeepSeekConversation(Conversation):
+    chatter: ChatDeepSeek | None = None
+    model: str = "deepseek-chat"
+    temperature: PositiveFloat = 0.1
+    request_timeout: PositiveInt = 60
+    base_url: str = "https://api.deepseek.com/v1"
+    max_retries: PositiveInt = 3
+    api_key: str | None = None
+    def __init__(
+        self,        
+    ):
+        super().__init__()
+
+    def set_api_key(self, key: str):
+        
+        try:
+            self.chatter = ChatDeepSeek(
+                model=self.model,
+                api_key=key,
+                temperature=self.temperature,
+                max_retries=self.max_retries,
+                timeout=self.request_timeout,
+                base_url=self.base_url,
+            )
+            # verify chat
+            ai_msg = self.chatter.invoke(
+                 [("system", "Hi")]
+            )
+            return True
+        except AuthenticationError as e:
+            self.chatter = None
+            return False
+
+    def chat(
+        self,
+        question: str,
+        messages: list[BaseMessage] = None
+    ):
+        msgs = messages + [("user", question)] if messages is not None else \
+               [("user", question)]
+        
+        try:
+            res_msg = self.chatter.invoke(msgs)
+            return res_msg
+        except Exception as e:
+            return str(e)
+
+
+