lattereview 0.1.0__tar.gz

lattereview-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Pouria Rouzrokh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lattereview-0.1.0/PKG-INFO

@@ -0,0 +1,184 @@
+Metadata-Version: 2.1
+Name: lattereview
+Version: 0.1.0
+Summary: A framework for multi-agent review workflows using large language models
+Home-page: https://github.com/PouriaRouzrokh/LatteReview
+Author: Pouria Rouzrokh
+Author-email: po.rouzrokh@gmail.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: litellm>=1.55.2
+Requires-Dist: nest-asyncio>=1.6.0
+Requires-Dist: ollama>=0.4.4
+Requires-Dist: openai>=1.57.4
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: pydantic>=2.10.3
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: tokencost>=0.1.17
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: openpyxl>=3.1.5
+
+# LatteReview 🤖☕
+
+[![PyPI version](https://badge.fury.io/py/lattereview.svg)](https://badge.fury.io/py/lattereview)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Maintained: yes](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/prouzrokh/lattereview)
+
+LatteReview is a powerful Python package designed to automate academic literature review processes through AI-powered agents. Just like enjoying a cup of latte ☕, reviewing numerous research articles should be a pleasant, efficient experience that doesn't consume your entire day!
+
+## 🎯 Key Features
+
+- Multi-agent review system with customizable roles and expertise
+- Support for multiple review rounds with hierarchical decision-making
+- Flexible model integration (OpenAI, Gemini, Claude, Groq, local models via Ollama)
+- Asynchronous processing for high-performance batch reviews
+- Structured output format with detailed scoring and reasoning
+- Comprehensive cost tracking and memory management
+- Extensible architecture for custom review workflows
+
+## 🛠️ Installation
+
+```bash
+pip install lattereview
+```
+
+## 🚀 Quick Start
+
+Here's a simple example of how to set up a review workflow with two primary reviewers and an expert reviewer for conflict resolution:
+
+```python
+from lattereview.providers import LiteLLMProvider
+from lattereview.agents import ScoringReviewer
+from lattereview.review_workflow import ReviewWorkflow
+import pandas as pd
+import asyncio
+
+# First Reviewer: Conservative approach
+reviewer1 = ScoringReviewer(
+    provider=LiteLLMProvider(model="gpt-4o-mini"),
+    name="Alice",
+    backstory="a radiologist with expertise in systematic reviews",
+    input_description="article title and abstract",
+    scoring_task="Evaluate how relevant the article is to artificial intelligence applications in radiology",
+    score_set=[1, 2, 3, 4, 5],
+    scoring_rules="Rate the relevance on a scale of 1 to 5, where 1 means the article is not at all relevant to AI in radiology, and 5 means it directly focuses on AI applications in radiology.",
+    model_args={"temperature": 0.1}  # Low temperature for consistent scoring
+)
+
+# Second Reviewer: More exploratory approach
+reviewer2 = ScoringReviewer(
+    provider=LiteLLMProvider(model="gemini/gemini-1.5-flash"),
+    name="Bob",
+    backstory="a computer scientist specializing in medical AI",
+    input_description="article title and abstract",
+    scoring_task="Evaluate how relevant the article is to artificial intelligence applications in radiology",
+    score_set=[1, 2, 3, 4, 5],
+    scoring_rules="Rate the relevance on a scale of 1 to 5, where 1 means the article is not at all relevant to AI in radiology, and 5 means it directly focuses on AI applications in radiology.",
+    model_args={"temperature": 0.8}  # Higher temperature for creative interpretation
+)
+
+# Expert Reviewer: Resolves disagreements
+expert = ScoringReviewer(
+    provider=LiteLLMProvider(model="gpt-4o"),
+    name="Carol",
+    backstory="a professor of AI in medical imaging",
+    input_description="article title, abstract, and previous reviews",
+    scoring_task="Review Alice and Bob's relevance assessments of this article to AI in radiology",
+    score_set=[1, 2],
+    scoring_rules='Score 1 if you agree with Alice\'s assessment, 2 if you agree with Bob\'s assessment',
+    model_args={"temperature": 0.1}  # Low temperature for careful judgment
+)
+
+# Define the multi-round review workflow
+workflow = ReviewWorkflow([
+    {
+        "round": 'A',  # First round: Initial review by both reviewers
+        "reviewers": [reviewer1, reviewer2],
+        "inputs": ["title", "abstract"]
+    },
+    {
+        "round": 'B',  # Second round: Expert reviews only disagreements
+        "reviewers": [expert],
+        "inputs": ["title", "abstract", "round-A_Alice_output", "round-A_Bob_output"],
+        "filter": lambda row: row["round-A_Alice_score"] != row["round-A_Bob_score"]
+    }
+])
+
+# Run the workflow on your data
+data = pd.read_excel("articles.xlsx")
+results = asyncio.run(workflow(data))
+```
+
+## 🔌 Model Support
+
+LatteReview offers flexible model integration through multiple providers:
+
+- **LiteLLMProvider** (Recommended): Supports OpenAI, Anthropic (Claude), Gemini, Groq, and more
+- **OpenAIProvider**: Direct integration with OpenAI and Gemini APIs
+- **OllamaProvider**: Optimized for local models via Ollama
+
+Note: Models should support async operations and structured JSON outputs for optimal performance.
+
+## 📖 Documentation
+
+Full documentation and API reference will be available in our upcoming preprint paper. [Link to be added]
+
+## 🛣️ Roadmap
+
+- [ ] Development of `AbstractionReviewer` class for automated paper summarization
+- [ ] Support for image-based inputs and multimodal analysis
+- [ ] Development of a no-code web application
+- [ ] Integration of RAG (Retrieval-Augmented Generation) tools
+- [ ] Addition of graph-based analysis tools
+- [ ] Enhanced visualization capabilities
+- [ ] Support for additional model providers
+
+## 👨‍💻 Author
+
+**Pouria Rouzrokh, MD, MPH, MHPE**  
+Medical Practitioner and Machine Learning Engineer  
+Incoming Radiology Resident @Yale University  
+Former Data Scientist @Mayo Clinic AI Lab
+
+Find my work:
+[![Twitter Follow](https://img.shields.io/twitter/follow/prouzrokh?style=social)](https://twitter.com/prouzrokh)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue)](https://linkedin.com/in/pouria-rouzrokh)
+[![Google Scholar](https://img.shields.io/badge/Google%20Scholar-Profile-green)](https://scholar.google.com/citations?user=Ksv9I0sAAAAJ&hl=en)
+[![Email](https://img.shields.io/badge/Email-Contact-red)](mailto:po.rouzrokh@gmail.com)
+
+## ❤️ Support LatteReview
+
+If you find LatteReview helpful in your research or work, consider supporting its continued development. Since we're already sharing a virtual coffee break while reviewing papers, maybe you'd like to treat me to a real one? ☕ 😊
+
+### Ways to Support:
+
+- [Treat me to a coffee](http://ko-fi.com/pouriarouzrokh) on Ko-fi ☕
+- [Star the repository](https://github.com/PouriaRouzrokh/LatteReview) to help others discover the project
+- Submit bug reports, feature requests, or contribute code
+- Share your experience using LatteReview in your research
+
+## 📜 License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## 🤝 Contributing
+
+We welcome contributions! Please feel free to submit a Pull Request.
+
+## 📚 Citation
+
+If you use LatteReview in your research, please cite our paper:
+
+```bibtex
+# Preprint citation to be added
+```

lattereview-0.1.0/README.md

@@ -0,0 +1,156 @@
+# LatteReview 🤖☕
+
+[![PyPI version](https://badge.fury.io/py/lattereview.svg)](https://badge.fury.io/py/lattereview)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Maintained: yes](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/prouzrokh/lattereview)
+
+LatteReview is a powerful Python package designed to automate academic literature review processes through AI-powered agents. Just like enjoying a cup of latte ☕, reviewing numerous research articles should be a pleasant, efficient experience that doesn't consume your entire day!
+
+## 🎯 Key Features
+
+- Multi-agent review system with customizable roles and expertise
+- Support for multiple review rounds with hierarchical decision-making
+- Flexible model integration (OpenAI, Gemini, Claude, Groq, local models via Ollama)
+- Asynchronous processing for high-performance batch reviews
+- Structured output format with detailed scoring and reasoning
+- Comprehensive cost tracking and memory management
+- Extensible architecture for custom review workflows
+
+## 🛠️ Installation
+
+```bash
+pip install lattereview
+```
+
+## 🚀 Quick Start
+
+Here's a simple example of how to set up a review workflow with two primary reviewers and an expert reviewer for conflict resolution:
+
+```python
+from lattereview.providers import LiteLLMProvider
+from lattereview.agents import ScoringReviewer
+from lattereview.review_workflow import ReviewWorkflow
+import pandas as pd
+import asyncio
+
+# First Reviewer: Conservative approach
+reviewer1 = ScoringReviewer(
+    provider=LiteLLMProvider(model="gpt-4o-mini"),
+    name="Alice",
+    backstory="a radiologist with expertise in systematic reviews",
+    input_description="article title and abstract",
+    scoring_task="Evaluate how relevant the article is to artificial intelligence applications in radiology",
+    score_set=[1, 2, 3, 4, 5],
+    scoring_rules="Rate the relevance on a scale of 1 to 5, where 1 means the article is not at all relevant to AI in radiology, and 5 means it directly focuses on AI applications in radiology.",
+    model_args={"temperature": 0.1}  # Low temperature for consistent scoring
+)
+
+# Second Reviewer: More exploratory approach
+reviewer2 = ScoringReviewer(
+    provider=LiteLLMProvider(model="gemini/gemini-1.5-flash"),
+    name="Bob",
+    backstory="a computer scientist specializing in medical AI",
+    input_description="article title and abstract",
+    scoring_task="Evaluate how relevant the article is to artificial intelligence applications in radiology",
+    score_set=[1, 2, 3, 4, 5],
+    scoring_rules="Rate the relevance on a scale of 1 to 5, where 1 means the article is not at all relevant to AI in radiology, and 5 means it directly focuses on AI applications in radiology.",
+    model_args={"temperature": 0.8}  # Higher temperature for creative interpretation
+)
+
+# Expert Reviewer: Resolves disagreements
+expert = ScoringReviewer(
+    provider=LiteLLMProvider(model="gpt-4o"),
+    name="Carol",
+    backstory="a professor of AI in medical imaging",
+    input_description="article title, abstract, and previous reviews",
+    scoring_task="Review Alice and Bob's relevance assessments of this article to AI in radiology",
+    score_set=[1, 2],
+    scoring_rules='Score 1 if you agree with Alice\'s assessment, 2 if you agree with Bob\'s assessment',
+    model_args={"temperature": 0.1}  # Low temperature for careful judgment
+)
+
+# Define the multi-round review workflow
+workflow = ReviewWorkflow([
+    {
+        "round": 'A',  # First round: Initial review by both reviewers
+        "reviewers": [reviewer1, reviewer2],
+        "inputs": ["title", "abstract"]
+    },
+    {
+        "round": 'B',  # Second round: Expert reviews only disagreements
+        "reviewers": [expert],
+        "inputs": ["title", "abstract", "round-A_Alice_output", "round-A_Bob_output"],
+        "filter": lambda row: row["round-A_Alice_score"] != row["round-A_Bob_score"]
+    }
+])
+
+# Run the workflow on your data
+data = pd.read_excel("articles.xlsx")
+results = asyncio.run(workflow(data))
+```
+
+## 🔌 Model Support
+
+LatteReview offers flexible model integration through multiple providers:
+
+- **LiteLLMProvider** (Recommended): Supports OpenAI, Anthropic (Claude), Gemini, Groq, and more
+- **OpenAIProvider**: Direct integration with OpenAI and Gemini APIs
+- **OllamaProvider**: Optimized for local models via Ollama
+
+Note: Models should support async operations and structured JSON outputs for optimal performance.
+
+## 📖 Documentation
+
+Full documentation and API reference will be available in our upcoming preprint paper. [Link to be added]
+
+## 🛣️ Roadmap
+
+- [ ] Development of `AbstractionReviewer` class for automated paper summarization
+- [ ] Support for image-based inputs and multimodal analysis
+- [ ] Development of a no-code web application
+- [ ] Integration of RAG (Retrieval-Augmented Generation) tools
+- [ ] Addition of graph-based analysis tools
+- [ ] Enhanced visualization capabilities
+- [ ] Support for additional model providers
+
+## 👨‍💻 Author
+
+**Pouria Rouzrokh, MD, MPH, MHPE**  
+Medical Practitioner and Machine Learning Engineer  
+Incoming Radiology Resident @Yale University  
+Former Data Scientist @Mayo Clinic AI Lab
+
+Find my work:
+[![Twitter Follow](https://img.shields.io/twitter/follow/prouzrokh?style=social)](https://twitter.com/prouzrokh)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue)](https://linkedin.com/in/pouria-rouzrokh)
+[![Google Scholar](https://img.shields.io/badge/Google%20Scholar-Profile-green)](https://scholar.google.com/citations?user=Ksv9I0sAAAAJ&hl=en)
+[![Email](https://img.shields.io/badge/Email-Contact-red)](mailto:po.rouzrokh@gmail.com)
+
+## ❤️ Support LatteReview
+
+If you find LatteReview helpful in your research or work, consider supporting its continued development. Since we're already sharing a virtual coffee break while reviewing papers, maybe you'd like to treat me to a real one? ☕ 😊
+
+### Ways to Support:
+
+- [Treat me to a coffee](http://ko-fi.com/pouriarouzrokh) on Ko-fi ☕
+- [Star the repository](https://github.com/PouriaRouzrokh/LatteReview) to help others discover the project
+- Submit bug reports, feature requests, or contribute code
+- Share your experience using LatteReview in your research
+
+## 📜 License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## 🤝 Contributing
+
+We welcome contributions! Please feel free to submit a Pull Request.
+
+## 📚 Citation
+
+If you use LatteReview in your research, please cite our paper:
+
+```bibtex
+# Preprint citation to be added
+```

lattereview-0.1.0/lattereview/agents/__init__.py

@@ -0,0 +1 @@
+from .scoring_reviewer import ScoringReviewer

lattereview-0.1.0/lattereview/agents/base_agent.py

@@ -0,0 +1,160 @@
+"""Base agent class with consistent error handling and type safety."""
+
+from typing import List, Optional, Dict, Any, Union
+from enum import Enum
+from pydantic import BaseModel, Field
+
+DEFAULT_CONCURRENT_REQUESTS = 20
+
+
+class ReasoningType(Enum):
+    """Enumeration for reasoning types."""
+
+    NONE = "none"
+    BRIEF = "brief"
+    LONG = "long"
+    COT = "cot"
+
+
+class AgentError(Exception):
+    """Base exception for agent-related errors."""
+
+    pass
+
+
+class BaseAgent(BaseModel):
+    response_format: Dict[str, Any]
+    provider: Optional[Any] = None
+    model_args: Dict[str, Any] = Field(default_factory=dict)
+    max_concurrent_requests: int = DEFAULT_CONCURRENT_REQUESTS
+    name: str = "BaseAgent"
+    backstory: str = "a generic base agent"
+    input_description: str = "article title/abstract"
+    examples: Union[str, List[Union[str, Dict[str, Any]]]] = None
+    reasoning: ReasoningType = ReasoningType.BRIEF
+    system_prompt: Optional[str] = None
+    item_prompt: Optional[str] = None
+    cost_so_far: float = 0
+    memory: List[Dict[str, Any]] = []
+    identity: Dict[str, Any] = {}
+
+    def __init__(self, **data: Any) -> None:
+        """Initialize the base agent with error handling."""
+        try:
+            super().__init__(**data)
+            if isinstance(self.reasoning, str):
+                self.reasoning = ReasoningType(self.reasoning.lower())
+            if self.reasoning == ReasoningType.NONE:
+                self.response_format.pop("reasoning", None)
+            self.setup()
+        except Exception as e:
+            raise AgentError(f"Error initializing agent: {str(e)}")
+
+    def setup(self) -> None:
+        """Setup the agent before use."""
+        raise NotImplementedError("This method must be implemented by subclasses.")
+
+    def build_system_prompt(self) -> str:
+        """Build the system prompt for the agent."""
+        try:
+            return self._clean_text(
+                f"""
+                Your name is <<{self.name}>> and you are <<{self.backstory}>>.
+                Your task is to review input itmes with the following description: <<{self.input_description}>>.
+                Your final output should have the following keys: \
+                    {", ".join(f"{k} ({v})" for k, v in self.response_format.items())}.
+                """
+            )
+        except Exception as e:
+            raise AgentError(f"Error building system prompt: {str(e)}")
+
+    def build_item_prompt(self, base_prompt: str, item_dict: Dict[str, Any]) -> str:
+        """Build the item prompt with variable substitution."""
+        try:
+            prompt = base_prompt
+            if "examples" in item_dict:
+                item_dict["examples"] = self.process_examples(item_dict["examples"])
+            if "reasoning" in item_dict:
+                item_dict["reasoning"] = self.process_reasoning(item_dict["reasoning"])
+
+            for key, value in item_dict.items():
+                if value is not None:
+                    prompt = prompt.replace(f"${{{key}}}$", str(value))
+                else:
+                    prompt = prompt.replace(f"${{{key}}}$", "")
+
+            return self._clean_text(prompt)
+        except Exception as e:
+            raise AgentError(f"Error building item prompt: {str(e)}")
+
+    def process_reasoning(self, reasoning: Union[str, ReasoningType]) -> str:
+        """Process the reasoning type into a prompt string."""
+        try:
+            if isinstance(reasoning, str):
+                reasoning = ReasoningType(reasoning.lower())
+
+            reasoning_map = {
+                ReasoningType.NONE: "",
+                ReasoningType.BRIEF:
+                    "You must also provide a brief (1 sentence) reasoning for your scoring. First reason then score!",
+                ReasoningType.LONG:
+                    "You must also provide a detailed reasoning for your scoring. First reason then score!",
+                ReasoningType.COT:
+                    "You must also provide a reasoning for your scoring . Think step by step in your reasoning. \
+                        First reason then score!",
+            }
+
+            return self._clean_text(reasoning_map.get(reasoning, ""))
+        except Exception as e:
+            raise AgentError(f"Error processing reasoning: {str(e)}")
+
+    def process_examples(self, examples: Union[str, Dict[str, Any], List[Union[str, Dict[str, Any]]]]) -> str:
+        """Process examples into a formatted string."""
+        try:
+            if not examples:
+                return ""
+
+            if not isinstance(examples, list):
+                examples = [examples]
+
+            examples_str = []
+            for example in examples:
+                if isinstance(example, dict):
+                    examples_str.append("***" + "".join(f"{k}: {v}\n" for k, v in example.items()))
+                elif isinstance(example, str):
+                    examples_str.append("***" + example)
+                else:
+                    raise ValueError(f"Invalid example type: {type(example)}")
+
+            return self._clean_text(
+                "<<Here is one or more examples of the performance you are expected to have: \n"
+                + "".join(examples_str)
+                + ">>"
+            )
+        except Exception as e:
+            raise AgentError(f"Error processing examples: {str(e)}")
+
+    def reset_memory(self) -> None:
+        """Reset the agent's memory and cost tracking."""
+        try:
+            self.memory = []
+            self.cost_so_far = 0
+            self.identity = {}
+        except Exception as e:
+            raise AgentError(f"Error resetting memory: {str(e)}")
+
+    def _clean_text(self, text: str) -> str:
+        """Remove extra spaces and blank lines from text."""
+        try:
+            lines = [line.strip() for line in text.splitlines() if line.strip()]
+            return " ".join(" ".join(line.split()) for line in lines)
+        except Exception as e:
+            raise AgentError(f"Error cleaning text: {str(e)}")
+
+    async def review_items(self, items: List[str]) -> List[Dict[str, Any]]:
+        """Review a list of items asynchronously."""
+        raise NotImplementedError("This method must be implemented by subclasses.")
+
+    async def review_item(self, item: str) -> Dict[str, Any]:
+        """Review a single item asynchronously."""
+        raise NotImplementedError("This method must be implemented by subclasses.")

lattereview-0.1.0/lattereview/agents/scoring_reviewer.py

@@ -0,0 +1,129 @@
+"""Reviewer agent implementation with consistent error handling and type safety."""
+
+import asyncio
+from pathlib import Path
+import datetime
+from typing import List, Dict, Any, Optional
+from pydantic import Field
+from .base_agent import BaseAgent, AgentError, ReasoningType
+from tqdm.asyncio import tqdm
+import warnings
+
+DEFAULT_MAX_RETRIES = 3
+
+
+class ScoringReviewer(BaseAgent):
+    response_format: Dict[str, Any] = {
+        "reasoning": str,
+        "score": int,
+    }
+    scoring_task: Optional[str] = None
+    score_set: List[int] = [1, 2]
+    scoring_rules: str = "Your scores should follow the defined schema."
+    generic_item_prompt: Optional[str] = Field(default=None)
+    reasoning: ReasoningType = ReasoningType.BRIEF
+    max_retries: int = DEFAULT_MAX_RETRIES
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize after Pydantic model initialization."""
+        try:
+            assert self.reasoning != ReasoningType.NONE, "Reasoning type cannot be 'none' for ScoreReviewer"
+            assert (
+                0 not in self.score_set
+            ), "Score set must not contain 0. This value is reserved for uncertain scorings / errors."
+            prompt_path = Path(__file__).parent.parent / "generic_prompts" / "review_prompt.txt"
+            if not prompt_path.exists():
+                raise FileNotFoundError(f"Review prompt template not found at {prompt_path}")
+            self.generic_item_prompt = prompt_path.read_text(encoding="utf-8")
+            self.setup()
+        except Exception as e:
+            raise AgentError(f"Error initializing agent: {str(e)}")
+
+    def setup(self) -> None:
+        """Build the agent's identity and configure the provider."""
+        try:
+            self.system_prompt = self.build_system_prompt()
+            self.score_set = str(self.score_set)
+            keys_to_replace = ["scoring_task", "score_set", "scoring_rules", "reasoning", "examples"]
+
+            self.item_prompt = self.build_item_prompt(
+                self.generic_item_prompt, {key: getattr(self, key) for key in keys_to_replace}
+            )
+
+            self.identity = {
+                "system_prompt": self.system_prompt,
+                "item_prompt": self.item_prompt,
+                "model_args": self.model_args,
+            }
+
+            if not self.provider:
+                raise AgentError("Provider not initialized")
+
+            self.provider.set_response_format(self.response_format)
+            self.provider.system_prompt = self.system_prompt
+        except Exception as e:
+            raise AgentError(f"Error in setup: {str(e)}")
+
+    async def review_items(self, items: List[str], tqdm_keywords: dict = None) -> List[Dict[str, Any]]:
+        """Review a list of items asynchronously with concurrency control and progress bar."""
+        try:
+            self.setup()
+            semaphore = asyncio.Semaphore(self.max_concurrent_requests)
+
+            async def limited_review_item(item: str, index: int) -> tuple[int, Dict[str, Any], Dict[str, float]]:
+                async with semaphore:
+                    response, cost = await self.review_item(item)
+                    return index, response, cost
+
+            # Building the tqdm desc
+            if tqdm_keywords:
+                tqdm_desc = f"""{[f'{k}: {v}' for k, v in tqdm_keywords.items()]} - \
+                    {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"""
+            else:
+                tqdm_desc = f"Reviewing {len(items)} items - {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+
+            # Create tasks with indices
+            tasks = [limited_review_item(item, i) for i, item in enumerate(items)]
+
+            # Collect results with indices
+            responses_costs = []
+            async for result in tqdm(asyncio.as_completed(tasks), total=len(items), desc=tqdm_desc):
+                responses_costs.append(await result)
+
+            # Sort by original index and separate response and cost
+            responses_costs.sort(key=lambda x: x[0])  # Sort by index
+            results = []
+
+            for i, response, cost in responses_costs:
+                if isinstance(cost, dict):
+                    cost = cost["total_cost"]
+                self.cost_so_far += cost
+                results.append(response)
+                self.memory.append(
+                    {
+                        "identity": self.identity,
+                        "item": items[i],
+                        "response": response,
+                        "cost": cost,
+                        "model_args": self.model_args,
+                    }
+                )
+
+            return results, cost
+        except Exception as e:
+            raise AgentError(f"Error reviewing items: {str(e)}")
+
+    async def review_item(self, item: str) -> tuple[Dict[str, Any], Dict[str, float]]:
+        """Review a single item asynchronously with error handling."""
+        num_tried = 0
+        while num_tried < self.max_retries:
+            try:
+                item_prompt = self.build_item_prompt(self.item_prompt, {"item": item})
+                response, cost = await self.provider.get_json_response(item_prompt, **self.model_args)
+                return response, cost
+            except Exception as e:
+                warnings.warn(f"Error reviewing item: {str(e)}. Retrying {num_tried}/{self.max_retries}")
+        raise AgentError("Error reviewing item!")

lattereview-0.1.0/lattereview/generic_prompts/review_prompt.txt

@@ -0,0 +1,15 @@
+Review the input item below and evaluate it against the following criteria:
+
+Scoring task: <<${scoring_task}$>>
+
+Input item: <<${item}$>>
+
+The possible scores for you to choose from are: ${score_set}$.
+
+Your scoring should be based on the following rules: <<${scoring_rules}$>>
+
+If you are highly uncertain about what score to return, return a score of "0". 
+
+${reasoning}$
+
+${examples}$

lattereview-0.1.0/lattereview/providers/__init__.py

@@ -0,0 +1,3 @@
+from .litellm_provider import LiteLLMProvider
+from .ollama_provider import OllamaProvider
+from .openai_provider import OpenAIProvider