cat-llm 0.0.61__tar.gz → 0.0.62__tar.gz

{cat_llm-0.0.61 → cat_llm-0.0.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cat-llm
-Version: 0.0.61
+Version: 0.0.62
 Summary: A tool for categorizing text data and images using LLMs and vision models
 Project-URL: Documentation, https://github.com/chrissoria/cat-llm#readme
 Project-URL: Issues, https://github.com/chrissoria/cat-llm/issues
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.8
+Requires-Dist: anthropic
 Requires-Dist: openai
 Requires-Dist: pandas
 Requires-Dist: requests

{cat_llm-0.0.61 → cat_llm-0.0.62}/pyproject.toml

@@ -28,7 +28,8 @@ dependencies = [
   "pandas",
   "tqdm",
   "requests",
-  "openai"
+  "openai",
+  "anthropic"
 ]
 
 [project.urls]

{cat_llm-0.0.61 → cat_llm-0.0.62}/src/catllm/__about__.py

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2025-present Christopher Soria <chrissoria@berkeley.edu>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.61"
+__version__ = "0.0.62"
 __author__ = "Chris Soria"
 __email__ = "chrissoria@berkeley.edu"
 __title__ = "cat-llm"

{cat_llm-0.0.61 → cat_llm-0.0.62}/src/catllm/build_web_research.py

@@ -20,6 +20,8 @@ def build_web_research_dataset(
     import regex
     from tqdm import tqdm
     import time
+
+    model_source = model_source.lower() # eliminating case sensitivity 
     
     categories_str = "\n".join(f"{i + 1}. {cat}" for i, cat in enumerate(categories))
     print(categories_str)
@@ -35,6 +37,8 @@ def build_web_research_dataset(
     link1 = []
     extracted_jsons = []
 
+    max_retries = 5 #API rate limit error handler retries
+
     for idx, item in enumerate(tqdm(search_input, desc="Building dataset")):
         if idx > 0:  # Skip delay for first item only
             time.sleep(time_delay)
@@ -68,32 +72,43 @@ def build_web_research_dataset(
         }}
         </format>"""
             #print(prompt)
-            if model_source == "Anthropic":
+            if model_source == "anthropic":
                 import anthropic
                 client = anthropic.Anthropic(api_key=api_key)
-                try:
-                    message = client.messages.create(
-                    model=user_model,
-                    max_tokens=1024,
-                    messages=[{"role": "user", "content": prompt}],
-                    **({"temperature": creativity} if creativity is not None else {}),
-                    tools=[{
-                    "type": "web_search_20250305", 
-                    "name": "web_search"
-                    }]
-                )
-                    reply = " ".join(
-                        block.text
-                        for block in message.content
-                        if getattr(block, "type", "") == "text"
-                    ).strip()
-                    link1.append(reply)
-                    
-                except Exception as e:
-                    print(f"An error occurred: {e}")
-                    link1.append(f"Error processing input: {e}")
 
-            elif model_source == "Google":
+                attempt = 0
+                while attempt < max_retries:
+                    try:
+                        message = client.messages.create(
+                        model=user_model,
+                        max_tokens=1024,
+                        messages=[{"role": "user", "content": prompt}],
+                        **({"temperature": creativity} if creativity is not None else {}),
+                        tools=[{
+                        "type": "web_search_20250305", 
+                        "name": "web_search"
+                        }]
+                    )
+                        reply = " ".join(
+                            block.text
+                            for block in message.content
+                            if getattr(block, "type", "") == "text"
+                        ).strip()
+                        link1.append(reply)
+                        break
+                    except anthropic.error.RateLimitError as e:
+                        wait_time = 2 ** attempt  # Exponential backoff, keeps doubling after each attempt
+                        print(f"Rate limit error encountered. Retrying in {wait_time} seconds...")
+                        time.sleep(wait_time) #in case user wants to try and buffer the amount of errors by adding a wait time before attemps
+                        attempt += 1
+                    except Exception as e:
+                        print(f"A Non-rate-limit error occurred: {e}")
+                        link1.append(f"Error processing input: {e}")
+                        break #stop retrying 
+                else:
+                    link1.append("Max retries exceeded for rate limit errors.")
+
+            elif model_source == "google":
                 import requests
                 url = f"https://generativelanguage.googleapis.com/v1beta/models/{user_model}:generateContent"
                 try: