cat-llm 0.0.61__py3-none-any.whl → 0.0.62__py3-none-any.whl

{cat_llm-0.0.61.dist-info → cat_llm-0.0.62.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cat-llm
-Version: 0.0.61
+Version: 0.0.62
 Summary: A tool for categorizing text data and images using LLMs and vision models
 Project-URL: Documentation, https://github.com/chrissoria/cat-llm#readme
 Project-URL: Issues, https://github.com/chrissoria/cat-llm/issues
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.8
+Requires-Dist: anthropic
 Requires-Dist: openai
 Requires-Dist: pandas
 Requires-Dist: requests

{cat_llm-0.0.61.dist-info → cat_llm-0.0.62.dist-info}/RECORD

@@ -1,7 +1,7 @@
 catllm/CERAD_functions.py,sha256=05n7h27TuAp3klkOnrH--m1wMreYqYuObM9NIab934o,22603
-catllm/__about__.py,sha256=kVSa5EnJdo6swDUoImNoZblgcdNnyj8uZIq8r4WrGQo,408
+catllm/__about__.py,sha256=R0Mt1NOAMAQCF7SHD4XDl2P4gF92EnfjYXaJ1Xo0vdc,408
 catllm/__init__.py,sha256=sf02zp7N0NW0mAQi7eQ4gliWR1EwoqvXkHN2HwwjcTE,372
-catllm/build_web_research.py,sha256=1mp1z59ed5-wfG79xMDH0jjjQYZSuo1ko-i0_GO6iE4,8253
+catllm/build_web_research.py,sha256=nAKfkg7lihjXrYrLvltsKCvpb5zRFYpNp95A-0zpDb8,9159
 catllm/image_functions.py,sha256=KDb2UxDLrioerlqKZjKAX7lqfW-S_TSQCK6YxtJRKwI,35958
 catllm/text_functions.py,sha256=xfpwAYivnPnDlsU21vp1Pma9mDR24tn1lcBZQfsyIrc,18467
 catllm/images/circle.png,sha256=JWujAWAh08-TajAoEr_TAeFNLlfbryOLw6cgIBREBuQ,86202
@@ -9,7 +9,7 @@ catllm/images/cube.png,sha256=nFec3e5bmRe4zrBCJ8QK-HcJLrG7u7dYdKhmdMfacfE,77275
 catllm/images/diamond.png,sha256=rJDZKtsnBGRO8FPA0iHuA8FvHFGi9PkI_DWSFdw6iv0,99568
 catllm/images/overlapping_pentagons.png,sha256=VO5plI6eoVRnjfqinn1nNzsCP2WQhuQy71V0EASouW4,71208
 catllm/images/rectangles.png,sha256=2XM16HO9EYWj2yHgN4bPXaCwPfl7iYQy0tQUGaJX9xg,40692
-cat_llm-0.0.61.dist-info/METADATA,sha256=Dm51S7hYCKkZwBXXdCBUmL6N5j2pqP8ZLe9OSXvzATw,22370
-cat_llm-0.0.61.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cat_llm-0.0.61.dist-info/licenses/LICENSE,sha256=Vje2sS5WV4TnIwY5uQHrF4qnBAM3YOk1pGpdH0ot-2o,34969
-cat_llm-0.0.61.dist-info/RECORD,,
+cat_llm-0.0.62.dist-info/METADATA,sha256=jstvau7l_g2KqYSheIcZJxC8DX2Bf_lA_wLNzPO5-qw,22395
+cat_llm-0.0.62.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cat_llm-0.0.62.dist-info/licenses/LICENSE,sha256=Vje2sS5WV4TnIwY5uQHrF4qnBAM3YOk1pGpdH0ot-2o,34969
+cat_llm-0.0.62.dist-info/RECORD,,

catllm/__about__.py

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2025-present Christopher Soria <chrissoria@berkeley.edu>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.61"
+__version__ = "0.0.62"
 __author__ = "Chris Soria"
 __email__ = "chrissoria@berkeley.edu"
 __title__ = "cat-llm"

catllm/build_web_research.py

@@ -20,6 +20,8 @@ def build_web_research_dataset(
     import regex
     from tqdm import tqdm
     import time
+
+    model_source = model_source.lower() # eliminating case sensitivity 
     
     categories_str = "\n".join(f"{i + 1}. {cat}" for i, cat in enumerate(categories))
     print(categories_str)
@@ -35,6 +37,8 @@ def build_web_research_dataset(
     link1 = []
     extracted_jsons = []
 
+    max_retries = 5 #API rate limit error handler retries
+
     for idx, item in enumerate(tqdm(search_input, desc="Building dataset")):
         if idx > 0:  # Skip delay for first item only
             time.sleep(time_delay)
@@ -68,32 +72,43 @@ def build_web_research_dataset(
         }}
         </format>"""
             #print(prompt)
-            if model_source == "Anthropic":
+            if model_source == "anthropic":
                 import anthropic
                 client = anthropic.Anthropic(api_key=api_key)
-                try:
-                    message = client.messages.create(
-                    model=user_model,
-                    max_tokens=1024,
-                    messages=[{"role": "user", "content": prompt}],
-                    **({"temperature": creativity} if creativity is not None else {}),
-                    tools=[{
-                    "type": "web_search_20250305", 
-                    "name": "web_search"
-                    }]
-                )
-                    reply = " ".join(
-                        block.text
-                        for block in message.content
-                        if getattr(block, "type", "") == "text"
-                    ).strip()
-                    link1.append(reply)
-                    
-                except Exception as e:
-                    print(f"An error occurred: {e}")
-                    link1.append(f"Error processing input: {e}")
 
-            elif model_source == "Google":
+                attempt = 0
+                while attempt < max_retries:
+                    try:
+                        message = client.messages.create(
+                        model=user_model,
+                        max_tokens=1024,
+                        messages=[{"role": "user", "content": prompt}],
+                        **({"temperature": creativity} if creativity is not None else {}),
+                        tools=[{
+                        "type": "web_search_20250305", 
+                        "name": "web_search"
+                        }]
+                    )
+                        reply = " ".join(
+                            block.text
+                            for block in message.content
+                            if getattr(block, "type", "") == "text"
+                        ).strip()
+                        link1.append(reply)
+                        break
+                    except anthropic.error.RateLimitError as e:
+                        wait_time = 2 ** attempt  # Exponential backoff, keeps doubling after each attempt
+                        print(f"Rate limit error encountered. Retrying in {wait_time} seconds...")
+                        time.sleep(wait_time) #in case user wants to try and buffer the amount of errors by adding a wait time before attemps
+                        attempt += 1
+                    except Exception as e:
+                        print(f"A Non-rate-limit error occurred: {e}")
+                        link1.append(f"Error processing input: {e}")
+                        break #stop retrying 
+                else:
+                    link1.append("Max retries exceeded for rate limit errors.")
+
+            elif model_source == "google":
                 import requests
                 url = f"https://generativelanguage.googleapis.com/v1beta/models/{user_model}:generateContent"
                 try: