cat-llm 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl

{cat_llm-0.0.7.dist-info → cat_llm-0.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cat-llm
-Version: 0.0.7
+Version: 0.0.9
 Summary: A tool for categorizing text data and images using LLMs and vision models
 Project-URL: Documentation, https://github.com/Christopher Soria/cat-llm#readme
 Project-URL: Issues, https://github.com/chrissoria/cat-llm/issues
@@ -23,6 +23,8 @@ Requires-Dist: pandas
 Requires-Dist: tqdm
 Description-Content-Type: text/markdown
 
+![catllm Logo](https://raw.githubusercontent.com/ChristopherSoria/cat-llm/main/src/catllm/images/logo.png)
+
 # cat_llm
 
 [![PyPI - Version](https://img.shields.io/pypi/v/cat-llm.svg)](https://pypi.org/project/cat-llm)

cat_llm-0.0.9.dist-info/RECORD

@@ -0,0 +1,4 @@
+cat_llm-0.0.9.dist-info/METADATA,sha256=eXrV7lw17_HuPwLbsO9-73Mq4eTNlDsVnC9_lU_M_cI,1704
+cat_llm-0.0.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cat_llm-0.0.9.dist-info/licenses/LICENSE,sha256=wJLsvOr6lrFUDcoPXExa01HOKFWrS3JC9f0RudRw8uw,1075
+cat_llm-0.0.9.dist-info/RECORD,,

cat_llm-0.0.7.dist-info/RECORD

@@ -1,7 +0,0 @@
-catllm/__about__.py,sha256=xo5JKgmkegVL-708Re6bKW-0jRXLvw4H4R8v09TfBsw,431
-catllm/__init__.py,sha256=xDin9x4jymeccuxE9Xf-27ncR9h7247IwLbeYN-m3j8,266
-catllm/cat_llm.py,sha256=pRwb-cLsZXDInd0P4Ti4jXgOf4gAiucldRVKLn7762k,38569
-cat_llm-0.0.7.dist-info/METADATA,sha256=6jXSQDKveC3nre0R8gNZPuAxWPhoRTotqjiwzs-s1VY,1596
-cat_llm-0.0.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cat_llm-0.0.7.dist-info/licenses/LICENSE,sha256=wJLsvOr6lrFUDcoPXExa01HOKFWrS3JC9f0RudRw8uw,1075
-cat_llm-0.0.7.dist-info/RECORD,,

catllm/__about__.py

@@ -1,10 +0,0 @@
-# SPDX-FileCopyrightText: 2025-present Christopher Soria <chrissoria@berkeley.edu>
-#
-# SPDX-License-Identifier: MIT
-__version__ = "0.0.7"
-__author__ = "Chris Soria"
-__email__ = "chrissoria@berkeley.edu"
-__title__ = "cat-llm"
-__description__ = "A tool for categorizing text data and images using LLMs and vision models"
-__url__ = "https://github.com/chrissoria/cat-llm"  # Replace with your actual repository URL
-__license__ = "MIT"

catllm/__init__.py

@@ -1,14 +0,0 @@
-# SPDX-FileCopyrightText: 2025-present Christopher Soria <chrissoria@berkeley.edu>
-#
-# SPDX-License-Identifier: MIT
-
-from .__about__ import (
-    __version__,
-    __author__,
-    __description__,
-    __title__,
-    __url__,
-    __license__,
-)
-
-from .cat_llm import *

catllm/cat_llm.py

@@ -1,952 +0,0 @@
-#multi-class text classification
-def extract_multi_class(
-    survey_question, 
-    survey_input,
-    categories,
-    api_key,
-    columns="numbered",
-    user_model="gpt-4o-2024-11-20",
-    creativity=0,
-    to_csv=False,
-    safety=False,
-    filename="categorized_data.csv",
-    save_directory=None,
-    model_source="OpenAI"
-):
-    import os
-    import json
-    import pandas as pd
-    import regex
-    from tqdm import tqdm
-    
-    categories_str = "\n".join(f"{i + 1}. {cat}" for i, cat in enumerate(categories))
-    cat_num = len(categories)
-    category_dict = {str(i+1): "0" for i in range(cat_num)}
-    example_JSON = json.dumps(category_dict, indent=4)
-
-    # ensure number of categories is what user wants
-    print("\nPlease verify the categories you entered:")
-    for i, cat in enumerate(categories, 1):
-        print(f"{i}. {cat}")
-    response = input("\nIf the list above is correct, type 'next' and press Enter to continue: ")
-    while response.strip().lower() != "next":
-        response = input("Please type 'next' to continue: ")
-    
-    link1 = []
-    extracted_jsons = []
-
-    for idx, response in enumerate(tqdm(survey_input, desc="Categorizing responses")):
-        reply = None  
-
-        if pd.isna(response): 
-            link1.append("Skipped NaN input")
-            default_json = example_JSON 
-            extracted_jsons.append(default_json)
-            #print(f"Skipped NaN input.")
-        else:
-            prompt = f"""A respondent was asked: {survey_question}. \
-Categorize this survey response "{response}" into the following categories that apply: \
-{categories_str} \
-Provide your work in JSON format where the number belonging to each category is the key and a 1 if the category is present and a 0 if it is not present as key values."""
-            #print(prompt)
-            if model_source == ("OpenAI"):
-                from openai import OpenAI
-                client = OpenAI(api_key=api_key)
-                try:
-                    response_obj = client.chat.completions.create(
-                    model=user_model,
-                    messages=[{'role': 'user', 'content': prompt}],
-                    temperature=creativity
-                )
-                    reply = response_obj.choices[0].message.content
-                    link1.append(reply)
-                except Exception as e:
-                    print(f"An error occurred: {e}")
-                    link1.append(f"Error processing input: {e}")
-            elif model_source == "Perplexity":
-                from openai import OpenAI
-                client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
-                try:
-                    response_obj = client.chat.completions.create(
-                        model=user_model,
-                        messages=[{'role': 'user', 'content': prompt}],
-                        temperature=creativity
-                    )
-                    reply = response_obj.choices[0].message.content
-                    link1.append(reply)
-                except Exception as e:
-                    print(f"An error occurred: {e}")
-                    link1.append(f"Error processing input: {e}")
-            elif model_source == "Anthropic":
-                import anthropic
-                client = anthropic.Anthropic(api_key=api_key)
-                try:
-                    message = client.messages.create(
-                    model=user_model,
-                    max_tokens=1024,
-                    temperature=creativity,
-                    messages=[{"role": "user", "content": prompt}]
-                )
-                    reply = message.content[0].text  # Anthropic returns content as list
-                    link1.append(reply)
-                except Exception as e:
-                    print(f"An error occurred: {e}")
-                    link1.append(f"Error processing input: {e}")
-            elif model_source == "Mistral":
-                from mistralai import Mistral
-                client = Mistral(api_key=api_key)
-                try:
-                    response = client.chat.complete(
-                    model=user_model,
-                    messages=[
-                        {'role': 'user', 'content': prompt}
-                    ],
-                    temperature=creativity
-                )
-                    reply = response.choices[0].message.content
-                    link1.append(reply)
-                except Exception as e:
-                    print(f"An error occurred: {e}")
-                    link1.append(f"Error processing input: {e}")
-            else:
-                raise ValueError("Unknown source! Choose from OpenAI, Anthropic, Perplexity, or Mistral")
-            # in situation that no JSON is found
-            if reply is not None:
-                extracted_json = regex.findall(r'\{(?:[^{}]|(?R))*\}', reply, regex.DOTALL)
-                if extracted_json:
-                    cleaned_json = extracted_json[0].replace('[', '').replace(']', '').replace('\n', '').replace(" ", '').replace("  ", '')
-                    extracted_jsons.append(cleaned_json)
-                    #print(cleaned_json)
-                else:
-                    error_message = """{"1":"e"}"""
-                    extracted_jsons.append(error_message)
-                    print(error_message)
-            else:
-                error_message = """{"1":"e"}"""
-                extracted_jsons.append(error_message)
-                #print(error_message)
-
-        # --- Safety Save ---
-        if safety:
-            # Save progress so far
-            temp_df = pd.DataFrame({
-                'survey_response': survey_input[:idx+1],
-                'link1': link1,
-                'json': extracted_jsons
-            })
-            # Normalize processed jsons so far
-            normalized_data_list = []
-            for json_str in extracted_jsons:
-                try:
-                    parsed_obj = json.loads(json_str)
-                    normalized_data_list.append(pd.json_normalize(parsed_obj))
-                except json.JSONDecodeError:
-                    normalized_data_list.append(pd.DataFrame({"1": ["e"]}))
-            normalized_data = pd.concat(normalized_data_list, ignore_index=True)
-            temp_df = pd.concat([temp_df, normalized_data], axis=1)
-            # Save to CSV
-            if save_directory is None:
-                save_directory = os.getcwd()
-            temp_df.to_csv(os.path.join(save_directory, filename), index=False)
-
-    # --- Final DataFrame ---
-    normalized_data_list = []
-    for json_str in extracted_jsons:
-        try:
-            parsed_obj = json.loads(json_str)
-            normalized_data_list.append(pd.json_normalize(parsed_obj))
-        except json.JSONDecodeError:
-            normalized_data_list.append(pd.DataFrame({"1": ["e"]}))
-    normalized_data = pd.concat(normalized_data_list, ignore_index=True)
-
-    categorized_data = pd.DataFrame({
-        'survey_response': survey_input.reset_index(drop=True),
-        'link1': pd.Series(link1).reset_index(drop=True),
-        'json': pd.Series(extracted_jsons).reset_index(drop=True)
-    })
-    categorized_data = pd.concat([categorized_data, normalized_data], axis=1)
-    
-    if columns != "numbered": #if user wants text columns
-        categorized_data.columns = list(categorized_data.columns[:3]) + categories[:len(categorized_data.columns) - 3]
-
-    if to_csv:
-        if save_directory is None:
-            save_directory = os.getcwd()
-        categorized_data.to_csv(os.path.join(save_directory, filename), index=False)
-    
-    return categorized_data
-
-# image multi-class (binary) function
-def extract_image_multi_class(
-    image_description, 
-    image_input,
-    categories,
-    api_key,
-    columns="numbered",
-    user_model="gpt-4o-2024-11-20",
-    creativity=0,
-    to_csv=False,
-    safety=False,
-    filename="categorized_data.csv",
-    save_directory=None,
-    model_source="OpenAI"
-):
-    import os
-    import json
-    import pandas as pd
-    import regex
-    from tqdm import tqdm
-    import glob
-    import base64
-    from pathlib import Path
-
-    if save_directory is not None and not os.path.isdir(save_directory):
-    # Directory doesn't exist - raise an exception to halt execution
-        raise FileNotFoundError(f"Directory {save_directory} doesn't exist")
-
-    image_extensions = [
-    '*.png', '*.jpg', '*.jpeg',
-    '*.gif', '*.webp', '*.svg', '*.svgz', '*.avif', '*.apng',
-    '*.tif', '*.tiff', '*.bmp',
-    '*.heif', '*.heic', '*.ico',
-    '*.psd'
-    ]
-
-    if not isinstance(image_input, list):
-        # If image_input is a filepath (string)
-        image_files = []
-        for ext in image_extensions:
-            image_files.extend(glob.glob(os.path.join(image_input, ext)))
-    
-        print(f"Found {len(image_files)} images.")
-    else:
-        # If image_files is already a list
-        image_files = image_input
-        print(f"Provided a list of {len(image_input)} images.")
-    
-    categories_str = "\n".join(f"{i + 1}. {cat}" for i, cat in enumerate(categories))
-    cat_num = len(categories)
-    category_dict = {str(i+1): "0" for i in range(cat_num)}
-    example_JSON = json.dumps(category_dict, indent=4)
-
-    # ensure number of categories is what user wants
-    print("Categories to classify:")
-    for i, cat in enumerate(categories, 1):
-        print(f"{i}. {cat}")
-    
-    link1 = []
-    extracted_jsons = []
-
-    for i, img_path in enumerate(tqdm(image_files, desc="Categorising images"), start=0):
-    # Check validity first
-        if img_path is None or not os.path.exists(img_path):
-            link1.append("Skipped NaN input or invalid path")
-            extracted_jsons.append("""{"no_valid_image": 1}""")
-            continue  # Skip the rest of the loop iteration
-        
-    # Only open the file if path is valid
-        with open(img_path, "rb") as f:
-            encoded = base64.b64encode(f.read()).decode("utf-8")
-    
-    # Handle extension safely
-        ext = Path(img_path).suffix.lstrip(".").lower()
-        encoded_image = f"data:image/{ext};base64,{encoded}"
-    
-        prompt = [
-            {
-                "type": "text",
-                "text": (
-                    f"You are an image-tagging assistant.\n"
-                    f"Task ► Examine the attached image and decide, **for each category below**, "
-                    f"whether it is PRESENT (1) or NOT PRESENT (0).\n\n"
-                    f"Image is expected to show: {image_description}\n\n"
-                    f"Categories:\n{categories_str}\n\n"
-                    f"Output format ► Respond with **only** a JSON object whose keys are the "
-                    f"quoted category numbers ('1', '2', …) and whose values are 1 or 0. "
-                    f"No additional keys, comments, or text.\n\n"
-                    f"Example (three categories):\n"
-                    f"{example_JSON}"
-                ),
-            },
-            {
-                "type": "image_url",
-                "image_url": {"url": encoded_image, "detail": "high"},
-            },
-        ]
-        if model_source == "OpenAI":
-            from openai import OpenAI
-            client = OpenAI(api_key=api_key)
-            try:
-                response_obj = client.chat.completions.create(
-                    model=user_model,
-                    messages=[{'role': 'user', 'content': prompt}],
-                    temperature=creativity
-                )
-                reply = response_obj.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-
-        elif model_source == "Perplexity":
-            from openai import OpenAI
-            client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
-            try:
-                response_obj = client.chat.completions.create(
-                    model=user_model,
-                    messages=[{'role': 'user', 'content': prompt}],
-                    temperature=creativity
-                )
-                reply = response_obj.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        elif model_source == "Anthropic":
-            import anthropic
-            client = anthropic.Anthropic(api_key=api_key)
-            try:
-                message = client.messages.create(
-                    model=user_model,
-                    max_tokens=1024,
-                    temperature=creativity,
-                    messages=[{"role": "user", "content": prompt}]
-                )
-                reply = message.content[0].text  # Anthropic returns content as list
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        elif model_source == "Mistral":
-            from mistralai import Mistral
-            client = Mistral(api_key=api_key)
-            try:
-                response = client.chat.complete(
-                    model=user_model,
-                    messages=[
-                    {'role': 'user', 'content': prompt}
-                ],
-                temperature=creativity
-                )
-                reply = response.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        else:
-            raise ValueError("Unknown source! Choose from OpenAI, Anthropic, Perplexity, or Mistral")
-            # in situation that no JSON is found
-        if reply is not None:
-            extracted_json = regex.findall(r'\{(?:[^{}]|(?R))*\}', reply, regex.DOTALL)
-            if extracted_json:
-                cleaned_json = extracted_json[0].replace('[', '').replace(']', '').replace('\n', '').replace(" ", '').replace("  ", '')
-                extracted_jsons.append(cleaned_json)
-                #print(cleaned_json)
-            else:
-                error_message = """{"1":"e"}"""
-                extracted_jsons.append(error_message)
-                print(error_message)
-        else:
-            error_message = """{"1":"e"}"""
-            extracted_jsons.append(error_message)
-            #print(error_message)
-
-        # --- Safety Save ---
-        if safety:
-            #print(f"Saving CSV to: {save_directory}")
-            # Save progress so far
-            temp_df = pd.DataFrame({
-                'image_input': image_files[:i+1],
-                'link1': link1,
-                'json': extracted_jsons
-            })
-            # Normalize processed jsons so far
-            normalized_data_list = []
-            for json_str in extracted_jsons:
-                try:
-                    parsed_obj = json.loads(json_str)
-                    normalized_data_list.append(pd.json_normalize(parsed_obj))
-                except json.JSONDecodeError:
-                    normalized_data_list.append(pd.DataFrame({"1": ["e"]}))
-            normalized_data = pd.concat(normalized_data_list, ignore_index=True)
-            temp_df = pd.concat([temp_df, normalized_data], axis=1)
-            # Save to CSV
-            if save_directory is None:
-                save_directory = os.getcwd()
-            temp_df.to_csv(os.path.join(save_directory, filename), index=False)
-
-    # --- Final DataFrame ---
-    normalized_data_list = []
-    for json_str in extracted_jsons:
-        try:
-            parsed_obj = json.loads(json_str)
-            normalized_data_list.append(pd.json_normalize(parsed_obj))
-        except json.JSONDecodeError:
-            normalized_data_list.append(pd.DataFrame({"1": ["e"]}))
-    normalized_data = pd.concat(normalized_data_list, ignore_index=True)
-
-    categorized_data = pd.DataFrame({
-        'image_input': image_files,
-        'link1': pd.Series(link1).reset_index(drop=True),
-        'json': pd.Series(extracted_jsons).reset_index(drop=True)
-    })
-    categorized_data = pd.concat([categorized_data, normalized_data], axis=1)
-    
-    if columns != "numbered": #if user wants text columns
-        categorized_data.columns = list(categorized_data.columns[:3]) + categories[:len(categorized_data.columns) - 3]
-
-    if to_csv:
-        if save_directory is None:
-            save_directory = os.getcwd()
-        categorized_data.to_csv(os.path.join(save_directory, filename), index=False)
-    
-    return categorized_data
-
-#image score function
-def extract_image_score(
-    reference_image_description,
-    image_input,
-    reference_image,
-    api_key,
-    columns="numbered",
-    user_model="gpt-4o-2024-11-20",
-    creativity=0,
-    to_csv=False,
-    safety=False,
-    filename="categorized_data.csv",
-    save_directory=None,
-    model_source="OpenAI"
-):
-    import os
-    import json
-    import pandas as pd
-    import regex
-    from tqdm import tqdm
-    import glob
-    import base64
-    from pathlib import Path
-
-    if save_directory is not None and not os.path.isdir(save_directory):
-    # Directory doesn't exist - raise an exception to halt execution
-        raise FileNotFoundError(f"Directory {save_directory} doesn't exist")
-
-    image_extensions = [
-    '*.png', '*.jpg', '*.jpeg',
-    '*.gif', '*.webp', '*.svg', '*.svgz', '*.avif', '*.apng',
-    '*.tif', '*.tiff', '*.bmp',
-    '*.heif', '*.heic', '*.ico',
-    '*.psd'
-    ]
-
-    if not isinstance(image_input, list):
-        # If image_input is a filepath (string)
-        image_files = []
-        for ext in image_extensions:
-            image_files.extend(glob.glob(os.path.join(image_input, ext)))
-    
-        print(f"Found {len(image_files)} images.")
-    else:
-        # If image_files is already a list
-        image_files = image_input
-        print(f"Provided a list of {len(image_input)} images.")
-    
-    with open(reference_image, 'rb') as f:
-        reference_image = f"data:image/{reference_image.split('.')[-1]};base64,{base64.b64encode(f.read()).decode('utf-8')}"
-    
-    link1 = []
-    extracted_jsons = []
-
-    for i, img_path in enumerate(tqdm(image_files, desc="Categorising images"), start=0):
-    # Check validity first
-        if img_path is None or not os.path.exists(img_path):
-            link1.append("Skipped NaN input or invalid path")
-            extracted_jsons.append("""{"no_valid_image": 1}""")
-            continue  # Skip the rest of the loop iteration
-        
-    # Only open the file if path is valid
-        with open(img_path, "rb") as f:
-            encoded = base64.b64encode(f.read()).decode("utf-8")
-    
-    # Handle extension safely
-        ext = Path(img_path).suffix.lstrip(".").lower()
-        encoded_image = f"data:image/{ext};base64,{encoded}"
-
-        prompt = [
-            {
-                "type": "text",
-                "text": (
-                    f"You are a visual similarity assessment system.\n"
-            f"Task ► Compare these two images:\n"
-            f"1. REFERENCE (left): {reference_image_description}\n"
-            f"2. INPUT (right): User-provided drawing\n\n"
-            f"Rating criteria:\n"
-            f"1: No meaningful similarity (fundamentally different)\n"
-            f"2: Barely recognizable similarity (25% match)\n" 
-            f"3: Partial match (50% key features)\n"
-            f"4: Strong alignment (75% features)\n"
-            f"5: Near-perfect match (90%+ similarity)\n\n"
-            f"Output format ► Return ONLY:\n"
-            "{\n"
-            '  "score": [1-5],\n'
-            '  "summary": "reason you scored"\n'
-            "}\n\n"
-            f"Critical rules:\n"
-            f"- Score must reflect shape, proportions, and key details\n"
-            f"- List only concrete matching elements from reference\n"
-            f"- No markdown or additional text"
-                ),
-            },
-            {"type": "image_url", 
-             "image_url": {"url": reference_image, "detail": "high"}
-             },
-            {
-                "type": "image_url",
-                
-                "image_url": {"url": encoded_image, "detail": "high"},
-            },
-        ]
-        if model_source == "OpenAI":
-            from openai import OpenAI
-            client = OpenAI(api_key=api_key)
-            try:
-                response_obj = client.chat.completions.create(
-                    model=user_model,
-                    messages=[{'role': 'user', 'content': prompt}],
-                    temperature=creativity
-                )
-                reply = response_obj.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-
-        elif model_source == "Perplexity":
-            from openai import OpenAI
-            client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
-            try:
-                response_obj = client.chat.completions.create(
-                    model=user_model,
-                    messages=[{'role': 'user', 'content': prompt}],
-                    temperature=creativity
-                )
-                reply = response_obj.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        elif model_source == "Anthropic":
-            import anthropic
-            client = anthropic.Anthropic(api_key=api_key)
-            try:
-                message = client.messages.create(
-                    model=user_model,
-                    max_tokens=1024,
-                    temperature=creativity,
-                    messages=[{"role": "user", "content": prompt}]
-                )
-                reply = message.content[0].text  # Anthropic returns content as list
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        elif model_source == "Mistral":
-            from mistralai import Mistral
-            client = Mistral(api_key=api_key)
-            try:
-                response = client.chat.complete(
-                    model=user_model,
-                    messages=[
-                    {'role': 'user', 'content': prompt}
-                ],
-                temperature=creativity
-                )
-                reply = response.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        else:
-            raise ValueError("Unknown source! Choose from OpenAI, Anthropic, Perplexity, or Mistral")
-            # in situation that no JSON is found
-        if reply is not None:
-            extracted_json = regex.findall(r'\{(?:[^{}]|(?R))*\}', reply, regex.DOTALL)
-            if extracted_json:
-                cleaned_json = extracted_json[0].replace('[', '').replace(']', '').replace('\n', '').replace(" ", '').replace("  ", '')
-                extracted_jsons.append(cleaned_json)
-                #print(cleaned_json)
-            else:
-                error_message = """{"1":"e"}"""
-                extracted_jsons.append(error_message)
-                print(error_message)
-        else:
-            error_message = """{"1":"e"}"""
-            extracted_jsons.append(error_message)
-            #print(error_message)
-
-        # --- Safety Save ---
-        if safety:
-            # Save progress so far
-            temp_df = pd.DataFrame({
-                'image_input': image_files[:i+1],
-                'link1': link1,
-                'json': extracted_jsons
-            })
-            # Normalize processed jsons so far
-            normalized_data_list = []
-            for json_str in extracted_jsons:
-                try:
-                    parsed_obj = json.loads(json_str)
-                    normalized_data_list.append(pd.json_normalize(parsed_obj))
-                except json.JSONDecodeError:
-                    normalized_data_list.append(pd.DataFrame({"1": ["e"]}))
-            normalized_data = pd.concat(normalized_data_list, ignore_index=True)
-            temp_df = pd.concat([temp_df, normalized_data], axis=1)
-            # Save to CSV
-            if save_directory is None:
-                save_directory = os.getcwd()
-            temp_df.to_csv(os.path.join(save_directory, filename), index=False)
-
-    # --- Final DataFrame ---
-    normalized_data_list = []
-    for json_str in extracted_jsons:
-        try:
-            parsed_obj = json.loads(json_str)
-            normalized_data_list.append(pd.json_normalize(parsed_obj))
-        except json.JSONDecodeError:
-            normalized_data_list.append(pd.DataFrame({"1": ["e"]}))
-    normalized_data = pd.concat(normalized_data_list, ignore_index=True)
-
-    categorized_data = pd.DataFrame({
-        'image_input': image_files,
-        'link1': pd.Series(link1).reset_index(drop=True),
-        'json': pd.Series(extracted_jsons).reset_index(drop=True)
-    })
-    categorized_data = pd.concat([categorized_data, normalized_data], axis=1)
-
-    if to_csv:
-        if save_directory is None:
-            save_directory = os.getcwd()
-        categorized_data.to_csv(os.path.join(save_directory, filename), index=False)
-    
-    return categorized_data
-
-# image features function
-def extract_image_features(
-    image_description, 
-    image_input,
-    features_to_extract,
-    api_key,
-    columns="numbered",
-    user_model="gpt-4o-2024-11-20",
-    creativity=0,
-    to_csv=False,
-    safety=False,
-    filename="categorized_data.csv",
-    save_directory=None,
-    model_source="OpenAI"
-):
-    import os
-    import json
-    import pandas as pd
-    import regex
-    from tqdm import tqdm
-    import glob
-    import base64
-    from pathlib import Path
-
-    if save_directory is not None and not os.path.isdir(save_directory):
-    # Directory doesn't exist - raise an exception to halt execution
-        raise FileNotFoundError(f"Directory {save_directory} doesn't exist")
-
-    image_extensions = [
-    '*.png', '*.jpg', '*.jpeg',
-    '*.gif', '*.webp', '*.svg', '*.svgz', '*.avif', '*.apng',
-    '*.tif', '*.tiff', '*.bmp',
-    '*.heif', '*.heic', '*.ico',
-    '*.psd'
-    ]
-
-    if not isinstance(image_input, list):
-        # If image_input is a filepath (string)
-        image_files = []
-        for ext in image_extensions:
-            image_files.extend(glob.glob(os.path.join(image_input, ext)))
-    
-        print(f"Found {len(image_files)} images.")
-    else:
-        # If image_files is already a list
-        image_files = image_input
-        print(f"Provided a list of {len(image_input)} images.")
-    
-    categories_str = "\n".join(f"{i + 1}. {cat}" for i, cat in enumerate(features_to_extract))
-    cat_num = len(features_to_extract)
-    category_dict = {str(i+1): "0" for i in range(cat_num)}
-    example_JSON = json.dumps(category_dict, indent=4)
-
-    # ensure number of categories is what user wants
-    print("\nPlease verify the categories you entered:")
-    for i, cat in enumerate(features_to_extract, 1):
-        print(f"{i}. {cat}")
-    print("\nIf the list above is correct, type 'next' and press Enter to continue.")
-    print("Type 'exit' to cancel the operation.")
-
-    print("\nPlease verify the categories you entered:")
-    for i, cat in enumerate(features_to_extract, 1):
-        print(f"{i}. {cat}")
-    response = input("\nIf the list above is correct, type 'next' and press Enter to continue: ")
-    while response.strip().lower() != "next":
-        response = input("Please type 'next' to continue: ")
-    
-    link1 = []
-    extracted_jsons = []
-
-    for i, img_path in enumerate(
-        tqdm(image_files, desc="Categorising images"), start=0):
-    # encode this specific image once
-        with open(img_path, "rb") as f:
-            encoded = base64.b64encode(f.read()).decode("utf-8")
-        ext = Path(img_path).suffix.lstrip(".").lower()
-        encoded_image = f"data:image/{ext};base64,{encoded}"
-
-        prompt = [
-            {
-                "type": "text",
-                "text": (
-                    f"You are a visual question answering assistant.\n"
-            f"Task ► Analyze the attached image and answer these specific questions:\n\n"
-            f"Image context: {image_description}\n\n"
-            f"Questions to answer:\n{categories_str}\n\n"
-            f"Output format ► Return **only** a JSON object where:\n"
-            f"- Keys are question numbers ('1', '2', ...)\n"
-            f"- Values are concise answers (numbers, short phrases)\n\n"
-            f"Example for 3 questions:\n"
-            "{\n"
-            '  "1": "4",\n'
-            '  "2": "blue",\n'
-            '  "3": "yes"\n'
-            "}\n\n"
-            f"Important rules:\n"
-            f"1. Answer directly - no explanations\n"
-            f"2. Use exact numerical values when possible\n"
-            f"3. For yes/no questions, use 'yes' or 'no'\n"
-            f"4. Never add extra keys or formatting"
-                ),
-            },
-            {
-                "type": "image_url",
-                "image_url": {"url": encoded_image, "detail": "high"},
-            },
-        ]
-        if model_source == "OpenAI":
-            from openai import OpenAI
-            client = OpenAI(api_key=api_key)
-            try:
-                response_obj = client.chat.completions.create(
-                    model=user_model,
-                    messages=[{'role': 'user', 'content': prompt}],
-                    temperature=creativity
-                )
-                reply = response_obj.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-
-        elif model_source == "Perplexity":
-            from openai import OpenAI
-            client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
-            try:
-                response_obj = client.chat.completions.create(
-                    model=user_model,
-                    messages=[{'role': 'user', 'content': prompt}],
-                    temperature=creativity
-                )
-                reply = response_obj.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        elif model_source == "Anthropic":
-            import anthropic
-            client = anthropic.Anthropic(api_key=api_key)
-            try:
-                message = client.messages.create(
-                    model=user_model,
-                    max_tokens=1024,
-                    temperature=creativity,
-                    messages=[{"role": "user", "content": prompt}]
-                )
-                reply = message.content[0].text  # Anthropic returns content as list
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        elif model_source == "Mistral":
-            from mistralai import Mistral
-            client = Mistral(api_key=api_key)
-            try:
-                response = client.chat.complete(
-                    model=user_model,
-                    messages=[
-                    {'role': 'user', 'content': prompt}
-                ],
-                temperature=creativity
-                )
-                reply = response.choices[0].message.content
-                link1.append(reply)
-            except Exception as e:
-                print(f"An error occurred: {e}")
-                link1.append(f"Error processing input: {e}")
-        else:
-            raise ValueError("Unknown source! Choose from OpenAI, Anthropic, Perplexity, or Mistral")
-            # in situation that no JSON is found
-        if reply is not None:
-            extracted_json = regex.findall(r'\{(?:[^{}]|(?R))*\}', reply, regex.DOTALL)
-            if extracted_json:
-                cleaned_json = extracted_json[0].replace('[', '').replace(']', '').replace('\n', '').replace(" ", '').replace("  ", '')
-                extracted_jsons.append(cleaned_json)
-                #print(cleaned_json)
-            else:
-                error_message = """{"1":"e"}"""
-                extracted_jsons.append(error_message)
-                print(error_message)
-        else:
-            error_message = """{"1":"e"}"""
-            extracted_jsons.append(error_message)
-            #print(error_message)
-
-        # --- Safety Save ---
-        if safety:
-            #print(f"Saving CSV to: {save_directory}")
-            # Save progress so far
-            temp_df = pd.DataFrame({
-                'image_input': image_files[:i+1],
-                'link1': link1,
-                'json': extracted_jsons
-            })
-            # Normalize processed jsons so far
-            normalized_data_list = []
-            for json_str in extracted_jsons:
-                try:
-                    parsed_obj = json.loads(json_str)
-                    normalized_data_list.append(pd.json_normalize(parsed_obj))
-                except json.JSONDecodeError:
-                    normalized_data_list.append(pd.DataFrame({"1": ["e"]}))
-            normalized_data = pd.concat(normalized_data_list, ignore_index=True)
-            temp_df = pd.concat([temp_df, normalized_data], axis=1)
-            # Save to CSV
-            if save_directory is None:
-                save_directory = os.getcwd()
-            temp_df.to_csv(os.path.join(save_directory, filename), index=False)
-
-    # --- Final DataFrame ---
-    normalized_data_list = []
-    for json_str in extracted_jsons:
-        try:
-            parsed_obj = json.loads(json_str)
-            normalized_data_list.append(pd.json_normalize(parsed_obj))
-        except json.JSONDecodeError:
-            normalized_data_list.append(pd.DataFrame({"1": ["e"]}))
-    normalized_data = pd.concat(normalized_data_list, ignore_index=True)
-
-    categorized_data = pd.DataFrame({
-        'image_input': image_files,
-        'link1': pd.Series(link1).reset_index(drop=True),
-        'json': pd.Series(extracted_jsons).reset_index(drop=True)
-    })
-    categorized_data = pd.concat([categorized_data, normalized_data], axis=1)
-    
-    if columns != "numbered": #if user wants text columns
-        categorized_data.columns = list(categorized_data.columns[:3]) + categories[:len(categorized_data.columns) - 3]
-
-    if to_csv:
-        if save_directory is None:
-            save_directory = os.getcwd()
-        categorized_data.to_csv(os.path.join(save_directory, filename), index=False)
-    
-    return categorized_data
-
-#extract categories from corpus
-def explore_corpus(
-    survey_question, 
-    survey_input,
-    api_key,
-    cat_num=10,
-    divisions=5,
-    user_model="gpt-4o-2024-11-20",
-    creativity=0,
-    to_csv=False,
-    filename="categorized_data.csv",
-    save_directory=None,
-    model_source="OpenAI"
-):
-    import os
-    import pandas as pd
-    import random
-    from openai import OpenAI
-    from openai import OpenAI, BadRequestError
-    from tqdm import tqdm
-
-    print(f"Exploring class for question: '{survey_question}'.\n          {cat_num * divisions} unique categories to be extracted.")
-    print()
-
-    chunk_size = round(max(1, len(survey_input) / divisions),0)
-    chunk_size = int(chunk_size)
-
-    if chunk_size < (cat_num/2):
-        raise ValueError(f"Cannot extract {cat_num} categories from chunks of only {chunk_size} responses. \n" 
-                    f"Choose one solution: \n"
-                    f"(1) Reduce 'divisions' parameter (currently {divisions}) to create larger chunks, or \n"
-                    f"(2) Reduce 'cat_num' parameter (currently {cat_num}) to extract fewer categories per chunk.")
-
-    random_chunks = []
-    for i in range(divisions):
-        chunk = survey_input.sample(n=chunk_size).tolist()
-        random_chunks.append(chunk)
-    
-    responses = []
-    responses_list = []
-    
-    for i in tqdm(range(divisions), desc="Processing chunks"):
-        survey_participant_chunks = '; '.join(random_chunks[i])
-        prompt = f"""Identify {cat_num} broad categories of responses to the question "{survey_question}" in the following list of responses. \
-Responses are each separated by a semicolon. \
-Responses are contained within triple backticks here: ```{survey_participant_chunks}``` \
-Number your categories from 1 through {cat_num} and be concise with the category labels and provide no description of the categories."""
-        
-        if model_source == "OpenAI":
-            client = OpenAI(api_key=api_key)
-            try:
-                response_obj = client.chat.completions.create(
-                    model=user_model,
-                    messages=[{'role': 'user', 'content': prompt}],
-                    temperature=creativity
-                )
-                reply = response_obj.choices[0].message.content
-                responses.append(reply)
-            except BadRequestError as e:
-                if "context_length_exceeded" in str(e) or "maximum context length" in str(e):
-                    error_msg = (f"Token limit exceeded for model {user_model}. "
-                        f"Try increasing the 'iterations' parameter to create smaller chunks.")
-                    raise ValueError(error_msg)
-                else:
-                    print(f"OpenAI API error: {e}")
-            except Exception as e:
-                print(f"An error occurred: {e}")
-        else:
-            raise ValueError(f"Unsupported model_source: {model_source}")
-        
-        # Extract just the text as a list
-        items = [line.split('. ', 1)[1] for line in responses[i].split('\n') if '. ' in line]
-        responses_list.append(items)
-
-    flat_list = [item.lower() for sublist in responses_list for item in sublist]
-
-    #convert flat_list to a df
-    df = pd.DataFrame(flat_list, columns=['Category'])
-    counts = pd.Series(flat_list).value_counts()  # Use original list before conversion
-    df['counts'] = df['Category'].map(counts)
-    df = df.sort_values(by='counts', ascending=False).reset_index(drop=True)
-    df = df.drop_duplicates(subset='Category', keep='first').reset_index(drop=True)
-    
-    return df