npcsh 0.3.31__py3-none-any.whl → 1.0.0__py3-none-any.whl

npcsh/conversation.py

@@ -1,291 +0,0 @@
-########
-########
-########
-########
-######## CONVERSATION
-########
-from typing import Any, Dict, Generator, List
-import os
-import anthropic
-
-from openai import OpenAI
-from google.generativeai import types
-import google.generativeai as genai
-from .npc_sysenv import get_system_message
-
-
-def get_ollama_conversation(
-    messages: List[Dict[str, str]],
-    model: str,
-    npc: Any = None,
-    tools: list = None,
-    images=None,
-    **kwargs,
-) -> List[Dict[str, str]]:
-    """
-    Function Description:
-        This function generates a conversation using the Ollama API.
-    Args:
-        messages (List[Dict[str, str]]): The list of messages in the conversation.
-        model (str): The model to use for the conversation.
-    Keyword Args:
-        npc (Any): The NPC object.
-    Returns:
-        List[Dict[str, str]]: The list of messages in the conversation.
-    """
-    import ollama
-
-    messages_copy = messages.copy()
-    if messages_copy[0]["role"] != "system":
-        if npc is not None:
-            system_message = get_system_message(npc)
-            messages_copy.insert(0, {"role": "system", "content": system_message})
-
-    response = ollama.chat(model=model, messages=messages_copy)
-    messages_copy.append(response["message"])
-    return messages_copy
-
-
-def get_openai_conversation(
-    messages: List[Dict[str, str]],
-    model: str,
-    npc: Any = None,
-    tools: list = None,
-    api_key: str = None,
-    images=None,
-    **kwargs,
-) -> List[Dict[str, str]]:
-    """
-    Function Description:
-        This function generates a conversation using the OpenAI API.
-    Args:
-        messages (List[Dict[str, str]]): The list of messages in the conversation.
-        model (str): The model to use for the conversation.
-    Keyword Args:
-        npc (Any): The NPC object.
-        api_key (str): The API key for accessing the OpenAI API.
-    Returns:
-        List[Dict[str, str]]: The list of messages in the conversation.
-    """
-
-    try:
-        if api_key is None:
-            api_key = os.environ["OPENAI_API_KEY"]
-        client = OpenAI(api_key=api_key)
-
-        system_message = (
-            get_system_message(npc) if npc else "You are a helpful assistant."
-        )
-
-        if messages is None:
-            messages = []
-
-        # Ensure the system message is at the beginning
-        if not any(msg["role"] == "system" for msg in messages):
-            messages.insert(0, {"role": "system", "content": system_message})
-
-        # messages should already include the user's latest message
-
-        # Make the API call with the messages including the latest user input
-        completion = client.chat.completions.create(
-            model=model, messages=messages, **kwargs
-        )
-
-        response_message = completion.choices[0].message
-        messages.append({"role": "assistant", "content": response_message.content})
-
-        return messages
-
-    except Exception as e:
-        return f"Error interacting with OpenAI: {e}"
-
-
-def get_openai_like_conversation(
-    messages: List[Dict[str, str]],
-    model: str,
-    api_url: str,
-    npc: Any = None,
-    images=None,
-    tools: list = None,
-    api_key: str = None,
-    **kwargs,
-) -> List[Dict[str, str]]:
-    """
-    Function Description:
-        This function generates a conversation using an OpenAI-like API.
-    Args:
-        messages (List[Dict[str, str]]): The list of messages in the conversation.
-        model (str): The model to use for the conversation.
-    Keyword Args:
-        npc (Any): The NPC object.
-        api_url (str): The URL of the API endpoint.
-        api_key (str): The API key for accessing the API.
-    Returns:
-        List[Dict[str, str]]: The list of messages in the conversation.
-    """
-
-    if api_url is None:
-        raise ValueError("api_url is required for openai-like provider")
-    if api_key is None:
-        api_key = "dummy_api_key"
-    try:
-        client = OpenAI(api_key=api_key, base_url=api_url)
-
-        system_message = (
-            get_system_message(npc) if npc else "You are a helpful assistant."
-        )
-
-        if messages is None:
-            messages = []
-
-        # Ensure the system message is at the beginning
-        if not any(msg["role"] == "system" for msg in messages):
-            messages.insert(0, {"role": "system", "content": system_message})
-
-        # messages should already include the user's latest message
-
-        # Make the API call with the messages including the latest user input
-
-        completion = client.chat.completions.create(
-            model=model, messages=messages, **kwargs
-        )
-        response_message = completion.choices[0].message
-        messages.append({"role": "assistant", "content": response_message.content})
-
-        return messages
-
-    except Exception as e:
-        return f"Error interacting with OpenAI: {e}"
-
-    return messages
-
-
-def get_anthropic_conversation(
-    messages: List[Dict[str, str]],
-    model: str,
-    npc: Any = None,
-    tools: list = None,
-    images=None,
-    api_key: str = None,
-    **kwargs,
-) -> List[Dict[str, str]]:
-    """
-    Function Description:
-        This function generates a conversation using the Anthropic API.
-    Args:
-        messages (List[Dict[str, str]]): The list of messages in the conversation.
-        model (str): The model to use for the conversation.
-    Keyword Args:
-        npc (Any): The NPC object.
-        api_key (str): The API key for accessing the Anthropic API.
-    Returns:
-        List[Dict[str, str]]: The list of messages in the conversation.
-    """
-
-    try:
-        if api_key is None:
-            api_key = os.getenv("ANTHROPIC_API_KEY", None)
-        system_message = get_system_message(npc) if npc else ""
-        client = anthropic.Anthropic(api_key=api_key)
-        last_user_message = None
-        for msg in reversed(messages):
-            if msg["role"] == "user":
-                last_user_message = msg["content"]
-                break
-
-        if last_user_message is None:
-            raise ValueError("No user message found in the conversation history.")
-
-        # if a sys message is in messages, remove it
-        if messages[0]["role"] == "system":
-            messages.pop(0)
-
-        message = client.messages.create(
-            model=model,
-            system=system_message,  # Include system message in each turn for Anthropic
-            messages=messages,  # Send only the last user message
-            max_tokens=8192,
-            **kwargs,
-        )
-
-        messages.append({"role": "assistant", "content": message.content[0].text})
-
-        return messages
-
-    except Exception as e:
-        return f"Error interacting with Anthropic conversations: {e}"
-
-
-def get_gemini_conversation(
-    messages: List[Dict[str, str]],
-    model: str,
-    npc: Any = None,
-    tools: list = None,
-    api_key: str = None,
-) -> List[Dict[str, str]]:
-    """
-    Function Description:
-        This function generates a conversation using the Gemini API.
-    Args:
-        messages (List[Dict[str, str]]): The list of messages in the conversation.
-        model (str): The model to use for the conversation.
-    Keyword Args:
-        npc (Any): The NPC object.
-    Returns:
-        List[Dict[str, str]]: The list of messages in the conversation.
-    """
-    # Make the API call to Gemini
-
-    # print(messages)
-    response = get_gemini_response(
-        messages[-1]["content"], model, messages=messages[1:], npc=npc
-    )
-    # print(response)
-    return response.get("messages", [])
-
-
-def get_deepseek_conversation(
-    messages: List[Dict[str, str]],
-    model: str,
-    npc: Any = None,
-    tools: list = None,
-    api_key: str = None,
-) -> List[Dict[str, str]]:
-    """
-    Function Description:
-        This function generates a conversation using the DeepSeek API.
-    Args:
-        messages (List[Dict[str, str]]): The list of messages in the conversation.
-        model (str): The model to use for the conversation.
-    Keyword Args:
-        npc (Any): The NPC object.
-    Returns:
-        List[Dict[str, str]]: The list of messages in the conversation.
-    """
-
-    system_message = get_system_message(npc) if npc else "You are a helpful assistant."
-
-    # Prepare the messages list
-    if messages is None or len(messages) == 0:
-        messages = [{"role": "system", "content": system_message}]
-    elif not any(msg["role"] == "system" for msg in messages):
-        messages.insert(0, {"role": "system", "content": system_message})
-
-    # Make the API call to DeepSeek
-    try:
-        response = get_deepseek_response(
-            messages[-1]["content"], model, messages=messages, npc=npc
-        )
-        messages.append(
-            {"role": "assistant", "content": response.get("response", "No response")}
-        )
-
-    except Exception as e:
-        messages.append(
-            {
-                "role": "assistant",
-                "content": f"Error interacting with DeepSeek: {str(e)}",
-            }
-        )
-
-    return messages

npcsh/data_models.py

@@ -1,46 +0,0 @@
-from pydantic import BaseModel
-from typing import List, Dict
-
-
-class NPC(BaseModel):
-    name: str
-    primary_directive: str
-    model: str
-    provider: str
-    api_url: str
-    tools: List[str]
-    use_default_tools: bool
-
-
-class Tool(BaseModel):
-    tool_name: str
-    description: str
-    steps: List[Dict[str, str]]
-
-
-class ToolStep(BaseModel):
-    engine: str
-    code: str
-
-
-class Context(BaseModel):
-    databases: List[str]
-    files: List[str]
-    vars: List[Dict[str, str]]
-
-
-class Pipeline(BaseModel):
-    steps: List[Dict[str, str]]
-
-
-class PipelineStep(BaseModel):
-    tool: str
-    args: List[str]
-    model: str
-    provider: str
-    task: str
-    npc: str
-
-
-class Fabrication(BaseModel):
-    spell: str

npcsh/dataframes.py

@@ -1,163 +0,0 @@
-## functions for dataframes
-import os
-import sqlite3
-import json
-import pandas as pd
-import numpy as np
-import io
-from PIL import Image
-from typing import Optional
-
-from .llm_funcs import get_llm_response
-from .audio import process_audio
-from .video import process_video
-
-from .load_data import load_pdf, load_csv, load_json, load_excel, load_txt, load_image
-
-
-def load_data_into_table(
-    file_path: str, table_name: str, cursor: sqlite3.Cursor, conn: sqlite3.Connection
-) -> None:
-    """
-    Function Description:
-        This function is used to load data into a table.
-    Args:
-        file_path : str : The file path.
-        table_name : str : The table name.
-        cursor : sqlite3.Cursor : The SQLite cursor.
-        conn : sqlite3.Connection : The SQLite connection.
-    Keyword Args:
-        None
-    Returns:
-        None
-    """
-    try:
-        if not os.path.exists(file_path):
-            raise FileNotFoundError(f"File not found: {file_path}")
-
-        # Determine file type and load data
-        if file_path.endswith(".csv"):
-            df = pd.read_csv(file_path)
-        elif file_path.endswith(".pdf"):
-            df = load_pdf(file_path)
-        elif file_path.endswith((".txt", ".log", ".md")):
-            df = load_txt(file_path)
-        elif file_path.endswith((".xls", ".xlsx")):
-            df = load_excel(file_path)
-        elif file_path.lower().endswith(
-            (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff")
-        ):
-            # Handle images as NumPy arrays
-            df = load_image(file_path)
-        elif file_path.lower().endswith(
-            (".mp4", ".avi", ".mov", ".mkv")
-        ):  # Video files
-            video_frames, audio_array = process_video(file_path)
-            # Store video frames and audio
-            df = pd.DataFrame(
-                {
-                    "video_frames": [video_frames.tobytes()],
-                    "shape": [video_frames.shape],
-                    "dtype": [video_frames.dtype.str],
-                    "audio_array": (
-                        [audio_array.tobytes()] if audio_array is not None else None
-                    ),
-                    "audio_rate": [sr] if audio_array is not None else None,
-                }
-            )
-
-        elif file_path.lower().endswith((".mp3", ".wav", ".ogg")):  # Audio files
-            audio_array, sr = process_audio(file_path)
-            df = pd.DataFrame(
-                {
-                    "audio_array": [audio_array.tobytes()],
-                    "audio_rate": [sr],
-                }
-            )
-        else:
-            # Attempt to load as text if no other type matches
-            try:
-                df = load_txt(file_path)
-            except Exception as e:
-                print(f"Could not load file: {e}")
-                return
-
-        # Store DataFrame in the database
-        df.to_sql(table_name, conn, if_exists="replace", index=False)
-        print(f"Data from '{file_path}' loaded into table '{table_name}'")
-
-    except Exception as e:
-        raise e  # Re-raise the exception for handling in enter_observation_mode
-
-
-def create_new_table(cursor: sqlite3.Cursor, conn: sqlite3.Connection) -> None:
-    """
-    Function Description:
-        This function is used to create a new table.
-    Args:
-        cursor : sqlite3.Cursor : The SQLite cursor.
-        conn : sqlite3.Connection : The SQLite connection.
-    Keyword Args:
-        None
-    Returns:
-        None
-    """
-
-    table_name = input("Enter new table name: ").strip()
-    columns = input("Enter column names separated by commas: ").strip()
-
-    create_query = (
-        f"CREATE TABLE {table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, {columns})"
-    )
-    cursor.execute(create_query)
-    conn.commit()
-    print(f"Table '{table_name}' created successfully.")
-
-
-def delete_table(cursor: sqlite3.Cursor, conn: sqlite3.Connection) -> None:
-    """
-    Function Description:
-        This function is used to delete a table.
-    Args:
-        cursor : sqlite3.Cursor : The SQLite cursor.
-        conn : sqlite3.Connection : The SQLite connection.
-    Keyword Args:
-        None
-    Returns:
-        None
-    """
-
-    table_name = input("Enter table name to delete: ").strip()
-    cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
-    conn.commit()
-    print(f"Table '{table_name}' deleted successfully.")
-
-
-def add_observation(
-    cursor: sqlite3.Cursor, conn: sqlite3.Connection, table_name: str
-) -> None:
-    """
-    Function Description:
-        This function is used to add an observation.
-    Args:
-        cursor : sqlite3.Cursor : The SQLite cursor.
-        conn : sqlite3.Connection : The SQLite connection.
-        table_name : str : The table name.
-    Keyword Args:
-        None
-    Returns:
-        None
-    """
-
-    cursor.execute(f"PRAGMA table_info({table_name})")
-    columns = [column[1] for column in cursor.fetchall() if column[1] != "id"]
-
-    values = []
-    for column in columns:
-        value = input(f"Enter value for {column}: ").strip()
-        values.append(value)
-
-    insert_query = f"INSERT INTO {table_name} ({','.join(columns)}) VALUES ({','.join(['?' for _ in columns])})"
-    cursor.execute(insert_query, values)
-    conn.commit()
-    print("Observation added successfully.")

npcsh/embeddings.py

@@ -1,168 +0,0 @@
-#######
-#######
-#######
-#######
-####### EMBEDDINGS
-#######
-from typing import List, Dict, Optional
-import numpy as np
-from npcsh.npc_sysenv import (
-    NPCSH_VECTOR_DB_PATH,
-    NPCSH_EMBEDDING_MODEL,
-    NPCSH_EMBEDDING_PROVIDER,
-    chroma_client,
-)
-from openai import OpenAI
-import anthropic
-
-
-def get_ollama_embeddings(
-    texts: List[str], model: str = "nomic-embed-text"
-) -> List[List[float]]:
-    """Generate embeddings using Ollama."""
-    import ollama
-
-    embeddings = []
-    for text in texts:
-        response = ollama.embeddings(model=model, prompt=text)
-        embeddings.append(response["embedding"])
-    return embeddings
-
-
-def get_openai_embeddings(
-    texts: List[str], model: str = "text-embedding-3-small"
-) -> List[List[float]]:
-    """Generate embeddings using OpenAI."""
-    client = OpenAI(api_key=openai_api_key)
-    response = client.embeddings.create(input=texts, model=model)
-    return [embedding.embedding for embedding in response.data]
-
-
-def get_openai_like_embeddings(
-    texts: List[str], model, api_url=None, api_key=None
-) -> List[List[float]]:
-    """Generate embeddings using OpenAI."""
-    client = OpenAI(api_key=openai_api_key, base_url=api_url)
-    response = client.embeddings.create(input=texts, model=model)
-    return [embedding.embedding for embedding in response.data]
-
-
-def get_anthropic_embeddings(
-    texts: List[str], model: str = "claude-3-haiku-20240307"
-) -> List[List[float]]:
-    """Generate embeddings using Anthropic."""
-    client = anthropic.Anthropic(api_key=anthropic_api_key)
-    embeddings = []
-    for text in texts:
-        # response = client.messages.create(
-        #    model=model, max_tokens=1024, messages=[{"role": "user", "content": text}]
-        # )
-        # Placeholder for actual embedding
-        embeddings.append([0.0] * 1024)  # Replace with actual embedding when available
-    return embeddings
-
-
-def store_embeddings_for_model(
-    texts,
-    embeddings,
-    metadata=None,
-    model: str = NPCSH_EMBEDDING_MODEL,
-    provider: str = NPCSH_EMBEDDING_PROVIDER,
-):
-    collection_name = f"{provider}_{model}_embeddings"
-    collection = chroma_client.get_collection(collection_name)
-
-    # Create meaningful metadata for each document (adjust as necessary)
-    if metadata is None:
-        metadata = [{"text_length": len(text)} for text in texts]  # Example metadata
-        print(
-            "metadata is none, creating metadata for each document as the length of the text"
-        )
-    # Add embeddings to the collection with metadata
-    collection.add(
-        ids=[str(i) for i in range(len(texts))],
-        embeddings=embeddings,
-        metadatas=metadata,  # Passing populated metadata
-        documents=texts,
-    )
-
-
-def delete_embeddings_from_collection(collection, ids):
-    """Delete embeddings by id from Chroma collection."""
-    if ids:
-        collection.delete(ids=ids)  # Only delete if ids are provided
-
-
-def search_similar_texts(
-    query: str,
-    docs_to_embed: Optional[List[str]] = None,
-    top_k: int = 5,
-    db_path: str = NPCSH_VECTOR_DB_PATH,
-    embedding_model: str = NPCSH_EMBEDDING_MODEL,
-    embedding_provider: str = NPCSH_EMBEDDING_PROVIDER,
-) -> List[Dict[str, any]]:
-    """
-    Search for similar texts using either a Chroma database or direct embedding comparison.
-    """
-
-    print(f"\nQuery to embed: {query}")
-    embedded_search_term = get_ollama_embeddings([query], embedding_model)[0]
-    # print(f"Query embedding: {embedded_search_term}")
-
-    if docs_to_embed is None:
-        # Fetch from the database if no documents to embed are provided
-        collection_name = f"{embedding_provider}_{embedding_model}_embeddings"
-        collection = chroma_client.get_collection(collection_name)
-        results = collection.query(
-            query_embeddings=[embedded_search_term], n_results=top_k
-        )
-        # Constructing and returning results
-        return [
-            {"id": id, "score": float(distance), "text": document}
-            for id, distance, document in zip(
-                results["ids"][0], results["distances"][0], results["documents"][0]
-            )
-        ]
-
-    print(f"\nNumber of documents to embed: {len(docs_to_embed)}")
-
-    # Get embeddings for provided documents
-    raw_embeddings = get_ollama_embeddings(docs_to_embed, embedding_model)
-
-    output_embeddings = []
-    for idx, emb in enumerate(raw_embeddings):
-        if emb:  # Exclude any empty embeddings
-            output_embeddings.append(emb)
-
-    # Convert to numpy arrays for calculations
-    doc_embeddings = np.array(output_embeddings)
-    query_embedding = np.array(embedded_search_term)
-
-    # Check for zero-length embeddings
-    if len(doc_embeddings) == 0:
-        raise ValueError("No valid document embeddings found")
-
-    # Normalize embeddings to avoid division by zeros
-    doc_norms = np.linalg.norm(doc_embeddings, axis=1, keepdims=True)
-    query_norm = np.linalg.norm(query_embedding)
-
-    # Ensure no zero vectors are being used in cosine similarity
-    if query_norm == 0:
-        raise ValueError("Query embedding is zero-length")
-
-    # Calculate cosine similarities
-    cosine_similarities = np.dot(doc_embeddings, query_embedding) / (
-        doc_norms.flatten() * query_norm
-    )
-
-    # Get indices of top K documents
-    top_indices = np.argsort(cosine_similarities)[::-1][:top_k]
-
-    return [
-        {
-            "id": str(idx),
-            "score": float(cosine_similarities[idx]),
-            "text": docs_to_embed[idx],
-        }
-        for idx in top_indices
-    ]