lollms-client 0.25.5__py3-none-any.whl → 0.26.0__py3-none-any.whl

lollms_client/llm_bindings/groq/__init__.py

@@ -0,0 +1,292 @@
+import base64
+import os
+from io import BytesIO
+from pathlib import Path
+from typing import Optional, Callable, List, Union, Dict
+
+from lollms_client.lollms_discussion import LollmsDiscussion, LollmsMessage
+from lollms_client.lollms_llm_binding import LollmsLLMBinding
+from lollms_client.lollms_types import MSG_TYPE
+from ascii_colors import ASCIIColors, trace_exception
+
+import pipmaster as pm
+
+# Ensure the required packages are installed
+pm.ensure_packages(["groq", "pillow", "tiktoken"])
+
+import groq
+from PIL import Image, ImageDraw
+import tiktoken
+
+BindingName = "GroqBinding"
+
+class GroqBinding(LollmsLLMBinding):
+    """
+    Groq API binding implementation.
+
+    This binding allows communication with Groq's LPU-powered inference service,
+    known for its high-speed generation. It uses an OpenAI-compatible API structure.
+    """
+
+    def __init__(self,
+                 model_name: str = "llama3-8b-8192",
+                 groq_api_key: str = None,
+                 **kwargs
+                 ):
+        """
+        Initialize the GroqBinding.
+
+        Args:
+            model_name (str): The name of the Groq model to use.
+            groq_api_key (str): The API key for the Groq service.
+        """
+        super().__init__(binding_name=BindingName)
+        self.model_name = model_name
+        self.groq_api_key = groq_api_key or os.getenv("GROQ_API_KEY")
+
+        if not self.groq_api_key:
+            raise ValueError("Groq API key is required. Set it via 'groq_api_key' or GROQ_API_KEY env var.")
+
+        try:
+            self.client = groq.Groq(api_key=self.groq_api_key)
+        except Exception as e:
+            ASCIIColors.error(f"Failed to configure Groq client: {e}")
+            self.client = None
+            raise ConnectionError(f"Could not configure Groq client: {e}") from e
+
+    def _construct_parameters(self,
+                              temperature: float,
+                              top_p: float,
+                              n_predict: int,
+                              seed: Optional[int]) -> Dict[str, any]:
+        """Builds a parameters dictionary for the Groq API."""
+        params = {}
+        # Groq API mirrors OpenAI's parameters
+        if temperature is not None: params['temperature'] = float(temperature)
+        if top_p is not None: params['top_p'] = top_p
+        if n_predict is not None: params['max_tokens'] = n_predict
+        if seed is not None: params['seed'] = seed
+        return params
+
+    def _prepare_messages(self, discussion: LollmsDiscussion, branch_tip_id: Optional[str] = None) -> List[Dict[str, any]]:
+        """Prepares the message list for the Groq API from a LollmsDiscussion."""
+        history = []
+        if discussion.system_prompt:
+            history.append({"role": "system", "content": discussion.system_prompt})
+
+        for msg in discussion.get_messages(branch_tip_id):
+            role = 'user' if msg.sender_type == "user" else 'assistant'
+            # Note: Groq models currently do not support image inputs.
+            # We only process the text content.
+            if msg.content:
+                history.append({'role': role, 'content': msg.content})
+        return history
+
+    def generate_text(self, prompt: str, **kwargs) -> Union[str, dict]:
+        """
+        Generate text using Groq. This is a wrapper around the chat method.
+        """
+        # Create a temporary discussion to leverage the `chat` method's logic
+        temp_discussion = LollmsDiscussion.from_messages([
+            LollmsMessage.new_message(sender_type="user", content=prompt)
+        ])
+        if kwargs.get("system_prompt"):
+            temp_discussion.system_prompt = kwargs.get("system_prompt")
+        
+        return self.chat(temp_discussion, **kwargs)
+
+    def chat(self,
+             discussion: LollmsDiscussion,
+             branch_tip_id: Optional[str] = None,
+             n_predict: Optional[int] = 2048,
+             stream: Optional[bool] = False,
+             temperature: float = 0.7,
+             top_p: float = 0.9,
+             seed: Optional[int] = None,
+             streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
+             **kwargs
+             ) -> Union[str, dict]:
+        """
+        Conduct a chat session with a Groq model.
+        """
+        if not self.client:
+            return {"status": "error", "message": "Groq client not initialized."}
+
+        messages = self._prepare_messages(discussion, branch_tip_id)
+        api_params = self._construct_parameters(temperature, top_p, n_predict, seed)
+        full_response_text = ""
+
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=messages,
+                stream=stream,
+                **api_params
+            )
+
+            if stream:
+                for chunk in response:
+                    delta = chunk.choices[0].delta.content
+                    if delta:
+                        full_response_text += delta
+                        if streaming_callback:
+                            if not streaming_callback(delta, MSG_TYPE.MSG_TYPE_CHUNK):
+                                break
+                return full_response_text
+            else:
+                return response.choices[0].message.content
+
+        except Exception as ex:
+            error_message = f"An unexpected error occurred with Groq API: {str(ex)}"
+            trace_exception(ex)
+            return {"status": "error", "message": error_message}
+
+    def tokenize(self, text: str) -> list:
+        """Tokenize text using tiktoken for a rough estimate."""
+        try:
+            # Most models on Groq (like Llama) use tokenizers that are
+            # reasonably approximated by cl100k_base.
+            encoding = tiktoken.get_encoding("cl100k_base")
+            return encoding.encode(text)
+        except Exception:
+            return list(text.encode('utf-8'))
+
+    def detokenize(self, tokens: list) -> str:
+        """Detokenize tokens using tiktoken."""
+        try:
+            encoding = tiktoken.get_encoding("cl100k_base")
+            return encoding.decode(tokens)
+        except Exception:
+            return bytes(tokens).decode('utf-8', errors='ignore')
+
+    def count_tokens(self, text: str) -> int:
+        """Count tokens in a text using the fallback tokenizer."""
+        return len(self.tokenize(text))
+
+    def embed(self, text: str, **kwargs) -> List[float]:
+        """
+        Groq does not provide an embedding API. This method is not implemented.
+        """
+        ASCIIColors.warning("Groq does not offer a public embedding API. This method is not implemented.")
+        raise NotImplementedError("Groq binding does not support embeddings.")
+
+    def get_model_info(self) -> dict:
+        """Return information about the current Groq setup."""
+        return {
+            "name": self.binding_name,
+            "version": groq.__version__,
+            "host_address": "https://api.groq.com/openai/v1",
+            "model_name": self.model_name,
+            "supports_structured_output": False,
+            "supports_vision": False, # Groq models do not currently support vision
+        }
+
+    def listModels(self) -> List[Dict[str, str]]:
+        """Lists available models from the Groq service."""
+        if not self.client:
+            ASCIIColors.error("Groq client not initialized. Cannot list models.")
+            return []
+        try:
+            ASCIIColors.debug("Listing Groq models...")
+            models = self.client.models.list()
+            model_info_list = []
+            for m in models.data:
+                model_info_list.append({
+                    'model_name': m.id,
+                    'display_name': m.id.replace('-', ' ').title(),
+                    'description': f"Context window: {m.context_window}, Active: {m.active}",
+                    'owned_by': m.owned_by
+                })
+            return model_info_list
+        except Exception as ex:
+            trace_exception(ex)
+            return []
+
+    def load_model(self, model_name: str) -> bool:
+        """Sets the model name for subsequent operations."""
+        self.model_name = model_name
+        ASCIIColors.info(f"Groq model set to: {model_name}. It will be used on the next API call.")
+        return True
+
+if __name__ == '__main__':
+    # Environment variable to set for testing:
+    # GROQ_API_KEY: Your Groq API key
+
+    if "GROQ_API_KEY" not in os.environ:
+        ASCIIColors.red("Error: GROQ_API_KEY environment variable not set.")
+        print("Please get your key from https://console.groq.com/keys and set it.")
+        exit(1)
+
+    ASCIIColors.yellow("--- Testing GroqBinding ---")
+
+    # Use a fast and common model for testing
+    test_model_name = "llama3-8b-8192"
+
+    try:
+        # --- Initialization ---
+        ASCIIColors.cyan("\n--- Initializing Binding ---")
+        binding = GroqBinding(model_name=test_model_name)
+        ASCIIColors.green("Binding initialized successfully.")
+        ASCIIColors.info(f"Using groq library version: {groq.__version__}")
+
+        # --- List Models ---
+        ASCIIColors.cyan("\n--- Listing Models ---")
+        models = binding.listModels()
+        if models:
+            ASCIIColors.green(f"Found {len(models)} models on Groq. Available models:")
+            for m in models:
+                print(f"- {m['model_name']} (owned by {m['owned_by']})")
+        else:
+            ASCIIColors.warning("No models found or failed to list models.")
+        
+        # --- Count Tokens ---
+        ASCIIColors.cyan("\n--- Counting Tokens ---")
+        sample_text = "The quick brown fox jumps over the lazy dog."
+        token_count = binding.count_tokens(sample_text)
+        ASCIIColors.green(f"Token count for '{sample_text}': {token_count}")
+
+        # --- Text Generation (Non-Streaming) ---
+        ASCIIColors.cyan("\n--- Text Generation (Non-Streaming) ---")
+        prompt_text = "What is the capital of France? Be concise."
+        generated_text = binding.generate_text(prompt_text, n_predict=20, stream=False)
+        if isinstance(generated_text, str):
+            ASCIIColors.green(f"Generated text:\n{generated_text}")
+        else:
+            ASCIIColors.error(f"Generation failed: {generated_text}")
+
+        # --- Text Generation (Streaming) ---
+        ASCIIColors.cyan("\n--- Text Generation (Streaming) ---")
+        full_streamed_text = ""
+        def stream_callback(chunk: str, msg_type: int):
+            nonlocal full_streamed_text
+            ASCIIColors.green(chunk, end="", flush=True)
+            full_streamed_text += chunk
+            return True
+        
+        stream_prompt = "Write a very short, 3-line poem about speed."
+        result = binding.generate_text(stream_prompt, n_predict=50, stream=True, streaming_callback=stream_callback)
+        print("\n--- End of Stream ---")
+        ASCIIColors.green(f"Full streamed text (for verification): {result}")
+
+        # --- Embeddings Test ---
+        ASCIIColors.cyan("\n--- Embeddings ---")
+        try:
+            binding.embed("This should fail.")
+        except NotImplementedError as e:
+            ASCIIColors.green(f"Successfully caught expected error for embeddings: {e}")
+        except Exception as e:
+            ASCIIColors.error(f"Caught an unexpected error for embeddings: {e}")
+
+        # --- Vision Test (should be unsupported) ---
+        ASCIIColors.cyan("\n--- Vision Test (Expecting No Support) ---")
+        model_info = binding.get_model_info()
+        if not model_info.get("supports_vision"):
+            ASCIIColors.green("Binding correctly reports no support for vision.")
+        else:
+            ASCIIColors.warning("Binding reports support for vision, which is unexpected for Groq.")
+
+    except Exception as e:
+        ASCIIColors.error(f"An error occurred during testing: {e}")
+        trace_exception(e)
+
+    ASCIIColors.yellow("\nGroqBinding test finished.")

lollms_client/llm_bindings/hugging_face_inference_api/__init__.py

@@ -0,0 +1,307 @@
+import os
+from typing import Optional, Callable, List, Union, Dict
+
+from lollms_client.lollms_discussion import LollmsDiscussion, LollmsMessage
+from lollms_client.lollms_llm_binding import LollmsLLMBinding
+from lollms_client.lollms_types import MSG_TYPE
+from ascii_colors import ASCIIColors, trace_exception
+
+import pipmaster as pm
+
+# Ensure the required packages are installed
+pm.ensure_packages(["huggingface_hub", "tiktoken"])
+
+from huggingface_hub import HfApi, InferenceClient
+import tiktoken
+
+BindingName = "HuggingFaceInferenceAPIBinding"
+
+class HuggingFaceInferenceAPIBinding(LollmsLLMBinding):
+    """
+    Hugging Face Inference API binding implementation.
+
+    This binding communicates with the Hugging Face serverless Inference API,
+    allowing access to thousands of models hosted on the Hub.
+    """
+
+    def __init__(self,
+                 model_name: str = "mistralai/Mistral-7B-Instruct-v0.2",
+                 hf_api_key: str = None,
+                 **kwargs
+                 ):
+        """
+        Initialize the HuggingFaceInferenceAPIBinding.
+
+        Args:
+            model_name (str): The repository ID of the model on the Hugging Face Hub.
+            hf_api_key (str): The Hugging Face API key.
+        """
+        super().__init__(binding_name=BindingName)
+        self.model_name = model_name
+        self.hf_api_key = hf_api_key or os.getenv("HUGGING_FACE_HUB_TOKEN")
+
+        if not self.hf_api_key:
+            raise ValueError("Hugging Face API key is required. Set it via 'hf_api_key' or HUGGING_FACE_HUB_TOKEN env var.")
+
+        try:
+            self.client = InferenceClient(model=self.model_name, token=self.hf_api_key)
+            self.hf_api = HfApi(token=self.hf_api_key)
+        except Exception as e:
+            ASCIIColors.error(f"Failed to configure Hugging Face client: {e}")
+            self.client = None
+            self.hf_api = None
+            raise ConnectionError(f"Could not configure Hugging Face client: {e}") from e
+
+    def _construct_parameters(self,
+                              temperature: float,
+                              top_p: float,
+                              n_predict: int,
+                              repeat_penalty: float,
+                              seed: Optional[int]) -> Dict[str, any]:
+        """Builds a parameters dictionary for the HF Inference API."""
+        params = {"details": False, "do_sample": True}
+        if temperature is not None and temperature > 0:
+            params['temperature'] = float(temperature)
+        else:
+            # A temperature of 0 can cause issues, a small epsilon is better
+            params['temperature'] = 0.001
+            params['do_sample'] = False
+
+
+        if top_p is not None: params['top_p'] = top_p
+        if n_predict is not None: params['max_new_tokens'] = n_predict
+        if repeat_penalty is not None: params['repetition_penalty'] = repeat_penalty
+        if seed is not None: params['seed'] = seed
+        return params
+
+    def _format_chat_prompt(self, discussion: LollmsDiscussion, branch_tip_id: Optional[str] = None) -> str:
+        """
+        Formats a discussion into a single prompt string, attempting to use the model's chat template.
+        """
+        messages = []
+        if discussion.system_prompt:
+             messages.append({"role": "system", "content": discussion.system_prompt})
+
+        for msg in discussion.get_messages(branch_tip_id):
+            role = 'user' if msg.sender_type == "user" else 'assistant'
+            messages.append({"role": role, "content": msg.content})
+
+        try:
+            # This is the preferred way, as it respects the model's specific formatting.
+            return self.client.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        except Exception:
+            # Fallback for models without a chat template or if the client fails to fetch it
+            ASCIIColors.warning("Could not apply chat template. Using generic formatting.")
+            full_prompt = ""
+            if discussion.system_prompt:
+                full_prompt += f"<|system|>\n{discussion.system_prompt}\n"
+            for msg in messages:
+                if msg['role'] == 'user':
+                    full_prompt += f"<|user|>\n{msg['content']}\n"
+                else:
+                    full_prompt += f"<|assistant|>\n{msg['content']}\n"
+            full_prompt += "<|assistant|>\n"
+            return full_prompt
+
+    def generate_text(self,
+                     prompt: str,
+                     n_predict: Optional[int] = 1024,
+                     stream: Optional[bool] = False,
+                     temperature: float = 0.7,
+                     top_p: float = 0.9,
+                     repeat_penalty: float = 1.1,
+                     seed: Optional[int] = None,
+                     streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
+                     **kwargs
+                     ) -> Union[str, dict]:
+        """
+        Generate text using the Hugging Face Inference API.
+        """
+        if not self.client:
+            return {"status": "error", "message": "HF Inference client not initialized."}
+
+        api_params = self._construct_parameters(temperature, top_p, n_predict, repeat_penalty, seed)
+        full_response_text = ""
+
+        try:
+            if stream:
+                for chunk in self.client.text_generation(prompt, stream=True, **api_params):
+                    full_response_text += chunk
+                    if streaming_callback:
+                        if not streaming_callback(chunk, MSG_TYPE.MSG_TYPE_CHUNK):
+                            break
+                return full_response_text
+            else:
+                return self.client.text_generation(prompt, **api_params)
+
+        except Exception as ex:
+            error_message = f"An unexpected error occurred with HF Inference API: {str(ex)}"
+            trace_exception(ex)
+            return {"status": "error", "message": error_message}
+
+    def chat(self, discussion: LollmsDiscussion, **kwargs) -> Union[str, dict]:
+        """
+        Conduct a chat session using the Inference API by formatting the discussion into a single prompt.
+        """
+        prompt = self._format_chat_prompt(discussion)
+        return self.generate_text(prompt, **kwargs)
+
+    def tokenize(self, text: str) -> list:
+        """Tokenize text using tiktoken as a fallback."""
+        try:
+            encoding = tiktoken.get_encoding("cl100k_base")
+            return encoding.encode(text)
+        except Exception:
+            return list(text.encode('utf-8'))
+
+    def detokenize(self, tokens: list) -> str:
+        """Detokenize tokens using tiktoken."""
+        try:
+            encoding = tiktoken.get_encoding("cl100k_base")
+            return encoding.decode(tokens)
+        except Exception:
+            return bytes(tokens).decode('utf-8', errors='ignore')
+
+    def count_tokens(self, text: str) -> int:
+        return len(self.tokenize(text))
+
+    def embed(self, text: str, **kwargs) -> List[float]:
+        """
+        Get embeddings using a dedicated sentence-transformer model from the Inference API.
+        """
+        if not self.client:
+            raise Exception("HF Inference client not initialized.")
+        
+        # User should specify a sentence-transformer model
+        embedding_model = kwargs.get("model")
+        if not embedding_model:
+            raise ValueError("A sentence-transformer model ID must be provided via the 'model' kwarg.")
+
+        try:
+            # This is a different endpoint on the InferenceClient
+            response = self.client.feature_extraction(text, model=embedding_model)
+            # The output for many models is a nested list, we need the first element.
+            if isinstance(response, list) and isinstance(response[0], list):
+                return response[0]
+            return response
+        except Exception as ex:
+            trace_exception(ex)
+            raise Exception(f"HF Inference API embedding failed: {str(ex)}") from ex
+
+    def get_model_info(self) -> dict:
+        return {
+            "name": self.binding_name,
+            "version": "unknown",
+            "host_address": "https://api-inference.huggingface.co",
+            "model_name": self.model_name,
+            "supports_structured_output": False,
+            "supports_vision": False, # Vision models use a different API call
+        }
+
+    def listModels(self) -> List[Dict[str, str]]:
+        """Lists text-generation models from the Hugging Face Hub."""
+        if not self.hf_api:
+            ASCIIColors.error("HF API client not initialized. Cannot list models.")
+            return []
+        try:
+            ASCIIColors.debug("Listing Hugging Face text-generation models...")
+            # We filter for the 'text-generation' pipeline tag
+            models = self.hf_api.list_models(filter="text-generation", sort="downloads", direction=-1, limit=100)
+            model_info_list = []
+            for m in models:
+                model_info_list.append({
+                    'model_name': m.modelId,
+                    'display_name': m.modelId,
+                    'description': f"Downloads: {m.downloads}, Likes: {m.likes}",
+                    'owned_by': m.author or "Hugging Face Community"
+                })
+            return model_info_list
+        except Exception as ex:
+            trace_exception(ex)
+            return []
+
+    def load_model(self, model_name: str) -> bool:
+        """Sets the model for subsequent operations and re-initializes the client."""
+        self.model_name = model_name
+        try:
+            self.client = InferenceClient(model=self.model_name, token=self.hf_api_key)
+            ASCIIColors.info(f"Hugging Face model set to: {model_name}. It will be used on the next API call.")
+            return True
+        except Exception as e:
+            ASCIIColors.error(f"Failed to re-initialize client for model {model_name}: {e}")
+            self.client = None
+            return False
+
+if __name__ == '__main__':
+    # Environment variable to set for testing:
+    # HUGGING_FACE_HUB_TOKEN: Your Hugging Face API key with read access.
+
+    if "HUGGING_FACE_HUB_TOKEN" not in os.environ:
+        ASCIIColors.red("Error: HUGGING_FACE_HUB_TOKEN environment variable not set.")
+        print("Please get your token from https://huggingface.co/settings/tokens and set it.")
+        exit(1)
+
+    ASCIIColors.yellow("--- Testing HuggingFaceInferenceAPIBinding ---")
+
+    test_model_name = "mistralai/Mistral-7B-Instruct-v0.2"
+    test_embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
+
+    try:
+        # --- Initialization ---
+        ASCIIColors.cyan("\n--- Initializing Binding ---")
+        binding = HuggingFaceInferenceAPIBinding(model_name=test_model_name)
+        ASCIIColors.green("Binding initialized successfully.")
+
+        # --- List Models ---
+        ASCIIColors.cyan("\n--- Listing Models ---")
+        models = binding.listModels()
+        if models:
+            ASCIIColors.green(f"Successfully fetched {len(models)} text-generation models.")
+            ASCIIColors.info("Top 5 most downloaded models:")
+            for m in models[:5]:
+                print(f"- {m['model_name']} ({m['description']})")
+        else:
+            ASCIIColors.warning("No models found or failed to list models.")
+        
+        # --- Text Generation (Non-Streaming) ---
+        ASCIIColors.cyan("\n--- Text Generation (Non-Streaming) ---")
+        prompt_text = "In a world where AI companions are common, a detective is assigned a new AI partner. Write the first paragraph of their first meeting."
+        ASCIIColors.info("Waiting for model to load (this might take a moment for cold starts)...")
+        generated_text = binding.generate_text(prompt_text, n_predict=150, stream=False)
+        if isinstance(generated_text, str):
+            ASCIIColors.green(f"Generated text:\n{generated_text}")
+        else:
+            ASCIIColors.error(f"Generation failed: {generated_text}")
+
+        # --- Chat (Streaming) ---
+        ASCIIColors.cyan("\n--- Chat (Streaming) ---")
+        chat_discussion = LollmsDiscussion.from_messages([
+            LollmsMessage.new_message(sender_type="system", content="You are a helpful and pirate-themed assistant named Captain Coder."),
+            LollmsMessage.new_message(sender_type="user", content="Ahoy there! Tell me, what be the best language for a scallywag to learn for data science?"),
+        ])
+        full_streamed_text = ""
+        def stream_callback(chunk: str, msg_type: int):
+            nonlocal full_streamed_text
+            ASCIIColors.green(chunk, end="", flush=True)
+            full_streamed_text += chunk
+            return True
+        
+        result = binding.chat(chat_discussion, n_predict=100, stream=True, streaming_callback=stream_callback)
+        print("\n--- End of Stream ---")
+        ASCIIColors.green(f"Full streamed text (for verification): {result}")
+
+        # --- Embeddings Test ---
+        ASCIIColors.cyan("\n--- Embeddings ---")
+        try:
+            embedding_text = "Hugging Face is the home of open-source AI."
+            embedding_vector = binding.embed(embedding_text, model=test_embedding_model)
+            ASCIIColors.green(f"Embedding for '{embedding_text}' (first 5 dims): {embedding_vector[:5]}...")
+            ASCIIColors.info(f"Embedding vector dimension: {len(embedding_vector)}")
+        except Exception as e:
+            ASCIIColors.error(f"Embedding test failed: {e}")
+
+    except Exception as e:
+        ASCIIColors.error(f"An error occurred during testing: {e}")
+        trace_exception(e)
+
+    ASCIIColors.yellow("\nHuggingFaceInferenceAPIBinding test finished.")

lollms_client/llm_bindings/lollms/__init__.py

@@ -281,6 +281,7 @@ class LollmsLLMBinding(LollmsLLMBinding):
         Returns:
             list: List of tokens.
         """
+        response=None
         try:
             # Prepare the request payload
             payload = {