polyrouter 1.0.0__py3-none-any.whl

polyrouter/Exceptions.py

@@ -0,0 +1,49 @@
+# this file will be responsible for defination of all the user defined Exceptions
+import logging
+
+logger = logging.getLogger(__name__)
+
+class LLMError(Exception):
+    def __init__(self, message):
+        logger.error(message)
+        super().__init__(message)
+
+
+class AllModelsFailedError(LLMError):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class ModelRateLimit(LLMError):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class AllClientsExhaustedError(LLMError):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class InvalidAPIKey(LLMError):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class InvalidJSONResponseError(LLMError):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class NoAPIKeysError(LLMError):
+    def __init__(self,message):
+        super().__init__(message)
+
+
+class NoModelMentioned(LLMError):
+    def __init__(self,message):
+        super().__init__(message)
+
+
+class UnknownError(LLMError):
+    def __init__(self,message):
+        super().__init__(message)

polyrouter/LLMClients.py

@@ -0,0 +1,565 @@
+# File act as a interface between user defined universal call functions and llm clients specific invoke functions
+
+
+# ALL the clients to which i can make calls
+
+from abc import ABC, abstractmethod
+import os
+import json
+import threading
+import logging
+from groq import Groq
+from google import genai
+from google.genai import types
+from .Exceptions import (
+    NoAPIKeysError, NoModelMentioned, UnknownError
+)
+from cerebras.cloud.sdk import Cerebras
+
+logger = logging.getLogger(__name__)
+
+class LLM(ABC):
+    """
+    Each LLM subclass manages its own keys and models.
+    call() returns None when fully exhausted (all keys x all models tried).
+    InitLLM uses this signal to switch to the next client.
+    """
+    @abstractmethod
+    def call(self, user_input, json_mode=False):
+        pass
+
+
+# If a particulat api key model crashes it should work for and it works for other api key, then dont crash the whole system , just switch api key
+# always take the first model whose TPM is available (otherwise client building fails -> assume wrong client and crash)
+# or hardcode model checking in try block e.g. model=GROQ_MODEL[0] -> model="openai/gpt-oss-120b"
+
+# ------------------------------------------- GROQ ------------------------------------------- 
+
+class GroqLLM(LLM):
+    def __init__(self, GROQ_MODEL, GROQ_KEY, prompt="You are a helpful assistant", temperature=0.5, max_output_tokens=500,DEBUG=0,IN_DEPTH_DEBUG=0, test_mode=0):
+        if len(GROQ_MODEL) == 0:
+            raise NoModelMentioned("GROQ : No models listed in GROQ_MODELS.")
+        if len(GROQ_KEY) == 0:
+            raise NoAPIKeysError("GROQ : No api key listed in GROQ_KEYS.")
+        
+        self.DEBUG = DEBUG
+        self.IN_DEPTH_DEBUG = IN_DEPTH_DEBUG
+        self.prompt = prompt
+        self.temperature = temperature
+        self.max_output_tokens = max_output_tokens
+        self._lock = threading.Lock()
+        self.clients = []
+
+        # storing all client so that we dont get any issue once clients are build
+        for api_key in GROQ_KEY:
+            if api_key and api_key.startswith("gsk"):
+
+                # check if key is really valid or not -> pipeline doenst break inbetween
+                try:
+                    client = Groq(api_key=api_key)
+                    response = client.chat.completions.create(
+                        model=GROQ_MODEL[0],
+                        messages=[
+                            {"role": "user", "content": "Reply: ok"}
+                        ],
+                        max_completion_tokens=100
+                    )
+                    self.clients.append(client)
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("GROQ : Working Key ....%s", api_key[10:])
+                        logger.debug("Output: %s", response.choices[0].message.content)
+
+                    # # What is the current api fails for 1/2 model and works on other -> just give a warning to user
+                    if(test_mode): 
+                        for model in GROQ_MODEL:
+                            try:
+                                response = client.chat.completions.create(
+                                    model=model,
+                                    messages=[
+                                        {"role": "user", "content": "Reply: ok"}
+                                    ],
+                                    max_completion_tokens=100
+                                )
+
+                                if self.IN_DEPTH_DEBUG:
+                                    logger.debug("GROQ : Working Model %s", model)
+                                    logger.debug("Output: %s", response.choices[0].message.content)
+
+
+                            except Exception as e:
+                                logger.error("GROQ : Model usage failed ....%s : %s", api_key[10:], model)
+                                # # Uncomment if you want to stop code if 'model' fails
+                                # raise UnknownError(f"GROQ : Model usage failed -> ....{api_key[10:]} : {model}")
+                        
+                except Exception as e:
+                    logger.error("GROQ : Model usage failed ....%s", api_key[10:])
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("Exception details", exc_info=True)
+                    # # Uncomment if you want to fail if api key fails
+                    # raise UnknownError(f"GROQ : API Error Initialising or 1st Model in GROQ_LLM failed : with API key ....{api_key[10:]}")
+                    # print("===============================================================")
+
+            else:
+                raise NoAPIKeysError(f"GROQ : Key ....{api_key[10:]} is missing or has an invalid format.")
+
+        if len(self.clients) == 0:
+            raise NoAPIKeysError("GROQ : No valid API keys found.")
+
+        self.models = list(GROQ_MODEL)  
+        self.current_model_idx = 0
+        self.current_client_idx = 0
+
+
+    def _get_current(self):
+        with self._lock:
+            if self.DEBUG:
+                logger.info("Using Model: %s", self.models[self.current_model_idx])
+            return (
+                self.clients[self.current_client_idx],
+                self.models[self.current_model_idx]
+            )
+
+
+    def _rotate_client(self):
+        """Move to next key. Returns True if a full cycle of all keys is complete."""
+        with self._lock:
+            self.current_client_idx += 1
+            if self.current_client_idx == len(self.clients):
+                self.current_client_idx = 0
+
+
+    def _rotate_model(self):
+        """Move to next model. Returns True if all models are exhausted."""
+        with self._lock:
+            self.current_model_idx += 1
+            self.current_client_idx = 0
+            if self.current_model_idx == len(self.models):
+                self.current_model_idx = 0
+
+
+    # return number of tokens left with current client 
+    # if number of tokens left < 500, maybe for next response they are not enough, so rotate proactively
+    def _get_remaining_tokens(self, response) -> int:
+        tokens_str = response.headers.get('x-ratelimit-remaining-tokens')
+        if self.IN_DEPTH_DEBUG:
+            logger.debug("GROQ Tokens Remaining: %s", tokens_str)
+        return int(tokens_str) if tokens_str is not None else 9999
+
+
+    def call(self, user_input, json_mode=False):
+        """
+        Rotation strategy:
+          1. Try current key on current model.
+          2. On failure -> rotate key.
+          3. All keys exhausted on current model -> rotate model, reset keys.
+          4. All models exhausted -> return None (signals InitLLM to switch client).
+        """
+        models_tried = 0
+
+        while models_tried < len(self.models):
+            keys_tried = 0
+
+            while keys_tried < len(self.clients):
+                client, model = self._get_current()
+                try:
+                    kwargs = {
+                        "model": model,
+                        "temperature": self.temperature,
+                        "max_tokens": self.max_output_tokens,
+                        "messages": [
+                            {"role": "system", "content": self.prompt},
+                            {"role": "user", "content": user_input}
+                        ]
+                    }
+                    if json_mode:
+                        kwargs["response_format"] = {"type": "json_object"}
+
+                    response = client.chat.completions.with_raw_response.create(**kwargs)
+
+                    if self._get_remaining_tokens(response) < 500:
+                        if self.DEBUG:
+                            logger.info("GROQ Low tokens on key %s, rotating proactively.", self.current_client_idx)
+                        keys_tried += 1
+                        self._rotate_client()
+
+                    parsed = response.parse()
+                    output_text = parsed.choices[0].message.content
+
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("%s : %s", user_input, output_text)
+
+                    if json_mode:
+                        return json.loads(output_text)
+                    return output_text
+
+                except Exception as e:
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("GROQ Unknown error on key %s", self.current_client_idx, exc_info=True)
+                    keys_tried += 1
+                    self._rotate_client()
+
+            if self.IN_DEPTH_DEBUG:
+                logger.debug("GROQ All keys exhausted for model '%s'. Rotating model.", self.models[self.current_model_idx])
+
+            models_tried += 1
+            self._rotate_model()
+
+
+        if self.DEBUG:
+            logger.info("GROQ All keys x all models exhausted.")
+        return None
+
+
+
+# ------------------------------------------- GEMINI ------------------------------------------- 
+# cannot move proactively as gemini doesn't provide the TPM left
+# check if it is giving response, if it stucks then rotate model 
+class GeminiLLM(LLM):
+    def __init__(self,GEMINI_MODEL, GEMINI_KEY, prompt="You are a helpful assistant", temperature=0.5, max_output_tokens=500,DEBUG=0,IN_DEPTH_DEBUG=0,test_mode=0):
+        if len(GEMINI_MODEL) == 0:
+            raise NoModelMentioned("GEMINI : No models listed in GEMINI_MODEL.")
+        if len(GEMINI_KEY) == 0:
+            raise NoAPIKeysError("GEMINI : No api key listed in GEMINI_KEYS.")
+
+        self.DEBUG = DEBUG
+        self.IN_DEPTH_DEBUG = IN_DEPTH_DEBUG
+        self.prompt = prompt
+        self.temperature = temperature
+        self.max_output_tokens = max_output_tokens
+        self._lock = threading.Lock()
+        self.clients = []
+
+        # storing all client so that we dont get any issue once clients are build 
+        for api_key in GEMINI_KEY:
+            if api_key and api_key.startswith("AIz"):
+
+                # check if key is really valid or not -> pipeline doenst break inbetween 
+                try:
+                    client = genai.Client(api_key=api_key)
+                    contents = [
+                        types.Content(
+                            role="user",
+                            parts=[types.Part(text="Reply: ok")]
+                        )
+                    ]
+                    response = client.models.generate_content(
+                        model=GEMINI_MODEL[0],
+                        contents=contents,
+                        config=types.GenerateContentConfig(
+                            temperature=0.1,
+                            max_output_tokens=100,
+                        )
+                    )
+                    self.clients.append(client)
+
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("GEMINI : Working Key ....%s", api_key[10:])
+                        logger.debug("Output: %s", response.text)
+
+                    # # What is the current api fails for 1/2 model and works on other -> just give a warning to user 
+                    if(test_mode):
+                        for model in GEMINI_MODEL:
+                            try:
+                                response = client.models.generate_content(
+                                    model=model,
+                                    contents=contents,
+                                    config=types.GenerateContentConfig(
+                                        temperature=0.1,
+                                        max_output_tokens=100,
+                                    )
+                                )
+
+                                if self.IN_DEPTH_DEBUG:
+                                    logger.debug("GEMINI : Working Model %s", model)
+                                    logger.debug("Output: %s", response.text)
+
+                            
+                            except Exception as e:
+                                logger.error("GEMINI : Model usage failed ....%s : %s", api_key[10:], model)
+                                # # Uncomment if you want to stop in between if model fails
+                                # raise UnknownError(f"GEMINI : Model usage failed ....{api_key[10:]} : {model}")
+                        
+
+                except Exception as e:
+                    logger.error("GEMINI : API usage failed ....%s", api_key[10:])
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("Exception details", exc_info=True)
+                    # # Uncomment if you want to fail if api key fails
+                    # raise UnknownError(f"GEMINI : API Error Initialising or 1st Model in GEMINI_LLM failed : with API key ....{api_key[10:]}")
+                    # print("===============================================================")
+
+            else:
+                raise NoAPIKeysError(f"GEMINI : Key ....{api_key[10:]} is missing or has an invalid format.")
+
+        if len(self.clients) == 0:
+            raise NoAPIKeysError("GEMINI : No valid API keys found.")
+
+        self.models = list(GEMINI_MODEL)  
+        self.current_model_idx = 0
+        self.current_client_idx = 0
+
+
+    def _get_current(self):
+        with self._lock:
+            if self.DEBUG:
+                logger.info("Using Model: %s", self.models[self.current_model_idx])
+            return (
+                self.clients[self.current_client_idx],
+                self.models[self.current_model_idx]
+            )
+
+
+    def _rotate_client(self):
+        """Move to next key. Returns True if a full cycle of all keys is complete."""
+        with self._lock:
+            self.current_client_idx += 1
+            if self.current_client_idx == len(self.clients):
+                self.current_client_idx = 0
+
+
+    def _rotate_model(self):
+        """Move to next model. Returns True if all models are exhausted."""
+        with self._lock:
+            self.current_model_idx += 1
+            self.current_client_idx = 0
+            if self.current_model_idx == len(self.models):
+                self.current_model_idx = 0
+
+
+    def call(self, user_input, json_mode=False):
+        """
+        Rotation strategy:
+          1. Try current key on current model.
+          2. On failure -> rotate key.
+          3. All keys exhausted on current model -> rotate model, reset keys.
+          4. All models exhausted -> return None (signals InitLLM to switch client).
+        """
+        models_tried = 0
+
+        while models_tried < len(self.models):
+            keys_tried = 0
+
+            while keys_tried < len(self.clients):
+                client, model = self._get_current()
+                try:
+
+                    contents = [types.Content(role="user", parts=[types.Part(text=user_input)])]
+                    config = types.GenerateContentConfig(
+                        system_instruction=self.prompt,
+                        temperature=self.temperature,
+                        max_output_tokens=self.max_output_tokens,
+                        response_mime_type="application/json" if json_mode else None
+                    )
+                    response = client.models.generate_content(model=model, contents=contents, config=config)
+
+                    output_text = response.text
+
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("%s : %s", user_input, output_text)
+
+                    if json_mode:
+                        return json.loads(output_text)
+                    return output_text
+
+                except Exception as e:
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("GEMINI Unknown error on key %s", self.current_client_idx, exc_info=True)
+                    keys_tried += 1
+                    self._rotate_client()
+
+            if self.IN_DEPTH_DEBUG:
+                logger.debug("GEMINI All keys exhausted for model '%s'. Rotating model.", self.models[self.current_model_idx])
+
+            models_tried += 1
+            self._rotate_model()
+
+
+        if self.DEBUG:
+            logger.info("GEMINI All keys x all models exhausted.")
+        return None
+    
+
+
+
+
+
+# ------------------------------------------- CEREBRAS ------------------------------------------- 
+# cannot move proactively as gemini doesn't provide the TPM left
+# check if it is giving response, if it stucks then rotate model 
+class CereBrasLLM(LLM):
+    def __init__(self, CEREBRAS_MODEL, CEREBRAS_KEY, prompt="You are a helpful assistant", temperature=0.5, max_output_tokens=500,DEBUG=0,IN_DEPTH_DEBUG=0, test_mode=0):
+        if len(CEREBRAS_MODEL) == 0:
+            raise NoModelMentioned("CEREBRAS : No models listed in CEREBRAS_MODEL.")
+        if len(CEREBRAS_KEY) == 0:
+            raise NoAPIKeysError("CEREBRAS : No api key listed in CEREBRAS_KEYS.")
+
+        self.DEBUG = DEBUG
+        self.IN_DEPTH_DEBUG = IN_DEPTH_DEBUG
+        self.prompt = prompt
+        self.temperature = temperature
+        self.max_output_tokens = max_output_tokens
+        self._lock = threading.Lock()
+        self.clients = []
+
+        # storing all client so that we dont get any issue once clients are build
+        for api_key in CEREBRAS_KEY:
+            if api_key and api_key.startswith("csk"):
+
+                # check if key is really valid or not -> pipeline doenst break inbetween
+                try:
+                    client = Cerebras(api_key=api_key)
+                    response = client.chat.completions.create(
+                        model=CEREBRAS_MODEL[0],
+                        max_completion_tokens=200,
+                        temperature=0.2,
+                        top_p=1,
+                        messages=[
+                            {
+                                "role": "system",
+                                "content": "Output exactly the requested text. No extra words."
+                            },
+                            {
+                                "role": "user",
+                                "content": "Return exactly: ok"
+                            }
+                        ],
+                    )
+                    self.clients.append(client)
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("CEREBRAS : Working Key ....%s", api_key[10:])
+                        logger.debug("Output: %s", response.choices[0].message.content)
+
+                    # # What is the current api fails for 1/2 model and works on other -> just give a warning to user 
+                    if(test_mode):
+                        for model in CEREBRAS_MODEL:
+                            try:
+                                response = client.chat.completions.create(
+                                    model=model,
+                                    max_completion_tokens=200,
+                                    temperature=0.2,
+                                    top_p=1,
+                                    messages=[
+                                        {
+                                            "role": "system",
+                                            "content": "Output exactly the requested text. No extra words."
+                                        },
+                                        {
+                                            "role": "user",
+                                            "content": "Return exactly: ok"
+                                        }
+                                    ],
+                                )
+
+                                if self.IN_DEPTH_DEBUG:
+                                    logger.debug("CEREBRAS : Working Model %s", model)
+                                    logger.debug("Output: %s", response.choices[0].message.content)
+
+
+                            except Exception as e:
+                                logger.error("CEREBRAS : Model usage failed ....%s : %s", api_key[10:], model)
+                                # # Uncomment if you want to stop code if 'model' fails
+                                # raise UnknownError(f"CEREBRAS : Model usage failed ....{api_key[10:]} : {model}")
+                        
+                except Exception as e:
+                    logger.error("CEREBRAS : Model usage failed ....%s", api_key[10:])
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("Exception details", exc_info=True)
+                    # # Uncomment if you want to fail if api key fails
+                    # raise UnknownError(f"CEREBRAS : API Error Initialising or 1st Model in CEREBRAS_LLM failed : with API key ....{api_key[10:]}")
+                    # print("===============================================================")
+
+            else:
+                raise NoAPIKeysError(f"CEREBRAS : Key ....{api_key[10:]} is missing or has an invalid format.")
+
+        if len(self.clients) == 0:
+            raise NoAPIKeysError("CEREBRAS : No valid API keys found.")
+
+        self.models = list(CEREBRAS_MODEL)  
+        self.current_model_idx = 0
+        self.current_client_idx = 0
+
+
+    def _get_current(self):
+        with self._lock:
+            if self.DEBUG:
+                logger.info("Using Model: %s", self.models[self.current_model_idx])
+            return (
+                self.clients[self.current_client_idx],
+                self.models[self.current_model_idx]
+            )
+
+
+    def _rotate_client(self):
+        """Move to next key. Returns True if a full cycle of all keys is complete."""
+        with self._lock:
+            self.current_client_idx += 1
+            if self.current_client_idx == len(self.clients):
+                self.current_client_idx = 0
+
+
+    def _rotate_model(self):
+        """Move to next model. Returns True if all models are exhausted."""
+        with self._lock:
+            self.current_model_idx += 1
+            self.current_client_idx = 0
+            if self.current_model_idx == len(self.models):
+                self.current_model_idx = 0
+
+
+    def call(self, user_input, json_mode=False):
+        """
+        Rotation strategy:
+          1. Try current key on current model.
+          2. On failure -> rotate key.
+          3. All keys exhausted on current model -> rotate model, reset keys.
+          4. All models exhausted -> return None (signals InitLLM to switch client).
+        """
+        models_tried = 0
+
+        while models_tried < len(self.models):
+            keys_tried = 0
+
+            while keys_tried < len(self.clients):
+                client, model = self._get_current()
+                try:
+                    kwargs = {
+                        "model": model,
+                        "temperature": self.temperature,
+                        "max_tokens": self.max_output_tokens,
+                        "messages": [
+                            {"role": "system", "content": self.prompt},
+                            {"role": "user", "content": user_input}
+                        ]
+                    }
+
+                    if json_mode:
+                        kwargs["response_format"] = {"type": "json_object"}
+
+                    response = client.chat.completions.with_raw_response.create(**kwargs)
+
+                    parsed_response = response.parse()
+                    output_text = parsed_response.choices[0].message.content
+
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("%s : %s", user_input, output_text)
+
+                    if json_mode:
+                        return json.loads(output_text)
+                    return output_text
+
+                except Exception as e:
+                    if self.IN_DEPTH_DEBUG:
+                        logger.debug("CEREBRAS Unknown error on key %s", self.current_client_idx, exc_info=True)
+                    keys_tried += 1
+                    self._rotate_client()
+
+            if self.IN_DEPTH_DEBUG:
+                logger.debug("CEREBRAS All keys exhausted for model '%s'. Rotating model.", self.models[self.current_model_idx])
+
+            models_tried += 1
+            self._rotate_model()
+
+
+        if self.DEBUG:
+            logger.info("CEREBRAS All keys x all models exhausted.")
+        return None

polyrouter/LLMOrchestrator.py

@@ -0,0 +1,83 @@
+# File respnsible for handling LLMClients and handle dynamic model and client routing
+
+
+from .LLMClients import LLM, GroqLLM, GeminiLLM, CereBrasLLM
+from .Exceptions import AllClientsExhaustedError, AllModelsFailedError
+import threading
+
+CURR_DIR = "LLMOrchestrator"
+
+class LLMOrchestrator:
+    """
+    - Holds all LLM Client Object.
+    - Stays on current client until it returns None (fully exhausted).
+    - Moves to next client and stays there until it too is exhausted.
+    - Raises AllModelsFailedError only when every client is exhausted.
+    """
+ 
+    def __init__(self, groq=None, gemini=None, cerebras=None, debug = 0, verbose = 0, prompt="You are a helpful assistant", temperature=0.5, max_output_tokens=1000,test_mode=0):
+        self.clients: list[LLM] = []
+        if(groq and groq.get("groq_models") and groq.get("groq_keys")):
+            # if(debug):
+            #     print("GROQ_KEYS: ",groq["groq_keys"])
+            self.clients.append(
+                GroqLLM(groq["groq_models"], groq["groq_keys"], prompt=prompt, temperature=temperature,max_output_tokens=max_output_tokens,DEBUG=debug,IN_DEPTH_DEBUG=verbose,test_mode=test_mode),
+            )
+
+        if(gemini and gemini.get("gemini_models") and gemini.get("gemini_keys")):
+            # if(debug):
+            #     print("GEMINI_KEYS: ",gemini["gemini_keys"])
+            self.clients.append(
+                GeminiLLM(gemini["gemini_models"], gemini["gemini_keys"], prompt=prompt, temperature=temperature,max_output_tokens=max_output_tokens,DEBUG=debug,IN_DEPTH_DEBUG=verbose,test_mode=test_mode),
+            )
+
+        if(cerebras and cerebras.get("cerebras_models") and cerebras.get("cerebras_keys")):
+            # if(debug):
+            #     print("CEREBRAS_KEYS: ",cerebras["cerebras_keys"])
+            self.clients.append(
+                CereBrasLLM(cerebras["cerebras_models"], cerebras["cerebras_keys"], prompt=prompt, temperature=temperature,max_output_tokens=max_output_tokens,DEBUG=debug,IN_DEPTH_DEBUG=verbose,test_mode=test_mode),
+            )
+ 
+        if len(self.clients) == 0:
+            raise AllClientsExhaustedError("InitLLM : No LLM clients configured. Either models or keys are empty")
+ 
+        self.current_idx = 0
+        self.DEBUG = debug
+        self.IN_DEPTH_DEBUG = verbose
+        self._lock = threading.Lock()
+ 
+    def _rotate_client(self) -> bool:
+        """Move to next available client. Returns False if all are exhausted."""
+        with self._lock:
+            self.current_idx += 1
+            return self.current_idx < len(self.clients)
+ 
+    def call(self, user_input, json_mode=False):
+        """
+        Try current client. If it returns None (exhausted),
+        move to next and stay there. Repeat until all exhausted.
+        """
+        while True:
+            with self._lock:
+                current_client = self.clients[self.current_idx]
+ 
+            result = current_client.call(user_input, json_mode=json_mode)
+ 
+            if result is not None:
+                return result
+ 
+            # Current client fully exhausted — move to next
+            if self.DEBUG:
+                print(f"[{CURR_DIR}] Client {self.current_idx} ({type(current_client).__name__}) exhausted. Switching.")
+ 
+            has_next = self._rotate_client()
+ 
+            if not has_next:
+                break
+ 
+        # instead of this we can put a while loop which breaks and raise error after 5 or 10 repetative failures
+        # cause every failure tells a story -> maybe that failure is no more a failure
+        # are bhaiya recover ho gaya hoga so (TPM resets after 1 min in groq sooo) (not in gemini)
+        raise AllModelsFailedError(
+            f"[{CURR_DIR}] All clients and models failed -> {CURR_DIR}"
+        )

polyrouter/__init__.py

@@ -0,0 +1,41 @@
+# goal -> keep it simple and light weight
+# initialise the public scope classes which users can access
+# metadata for the library
+
+from .LLMOrchestrator import LLMOrchestrator
+import logging
+
+# prevent unexpected output if user dont want to see logs
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+from .Exceptions import (
+    AllModelsFailedError,
+    AllClientsExhaustedError,
+    ModelRateLimit,
+    InvalidAPIKey,
+    UnknownError, 
+    NoModelMentioned, 
+    NoAPIKeysError,
+    InvalidJSONResponseError
+)
+
+
+# Package Metadata
+__version__ = "0.1.0"
+__author__ = "Pratham Tomar"
+
+
+
+# if something is not defines in __all__ it means that class is private for the users
+# instead of from polyrouter.LLMOrchestrator import LLMOrchestrator -> polyrouter import LLMOrchestrator
+__all__ = [
+    "LLMOrchestrator",
+    "AllModelsFailedError",
+    "AllClientsExhaustedError",
+    "ModelRateLimit",
+    "InvalidAPIKey",
+    "UnknownError", 
+    "NoModelMentioned", 
+    "NoAPIKeysError",
+    "InvalidJSONResponseError"
+]

polyrouter-1.0.0.dist-info/METADATA

@@ -0,0 +1,304 @@
+Metadata-Version: 2.4
+Name: polyrouter
+Version: 1.0.0
+Summary: A routing/orchestration library (adjust as needed)
+Author-email: Pratham Tomar <prathamtomar1733@gmail.com>
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: cerebras_cloud_sdk>=1.67.0
+Requires-Dist: google-genai>=2.4.0
+Requires-Dist: groq>=1.2.0
+
+# PolyRouter
+
+![Python](https://img.shields.io/badge/Python-3-blue)
+![Architecture](https://img.shields.io/badge/Architecture-Config--Driven%20LLM%20Router-0A84FF)
+![License](https://img.shields.io/badge/License-TBD-lightgrey)
+
+PolyRouter is a lightweight Python library that routes requests across multiple LLM providers. It helps applications achieve deterministic failover by rotating API keys, client providers, and model candidates when requests fail.
+
+## Overview
+
+This repository provides an orchestration layer that can be embedded in your application to manage provider rotation, API-key pools, and model fallbacks. Keys are loaded from the environment (see `.env`), and the orchestrator tries configured provider/model/key combinations until a request succeeds or all combinations are exhausted.
+
+Example provider adapters included in this snapshot:
+
+- Groq
+- Google Gemini
+- Cerebras
+
+## Badges
+
+| Badge                                                                                           | Meaning                                |
+| ----------------------------------------------------------------------------------------------- | -------------------------------------- |
+| ![Python](https://img.shields.io/badge/Python-3-blue)                                           | Python implementation                  |
+| ![Architecture](https://img.shields.io/badge/Architecture-Config--Driven%20LLM%20Router-0A84FF) | Multi-provider failover design         |
+| ![License](https://img.shields.io/badge/License-TBD-lightgrey)                                  | Update once a formal license is chosen |
+
+## Key Features
+
+| Capability                | Technical Detail                                                                                                                 |
+| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------- |
+| Provider rotation         | Requests can move across Groq, Gemini, and Cerebras client pools without changing application code.                              |
+| Key pool management       | Each provider can be backed by multiple API keys, allowing the runtime to continue when a single key expires or is rate-limited. |
+| Model pool fallback       | Ordered model lists in `config.py` act as a preference chain, so the router can try alternate models before surfacing a failure. |
+| Debug visibility          | `DEBUG` and `IN_DEPTH_DEBUG` control log verbosity so you can switch between concise operational logs and deep request tracing.  |
+| Centralized configuration | Provider counts, model lists, and debug mode live in one place instead of being duplicated across call sites.                    |
+| Failure isolation         | Provider-specific errors do not have to terminate the entire workflow if another valid key/model combination is still available. |
+
+## How It Works
+
+```mermaid
+flowchart TD
+	A[Application request] --> B[Load config.py]
+	B --> C[Load .env API keys]
+	C --> D[Try primary provider]
+	D --> E{Request succeeds?}
+	E -->|Yes| F[Return response]
+	E -->|No| G[Rotate key / model / client]
+	G --> H{Any combinations left?}
+	H -->|Yes| D
+	H -->|No| I[Raise exhaustion error]
+```
+
+The intended behavior is simple:
+
+1. Read provider preferences, model lists, and key counts from `config.py`.
+2. Load API credentials from the environment.
+3. Attempt a request with the active provider/model combination.
+4. On provider failure, rotate through the next key or model.
+5. When a provider pool is exhausted, move to the next client family.
+6. Stop only when every configured combination has been tried.
+
+> The repo is built for operational resilience, not for single-provider purity.
+
+## Project Structure
+
+```text
+PolyRouter/
+├── examples/                # Example usage scenarios
+│   └── basic_usage.py
+├── polyrouter/              # Library source
+│   ├── Exceptions.py
+│   ├── LLMClients.py
+│   ├── LLMOrchestrator.py
+│   └── __init__.py
+├── .env                     # (example present in repo snapshot)
+├── requirements-dev.txt     # Development / runtime deps
+└── README.md
+```
+
+## Installation
+
+Local setup (recommended):
+
+```bash
+git clone <repository-url>
+cd PolyRouter
+python3 -m venv .venv
+source .venv/bin/activate
+pip install --upgrade pip
+pip install -r requirements-dev.txt
+```
+
+### Environment variables
+
+This repo includes a `.env` file in the snapshot. In normal usage copy and populate a local `.env` (do not commit secrets):
+
+```bash
+cp .env .env.local
+# edit .env.local and export provider API keys, e.g.:
+# GROQ_API_KEY0=...
+# GEMINI_API_KEY0=...
+```
+
+The examples use environment variables named like `GROQ_API_KEY0`, `GEMINI_API_KEY0`, etc.
+
+### Dependencies
+
+Dependencies used by examples and adapters may include provider SDKs and `python-dotenv`. Install via `requirements-dev.txt`.
+
+<details>
+<summary>Build / verification steps</summary>
+
+This repository is a Python library-style project rather than a packaged service, so the essential validation step is import verification plus a smoke test in your host application.
+
+```bash
+python -m compileall .
+python - <<'PY'
+from config import DEBUG, IN_DEPTH_DEBUG, GROQ_MODEL
+print("config loaded:", DEBUG, IN_DEPTH_DEBUG, GROQ_MODEL)
+PY
+```
+
+</details>
+
+## Configuration
+
+`config.py` is the primary customization point.
+
+| Setting          | Role                                                         |
+| ---------------- | ------------------------------------------------------------ |
+| `DEBUG`          | Enables the main debug statement stream.                     |
+| `IN_DEPTH_DEBUG` | Enables detailed trace output for low-level troubleshooting. |
+| `GROQ_MODEL`     | Ordered Groq model preference list.                          |
+| `GEMINI_MODEL`   | Ordered Gemini model preference list.                        |
+| `CEREBRAS_MODEL` | Ordered Cerebras model preference list.                      |
+| `GROQ_KEY`       | Number of Groq API keys to scan.                             |
+| `GEMINI_KEY`     | Number of Gemini API keys to scan.                           |
+| `CEREBRAS_KEY`   | Number of Cerebras API keys to scan.                         |
+
+Recommended operating model:
+
+```python
+DEBUG = 1
+IN_DEPTH_DEBUG = 0
+
+GROQ_MODEL = ["openai/gpt-oss-120b", "llama-3.3-70b-versatile"]
+GEMINI_MODEL = ["gemini-2.5-flash"]
+CEREBRAS_MODEL = ["gpt-oss-120b"]
+
+GROQ_KEY = 2
+GEMINI_KEY = 1
+CEREBRAS_KEY = 1
+```
+
+## Usage
+
+See `examples/basic_usage.py` for a minimal example. The orchestrator can be constructed directly in your application; you do not need a `config.py` file if you prefer to pass provider settings programmatically.
+
+Minimal usage:
+
+```python
+from dotenv import load_dotenv
+from polyrouter.LLMOrchestrator import LLMOrchestrator
+import os
+
+load_dotenv()
+
+llm = LLMOrchestrator(
+	groq={
+		"groq_models": ["openai/gpt-oss-120b"],
+		"groq_keys": [os.getenv("GROQ_API_KEY0")],
+	},
+	debug=True,
+	verbose=True,
+	prompt="You are a helpful assistant",
+)
+
+response = llm.request()
+print(response)
+```
+
+When `debug`/`verbose` are enabled the orchestrator prints provider, model and key selection and rotation decisions.
+
+## API / CLI
+
+No standalone CLI or HTTP API is exposed in this repository snapshot.
+
+The public surface is intentionally library-oriented:
+
+- `config.py` controls behavior
+- `LLMClients.py` defines the client abstraction
+- `LLMOrchestrator.py` is the orchestration boundary
+
+If you add a CLI later, document it here with exact command syntax and exit codes.
+
+## Deployment
+
+Because this project is a routing library, deployment usually means shipping it as part of a larger Python service or worker.
+
+### Recommended deployment checklist
+
+1. Pin dependencies with `requirements.txt`.
+2. Inject secrets through the runtime environment, not source control.
+3. Set `DEBUG = 0` and `IN_DEPTH_DEBUG = 0` for production unless you are actively diagnosing issues.
+4. Validate all required API keys are present before starting the process.
+5. Run the host service behind your preferred process manager, container runtime, or platform scheduler.
+
+### Containerized deployment
+
+If you package the project into a container, copy only the source files, install requirements, and mount secrets through environment variables or secret storage.
+
+```dockerfile
+FROM python:3.11-slim
+
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+CMD ["python", "-c", "import config; print('LLM-Gateway-Service ready')"]
+```
+
+## Screenshots
+
+> Screenshot placeholder: add an architecture diagram or runtime trace capture here once the project has a visual demo surface.
+
+Suggested assets for a production repository:
+
+- request-routing diagram
+- provider rotation log snippet
+- environment setup screenshot
+
+## Troubleshooting
+
+| Symptom                                | Likely Cause                                                 | Resolution                                                                 |
+| -------------------------------------- | ------------------------------------------------------------ | -------------------------------------------------------------------------- |
+| Import error for a provider SDK        | Dependencies are missing from the active virtual environment | Re-run `pip install -r requirements.txt` inside the activated environment. |
+| Requests stop after one provider fails | No fallback keys or models are configured                    | Add more keys to `.env` and expand the model pool in `config.py`.          |
+| All requests fail immediately          | Environment variables are missing or misnamed                | Verify `.env` matches `.env.template` exactly.                             |
+| Debug logs are too noisy               | Verbosity flags are enabled                                  | Set `DEBUG = 0` and `IN_DEPTH_DEBUG = 0` for normal operation.             |
+| A specific model keeps failing         | The model is unsupported, rate-limited, or exhausted         | Remove it from the preference list or move it later in the rotation order. |
+
+## Contributing
+
+Contributions are welcome if they improve correctness, observability, or provider coverage.
+
+Please keep pull requests focused and include:
+
+- a concise description of the routing behavior being changed
+- reproduction steps for any failure-handling update
+- updates to `.env.template` and `config.py` when configuration contracts change
+- tests or a clear validation checklist when the orchestration flow changes
+
+Guidelines:
+
+1. Do not hard-code secrets.
+2. Preserve backward-compatible configuration names whenever possible.
+3. Keep provider rotation behavior deterministic and well logged.
+4. Prefer small, isolated changes to client adapters and error handling.
+
+## Roadmap
+
+Planned improvements that would strengthen the project further:
+
+- formal public orchestration API with documented inputs and return types
+- structured logging with request IDs and provider attempt history
+- health checks for provider pools and exhausted key detection
+- test coverage for failover, invalid-key handling, and model rotation
+- optional CLI for smoke testing provider credentials
+- metrics hooks for success rate, fallback rate, and exhaustion rate
+
+## Acknowledgements
+
+LLM-Gateway-Service builds on the ecosystem provided by:
+
+- Groq
+- Google Gemini / Google Gen AI SDK
+- Cerebras Cloud SDK
+- python-dotenv
+- tenacity
+
+It also follows a common open-source reliability pattern: fail over without forcing callers to understand vendor-specific error recovery.
+
+## License
+
+License: TBD.
+
+Add the repository's chosen license here once it is finalized, and keep the license file in sync with this section.

polyrouter-1.0.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+polyrouter/Exceptions.py,sha256=4JmiViSyMJWBZ84zfBLeMGFtPTJI66vBBHCiDqO_KIc,1089
+polyrouter/LLMClients.py,sha256=KA9zXQcchds73Sd3LCNEtJEIwDuduDKVau2YorwVPUk,24048
+polyrouter/LLMOrchestrator.py,sha256=_Q_SRd5k0ygTFOUhiSF1GEzYeCRuBr61bRoysIem00U,3750
+polyrouter/__init__.py,sha256=vtX_PhGGQ1DUgwtzCsIG7C9TyH5sz-SZMNDhnma1tRg,1031
+polyrouter-1.0.0.dist-info/METADATA,sha256=7cs2hNvmz5FTS57DdNZXtQJVprktkSdpOxR7579EBpA,12341
+polyrouter-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+polyrouter-1.0.0.dist-info/top_level.txt,sha256=xNqKkGjeByTilrvhWqnm-htkJmity5rtql6EtlznTqM,11
+polyrouter-1.0.0.dist-info/RECORD,,

polyrouter-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

polyrouter-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+polyrouter