llmstudio 0.0.1b1__tar.gz

llmstudio-0.0.1b1/PKG-INFO

@@ -0,0 +1,101 @@
+Metadata-Version: 2.1
+Name: llmstudio
+Version: 0.0.1b1
+Summary: Prompt Perfection at Your Fingertips
+Home-page: https://llmstudio.ai/
+License: MIT
+Keywords: ml,ai,llm,llmops,openai,langchain,chatgpt,llmstudio,tensorops
+Author: Cláudio Lemos
+Author-email: claudio@tensorops.ai
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: proxy
+Provides-Extra: tracker
+Requires-Dist: langchain (>=0,<1)
+Requires-Dist: langchain-experimental (>=0,<1)
+Requires-Dist: llmstudio-core (==0.0.1b1)
+Requires-Dist: llmstudio-proxy (==0.0.1b1) ; extra == "proxy"
+Requires-Dist: llmstudio-tracker (==0.0.1b1) ; extra == "tracker"
+Requires-Dist: pydantic (>=2.5,<3.0)
+Requires-Dist: python-dotenv (>=0,<1)
+Project-URL: Bug Tracker, https://github.com/tensoropsai/llmstudio/issues
+Project-URL: Documentation, https://docs.llmstudio.ai
+Project-URL: Repository, https://github.com/tensoropsai/llmstudio
+Description-Content-Type: text/markdown
+
+# LLMstudio by [TensorOps](http://tensorops.ai "TensorOps")
+
+Prompt Engineering at your fingertips
+
+![LLMstudio logo](https://imgur.com/Xqsj6V2.gif)
+
+## 🌟 Features
+
+![LLMstudio UI](https://imgur.com/wrwiIUs.png)
+
+- **LLM Proxy Access**: Seamless access to all the latest LLMs by OpenAI, Anthropic, Google.
+- **Custom and Local LLM Support**: Use custom or local open-source LLMs through Ollama.
+- **Prompt Playground UI**: A user-friendly interface for engineering and fine-tuning your prompts.
+- **Python SDK**: Easily integrate LLMstudio into your existing workflows.
+- **Monitoring and Logging**: Keep track of your usage and performance for all requests.
+- **LangChain Integration**: LLMstudio integrates with your already existing LangChain projects.
+- **Batch Calling**: Send multiple requests at once for improved efficiency.
+- **Smart Routing and Fallback**: Ensure 24/7 availability by routing your requests to trusted LLMs.
+- **Type Casting (soon)**: Convert data types as needed for your specific use case.
+
+## 🚀 Quickstart
+
+Don't forget to check out [https://docs.llmstudio.ai](docs) page.
+
+## Installation
+
+Install the latest version of **LLMstudio** using `pip`. We suggest that you create and activate a new environment using `conda`
+
+```bash
+pip install llmstudio
+```
+
+Install `bun` if you want to use the UI
+
+```bash
+curl -fsSL https://bun.sh/install | bash
+```
+
+Create a `.env` file at the same path you'll run **LLMstudio**
+
+```bash
+OPENAI_API_KEY="sk-api_key"
+ANTHROPIC_API_KEY="sk-api_key"
+```
+
+Now you should be able to run **LLMstudio** using the following command.
+
+```bash
+llmstudio server --ui
+```
+
+When the `--ui` flag is set, you'll be able to access the UI at [http://localhost:3000](http://localhost:3000)
+
+## 📖 Documentation
+
+- [Visit our docs to learn how the SDK works](https://docs.LLMstudio.ai) (coming soon)
+- Checkout our [notebook examples](https://github.com/TensorOpsAI/LLMstudio/tree/main/examples) to follow along with interactive tutorials
+
+## 👨‍💻 Contributing
+
+- Head on to our [Contribution Guide](https://github.com/TensorOpsAI/LLMstudio/tree/main/CONTRIBUTING.md) to see how you can help LLMstudio.
+- Join our [Discord](https://discord.gg/GkAfPZR9wy) to talk with other LLMstudio enthusiasts.
+
+## Training
+
+[![Banner](https://imgur.com/XTRFZ4m.png)](https://www.tensorops.ai/llm-studio-workshop)
+
+---
+
+Thank you for choosing LLMstudio. Your journey to perfecting AI interactions starts here.
+

llmstudio-0.0.1b1/README.md

@@ -0,0 +1,70 @@
+# LLMstudio by [TensorOps](http://tensorops.ai "TensorOps")
+
+Prompt Engineering at your fingertips
+
+![LLMstudio logo](https://imgur.com/Xqsj6V2.gif)
+
+## 🌟 Features
+
+![LLMstudio UI](https://imgur.com/wrwiIUs.png)
+
+- **LLM Proxy Access**: Seamless access to all the latest LLMs by OpenAI, Anthropic, Google.
+- **Custom and Local LLM Support**: Use custom or local open-source LLMs through Ollama.
+- **Prompt Playground UI**: A user-friendly interface for engineering and fine-tuning your prompts.
+- **Python SDK**: Easily integrate LLMstudio into your existing workflows.
+- **Monitoring and Logging**: Keep track of your usage and performance for all requests.
+- **LangChain Integration**: LLMstudio integrates with your already existing LangChain projects.
+- **Batch Calling**: Send multiple requests at once for improved efficiency.
+- **Smart Routing and Fallback**: Ensure 24/7 availability by routing your requests to trusted LLMs.
+- **Type Casting (soon)**: Convert data types as needed for your specific use case.
+
+## 🚀 Quickstart
+
+Don't forget to check out [https://docs.llmstudio.ai](docs) page.
+
+## Installation
+
+Install the latest version of **LLMstudio** using `pip`. We suggest that you create and activate a new environment using `conda`
+
+```bash
+pip install llmstudio
+```
+
+Install `bun` if you want to use the UI
+
+```bash
+curl -fsSL https://bun.sh/install | bash
+```
+
+Create a `.env` file at the same path you'll run **LLMstudio**
+
+```bash
+OPENAI_API_KEY="sk-api_key"
+ANTHROPIC_API_KEY="sk-api_key"
+```
+
+Now you should be able to run **LLMstudio** using the following command.
+
+```bash
+llmstudio server --ui
+```
+
+When the `--ui` flag is set, you'll be able to access the UI at [http://localhost:3000](http://localhost:3000)
+
+## 📖 Documentation
+
+- [Visit our docs to learn how the SDK works](https://docs.LLMstudio.ai) (coming soon)
+- Checkout our [notebook examples](https://github.com/TensorOpsAI/LLMstudio/tree/main/examples) to follow along with interactive tutorials
+
+## 👨‍💻 Contributing
+
+- Head on to our [Contribution Guide](https://github.com/TensorOpsAI/LLMstudio/tree/main/CONTRIBUTING.md) to see how you can help LLMstudio.
+- Join our [Discord](https://discord.gg/GkAfPZR9wy) to talk with other LLMstudio enthusiasts.
+
+## Training
+
+[![Banner](https://imgur.com/XTRFZ4m.png)](https://www.tensorops.ai/llm-studio-workshop)
+
+---
+
+Thank you for choosing LLMstudio. Your journey to perfecting AI interactions starts here.

llmstudio-0.0.1b1/llmstudio/cli.py

@@ -0,0 +1,38 @@
+import os
+import signal
+import threading
+
+import click
+from llmstudio.server import start_servers
+
+
+def handle_shutdown(signum, frame):
+    print("Shutting down gracefully...")
+    os._exit(0)
+
+
+@click.group()
+def main():
+    pass
+
+
+@main.command()
+@click.option("--proxy", is_flag=True, help="Start the Proxy server.")
+@click.option("--tracker", is_flag=True, help="Start the Tracker server.")
+def server(proxy, tracker):
+    signal.signal(signal.SIGINT, handle_shutdown)
+
+    start_servers(proxy=proxy, tracker=tracker)
+
+    print("Servers are running. Press CTRL+C to stop.")
+
+    stop_event = threading.Event()
+    try:
+        stop_event.wait()  # Wait indefinitely until the event is set
+    except KeyboardInterrupt:
+        print("Shutting down servers...")
+
+
+if __name__ == "__main__":
+    # main()
+    server()

llmstudio-0.0.1b1/llmstudio/langchain.py

@@ -0,0 +1,137 @@
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+)
+
+from langchain.schema.messages import BaseMessage
+from langchain.schema.output import ChatGeneration, ChatResult
+from langchain_community.adapters.openai import (
+    convert_dict_to_message,
+    convert_message_to_dict,
+)
+from langchain_core.language_models.base import LanguageModelInput
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.runnables import Runnable
+from langchain_core.tools import BaseTool
+from langchain_core.utils.function_calling import convert_to_openai_tool
+from llmstudio.providers import LLM
+from openai import BaseModel
+
+
+class ChatLLMstudio(BaseChatModel):
+    llm: LLM
+    model: str
+    is_stream: bool = False
+    retries: int = 0
+    parameters: dict = {}
+
+    @property
+    def _llm_type(self):
+        return "LLMstudio"
+
+    def _create_message_dicts(
+        self, messages: List[BaseMessage], stop: Optional[List[str]]
+    ) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
+        message_dicts = [convert_message_to_dict(m) for m in messages]
+        return message_dicts
+
+    def _create_chat_result(self, response: Any) -> ChatResult:
+        generations = []
+        if not isinstance(response, dict):
+            response = response.dict()
+        for res in response["choices"]:
+            message = convert_dict_to_message(res["message"])
+            generation_info = dict(finish_reason=res.get("finish_reason"))
+            if "logprobs" in res:
+                generation_info["logprobs"] = res["logprobs"]
+            gen = ChatGeneration(
+                message=message,
+                generation_info=generation_info,
+            )
+            generations.append(gen)
+        token_usage = response.get("usage", {})
+        llm_output = {
+            "token_usage": token_usage,
+            "model_name": response["model"],
+            "system_fingerprint": response.get("system_fingerprint", ""),
+        }
+        return ChatResult(generations=generations, llm_output=llm_output)
+
+    def bind_tools(
+        self,
+        tools: Sequence[Union[Dict[str, Any], Type[BaseModel], Callable, BaseTool]],
+        *,
+        tool_choice: Optional[
+            Union[dict, str, Literal["auto", "any", "none"], bool]
+        ] = None,
+        **kwargs: Any,
+    ) -> Runnable[LanguageModelInput, BaseMessage]:
+        """Bind tool-like objects to this chat model.
+
+        Args:
+            tools: A list of tool definitions to bind to this chat model.
+                Can be a dictionary, pydantic model, callable, or BaseTool. Pydantic
+                models, callables, and BaseTools will be automatically converted to
+                their schema dictionary representation.
+            tool_choice: Which tool to require the model to call.
+                Must be the name of the single provided function,
+                "auto" to automatically determine which function to call
+                with the option to not call any function, "any" to enforce that some
+                function is called, or a dict of the form:
+                {"type": "function", "function": {"name": <<tool_name>>}}.
+            **kwargs: Any additional parameters to pass to the
+                :class:`~langchain.runnable.Runnable` constructor.
+        """
+        formatted_tools = [convert_to_openai_tool(tool) for tool in tools]
+        if tool_choice is not None and tool_choice:
+            if isinstance(tool_choice, str) and (
+                tool_choice not in ("auto", "any", "none")
+            ):
+                tool_choice = {"type": "function", "function": {"name": tool_choice}}
+            if isinstance(tool_choice, dict) and (len(formatted_tools) != 1):
+                raise ValueError(
+                    "When specifying `tool_choice`, you must provide exactly one "
+                    f"tool. Received {len(formatted_tools)} tools."
+                )
+            if isinstance(tool_choice, dict) and (
+                formatted_tools[0]["function"]["name"]
+                != tool_choice["function"]["name"]
+            ):
+                raise ValueError(
+                    f"Tool choice {tool_choice} was specified, but the only "
+                    f"provided tool was {formatted_tools[0]['function']['name']}."
+                )
+            if isinstance(tool_choice, bool):
+                if len(tools) > 1:
+                    raise ValueError(
+                        "tool_choice can only be True when there is one tool. Received "
+                        f"{len(tools)} tools."
+                    )
+                tool_name = formatted_tools[0]["function"]["name"]
+                tool_choice = {
+                    "type": "function",
+                    "function": {"name": tool_name},
+                }
+
+            kwargs["tool_choice"] = tool_choice
+        return super().bind(tools=formatted_tools, **kwargs)
+
+    def _generate(self, messages: List[BaseMessage], **kwargs) -> ChatResult:
+        messages_dicts = self._create_message_dicts(messages, [])
+        response = self.llm.chat(
+            messages_dicts,
+            model=kwargs.get("model", self.model),
+            is_stream=kwargs.get("is_stream", self.is_stream),
+            retries=kwargs.get("retries", self.retries),
+            parameters=kwargs.get("parameters", self.parameters),
+            **kwargs,
+        )
+        return self._create_chat_result(response)

llmstudio-0.0.1b1/llmstudio/providers/__init__.py

@@ -0,0 +1,31 @@
+try:
+    from llmstudio_tracker.tracker import Tracker, TrackingConfig
+except ImportError:
+    name = None
+    message = f"{name} requires the 'llmstudio-tracker' package. Add it to the dependencies, or install it using 'pip install llmstudio[tracker]'."
+
+    class Tracker:
+        def __init__(self, *args, **kwargs):
+            raise ImportError(message.format(name="Tracker"))
+
+    class TrackingConfig:
+        def __init__(self, *args, **kwargs):
+            raise ImportError(message.format(name="TrackingConfig"))
+
+
+try:
+    from llmstudio_proxy.provider import LLMProxyProvider, ProxyConfig
+except ImportError:
+    name = None
+    message = f"{name} requires the 'llmstudio-proxy' package. Add it to the dependencies, or install it using 'pip install llmstudio[proxy]'."
+
+    class LLMProxyProvider:
+        def __init__(self, *args, **kwargs):
+            raise ImportError(message.format(name="LLMProxyProvider"))
+
+    class ProxyConfig:
+        def __init__(self, *args, **kwargs):
+            raise ImportError(message.format(name="ProxyConfig"))
+
+
+from llmstudio.providers.provider import LLM

llmstudio-0.0.1b1/llmstudio/providers/provider.py

@@ -0,0 +1,143 @@
+from typing import Any, Coroutine, Dict, Optional
+
+from llmstudio.providers import LLMProxyProvider, ProxyConfig, Tracker, TrackingConfig
+from llmstudio.utils import create_session_id
+from llmstudio_core.providers import LLMCore
+from llmstudio_core.providers.provider import Provider
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
+
+
+class LLM(Provider):
+    def __init__(
+        self,
+        provider: str,
+        proxy_config: Optional[ProxyConfig] = None,
+        tracking_config: Optional[TrackingConfig] = None,
+        session_id: Optional[str] = None,
+        api_key: Optional[str] = None,
+        api_endpoint: Optional[str] = None,
+        api_version: Optional[str] = None,
+        base_url: Optional[str] = None,
+        **kwargs,
+    ):
+        """
+        Initializes an LLM provider instance to route your calls to the configured provider client.
+
+        This constructor sets up the LLM provider with optional proxy and tracking configurations.
+        If a proxy configuration is provided, it initializes an LLMProxyProvider; otherwise, it uses
+        LLMCore. It also sets up a tracker if a tracking configuration is provided. The session_id
+        is used to uniquely identify interactions within a session, and it requires a tracking
+        configuration to be specified.
+
+        Parameters:
+        - provider (str): The name of the LLM provider (e.g., "openai", "vertexai").
+        - proxy_config (Optional[ProxyConfig]): Configuration for proxy settings, applicable if proxy server is running.
+        - tracking_config (Optional[TrackingConfig]): Configuration for tracking interactions with the LLM. Applicable if Tracking server is running.
+        - session_id (Optional[str]): A unique identifier for the session, used for tracking purposes.
+
+        # If running without a Proxy Server:
+        # All providers:
+        - api_key (Optional[str]): The API key for authenticating requests to the LLM provider.
+
+        # Azure
+        - api_endpoint (Optional[str]): The specific API endpoint to use for requests.
+        - api_version (Optional[str]): The version of the API to use.
+        - base_url (Optional[str]): The base URL for the API requests.
+
+        - **kwargs: Additional keyword arguments for provider customization.
+
+        Raises:
+        - ValueError: If a session_id is provided without a tracking_config.
+        """
+        if proxy_config is not None:
+            self._provider = LLMProxyProvider(
+                provider=provider, proxy_config=proxy_config
+            )
+        else:
+            self._provider = LLMCore(
+                provider=provider,
+                api_key=api_key,
+                api_endpoint=api_endpoint,
+                api_version=api_version,
+                base_url=base_url,
+                **kwargs,
+            )
+
+        self._session_id = None
+        self._tracker = None
+        if tracking_config is not None:
+            self._tracker = Tracker(tracking_config=tracking_config)
+            self._session_id = create_session_id() if session_id is None else session_id
+
+        if (session_id is not None) and tracking_config is None:
+            raise ValueError(
+                f"'session_id' requires the 'tracking_config' specified and 'llmstudio[tracker]' installation."
+            )
+
+    def _provider_config_name(self):
+        return self._provider._provider_config_name()
+
+    def chat(
+        self,
+        chat_input: Any,
+        model: str,
+        is_stream: bool | None = False,
+        retries: int | None = 0,
+        parameters: Optional[dict] = {},
+        **kwargs,
+    ) -> ChatCompletionChunk | ChatCompletion:
+        result = self._provider.chat(
+            chat_input, model, is_stream, retries, parameters, **kwargs
+        )
+
+        if isinstance(result, (ChatCompletionChunk, ChatCompletion)):
+            if self._tracker:
+                result_dict = self._add_session_id(result, self._session_id)
+                self._tracker.log(result_dict)
+            return result
+        else:
+
+            def generator_wrapper():
+                for item in result:
+                    yield item
+                    if self._tracker and item.metrics:
+                        result_dict = self._add_session_id(item, self._session_id)
+                        self._tracker.log(result_dict)
+
+            return generator_wrapper()
+
+    async def achat(
+        self,
+        chat_input: Any,
+        model: str,
+        is_stream: bool | None = False,
+        retries: int | None = 0,
+        parameters: Optional[dict] = {},
+        **kwargs,
+    ) -> Coroutine[Any, Any, Coroutine[Any, Any, ChatCompletionChunk | ChatCompletion]]:
+        result = await self._provider.achat(
+            chat_input, model, is_stream, retries, parameters, **kwargs
+        )
+        if isinstance(result, (ChatCompletionChunk, ChatCompletion)):
+            if self._tracker:
+                result_dict = self._add_session_id(result, self._session_id)
+                self._tracker.log(result_dict)
+            return result
+        else:
+
+            async def async_generator_wrapper():
+                async for item in result:
+                    yield item
+                    if self._tracker and item.metrics:
+                        result_dict = self._add_session_id(item, self._session_id)
+                        self._tracker.log(result_dict)
+
+            return async_generator_wrapper()
+
+    @staticmethod
+    def _add_session_id(
+        result: ChatCompletionChunk | ChatCompletion, session_id: str
+    ) -> Dict[str, Any]:
+        result_dict = result.model_dump()
+        result_dict["session_id"] = session_id
+        return result_dict

llmstudio-0.0.1b1/llmstudio/server.py

@@ -0,0 +1,14 @@
+_proxy_server_started = False
+_tracker_server_started = False
+
+
+def start_servers(proxy=True, tracker=True):
+    global _proxy_server_started, _tracker_server_started
+    if proxy and not _proxy_server_started:
+        from llmstudio_proxy.server import setup_engine_server
+
+        setup_engine_server()
+    if tracker and not _tracker_server_started:
+        from llmstudio_tracker.server import setup_tracking_server
+
+        setup_tracking_server()

llmstudio-0.0.1b1/llmstudio/utils.py

@@ -0,0 +1,11 @@
+import datetime
+from uuid import uuid4
+
+
+def create_session_id() -> str:
+    hash = str(uuid4())
+    try:
+        timestamp = datetime.datetime.now(datetime.UTC).strftime("%Y%m%d-%H%M%S")
+    except AttributeError:  # python < 3.12
+        timestamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S")
+    return f"{timestamp}-{hash}"

llmstudio-0.0.1b1/pyproject.toml

@@ -0,0 +1,48 @@
+[tool.poetry]
+name = "llmstudio"
+version = "0.0.1b1"
+description = "Prompt Perfection at Your Fingertips"
+authors = ["Cláudio Lemos <claudio@tensorops.ai>"]
+license = "MIT"
+homepage = "https://llmstudio.ai/"
+repository = "https://github.com/tensoropsai/llmstudio"
+documentation = "https://docs.llmstudio.ai"
+readme = "README.md"
+keywords = ["ml", "ai", "llm", "llmops", "openai", "langchain", "chatgpt", "llmstudio", "tensorops"]
+
+[tool.poetry.urls]
+"Bug Tracker" = "https://github.com/tensoropsai/llmstudio/issues"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+pydantic = "^2.5"
+python-dotenv = "^0"
+langchain = "^0"
+langchain-experimental = "^0"
+llmstudio-core = "0.0.1b1"
+llmstudio-tracker = { version = "0.0.1b1", optional = true }
+llmstudio-proxy = { version = "0.0.1b1", optional = true }
+
+[tool.poetry.extras]
+proxy = ["llmstudio-proxy"]
+tracker = ["llmstudio-tracker"]
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.4"
+pytest-asyncio = "^0"
+mypy = "^1.5"
+ruff = "^0.0.284"
+black = "^23.7"
+pre-commit = "^3.7"
+flake8= "^7"
+ipykernel = "^6"
+llmstudio-core = { path = "../core/", develop = true }
+llmstudio-tracker = { path = "../tracker/", develop = true }
+llmstudio-proxy = { path = "../proxy/", develop = true }
+
+[tool.poetry.scripts]
+llmstudio = "llmstudio.cli:main"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"