yta-fastapi-docker-llamacpp 0.0.1__tar.gz

yta_fastapi_docker_llamacpp-0.0.1/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2018 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

yta_fastapi_docker_llamacpp-0.0.1/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: yta-fastapi-docker-llamacpp
+Version: 0.0.1
+Summary: Youtube Autonomous FastAPI Docker Llama.cpp Module
+License-File: LICENSE
+Author: danialcala94
+Author-email: danielalcalavalera@gmail.com
+Requires-Python: >=3.10,<3.14
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: fastapi (>=0.0.1,<9999.0.0)
+Requires-Dist: uvicorn (>=0.0.1,<9999.0.0)
+Requires-Dist: yta_fastapi_docker_pydantic_models (>=0.0.4,<1.0.0)
+Requires-Dist: yta_httpx (>=0.0.27,<1.0.0)
+Description-Content-Type: text/markdown
+
+# Youtube Autonomous FastAPI Docker Llama.cpp Module
+
+The module that is providing the functionality related to the Llama.cpp models hub (having the models and using them) through a FastAPI that is included and isolated in a Docker container.
+
+This module is meant to be exposed as a container inside the internal network, to be connected with its own FastAPI that is exposing the functionality outside.
+
+### Endpoints
+
+#### GET
+No endpoints by now.
+
+#### POST
+No endpoints by now.
+
+## Instructions
+I've followed these steps to make `llama.cpp` available in my laptop as a container running with cuda, and I've adapted this workflow to this project so its done automatically:
+
+1. Nos aseguramos de tener la imagen de Nvidia en docker:
+`$docker run --rm --gpus all nvidia/cuda:12.9.1-runtime-ubuntu24.04 nvidia-smi`
+
+2. Creamos una carpeta `models` para tener los modelos ahí guardados (en mi caso en un SSD externo para ahorrar espacio) en `D:/llama/models`.
+
+3. Descargamos el modelo GGUF que necesitemos (para ello, ver que tipo y qué características en función de nuestro PC), en cmd desde la carpeta `models` del paso 2:
+`$huggingface-cli download unsloth/gemma-4-E2B-it-GGUF gemma-4-E2B-it-UD-Q4_K_XL.gguf --local-dir ./`
+
+1. Descargamos el contenedor 'llama.cpp' adaptado a CUDA, estando en el cmd de la carpeta `models`:
+`$docker run --rm --gpus all -p 8080:8080 -v "${PWD}:/models" ghcr.io/ggml-org/llama.cpp:server-cuda -m  --host 0.0.0.0 -ngl 999`

yta_fastapi_docker_llamacpp-0.0.1/README.md

@@ -0,0 +1,27 @@
+# Youtube Autonomous FastAPI Docker Llama.cpp Module
+
+The module that is providing the functionality related to the Llama.cpp models hub (having the models and using them) through a FastAPI that is included and isolated in a Docker container.
+
+This module is meant to be exposed as a container inside the internal network, to be connected with its own FastAPI that is exposing the functionality outside.
+
+### Endpoints
+
+#### GET
+No endpoints by now.
+
+#### POST
+No endpoints by now.
+
+## Instructions
+I've followed these steps to make `llama.cpp` available in my laptop as a container running with cuda, and I've adapted this workflow to this project so its done automatically:
+
+1. Nos aseguramos de tener la imagen de Nvidia en docker:
+`$docker run --rm --gpus all nvidia/cuda:12.9.1-runtime-ubuntu24.04 nvidia-smi`
+
+2. Creamos una carpeta `models` para tener los modelos ahí guardados (en mi caso en un SSD externo para ahorrar espacio) en `D:/llama/models`.
+
+3. Descargamos el modelo GGUF que necesitemos (para ello, ver que tipo y qué características en función de nuestro PC), en cmd desde la carpeta `models` del paso 2:
+`$huggingface-cli download unsloth/gemma-4-E2B-it-GGUF gemma-4-E2B-it-UD-Q4_K_XL.gguf --local-dir ./`
+
+1. Descargamos el contenedor 'llama.cpp' adaptado a CUDA, estando en el cmd de la carpeta `models`:
+`$docker run --rm --gpus all -p 8080:8080 -v "${PWD}:/models" ghcr.io/ggml-org/llama.cpp:server-cuda -m  --host 0.0.0.0 -ngl 999`

yta_fastapi_docker_llamacpp-0.0.1/pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "yta-fastapi-docker-llamacpp"
+version = "0.0.1"
+description = "Youtube Autonomous FastAPI Docker Llama.cpp Module"
+authors = [
+    {name = "danialcala94", email = "danielalcalavalera@gmail.com"}
+]
+readme = "README.md"
+# 3.12.11 is working
+requires-python = ">=3.10,<3.14"
+
+[tool.poetry.dependencies]
+# Mandatory
+fastapi = { version = ">=0.0.1,<9999.0.0", optional = false }
+uvicorn = { version = ">=0.0.1,<9999.0.0", optional = false }
+yta_httpx = { version = ">=0.0.27,<1.0.0", optional = false }
+yta_fastapi_docker_pydantic_models = { version = ">=0.0.4,<1.0.0", optional = false }
+# Optional
+
+[tool.poetry]
+packages = [{include = "yta_fastapi_docker_llamacpp", from = "src"}]
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.5"
+pytest-asyncio = ">=1.3.0"
+httpx = ">=0.0.1"
+yta_testing = ">=0.0.1"
+
+[tool.pytest.ini_options]
+markers = [
+    "mandatory: mandatory tests for release",
+    "additional: exhaustive and demanding tests"
+]
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

yta_fastapi_docker_llamacpp-0.0.1/src/yta_fastapi_docker_llamacpp/app/llamacpp/gemma4.py

@@ -0,0 +1,66 @@
+"""
+This module is using the Gemma4 model to handle
+the requests:
+- `gemma-4-E2B-it-UD-Q4_K_XL.gguf`
+"""
+from yta_httpx.client import HttpClient
+from typing import Union
+
+
+LLAMACPP_DOCKER_API_ENDPOINT = 'http://localhost:8080/'
+
+class Gemma4Llamacpp:
+    """
+    Class to wrap the functionality of Gemma4 model.
+    """
+
+    async def ask(
+        self,
+        prompt: str,
+        schema: Union[dict, None] = None
+    ) -> str:
+        """
+        Create an ephemeral chat to ask the `prompt` given
+        and waits for a response (that will match the
+        `schema`, if given), that will be returned as a 
+        string.
+
+        If you are providing a `schema` that expects a json
+        response, parse the `str` this method is returning.
+        """
+        payload = {
+            "temperature": 0,
+            "top_p": 1,
+            # TODO: This is rejecting some answers
+            # "max_tokens": 256,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ],
+        }
+
+        if schema is not None:
+            payload['response_format'] = {
+                "type": "json_schema",
+                "json_schema": {
+                    # TODO: What about this schema name (?)
+                    "name": "scene_analysis",
+                    "schema": schema
+                }
+            }
+
+        # url = 'http://localhost:8080/v1/chat/completions'
+        url = f'{LLAMACPP_DOCKER_API_ENDPOINT}v1/chat/completions'
+
+        async with HttpClient() as http_client:
+            response = await http_client.post.complete(
+                url = url,
+                json = payload,
+                timeout = 300
+            )
+            result = response.json()
+            message_content = result['choices'][0]['message']['content']
+
+        return message_content

yta_fastapi_docker_llamacpp-0.0.1/src/yta_fastapi_docker_llamacpp/app/main.py

@@ -0,0 +1,10 @@
+from yta_fastapi_docker_llamacpp.app.routers import router as general_router
+from yta_fastapi_docker_llamacpp.app.routers.llamacpp import router as llamacpp_router
+from fastapi import FastAPI
+
+
+app = FastAPI()
+
+# Include all the routers we have
+app.include_router(general_router)
+app.include_router(llamacpp_router)

yta_fastapi_docker_llamacpp-0.0.1/src/yta_fastapi_docker_llamacpp/app/routers/__init__.py

@@ -0,0 +1,24 @@
+"""
+Check this project to get inspiration about FastAPI:
+- https://github.com/Implosiv3/render-fastapi/blob/master/render-fastapi/routers/download/__init__.py
+"""
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+
+
+PREFIX = f''
+
+router = APIRouter(
+    prefix = PREFIX
+)
+
+@router.get('/check-status')
+def route_check_status(
+) -> JSONResponse:
+    return JSONResponse(
+        {
+            'error': False,
+            'message': 'Hello World!'
+        },
+        status_code = 200
+    )

yta_fastapi_docker_llamacpp-0.0.1/src/yta_fastapi_docker_llamacpp/app/routers/llamacpp.py

@@ -0,0 +1,36 @@
+from yta_fastapi_docker_llamacpp.app.llamacpp.gemma4 import Gemma4Llamacpp
+from yta_fastapi_docker_pydantic_models.llamacpp import GemmaRequest
+from fastapi.responses import JSONResponse
+from fastapi import APIRouter
+
+
+PREFIX = f'/llamacpp'
+
+router = APIRouter(
+    prefix = PREFIX
+)
+
+# TODO: Maybe this should be in another router
+@router.post('/gemma4')
+def post_gemma4(
+    request: GemmaRequest
+) -> JSONResponse:
+    """
+    The request must include a `prompt`, that is
+    mandatory, and could include a `schema` that
+    is optional.
+    """
+    gemma4_agent = Gemma4Llamacpp()
+
+    response = gemma4_agent.ask(
+        prompt = request.prompt,
+        schema = request.schema
+    )
+
+    return JSONResponse(
+        {
+            'error': False,
+            'data': response
+        },
+        status_code = 200
+    )