nexaai 0.0.8.3__tar.gz → 0.0.8.4__tar.gz

{nexaai-0.0.8.3 → nexaai-0.0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nexaai
-Version: 0.0.8.3
+Version: 0.0.8.4
 Summary: Nexa AI SDK
 Author-Email: Nexa AI <octopus@nexa4ai.com>
 License: MIT

nexaai-0.0.8.4/nexa/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.8.4"

{nexaai-0.0.8.3 → nexaai-0.0.8.4}/nexa/cli/entry.py

@@ -165,6 +165,7 @@ def main():
     server_parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to bind the server to")
     server_parser.add_argument("--port", type=int, default=8000, help="Port to bind the server to")
     server_parser.add_argument("--reload", action="store_true", help="Enable automatic reloading on code changes")
+    server_parser.add_argument("--nctx", type=int, default=2048, help="Length of context window")
 
     # Other commands
     subparsers.add_parser("pull", help="Pull a model from official or hub.").add_argument("model_path", type=str, help="Path or identifier for the model in Nexa Model Hub")

{nexaai-0.0.8.3 → nexaai-0.0.8.4}/nexa/constants.py

@@ -46,10 +46,12 @@ NEXA_RUN_MODEL_MAP_TEXT = {
     "gemma2": "gemma-2-2b-instruct:q4_0",
     "qwen1.5": "Qwen1.5-7B-Instruct:q4_0",
     "qwen2": "Qwen2-1.5B-Instruct:q4_0",
+    "qwen2.5": "Qwen2.5-1.5B-Instruct:q4_0",
     "mistral": "Mistral-7B-Instruct-v0.3:q4_0",
     "codegemma": "codegemma-2b:q4_0",
     "codellama": "CodeLlama-7b-Instruct:q4_0",
-    "codeqwen": "CodeQwen1.5-7B-Instruct:q4_0",
+    "codeqwen": "Qwen2.5-Coder-1.5B-Instruct:q4_0",
+    "mathqwen": "Qwen2.5-Math-1.5B-Instruct:q4_0",
     "deepseek-coder": "deepseek-coder-1.3b-instruct:q4_0",
     "dolphin-mistral": "dolphin-2.8-mistral-7b:q4_0",
     "phi2": "Phi-2:q4_0",
@@ -202,6 +204,7 @@ NEXA_RUN_CHAT_TEMPLATE_MAP = {
     "qwen1.5-7b-instruct": "qwen",
     "codeqwen1.5-7b-instruct": "qwen",
     "qwen2": "qwen",
+    "qwen2.5": "qwen",
     "qwen2-0.5b-instruct": "qwen",
     "qwen2-1.5b-instruct": "qwen",
     "qwen2-7b-instruct": "qwen",
@@ -301,6 +304,13 @@ NEXA_OFFICIAL_MODELS_TYPE = {
     "gemma-7b-instruct": ModelType.NLP,
     "gemma-7b": ModelType.NLP,
     "Qwen2-1.5B": ModelType.NLP,
+    "Qwen2.5-0.5B-Instruct": ModelType.NLP,
+    "Qwen2.5-1.5B-Instruct": ModelType.NLP,
+    "Qwen2.5-3B-Instruct": ModelType.NLP,
+    "Qwen2.5-Coder-1.5B-Instruct": ModelType.NLP,
+    "Qwen2.5-Coder-7B-Instruct": ModelType.NLP,
+    "Qwen2.5-Math-1.5B-Instruct": ModelType.NLP,
+    "Qwen2.5-Math-7B-Instruct": ModelType.NLP,
     "codegemma-7b": ModelType.NLP,
     "TinyLlama-1.1B-Chat-v1.0": ModelType.NLP,
     "CodeLlama-7b-Instruct": ModelType.NLP,

{nexaai-0.0.8.3 → nexaai-0.0.8.4}/nexa/gguf/server/nexa_service.py

@@ -18,11 +18,8 @@ from pydantic import BaseModel
 from nexa.constants import (
     NEXA_RUN_CHAT_TEMPLATE_MAP,
     NEXA_RUN_COMPLETION_TEMPLATE_MAP,
-    NEXA_RUN_MODEL_MAP_TEXT,
-    NEXA_RUN_MODEL_MAP_IMAGE,
     NEXA_RUN_MODEL_PRECISION_MAP,
     NEXA_RUN_MODEL_MAP_FUNCTION_CALLING,
-    NEXA_RUN_MODEL_MAP_VOICE,
 )
 from nexa.gguf.lib_utils import is_gpu_available
 from nexa.gguf.llama._utils_transformers import suppress_stdout_stderr
@@ -44,28 +41,91 @@ app.add_middleware(
 )
 
 model = None
-is_chat_mode = False
 chat_format = None
 completion_template = None
 hostname = socket.gethostname()
-conversation_history = [{"role": "system", "content": "You are a helpful assistant"}]
+chat_completion_system_prompt = [{"role": "system", "content": "You are a helpful assistant"}]
 function_call_system_prompt = [{"role": "system", "content": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. The assistant calls functions with appropriate input when necessary"}]
 model_path = None
+n_ctx = None
 
+# Request Classes
 class GenerationRequest(BaseModel):
     prompt: str = "Tell me a story"
     temperature: float = 1.0
     max_new_tokens: int = 128
     top_k: int = 50
     top_p: float = 1.0
-    stop_words: Optional[List[str]] = None
+    stop_words: Optional[List[str]] = []
+    logprobs: Optional[bool] = False
+    top_logprobs: Optional[int] = 4
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+class ImageResponse(BaseModel):
+    base64: str
+    url: str
+
+class ChatCompletionRequest(BaseModel):
+    messages: List[Message] = [
+        {"role": "user", "content": "Tell me a story"}]
+    max_tokens: Optional[int] = 128
+    temperature: Optional[float] = 0.1
+    stream: Optional[bool] = False
+    stop_words: Optional[List[str]] = []
     logprobs: Optional[bool] = False
     top_logprobs: Optional[int] = 4
 
+class FunctionDefinitionRequestClass(BaseModel):
+    type: str = "function"
+    function: Dict[str, Any]
+
+    class Config:
+        extra = "allow"
+
+class FunctionCallRequest(BaseModel):
+    messages: List[Message] = [
+        Message(role="user", content="Extract Jason is 25 years old")]
+    tools: List[FunctionDefinitionRequestClass] = [
+        FunctionDefinitionRequestClass(
+            type="function",
+            function={
+                "name": "UserDetail",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "name": {
+                            "type": "string",
+                            "description": "The user's name"
+                        },
+                        "age": {
+                            "type": "integer",
+                            "description": "The user's age"
+                        }
+                    },
+                    "required": ["name", "age"]
+                }
+            }
+        )
+    ]
+    tool_choice: Optional[str] = "auto"
+
+class ImageGenerationRequest(BaseModel):
+    prompt: str = "A girl, standing in a field of flowers, vivid"
+    image_path: Optional[str] = ""
+    cfg_scale: float = 7.0
+    width: int = 256
+    height: int = 256
+    sample_steps: int = 20
+    seed: int = 0
+    negative_prompt: Optional[str] = ""
+
+# helper functions
 async def load_model():
-    global model, chat_format, completion_template, model_path
+    global model, chat_format, completion_template, model_path, n_ctx
     downloaded_path, run_type = pull_model(model_path)
-
     if run_type == "NLP":
         if model_path in NEXA_RUN_MODEL_MAP_FUNCTION_CALLING:
             chat_format = "chatml-function-calling"
@@ -76,7 +136,8 @@ async def load_model():
                         verbose=False,
                         chat_format=chat_format,
                         n_gpu_layers=-1 if is_gpu_available() else 0,
-                        logits_all=True
+                        logits_all=True,
+                        n_ctx=n_ctx
                     )
                 except Exception as e:
                     logging.error(
@@ -87,13 +148,15 @@ async def load_model():
                         verbose=False,
                         chat_format=chat_format,
                         n_gpu_layers=0,  # hardcode to use CPU,
-                        logits_all=True
+                        logits_all=True,
+                        n_ctx=n_ctx
                     )
 
                 logging.info(f"model loaded as {model}")
         else:
-            chat_format = NEXA_RUN_CHAT_TEMPLATE_MAP.get(model_path, None)
-            completion_template = NEXA_RUN_COMPLETION_TEMPLATE_MAP.get(model_path, None)
+            model_name = model_path.split(":")[0].lower()
+            chat_format = NEXA_RUN_CHAT_TEMPLATE_MAP.get(model_name, None)
+            completion_template = NEXA_RUN_COMPLETION_TEMPLATE_MAP.get(model_name, None)
             with suppress_stdout_stderr():
                 try:
                     model = Llama(
@@ -101,7 +164,8 @@ async def load_model():
                         verbose=False,
                         chat_format=chat_format,
                         n_gpu_layers=-1 if is_gpu_available() else 0,
-                        logits_all=True  # switch on
+                        logits_all=True,
+                        n_ctx=n_ctx
                     )
                 except Exception as e:
                     logging.error(
@@ -112,9 +176,11 @@ async def load_model():
                         verbose=False,
                         chat_format=chat_format,
                         n_gpu_layers=0,  # hardcode to use CPU
-                        logits_all=True
+                        logits_all=True,
+                        n_ctx=n_ctx
                     )
                 logging.info(f"model loaded as {model}")
+                chat_format = model.metadata.get("tokenizer.chat_template", None)               
     elif run_type == "Computer Vision":
         with suppress_stdout_stderr():
             model = StableDiffusion(
@@ -136,18 +202,10 @@ async def load_model():
     else:
         raise ValueError(f"Model {model_path} not found in Model Hub")
 
-
-@app.on_event("startup")
-async def startup_event():
-    global model_path
-    model_path = os.getenv("MODEL_PATH", "gemma")
-    logging.info(f"Model Path: {model_path}")
-    await load_model()
-
 async def nexa_run_text_generation(
     prompt, temperature, stop_words, max_new_tokens, top_k, top_p, logprobs=None, top_logprobs=None
 ) -> Dict[str, Any]:
-    global model, chat_format, completion_template, conversation_history
+    global model, chat_format, completion_template
     if model is None:
         raise ValueError("Model is not loaded. Please check the model path and try again.")
 
@@ -155,10 +213,10 @@ async def nexa_run_text_generation(
     logprobs_or_none = None  # init to store the logprobs if requested
 
     if chat_format:
-        conversation_history.append({"role": "user", "content": prompt})
+        messages = chat_completion_system_prompt + [{"role": "user", "content": prompt}]
 
         params = {
-            'messages': conversation_history,
+            'messages': messages,
             'temperature': temperature,
             'max_tokens': max_new_tokens,
             'top_k': top_k,
@@ -182,12 +240,15 @@ async def nexa_run_text_generation(
                 else:
                     for key in logprobs_or_none:  # tokens, token_logprobs, top_logprobs, text_offset
                         if key in chunk["choices"][0]["logprobs"]:
-                            logprobs_or_none[key].extend(chunk["choices"][0]["logprobs"][key])  # accumulate data from each chunk
-
+                            logprobs_or_none[key].extend(chunk["choices"][0]["logprobs"][key])  # accumulate data from each chunk                            
     else:
-        prompt = completion_template.format(prompt) if completion_template else prompt
+        if completion_template:
+            formatted_prompt = completion_template.format(input=prompt)
+        else:
+            formatted_prompt = prompt
+
         streamer = model.create_completion(
-            prompt=prompt,
+            prompt=formatted_prompt,
             temperature=temperature,
             max_tokens=max_new_tokens,
             top_k=top_k,
@@ -210,117 +271,12 @@ async def nexa_run_text_generation(
                         if key in chunk["choices"][0]["logprobs"]:
                             logprobs_or_none[key].extend(chunk["choices"][0]["logprobs"][key])  # accumulate data from each chunk
 
-    if is_chat_mode:
-        conversation_history.append({"role": "assistant", "content": generated_text})
-
     result = {
         "result": generated_text,
         "logprobs": logprobs_or_none
     }
     return result
 
-
-@app.get("/", response_class=HTMLResponse)
-async def read_root(request: Request):
-    return HTMLResponse(
-        content=f"<h1>Welcome to Nexa AI</h1><p>Hostname: {hostname}</p>"
-    )
-
-
-@app.post("/v1/completions")
-async def generate_text(request: GenerationRequest):
-    logging.info(f"[/v1/completions] Request logprobs: {request.logprobs}, top_logprobs: {request.top_logprobs}")
-    try:
-        result = await nexa_run_text_generation(**request.dict())
-
-        return JSONResponse(content={
-            "choices": [{
-                "text": result["result"],
-                "logprobs": result["logprobs"]
-            }]
-        })
-    except Exception as e:
-        logging.error(f"Error in text generation: {e}")
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-class Message(BaseModel):
-    role: str
-    content: str
-
-class ImageResponse(BaseModel):
-    base64: str
-    url: str
-
-class ChatCompletionRequest(BaseModel):
-    messages: List[Message] = [
-        {"role": "user", "content": "Tell me a story"}]
-    max_tokens: Optional[int] = 128
-    temperature: Optional[float] = 0.1
-    stream: Optional[bool] = False
-    stop_words: Optional[List[str]] = []
-    logprobs: Optional[bool] = False
-    top_logprobs: Optional[int] = 4
-
-class FunctionDefinitionRequestClass(BaseModel):
-    type: str = "function"
-    function: Dict[str, Any]
-
-    class Config:
-        extra = "allow"
-
-class FunctionCallRequest(BaseModel):
-    messages: List[Message] = [
-        Message(role="user", content="Extract Jason is 25 years old")]
-    tools: List[FunctionDefinitionRequestClass] = [
-        FunctionDefinitionRequestClass(
-            type="function",
-            function={
-                "name": "UserDetail",
-                "parameters": {
-                    "type": "object",
-                    "properties": {
-                        "name": {
-                            "type": "string",
-                            "description": "The user's name"
-                        },
-                        "age": {
-                            "type": "integer",
-                            "description": "The user's age"
-                        }
-                    },
-                    "required": ["name", "age"]
-                }
-            }
-        )
-    ]
-    tool_choice: Optional[str] = "auto"
-
-def _resp_async_generator(streamer):
-    _id = str(uuid.uuid4())
-
-    for token in streamer:
-        chunk = {
-            "id": _id,
-            "object": "chat.completion.chunk",
-            "created": time.time(),
-            "choices": [{"delta": {"content": token}}],
-        }
-        yield f"data: {json.dumps(chunk)}\n\n"
-
-    yield "data: [DONE]\n\n"
-
-
-class ImageGenerationRequest(BaseModel):
-    prompt: str = "A girl, standing in a field of flowers, vivid"
-    image_path: Optional[str] = ""
-    cfg_scale: float = 7.0
-    width: int = 256
-    height: int = 256
-    sample_steps: int = 20
-    seed: int = 0
-    negative_prompt: Optional[str] = ""
-
 async def nexa_run_image_generation(
     prompt,
     image_path,
@@ -367,58 +323,66 @@ def base64_encode_image(image_path):
     with open(image_path, "rb") as image_file:
         return base64.b64encode(image_file.read()).decode("utf-8")
 
-@app.post("/v1/txt2img")
-async def txt2img(request: ImageGenerationRequest):
-    try:
-        generation_kwargs = request.dict()
 
-        generated_images = await nexa_run_image_generation(**generation_kwargs)
+def run_nexa_ai_service(model_path_arg, **kwargs):
+    global model_path, n_ctx
+    model_path = model_path_arg or "gemma"
+    os.environ["MODEL_PATH"] = model_path
+    n_ctx = kwargs.get("nctx", 2048)
+    host = kwargs.get("host", "0.0.0.0")
+    port = kwargs.get("port", 8000)
+    reload = kwargs.get("reload", False)
+    uvicorn.run(app, host=host, port=port, reload=reload)
 
-        resp = {"created": time.time(), "data": []}
+# Endpoints
+@app.on_event("startup")
+async def startup_event():
+    global model_path
+    model_path = os.getenv("MODEL_PATH", "gemma")
+    logging.info(f"Model Path: {model_path}")
+    await load_model()
 
-        for image in generated_images:
-            id = int(time.time())
-            if not os.path.exists("nexa_server_output"):
-                os.makedirs("nexa_server_output")
-            image_path = os.path.join("nexa_server_output", f"txt2img_{id}.png")
-            image.save(image_path)
-            img = ImageResponse(base64=base64_encode_image(image_path), url=os.path.abspath(image_path))
-            resp["data"].append(img)
 
-        return resp
+@app.get("/", response_class=HTMLResponse, tags=["Root"])
+async def read_root(request: Request):
+    return HTMLResponse(
+        content=f"<h1>Welcome to Nexa AI</h1><p>Hostname: {hostname}</p>"
+    )
 
-    except Exception as e:
-        logging.error(f"Error in txt2img generation: {e}")
-        raise HTTPException(status_code=500, detail=str(e))
 
-@app.post("/v1/img2img")
-async def img2img(request: ImageGenerationRequest):
-    try:
-        generation_kwargs = request.dict()
+def _resp_async_generator(streamer):
+    _id = str(uuid.uuid4())
 
-        generated_images = await nexa_run_image_generation(**generation_kwargs)
-        resp = {"created": time.time(), "data": []}
+    for token in streamer:
+        chunk = {
+            "id": _id,
+            "object": "chat.completion.chunk",
+            "created": time.time(),
+            "choices": [{"delta": {"content": token}}],
+        }
+        yield f"data: {json.dumps(chunk)}\n\n"
 
-        for image in generated_images:
-            id = int(time.time())
-            if not os.path.exists("nexa_server_output"):
-                os.makedirs("nexa_server_output")
-            image_path = os.path.join("nexa_server_output", f"img2img_{id}.png")
-            image.save(image_path)
-            img = ImageResponse(base64=base64_encode_image(image_path), url=os.path.abspath(image_path))
-            resp["data"].append(img)
+    yield "data: [DONE]\n\n"
 
-        return resp
+@app.post("/v1/completions", tags=["NLP"])
+async def generate_text(request: GenerationRequest):
 
+    try:
+        result = await nexa_run_text_generation(**request.dict())
 
+        return JSONResponse(content={
+            "choices": [{
+                "text": result["result"],
+                "logprobs": result.get("logprobs")
+            }]
+        })
     except Exception as e:
-        logging.error(f"Error in img2img generation: {e}")
+        logging.error(f"Error in text generation: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@app.post("/v1/chat/completions")
+@app.post("/v1/chat/completions", tags=["NLP"])
 async def chat_completions(request: ChatCompletionRequest):
-    logging.info(f"[/v1/chat/completions] Request logprobs: {request.logprobs}, top_logprobs: {request.top_logprobs}")
     try:
         generation_kwargs = GenerationRequest(
             prompt="" if len(request.messages) == 0 else request.messages[-1].content,
@@ -455,7 +419,7 @@ async def chat_completions(request: ChatCompletionRequest):
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@app.post("/v1/function-calling")
+@app.post("/v1/function-calling", tags=["NLP"])
 async def function_call(request: FunctionCallRequest):
     try:
         messages = function_call_system_prompt + [
@@ -474,8 +438,57 @@ async def function_call(request: FunctionCallRequest):
     except Exception as e:
         logging.error(f"Error in function calling: {e}")
         raise HTTPException(status_code=500, detail=str(e))
+    
+
+@app.post("/v1/txt2img", tags=["Computer Vision"])
+async def txt2img(request: ImageGenerationRequest):
+    try:
+        generation_kwargs = request.dict()
+
+        generated_images = await nexa_run_image_generation(**generation_kwargs)
+
+        resp = {"created": time.time(), "data": []}
+
+        for image in generated_images:
+            id = int(time.time())
+            if not os.path.exists("nexa_server_output"):
+                os.makedirs("nexa_server_output")
+            image_path = os.path.join("nexa_server_output", f"txt2img_{id}.png")
+            image.save(image_path)
+            img = ImageResponse(base64=base64_encode_image(image_path), url=os.path.abspath(image_path))
+            resp["data"].append(img)
+
+        return resp
+
+    except Exception as e:
+        logging.error(f"Error in txt2img generation: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/v1/img2img", tags=["Computer Vision"])
+async def img2img(request: ImageGenerationRequest):
+    try:
+        generation_kwargs = request.dict()
+
+        generated_images = await nexa_run_image_generation(**generation_kwargs)
+        resp = {"created": time.time(), "data": []}
+
+        for image in generated_images:
+            id = int(time.time())
+            if not os.path.exists("nexa_server_output"):
+                os.makedirs("nexa_server_output")
+            image_path = os.path.join("nexa_server_output", f"img2img_{id}.png")
+            image.save(image_path)
+            img = ImageResponse(base64=base64_encode_image(image_path), url=os.path.abspath(image_path))
+            resp["data"].append(img)
+
+        return resp
+
+
+    except Exception as e:
+        logging.error(f"Error in img2img generation: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
 
-@app.post("/v1/audio/transcriptions")
+@app.post("/v1/audio/transcriptions", tags=["Audio"])
 async def transcribe_audio(
     file: UploadFile = File(...),
     beam_size: Optional[int] = Query(5, description="Beam size for transcription"),
@@ -503,7 +516,7 @@ async def transcribe_audio(
     finally:
         os.unlink(temp_audio_path)
 
-@app.post("/v1/audio/translations")
+@app.post("/v1/audio/translations", tags=["Audio"])
 async def translate_audio(
     file: UploadFile = File(...),
     beam_size: Optional[int] = Query(5, description="Beam size for translation"),
@@ -528,22 +541,14 @@ async def translate_audio(
     finally:
         os.unlink(temp_audio_path)
 
-
-def run_nexa_ai_service(model_path_arg, **kwargs):
-    global model_path
-    model_path = model_path_arg or "gemma"
-    os.environ["MODEL_PATH"] = model_path
-    host = kwargs.get("host", "0.0.0.0")
-    port = kwargs.get("port", 8000)
-    reload = kwargs.get("reload", False)
-    uvicorn.run(app, host=host, port=port, reload=reload)
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Run the Nexa AI Text Generation Service"
     )
     parser.add_argument("model_path", type=str, nargs='?', default="gemma", help="Folder Path on Amazon S3")
+    parser.add_argument(
+        "--nctx", type=int, default=2048, help="Length of context window"
+    )
     parser.add_argument(
         "--host", type=str, default="0.0.0.0", help="Host to bind the server to"
     )
@@ -556,4 +561,4 @@ if __name__ == "__main__":
         help="Enable automatic reloading on code changes",
     )
     args = parser.parse_args()
-    run_nexa_ai_service(args.model_path, host=args.host, port=args.port, reload=args.reload)
+    run_nexa_ai_service(args.model_path, nctx=args.nctx, host=args.host, port=args.port, reload=args.reload)

nexaai-0.0.8.3/nexa/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.8.3"