ggufly 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ggufly/__init__.py +67 -0
- ggufly/api.py +181 -0
- ggufly/cli.py +153 -0
- ggufly/client.py +251 -0
- ggufly/config.py +43 -0
- ggufly/engine.py +376 -0
- ggufly/py.typed +1 -0
- ggufly/resolve.py +65 -0
- ggufly/server.py +230 -0
- ggufly/utils.py +115 -0
- ggufly-0.2.0.dist-info/METADATA +218 -0
- ggufly-0.2.0.dist-info/RECORD +16 -0
- ggufly-0.2.0.dist-info/WHEEL +5 -0
- ggufly-0.2.0.dist-info/entry_points.txt +2 -0
- ggufly-0.2.0.dist-info/licenses/LICENSE +21 -0
- ggufly-0.2.0.dist-info/top_level.txt +1 -0
ggufly/__init__.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
r"""ggufly: modelos GGUF locales con llama.cpp precompilado y
|
|
2
|
+
respuestas simultaneas compartiendo memoria (un solo modelo cargado).
|
|
3
|
+
|
|
4
|
+
NOTA: ggufly NO descarga modelos. El usuario obtiene sus .gguf por su cuenta
|
|
5
|
+
(p.ej. `huggingface-cli download <repo>`) y pasa la ruta local.
|
|
6
|
+
|
|
7
|
+
Ejemplo:
|
|
8
|
+
import ggufly
|
|
9
|
+
m = ggufly.load(r"C:\modelos\qwen2.5-0.5b-instruct-q4_k_m.gguf")
|
|
10
|
+
print(m.chat("Hola!")["choices"][0]["message"]["content"])
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
__version__ = "0.2.0"
|
|
17
|
+
__all__ = [
|
|
18
|
+
"Model",
|
|
19
|
+
"load",
|
|
20
|
+
"run_many",
|
|
21
|
+
"arun_many",
|
|
22
|
+
"ensure_engine",
|
|
23
|
+
"installed_engine",
|
|
24
|
+
"serve",
|
|
25
|
+
"start_server",
|
|
26
|
+
"shutdown_all",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
from .api import Model, load, run_many, arun_many
|
|
30
|
+
from .engine import ensure_engine, installed_engine
|
|
31
|
+
from .server import Server, ServerConfig, get_or_start, shutdown_all
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def start_server(
|
|
35
|
+
model: str,
|
|
36
|
+
*,
|
|
37
|
+
mmproj: Optional[str] = None,
|
|
38
|
+
port: Optional[int] = None,
|
|
39
|
+
n_ctx: int = 8192,
|
|
40
|
+
n_parallel: int = 4,
|
|
41
|
+
n_gpu_layers: int = -1,
|
|
42
|
+
backend: Optional[str] = None,
|
|
43
|
+
extra_args: Optional[list] = None,
|
|
44
|
+
host: str = "127.0.0.1",
|
|
45
|
+
) -> Server:
|
|
46
|
+
"""Levanta un llama-server gestionado y devuelve el objeto Server.
|
|
47
|
+
|
|
48
|
+
Util si quieres exponer la API OpenAI-compatible de llama.cpp a otras
|
|
49
|
+
apps (el endpoint esta en Server.base_url).
|
|
50
|
+
"""
|
|
51
|
+
cfg = ServerConfig(
|
|
52
|
+
model=model,
|
|
53
|
+
mmproj=mmproj,
|
|
54
|
+
port=port,
|
|
55
|
+
n_ctx=n_ctx,
|
|
56
|
+
n_parallel=n_parallel,
|
|
57
|
+
n_gpu_layers=n_gpu_layers,
|
|
58
|
+
backend=backend,
|
|
59
|
+
extra_args=tuple(extra_args or ()),
|
|
60
|
+
host=host,
|
|
61
|
+
)
|
|
62
|
+
return get_or_start(cfg)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def serve(model: str, **kwargs) -> Server:
|
|
66
|
+
"""Alias de start_server."""
|
|
67
|
+
return start_server(model, **kwargs)
|
ggufly/api.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""API de alto nivel: un objeto `Model` que enciende/reutiliza un solo
|
|
2
|
+
llama-server compartido y expone chat, streaming, vision y embeddings.
|
|
3
|
+
|
|
4
|
+
Uso basico:
|
|
5
|
+
import ggufly
|
|
6
|
+
m = ggufly.load("Qwen/Qwen2.5-VL-7B-Instruct-GGUF") # auto-descarga
|
|
7
|
+
r = m.chat("Hola, ¿quien eres?")
|
|
8
|
+
print(r["choices"][0]["message"]["content"])
|
|
9
|
+
|
|
10
|
+
Vision:
|
|
11
|
+
r = m.chat("¿Que hay en la imagen?", images=["foto.jpg"])
|
|
12
|
+
|
|
13
|
+
Concurrencia (comparte el mismo modelo en VRAM):
|
|
14
|
+
import concurrent.futures as cf
|
|
15
|
+
with cf.ThreadPoolExecutor(8) as ex:
|
|
16
|
+
outs = list(ex.map(lambda q: m.chat(q, max_tokens=200), preguntas))
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
|
|
23
|
+
|
|
24
|
+
from .client import AsyncChatClient, ChatClient
|
|
25
|
+
from .resolve import resolve
|
|
26
|
+
from .server import ServerConfig, get_or_start
|
|
27
|
+
|
|
28
|
+
Message = Dict[str, Any]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _as_messages(prompt_or_messages: Union[str, List[Message]]) -> List[Message]:
|
|
32
|
+
if isinstance(prompt_or_messages, str):
|
|
33
|
+
return [{"role": "user", "content": prompt_or_messages}]
|
|
34
|
+
return prompt_or_messages
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Model:
|
|
38
|
+
"""Modelo GGUF cargado (texto y/o vision) con respuestas simultaneas."""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
source: str,
|
|
43
|
+
*,
|
|
44
|
+
mmproj: Optional[str] = None,
|
|
45
|
+
backend: Optional[str] = None,
|
|
46
|
+
n_ctx: int = 8192,
|
|
47
|
+
n_parallel: int = 4,
|
|
48
|
+
n_gpu_layers: int = -1,
|
|
49
|
+
flash_attn: bool = True,
|
|
50
|
+
threads: Optional[int] = None,
|
|
51
|
+
port: Optional[int] = None,
|
|
52
|
+
extra_args: Optional[List[str]] = None,
|
|
53
|
+
):
|
|
54
|
+
model_path, mmproj_path = resolve(source, mmproj=mmproj)
|
|
55
|
+
self.model_path: Path = model_path
|
|
56
|
+
self.mmproj_path: Optional[Path] = mmproj_path
|
|
57
|
+
self.config = ServerConfig(
|
|
58
|
+
model=str(model_path),
|
|
59
|
+
mmproj=str(mmproj_path) if mmproj_path else None,
|
|
60
|
+
backend=backend,
|
|
61
|
+
n_ctx=n_ctx,
|
|
62
|
+
n_parallel=n_parallel,
|
|
63
|
+
n_gpu_layers=n_gpu_layers,
|
|
64
|
+
flash_attn=flash_attn,
|
|
65
|
+
threads=threads,
|
|
66
|
+
port=port,
|
|
67
|
+
extra_args=tuple(extra_args or ()),
|
|
68
|
+
)
|
|
69
|
+
self.server = get_or_start(self.config)
|
|
70
|
+
self.base_url = self.server.base_url
|
|
71
|
+
self._sync = ChatClient(self.base_url)
|
|
72
|
+
self._async = AsyncChatClient(self.base_url)
|
|
73
|
+
|
|
74
|
+
# ------------------------------------------------------------ propiedades
|
|
75
|
+
@property
|
|
76
|
+
def has_vision(self) -> bool:
|
|
77
|
+
return self.mmproj_path is not None
|
|
78
|
+
|
|
79
|
+
def slots(self) -> dict:
|
|
80
|
+
"""Estado de los slots de batching (activos, tamaños de ctx, etc)."""
|
|
81
|
+
return self._sync.slots()
|
|
82
|
+
|
|
83
|
+
def __repr__(self) -> str:
|
|
84
|
+
return (
|
|
85
|
+
f"<Model {self.model_path.name} vision={self.has_vision} "
|
|
86
|
+
f"slots={self.config.n_parallel} at {self.base_url}>"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# ----------------------------------------------------------------- sync
|
|
90
|
+
def chat(
|
|
91
|
+
self,
|
|
92
|
+
prompt_or_messages: Union[str, List[Message]],
|
|
93
|
+
*,
|
|
94
|
+
images: Optional[List[Any]] = None,
|
|
95
|
+
stream: bool = False,
|
|
96
|
+
**gen: Any,
|
|
97
|
+
) -> Union[Dict[str, Any], Iterator[str]]:
|
|
98
|
+
"""Chat con el modelo. Con stream=True devuelve un iterador de chunks.
|
|
99
|
+
|
|
100
|
+
Args de generacion habituales: max_tokens, temperature, top_p,
|
|
101
|
+
repeat_penalty, stop, seed...
|
|
102
|
+
"""
|
|
103
|
+
msgs = _as_messages(prompt_or_messages)
|
|
104
|
+
return self._sync.chat(msgs, images=images, stream=stream, **gen)
|
|
105
|
+
|
|
106
|
+
def generate(self, prompt: str, *, stream: bool = False, **gen: Any) -> Union[dict, Iterator[str]]:
|
|
107
|
+
"""Completado crudo (/completion) sin plantilla de chat."""
|
|
108
|
+
return self._sync.completion(prompt, stream=stream, **gen)
|
|
109
|
+
|
|
110
|
+
def embed(self, texts: Union[str, List[str]]) -> List[List[float]]:
|
|
111
|
+
"""Embeddings (requiere un modelo de embeddings GGUF)."""
|
|
112
|
+
return self._sync.embeddings(texts)
|
|
113
|
+
|
|
114
|
+
# ---------------------------------------------------------------- async
|
|
115
|
+
async def achat(
|
|
116
|
+
self,
|
|
117
|
+
prompt_or_messages: Union[str, List[Message]],
|
|
118
|
+
*,
|
|
119
|
+
images: Optional[List[Any]] = None,
|
|
120
|
+
stream: bool = False,
|
|
121
|
+
**gen: Any,
|
|
122
|
+
) -> Union[Dict[str, Any], AsyncIterator[str]]:
|
|
123
|
+
msgs = _as_messages(prompt_or_messages)
|
|
124
|
+
return await self._async.chat(msgs, images=images, stream=stream, **gen)
|
|
125
|
+
|
|
126
|
+
async def agenerate(self, prompt: str, *, stream: bool = False, **gen: Any) -> Union[dict, AsyncIterator[str]]:
|
|
127
|
+
return await self._async.completion(prompt, stream=stream, **gen)
|
|
128
|
+
|
|
129
|
+
async def aembed(self, texts: Union[str, List[str]]) -> List[List[float]]:
|
|
130
|
+
return await self._async.embeddings(texts)
|
|
131
|
+
|
|
132
|
+
# ----------------------------------------------------------- utilidades
|
|
133
|
+
def wait_stream(self, stream_iter) -> str:
|
|
134
|
+
"""Consume un stream y devuelve el texto completo."""
|
|
135
|
+
return "".join(stream_iter)
|
|
136
|
+
|
|
137
|
+
async def await_stream(self, async_iter) -> str:
|
|
138
|
+
parts = []
|
|
139
|
+
async for chunk in async_iter:
|
|
140
|
+
parts.append(chunk)
|
|
141
|
+
return "".join(parts)
|
|
142
|
+
|
|
143
|
+
def close(self) -> None:
|
|
144
|
+
"""Para el servidor solo si este objeto fue quien lo creo (no recomendado
|
|
145
|
+
si compartes el modelo entre varios `Model`). Usa ggufly.shutdown_all()."""
|
|
146
|
+
from .server import shutdown_all
|
|
147
|
+
|
|
148
|
+
shutdown_all()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def load(source: str, **kwargs: Any) -> Model:
|
|
152
|
+
"""Atajo para Model(...)."""
|
|
153
|
+
return Model(source, **kwargs)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def run_many(model: "Model", prompts: List[str], *, workers: int = 4, **gen: Any) -> List[Dict[str, Any]]:
|
|
157
|
+
"""Ejecuta varios prompts en paralelo reutilizando el mismo modelo en VRAM."""
|
|
158
|
+
import concurrent.futures as cf
|
|
159
|
+
|
|
160
|
+
with cf.ThreadPoolExecutor(max_workers=workers) as ex:
|
|
161
|
+
futures = [ex.submit(lambda p=p: model.chat(p, **gen)) for p in prompts]
|
|
162
|
+
return [f.result() for f in futures]
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
async def arun_many(
|
|
166
|
+
model: "Model",
|
|
167
|
+
prompts: List[str],
|
|
168
|
+
*,
|
|
169
|
+
concurrency: Optional[int] = None,
|
|
170
|
+
**gen: Any,
|
|
171
|
+
) -> List[Dict[str, Any]]:
|
|
172
|
+
"""Igual que run_many pero async (semáforo opcional)."""
|
|
173
|
+
sem = asyncio.Semaphore(concurrency) if concurrency else None
|
|
174
|
+
|
|
175
|
+
async def one(p: str):
|
|
176
|
+
if sem:
|
|
177
|
+
async with sem:
|
|
178
|
+
return await model.achat(p, **gen)
|
|
179
|
+
return await model.achat(p, **gen)
|
|
180
|
+
|
|
181
|
+
return await asyncio.gather(*[one(p) for p in prompts])
|
ggufly/cli.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""CLI de ggufly.
|
|
2
|
+
|
|
3
|
+
ggufly engines status|install [--backend cuda|vulkan|cpu]
|
|
4
|
+
ggufly serve <ruta-modelo.gguf> [--mmproj X] [--port P] [--parallel N] ...
|
|
5
|
+
ggufly chat <ruta-modelo.gguf> [--image FOTO]
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
from typing import Optional
|
|
13
|
+
|
|
14
|
+
from .utils import setup_logging
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _add_model_args(p: argparse.ArgumentParser) -> None:
|
|
18
|
+
p.add_argument("model", help="Ruta local a un archivo .gguf o carpeta que lo contenga")
|
|
19
|
+
p.add_argument("--mmproj", default=None, help="Archivo mmproj para vision")
|
|
20
|
+
p.add_argument("--backend", default=None, choices=["cuda", "vulkan", "cpu"])
|
|
21
|
+
p.add_argument("--ctx", type=int, default=8192, help="Contexto total")
|
|
22
|
+
p.add_argument("--parallel", type=int, default=4, help="Slots simultaneos (comparten VRAM)")
|
|
23
|
+
p.add_argument("--ngl", type=int, default=-1, help="Capas en GPU (-1 = todas)")
|
|
24
|
+
p.add_argument("--port", type=int, default=None)
|
|
25
|
+
p.add_argument("--threads", type=int, default=None)
|
|
26
|
+
p.add_argument("extra", nargs="*", help="Argumentos extra para llama-server (-- --key value)")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main(argv: Optional[list] = None) -> int:
|
|
30
|
+
ap = argparse.ArgumentParser(prog="ggufly", description="Modelos GGUF sin compilar nada")
|
|
31
|
+
ap.add_argument("-v", "--verbose", action="store_true")
|
|
32
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
33
|
+
|
|
34
|
+
p_eng = sub.add_parser("engines", help="Instalar/ver el motor llama.cpp")
|
|
35
|
+
eng_sub = p_eng.add_subparsers(dest="eng_cmd", required=True)
|
|
36
|
+
eng_sub.add_parser("status")
|
|
37
|
+
p_inst = eng_sub.add_parser("install")
|
|
38
|
+
p_inst.add_argument("--backend", choices=["cuda", "vulkan", "cpu"], default=None)
|
|
39
|
+
|
|
40
|
+
p_serve = sub.add_parser("serve", help="Servir el modelo (API OpenAI-compatible)")
|
|
41
|
+
_add_model_args(p_serve)
|
|
42
|
+
|
|
43
|
+
p_chat = sub.add_parser("chat", help="Chat interactivo en la terminal")
|
|
44
|
+
_add_model_args(p_chat)
|
|
45
|
+
|
|
46
|
+
args = ap.parse_args(argv)
|
|
47
|
+
setup_logging(args.verbose)
|
|
48
|
+
|
|
49
|
+
if args.cmd == "engines":
|
|
50
|
+
return _cmd_engines(args)
|
|
51
|
+
if args.cmd == "serve":
|
|
52
|
+
return _cmd_serve(args)
|
|
53
|
+
if args.cmd == "chat":
|
|
54
|
+
return _cmd_chat(args)
|
|
55
|
+
return 1
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _cmd_engines(args) -> int:
|
|
59
|
+
from .engine import installed_engine, ensure_engine
|
|
60
|
+
|
|
61
|
+
if args.eng_cmd == "status":
|
|
62
|
+
e = installed_engine()
|
|
63
|
+
if e:
|
|
64
|
+
print(f"Instalado: backend={e.backend} dir={e.directory}")
|
|
65
|
+
print(f" {e.version()}")
|
|
66
|
+
else:
|
|
67
|
+
print("Motor no instalado. Usa: ggufly engines install")
|
|
68
|
+
return 0
|
|
69
|
+
|
|
70
|
+
e = ensure_engine(args.backend, allow_build=True)
|
|
71
|
+
print(f"Instalado backend={e.backend} en {e.directory}")
|
|
72
|
+
print(f" {e.version()}")
|
|
73
|
+
return 0
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _cmd_serve(args) -> int:
|
|
77
|
+
from .server import ServerConfig, get_or_start
|
|
78
|
+
|
|
79
|
+
extra = tuple(args.extra or ())
|
|
80
|
+
cfg = ServerConfig(
|
|
81
|
+
model=args.model,
|
|
82
|
+
mmproj=args.mmproj,
|
|
83
|
+
backend=args.backend,
|
|
84
|
+
n_ctx=args.ctx,
|
|
85
|
+
n_parallel=args.parallel,
|
|
86
|
+
n_gpu_layers=args.ngl,
|
|
87
|
+
threads=args.threads,
|
|
88
|
+
port=args.port,
|
|
89
|
+
extra_args=extra,
|
|
90
|
+
)
|
|
91
|
+
srv = get_or_start(cfg)
|
|
92
|
+
print(f"API lista: {srv.base_url}/v1/chat/completions")
|
|
93
|
+
if args.mmproj:
|
|
94
|
+
print(f"Vision activa (mmproj): {args.mmproj}")
|
|
95
|
+
try:
|
|
96
|
+
while True:
|
|
97
|
+
input()
|
|
98
|
+
except (KeyboardInterrupt, EOFError):
|
|
99
|
+
pass
|
|
100
|
+
finally:
|
|
101
|
+
srv.stop()
|
|
102
|
+
return 0
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _cmd_chat(args) -> int:
|
|
106
|
+
from .api import Model
|
|
107
|
+
|
|
108
|
+
m = Model(
|
|
109
|
+
args.model,
|
|
110
|
+
mmproj=args.mmproj,
|
|
111
|
+
backend=args.backend,
|
|
112
|
+
n_ctx=args.ctx,
|
|
113
|
+
n_parallel=args.parallel,
|
|
114
|
+
n_gpu_layers=args.ngl,
|
|
115
|
+
threads=args.threads,
|
|
116
|
+
port=args.port,
|
|
117
|
+
extra_args=list(args.extra or ()),
|
|
118
|
+
)
|
|
119
|
+
print(f"Modelo: {m.model_path.name} vision={m.has_vision} slots={args.parallel}")
|
|
120
|
+
print("Escribe /image <ruta> para adjuntar una imagen al siguiente mensaje; /exit para salir.")
|
|
121
|
+
history = []
|
|
122
|
+
pending_images: list = []
|
|
123
|
+
while True:
|
|
124
|
+
try:
|
|
125
|
+
user = input("tu> ").strip()
|
|
126
|
+
except (KeyboardInterrupt, EOFError):
|
|
127
|
+
break
|
|
128
|
+
if not user:
|
|
129
|
+
continue
|
|
130
|
+
if user == "/exit":
|
|
131
|
+
break
|
|
132
|
+
if user.startswith("/image "):
|
|
133
|
+
pending_images.append(user[7:].strip())
|
|
134
|
+
print("(imagen agregada)")
|
|
135
|
+
continue
|
|
136
|
+
history.append({"role": "user", "content": user})
|
|
137
|
+
print("asistente> ", end="", flush=True)
|
|
138
|
+
full = ""
|
|
139
|
+
try:
|
|
140
|
+
for chunk in m.chat(history, images=pending_images or None, stream=True):
|
|
141
|
+
print(chunk, end="", flush=True)
|
|
142
|
+
full += chunk
|
|
143
|
+
except Exception as e:
|
|
144
|
+
print(f"\n[error] {e}", file=sys.stderr)
|
|
145
|
+
print()
|
|
146
|
+
pending_images = []
|
|
147
|
+
if full:
|
|
148
|
+
history.append({"role": "assistant", "content": full})
|
|
149
|
+
return 0
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
if __name__ == "__main__":
|
|
153
|
+
raise SystemExit(main())
|
ggufly/client.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""Cliente HTTP (OpenAI-compatible) contra llama-server, con streaming SSE."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import base64
|
|
6
|
+
import mimetypes
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
Message = Dict[str, Any]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def encode_image(obj: Union[str, Path, bytes]) -> str:
|
|
16
|
+
"""Convierte una imagen (ruta, URL o bytes) a data-URI base64.
|
|
17
|
+
|
|
18
|
+
- Ruta local -> lee el archivo.
|
|
19
|
+
- http(s):// -> descarga.
|
|
20
|
+
- bytes -> usa tal cual.
|
|
21
|
+
"""
|
|
22
|
+
mime = None
|
|
23
|
+
if isinstance(obj, (str, Path)):
|
|
24
|
+
s = str(obj)
|
|
25
|
+
if s.startswith(("http://", "https://")):
|
|
26
|
+
r = httpx.get(s, timeout=60.0, follow_redirects=True)
|
|
27
|
+
r.raise_for_status()
|
|
28
|
+
data = r.content
|
|
29
|
+
mime = r.headers.get("content-type")
|
|
30
|
+
else:
|
|
31
|
+
p = Path(obj)
|
|
32
|
+
data = p.read_bytes()
|
|
33
|
+
mime = mimetypes.guess_type(p.name)[0]
|
|
34
|
+
else:
|
|
35
|
+
data = bytes(obj)
|
|
36
|
+
mime = mime or "image/png"
|
|
37
|
+
if not mime.startswith("image/"):
|
|
38
|
+
mime = "image/png"
|
|
39
|
+
return f"data:{mime};base64,{base64.b64encode(data).decode()}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _BaseClient:
|
|
43
|
+
def _prepare_messages(
|
|
44
|
+
self,
|
|
45
|
+
messages: List[Message],
|
|
46
|
+
images: Optional[List[Any]] = None,
|
|
47
|
+
) -> List[Message]:
|
|
48
|
+
"""Inyecta imagenes en el ultimo mensaje user como content parts."""
|
|
49
|
+
msgs = [dict(m) for m in messages]
|
|
50
|
+
if images:
|
|
51
|
+
parts: List[Dict[str, Any]] = []
|
|
52
|
+
for img in images:
|
|
53
|
+
parts.append({"type": "image_url", "image_url": {"url": encode_image(img)}})
|
|
54
|
+
if not msgs or msgs[-1].get("role") != "user":
|
|
55
|
+
msgs.append({"role": "user", "content": []})
|
|
56
|
+
last = msgs[-1]
|
|
57
|
+
if isinstance(last.get("content"), str):
|
|
58
|
+
parts.append({"type": "text", "text": last["content"]})
|
|
59
|
+
elif isinstance(last.get("content"), list):
|
|
60
|
+
parts.extend(last["content"])
|
|
61
|
+
last["content"] = parts
|
|
62
|
+
return msgs
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def _iter_sse_lines(line_iter) -> Iterator[str]:
|
|
66
|
+
for line in line_iter:
|
|
67
|
+
if line.startswith("data:"):
|
|
68
|
+
yield line[5:].strip()
|
|
69
|
+
|
|
70
|
+
@staticmethod
|
|
71
|
+
def _parse_sse_payload(data: str) -> Optional[dict]:
|
|
72
|
+
if data == "[DONE]":
|
|
73
|
+
return None
|
|
74
|
+
try:
|
|
75
|
+
return json.loads(data)
|
|
76
|
+
except json.JSONDecodeError:
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
@staticmethod
|
|
80
|
+
def _delta_text(obj: dict) -> str:
|
|
81
|
+
choices = obj.get("choices") or []
|
|
82
|
+
if not choices:
|
|
83
|
+
return ""
|
|
84
|
+
d = choices[0].get("delta") or {}
|
|
85
|
+
return d.get("content") or ""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class ChatClient(_BaseClient):
|
|
89
|
+
"""Cliente sincrono."""
|
|
90
|
+
|
|
91
|
+
def __init__(self, base_url: str, timeout: float = 600.0):
|
|
92
|
+
self.base_url = base_url.rstrip("/")
|
|
93
|
+
self._client = httpx.Client(timeout=timeout)
|
|
94
|
+
|
|
95
|
+
# ------------------------------------------------------------- chat
|
|
96
|
+
def chat(
|
|
97
|
+
self,
|
|
98
|
+
messages: List[Message],
|
|
99
|
+
*,
|
|
100
|
+
images: Optional[List[Any]] = None,
|
|
101
|
+
stream: bool = False,
|
|
102
|
+
**gen: Any,
|
|
103
|
+
) -> Union[dict, Iterator[str]]:
|
|
104
|
+
body: Dict[str, Any] = {
|
|
105
|
+
"messages": self._prepare_messages(messages, images),
|
|
106
|
+
"stream": stream,
|
|
107
|
+
**gen,
|
|
108
|
+
}
|
|
109
|
+
if not stream:
|
|
110
|
+
r = self._client.post(f"{self.base_url}/v1/chat/completions", json=body)
|
|
111
|
+
r.raise_for_status()
|
|
112
|
+
return r.json()
|
|
113
|
+
|
|
114
|
+
def gen_stream():
|
|
115
|
+
with self._client.stream(
|
|
116
|
+
"POST", f"{self.base_url}/v1/chat/completions", json=body
|
|
117
|
+
) as resp:
|
|
118
|
+
resp.raise_for_status()
|
|
119
|
+
for payload in self._iter_sse_lines(resp.iter_lines()):
|
|
120
|
+
obj = self._parse_sse_payload(payload)
|
|
121
|
+
if obj is None:
|
|
122
|
+
continue
|
|
123
|
+
usage = obj.get("usage")
|
|
124
|
+
text = self._delta_text(obj)
|
|
125
|
+
if text or usage:
|
|
126
|
+
yield text
|
|
127
|
+
|
|
128
|
+
return gen_stream()
|
|
129
|
+
|
|
130
|
+
# --------------------------------------------------------- completion
|
|
131
|
+
def completion(self, prompt: str, *, stream: bool = False, **gen: Any) -> Union[dict, Iterator[str]]:
|
|
132
|
+
body = {"prompt": prompt, "stream": stream, **gen}
|
|
133
|
+
if not stream:
|
|
134
|
+
r = self._client.post(f"{self.base_url}/completion", json=body)
|
|
135
|
+
r.raise_for_status()
|
|
136
|
+
return r.json()
|
|
137
|
+
|
|
138
|
+
def gen_stream():
|
|
139
|
+
with self._client.stream("POST", f"{self.base_url}/completion", json=body) as resp:
|
|
140
|
+
resp.raise_for_status()
|
|
141
|
+
for payload in self._iter_sse_lines(resp.iter_lines()):
|
|
142
|
+
obj = self._parse_sse_payload(payload)
|
|
143
|
+
if obj is None:
|
|
144
|
+
continue
|
|
145
|
+
text = obj.get("content") or ""
|
|
146
|
+
if text or obj.get("stop"):
|
|
147
|
+
yield text
|
|
148
|
+
|
|
149
|
+
return gen_stream()
|
|
150
|
+
|
|
151
|
+
# ---------------------------------------------------------- embeddings
|
|
152
|
+
def embeddings(self, texts: Union[str, List[str]]) -> List[List[float]]:
|
|
153
|
+
payload = {"input": texts}
|
|
154
|
+
r = self._client.post(f"{self.base_url}/v1/embeddings", json=payload)
|
|
155
|
+
r.raise_for_status()
|
|
156
|
+
data = r.json()["data"]
|
|
157
|
+
if isinstance(texts, str):
|
|
158
|
+
return [data[0]["embedding"]]
|
|
159
|
+
return [d["embedding"] for d in data]
|
|
160
|
+
|
|
161
|
+
# --------------------------------------------------------------- extras
|
|
162
|
+
def tokenize(self, text: str) -> List[int]:
|
|
163
|
+
r = self._client.post(f"{self.base_url}/tokenize", json={"content": text})
|
|
164
|
+
r.raise_for_status()
|
|
165
|
+
return r.json()["tokens"]
|
|
166
|
+
|
|
167
|
+
def detokenize(self, tokens: List[int]) -> str:
|
|
168
|
+
r = self._client.post(f"{self.base_url}/detokenize", json={"tokens": tokens})
|
|
169
|
+
r.raise_for_status()
|
|
170
|
+
return r.json()["content"]
|
|
171
|
+
|
|
172
|
+
def slots(self) -> dict:
|
|
173
|
+
r = self._client.get(f"{self.base_url}/slots")
|
|
174
|
+
r.raise_for_status()
|
|
175
|
+
return r.json()
|
|
176
|
+
|
|
177
|
+
def close(self) -> None:
|
|
178
|
+
self._client.close()
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class AsyncChatClient(_BaseClient):
|
|
182
|
+
"""Cliente asincrono."""
|
|
183
|
+
|
|
184
|
+
def __init__(self, base_url: str, timeout: float = 600.0):
|
|
185
|
+
self.base_url = base_url.rstrip("/")
|
|
186
|
+
self._client = httpx.AsyncClient(timeout=timeout)
|
|
187
|
+
|
|
188
|
+
async def chat(
|
|
189
|
+
self,
|
|
190
|
+
messages: List[Message],
|
|
191
|
+
*,
|
|
192
|
+
images: Optional[List[Any]] = None,
|
|
193
|
+
stream: bool = False,
|
|
194
|
+
**gen: Any,
|
|
195
|
+
) -> Union[dict, AsyncIterator[str]]:
|
|
196
|
+
body = {"messages": self._prepare_messages(messages, images), "stream": stream, **gen}
|
|
197
|
+
if not stream:
|
|
198
|
+
r = await self._client.post(f"{self.base_url}/v1/chat/completions", json=body)
|
|
199
|
+
r.raise_for_status()
|
|
200
|
+
return r.json()
|
|
201
|
+
|
|
202
|
+
async def gen_stream():
|
|
203
|
+
async with self._client.stream(
|
|
204
|
+
"POST", f"{self.base_url}/v1/chat/completions", json=body
|
|
205
|
+
) as resp:
|
|
206
|
+
resp.raise_for_status()
|
|
207
|
+
async for line in resp.aiter_lines():
|
|
208
|
+
if not line.startswith("data:"):
|
|
209
|
+
continue
|
|
210
|
+
obj = self._parse_sse_payload(line[5:].strip())
|
|
211
|
+
if obj is None:
|
|
212
|
+
continue
|
|
213
|
+
usage = obj.get("usage")
|
|
214
|
+
text = self._delta_text(obj)
|
|
215
|
+
if text or usage:
|
|
216
|
+
yield text
|
|
217
|
+
|
|
218
|
+
return gen_stream()
|
|
219
|
+
|
|
220
|
+
async def completion(self, prompt: str, *, stream: bool = False, **gen: Any) -> Union[dict, AsyncIterator[str]]:
|
|
221
|
+
body = {"prompt": prompt, "stream": stream, **gen}
|
|
222
|
+
if not stream:
|
|
223
|
+
r = await self._client.post(f"{self.base_url}/completion", json=body)
|
|
224
|
+
r.raise_for_status()
|
|
225
|
+
return r.json()
|
|
226
|
+
|
|
227
|
+
async def gen_stream():
|
|
228
|
+
async with self._client.stream("POST", f"{self.base_url}/completion", json=body) as resp:
|
|
229
|
+
resp.raise_for_status()
|
|
230
|
+
async for line in resp.aiter_lines():
|
|
231
|
+
if not line.startswith("data:"):
|
|
232
|
+
continue
|
|
233
|
+
obj = self._parse_sse_payload(line[5:].strip())
|
|
234
|
+
if obj is None:
|
|
235
|
+
continue
|
|
236
|
+
text = obj.get("content") or ""
|
|
237
|
+
if text or obj.get("stop"):
|
|
238
|
+
yield text
|
|
239
|
+
|
|
240
|
+
return gen_stream()
|
|
241
|
+
|
|
242
|
+
async def embeddings(self, texts: Union[str, List[str]]) -> List[List[float]]:
|
|
243
|
+
r = await self._client.post(f"{self.base_url}/v1/embeddings", json={"input": texts})
|
|
244
|
+
r.raise_for_status()
|
|
245
|
+
data = r.json()["data"]
|
|
246
|
+
if isinstance(texts, str):
|
|
247
|
+
return [data[0]["embedding"]]
|
|
248
|
+
return [d["embedding"] for d in data]
|
|
249
|
+
|
|
250
|
+
async def close(self) -> None:
|
|
251
|
+
await self._client.aclose()
|
ggufly/config.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Rutas y configuracion global de ggufly.
|
|
2
|
+
|
|
3
|
+
Variables de entorno soportadas:
|
|
4
|
+
GGUFLY_HOME -> raiz de datos (por defecto: %LOCALAPPDATA%/ggufly o ~/.cache/ggufly)
|
|
5
|
+
GGUFLY_MODELS -> carpeta de modelos descargados
|
|
6
|
+
GGUFLY_ENGINE -> carpeta de binarios de llama.cpp
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
APP_NAME = "ggufly"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def home_dir() -> Path:
|
|
18
|
+
env = os.environ.get("GGUFLY_HOME")
|
|
19
|
+
if env:
|
|
20
|
+
return Path(env)
|
|
21
|
+
if sys.platform == "win32":
|
|
22
|
+
base = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local")
|
|
23
|
+
return Path(base) / APP_NAME
|
|
24
|
+
xdg = os.environ.get("XDG_CACHE_HOME")
|
|
25
|
+
base = Path(xdg) if xdg else Path.home() / ".cache"
|
|
26
|
+
return base / APP_NAME
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def models_dir() -> Path:
|
|
30
|
+
return Path(os.environ.get("GGUFLY_MODELS") or (home_dir() / "models"))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def engine_dir() -> Path:
|
|
34
|
+
return Path(os.environ.get("GGUFLY_ENGINE") or (home_dir() / "engine"))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def logs_dir() -> Path:
|
|
38
|
+
return home_dir() / "logs"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def ensure_dirs() -> None:
|
|
42
|
+
for d in (home_dir(), models_dir(), logs_dir()):
|
|
43
|
+
d.mkdir(parents=True, exist_ok=True)
|