ggufly 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ggufly/__init__.py ADDED
@@ -0,0 +1,67 @@
1
+ r"""ggufly: modelos GGUF locales con llama.cpp precompilado y
2
+ respuestas simultaneas compartiendo memoria (un solo modelo cargado).
3
+
4
+ NOTA: ggufly NO descarga modelos. El usuario obtiene sus .gguf por su cuenta
5
+ (p.ej. `huggingface-cli download <repo>`) y pasa la ruta local.
6
+
7
+ Ejemplo:
8
+ import ggufly
9
+ m = ggufly.load(r"C:\modelos\qwen2.5-0.5b-instruct-q4_k_m.gguf")
10
+ print(m.chat("Hola!")["choices"][0]["message"]["content"])
11
+ """
12
+ from __future__ import annotations
13
+
14
+ from typing import Optional
15
+
16
+ __version__ = "0.2.0"
17
+ __all__ = [
18
+ "Model",
19
+ "load",
20
+ "run_many",
21
+ "arun_many",
22
+ "ensure_engine",
23
+ "installed_engine",
24
+ "serve",
25
+ "start_server",
26
+ "shutdown_all",
27
+ ]
28
+
29
+ from .api import Model, load, run_many, arun_many
30
+ from .engine import ensure_engine, installed_engine
31
+ from .server import Server, ServerConfig, get_or_start, shutdown_all
32
+
33
+
34
+ def start_server(
35
+ model: str,
36
+ *,
37
+ mmproj: Optional[str] = None,
38
+ port: Optional[int] = None,
39
+ n_ctx: int = 8192,
40
+ n_parallel: int = 4,
41
+ n_gpu_layers: int = -1,
42
+ backend: Optional[str] = None,
43
+ extra_args: Optional[list] = None,
44
+ host: str = "127.0.0.1",
45
+ ) -> Server:
46
+ """Levanta un llama-server gestionado y devuelve el objeto Server.
47
+
48
+ Util si quieres exponer la API OpenAI-compatible de llama.cpp a otras
49
+ apps (el endpoint esta en Server.base_url).
50
+ """
51
+ cfg = ServerConfig(
52
+ model=model,
53
+ mmproj=mmproj,
54
+ port=port,
55
+ n_ctx=n_ctx,
56
+ n_parallel=n_parallel,
57
+ n_gpu_layers=n_gpu_layers,
58
+ backend=backend,
59
+ extra_args=tuple(extra_args or ()),
60
+ host=host,
61
+ )
62
+ return get_or_start(cfg)
63
+
64
+
65
+ def serve(model: str, **kwargs) -> Server:
66
+ """Alias de start_server."""
67
+ return start_server(model, **kwargs)
ggufly/api.py ADDED
@@ -0,0 +1,181 @@
1
+ """API de alto nivel: un objeto `Model` que enciende/reutiliza un solo
2
+ llama-server compartido y expone chat, streaming, vision y embeddings.
3
+
4
+ Uso basico:
5
+ import ggufly
6
+ m = ggufly.load("Qwen/Qwen2.5-VL-7B-Instruct-GGUF") # auto-descarga
7
+ r = m.chat("Hola, ¿quien eres?")
8
+ print(r["choices"][0]["message"]["content"])
9
+
10
+ Vision:
11
+ r = m.chat("¿Que hay en la imagen?", images=["foto.jpg"])
12
+
13
+ Concurrencia (comparte el mismo modelo en VRAM):
14
+ import concurrent.futures as cf
15
+ with cf.ThreadPoolExecutor(8) as ex:
16
+ outs = list(ex.map(lambda q: m.chat(q, max_tokens=200), preguntas))
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import asyncio
21
+ from pathlib import Path
22
+ from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
23
+
24
+ from .client import AsyncChatClient, ChatClient
25
+ from .resolve import resolve
26
+ from .server import ServerConfig, get_or_start
27
+
28
+ Message = Dict[str, Any]
29
+
30
+
31
+ def _as_messages(prompt_or_messages: Union[str, List[Message]]) -> List[Message]:
32
+ if isinstance(prompt_or_messages, str):
33
+ return [{"role": "user", "content": prompt_or_messages}]
34
+ return prompt_or_messages
35
+
36
+
37
+ class Model:
38
+ """Modelo GGUF cargado (texto y/o vision) con respuestas simultaneas."""
39
+
40
+ def __init__(
41
+ self,
42
+ source: str,
43
+ *,
44
+ mmproj: Optional[str] = None,
45
+ backend: Optional[str] = None,
46
+ n_ctx: int = 8192,
47
+ n_parallel: int = 4,
48
+ n_gpu_layers: int = -1,
49
+ flash_attn: bool = True,
50
+ threads: Optional[int] = None,
51
+ port: Optional[int] = None,
52
+ extra_args: Optional[List[str]] = None,
53
+ ):
54
+ model_path, mmproj_path = resolve(source, mmproj=mmproj)
55
+ self.model_path: Path = model_path
56
+ self.mmproj_path: Optional[Path] = mmproj_path
57
+ self.config = ServerConfig(
58
+ model=str(model_path),
59
+ mmproj=str(mmproj_path) if mmproj_path else None,
60
+ backend=backend,
61
+ n_ctx=n_ctx,
62
+ n_parallel=n_parallel,
63
+ n_gpu_layers=n_gpu_layers,
64
+ flash_attn=flash_attn,
65
+ threads=threads,
66
+ port=port,
67
+ extra_args=tuple(extra_args or ()),
68
+ )
69
+ self.server = get_or_start(self.config)
70
+ self.base_url = self.server.base_url
71
+ self._sync = ChatClient(self.base_url)
72
+ self._async = AsyncChatClient(self.base_url)
73
+
74
+ # ------------------------------------------------------------ propiedades
75
+ @property
76
+ def has_vision(self) -> bool:
77
+ return self.mmproj_path is not None
78
+
79
+ def slots(self) -> dict:
80
+ """Estado de los slots de batching (activos, tamaños de ctx, etc)."""
81
+ return self._sync.slots()
82
+
83
+ def __repr__(self) -> str:
84
+ return (
85
+ f"<Model {self.model_path.name} vision={self.has_vision} "
86
+ f"slots={self.config.n_parallel} at {self.base_url}>"
87
+ )
88
+
89
+ # ----------------------------------------------------------------- sync
90
+ def chat(
91
+ self,
92
+ prompt_or_messages: Union[str, List[Message]],
93
+ *,
94
+ images: Optional[List[Any]] = None,
95
+ stream: bool = False,
96
+ **gen: Any,
97
+ ) -> Union[Dict[str, Any], Iterator[str]]:
98
+ """Chat con el modelo. Con stream=True devuelve un iterador de chunks.
99
+
100
+ Args de generacion habituales: max_tokens, temperature, top_p,
101
+ repeat_penalty, stop, seed...
102
+ """
103
+ msgs = _as_messages(prompt_or_messages)
104
+ return self._sync.chat(msgs, images=images, stream=stream, **gen)
105
+
106
+ def generate(self, prompt: str, *, stream: bool = False, **gen: Any) -> Union[dict, Iterator[str]]:
107
+ """Completado crudo (/completion) sin plantilla de chat."""
108
+ return self._sync.completion(prompt, stream=stream, **gen)
109
+
110
+ def embed(self, texts: Union[str, List[str]]) -> List[List[float]]:
111
+ """Embeddings (requiere un modelo de embeddings GGUF)."""
112
+ return self._sync.embeddings(texts)
113
+
114
+ # ---------------------------------------------------------------- async
115
+ async def achat(
116
+ self,
117
+ prompt_or_messages: Union[str, List[Message]],
118
+ *,
119
+ images: Optional[List[Any]] = None,
120
+ stream: bool = False,
121
+ **gen: Any,
122
+ ) -> Union[Dict[str, Any], AsyncIterator[str]]:
123
+ msgs = _as_messages(prompt_or_messages)
124
+ return await self._async.chat(msgs, images=images, stream=stream, **gen)
125
+
126
+ async def agenerate(self, prompt: str, *, stream: bool = False, **gen: Any) -> Union[dict, AsyncIterator[str]]:
127
+ return await self._async.completion(prompt, stream=stream, **gen)
128
+
129
+ async def aembed(self, texts: Union[str, List[str]]) -> List[List[float]]:
130
+ return await self._async.embeddings(texts)
131
+
132
+ # ----------------------------------------------------------- utilidades
133
+ def wait_stream(self, stream_iter) -> str:
134
+ """Consume un stream y devuelve el texto completo."""
135
+ return "".join(stream_iter)
136
+
137
+ async def await_stream(self, async_iter) -> str:
138
+ parts = []
139
+ async for chunk in async_iter:
140
+ parts.append(chunk)
141
+ return "".join(parts)
142
+
143
+ def close(self) -> None:
144
+ """Para el servidor solo si este objeto fue quien lo creo (no recomendado
145
+ si compartes el modelo entre varios `Model`). Usa ggufly.shutdown_all()."""
146
+ from .server import shutdown_all
147
+
148
+ shutdown_all()
149
+
150
+
151
+ def load(source: str, **kwargs: Any) -> Model:
152
+ """Atajo para Model(...)."""
153
+ return Model(source, **kwargs)
154
+
155
+
156
+ def run_many(model: "Model", prompts: List[str], *, workers: int = 4, **gen: Any) -> List[Dict[str, Any]]:
157
+ """Ejecuta varios prompts en paralelo reutilizando el mismo modelo en VRAM."""
158
+ import concurrent.futures as cf
159
+
160
+ with cf.ThreadPoolExecutor(max_workers=workers) as ex:
161
+ futures = [ex.submit(lambda p=p: model.chat(p, **gen)) for p in prompts]
162
+ return [f.result() for f in futures]
163
+
164
+
165
+ async def arun_many(
166
+ model: "Model",
167
+ prompts: List[str],
168
+ *,
169
+ concurrency: Optional[int] = None,
170
+ **gen: Any,
171
+ ) -> List[Dict[str, Any]]:
172
+ """Igual que run_many pero async (semáforo opcional)."""
173
+ sem = asyncio.Semaphore(concurrency) if concurrency else None
174
+
175
+ async def one(p: str):
176
+ if sem:
177
+ async with sem:
178
+ return await model.achat(p, **gen)
179
+ return await model.achat(p, **gen)
180
+
181
+ return await asyncio.gather(*[one(p) for p in prompts])
ggufly/cli.py ADDED
@@ -0,0 +1,153 @@
1
+ """CLI de ggufly.
2
+
3
+ ggufly engines status|install [--backend cuda|vulkan|cpu]
4
+ ggufly serve <ruta-modelo.gguf> [--mmproj X] [--port P] [--parallel N] ...
5
+ ggufly chat <ruta-modelo.gguf> [--image FOTO]
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import json
11
+ import sys
12
+ from typing import Optional
13
+
14
+ from .utils import setup_logging
15
+
16
+
17
+ def _add_model_args(p: argparse.ArgumentParser) -> None:
18
+ p.add_argument("model", help="Ruta local a un archivo .gguf o carpeta que lo contenga")
19
+ p.add_argument("--mmproj", default=None, help="Archivo mmproj para vision")
20
+ p.add_argument("--backend", default=None, choices=["cuda", "vulkan", "cpu"])
21
+ p.add_argument("--ctx", type=int, default=8192, help="Contexto total")
22
+ p.add_argument("--parallel", type=int, default=4, help="Slots simultaneos (comparten VRAM)")
23
+ p.add_argument("--ngl", type=int, default=-1, help="Capas en GPU (-1 = todas)")
24
+ p.add_argument("--port", type=int, default=None)
25
+ p.add_argument("--threads", type=int, default=None)
26
+ p.add_argument("extra", nargs="*", help="Argumentos extra para llama-server (-- --key value)")
27
+
28
+
29
+ def main(argv: Optional[list] = None) -> int:
30
+ ap = argparse.ArgumentParser(prog="ggufly", description="Modelos GGUF sin compilar nada")
31
+ ap.add_argument("-v", "--verbose", action="store_true")
32
+ sub = ap.add_subparsers(dest="cmd", required=True)
33
+
34
+ p_eng = sub.add_parser("engines", help="Instalar/ver el motor llama.cpp")
35
+ eng_sub = p_eng.add_subparsers(dest="eng_cmd", required=True)
36
+ eng_sub.add_parser("status")
37
+ p_inst = eng_sub.add_parser("install")
38
+ p_inst.add_argument("--backend", choices=["cuda", "vulkan", "cpu"], default=None)
39
+
40
+ p_serve = sub.add_parser("serve", help="Servir el modelo (API OpenAI-compatible)")
41
+ _add_model_args(p_serve)
42
+
43
+ p_chat = sub.add_parser("chat", help="Chat interactivo en la terminal")
44
+ _add_model_args(p_chat)
45
+
46
+ args = ap.parse_args(argv)
47
+ setup_logging(args.verbose)
48
+
49
+ if args.cmd == "engines":
50
+ return _cmd_engines(args)
51
+ if args.cmd == "serve":
52
+ return _cmd_serve(args)
53
+ if args.cmd == "chat":
54
+ return _cmd_chat(args)
55
+ return 1
56
+
57
+
58
+ def _cmd_engines(args) -> int:
59
+ from .engine import installed_engine, ensure_engine
60
+
61
+ if args.eng_cmd == "status":
62
+ e = installed_engine()
63
+ if e:
64
+ print(f"Instalado: backend={e.backend} dir={e.directory}")
65
+ print(f" {e.version()}")
66
+ else:
67
+ print("Motor no instalado. Usa: ggufly engines install")
68
+ return 0
69
+
70
+ e = ensure_engine(args.backend, allow_build=True)
71
+ print(f"Instalado backend={e.backend} en {e.directory}")
72
+ print(f" {e.version()}")
73
+ return 0
74
+
75
+
76
+ def _cmd_serve(args) -> int:
77
+ from .server import ServerConfig, get_or_start
78
+
79
+ extra = tuple(args.extra or ())
80
+ cfg = ServerConfig(
81
+ model=args.model,
82
+ mmproj=args.mmproj,
83
+ backend=args.backend,
84
+ n_ctx=args.ctx,
85
+ n_parallel=args.parallel,
86
+ n_gpu_layers=args.ngl,
87
+ threads=args.threads,
88
+ port=args.port,
89
+ extra_args=extra,
90
+ )
91
+ srv = get_or_start(cfg)
92
+ print(f"API lista: {srv.base_url}/v1/chat/completions")
93
+ if args.mmproj:
94
+ print(f"Vision activa (mmproj): {args.mmproj}")
95
+ try:
96
+ while True:
97
+ input()
98
+ except (KeyboardInterrupt, EOFError):
99
+ pass
100
+ finally:
101
+ srv.stop()
102
+ return 0
103
+
104
+
105
+ def _cmd_chat(args) -> int:
106
+ from .api import Model
107
+
108
+ m = Model(
109
+ args.model,
110
+ mmproj=args.mmproj,
111
+ backend=args.backend,
112
+ n_ctx=args.ctx,
113
+ n_parallel=args.parallel,
114
+ n_gpu_layers=args.ngl,
115
+ threads=args.threads,
116
+ port=args.port,
117
+ extra_args=list(args.extra or ()),
118
+ )
119
+ print(f"Modelo: {m.model_path.name} vision={m.has_vision} slots={args.parallel}")
120
+ print("Escribe /image <ruta> para adjuntar una imagen al siguiente mensaje; /exit para salir.")
121
+ history = []
122
+ pending_images: list = []
123
+ while True:
124
+ try:
125
+ user = input("tu> ").strip()
126
+ except (KeyboardInterrupt, EOFError):
127
+ break
128
+ if not user:
129
+ continue
130
+ if user == "/exit":
131
+ break
132
+ if user.startswith("/image "):
133
+ pending_images.append(user[7:].strip())
134
+ print("(imagen agregada)")
135
+ continue
136
+ history.append({"role": "user", "content": user})
137
+ print("asistente> ", end="", flush=True)
138
+ full = ""
139
+ try:
140
+ for chunk in m.chat(history, images=pending_images or None, stream=True):
141
+ print(chunk, end="", flush=True)
142
+ full += chunk
143
+ except Exception as e:
144
+ print(f"\n[error] {e}", file=sys.stderr)
145
+ print()
146
+ pending_images = []
147
+ if full:
148
+ history.append({"role": "assistant", "content": full})
149
+ return 0
150
+
151
+
152
+ if __name__ == "__main__":
153
+ raise SystemExit(main())
ggufly/client.py ADDED
@@ -0,0 +1,251 @@
1
+ """Cliente HTTP (OpenAI-compatible) contra llama-server, con streaming SSE."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import base64
6
+ import mimetypes
7
+ from pathlib import Path
8
+ from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
9
+
10
+ import httpx
11
+
12
+ Message = Dict[str, Any]
13
+
14
+
15
+ def encode_image(obj: Union[str, Path, bytes]) -> str:
16
+ """Convierte una imagen (ruta, URL o bytes) a data-URI base64.
17
+
18
+ - Ruta local -> lee el archivo.
19
+ - http(s):// -> descarga.
20
+ - bytes -> usa tal cual.
21
+ """
22
+ mime = None
23
+ if isinstance(obj, (str, Path)):
24
+ s = str(obj)
25
+ if s.startswith(("http://", "https://")):
26
+ r = httpx.get(s, timeout=60.0, follow_redirects=True)
27
+ r.raise_for_status()
28
+ data = r.content
29
+ mime = r.headers.get("content-type")
30
+ else:
31
+ p = Path(obj)
32
+ data = p.read_bytes()
33
+ mime = mimetypes.guess_type(p.name)[0]
34
+ else:
35
+ data = bytes(obj)
36
+ mime = mime or "image/png"
37
+ if not mime.startswith("image/"):
38
+ mime = "image/png"
39
+ return f"data:{mime};base64,{base64.b64encode(data).decode()}"
40
+
41
+
42
+ class _BaseClient:
43
+ def _prepare_messages(
44
+ self,
45
+ messages: List[Message],
46
+ images: Optional[List[Any]] = None,
47
+ ) -> List[Message]:
48
+ """Inyecta imagenes en el ultimo mensaje user como content parts."""
49
+ msgs = [dict(m) for m in messages]
50
+ if images:
51
+ parts: List[Dict[str, Any]] = []
52
+ for img in images:
53
+ parts.append({"type": "image_url", "image_url": {"url": encode_image(img)}})
54
+ if not msgs or msgs[-1].get("role") != "user":
55
+ msgs.append({"role": "user", "content": []})
56
+ last = msgs[-1]
57
+ if isinstance(last.get("content"), str):
58
+ parts.append({"type": "text", "text": last["content"]})
59
+ elif isinstance(last.get("content"), list):
60
+ parts.extend(last["content"])
61
+ last["content"] = parts
62
+ return msgs
63
+
64
+ @staticmethod
65
+ def _iter_sse_lines(line_iter) -> Iterator[str]:
66
+ for line in line_iter:
67
+ if line.startswith("data:"):
68
+ yield line[5:].strip()
69
+
70
+ @staticmethod
71
+ def _parse_sse_payload(data: str) -> Optional[dict]:
72
+ if data == "[DONE]":
73
+ return None
74
+ try:
75
+ return json.loads(data)
76
+ except json.JSONDecodeError:
77
+ return None
78
+
79
+ @staticmethod
80
+ def _delta_text(obj: dict) -> str:
81
+ choices = obj.get("choices") or []
82
+ if not choices:
83
+ return ""
84
+ d = choices[0].get("delta") or {}
85
+ return d.get("content") or ""
86
+
87
+
88
+ class ChatClient(_BaseClient):
89
+ """Cliente sincrono."""
90
+
91
+ def __init__(self, base_url: str, timeout: float = 600.0):
92
+ self.base_url = base_url.rstrip("/")
93
+ self._client = httpx.Client(timeout=timeout)
94
+
95
+ # ------------------------------------------------------------- chat
96
+ def chat(
97
+ self,
98
+ messages: List[Message],
99
+ *,
100
+ images: Optional[List[Any]] = None,
101
+ stream: bool = False,
102
+ **gen: Any,
103
+ ) -> Union[dict, Iterator[str]]:
104
+ body: Dict[str, Any] = {
105
+ "messages": self._prepare_messages(messages, images),
106
+ "stream": stream,
107
+ **gen,
108
+ }
109
+ if not stream:
110
+ r = self._client.post(f"{self.base_url}/v1/chat/completions", json=body)
111
+ r.raise_for_status()
112
+ return r.json()
113
+
114
+ def gen_stream():
115
+ with self._client.stream(
116
+ "POST", f"{self.base_url}/v1/chat/completions", json=body
117
+ ) as resp:
118
+ resp.raise_for_status()
119
+ for payload in self._iter_sse_lines(resp.iter_lines()):
120
+ obj = self._parse_sse_payload(payload)
121
+ if obj is None:
122
+ continue
123
+ usage = obj.get("usage")
124
+ text = self._delta_text(obj)
125
+ if text or usage:
126
+ yield text
127
+
128
+ return gen_stream()
129
+
130
+ # --------------------------------------------------------- completion
131
+ def completion(self, prompt: str, *, stream: bool = False, **gen: Any) -> Union[dict, Iterator[str]]:
132
+ body = {"prompt": prompt, "stream": stream, **gen}
133
+ if not stream:
134
+ r = self._client.post(f"{self.base_url}/completion", json=body)
135
+ r.raise_for_status()
136
+ return r.json()
137
+
138
+ def gen_stream():
139
+ with self._client.stream("POST", f"{self.base_url}/completion", json=body) as resp:
140
+ resp.raise_for_status()
141
+ for payload in self._iter_sse_lines(resp.iter_lines()):
142
+ obj = self._parse_sse_payload(payload)
143
+ if obj is None:
144
+ continue
145
+ text = obj.get("content") or ""
146
+ if text or obj.get("stop"):
147
+ yield text
148
+
149
+ return gen_stream()
150
+
151
+ # ---------------------------------------------------------- embeddings
152
+ def embeddings(self, texts: Union[str, List[str]]) -> List[List[float]]:
153
+ payload = {"input": texts}
154
+ r = self._client.post(f"{self.base_url}/v1/embeddings", json=payload)
155
+ r.raise_for_status()
156
+ data = r.json()["data"]
157
+ if isinstance(texts, str):
158
+ return [data[0]["embedding"]]
159
+ return [d["embedding"] for d in data]
160
+
161
+ # --------------------------------------------------------------- extras
162
+ def tokenize(self, text: str) -> List[int]:
163
+ r = self._client.post(f"{self.base_url}/tokenize", json={"content": text})
164
+ r.raise_for_status()
165
+ return r.json()["tokens"]
166
+
167
+ def detokenize(self, tokens: List[int]) -> str:
168
+ r = self._client.post(f"{self.base_url}/detokenize", json={"tokens": tokens})
169
+ r.raise_for_status()
170
+ return r.json()["content"]
171
+
172
+ def slots(self) -> dict:
173
+ r = self._client.get(f"{self.base_url}/slots")
174
+ r.raise_for_status()
175
+ return r.json()
176
+
177
+ def close(self) -> None:
178
+ self._client.close()
179
+
180
+
181
+ class AsyncChatClient(_BaseClient):
182
+ """Cliente asincrono."""
183
+
184
+ def __init__(self, base_url: str, timeout: float = 600.0):
185
+ self.base_url = base_url.rstrip("/")
186
+ self._client = httpx.AsyncClient(timeout=timeout)
187
+
188
+ async def chat(
189
+ self,
190
+ messages: List[Message],
191
+ *,
192
+ images: Optional[List[Any]] = None,
193
+ stream: bool = False,
194
+ **gen: Any,
195
+ ) -> Union[dict, AsyncIterator[str]]:
196
+ body = {"messages": self._prepare_messages(messages, images), "stream": stream, **gen}
197
+ if not stream:
198
+ r = await self._client.post(f"{self.base_url}/v1/chat/completions", json=body)
199
+ r.raise_for_status()
200
+ return r.json()
201
+
202
+ async def gen_stream():
203
+ async with self._client.stream(
204
+ "POST", f"{self.base_url}/v1/chat/completions", json=body
205
+ ) as resp:
206
+ resp.raise_for_status()
207
+ async for line in resp.aiter_lines():
208
+ if not line.startswith("data:"):
209
+ continue
210
+ obj = self._parse_sse_payload(line[5:].strip())
211
+ if obj is None:
212
+ continue
213
+ usage = obj.get("usage")
214
+ text = self._delta_text(obj)
215
+ if text or usage:
216
+ yield text
217
+
218
+ return gen_stream()
219
+
220
+ async def completion(self, prompt: str, *, stream: bool = False, **gen: Any) -> Union[dict, AsyncIterator[str]]:
221
+ body = {"prompt": prompt, "stream": stream, **gen}
222
+ if not stream:
223
+ r = await self._client.post(f"{self.base_url}/completion", json=body)
224
+ r.raise_for_status()
225
+ return r.json()
226
+
227
+ async def gen_stream():
228
+ async with self._client.stream("POST", f"{self.base_url}/completion", json=body) as resp:
229
+ resp.raise_for_status()
230
+ async for line in resp.aiter_lines():
231
+ if not line.startswith("data:"):
232
+ continue
233
+ obj = self._parse_sse_payload(line[5:].strip())
234
+ if obj is None:
235
+ continue
236
+ text = obj.get("content") or ""
237
+ if text or obj.get("stop"):
238
+ yield text
239
+
240
+ return gen_stream()
241
+
242
+ async def embeddings(self, texts: Union[str, List[str]]) -> List[List[float]]:
243
+ r = await self._client.post(f"{self.base_url}/v1/embeddings", json={"input": texts})
244
+ r.raise_for_status()
245
+ data = r.json()["data"]
246
+ if isinstance(texts, str):
247
+ return [data[0]["embedding"]]
248
+ return [d["embedding"] for d in data]
249
+
250
+ async def close(self) -> None:
251
+ await self._client.aclose()
ggufly/config.py ADDED
@@ -0,0 +1,43 @@
1
+ """Rutas y configuracion global de ggufly.
2
+
3
+ Variables de entorno soportadas:
4
+ GGUFLY_HOME -> raiz de datos (por defecto: %LOCALAPPDATA%/ggufly o ~/.cache/ggufly)
5
+ GGUFLY_MODELS -> carpeta de modelos descargados
6
+ GGUFLY_ENGINE -> carpeta de binarios de llama.cpp
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ import sys
12
+ from pathlib import Path
13
+
14
+ APP_NAME = "ggufly"
15
+
16
+
17
+ def home_dir() -> Path:
18
+ env = os.environ.get("GGUFLY_HOME")
19
+ if env:
20
+ return Path(env)
21
+ if sys.platform == "win32":
22
+ base = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local")
23
+ return Path(base) / APP_NAME
24
+ xdg = os.environ.get("XDG_CACHE_HOME")
25
+ base = Path(xdg) if xdg else Path.home() / ".cache"
26
+ return base / APP_NAME
27
+
28
+
29
+ def models_dir() -> Path:
30
+ return Path(os.environ.get("GGUFLY_MODELS") or (home_dir() / "models"))
31
+
32
+
33
+ def engine_dir() -> Path:
34
+ return Path(os.environ.get("GGUFLY_ENGINE") or (home_dir() / "engine"))
35
+
36
+
37
+ def logs_dir() -> Path:
38
+ return home_dir() / "logs"
39
+
40
+
41
+ def ensure_dirs() -> None:
42
+ for d in (home_dir(), models_dir(), logs_dir()):
43
+ d.mkdir(parents=True, exist_ok=True)