abstract-hugpy-dev 0.0.1__py3-none-any.whl

abstract_hugpy_dev/__init__.py

@@ -0,0 +1,3 @@
+from .imports import *
+from .managers import *
+from .utils import *

abstract_hugpy_dev/flask_app/__init__.py

@@ -0,0 +1,2 @@
+from .app import *
+from .wsgi_app import *

abstract_hugpy_dev/flask_app/app/__init__.py

@@ -0,0 +1,7 @@
+from .functions import *
+from .routes.llm_storage_routes import llm_bp
+from .routes.chat_routes import chat_bp
+from .routes.search_routes import search_bp
+from .routes.upload_routes import upload_bp
+from .routes.worker_routes import worker_bp
+

abstract_hugpy_dev/flask_app/app/functions/__init__.py

@@ -0,0 +1,3 @@
+from .imports import *
+from .downloads import *
+from .chat import *

abstract_hugpy_dev/flask_app/app/functions/chat/__init__.py

@@ -0,0 +1 @@
+from .streaming import *

abstract_hugpy_dev/flask_app/app/functions/chat/imports.py

@@ -0,0 +1 @@
+from ..imports import *

abstract_hugpy_dev/flask_app/app/functions/chat/streaming.py

@@ -0,0 +1,168 @@
+from .imports import *
+
+from flask import Response, stream_with_context
+from pydantic import BaseModel
+from typing import Optional, List
+
+
+def sse_event(payload: dict) -> bytes:
+    return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n".encode("utf-8")
+
+
+def event_to_sse(ev) -> bytes:
+    """Serialize a dispatch StreamEvent to the browser's SSE wire shape.
+
+    token/done/error get their minimal browser payloads; everything else
+    (status / provisioning progress / continuation markers — including events
+    relayed from a GPU worker) rides through verbatim via model_dump().
+    """
+    t = getattr(ev, "type", None)
+    if t == "token":
+        return sse_event({"type": "token", "text": ev.text})
+    if t == "done":
+        return sse_event({"type": "done", "finish_reason": ev.finish_reason})
+    if t == "error":
+        return sse_event({"type": "error", "message": ev.message})
+    return sse_event(ev.model_dump())
+
+
+def chat_iter_sync(agen):
+    """Drive an async generator from Flask's synchronous WSGI context."""
+    loop = asyncio.new_event_loop()
+
+    try:
+        asyncio.set_event_loop(loop)
+
+        while True:
+            try:
+                item = loop.run_until_complete(agen.__anext__())
+
+                if isinstance(item, str):
+                    item = item.encode("utf-8")
+
+                yield item
+
+            except StopAsyncIteration:
+                break
+
+    finally:
+        try:
+            loop.run_until_complete(loop.shutdown_asyncgens())
+        except Exception:
+            pass
+
+        asyncio.set_event_loop(None)
+        loop.close()
+
+
+def _resolve_max_new_tokens(body: ChatBody) -> int:
+    """Default to the model's full context when the client didn't cap it.
+
+    A tool, not a service — so when max_new_tokens is omitted we give the model
+    as much room as it has. The engine auto-continues past this per-call cap, so
+    this is the per-pass budget, not a hard ceiling on total output.
+    """
+    if body.max_new_tokens:
+        return body.max_new_tokens
+    try:
+        from abstract_hugpy.imports.config.main import get_model_config
+        cfg = get_model_config(body.model_key) if body.model_key else None
+        ctx = getattr(cfg, "model_max_length", None)
+        if ctx and int(ctx) > 0:
+            return int(ctx)
+    except Exception:
+        pass
+    # Fall back to the global default cap.
+    try:
+        from abstract_hugpy.imports.src.constants.constants import DEFAULT_MAX_TOKENS
+        return int(DEFAULT_MAX_TOKENS)
+    except Exception:
+        return 4096
+
+
+async def stream_events(body: ChatBody):
+    """Build prompt_kwargs and stream the unified chat engine to SSE.
+
+    The route is deliberately dumb: it does NOT decide local vs worker. It hands
+    prompt_kwargs to execute_chat_stream, which drives resolve() — and resolve()
+    is the single place that picks in-process / placement-peer / live-GPU-worker
+    and falls back to local. So local and worker chat now stream identically
+    (token-by-token, with auto-continuation past the cap), and there is no
+    separate worker-offload path in this route anymore.
+    """
+    from abstract_hugpy.managers.dispatch import execute_chat_stream
+
+    prompt_kwargs = {}
+    if body.max_new_tokens:
+        # Explicit cap from the client -> honor it (bounded, per-call).
+        prompt_kwargs["max_new_tokens"] = body.max_new_tokens
+    else:
+        # No cap requested -> run unbounded: the runner generates chunk-by-chunk
+        # until the model naturally stops, so the response is never truncated by
+        # a token limit. (Per-chunk size uses the model's context.)
+        prompt_kwargs["unbounded"] = True
+        prompt_kwargs["max_new_tokens"] = _resolve_max_new_tokens(body)
+
+    if body.model_key:
+        prompt_kwargs["model_key"] = body.model_key
+
+    if body.temperature is not None:
+        prompt_kwargs["temperature"] = body.temperature
+
+    if body.do_sample is not None:
+        prompt_kwargs["do_sample"] = body.do_sample
+
+    if body.messages:
+        prompt_kwargs["messages"] = messages_to_dicts(body.messages)
+    else:
+        prompt_kwargs["prompt"] = body.prompt
+
+    if body.file:
+        prompt_kwargs["file"] = body.file
+    if body.images:
+        prompt_kwargs["images"] = body.images
+    if body.request_id:
+        # Stable id the engine threads through every continuation pass; also lets
+        # the browser correlate the stream.
+        prompt_kwargs["request_id"] = body.request_id
+
+    # Text-only chat to a multi-task (e.g. vision) model: route to its
+    # text-generation task instead of the default image-text-to-text, so a
+    # plain prompt uses the text runner. The vision runner requires an image
+    # and would otherwise fail validation. Only do this when no image is given
+    # and the model actually lists text-generation.
+    if not body.images and not body.file and body.model_key:
+        try:
+            from abstract_hugpy.imports.config.main import get_model_config
+            cfg = get_model_config(body.model_key)
+            tasks = getattr(cfg, "tasks", None) or []
+            primary = getattr(cfg, "primary_task", None)
+            if primary != "text-generation" and "text-generation" in tasks:
+                prompt_kwargs["task"] = "text-generation"
+        except Exception:
+            pass
+
+    logger.info("prompt_kwargs == %s", prompt_kwargs)
+
+    try:
+        async for event in execute_chat_stream(**prompt_kwargs):
+            yield event_to_sse(event)
+    except Exception as exc:
+        logger.exception("stream_events failed")
+        yield sse_event({"type": "error", "message": str(exc)})
+
+
+def chat_stream(mimetype=None, headers=None, **kwargs):
+    logger.info(kwargs)
+    body = ChatBody(**kwargs)
+
+    return Response(
+        stream_with_context(chat_iter_sync(stream_events(body))),
+        mimetype=mimetype or "text/event-stream",
+        headers=headers or {
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",
+            "Connection": "keep-alive",
+        },
+        direct_passthrough=True,
+    )

abstract_hugpy_dev/flask_app/app/functions/downloads/__init__.py

@@ -0,0 +1,4 @@
+from .downloads import *
+from .downloader import *
+from .cancelable_downloads import *
+

abstract_hugpy_dev/flask_app/app/functions/downloads/cancelable_downloads.py

@@ -0,0 +1,133 @@
+import multiprocessing as mp
+from datetime import datetime, timezone
+from flask import jsonify, abort
+from .imports import *
+from .downloader import *
+# ──────────────────────────────────────────────────────────────────────────
+# Subprocess worker — module-level so it's spawn-safe; underscore-private so
+# nothing imports it (multiprocessing references the function object directly).
+# ──────────────────────────────────────────────────────────────────────────
+def update_model_status(model: dict) -> dict:
+    model.update(model_status(model))
+    return model
+
+
+def _estimate_total_bytes(model: dict) -> int | None:
+    """Sum the sizes of exactly the files this download will fetch, so the
+    progress bar can show a real percentage. Respects filename (single GGUF),
+    include patterns, or full repo. Returns None on any failure -> the bar
+    falls back to indeterminate, which still works."""
+    hub_id = model.get("hub_id")
+    if not hub_id:
+        return None
+    repo_id, _ = split_hub_id(hub_id)
+    try:
+        info = hfApi.model_info(repo_id, files_metadata=True)
+    except Exception as exc:
+        logger.info("size estimate failed for %s: %s", hub_id, exc)
+        return None
+
+    filename = model.get("filename")
+    include = model.get("include")
+
+    def will_download(path: str) -> bool:
+        if filename:
+            return path == filename or path.endswith("/" + filename)
+        if include:
+            pats = include if isinstance(include, list) else [include]
+            return any(fnmatch.fnmatch(path, p) for p in pats)
+        return True
+
+    total = sum((s.size or 0) for s in (info.siblings or []) if will_download(s.rfilename))
+    return total or None
+
+
+def _download_worker(model_key: str, model: dict) -> None:
+    os.setpgrp()
+    download_one(model=model,model_key=model_key )   # writes hugpy.json via _stamp
+
+
+def _dir_bytes(path: str) -> int:
+    total = 0
+    for root, _, files in os.walk(path):
+        for f in files:
+            try:
+                total += os.path.getsize(os.path.join(root, f))
+            except OSError:
+                pass
+    return total
+# ──────────────────────────────────────────────────────────────────────────
+# Launch: spawn the worker, then sample dir-size for progress in a monitor
+# thread that also resolves the terminal state.
+# ──────────────────────────────────────────────────────────────────────────
+def start_cancellable_download(job: Job, model: dict, total_bytes: int | None = None) -> None:
+    logger.info(model)
+    dest = route_destination(model=model)
+    logger.info(dest)
+    proc = mp.Process(target=_download_worker, args=(job.model_key, model), daemon=True)
+    proc.start()
+
+    job_store.update(
+        job.id, status="running", message="Downloading…",
+        total_bytes=total_bytes, _proc=proc,
+    )
+
+    def monitor() -> None:
+        nonlocal total_bytes
+        # If the caller didn't know the size (registered-model downloads don't),
+        # estimate it here so the bar can show a percentage. One HF metadata
+        # call, on this background thread — never blocks the POST response.
+        if total_bytes is None:
+            total_bytes = _estimate_total_bytes(model)
+            if total_bytes:
+                job_store.update(job.id, total_bytes=total_bytes)
+
+        while proc.is_alive():
+            got = _dir_bytes(dest)
+            pct = (got / total_bytes) if total_bytes else 0.0
+            job_store.update(job.id, progress=min(pct, 0.999), downloaded_bytes=got)
+            time.sleep(1.0)
+
+        proc.join()
+
+        cur = job_store.get(job.id)
+        if cur and cur.status == "cancelled":
+            return
+
+        if proc.exitcode == 0:
+            job_store.update(
+                job.id, status="completed", progress=1.0,
+                downloaded_bytes=_dir_bytes(dest),
+                message=f"Installed at {dest}",
+            )
+            try:
+                record_downloaded_model(model, dest)
+                refresh_registry(run_discovery=False)
+            except Exception as exc:
+                logger.warning("post-download registry refresh failed: %s", exc)
+        else:
+            job_store.update(
+                job.id, status="failed", message="Download failed.",
+                error=f"worker exited with code {proc.exitcode}",
+            )
+
+    threading.Thread(target=monitor, daemon=True).start()
+
+
+
+def cancel_download(job_id: str) -> dict:
+    job = job_store.get(job_id)
+    if not job:
+        abort(404, description="Unknown job ID.")
+    if job.status not in ("queued", "running"):
+        return {"cancelled": False, "reason": f"job is {job.status}"}
+
+    proc = getattr(job, "_proc", None)
+    if proc is not None and proc.is_alive():
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+        except (ProcessLookupError, PermissionError):
+            pass
+
+    job_store.update(job_id, status="cancelled", message="Cancelled by user.")
+    return {"cancelled": True}

abstract_hugpy_dev/flask_app/app/functions/downloads/downloader.py

@@ -0,0 +1,29 @@
+from .imports import *
+def model_status(model: dict) -> dict:
+    destination = route_destination(model)              # was model_destination(...)
+    marker = os.path.join(destination, HUGPY_MARKER)    # was install_marker(...)
+    if os.path.exists(marker):
+        status = "installed"
+    elif os.path.exists(destination) and os.listdir(destination):
+        status = "partial"
+    else:
+        status = "not_installed"
+    return {"status": status, "destination": destination, "installed_marker": marker}
+
+def write_install_marker(destination: str, model_key: str, model: dict[str, Any]) -> None:
+    marker = install_marker(destination)
+    payload = {
+        "model_key": model_key,
+        "hub_id": model.get("hub_id"),
+        "framework": model.get("framework"),
+        "task": model.get("task"),
+        "filename": model.get("filename"),
+        "include": model.get("include"),
+        "installed_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+    with open(marker, "w", encoding="utf-8") as f:
+        f.write(json.dumps(payload, indent=2))
+
+
+

abstract_hugpy_dev/flask_app/app/functions/downloads/downloads.py

@@ -0,0 +1,34 @@
+from .imports import *
+# ---------------------------------------------------------------------------
+# In-process job store
+# ---------------------------------------------------------------------------
+
+jobs: Dict[str, Dict] = {}
+jobs_lock = threading.Lock()
+
+
+def make_job(model_key: str) -> str:
+    job_id = uuid.uuid4().hex[:10]
+    with jobs_lock:
+        jobs[job_id] = {
+            "job_id": job_id,
+            "model_key": model_key,
+            "status": "queued",
+            "message": "",
+        }
+    return job_id
+
+
+def run_download(job_id: str, model_key: str) -> None:
+    with jobs_lock:
+        jobs[job_id]["status"] = "running"
+    try:
+        dest = download_model(model_key, MODELS[model_key])
+        with jobs_lock:
+            jobs[job_id]["status"] = "done"
+            jobs[job_id]["message"] = str(dest)
+    except Exception as exc:
+        with jobs_lock:
+            jobs[job_id]["status"] = "error"
+            jobs[job_id]["message"] = str(exc)
+

abstract_hugpy_dev/flask_app/app/functions/downloads/imports.py

@@ -0,0 +1 @@
+from ..imports import *

abstract_hugpy_dev/flask_app/app/functions/imports/__init__.py

@@ -0,0 +1,4 @@
+from .init_imports import *
+from .options import *
+from .utils import *
+

abstract_hugpy_dev/flask_app/app/functions/imports/init_imports.py

@@ -0,0 +1,6 @@
+from abstract_flask import *
+from .....imports import *
+from .....managers import *
+from .....utils import *
+import signal
+import multiprocessing as mp

abstract_hugpy_dev/flask_app/app/functions/imports/options/__init__.py

@@ -0,0 +1,2 @@
+from .install import *
+from .search import *

abstract_hugpy_dev/flask_app/app/functions/imports/options/imports.py

@@ -0,0 +1,2 @@
+from ..init_imports import *
+from ..utils import *

abstract_hugpy_dev/flask_app/app/functions/imports/options/install.py

@@ -0,0 +1,67 @@
+from .imports import *
+
+def _human(n):
+    if not n: return ""
+    for u in ("B","KB","MB","GB","TB"):
+        if n < 1024: return f"{n:.1f} {u}"
+        n /= 1024
+    return f"{n:.1f} PB"
+
+def _gguf_options(files, free_bytes):
+    groups: dict[str, list] = {}
+    for f in files:
+        if not f.path.lower().endswith(".gguf"):
+            continue
+        m = GGUF_QUANT.search(f.path)
+        groups.setdefault(m.group(0).upper() if m else f.path, []).append(f)
+
+    opts = []
+    for label, group in sorted(groups.items()):
+        total = sum(g.size for g in group if g.size) or None
+        common = dict(
+            id=f"gguf:{label}",
+            framework="llama_cpp",
+            total_bytes=total,
+            fits_disk=(None if total is None or free_bytes is None
+                       else total < free_bytes),
+        )
+        if len(group) == 1:
+            opts.append(InstallOption(
+                label=f"GGUF · {label} · {_human(total)}",
+                filename=group[0].path, **common))
+        else:
+            opts.append(InstallOption(           # sharded → glob, not filename
+                label=f"GGUF · {label} · {len(group)} shards · {_human(total)}",
+                include=[f"*{label}*.gguf"], **common))
+    return opts
+
+def _transformers_option(files, free_bytes):
+    has_st  = any(f.path.endswith(".safetensors") for f in files)
+    has_bin = any(f.path.endswith(".bin") for f in files)
+    if not (has_st or has_bin):
+        return None
+    if has_st and has_bin:                        # skip duplicate .bin weights
+        include = ["*.safetensors", "*.json", "*.model", "tokenizer*", "*.txt"]
+        total = sum(f.size for f in files if f.size and (
+            f.path.endswith((".safetensors", ".json", ".model", ".txt"))
+            or "tokenizer" in f.path)) or None
+        label = f"Transformers · safetensors only · {_human(total)}"
+    else:
+        include, total = None, (sum(f.size for f in files if f.size) or None)
+        label = f"Transformers · full snapshot · {_human(total)}"
+    return InstallOption(
+        id="transformers", framework="transformers", label=label,
+        include=include, total_bytes=total,
+        fits_disk=(None if total is None or free_bytes is None
+                   else total < free_bytes),
+    )
+
+def resolve_options(hub_id, task, files, free_bytes) -> InstallOptions:
+    opts = _gguf_options(files, free_bytes)
+    tf = _transformers_option(files, free_bytes)
+    if tf:
+        opts.append(tf)
+    # prefer a mid GGUF quant, else transformers
+    rec = next((o.id for o in opts if o.id == "gguf:Q4_K_M"),
+               opts[0].id if opts else None)
+    return InstallOptions(hub_id=hub_id, task=task, options=opts, recommended=rec)

abstract_hugpy_dev/flask_app/app/functions/imports/options/search.py

@@ -0,0 +1,35 @@
+from .imports import *       # ← the missing name
+
+_SIZE_CACHE: dict[str, tuple[float, int | None]] = {}
+_SIZE_TTL = 600
+_size_lock = threading.Lock()
+
+def model_size(hub_id: str) -> int | None:
+    now = time.time()
+    with _size_lock:
+        hit = _SIZE_CACHE.get(hub_id)
+        if hit and now - hit[0] < _SIZE_TTL:
+            return hit[1]
+    try:
+        info = hfApi.model_info(hub_id, files_metadata=True)
+        total = sum(s.size for s in info.siblings if s.size) or None
+    except Exception as exc:
+        logger.warning("model_size(%s) failed: %s", hub_id, exc)   # don't hide it
+        total = None
+    with _size_lock:
+        _SIZE_CACHE[hub_id] = (now, total)
+    return total
+
+def free_bytes() -> int | None:
+    """Headroom on the filesystem where this repo will actually be written.
+
+    Uses MODELS_DIR (search package's storage root) — the same root
+    destination_for_model builds paths from — so fits_disk can't disagree
+    with where the file goes. Deliberately NOT list_peers(): that reports a
+    pydantic settings.storage_root that may point at a different mount.
+    """
+    try:
+        probe = MODELS_DIR if os.path.exists(MODELS_DIR) else "/"
+        return shutil.disk_usage(probe).free
+    except OSError:
+        return None

abstract_hugpy_dev/flask_app/app/functions/imports/utils/__init__.py

@@ -0,0 +1,5 @@
+from .schemas import *
+from .manifest import *
+from .peers import *
+from .workers import *
+from .constants import *

abstract_hugpy_dev/flask_app/app/functions/imports/utils/constants.py

@@ -0,0 +1,4 @@
+from .imports import *
+JOBSTATUS = Literal["queued", "running", "completed", "failed", "cancelled"]
+hfApi = HfApi(token=HF_TOKEN)
+GGUF_QUANT = re.compile(r"(Q\d+_[A-Z0-9_]+|F16|BF16|F32)", re.I)

abstract_hugpy_dev/flask_app/app/functions/imports/utils/imports.py

@@ -0,0 +1 @@
+from ..init_imports import *

abstract_hugpy_dev/flask_app/app/functions/imports/utils/manifest.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any
+
+
+def load_manifest(path: str) -> dict[str, dict[str, Any]]:
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Manifest not found: {path}")
+
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if not isinstance(data, dict):
+        raise ValueError("Manifest root must be a JSON object.")
+
+    return data
+
+
+def load_manifest_or_empty(path: str) -> dict[str, dict[str, Any]]:
+    if not os.path.exists(path):
+        return {}
+    return load_manifest(path)
+
+
+def save_manifest(path: str, manifest: dict[str, dict[str, Any]]) -> None:
+    parent = os.path.dirname(path)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(manifest, f, indent=2, sort_keys=True)
+
+
+def key_for_hub_id(hub_id: str) -> str:
+    slug = re.sub(r"[^A-Za-z0-9._-]+", "_", hub_id.strip())
+    return slug.strip("_") or hub_id
+
+
+def upsert_model(
+    path: str,
+    model: dict[str, Any],
+    *,
+    key: str | None = None,
+) -> tuple[str, dict[str, dict[str, Any]]]:
+    """Insert or update a manifest entry. Returns (key, full_manifest)."""
+    manifest = load_manifest_or_empty(path)
+    chosen_key = key or key_for_hub_id(model.get("hub_id") or model.get("name", ""))
+    existing = manifest.get(chosen_key, {})
+    manifest[chosen_key] = {**existing, **model}
+    save_manifest(path, manifest)
+    return chosen_key, manifest