glossdex 0.1.0__py3-none-any.whl

glossdex/__init__.py

@@ -0,0 +1,19 @@
+"""glossdex: glossed locally, indexed by meaning, gated by evidence."""
+
+from .describers import Describer, OllamaDescriber, TextOnlyDescriber
+from .embedders import Embedder, OllamaEmbedder, SentenceTransformersEmbedder
+from .engine import Context, Glossdex, Hit, IndexReport, SearchResult
+from .phrases import parse_phrases
+from .profiles import EMBEDDINGGEMMA, EMBEDDINGGEMMA_PASSAGES, Profile
+from .scoring import ScoringIndex, Trace, Unit, select
+from .sources import FolderSource, Item
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Glossdex", "SearchResult", "Hit", "Context", "IndexReport",
+    "Describer", "OllamaDescriber", "TextOnlyDescriber",
+    "Embedder", "OllamaEmbedder", "SentenceTransformersEmbedder",
+    "FolderSource", "Item", "Profile", "EMBEDDINGGEMMA", "EMBEDDINGGEMMA_PASSAGES",
+    "ScoringIndex", "Unit", "Trace", "select", "parse_phrases",
+]

glossdex/cli.py

@@ -0,0 +1,183 @@
+"""Command line: ``glossdex index|search|explain|serve|doctor``."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import urllib.request
+
+from . import __version__
+from .describers import DEFAULT_OLLAMA, OllamaDescriber, TextOnlyDescriber
+from .embedders import OllamaEmbedder
+from .engine import Glossdex
+
+
+def _engine(args) -> Glossdex:
+    if args.describer == "text-only":
+        describer = TextOnlyDescriber()
+    else:
+        describer = OllamaDescriber(model=args.vision_model, text_model=args.text_model,
+                                    host=args.ollama)
+    if args.embedder == "st":
+        from .embedders import SentenceTransformersEmbedder
+        embedder = SentenceTransformersEmbedder()
+    else:
+        embedder = OllamaEmbedder(model=args.embed_model, host=args.ollama)
+    return Glossdex.for_folder(args.folder, describer=describer, embedder=embedder)
+
+
+def _common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("folder", help="the folder to index or search")
+    p.add_argument("--ollama", default=os.environ.get("OLLAMA_HOST", DEFAULT_OLLAMA))
+    p.add_argument("--vision-model", default="gemma4:e4b",
+                   help="Ollama model that describes images (default: gemma4:e4b)")
+    p.add_argument("--text-model", default=None,
+                   help="Ollama model that lists facts of documents (default: the vision model)")
+    p.add_argument("--embed-model", default="embeddinggemma")
+    p.add_argument("--describer", choices=["ollama", "text-only"], default="ollama",
+                   help="text-only: no language model; a document's sentences are its phrases")
+    p.add_argument("--embedder", choices=["ollama", "st"], default="ollama",
+                   help="st: sentence-transformers (pip install 'glossdex[st]')")
+
+
+def cmd_index(args) -> int:
+    gx = _engine(args)
+
+    def on_item(item, err):
+        p = gx.progress
+        status = "FAILED " + err if err else "ok"
+        print(f"[{p['done']}/{p['total']}] {item.title}  {status}", flush=True)
+
+    rep = gx.index(rebuild=args.rebuild, on_item=on_item)
+    print(f"\nIndexed {rep.added}, unchanged {rep.unchanged}, removed {rep.removed}, "
+          f"failed {len(rep.failed)} in {rep.seconds:.1f}s. Index: {gx.store.path}")
+    return 1 if rep.failed and not rep.added else 0
+
+
+def _one_line(text: str, width: int = 100) -> str:
+    s = " ".join(text.split())
+    return s if len(s) <= width else s[: width - 1] + "…"
+
+
+def _print_result(res, show_trace: bool) -> None:
+    if not res.found:
+        print("Nothing found.")
+    for i, h in enumerate(res.hits, 1):
+        extra = "  (near miss)" if h.reason == "pad" else ""
+        words = f"  words {h.coverage:.0%}" if h.coverage > 0 else ""
+        print(f"{i:2d}. match {h.semantic:.3f}{words}  {h.title}{extra}\n"
+              f"      matched: {_one_line(h.matched)}")
+    if show_trace:
+        print("\n" + res.trace.verdict)
+
+
+def cmd_search(args) -> int:
+    gx = _engine(args)
+    res = gx.search(args.query, strictness=args.strictness, compare_top_k=args.top_k)
+    if args.json:
+        print(json.dumps(res.to_dict(), indent=2))
+    else:
+        _print_result(res, not args.quiet)
+    return 0
+
+
+def cmd_explain(args) -> int:
+    gx = _engine(args)
+    res = gx.search(args.query, strictness=args.strictness)
+    shown = {h.id for h in res.hits}
+    print(res.trace.verdict + "\n\nFull ranking before the cutoff:")
+    for i, h in enumerate(gx.explain(args.query, args.n), 1):
+        mark = "SHOWN" if h.id in shown else "     "
+        print(f"{i:2d}. {mark} score {h.score:.4f}  semantic {h.semantic:.4f}  "
+              f"coverage {h.coverage:.3f}  {h.title}\n      matched: {_one_line(h.matched)}")
+    print(f"\nProfile: {res.trace.profile}")
+    return 0
+
+
+def cmd_serve(args) -> int:
+    from .server import serve
+    serve(_engine(args), host=args.host, port=args.port, open_browser=not args.no_browser)
+    return 0
+
+
+def cmd_doctor(args) -> int:
+    ok = True
+    print(f"glossdex {__version__}, Python {sys.version.split()[0]}")
+    try:
+        with urllib.request.urlopen(f"{args.ollama}/api/tags", timeout=5) as r:
+            models = {m["name"] for m in json.loads(r.read())["models"]}
+        print(f"Ollama at {args.ollama}: running")
+        for m in (args.vision_model, args.embed_model):
+            have = any(x == m or x.split(":")[0] == m for x in models) or f"{m}:latest" in models
+            print(f"  {m}: {'installed' if have else 'MISSING, run: ollama pull ' + m}")
+            ok &= have
+    except Exception as e:
+        print(f"Ollama at {args.ollama}: NOT reachable ({e}). Install from https://ollama.com")
+        ok = False
+    try:
+        import PIL  # noqa: F401
+        print("Pillow: installed")
+    except ImportError:
+        print("Pillow: missing (images are sent at full size)")
+    try:
+        import pypdf  # noqa: F401
+        print("pypdf: installed (PDF files will be indexed)")
+    except ImportError:
+        print("pypdf: not installed (PDFs skipped; pip install 'glossdex[pdf]')")
+    print("Ready." if ok else "Not ready: fix the lines above.")
+    return 0 if ok else 1
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="glossdex", description=(
+        "Glossed locally, indexed by meaning, gated by evidence. Private semantic search over "
+        "a folder of images and documents that shows only what the evidence supports."))
+    ap.add_argument("--version", action="version", version=f"glossdex {__version__}")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p = sub.add_parser("index", help="describe and embed new or changed files")
+    _common(p)
+    p.add_argument("--rebuild", action="store_true", help="discard the index and start over")
+    p.set_defaults(fn=cmd_index)
+
+    for name, fn, helptext in (("search", cmd_search, "search the index"),
+                               ("explain", cmd_explain, "show the ranking and every decision")):
+        p = sub.add_parser(name, help=helptext)
+        _common(p)
+        p.add_argument("query")
+        p.add_argument("--strictness", type=float, default=None,
+                       help="0..1; lower shows more (default 0.70)")
+        if name == "search":
+            p.add_argument("--top-k", type=int, default=0,
+                           help="show a fixed top-k instead, for comparison")
+            p.add_argument("--json", action="store_true")
+            p.add_argument("--quiet", action="store_true")
+        else:
+            p.add_argument("-n", type=int, default=15)
+        p.set_defaults(fn=fn)
+
+    p = sub.add_parser("serve", help="open the local web UI")
+    _common(p)
+    p.add_argument("--host", default="127.0.0.1")
+    p.add_argument("--port", type=int, default=8484)
+    p.add_argument("--no-browser", action="store_true")
+    p.set_defaults(fn=cmd_serve)
+
+    p = sub.add_parser("doctor", help="check that Ollama and the models are ready")
+    p.add_argument("--ollama", default=os.environ.get("OLLAMA_HOST", DEFAULT_OLLAMA))
+    p.add_argument("--vision-model", default="gemma4:e4b")
+    p.add_argument("--embed-model", default="embeddinggemma")
+    p.set_defaults(fn=cmd_doctor)
+
+    args = ap.parse_args(argv)
+    try:
+        return args.fn(args)
+    except RuntimeError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

glossdex/describers.py

@@ -0,0 +1,155 @@
+"""Description-generator adapters: content item -> a gloss (a list of independent phrases).
+
+Any local model works. The built-in adapters talk to Ollama over its local HTTP API; nothing
+leaves the machine. Write your own by implementing :class:`Describer`.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import List, Optional, Protocol
+
+from .phrases import parse_phrases
+from .sources import Item
+
+DEFAULT_OLLAMA = "http://localhost:11434"
+
+IMAGE_PROMPT = (
+    "Describe this image for a search index as up to {n} short phrases.\n"
+    "Put each phrase on its own line and end it with a period.\n"
+    "One element per phrase: the main subject, what people or animals are doing, objects, "
+    "setting, colors, mood. Do not join two elements into one phrase.\n"
+    "If the image is a document, receipt, bill, sign or screen: name its type and copy its key "
+    "text (names, organizations, dates, amounts, identifiers).\n"
+    "Only state what is visible. No introduction, no conclusion, no markdown."
+)
+
+DOCUMENT_PROMPT = (
+    "List the key facts of this document as up to {n} concise, independent phrases, one fact "
+    "per line, each ending with a period: document type, persons or parties named, "
+    "organization, dates, amounts, identifiers, subject, findings or decisions, and requested "
+    "actions. Use the document's own words for names, numbers and dates. Do not add "
+    "information that is not in the document. No intro or outro.\n\nDocument:\n{text}"
+)
+
+
+@dataclass
+class Description:
+    text: str  # the raw model output (the whole description)
+    phrases: List[str]
+    model: str
+    seconds: float
+
+
+class Describer(Protocol):
+    name: str
+
+    def describe(self, item: Item) -> Description: ...
+
+
+class DescriberError(RuntimeError):
+    pass
+
+
+def _post(url: str, payload: dict, timeout: float) -> dict:
+    req = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"),
+                                 headers={"Content-Type": "application/json"})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        body = e.read().decode("utf-8", "replace")[:300]
+        raise DescriberError(f"Ollama returned {e.code}: {body}") from e
+    except urllib.error.URLError as e:
+        raise DescriberError(
+            f"Cannot reach Ollama at {url}. Is it running? (`ollama serve`)") from e
+
+
+def _image_b64(path: str, max_side: int) -> str:
+    try:
+        from PIL import Image, ImageOps
+    except ImportError:  # pragma: no cover
+        with open(path, "rb") as f:
+            return base64.b64encode(f.read()).decode("ascii")
+    with Image.open(path) as im:
+        im = ImageOps.exif_transpose(im).convert("RGB")
+        im.thumbnail((max_side, max_side))
+        buf = io.BytesIO()
+        im.save(buf, format="JPEG", quality=85)
+    return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+class OllamaDescriber:
+    """Describes images with a local vision-language model and text with the same (or another) model.
+
+    Defaults to Gemma 4 E4B, which reads both images and text.
+    """
+
+    def __init__(self, model: str = "gemma4:e4b", text_model: Optional[str] = None,
+                 host: str = DEFAULT_OLLAMA, max_phrases: int = 7, max_side: int = 1024,
+                 timeout: float = 600.0, image_prompt: str = IMAGE_PROMPT,
+                 document_prompt: str = DOCUMENT_PROMPT):
+        self.model = model
+        self.text_model = text_model or model
+        self.host = host.rstrip("/")
+        self.max_phrases = max_phrases
+        self.max_side = max_side
+        self.timeout = timeout
+        self.image_prompt = image_prompt
+        self.document_prompt = document_prompt
+        self.name = f"ollama:{self.model}"
+
+    def _generate(self, model: str, prompt: str, images: Optional[List[str]] = None) -> dict:
+        payload = {
+            "model": model,
+            "prompt": prompt,
+            "stream": False,
+            "think": False,
+            "options": {"temperature": 0, "num_predict": 60 * self.max_phrases + 120},
+        }
+        if images:
+            payload["images"] = images
+        try:
+            return _post(f"{self.host}/api/generate", payload, self.timeout)
+        except DescriberError as e:
+            if "think" in str(e):  # models without a thinking switch reject the field
+                payload.pop("think")
+                return _post(f"{self.host}/api/generate", payload, self.timeout)
+            raise
+
+    def describe(self, item: Item) -> Description:
+        start = time.time()
+        if item.kind == "image":
+            res = self._generate(self.model, self.image_prompt.format(n=self.max_phrases),
+                                 [_image_b64(item.path, self.max_side)])
+            model = self.model
+        else:
+            res = self._generate(self.text_model, self.document_prompt.format(
+                n=self.max_phrases, text=item.text))
+            model = self.text_model
+        text = (res.get("response") or "").strip()
+        truncated = res.get("done_reason") == "length"
+        return Description(text, parse_phrases(text, truncated), model, time.time() - start)
+
+
+class TextOnlyDescriber:
+    """No model at all: an item's own sentences are its phrases. Text and documents only.
+
+    Useful to try glossdex on documents in seconds, and for text that is already terse
+    (notes, tickets, logs). Images are skipped.
+    """
+
+    name = "text-only"
+
+    def describe(self, item: Item) -> Description:
+        if item.kind == "image":
+            raise DescriberError("text-only describer cannot describe images")
+        start = time.time()
+        text = item.text.strip()
+        return Description(text, parse_phrases(text), self.name, time.time() - start)

glossdex/embedders.py

@@ -0,0 +1,104 @@
+"""Embedding adapters: text -> vector, in a document mode and a query mode.
+
+``profile_key`` names the model whose score units the vectors are in; it selects the
+thresholds in :mod:`glossdex.profiles`. Two adapters that run the same model (for example
+EmbeddingGemma through Ollama or through sentence-transformers) share a key.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from typing import List, Protocol, Sequence
+
+import numpy as np
+
+from .describers import DEFAULT_OLLAMA
+
+#: EmbeddingGemma's task prompts for retrieval (from its model card).
+GEMMA_QUERY_PREFIX = "task: search result | query: "
+GEMMA_DOCUMENT_PREFIX = "title: none | text: "
+
+
+class Embedder(Protocol):
+    name: str
+    profile_key: str
+
+    def embed_documents(self, texts: Sequence[str]) -> np.ndarray: ...
+
+    def embed_query(self, text: str) -> np.ndarray: ...
+
+
+class EmbedderError(RuntimeError):
+    pass
+
+
+class OllamaEmbedder:
+    """EmbeddingGemma (or any embedding model) served by a local Ollama."""
+
+    def __init__(self, model: str = "embeddinggemma", host: str = DEFAULT_OLLAMA,
+                 profile_key: str = None, query_prefix: str = None, document_prefix: str = None,
+                 timeout: float = 120.0, batch: int = 32):
+        self.model = model
+        self.host = host.rstrip("/")
+        self.timeout = timeout
+        self.batch = batch
+        is_gemma = model.split(":")[0] == "embeddinggemma"
+        self.profile_key = profile_key or ("embeddinggemma-300m" if is_gemma else model)
+        self.query_prefix = query_prefix if query_prefix is not None else (
+            GEMMA_QUERY_PREFIX if is_gemma else "")
+        self.document_prefix = document_prefix if document_prefix is not None else (
+            GEMMA_DOCUMENT_PREFIX if is_gemma else "")
+        self.name = f"ollama:{model}"
+
+    def _embed(self, texts: List[str]) -> np.ndarray:
+        out = []
+        for i in range(0, len(texts), self.batch):
+            payload = {"model": self.model, "input": texts[i:i + self.batch]}
+            req = urllib.request.Request(f"{self.host}/api/embed",
+                                         data=json.dumps(payload).encode("utf-8"),
+                                         headers={"Content-Type": "application/json"})
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                    data = json.loads(resp.read().decode("utf-8"))
+            except urllib.error.HTTPError as e:
+                raise EmbedderError(f"Ollama returned {e.code}: "
+                                    f"{e.read().decode('utf-8', 'replace')[:300]}") from e
+            except urllib.error.URLError as e:
+                raise EmbedderError(f"Cannot reach Ollama at {self.host}. Is it running?") from e
+            out.extend(data["embeddings"])
+        return np.asarray(out, dtype=np.float32)
+
+    def embed_documents(self, texts: Sequence[str]) -> np.ndarray:
+        return self._embed([self.document_prefix + t for t in texts])
+
+    def embed_query(self, text: str) -> np.ndarray:
+        return self._embed([self.query_prefix + text])[0]
+
+
+class SentenceTransformersEmbedder:
+    """EmbeddingGemma through sentence-transformers (``pip install glossdex[st]``).
+
+    ``google/embeddinggemma-300m`` is gated on Hugging Face: accept its license on the model
+    page and run ``hf auth login`` once.
+    """
+
+    def __init__(self, model: str = "google/embeddinggemma-300m", device: str = None,
+                 profile_key: str = None):
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError as e:  # pragma: no cover
+            raise EmbedderError("pip install 'glossdex[st]' to use sentence-transformers") from e
+        self._model = SentenceTransformer(model, device=device)
+        self.profile_key = profile_key or (
+            "embeddinggemma-300m" if "embeddinggemma" in model else model)
+        self.name = f"st:{model}"
+
+    def embed_documents(self, texts: Sequence[str]) -> np.ndarray:
+        encode = getattr(self._model, "encode_document", self._model.encode)
+        return np.asarray(encode(list(texts)), dtype=np.float32)
+
+    def embed_query(self, text: str) -> np.ndarray:
+        encode = getattr(self._model, "encode_query", self._model.encode)
+        return np.asarray(encode([text])[0], dtype=np.float32)