patchvec 0.5.6__py3-none-any.whl

pave/config.py

@@ -0,0 +1,240 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+import os, re, yaml, threading, logging
+from pathlib import Path
+from typing import Any, Dict
+from dotenv import load_dotenv
+
+load_dotenv()
+
+_ENV_PREFIX = "PATCHVEC_"
+
+_DEFAULT_CONFIG_PATH = os.environ.get(_ENV_PREFIX + "CONFIG", "./config.yml")
+
+_DEFAULTS = {
+    "data_dir": "./data",
+    "auth": {"mode": "none", "api_keys": {}, "tenants_file": None},
+    "vector_store": {"type": "default"},
+}
+
+_ENV_PATTERN = re.compile(r"\$\{([^}:|]+)(?:\|([^}]*))?\}")
+
+# ---------------- utils ----------------
+def _deep_merge(a: Dict[str, Any], b: Dict[str, Any]) -> Dict[str, Any]:
+    out = dict(a)
+    for k, v in (b or {}).items():
+        if isinstance(v, dict) and isinstance(out.get(k), dict):
+            out[k] = _deep_merge(out[k], v)
+        elif v is not None:
+            out[k] = v
+    return out
+
+def _coerce(s: str) -> Any:
+    if isinstance(s, str):
+        low = s.lower()
+        if low in {"true", "false"}:
+            return low == "true"
+        try:
+            if s.isdigit():
+                return int(s)
+            return float(s)
+        except Exception:
+            return s
+    return s
+
+def _subst_env(value: Any) -> Any:
+    if not isinstance(value, str):
+        return value
+    def repl(m: re.Match) -> str:
+        key = m.group(1)
+        default = m.group(2) if m.group(2) is not None else ""
+        return os.environ.get(key, default)
+    return _ENV_PATTERN.sub(repl, value)
+
+def _resolve_env_in_obj(obj: Any) -> Any:
+    if isinstance(obj, dict):
+        return {k: _resolve_env_in_obj(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_resolve_env_in_obj(v) for v in obj]
+    return _subst_env(obj)
+
+def _env_to_dict(prefix: str = _ENV_PREFIX) -> Dict[str, Any]:
+    envmap: Dict[str, Any] = {}
+    for k, v in os.environ.items():
+        if not k.startswith(prefix):
+            continue
+        path = k[len(prefix):].lower().split("__")
+        cur = envmap
+        for part in path[:-1]:
+            cur = cur.setdefault(part, {})
+        cur[path[-1]] = _coerce(v)
+    return envmap
+
+def _load_yaml(path: str | Path) -> Dict[str, Any]:
+    p = Path(path)
+    if p.is_file():
+        with p.open("r", encoding="utf-8") as f:
+            return yaml.safe_load(f) or {}
+    return {}
+
+# --------------- singleton wrapper ----------------
+class Config:
+    """
+    Single backing dict; thread-safe. Can be constructed from a file path
+    or from a pre-built dict. `get(path, default)` persists default.
+    """
+    def __init__(
+            self,
+            data: Dict[str, Any] | None = None,
+            path: str | Path | None = None):
+        self._lock = threading.RLock()
+        if data is None:
+            data = self._load_dict(path or _DEFAULT_CONFIG_PATH)
+        self._cfg: Dict[str, Any] = dict(data)
+        self._data = self._cfg  # back-compat alias for old tests
+
+    # --- main loader, now a static/class member ---
+    @staticmethod
+    def _load_dict(path: str | Path) -> Dict[str, Any]:
+        file_cfg = _resolve_env_in_obj(_load_yaml(path))
+        tenants_file = file_cfg.get("auth", {}).get("tenants_file") \
+            if isinstance(file_cfg.get("auth"), dict) else None
+        if tenants_file:
+            tcfg = _resolve_env_in_obj(_load_yaml(tenants_file))
+            file_cfg = _deep_merge(file_cfg, tcfg)
+        env_cfg = _env_to_dict()
+        return _deep_merge(_deep_merge(_DEFAULTS, file_cfg), env_cfg)
+
+    # -------- path ops --------
+    def _get_from(self, store: Dict[str, Any], path: str):
+        cur: Any = store
+        for part in path.split("."):
+            if not isinstance(cur, dict) or part not in cur:
+                return None
+            cur = cur[part]
+        return cur
+
+    def _set_into(self, store: Dict[str, Any], path: str, value: Any) -> None:
+        cur = store
+        parts = path.split(".")
+        for p in parts[:-1]:
+            cur = cur.setdefault(p, {})
+        cur[parts[-1]] = value
+
+    # -------- public API --------
+    def get(self, path: str, default: Any = None) -> Any:
+        with self._lock:
+            val = self._get_from(self._cfg, path)
+            if val is not None:
+                return val
+            if default is not None:
+                # persist default on first read (previous behavior)
+                self._set_into(self._cfg, path, default)
+                return default
+            return None
+
+    def set(self, path: str, value: Any) -> None:
+        with self._lock:
+            self._set_into(self._cfg, path, value)
+
+    def as_dict(self) -> Dict[str, Any]:
+        with self._lock:
+            # shallow copy is enough for read-only views in tests/health
+            return _deep_merge({}, self._cfg)
+
+    def snapshot(self) -> Dict[str, Any]:
+        return self.as_dict()
+
+    # replace all config in place (keeps object identity for back-compat)
+    def replace(self, data: Dict[str, Any] | None = None,
+                path: str | Path | None = None) -> None:
+        fresh = Config(data=data, path=path)
+        with self._lock:
+            self._cfg.clear()
+            self._cfg.update(fresh._cfg)
+            self._data = self._cfg  # keep the back-compat alias valid
+
+    # attribute sugar (cfg.instance_name, cfg.auth, ...)
+    def __getattr__(self, item):
+        with self._lock:
+            v = self._cfg.get(item)
+            if isinstance(v, dict):
+                # lightweight view (shares the same store via a child Config)
+                child = Config({})
+                # point child to the same backing dict (no copy)
+                child._cfg = v
+                return child
+            return v
+
+# --- singleton access (API + CLI + tests share this) ---
+_CFG_SINGLETON = Config()
+
+def get_cfg() -> Config:
+    return _CFG_SINGLETON
+
+def reload_cfg(path: str | None = None) -> Config:
+    # hard reload from disk/env; keep the same object for back-compat
+    _CFG_SINGLETON.replace(path=path)
+    return _CFG_SINGLETON
+
+CFG = _CFG_SINGLETON
+
+# --- singleton logger ---
+def _init_logger() -> logging.Logger:
+    """
+    Initializes hierarchical logging levels:
+      - pave (base)
+      - watch namespaces (base -1 → more verbose)
+      - quiet namespaces (base +1 → less verbose)
+      - all others (base +2)
+    """
+    cfg = get_cfg()
+    base_level = getattr(logging, cfg.get("loglevel", "WARN").upper(), logging.INFO)
+
+    def shift(level, delta):
+        """Moves numeric loglevel by ±10 per step."""
+        return min(logging.CRITICAL, max(logging.DEBUG, level + 10 * delta))
+
+    root = logging.getLogger()
+    root.setLevel(shift(base_level, +2))  # default = quietest
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter(
+        "%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        "%H:%M:%S"
+    ))
+    root.handlers.clear()
+    root.addHandler(handler)
+
+    # main project
+    if cfg.get("dev",0):
+         logging.getLogger("pave").setLevel(logging.DEBUG)
+    else:
+         logging.getLogger("pave").setLevel(base_level)
+
+    #namespaces fixed to loglevel.DEBUG
+    debug = cfg.get("log.debug", [])
+    for ns in debug:
+        logging.getLogger(ns).setLevel(logging.DEBUG)
+
+    # namespaces that should be more verbose
+    watch = cfg.get("log.watch", ["txtai"])
+    for ns in watch:
+        logging.getLogger(ns).setLevel(shift(base_level, -1))
+
+    # namespaces that should be quieter
+    quiet = cfg.get("log.quiet", ["fastapi", "uvicorn", "sqlalchemy", "urllib"])
+    for ns in quiet:
+        logging.getLogger(ns).setLevel(shift(base_level, +1))
+
+    return logging.getLogger("pave")
+
+_LOGGER_SINGLETON = _init_logger()
+
+def get_logger() -> logging.Logger:
+    """Returns global PatchVec logger"""
+    return _LOGGER_SINGLETON
+
+LOG = _LOGGER_SINGLETON

pave/embedders/__init__.py

@@ -0,0 +1 @@
+# pkg

pave/embedders/base.py

@@ -0,0 +1,12 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import annotations
+from abc import ABC, abstractmethod
+
+class BaseEmbedder(ABC):
+    @abstractmethod
+    def encode(self, texts: list[str]) -> list[list[float]]: ...
+
+    @property
+    @abstractmethod
+    def dim(self) -> int | None: ...

pave/embedders/factory.py

@@ -0,0 +1,21 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+from .base import BaseEmbedder
+from ..config import CFG
+
+def get_embedder(cfg: CFG = CFG) -> BaseEmbedder:
+    etype = (cfg.get("embedder.type", "default") or "default").lower()
+    match etype:
+        case "default" | "txtai":  # vendor-neutral; bw compatible with 'txtai'
+            from .txtai_emb import TxtaiEmbedder
+            return TxtaiEmbedder()
+        case "sbert":
+            from .sbert_emb import SbertEmbedder
+            return SbertEmbedder()
+        case "openai":
+            from .openai_emb import OpenAIEmbedder
+            return OpenAIEmbedder()
+        case _:
+            raise RuntimeError(f"Unknown embedder.type: {etype}")

pave/embedders/openai_emb.py

@@ -0,0 +1,30 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import annotations
+import os
+from openai import OpenAI
+from ..config import CFG
+
+class OpenAIEmbedder:
+    def __init__(self):
+        self.model = CFG.get("embedder.model", "text-embedding-3-small")
+        self.batch_size = int(CFG.get("embedder.batch_size", 256))
+        self._dim = CFG.get("embedder.dim")
+        api_key = CFG.get("embedder.api_key") or os.environ.get("OPENAI_API_KEY")
+        if not api_key:
+            raise RuntimeError("OpenAI API key not configured")
+        self.client = OpenAI(api_key=api_key)
+
+    @property
+    def dim(self) -> int | None:
+        try:
+            return int(self._dim) if self._dim is not None else None
+        except Exception:
+            return None
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        kwargs = {"model": self.model, "input": texts}
+        if self._dim is not None:
+            kwargs["dimensions"] = int(self._dim)
+        res = self.client.embeddings.create(**kwargs)
+        return [d.embedding for d in res.data]

pave/embedders/sbert_emb.py

@@ -0,0 +1,24 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import annotations
+from sentence_transformers import SentenceTransformer
+from ..config import CFG
+
+class SbertEmbedder:
+    def __init__(self):
+        model_name = CFG.get("embedder.model", "sentence-transformers/all-MiniLM-L6-v2")
+        device = CFG.get("embedder.device", "auto")
+        self.batch_size = int(CFG.get("embedder.batch_size", 64))
+        self.model = SentenceTransformer(model_name, device=device)
+        try:
+            self._dim = int(self.model.get_sentence_embedding_dimension())
+        except Exception:
+            self._dim = None
+
+    @property
+    def dim(self) -> int | None:
+        return self._dim
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        vecs = self.model.encode(texts, batch_size=self.batch_size, show_progress_bar=False, convert_to_numpy=True)
+        return vecs.tolist()

pave/embedders/txtai_emb.py

@@ -0,0 +1,58 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+from typing import Any, Dict, List
+from txtai.embeddings import Embeddings
+from ..config import CFG
+
+class TxtaiEmbedder:
+    """
+    Generic txtai-powered embedder.
+    Uses txtai.Embeddings under the hood just for embedding (no indexing here).
+    You can switch models/backends via config.
+
+    Config (examples):
+      embedder:
+        type: txtai
+        txtai:
+          # simplest: sentence-transformers model path (default)
+          path: sentence-transformers/all-MiniLM-L6-v2
+
+          # or explicit method + path
+          # method: transformers        # or 'sentence-transformers'
+          # path: sentence-transformers/paraphrase-MiniLM-L3-v2
+
+          # any other txtai Embeddings config fields can be passed through here.
+    """
+    def __init__(self):
+        # Back-compat: fall back to top-level embed_model if txtai section missing
+        section = CFG.get("embedder.txtai", {}) or {}
+        path = section.get("path") or CFG.get("embedder.model") or CFG.get("embed_model")
+
+        # Build the embeddings config for txtai
+        cfg: Dict[str, Any] = dict(section)
+        if "path" not in cfg and path:
+            cfg["path"] = path
+        # If method omitted, txtai will infer based on path; that’s fine.
+
+        # Ensure we’re only using txtai to embed (no index path persistence here)
+        # txtai will lazy-load the model as needed.
+        self._emb = Embeddings(cfg)
+
+        # Try to capture dim if available (not guaranteed)
+        try:
+            # Some txtai models expose : self._emb.model.get_sentence_embedding_dimension()
+            # We don’t rely on it; just best-effort.
+            dim = getattr(getattr(self._emb, "model", None), "get_sentence_embedding_dimension", None)
+            self._dim = int(dim()) if callable(dim) else None
+        except Exception:
+            self._dim = None
+
+    @property
+    def dim(self) -> int | None:
+        return self._dim
+
+    def encode(self, texts: List[str]) -> List[List[float]]:
+        # txtai expects list[str], returns list[list[float]]
+        return self._emb.embed(texts)

pave/main.py

@@ -0,0 +1,303 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import json, os, logging
+import uvicorn
+
+from fastapi import FastAPI, Header, Body, File, UploadFile, Form, Path, \
+    Query, Depends, Request, HTTPException
+from fastapi.responses import JSONResponse, PlainTextResponse
+from pydantic import BaseModel
+from typing import Optional, Dict, Any, Annotated
+
+from pave.config import get_cfg, get_logger
+from pave.auth import AuthContext, auth_ctx, authorize_tenant, \
+    enforce_policy, resolve_bind
+from pave.metrics import inc, set_error, snapshot, to_prometheus
+from pave.stores.factory import get_store
+from pave.stores.base import BaseStore
+from pave.service import \
+    create_collection as svc_create_collection, \
+    delete_collection as svc_delete_collection, \
+    ingest_document as svc_ingest_document, \
+    do_search as svc_do_search
+
+
+VERSION = "0.5.6"
+
+class SearchBody(BaseModel):
+    q: str
+    k: int = 5
+    filters: Optional[Dict[str, Any]] = None
+
+# Dependency injection builder
+def build_app(cfg=get_cfg()) -> FastAPI:
+    app = FastAPI(
+        title=cfg.get("instance.name","Patchvec"),
+        description=cfg.get("instance.desc","Vector Search Microservice")
+    )
+    app.state.store = get_store(cfg)
+    app.state.cfg = cfg
+    app.state.version = VERSION
+
+    def current_store(request: Request) -> BaseStore:
+        return request.app.state.store
+
+
+    # -------------------- Health --------------------
+
+    def _readiness_check() -> Dict[str, Any]:
+        details: Dict[str, Any] = {
+            "data_dir": cfg.get("data_dir"),
+            "vector_store": cfg.get("vector_store.type"),
+            "writable": False,
+            "vector_backend_init": False,
+        }
+        try:
+            os.makedirs(cfg.data_dir, exist_ok=True)
+            testfile = os.path.join(cfg.data_dir, ".writetest")
+            with open(testfile, "w", encoding="utf-8") as f:
+                f.write("ok")
+            os.remove(testfile)
+            details["writable"] = True
+        except Exception as e:
+            details["writable"] = False
+            set_error(f"fs: {e}")
+        try:
+            request_store = app.state.store
+            request_store.load_or_init("_system", "health")
+            details["vector_backend_init"] = True
+        except Exception as e:
+            details["vector_backend_init"] = False
+            set_error(f"vec: {e}")
+        details["ok"] = bool(
+            details["writable"] and details["vector_backend_init"])
+        details["version"] = VERSION
+        return details
+
+    @app.get("/health")
+    def health():
+        inc("requests_total")
+        d = _readiness_check()
+        status = "ready" if d.get("ok") else "degraded"
+        return {"ok": d["ok"], "status": status, "version": VERSION}
+
+    @app.get("/health/live")
+    def health_live():
+        inc("requests_total")
+        return {"ok": True, "status": "live", "version": VERSION}
+
+    @app.get("/health/ready")
+    def health_ready():
+        inc("requests_total")
+        d = _readiness_check()
+        code = 200 if d.get("ok") else 503
+        return JSONResponse(d, status_code=code)
+
+    @app.get("/health/metrics")
+    def health_metrics():
+        inc("requests_total")
+        extra = {
+            "version": VERSION,
+            "vector_store": cfg.get("vector_store.type"),
+            "auth": cfg.get("auth.mode")
+        }
+        return snapshot(extra)
+
+    @app.get("/metrics")
+    def metrics_prom():
+        inc("requests_total")
+        txt = to_prometheus(build={
+            "version": VERSION,
+            "vector_store": cfg.get("vector_store.type"),
+            "auth":cfg.get("auth.mode")
+        })
+        return PlainTextResponse(txt, media_type="text/plain; version=0.0.4")
+
+    
+    # ----------------- Core API ------------------
+
+    @app.post("/collections/{tenant}/{name}")
+    def create_collection(
+        tenant: str,
+        name: str,
+        ctx: AuthContext = Depends(authorize_tenant),
+        store: BaseStore = Depends(current_store),
+    ):
+        inc("requests_total")
+        return svc_create_collection(store, tenant, name)
+
+    @app.delete("/collections/{tenant}/{name}")
+    def delete_collection_route(
+        tenant: str,
+        name: str,
+        ctx: AuthContext = Depends(authorize_tenant),
+        store: BaseStore = Depends(current_store),
+    ):
+        inc("requests_total")
+        return svc_delete_collection(store, tenant, name)
+
+    @app.post("/collections/{tenant}/{collection}/documents")
+    async def upload_document(
+        tenant: str,
+        collection: str,
+        file: UploadFile = File(...),
+        docid: Optional[str] = Form(None),
+        metadata: Optional[str] = Form(None),
+        # CSV controls as optional query params
+        # (kept out of form to not clash with file upload)
+        csv_has_header: Optional[str] = Query(None, pattern="^(auto|yes|no)$"),
+        csv_meta_cols: Optional[str] = Query(None),
+        csv_include_cols: Optional[str] = Query(None),
+        ctx: AuthContext = Depends(authorize_tenant),
+        store: BaseStore = Depends(current_store),
+    ):
+        meta_obj = None
+        if metadata:
+            try:
+                import json
+                meta_obj = json.loads(metadata)
+            except Exception as e:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"invalid metadata json: {e}"
+                )
+
+        content = await file.read()
+
+        csv_opts = None
+        if csv_has_header or csv_meta_cols or csv_include_cols:
+            csv_opts = {
+                "has_header": csv_has_header or "auto",
+                "meta_cols": csv_meta_cols or "",
+                "include_cols": csv_include_cols or "",
+            }
+
+        try:
+            out = svc_ingest_document(
+                store, tenant, collection, file.filename, content,
+                docid, meta_obj, csv_options=csv_opts
+            )
+            return out
+        except ValueError as ve:
+            # e.g., names provided but no header
+            raise HTTPException(status_code=400, detail=str(ve))
+
+    # POST search (supports filters)
+    @app.post("/collections/{tenant}/{name}/search")
+    def search_route_post(
+        tenant: str,
+        name: str,
+        body: SearchBody,
+        ctx: AuthContext = Depends(authorize_tenant),
+        store: BaseStore = Depends(current_store),
+    ):
+        inc("requests_total")
+        include_common = bool(cfg.common_enabled)
+        result = svc_do_search(
+            store, tenant, name, body.q, body.k, filters=body.filters,
+            include_common=include_common, common_tenant=cfg.common_tenant,
+            common_collection=cfg.common_collection
+        )
+        return JSONResponse(result)
+
+    # GET search (no filters)
+    @app.get("/collections/{tenant}/{name}/search")
+    def search_route_get(
+        tenant: str,
+        name: str,
+        q: str = Query(...),
+        k: int = Query(5, ge=1),
+        ctx: AuthContext = Depends(authorize_tenant),
+        store: BaseStore = Depends(current_store),
+    ):
+        inc("requests_total")
+        include_common = bool(cfg.common_enabled)
+        result = svc_do_search(
+            store, tenant, name, q, k, filters=None,
+            include_common=include_common, common_tenant=cfg.common_tenant,
+            common_collection=cfg.common_collection
+        )
+        return JSONResponse(result)
+
+    # Common collection search
+    @app.post("/search")
+    def search_common_post(
+        body: SearchBody,
+        ctx: AuthContext = Depends(auth_ctx),
+        store: BaseStore = Depends(current_store),
+    ):
+        inc("requests_total")
+        if not cfg.common_enabled:
+            return JSONResponse({"matches": []})
+        result = svc_do_search(
+            store, cfg.common_tenant, cfg.common_collection, body.q, body.k,
+            filters=body.filters
+        )
+        return JSONResponse(result)
+
+    @app.get("/search")
+    def search_common_get(
+        q: str = Query(...),
+        k: int = Query(5, ge=1),
+        ctx: AuthContext = Depends(auth_ctx), 
+        store: BaseStore = Depends(current_store),
+    ):
+        inc("requests_total")
+        if not cfg.common_enabled:
+            return JSONResponse({"matches": []})
+        result = svc_do_search(
+            store, cfg.common_tenant, cfg.common_collection, q, k, filters=None
+        )
+        return JSONResponse(result)
+
+    return app
+
+def main_srv():
+    """
+    HTTP server entrypoint.
+    Precedence: CFG (reads env first) > defaults.
+    """
+    cfg = get_cfg()
+    log = get_logger()
+    log.info("Welcome to PatchVEC 🍰")
+    # Policy:
+    # - fail fast without auth in prod;
+    # - auth=none only in dev with loopback;
+    # - raises on invalid config.
+    enforce_policy(cfg)
+
+    # resolve bind host/port
+    host, port = resolve_bind(cfg)
+    cfg.set("server.host", host)
+    cfg.set("server.port", port)
+
+    # flags from CFG
+    reload = bool(cfg.get("server.reload", False))
+    workers = int(cfg.get("server.workers", 1))
+    log_level = str(cfg.get("server.log_level", "info"))
+
+    if cfg.get("dev",0):
+        log_level = "debug"
+        cfg.set("server.log_level", log_level)
+        log.setLevel(logging.DEBUG)
+
+    # run server
+    uvicorn.run("pave.main:app",
+                host=host,
+                port=port,
+                reload=reload,
+                workers=workers,
+                log_level=log_level)
+
+# Default app instance for `uvicorn pave.main:app`
+app = build_app()
+
+# UI attach (minimal)
+from pave.ui import attach_ui
+attach_ui(app)
+
+if __name__ == "__main__":
+    main_srv()