patchvec 0.5.6__py3-none-any.whl

pave/metrics.py

@@ -0,0 +1,52 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import annotations
+import time, threading
+from typing import Dict, Any
+
+_started = time.time()
+_lock = threading.Lock()
+_counters: Dict[str, float] = {
+    "requests_total": 0.0,
+    "collections_created_total": 0.0,
+    "collections_deleted_total": 0.0,
+    "documents_indexed_total": 0.0,
+    "chunks_indexed_total": 0.0,
+    "purge_total": 0.0,
+    "search_total": 0.0,
+    "errors_total": 0.0,
+}
+
+_last_error: str | None = None
+
+def inc(name: str, value: float = 1.0):
+    with _lock:
+        _counters[name] = _counters.get(name, 0.0) + value
+
+def set_error(msg: str):
+    global _last_error
+    with _lock:
+        _last_error = msg
+        _counters["errors_total"] = _counters.get("errors_total", 0.0) + 1.0
+
+def snapshot(extra: Dict[str, Any] | None = None) -> Dict[str, Any]:
+    with _lock:
+        data = dict(_counters)
+        data.update({
+            "uptime_seconds": time.time() - _started,
+            "last_error": _last_error,
+        })
+        if extra:
+            data.update(extra)
+        return data
+
+def to_prometheus(extra: Dict[str, Any] | None = None, build: Dict[str, str] | None = None) -> str:
+    s = []
+    snap = snapshot(extra)
+    for k, v in snap.items():
+        if isinstance(v, (int, float)):
+            s.append(f"patchvec_{k} {float(v)}")
+    if build:
+        labels = ",".join([f'{key}="{val}"' for key, val in build.items()])
+        s.append(f"patchvec_build_info{{{labels}}} 1")
+    return "\n".join(s) + "\n"

pave/preprocess.py

@@ -0,0 +1,151 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+import io, csv, mimetypes
+from .config import CFG
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple
+from pypdf import PdfReader
+
+TXT_CHUNK_SIZE = int(CFG.get("preprocess.txt_chunk_size", 1000))
+TXT_CHUNK_OVERLAP = int(CFG.get("preprocess.txt_chunk_overlap", 200))
+
+def _chunks(text: str, size: int = TXT_CHUNK_SIZE, overlap: int = TXT_CHUNK_OVERLAP):
+    text = text or ""
+    step = max(size - overlap, 1)
+    i = 0
+    while i < len(text):
+        yield text[i : i + size]
+        i += step
+
+def _csv_parse_col_spec(spec: str) -> tuple[list[str], list[int]]:
+    names: list[str] = []
+    idxs: list[int] = []
+    if not spec:
+        return names, idxs
+    for tok in (t.strip() for t in spec.split(",") if t.strip()):
+        if tok.isdigit():
+            i = int(tok)
+            if i <= 0:
+                raise ValueError("CSV column indices are 1-based")
+            idxs.append(i - 1)
+        else:
+            names.append(tok)
+    return names, idxs
+
+def _csv_stringify_row(row: Dict[str, Any], keys: List[str]) -> str:
+    return "\n".join(f"{k}: {'' if row.get(k) is None else row.get(k)}" for k in keys)
+
+def _preprocess_csv(filename: str, content: bytes, csv_options: Dict[str, Any]) -> Iterator[Tuple[str, str, Dict[str, Any]]]:
+    has_header = (csv_options.get("has_header") or "auto").lower()  # auto|yes|no
+    meta_spec = csv_options.get("meta_cols") or ""
+    inc_spec  = csv_options.get("include_cols") or ""
+
+    meta_names, meta_idxs = _csv_parse_col_spec(meta_spec)
+    inc_names , inc_idxs  = _csv_parse_col_spec(inc_spec)
+
+    # decode
+    try:
+        text = content.decode("utf-8")
+    except UnicodeDecodeError:
+        text = content.decode("latin-1")
+
+    sio = io.StringIO(text)
+    sniffer = csv.Sniffer()
+    try:
+        dialect = sniffer.sniff(text[:4096])
+    except Exception:
+        dialect = csv.excel
+
+    reader = csv.reader(sio, dialect)
+    first = next(reader, None)
+    if first is None:
+        return
+
+    header_row: Optional[List[str]] = None
+    if has_header == "yes":
+        header_row = [str(h).strip() for h in first]
+    elif has_header == "no":
+        header_row = None
+    else:
+        try:
+            header_row = [str(h).strip() for h in first] if sniffer.has_header(text[:4096]) else None
+        except Exception:
+            header_row = None
+
+    if header_row is not None:
+        cols = header_row
+        data_rows = reader
+    else:
+        cols = [f"col_{i}" for i in range(len(first))]
+        data_rows = [first, *list(reader)]
+
+    ncols = len(cols)
+    name_to_idx = {c: i for i, c in enumerate(cols)}
+
+    # refuse if names referenced but no header
+    if (meta_names or inc_names) and header_row is None:
+        raise ValueError("CSV has no header but column names were provided. Use 1-based indices or supply a header.")
+
+    def resolve(names: list[str], idxs: list[int]) -> List[str]:
+        out: list[str] = []
+        for nm in names:
+            if nm not in name_to_idx:
+                raise ValueError(f"CSV column '{nm}' not found in header")
+            out.append(nm)
+        for i in idxs:
+            if i < 0 or i >= ncols:
+                raise ValueError(f"CSV column index {i+1} out of range (1..{ncols})")
+            out.append(cols[i])
+        seen = set(); out2=[]
+        for k in out:
+            if k not in seen:
+                seen.add(k); out2.append(k)
+        return out2
+
+    meta_keys = resolve(meta_names, meta_idxs)
+    if inc_names or inc_idxs:
+        include_keys = resolve(inc_names, inc_idxs)
+    else:
+        # DEFAULT: include all columns EXCEPT meta
+        meta_set = set(meta_keys)
+        include_keys = [c for c in cols if c not in meta_set]
+
+    rowno = 0
+    for row in data_rows:
+        rowno += 1
+        if len(row) < ncols:
+            row = row + [""] * (ncols - len(row))
+        elif len(row) > ncols:
+            row = row[:ncols]
+        asdict = {cols[i]: row[i] for i in range(ncols)}
+        text_part = _csv_stringify_row(asdict, include_keys)
+        extra = {k: asdict.get(k, "") for k in meta_keys}
+        extra["row"] = rowno
+        extra["has_header"] = bool(header_row is not None)
+        yield (f"row_{rowno-1}", text_part, extra)
+
+def preprocess(filename: str, content: bytes, csv_options: Dict[str, Any] \
+               | None = None) -> Iterator[Tuple[str, str, Dict[str, Any]]]:
+    """
+    Yields (local_id, text, extra_meta):
+    - PDF: one chunk per page
+    - TXT: charcount-based chunks
+    - CSV: one chunk per row ("; " join)
+    """
+    mt, _ = mimetypes.guess_type(filename)
+    ext = filename.lower().rsplit(".", 1)[-1] if "." in filename else ""
+    if ext == "pdf":
+        reader = PdfReader(io.BytesIO(content))
+        for i, page in enumerate(reader.pages):
+            text = page.extract_text() or ""
+            yield f"page_{i}", text, {"page": i}
+    elif ext == "txt":
+        text = content.decode("utf-8", errors="ignore")
+        for i, chunk in enumerate(_chunks(text)):
+            yield f"chunk_{i}", chunk, {"chunk": i}
+    elif ext == "csv" or mt == "text/csv":
+        yield from _preprocess_csv(filename, content, csv_options or {})
+        return
+    else:
+        raise ValueError(f"unsupported file type: {ext or 'unknown'}")

pave/service.py

@@ -0,0 +1,92 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+import uuid, json, re
+from typing import Dict, Any, Iterable, Tuple, List
+from datetime import datetime, timezone as tz
+from pave.preprocess import preprocess
+from pave.metrics import inc as m_inc
+
+# Pure-ish service functions operating on a store adapter
+
+def create_collection(store, tenant: str, name: str) -> Dict[str, Any]:
+    store.load_or_init(tenant, name)
+    store.save(tenant, name)
+    m_inc("collections_created_total", 1.0)
+    return {
+        "ok": True,
+        "tenant": tenant,
+        "collection": name
+    }
+
+def delete_collection(store, tenant: str, name: str) -> Dict[str, Any]:
+    store.delete_collection(tenant, name)
+    m_inc("collections_deleted_total", 1.0)
+    return {
+        "ok": True,
+        "tenant": tenant,
+        "deleted": name
+    }
+
+def _default_docid(filename: str) -> str:
+    # Uppercase
+    base = filename.upper()
+    # replace space and dot with underscore
+    base = base.replace(" ", "_").replace(".", "_")
+    # replace all non A-Z0-9_ with underscore
+    base = re.sub(r"[^A-Z0-9_]", "_", base)
+    # collapse multiple underscores
+    base = re.sub(r"_+", "_", base).strip("_")
+    if base != '': return base
+    return "PVDOC_"+str(uuid.uuid4())
+
+def ingest_document(store, tenant: str, collection: str, filename: str, content: bytes,
+                    docid: str | None, metadata: Dict[str, Any] | None,
+                    csv_options: Dict[str, Any] | None = None) -> Dict[str, Any]:
+    baseid = docid or _default_docid(filename)
+    if baseid and store.has_doc(tenant, collection, baseid):
+        purged = store.purge_doc(tenant, collection, baseid)
+        m_inc("purge_total", purged)
+    meta_doc = metadata or {}
+    records = []
+    for local_id, text, extra in preprocess(filename, content, csv_options=csv_options):
+        rid = f"{baseid}::{local_id}"
+        now = datetime.now(tz.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
+        meta = {"docid": baseid, "filename": filename, "ingested_at": now}
+        meta.update(meta_doc)
+        meta.update(extra)
+        records.append((rid, text, meta))
+    if not records:
+        return {"ok": False, "error": "no text extracted"}
+    count = store.index_records(tenant, collection, baseid, records)
+    m_inc("documents_indexed_total", 1.0)
+    m_inc("chunks_indexed_total", float(count or 0))
+    return {
+        "ok": True,
+        "tenant": tenant,
+        "collection": collection,
+        "docid": baseid,
+        "chunks": count
+    }
+
+def do_search(store, tenant: str, collection: str, q: str, k: int = 5,
+              filters: Dict[str, Any] | None = None, include_common: bool = False,
+              common_tenant: str | None = None, common_collection: str | None = None
+              ) -> Dict[str, Any]:
+    m_inc("search_total", 1.0)
+    if include_common and common_tenant and common_collection:
+        matches: List[Dict[str, Any]] = []
+        matches.extend(store.search(
+            tenant, collection, q, max(10, k * 2), filters=filters))
+        matches.extend(store.search(
+            common_tenant, common_collection, q, max(10, k * 2), filters=filters))
+        from heapq import nlargest
+        top = nlargest(k, matches, key=lambda x: x["score"])
+        m_inc("matches_total", float(len(top) or 0))
+        return {"matches": top}
+    top = store.search(tenant, collection, q, k, filters=filters)
+    m_inc("matches_total", float(len(top) or 0))
+    return {
+        "matches": top
+    }

pave/stores/__init__.py

@@ -0,0 +1 @@
+# pkg

pave/stores/base.py

@@ -0,0 +1,33 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import Iterable, Dict, Any, List
+
+
+Record = tuple[str, str, Dict[str, Any]]  # (rid, text, meta)
+
+class BaseStore(ABC):
+    @abstractmethod
+    def load_or_init(self, tenant: str, collection: str) -> None: ...
+
+    @abstractmethod
+    def save(self, tenant: str, collection: str) -> None: ...
+
+    @abstractmethod
+    def delete_collection(self, tenant: str, collection: str) -> None: ...
+
+    @abstractmethod
+    def has_doc(self, tenant: str, collection: str, docid: str) -> bool: ...
+
+    @abstractmethod
+    def purge_doc(self, tenant: str, collection: str, docid: str) -> int: ...
+
+    @abstractmethod
+    def index_records(self, tenant: str, collection: str, docid: str,
+                      records: Iterable[Record]) -> int: ...
+
+    @abstractmethod
+    def search(self, tenant: str, collection: str, query: str, k: int = 5,
+               filters: Dict[str, Any] | None = None) -> List[Dict[str, Any]]: ...

pave/stores/factory.py

@@ -0,0 +1,18 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+from .base import BaseStore
+from ..config import CFG
+
+def get_store(cfg: CFG = CFG) -> BaseStore:
+    stype = cfg.get("vector_store.type")
+    match stype:
+        case "default" | "txtai":  # vendor-neutral default
+            from .txtai_store import TxtaiStore
+            return TxtaiStore()
+        case "qdrant":
+            from .qdrant_store import QdrantStore
+            return QdrantStore()
+        case _:
+            raise RuntimeError(f"Unknown vector_store.type: {stype}")

pave/stores/qdrant_store.py

@@ -0,0 +1,26 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigopitanga@posteo.net>
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import annotations
+from typing import Iterable, Dict, Any, List
+from .base import BaseStore, Record
+
+class QdrantStore(BaseStore):
+    """Stub adapter for Qdrant. To be implemented."""
+
+    def load_or_init(self, tenant: str, collection: str) -> None:
+        raise NotImplementedError("to be implemented")
+
+    def save(self, tenant: str, collection: str) -> None:
+        raise NotImplementedError("to be implemented")
+
+    def delete_collection(self, tenant: str, collection: str) -> None:
+        raise NotImplementedError("to be implemented")
+
+    def purge_doc(self, tenant: str, collection: str, docid: str) -> int:
+        raise NotImplementedError("to be implemented")
+
+    def index_records(self, tenant: str, collection: str, docid: str, records: Iterable[Record]) -> int:
+        raise NotImplementedError("to be implemented")
+
+    def search(self, tenant: str, collection: str, text: str, k: int = 5, filters: Dict[str, Any] | None = None) -> List[Dict[str, Any]]:
+        raise NotImplementedError("to be implemented")