patchvec 0.5.8__py3-none-any.whl → 0.5.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {patchvec-0.5.8.dist-info → patchvec-0.5.9.dist-info}/METADATA +14 -12
- patchvec-0.5.9.dist-info/RECORD +43 -0
- {patchvec-0.5.8.dist-info → patchvec-0.5.9.dist-info}/WHEEL +1 -1
- pave/assets/config.yml.example +224 -0
- pave/assets/tenants.yml.example +19 -0
- pave/auth.py +5 -5
- pave/backends/__init__.py +13 -0
- pave/backends/base.py +29 -0
- pave/backends/faiss.py +153 -0
- pave/backends/qdrant.py +46 -0
- pave/cli.py +167 -41
- pave/config.py +120 -23
- pave/embedders/__init__.py +5 -2
- pave/embedders/base.py +16 -7
- pave/embedders/factory.py +10 -9
- pave/embedders/openai.py +47 -0
- pave/embedders/sbert.py +69 -0
- pave/filters.py +164 -0
- pave/main.py +72 -597
- pave/metadb.py +603 -0
- pave/metrics.py +2 -2
- pave/preprocess.py +2 -1
- pave/routes/__init__.py +16 -0
- pave/routes/admin.py +131 -0
- pave/routes/collections.py +123 -0
- pave/routes/documents.py +164 -0
- pave/routes/health.py +107 -0
- pave/routes/search.py +177 -0
- pave/runtime_paths.py +89 -0
- pave/service.py +78 -232
- pave/stores/__init__.py +4 -2
- pave/stores/base.py +22 -8
- pave/stores/local.py +807 -0
- patchvec-0.5.8.dist-info/RECORD +0 -32
- pave/embedders/openai_emb.py +0 -30
- pave/embedders/sbert_emb.py +0 -24
- pave/embedders/txtai_emb.py +0 -58
- pave/meta_store.py +0 -320
- pave/stores/factory.py +0 -18
- pave/stores/qdrant_store.py +0 -37
- pave/stores/txtai_store.py +0 -950
- {patchvec-0.5.8.dist-info → patchvec-0.5.9.dist-info}/entry_points.txt +0 -0
- {patchvec-0.5.8.dist-info → patchvec-0.5.9.dist-info}/licenses/LICENSE +0 -0
- {patchvec-0.5.8.dist-info → patchvec-0.5.9.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: patchvec
|
|
3
|
-
Version: 0.5.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.5.9
|
|
4
|
+
Summary: PaveDB — A lightweight, pluggable vector search microservice.
|
|
5
5
|
Author: Rodrigo Rodrigues da Silva
|
|
6
6
|
Author-email: rodrigo@flowlexi.com
|
|
7
7
|
License: AGPL-3.0-or-later
|
|
@@ -24,7 +24,6 @@ Description-Content-Type: text/markdown
|
|
|
24
24
|
License-File: LICENSE
|
|
25
25
|
Requires-Dist: fastapi>=0.115.0
|
|
26
26
|
Requires-Dist: uvicorn[standard]>=0.30.6
|
|
27
|
-
Requires-Dist: txtai>=6.3.0
|
|
28
27
|
Requires-Dist: pydantic>=2.8.2
|
|
29
28
|
Requires-Dist: python-multipart>=0.0.9
|
|
30
29
|
Requires-Dist: pypdf>=5.0.0
|
|
@@ -32,14 +31,14 @@ Requires-Dist: pyyaml>=6.0.2
|
|
|
32
31
|
Requires-Dist: python-dotenv>=1.0.1
|
|
33
32
|
Requires-Dist: faiss-cpu>=1.7.1
|
|
34
33
|
Requires-Dist: torch>=2.10.0
|
|
34
|
+
Requires-Dist: sentence-transformers>=2.7.0
|
|
35
35
|
Provides-Extra: cpu
|
|
36
|
-
Provides-Extra: sbert
|
|
37
|
-
Requires-Dist: sentence-transformers>=2.7.0; extra == "sbert"
|
|
38
36
|
Provides-Extra: openai
|
|
39
37
|
Requires-Dist: openai>=1.0.0; extra == "openai"
|
|
40
38
|
Provides-Extra: test
|
|
41
39
|
Requires-Dist: pytest; extra == "test"
|
|
42
40
|
Requires-Dist: httpx; extra == "test"
|
|
41
|
+
Requires-Dist: datasets>=3.5.0; extra == "test"
|
|
43
42
|
Dynamic: author
|
|
44
43
|
Dynamic: author-email
|
|
45
44
|
Dynamic: classifier
|
|
@@ -67,7 +66,7 @@ Upload → chunk → index (with metadata) → search via REST and CLI.
|
|
|
67
66
|
- Metadata filters on search (`{"filters": {"docid": "DOC-1"}}`)
|
|
68
67
|
- REST and CLI entry points
|
|
69
68
|
- Health/metrics endpoints + Prometheus exporter
|
|
70
|
-
- Pluggable embeddings and stores; default
|
|
69
|
+
- Pluggable embeddings and stores; default stack is local FAISS + SBERT
|
|
71
70
|
|
|
72
71
|
## Requirements
|
|
73
72
|
- Python 3.10–3.14
|
|
@@ -102,27 +101,30 @@ export PATCHVEC_AUTH__GLOBAL_KEY="your-secret"
|
|
|
102
101
|
```
|
|
103
102
|
|
|
104
103
|
## Minimal config (optional)
|
|
105
|
-
By default PatchVec runs with sensible local defaults.
|
|
106
|
-
|
|
104
|
+
By default PatchVec runs with sensible local defaults. For a user install,
|
|
105
|
+
customize `~/patchvec/config.yml`:
|
|
107
106
|
```yaml
|
|
108
107
|
vector_store:
|
|
109
|
-
type:
|
|
108
|
+
type: faiss
|
|
110
109
|
embedder:
|
|
111
|
-
type:
|
|
110
|
+
type: sbert
|
|
112
111
|
auth:
|
|
113
112
|
mode: static
|
|
114
113
|
global_key: ${PATCHVEC_GLOBAL_KEY}
|
|
115
114
|
```
|
|
116
115
|
Then export:
|
|
117
116
|
```bash
|
|
118
|
-
export PATCHVEC_CONFIG=./config.yml
|
|
119
117
|
export PATCHVEC_GLOBAL_KEY="your-secret"
|
|
120
118
|
```
|
|
119
|
+
If you keep the file elsewhere, point the runtime at it explicitly:
|
|
120
|
+
```bash
|
|
121
|
+
export PATCHVEC_CONFIG=/path/to/config.yml
|
|
122
|
+
```
|
|
121
123
|
|
|
122
124
|
## CLI example
|
|
123
125
|
```bash
|
|
124
126
|
pavecli create-collection demo books
|
|
125
|
-
pavecli
|
|
127
|
+
pavecli ingest demo books demo/20k_leagues.txt --docid=verne-20k \
|
|
126
128
|
--metadata='{"lang":"en"}'
|
|
127
129
|
pavecli search demo books "captain nemo" -k 5
|
|
128
130
|
```
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
patchvec-0.5.9.dist-info/licenses/LICENSE,sha256=2KbMMavBa2dIx6IfIWEfWh7DP2fSLKI9faHBm5VJa-4,34020
|
|
2
|
+
pave/__init__.py,sha256=g9xY4EojFJqm-dH3UjXIFtzi_T5_3tUFNY3ipocWCA0,227
|
|
3
|
+
pave/auth.py,sha256=j1q-6RGSlOvQDCkM_uJIzllz7H0vVIgS8e5KXtMTDWA,5242
|
|
4
|
+
pave/cli.py,sha256=6HaN7HEiTd4ALroMPEYwF58u74EDo-hyRvSPxdeDETY,11027
|
|
5
|
+
pave/config.py,sha256=4fil_ovxlcZbQSLz-IbdFz_vUtWGdhAqgzOdo-NbW0c,11663
|
|
6
|
+
pave/filters.py,sha256=QDThvoUJXLGnoge4Gie2YCboanT4wmC-CGECbbBPd_8,5163
|
|
7
|
+
pave/log.py,sha256=L8pPCP32Mbxt9oLoKSY4NIfptqYG6R-cW3V0NddozTY,8203
|
|
8
|
+
pave/main.py,sha256=aDsLKeSEvMqHKd_d1sCegJ8_LyVQy1MnpK0qhISx7c8,11926
|
|
9
|
+
pave/metadb.py,sha256=jy2hMxEv0Rw759eoG3S08AnZE0BfXgVg4JJB7ilZlDk,21709
|
|
10
|
+
pave/metrics.py,sha256=COqwrYtOepPudMQWimaV1Vm47rcTTFA2NxvY0Vb3sus,6457
|
|
11
|
+
pave/preprocess.py,sha256=Wu7VEhcUdXX8KggPbPTWiKmcOCEpPmlhuS8SbBK5Ztw,5444
|
|
12
|
+
pave/runtime_paths.py,sha256=v8XydI1uh1P-v_C5dWi-2Rt_7WnMObreXexxgMI1UxE,2349
|
|
13
|
+
pave/schemas.py,sha256=ZrtslEt87u9ck_X24YDGuH73fCGQLu9fkYqefLEyVC4,1152
|
|
14
|
+
pave/service.py,sha256=ZZ2hVvOuEb258fi7dMc2dtLdU5RXBN7uEB6PFpo6HdE,12021
|
|
15
|
+
pave/ui.py,sha256=x5IrGyz78PXpCMSj_Y8vX4mx3b3yAErE60AXBCPgz7E,6343
|
|
16
|
+
pave/assets/__init__.py,sha256=Cxqbj0YBvSMcD8_OvKII2OhB_5ZiJujcyh7OZucbFdo,113
|
|
17
|
+
pave/assets/config.yml.example,sha256=xQjInMykLN8b2yRhc2FrlkRknNsa4I1zNFZSGtd_BeI,8843
|
|
18
|
+
pave/assets/patchvec_icon_192.png,sha256=kUkXir9MtwxWTd_9hPTdycDwdRCimjprJhmaIP6R-XM,9733
|
|
19
|
+
pave/assets/tenants.yml.example,sha256=dshjFtnMiaeBATWZD8bHCuzpbMSS4N6vcUuUD6QjaZo,644
|
|
20
|
+
pave/assets/ui.html,sha256=E8mrkuDEQi2-6vyG6AAkXLxfOb7VIqYe22_uLoPLKGo,5070
|
|
21
|
+
pave/backends/__init__.py,sha256=WkrPPDdSCgXUR-0CeVdjwmflulae1bayYLPWuuqydVc,322
|
|
22
|
+
pave/backends/base.py,sha256=BxGwzv7FPQjonFyzG_Gnb8zUGUUdQ0OJMU9GTqBtrJc,639
|
|
23
|
+
pave/backends/faiss.py,sha256=YvajvqpVcRcZMmIeXnq7WAL_f6W8GaqnH_eDAtXU4u8,4666
|
|
24
|
+
pave/backends/qdrant.py,sha256=EzHN66C7ycLoUxEhm4RhoP6ZTVVqHwhW3oj8QSAVRRI,1218
|
|
25
|
+
pave/embedders/__init__.py,sha256=LaATyOd6M4nr6YGhppj_E1uQ-OxH9ZsgHBIB2uJMaFQ,208
|
|
26
|
+
pave/embedders/base.py,sha256=c5gE7G2Pa7Zu5kS8sKhXNu8ynddhHYn-lMPB5yDWKWo,505
|
|
27
|
+
pave/embedders/factory.py,sha256=xurK3L5Y4Z5x0W20rRZxXmj7lex0EDZ2fmRPysjZYIc,619
|
|
28
|
+
pave/embedders/openai.py,sha256=8CwgRglQEbIuc3-TdbC_yTjqd5oMFHoboBfKW_ATLdU,1498
|
|
29
|
+
pave/embedders/sbert.py,sha256=HsZwFfXtwYSOTtTf1m7SgqKKGvEDc852BL6PuAKBB_M,2039
|
|
30
|
+
pave/routes/__init__.py,sha256=rpFdUG6Sk7fhy51kxewmo0WHle_UUphis4rWo6kdcks,539
|
|
31
|
+
pave/routes/admin.py,sha256=4s_ocNYy929kEFLbsipdzLkAnPhNSPY8wHAldj7qpSs,3926
|
|
32
|
+
pave/routes/collections.py,sha256=W4ZkLegKRlrvGDgXcBB15fZBMJLApAz09wyfQNv_90o,3883
|
|
33
|
+
pave/routes/documents.py,sha256=NxyXlS4ZJ3RXaji9kFtkHlUzNDJE4BWpck8eU7qzfpo,5438
|
|
34
|
+
pave/routes/health.py,sha256=PWmhIp8klS2P9l8e6ShkZOgEk04npWPDPSD5J1P8koY,3524
|
|
35
|
+
pave/routes/search.py,sha256=JX-Esnl93xKnRuN5cP-ajN9LmXnL8JUMysbp6PsC2kM,5295
|
|
36
|
+
pave/stores/__init__.py,sha256=fpaYUUTfVr_h6Fn28iugATtjoeuXH78nQ_s4OLyqTcE,169
|
|
37
|
+
pave/stores/base.py,sha256=7IrmV9nqoI_3kad2ZkFa8LI8i8_f5oZ24bSOPddXqoA,3064
|
|
38
|
+
pave/stores/local.py,sha256=vlDa2gjiSPWelHHj9sVN7gt6JA_g2N6jQZAoDRO4Yxw,28729
|
|
39
|
+
patchvec-0.5.9.dist-info/METADATA,sha256=xXE2HKA125nkhpY3CQFwyCYQSKpyyyVKTzEogd5sZVw,4798
|
|
40
|
+
patchvec-0.5.9.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
41
|
+
patchvec-0.5.9.dist-info/entry_points.txt,sha256=L6cnN4Byk5eDON2d4ipquuLowUjo7-5wpxLb-kThytQ,75
|
|
42
|
+
patchvec-0.5.9.dist-info/top_level.txt,sha256=AbaAN6M4jryKL79DrYPnt49TpE2sjp97qYmm31RD_-U,5
|
|
43
|
+
patchvec-0.5.9.dist-info/RECORD,,
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# PatchVec sample configuration
|
|
2
|
+
# Copy to config.yml and adjust.
|
|
3
|
+
# Secrets (API keys) should live in a separate untracked file — see tenants.yml.example.
|
|
4
|
+
#
|
|
5
|
+
# Default user-install paths:
|
|
6
|
+
# config: ~/pavedb/config.yml
|
|
7
|
+
# tenants: ~/pavedb/tenants.yml
|
|
8
|
+
# data: ~/pavedb/data
|
|
9
|
+
#
|
|
10
|
+
# Distro-like install example:
|
|
11
|
+
# config: /etc/pavedb/config.yml
|
|
12
|
+
# tenants: /var/pavedb/tenants.yml
|
|
13
|
+
# data: /var/pavedb/data
|
|
14
|
+
#
|
|
15
|
+
# Config file location override:
|
|
16
|
+
# PAVEDB_CONFIG=/etc/pavedb/config.yml
|
|
17
|
+
#
|
|
18
|
+
# In Docker/systemd deployments always set PAVEDB_CONFIG explicitly — the
|
|
19
|
+
# default path expands ~ relative to the process user, which may not be what
|
|
20
|
+
# you expect inside a container. Example compose snippet:
|
|
21
|
+
#
|
|
22
|
+
# environment:
|
|
23
|
+
# PAVEDB_CONFIG: /etc/pavedb/config.yml
|
|
24
|
+
# volumes:
|
|
25
|
+
# - ./config.yml:/etc/pavedb/config.yml:ro
|
|
26
|
+
#
|
|
27
|
+
# All keys can also be overridden inline via environment variables:
|
|
28
|
+
# PAVEDB_<KEY>=value (top-level, e.g. PAVEDB_DATA_DIR)
|
|
29
|
+
# PAVEDB_<SECTION>__<KEY>=val (nested, e.g. PAVEDB_LOG__LEVEL=debug)
|
|
30
|
+
# Legacy PATCHVEC_* vars still work in v0.5.9 but will be removed in v0.6.
|
|
31
|
+
#
|
|
32
|
+
# `auth.tenants_file` is optional. If unset, its default is `None` and no
|
|
33
|
+
# tenants sidecar file is loaded.
|
|
34
|
+
# If set, PaveDB loads that sidecar first. Then inline tenant config is
|
|
35
|
+
# applied with precedence:
|
|
36
|
+
# env vars > config.yml > tenants.yml > defaults
|
|
37
|
+
# Example: define tenant "acme" entirely from env:
|
|
38
|
+
# PAVEDB_AUTH__API_KEYS__acme=change-me
|
|
39
|
+
# PAVEDB_TENANTS__acme__MAX_CONCURRENT=5
|
|
40
|
+
|
|
41
|
+
# ---------------------------------------------------------------------------
|
|
42
|
+
# Storage
|
|
43
|
+
# ---------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
# Data directory — ~ is expanded at startup.
|
|
46
|
+
# Default (library/dev): ~/pavedb/data
|
|
47
|
+
# For distro-like installs use an absolute path, e.g. /var/pavedb/data.
|
|
48
|
+
data_dir: ~/pavedb/data
|
|
49
|
+
|
|
50
|
+
# ---------------------------------------------------------------------------
|
|
51
|
+
# Common collection
|
|
52
|
+
# ---------------------------------------------------------------------------
|
|
53
|
+
# When enabled, every tenant search also queries a shared "common" collection
|
|
54
|
+
# and merges the results. Useful for org-wide reference data (e.g. a shared
|
|
55
|
+
# code/term dictionary) that should be visible to all tenants.
|
|
56
|
+
# The common collection is owned by common_tenant and is never rate-limited.
|
|
57
|
+
|
|
58
|
+
common_enabled: false
|
|
59
|
+
common_tenant: global # tenant that owns the common collection
|
|
60
|
+
common_collection: common # collection name within that tenant
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
# Authentication
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
|
|
66
|
+
auth:
|
|
67
|
+
# none — no auth; all requests accepted. Dev/private deployments only.
|
|
68
|
+
# enforce_policy() will reject auth=none on non-loopback interfaces.
|
|
69
|
+
# static — Bearer token per tenant; keys defined below or in tenants_file.
|
|
70
|
+
mode: static
|
|
71
|
+
|
|
72
|
+
# Default tenant name when auth.mode=none (no key required).
|
|
73
|
+
default_access_tenant: public
|
|
74
|
+
|
|
75
|
+
# Global admin key — grants access to all tenants and admin routes.
|
|
76
|
+
# Always read from the environment; never hardcode in committed files.
|
|
77
|
+
global_key: ${PAVEDB_GLOBAL_KEY}
|
|
78
|
+
|
|
79
|
+
# External tenant→key mapping file.
|
|
80
|
+
# Keep the same key paths there (`auth.api_keys`, `tenants.*`).
|
|
81
|
+
# User install default: ~/pavedb/tenants.yml
|
|
82
|
+
# Distro-like install: /var/pavedb/tenants.yml
|
|
83
|
+
# tenants_file: ~/pavedb/tenants.yml
|
|
84
|
+
|
|
85
|
+
# Inline tenant→key mapping (fallback; keep empty in the repo).
|
|
86
|
+
api_keys: {}
|
|
87
|
+
|
|
88
|
+
# ---------------------------------------------------------------------------
|
|
89
|
+
# Vector store
|
|
90
|
+
# ---------------------------------------------------------------------------
|
|
91
|
+
|
|
92
|
+
vector_store:
|
|
93
|
+
# faiss — local FAISS index (built-in, no extra services required).
|
|
94
|
+
type: faiss
|
|
95
|
+
|
|
96
|
+
# Options for type=faiss.
|
|
97
|
+
faiss:
|
|
98
|
+
embed_model: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
|
|
99
|
+
max_query_chars: 4000
|
|
100
|
+
|
|
101
|
+
# ---------------------------------------------------------------------------
|
|
102
|
+
# Embedder
|
|
103
|
+
# ---------------------------------------------------------------------------
|
|
104
|
+
# Controls how text is converted to vectors before indexing and querying.
|
|
105
|
+
# The embedder is shared across all collections (per-collection config: v0.6).
|
|
106
|
+
|
|
107
|
+
embedder:
|
|
108
|
+
# sbert — direct sentence-transformers (recommended local default).
|
|
109
|
+
# openai — OpenAI embeddings API (requires API key, adds latency).
|
|
110
|
+
type: sbert
|
|
111
|
+
|
|
112
|
+
# Used by type=sbert.
|
|
113
|
+
sbert:
|
|
114
|
+
model: sentence-transformers/all-MiniLM-L6-v2
|
|
115
|
+
batch_size: 64
|
|
116
|
+
device: auto # cpu | cuda | mps | auto
|
|
117
|
+
|
|
118
|
+
# Used by type=openai.
|
|
119
|
+
openai:
|
|
120
|
+
model: text-embedding-3-small
|
|
121
|
+
batch_size: 256
|
|
122
|
+
api_key: ${PAVEDB_OPENAI_API_KEY}
|
|
123
|
+
dim: 1536
|
|
124
|
+
|
|
125
|
+
# ---------------------------------------------------------------------------
|
|
126
|
+
# Ingest limits
|
|
127
|
+
# ---------------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
ingest:
|
|
130
|
+
# Reject uploads larger than this (MB). 0 = unlimited.
|
|
131
|
+
# Also set your reverse proxy: e.g. nginx client_max_body_size >= this value.
|
|
132
|
+
max_file_size_mb: 500
|
|
133
|
+
|
|
134
|
+
# Max parallel ingests; excess requests get 503 immediately.
|
|
135
|
+
# Ingest is CPU/memory heavy — keep this well below your core count.
|
|
136
|
+
# 0 = unlimited (not recommended in production).
|
|
137
|
+
max_concurrent: 7
|
|
138
|
+
|
|
139
|
+
# Reverse-proxy timeout guidance for ingest endpoints:
|
|
140
|
+
# Ingest embeds the entire document before responding; large files can take
|
|
141
|
+
# tens of seconds. Raise these nginx directives accordingly:
|
|
142
|
+
#
|
|
143
|
+
# proxy_read_timeout 300; # wait up to 5 min for upstream response
|
|
144
|
+
# client_body_timeout 120; # wait up to 2 min for client to send body
|
|
145
|
+
#
|
|
146
|
+
# Your HTTP client read timeout should also be at least 300 s for ingest.
|
|
147
|
+
|
|
148
|
+
# ---------------------------------------------------------------------------
|
|
149
|
+
# Search limits
|
|
150
|
+
# ---------------------------------------------------------------------------
|
|
151
|
+
|
|
152
|
+
search:
|
|
153
|
+
# Max parallel searches; excess requests get 503 immediately.
|
|
154
|
+
# 0 = unlimited (not recommended in production).
|
|
155
|
+
max_concurrent: 42
|
|
156
|
+
|
|
157
|
+
# Per-search timeout in ms; returns 503 on expiry. 0 = no timeout.
|
|
158
|
+
timeout_ms: 30000
|
|
159
|
+
|
|
160
|
+
# ---------------------------------------------------------------------------
|
|
161
|
+
# Per-tenant concurrency limits
|
|
162
|
+
# ---------------------------------------------------------------------------
|
|
163
|
+
# Applies to all tenant-scoped routes (search, ingest, collection ops).
|
|
164
|
+
# Admin requests and global-key requests bypass these limits.
|
|
165
|
+
# Per-tenant overrides can be set in tenants.yml (see tenants.yml.example).
|
|
166
|
+
# Moving-window limits (max_rpm, max_rph) require SQLite — post-v0.5.8.
|
|
167
|
+
|
|
168
|
+
tenants:
|
|
169
|
+
default_max_concurrent: 10 # applies to every tenant; 0 = unlimited
|
|
170
|
+
|
|
171
|
+
# ---------------------------------------------------------------------------
|
|
172
|
+
# Text preprocessing
|
|
173
|
+
# ---------------------------------------------------------------------------
|
|
174
|
+
# Controls how plain-text (.txt) files are split into chunks before embedding.
|
|
175
|
+
# Overlap ensures context is preserved across chunk boundaries.
|
|
176
|
+
|
|
177
|
+
preprocess:
|
|
178
|
+
txt_chunk_size: 1000 # characters per chunk
|
|
179
|
+
txt_chunk_overlap: 200 # characters of overlap between adjacent chunks
|
|
180
|
+
|
|
181
|
+
# ---------------------------------------------------------------------------
|
|
182
|
+
# Server
|
|
183
|
+
# ---------------------------------------------------------------------------
|
|
184
|
+
# These defaults are overridden by env vars HOST / PORT / RELOAD / WORKERS.
|
|
185
|
+
|
|
186
|
+
server:
|
|
187
|
+
host: 0.0.0.0
|
|
188
|
+
port: 8086
|
|
189
|
+
reload: false
|
|
190
|
+
workers: 1
|
|
191
|
+
|
|
192
|
+
# Keep-alive timeout in seconds (passed to uvicorn --timeout-keep-alive).
|
|
193
|
+
# Must be lower than your reverse proxy's keepalive_timeout.
|
|
194
|
+
# nginx default is 75 s — set this to 65 when behind nginx.
|
|
195
|
+
timeout_keep_alive: 75
|
|
196
|
+
|
|
197
|
+
# ---------------------------------------------------------------------------
|
|
198
|
+
# Logging
|
|
199
|
+
# ---------------------------------------------------------------------------
|
|
200
|
+
|
|
201
|
+
log:
|
|
202
|
+
# Dev log level (stderr). DEBUG | INFO | WARNING | ERROR — default INFO.
|
|
203
|
+
# Overridden by PAVEDB_LOG__LEVEL env var (e.g. in Makefile: debug).
|
|
204
|
+
# Per-namespace overrides: log.debug / log.watch / log.quiet (list of loggers).
|
|
205
|
+
level: INFO
|
|
206
|
+
|
|
207
|
+
# Ops log — one structured JSON line per operation (search, ingest, delete…).
|
|
208
|
+
# Fields: ts, op, tenant, collection, latency_ms, status, error_code, hits.
|
|
209
|
+
# null (off) | stdout | /path/to/ops.jsonl
|
|
210
|
+
# stdout is recommended for Docker/12-factor; pave uses stderr for the dev log.
|
|
211
|
+
# File paths suit traditional deployments; no rotation is provided here.
|
|
212
|
+
ops_log: null
|
|
213
|
+
|
|
214
|
+
# Uvicorn access log destination.
|
|
215
|
+
# null (use uvicorn default) | stdout | /path/to/access.log
|
|
216
|
+
access_log: null
|
|
217
|
+
|
|
218
|
+
# ---------------------------------------------------------------------------
|
|
219
|
+
# Instance branding (optional)
|
|
220
|
+
# ---------------------------------------------------------------------------
|
|
221
|
+
|
|
222
|
+
instance:
|
|
223
|
+
name: PatchVec
|
|
224
|
+
desc: Vector Search Microservice (pluggable, functional)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Untracked secrets file with tenant API keys and per-tenant limits.
|
|
2
|
+
# Name it 'tenants.yml', adjust and DO NOT commit.
|
|
3
|
+
# The installed default should be innocuous; uncomment the examples you need.
|
|
4
|
+
#
|
|
5
|
+
# auth:
|
|
6
|
+
# api_keys:
|
|
7
|
+
# acme: change-me
|
|
8
|
+
# umbrella: change-me
|
|
9
|
+
|
|
10
|
+
# Per-tenant concurrency overrides.
|
|
11
|
+
# Omit a tenant entry to fall back to tenants.default_max_concurrent in config.yml.
|
|
12
|
+
# 0 = unlimited for that tenant.
|
|
13
|
+
# Moving-window limits (max_rpm, max_rph) will be added here after SQLite lands.
|
|
14
|
+
#
|
|
15
|
+
# tenants:
|
|
16
|
+
# acme:
|
|
17
|
+
# max_concurrent: 5 # overrides global default for acme; 0 = unlimited
|
|
18
|
+
# umbrella:
|
|
19
|
+
# max_concurrent: 20
|
pave/auth.py
CHANGED
|
@@ -24,7 +24,7 @@ def _raise_401():
|
|
|
24
24
|
"code": "auth_invalid",
|
|
25
25
|
"error": "missing or invalid authorization header",
|
|
26
26
|
},
|
|
27
|
-
headers={"WWW-Authenticate": 'Bearer realm="
|
|
27
|
+
headers={"WWW-Authenticate": 'Bearer realm="pavedb", error="invalid_token"'},
|
|
28
28
|
)
|
|
29
29
|
|
|
30
30
|
def _raise_403():
|
|
@@ -33,7 +33,7 @@ def _raise_403():
|
|
|
33
33
|
detail={"code": "auth_forbidden", "error": "forbidden"},
|
|
34
34
|
headers={
|
|
35
35
|
"WWW-Authenticate":
|
|
36
|
-
'Bearer realm="
|
|
36
|
+
'Bearer realm="pavedb", error="insufficient_scope"'
|
|
37
37
|
},
|
|
38
38
|
)
|
|
39
39
|
|
|
@@ -127,8 +127,8 @@ async def tenant_rate_limit(
|
|
|
127
127
|
# --- Startup security policy -------------------------------------------------
|
|
128
128
|
|
|
129
129
|
def _is_dev(cfg) -> bool:
|
|
130
|
-
# check dev flag
|
|
131
|
-
return bool(cfg.get("dev", False))
|
|
130
|
+
# check dev flag from config/env overlay
|
|
131
|
+
return bool(cfg.get("dev", False))
|
|
132
132
|
|
|
133
133
|
def enforce_policy(cfg) -> None:
|
|
134
134
|
"""
|
|
@@ -142,7 +142,7 @@ def enforce_policy(cfg) -> None:
|
|
|
142
142
|
if not dev:
|
|
143
143
|
raise RuntimeError(
|
|
144
144
|
"auth.mode=none not allowed in production. "
|
|
145
|
-
"Set auth.mode=static with a key or run with
|
|
145
|
+
"Set auth.mode=static with a key or run with PAVEDB_DEV=1 for dev."
|
|
146
146
|
)
|
|
147
147
|
host = str(cfg.get("server.host", "127.0.0.1")).strip()
|
|
148
148
|
if host not in ("127.0.0.1", "localhost"):
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# (C) 2026 Rodrigo Rodrigues da Silva <rodrigo@flowlexi.com>
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
from .base import SearchHit, VectorBackend
|
|
5
|
+
from .faiss import FaissBackend
|
|
6
|
+
from .qdrant import QdrantVectorBackend
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"FaissBackend",
|
|
10
|
+
"QdrantVectorBackend",
|
|
11
|
+
"SearchHit",
|
|
12
|
+
"VectorBackend",
|
|
13
|
+
]
|
pave/backends/base.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# (C) 2026 Rodrigo Rodrigues da Silva <rodrigo@flowlexi.com>
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
from numpy.typing import NDArray
|
|
8
|
+
from typing import Protocol
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
SearchHit = tuple[str, float]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class VectorBackend(Protocol):
|
|
15
|
+
def initialize(self) -> None: ...
|
|
16
|
+
|
|
17
|
+
def add(self, rids: list[str], vectors: NDArray[np.float32]) -> None: ...
|
|
18
|
+
|
|
19
|
+
def search(
|
|
20
|
+
self,
|
|
21
|
+
vector: NDArray[np.float32],
|
|
22
|
+
k: int,
|
|
23
|
+
) -> list[SearchHit]: ...
|
|
24
|
+
|
|
25
|
+
def delete(self, rids: list[str]) -> None: ...
|
|
26
|
+
|
|
27
|
+
def flush(self) -> None: ...
|
|
28
|
+
|
|
29
|
+
def close(self) -> None: ...
|
pave/backends/faiss.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# (C) 2026 Rodrigo Rodrigues da Silva <rodrigo@flowlexi.com>
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import faiss
|
|
10
|
+
import numpy as np
|
|
11
|
+
from numpy.typing import NDArray
|
|
12
|
+
|
|
13
|
+
from .base import SearchHit
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class FaissBackend:
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
dimension: int,
|
|
20
|
+
*,
|
|
21
|
+
storage_dir: Path,
|
|
22
|
+
) -> None:
|
|
23
|
+
self._dimension = int(dimension)
|
|
24
|
+
self._storage_dir = storage_dir
|
|
25
|
+
self._index = faiss.IndexIDMap2(faiss.IndexFlatIP(self._dimension))
|
|
26
|
+
self._rid_to_id: dict[str, int] = {}
|
|
27
|
+
self._id_to_rid: dict[int, str] = {}
|
|
28
|
+
self._next_id = 0
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def _index_file(self) -> Path:
|
|
32
|
+
return self._storage_dir / "faiss.index"
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def _map_file(self) -> Path:
|
|
36
|
+
return self._storage_dir / "id_map.json"
|
|
37
|
+
|
|
38
|
+
def initialize(self) -> None:
|
|
39
|
+
if not self._index_file.is_file() or not self._map_file.is_file():
|
|
40
|
+
return
|
|
41
|
+
|
|
42
|
+
loaded = faiss.read_index(str(self._index_file))
|
|
43
|
+
loaded_dim = int(getattr(loaded, "d", -1))
|
|
44
|
+
if loaded_dim != self._dimension:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
"FAISS index dimension mismatch: "
|
|
47
|
+
f"expected {self._dimension}, found {loaded_dim}"
|
|
48
|
+
)
|
|
49
|
+
self._index = loaded
|
|
50
|
+
data = json.loads(self._map_file.read_text(encoding="utf-8"))
|
|
51
|
+
rid_to_id_raw = data.get("rid_to_id", {})
|
|
52
|
+
id_to_rid_raw = data.get("id_to_rid", {})
|
|
53
|
+
self._rid_to_id = {
|
|
54
|
+
str(rid): int(iid)
|
|
55
|
+
for rid, iid in rid_to_id_raw.items()
|
|
56
|
+
}
|
|
57
|
+
self._id_to_rid = {
|
|
58
|
+
int(iid): str(rid)
|
|
59
|
+
for iid, rid in id_to_rid_raw.items()
|
|
60
|
+
}
|
|
61
|
+
self._next_id = int(data.get("next_id", 0))
|
|
62
|
+
|
|
63
|
+
def add(self, rids: list[str], vectors: NDArray[np.float32]) -> None:
|
|
64
|
+
if not rids:
|
|
65
|
+
return
|
|
66
|
+
if len(rids) != int(vectors.shape[0]):
|
|
67
|
+
raise ValueError("rids and vectors length mismatch")
|
|
68
|
+
|
|
69
|
+
unique_rids: list[str] = []
|
|
70
|
+
rid_to_vector: dict[str, NDArray[np.float32]] = {}
|
|
71
|
+
for idx, rid in enumerate(rids):
|
|
72
|
+
srid = str(rid)
|
|
73
|
+
if srid not in rid_to_vector:
|
|
74
|
+
unique_rids.append(srid)
|
|
75
|
+
rid_to_vector[srid] = vectors[idx]
|
|
76
|
+
|
|
77
|
+
reindex_rids = [rid for rid in unique_rids if rid in self._rid_to_id]
|
|
78
|
+
if reindex_rids:
|
|
79
|
+
self.delete(reindex_rids)
|
|
80
|
+
|
|
81
|
+
matrix = np.vstack([rid_to_vector[rid] for rid in unique_rids]).astype(
|
|
82
|
+
np.float32,
|
|
83
|
+
copy=False,
|
|
84
|
+
)
|
|
85
|
+
normed = matrix.copy()
|
|
86
|
+
faiss.normalize_L2(normed)
|
|
87
|
+
|
|
88
|
+
int_ids: list[int] = []
|
|
89
|
+
for rid in unique_rids:
|
|
90
|
+
iid = self._next_id
|
|
91
|
+
self._next_id += 1
|
|
92
|
+
self._rid_to_id[rid] = iid
|
|
93
|
+
self._id_to_rid[iid] = rid
|
|
94
|
+
int_ids.append(iid)
|
|
95
|
+
|
|
96
|
+
ids = np.array(int_ids, dtype=np.int64)
|
|
97
|
+
self._index.add_with_ids(normed, ids)
|
|
98
|
+
|
|
99
|
+
def search(
|
|
100
|
+
self,
|
|
101
|
+
vector: NDArray[np.float32],
|
|
102
|
+
k: int,
|
|
103
|
+
) -> list[SearchHit]:
|
|
104
|
+
if self._index.ntotal == 0:
|
|
105
|
+
return []
|
|
106
|
+
|
|
107
|
+
q = np.asarray(vector, dtype=np.float32).reshape(1, -1).copy()
|
|
108
|
+
faiss.normalize_L2(q)
|
|
109
|
+
|
|
110
|
+
limit = min(int(k), int(self._index.ntotal))
|
|
111
|
+
scores, ids = self._index.search(q, limit)
|
|
112
|
+
|
|
113
|
+
out: list[SearchHit] = []
|
|
114
|
+
for iid, score in zip(ids[0], scores[0], strict=False):
|
|
115
|
+
if int(iid) < 0:
|
|
116
|
+
continue
|
|
117
|
+
rid = self._id_to_rid.get(int(iid))
|
|
118
|
+
if rid is None:
|
|
119
|
+
continue
|
|
120
|
+
out.append((rid, float(score)))
|
|
121
|
+
return out
|
|
122
|
+
|
|
123
|
+
def delete(self, rids: list[str]) -> None:
|
|
124
|
+
if not rids:
|
|
125
|
+
return
|
|
126
|
+
int_ids: list[int] = []
|
|
127
|
+
for rid in rids:
|
|
128
|
+
srid = str(rid)
|
|
129
|
+
iid = self._rid_to_id.pop(srid, None)
|
|
130
|
+
if iid is None:
|
|
131
|
+
continue
|
|
132
|
+
self._id_to_rid.pop(iid, None)
|
|
133
|
+
int_ids.append(iid)
|
|
134
|
+
|
|
135
|
+
if int_ids:
|
|
136
|
+
self._index.remove_ids(np.array(int_ids, dtype=np.int64))
|
|
137
|
+
|
|
138
|
+
def flush(self) -> None:
|
|
139
|
+
self._storage_dir.mkdir(parents=True, exist_ok=True)
|
|
140
|
+
faiss.write_index(self._index, str(self._index_file))
|
|
141
|
+
|
|
142
|
+
payload = {
|
|
143
|
+
"rid_to_id": self._rid_to_id,
|
|
144
|
+
"id_to_rid": {str(iid): rid for iid, rid in self._id_to_rid.items()},
|
|
145
|
+
"next_id": self._next_id,
|
|
146
|
+
}
|
|
147
|
+
self._map_file.write_text(
|
|
148
|
+
json.dumps(payload, ensure_ascii=False),
|
|
149
|
+
encoding="utf-8",
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
def close(self) -> None:
|
|
153
|
+
self.flush()
|
pave/backends/qdrant.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# (C) 2026 Rodrigo Rodrigues da Silva <rodrigo@flowlexi.com>
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
from numpy.typing import NDArray
|
|
8
|
+
|
|
9
|
+
from .base import SearchHit
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class QdrantVectorBackend:
|
|
13
|
+
"""Stub for Qdrant vector backend. Not yet implemented."""
|
|
14
|
+
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
*,
|
|
18
|
+
url: str,
|
|
19
|
+
collection: str,
|
|
20
|
+
api_key: str | None = None,
|
|
21
|
+
) -> None:
|
|
22
|
+
self._url = url
|
|
23
|
+
self._collection = collection
|
|
24
|
+
self._api_key = api_key
|
|
25
|
+
|
|
26
|
+
def initialize(self) -> None:
|
|
27
|
+
raise NotImplementedError("QdrantVectorBackend")
|
|
28
|
+
|
|
29
|
+
def add(self, rids: list[str], vectors: NDArray[np.float32]) -> None:
|
|
30
|
+
raise NotImplementedError("QdrantVectorBackend")
|
|
31
|
+
|
|
32
|
+
def search(
|
|
33
|
+
self,
|
|
34
|
+
vector: NDArray[np.float32],
|
|
35
|
+
k: int,
|
|
36
|
+
) -> list[SearchHit]:
|
|
37
|
+
raise NotImplementedError("QdrantVectorBackend")
|
|
38
|
+
|
|
39
|
+
def delete(self, rids: list[str]) -> None:
|
|
40
|
+
raise NotImplementedError("QdrantVectorBackend")
|
|
41
|
+
|
|
42
|
+
def flush(self) -> None:
|
|
43
|
+
raise NotImplementedError("QdrantVectorBackend")
|
|
44
|
+
|
|
45
|
+
def close(self) -> None:
|
|
46
|
+
raise NotImplementedError("QdrantVectorBackend")
|