pavedb 0.9.0__py3-none-any.whl

pave/__init__.py

@@ -0,0 +1,6 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigo@flowlexi.com>
+# SPDX-License-Identifier: AGPL-3.0-or-later
+__all__ = [
+    "config", "auth", "preprocess", "metrics", "service", "main", "cli",
+    "embedders", "stores", "log",
+]

pave/assets/__init__.py

@@ -0,0 +1,4 @@
+# (C) 2025 Rodrigo Rodrigues da Silva <rodrigo@flowlexi.com>
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# pkg

pave/assets/config.yml.example

@@ -0,0 +1,229 @@
+# PaveDB sample configuration
+# Copy to config.yml and adjust.
+# Secrets (API keys) should live in a separate untracked file — see tenants.yml.example.
+#
+# Default user-install paths:
+#   config:  ~/pavedb/config.yml
+#   tenants: ~/pavedb/tenants.yml
+#   data:    ~/pavedb/data
+#
+# Distro-like install example:
+#   config:  /etc/pavedb/config.yml
+#   tenants: /var/pavedb/tenants.yml
+#   data:    /var/pavedb/data
+#
+# Config file location override:
+#   PAVEDB_CONFIG=/etc/pavedb/config.yml
+#
+# In Docker/systemd deployments always set PAVEDB_CONFIG explicitly — the
+# default path expands ~ relative to the process user, which may not be what
+# you expect inside a container. Example compose snippet:
+#
+#   environment:
+#     PAVEDB_CONFIG: /etc/pavedb/config.yml
+#   volumes:
+#     - ./config.yml:/etc/pavedb/config.yml:ro
+#
+# All keys can also be overridden inline via environment variables:
+#   PAVEDB_<KEY>=value          (top-level, e.g. PAVEDB_DATA_DIR)
+#   PAVEDB_<SECTION>__<KEY>=val (nested,    e.g. PAVEDB_LOG__LEVEL=debug)
+# `auth.tenants_file` is optional. If unset, its default is `None` and no
+# tenants sidecar file is loaded.
+# If set, PaveDB loads that sidecar first. Then inline tenant config is
+# applied with precedence:
+#   env vars > config.yml > tenants.yml > defaults
+# Example: define tenant "acme" entirely from env:
+#   PAVEDB_AUTH__API_KEYS__acme=change-me
+#   PAVEDB_TENANTS__acme__MAX_CONCURRENT=5
+
+# ---------------------------------------------------------------------------
+# Storage
+# ---------------------------------------------------------------------------
+
+# Data directory — ~ is expanded at startup.
+# Default (library/dev): ~/pavedb/data
+# For distro-like installs use an absolute path, e.g. /var/pavedb/data.
+data_dir: ~/pavedb/data
+
+# ---------------------------------------------------------------------------
+# Common collection
+# ---------------------------------------------------------------------------
+# When enabled, every tenant search also queries a shared "common" collection
+# and merges the results. Useful for org-wide reference data (e.g. a shared
+# code/term dictionary) that should be visible to all tenants.
+# The common collection is owned by common_tenant and is never rate-limited.
+
+common_enabled: false
+common_tenant: global        # tenant that owns the common collection
+common_collection: common    # collection name within that tenant
+
+# ---------------------------------------------------------------------------
+# Authentication
+# ---------------------------------------------------------------------------
+
+auth:
+  # none   — no auth; all requests accepted. Dev/private deployments only.
+  #          enforce_policy() will reject auth=none on non-loopback interfaces.
+  # static — Bearer token per tenant; keys defined below or in tenants_file.
+  mode: static
+
+  # Default tenant name when auth.mode=none (no key required).
+  default_access_tenant: public
+
+  # Global admin key — grants access to all tenants and admin routes.
+  # Always read from the environment; never hardcode in committed files.
+  global_key: ${PAVEDB_GLOBAL_KEY}
+
+  # External tenant→key mapping file.
+  # Keep the same key paths there (`auth.api_keys`, `tenants.*`).
+  # User install default:  ~/pavedb/tenants.yml
+  # Distro-like install:   /var/pavedb/tenants.yml
+  # tenants_file: ~/pavedb/tenants.yml
+
+  # Inline tenant→key mapping (fallback; keep empty in the repo).
+  api_keys: {}
+
+# ---------------------------------------------------------------------------
+# Vector store
+# ---------------------------------------------------------------------------
+
+vector_store:
+  # faiss   — local FAISS index (built-in, no extra services required).
+  type: faiss
+
+  # Options for type=faiss.
+  faiss:
+    embed_model: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+    max_query_chars: 4000
+
+# ---------------------------------------------------------------------------
+# Embedder
+# ---------------------------------------------------------------------------
+# Controls how text is converted to vectors before indexing and querying.
+# The embedder is shared across all collections (per-collection config: v0.6).
+
+embedder:
+  # sbert  — direct sentence-transformers (recommended local default).
+  # openai — OpenAI embeddings API (requires API key, adds latency).
+  type: sbert
+
+  # Used by type=sbert.
+  sbert:
+    runtime: auto   # auto | process | direct
+    model: sentence-transformers/all-MiniLM-L6-v2
+    batch_size: 64
+    device: auto   # cpu | cuda | mps | auto
+
+  # Used by type=openai.
+  openai:
+    model: text-embedding-3-small
+    batch_size: 256
+    api_key: ${PAVEDB_OPENAI_API_KEY}
+    dim: 1536
+
+# ---------------------------------------------------------------------------
+# Ingest limits
+# ---------------------------------------------------------------------------
+
+ingest:
+  # Reject uploads larger than this (MB). 0 = unlimited.
+  # Also set your reverse proxy: e.g. nginx client_max_body_size >= this value.
+  max_file_size_mb: 500
+
+  # Max parallel ingests; excess requests get 503 immediately.
+  # Ingest is CPU/memory heavy — keep this well below your core count.
+  # In-code default is 7 (conservative floor); 10 below is a typical-dev-host
+  # value. Rule of thumb: ~1× core count on dedicated hosts (e.g. 10 on a
+  # 10-core M-series, 16 on a 16-vCPU VM).
+  # 0 = unlimited (not recommended in production).
+  max_concurrent: 10
+
+# Reverse-proxy timeout guidance for ingest endpoints:
+# Ingest embeds the entire document before responding; large files can take
+# tens of seconds. Raise these nginx directives accordingly:
+#
+#   proxy_read_timeout   300;  # wait up to 5 min for upstream response
+#   client_body_timeout  120;  # wait up to 2 min for client to send body
+#
+# Your HTTP client read timeout should also be at least 300 s for ingest.
+
+# ---------------------------------------------------------------------------
+# Search limits
+# ---------------------------------------------------------------------------
+
+search:
+  # Max parallel searches; excess requests get 503 immediately.
+  # In-code default is 42 (conservative floor); 48 below is a typical-dev-host
+  # value. Rule of thumb: ~4–5× core count on dedicated hosts; throughput
+  # plateaus past that and only latency grows.
+  # 0 = unlimited (not recommended in production).
+  max_concurrent: 48
+
+  # Per-search timeout in ms; returns 503 on expiry. 0 = no timeout.
+  timeout_ms: 30000
+
+# ---------------------------------------------------------------------------
+# Per-tenant concurrency limits
+# ---------------------------------------------------------------------------
+# Applies to all tenant-scoped routes (search, ingest, collection ops).
+# Admin requests and global-key requests bypass these limits.
+# Per-tenant overrides can be set in tenants.yml (see tenants.yml.example).
+# Moving-window limits (max_rpm, max_rph) require SQLite — post-v0.5.8.
+
+tenants:
+  default_max_concurrent: 10  # applies to every tenant; 0 = unlimited
+
+# ---------------------------------------------------------------------------
+# Text preprocessing
+# ---------------------------------------------------------------------------
+# Controls how plain-text (.txt) files are split into chunks before embedding.
+# Overlap ensures context is preserved across chunk boundaries.
+
+preprocess:
+  txt_chunk_size: 1000     # characters per chunk
+  txt_chunk_overlap: 200   # characters of overlap between adjacent chunks
+
+# ---------------------------------------------------------------------------
+# Server
+# ---------------------------------------------------------------------------
+# These defaults are overridden by env vars HOST / PORT / RELOAD / WORKERS.
+
+server:
+  host: 0.0.0.0
+  port: 8086
+  reload: false
+  workers: 1
+
+  # Keep-alive timeout in seconds (passed to uvicorn --timeout-keep-alive).
+  # Must be lower than your reverse proxy's keepalive_timeout.
+  # nginx default is 75 s — set this to 65 when behind nginx.
+  timeout_keep_alive: 75
+
+# ---------------------------------------------------------------------------
+# Logging
+# ---------------------------------------------------------------------------
+
+log:
+  # Dev log level (stderr). DEBUG | INFO | WARNING | ERROR — default INFO.
+  # Overridden by PAVEDB_LOG__LEVEL env var (e.g. in Makefile: debug).
+  # Per-namespace overrides: log.debug / log.watch / log.quiet (list of loggers).
+  level: INFO
+
+  # Ops log — one structured JSON line per operation (search, ingest, delete…).
+  # Fields: ts, op, tenant, collection, latency_ms, status, error_code, hits.
+  # null (off) | stdout | /path/to/ops.jsonl
+  # stdout is recommended for Docker/12-factor; pave uses stderr for the dev log.
+  # File paths suit traditional deployments; no rotation is provided here.
+  ops_log: null
+
+  # Uvicorn access log destination.
+  # null (use uvicorn default) | stdout | /path/to/access.log
+  access_log: null
+
+# ---------------------------------------------------------------------------
+# Instance branding (optional)
+# ---------------------------------------------------------------------------
+
+instance:
+  name: PaveDB
+  desc: Vector Search Microservice (pluggable, functional)

pave/assets/tenants.yml.example

@@ -0,0 +1,19 @@
+# Untracked secrets file with tenant API keys and per-tenant limits.
+# Name it 'tenants.yml', adjust and DO NOT commit.
+# The installed default should be innocuous; uncomment the examples you need.
+#
+# auth:
+#   api_keys:
+#     acme: change-me
+#     umbrella: change-me
+
+# Per-tenant concurrency overrides.
+# Omit a tenant entry to fall back to tenants.default_max_concurrent in config.yml.
+# 0 = unlimited for that tenant.
+# Moving-window limits (max_rpm, max_rph) will be added here after SQLite lands.
+#
+# tenants:
+#   acme:
+#     max_concurrent: 5   # overrides global default for acme; 0 = unlimited
+#   umbrella:
+#     max_concurrent: 20

pave/assets/ui.html

@@ -0,0 +1,366 @@
+<!doctype html>
+<!-- (C) 2025 Rodrigo Rodrigues da Silva <rodrigo@flowlexi.com> -->
+<!-- SPDX-License-Identifier: AGPL-3.0-or-later -->
+
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width,initial-scale=1" />
+<!-- favicon robusto -->
+<link rel="icon" type="image/png" sizes="32x32"
+      href="/assets/pavedb_icon_192.png?v=__VERSION__" />
+<link rel="icon" type="image/png" sizes="192x192"
+      href="/assets/pavedb_icon_192.png?v=__VERSION__" />
+<link rel="icon" href="/favicon.ico?v=__VERSION__" />
+<title>__INST_NAME__ • Search</title>
+<style>
+  /* Flowlexi docs palette with restrained PaveDB accents */
+  :root{
+    --bg:#F4F2ED; --panel:#FFFFFF; --text:#1E1E1E; --muted:#4B5563;
+    --accent:#2ECC71; --border:#E5E7EB; --link:#11957D;
+    --link-accent:#0B7769;
+  }
+
+  *{ box-sizing:border-box }
+  html,body{ height:100% }
+  body{
+    min-height:100vh; display:flex; flex-direction:column; overflow:hidden;
+    margin:0; font:16px/1.45 system-ui,Segoe UI,Roboto,Inter,Arial;
+    background:var(--bg); color:var(--text);
+  }
+
+  .bar{
+    flex:0 0 auto; display:flex; gap:10px; align-items:center;
+    padding:12px; border-bottom:1px solid var(--border); background:var(--bg);
+  }
+  .left{ display:flex; gap:8px; align-items:center }
+  .right{
+    margin-left:auto; color:var(--muted); white-space:nowrap;
+    overflow:hidden; text-overflow:ellipsis;
+  }
+
+  .tab{
+    padding:8px 12px; border-radius:10px; border:1px solid var(--border);
+    cursor:pointer; background:var(--panel); color:var(--text);
+  }
+  .tab:hover{
+    background:#F1F3F4; color:var(--link-accent);
+  }
+  .tab.active{
+    background:#D9E3E2; color:var(--link-accent);
+    border-color:var(--link-accent);
+  }
+
+  .frames{ flex:1 1 auto; min-height:0; background:var(--bg) }
+  .frame{ display:none; width:100%; height:100%; border:0; background:var(--bg) }
+  .frame.active{ display:block }
+
+  .footer{
+    flex:0 0 auto; position:sticky; bottom:0;
+    display:flex; gap:12px; align-items:center; justify-content:center;
+    padding:10px; color:var(--muted); border-top:1px solid var(--border);
+    background:var(--bg);
+  }
+  .footer a{ color:var(--link); text-decoration:none }
+  .footer a:hover{ color:var(--link-accent) }
+</style>
+</head>
+<body>
+  <div class="bar">
+    <div class="left">
+      <button class="tab active" data-target="search"
+              data-title="__INST_NAME__ • Search">Search</button>
+      <button class="tab" data-target="data"
+              data-title="__INST_NAME__ • Data">Data</button>
+      <button class="tab" data-target="admin"
+              data-title="__INST_NAME__ • Admin">Admin</button>
+    </div>
+    <div class="right">
+      <strong>__INST_NAME__</strong> — <small>__INST_DESC__</small>
+    </div>
+  </div>
+
+  <div class="frames">
+    <iframe id="search" class="frame active" src="/ui/search" title="Search"></iframe>
+    <iframe id="data" class="frame" src="/ui/data" title="Data"></iframe>
+    <iframe id="admin" class="frame" src="/ui/admin" title="Admin"></iframe>
+  </div>
+
+  <div class="footer">
+    © 2025, 2026
+    <a href="https://flowlexi.com" target="_blank"
+       rel="noopener">Flowlexi</a> •
+    <a href="__REPO_URL__" target="_blank"
+       rel="noopener">🛣️ PaveDB __VERSION__</a> •
+    <a href="__LICENSE_URL__" target="_blank" rel="noopener">__LICENSE_NAME__</a>
+  </div>
+<script>
+  function activeFrame(){ return document.querySelector('.frame.active'); }
+
+  function scrubSwagger(frame){
+    const win = frame?.contentWindow;
+    const doc = win?.document;
+    if(!doc) return;
+    if(!doc.querySelector('.swagger-ui')){
+      setTimeout(()=>scrubSwagger(frame), 80);
+      return;
+    }
+
+    // CSS: remove topo/infos
+    const STYLE_ID = 'pv-clean';
+    if(!doc.getElementById(STYLE_ID)){
+      const s = doc.createElement('style');
+      s.id = STYLE_ID;
+      s.textContent = `
+        :root{
+          --bg:#F4F2ED; --panel:#FFFFFF; --text:#1E1E1E;
+          --muted:#4B5563; --accent:#2ECC71; --border:#E5E7EB;
+          --link:#11957D; --link-accent:#0B7769;
+        }
+        #operations-tag-default,
+        .swagger-ui .topbar,
+        .swagger-ui .download-url-wrapper,
+        .swagger-ui .information-container,
+        .swagger-ui .info,
+        .swagger-ui .servers { display:none !important; }
+        html, body {
+          background: var(--bg) !important;
+          color: var(--text) !important;
+        }
+        .swagger-ui {
+          color: var(--text) !important;
+        }
+        .swagger-ui .wrapper {
+          margin:0 !important; padding:0 0 8px !important;
+          max-width:none !important;
+        }
+        .swagger-ui a,
+        .swagger-ui .opblock-summary-path,
+        .swagger-ui .link {
+          color: var(--link) !important;
+        }
+        .swagger-ui a:hover,
+        .swagger-ui .opblock-summary-path:hover,
+        .swagger-ui .link:hover {
+          color: var(--link-accent) !important;
+        }
+        .swagger-ui section.models,
+        .swagger-ui .model-box {
+          background: var(--panel) !important;
+          border:1px solid var(--border) !important;
+          box-shadow:none !important;
+        }
+        .swagger-ui .scheme-container {
+          margin:8px 0 0 !important;
+          padding:0 16px 8px !important;
+          background: transparent !important;
+          border:0 !important;
+          box-shadow:none !important;
+        }
+        .swagger-ui .scheme-container .schemes,
+        .swagger-ui .scheme-container .auth-wrapper {
+          background: transparent !important;
+          border:0 !important;
+          box-shadow:none !important;
+        }
+        .swagger-ui .opblock {
+          background: var(--panel) !important;
+          border:1px solid var(--border) !important;
+          box-shadow:none !important;
+        }
+        .swagger-ui .opblock .opblock-summary,
+        .swagger-ui .responses-inner,
+        .swagger-ui table tbody tr td,
+        .swagger-ui .parameters-col_description input,
+        .swagger-ui .parameters-col_description textarea,
+        .swagger-ui .model-example {
+          border-color: var(--border) !important;
+        }
+        .swagger-ui .opblock-tag,
+        .swagger-ui .opblock-summary-description,
+        .swagger-ui .response-col_description__inner p,
+        .swagger-ui .parameter__type,
+        .swagger-ui .parameter__name,
+        .swagger-ui .responses-inner h4,
+        .swagger-ui .responses-inner h5,
+        .swagger-ui .model-title,
+        .swagger-ui label,
+        .swagger-ui .prop-name {
+          color: var(--text) !important;
+        }
+        .swagger-ui .opblock-description-wrapper p,
+        .swagger-ui .response-col_links,
+        .swagger-ui .responses-inner .markdown p,
+        .swagger-ui .renderedMarkdown p,
+        .swagger-ui .parameter__deprecated,
+        .swagger-ui .tab li,
+        .swagger-ui .model-title small,
+        .swagger-ui .prop-type {
+          color: var(--muted) !important;
+        }
+        .swagger-ui .copy-to-clipboard,
+        .swagger-ui .model-hint {
+          color: #6B7280 !important;
+        }
+        .swagger-ui .opblock-body pre.microlight,
+        .swagger-ui .highlight-code,
+        .swagger-ui textarea,
+        .swagger-ui input[type=text],
+        .swagger-ui input[type=password],
+        .swagger-ui select {
+          background: #F1F3F4 !important;
+          border:1px solid var(--border) !important;
+          color: var(--text) !important;
+          box-shadow:none !important;
+        }
+        .swagger-ui textarea:focus,
+        .swagger-ui input[type=text]:focus,
+        .swagger-ui input[type=password]:focus,
+        .swagger-ui select:focus {
+          border-color: var(--link) !important;
+          outline:none !important;
+        }
+        .swagger-ui .tab li.active {
+          background: #D9E3E2 !important;
+          color: var(--text) !important;
+        }
+        .swagger-ui .btn {
+          border-radius:8px !important;
+          box-shadow:none !important;
+        }
+        .swagger-ui .btn.authorize,
+        .swagger-ui .btn.execute,
+        .swagger-ui .btn.try-out__btn {
+          background: var(--link) !important;
+          border-color: var(--link) !important;
+          color: #FFFFFF !important;
+        }
+        .swagger-ui .btn.authorize:hover,
+        .swagger-ui .btn.execute:hover,
+        .swagger-ui .btn.try-out__btn:hover,
+        .swagger-ui .btn.authorize:focus,
+        .swagger-ui .btn.execute:focus,
+        .swagger-ui .btn.try-out__btn:focus {
+          background: var(--link-accent) !important;
+          border-color: var(--link-accent) !important;
+          color: #FFFFFF !important;
+        }
+        .swagger-ui .btn.cancel {
+          background: var(--panel) !important;
+          border-color: var(--border) !important;
+          color: var(--muted) !important;
+        }
+        .swagger-ui .btn.cancel:hover,
+        .swagger-ui .btn.cancel:focus {
+          background: #F1F3F4 !important;
+          color: var(--text) !important;
+        }
+        .swagger-ui .btn.authorize svg,
+        .swagger-ui .btn.execute svg {
+          fill: currentColor !important;
+        }
+        .swagger-ui .opblock.opblock-get .opblock-summary {
+          background: rgba(91,141,184,.08) !important;
+        }
+        .swagger-ui .opblock.opblock-post .opblock-summary {
+          background: rgba(17,149,125,.08) !important;
+        }
+        .swagger-ui .opblock.opblock-put .opblock-summary {
+          background: rgba(201,130,58,.08) !important;
+        }
+        .swagger-ui .opblock.opblock-patch .opblock-summary {
+          background: rgba(68,191,152,.08) !important;
+        }
+        .swagger-ui .opblock.opblock-delete .opblock-summary {
+          background: rgba(176,82,74,.08) !important;
+        }
+        .swagger-ui .opblock.opblock-head .opblock-summary {
+          background: rgba(122,107,152,.08) !important;
+        }
+        .swagger-ui .opblock.opblock-options .opblock-summary {
+          background: rgba(79,111,143,.08) !important;
+        }
+        .swagger-ui .opblock.opblock-get .opblock-summary-method {
+          background:#5B8DB8 !important;
+        }
+        .swagger-ui .opblock.opblock-post .opblock-summary-method {
+          background:#11957D !important;
+        }
+        .swagger-ui .opblock.opblock-put .opblock-summary-method {
+          background:#C9823A !important;
+        }
+        .swagger-ui .opblock.opblock-patch .opblock-summary-method {
+          background:#44BF98 !important;
+        }
+        .swagger-ui .opblock.opblock-delete .opblock-summary-method {
+          background:#B0524A !important;
+        }
+        .swagger-ui .opblock.opblock-head .opblock-summary-method {
+          background:#7A6B98 !important;
+        }
+        .swagger-ui .opblock.opblock-options .opblock-summary-method {
+          background:#4F6F8F !important;
+        }
+      `;
+      doc.head.appendChild(s);
+    }
+  }
+
+  const TAB_KEY = 'pavedb.ui.tab';
+  const tabs = document.querySelectorAll('.tab');
+  const frames = document.querySelectorAll('.frame');
+
+  function tabNames(){
+    return Array.from(tabs).map(function(tab){ return tab.dataset.target; });
+  }
+
+  function tabFromLocation(){
+    const qs = new URLSearchParams(window.location.search);
+    const queryTab = qs.get('tab');
+    if(queryTab) return queryTab;
+    const hash = window.location.hash.replace(/^#\/?/, '');
+    return hash || '';
+  }
+
+  function rememberedTab(){
+    try { return window.localStorage.getItem(TAB_KEY) || ''; }
+    catch (_err) { return ''; }
+  }
+
+  function writeTab(name){
+    try { window.localStorage.setItem(TAB_KEY, name); }
+    catch (_err) {}
+    const url = new URL(window.location.href);
+    url.searchParams.set('tab', name);
+    url.hash = '/' + name;
+    window.history.replaceState(null, '', url);
+  }
+
+  function activateTab(name, updateUrl){
+    if(tabNames().indexOf(name) === -1) name = 'search';
+    tabs.forEach(function(tab){ tab.classList.remove('active'); });
+    frames.forEach(function(frame){ frame.classList.remove('active'); });
+    const tab = document.querySelector('.tab[data-target="' + name + '"]');
+    const frame = document.getElementById(name);
+    if(tab) tab.classList.add('active');
+    if(frame) frame.classList.add('active');
+    if(tab && tab.dataset.title) document.title = tab.dataset.title;
+    if(frame) scrubSwagger(frame);
+    if(updateUrl) writeTab(name);
+  }
+
+  tabs.forEach(function(tab){
+    tab.addEventListener('click', function(){
+      activateTab(tab.dataset.target, true);
+    });
+  });
+
+  frames.forEach(function(frame){
+    frame.addEventListener('load', function(){ scrubSwagger(frame); });
+  });
+
+  const initialTab = tabFromLocation() || rememberedTab() || 'search';
+  activateTab(initialTab, false);
+</script>
+</body>
+</html>