verity-rag 0.1.0__tar.gz

verity_rag-0.1.0/.claude/settings.local.json

@@ -0,0 +1,40 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(python *)",
+      "Bash(uvicorn rag_agent.app:create_app --factory --host 0.0.0.0 --port 8100)",
+      "Bash(xargs ls -la)",
+      "Bash(uv run *)",
+      "Bash(docker compose up *)",
+      "Bash(podman compose *)",
+      "Bash(docker compose ps *)",
+      "Bash(podman rm *)",
+      "Bash(podman exec *)",
+      "Bash(podman inspect *)",
+      "Bash(podman run *)",
+      "Bash(python3 *)",
+      "Bash(wkhtmltopdf - /tmp/test_document.pdf)",
+      "Bash(curl -s -X POST 'http://localhost:8100/api/v1/documents/upload?collection_name=documents&replace=true' -H 'Content-Type: multipart/form-data' -F file=@/tmp/test_document.pdf)",
+      "Bash(curl *)",
+      "Bash(kill %1)",
+      "Bash(ip addr *)",
+      "Bash(awk '{print $2}')",
+      "Bash(ip route *)",
+      "Bash(awk '{print $3}')",
+      "Bash(xargs -I{} ip addr show {})",
+      "Bash(podman network *)",
+      "Bash(sudo iptables -I INPUT -s 10.88.0.0/16 -p tcp --dport 8012 -j ACCEPT)",
+      "Bash(podman-compose ps *)",
+      "Bash(docker-compose ps *)",
+      "Bash(podman ps *)",
+      "Bash(podman logs *)",
+      "Bash(podman info *)",
+      "Bash(podman version *)",
+      "Bash(podman-compose stop *)",
+      "Bash(podman-compose rm *)",
+      "Bash(podman-compose up *)",
+      "Bash(env)",
+      "Bash(wait)"
+    ]
+  }
+}

verity_rag-0.1.0/.dockerignore

@@ -0,0 +1,68 @@
+# Environment
+.env
+.env.local
+.env.*.local
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+.eggs/
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Virtual environments - CRITICAL for Docker build performance
+venv/
+.venv/
+env/
+*/env/
+*/venv/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.coverage.*
+*.log
+
+# Jupyter
+.ipynb_checkpoints/
+
+# Git
+.git/
+.gitignore
+
+# Docker
+Dockerfile*
+docker-compose*
+.dockerignore
+
+# Data
+data/
+uploads/
+media/
+*.db
+*.sqlite3
+milvus-data/
+
+# Vector stores & models
+*.bin
+*.pkl
+*.joblib
+*.onnx
+.huggingface/
+cache/

verity_rag-0.1.0/.env.dev

@@ -0,0 +1,52 @@
+# ── Dev-Fast: Local development without application containers ────
+# Data infra (Postgres, Valkey, Milvus) still runs in containers
+# but the API server and worker run directly on the host with hot-reload.
+#
+# Usage:
+#   make dev-up          # start data infra containers
+#   make dev-fast        # run API with hot-reload
+#   make dev-fast-worker # run worker directly
+#   make dev-down        # stop data infra containers
+
+# ── Infrastructure (uses docker-compose host port mappings) ─────
+DATABASE_URL=postgresql+asyncpg://rag:rag@localhost:5433/rag
+VALKEY_URL=redis://localhost:6380/0
+MILVUS_URI=http://localhost:19530
+MILVUS_TOKEN=
+
+# ── Embeddings ──────────────────────────────────────────────────
+# Set EMBEDDING_BASE_URL to empty to use local sentence-transformers
+# Or point to an OpenAI-compatible endpoint (Ollama, vLLM, TEI, etc.)
+EMBEDDING_BASE_URL=http://localhost:8012/v1
+EMBEDDING_API_KEY=
+EMBEDDING_MODEL=huoxu/all-MiniLM-L6-v2-Q8_0-GGUF:Q8_0
+EMBEDDING_DIM=384
+
+# ── LLM (for image description) ────────────────────────────────
+LLM_BASE_URL=
+LLM_API_KEY=
+LLM_MODEL=
+
+# ── Storage ────────────────────────────────────────────────────
+MEDIA_DIR=./.dev-media
+MAX_UPLOAD_SIZE_MB=50
+
+# ── Pipeline ───────────────────────────────────────────────────
+CHUNK_SIZE=512
+CHUNK_OVERLAP=50
+CHUNKING_STRATEGY=recursive
+ENABLE_HYBRID_SEARCH=false
+ENABLE_OCR=false
+ENABLE_IMAGE_DESCRIPTION=false
+
+# ── Reranker ───────────────────────────────────────────────────
+CROSS_ENCODER_MODEL=cross-encoder/ms-marco-MiniLM-L6-v2
+HF_TOKEN=
+
+# ── Server ─────────────────────────────────────────────────────
+HOST=0.0.0.0
+PORT=8100
+DEBUG=true
+CORS_ORIGINS='["*"]'
+LOG_LEVEL=DEBUG
+LOG_FORMAT=console

verity_rag-0.1.0/.env.example

@@ -0,0 +1,41 @@
+# ── Infrastructure ──────────────────────────────────────────────
+DATABASE_URL=postgresql+asyncpg://rag:rag@localhost:5433/rag
+VALKEY_URL=redis://localhost:6380/0
+MILVUS_URI=http://localhost:19530
+MILVUS_TOKEN=
+
+# ── Embeddings ──────────────────────────────────────────────────
+# OpenAI-compatible endpoint (works with Ollama, vLLM, TEI, OpenAI)
+EMBEDDING_BASE_URL=
+EMBEDDING_API_KEY=
+EMBEDDING_MODEL=all-MiniLM-L6-v2
+EMBEDDING_DIM=384
+
+# ── LLM (for image description) ────────────────────────────────
+LLM_BASE_URL=
+LLM_API_KEY=
+LLM_MODEL=
+
+# ── Storage ────────────────────────────────────────────────────
+MEDIA_DIR=/data/media
+MAX_UPLOAD_SIZE_MB=50
+
+# ── Pipeline ───────────────────────────────────────────────────
+CHUNK_SIZE=512
+CHUNK_OVERLAP=50
+CHUNKING_STRATEGY=recursive
+ENABLE_HYBRID_SEARCH=false
+ENABLE_OCR=false
+ENABLE_IMAGE_DESCRIPTION=true
+
+# ── Reranker ───────────────────────────────────────────────────
+CROSS_ENCODER_MODEL=cross-encoder/ms-marco-MiniLM-L6-v2
+HF_TOKEN=
+
+# ── Server ─────────────────────────────────────────────────────
+HOST=0.0.0.0
+PORT=8100
+DEBUG=false
+CORS_ORIGINS=["*"]
+LOG_LEVEL=INFO
+LOG_FORMAT=json

verity_rag-0.1.0/.gitignore

@@ -0,0 +1,60 @@
+# Environment
+.env
+.env.local
+.env.*.local
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+.eggs/
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Virtual environments
+venv/
+.venv/
+env/
+*/env/
+*/venv/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Project specific
+data/
+uploads/
+chroma_db/
+*.db
+*.sqlite3
+milvus-data/
+.dev-media/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.coverage.*
+*.log
+
+# Jupyter
+.ipynb_checkpoints/
+
+# Vector stores & models
+*.bin
+*.pkl
+*.joblib
+*.onnx
+.huggingface/
+cache/

verity_rag-0.1.0/Dockerfile

@@ -0,0 +1,33 @@
+# ── RAG Agent — API & Worker ──────────────────────────────────────
+FROM python:3.12-slim AS base
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1
+
+# System dependencies for PyMuPDF and other native libs
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        curl \
+        libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy application source, configuration, and frontend
+COPY pyproject.toml .
+COPY rag_agent/ rag_agent/
+COPY frontend/ frontend/
+COPY alembic.ini .
+
+# Install Python dependencies (requires source for hatchling wheel build)
+RUN pip install --upgrade pip && \
+    pip install .
+
+# Create media directory
+RUN mkdir -p /data/media
+
+EXPOSE 8100
+
+# Default: run the API server
+CMD ["uvicorn", "rag_agent.app:create_app", "--host", "0.0.0.0", "--port", "8100", "--factory"]

verity_rag-0.1.0/Makefile

@@ -0,0 +1,219 @@
+# ── RAG Agent Makefile ───────────────────────────────────────────
+# Common tasks for development and deployment
+# Requires: Podman or Docker (with compose), uv (https://github.com/astral-sh/uv)
+
+SHELL := /bin/bash
+export PYTHONPATH := $(PWD)
+
+# ── Container Runtime ────────────────────────────────────────────
+# Auto-detect: prefers podman, falls back to docker.
+# Override with:  make CONTAINER_RUNTIME=docker up
+CONTAINER_RUNTIME ?= $(shell if command -v podman &>/dev/null; then echo podman; elif command -v docker &>/dev/null; then echo docker; else echo docker; fi)
+COMPOSE := $(CONTAINER_RUNTIME) compose
+
+# ── Environment ──────────────────────────────────────────────────
+ifeq ($(shell uname -s),Darwin)
+	USER_ID := 501
+	GROUP_ID := 20
+else
+	USER_ID := $(shell id -u)
+	GROUP_ID := $(shell id -g)
+endif
+
+# ── Help ────────────────────────────────────────────────────────
+help:
+	@echo "RAG Agent Development Tasks"
+	@echo "──────────────────────────────────────────────────────────────"
+	@echo "setup         - Install Python dependencies with uv"
+	@echo "test          - Run test suite"
+	@echo "lint          - Run linters (ruff)"
+	@echo "format        - Format code (ruff format)"
+	@echo "typecheck     - Run type checking (mypy)"
+	@echo ""
+	@echo "── Container Stack ───────────────────────────────────────────"
+	@echo "up            - Start full container stack"
+	@echo "down          - Stop container stack"
+	@echo "logs          - Show service logs"
+	@echo "ps            - Show running containers"
+	@echo ""
+	@echo "── Dev-Fast (no app containers) ─────────────────────────────"
+	@echo "dev-up        - Start ONLY data infra (Postgres, Valkey, Milvus)"
+	@echo "dev-down      - Stop data infra containers"
+	@echo "dev-logs      - Show data infra logs"
+	@echo "dev-fast      - Run API directly with hot-reload + .env.dev"
+	@echo "dev-fast-worker - Run worker directly with .env.dev"
+	@echo "dev-migrate   - Run Alembic migrations against local infra"
+	@echo "dev-create-tables - Create tables directly (no Alembic)"
+	@echo "dev-setup     - Create media dir + run migrations"
+	@echo ""
+	@echo "── Local Commands ───────────────────────────────────────────"
+	@echo "worker        - Start ARQ worker (outside Docker)"
+	@echo "run           - Start API server (outside Docker)"
+	@echo "migrate       - Run Alembic migrations"
+	@echo "wait          - Wait for services to be ready"
+	@echo "clean         - Clean build artifacts and __pycache__"
+	@echo ""
+	@echo "── Shell Access ─────────────────────────────────────────────"
+	@echo "shell         - Open shell in API container"
+	@echo "db-shell      - Open PostgreSQL shell (container)"
+	@echo "valkey-cli    - Open Valkey CLI (container)"
+	@echo "milvus-cli    - Open Milvus CLI (container)"
+	@echo "──────────────────────────────────────────────────────────────"
+	@echo "Example: make dev-up && make dev-setup && make dev-fast"
+
+# ── Python Dependencies ──────────────────────────────────────────
+setup:
+	@echo "📦 Installing Python dependencies with uv..."
+	uv sync --all-extras
+
+.PHONY: setup
+
+# ── Testing ──────────────────────────────────────────────────────
+test:
+	@echo "🧪 Running test suite..."
+	uv run pytest tests/ -v
+
+.PHONY: test
+
+# ── Linting and Formatting ───────────────────────────────────────
+lint:
+	@echo "🔍 Running linters..."
+	uv run ruff check .
+
+format:
+	@echo "✏️  Formatting code..."
+	uv run ruff format .
+
+typecheck:
+	@echo "✅ Running type checking..."
+	uv run mypy rag_agent/
+
+.PHONY: lint format typecheck
+
+# ── Container Stack ──────────────────────────────────────────────
+up:
+	@echo "🐳 Starting container stack with $(CONTAINER_RUNTIME)..."
+	$(COMPOSE) up -d
+
+down:
+	@echo "⏹️  Stopping container stack..."
+	$(COMPOSE) down
+
+logs:
+	@echo "📋 Showing service logs..."
+	$(COMPOSE) logs -f --tail=100
+
+ps:
+	@echo "📋 Showing running containers..."
+	$(COMPOSE) ps
+
+.PHONY: up down logs ps
+
+# ── Development ──────────────────────────────────────────────────
+run:
+	@echo "🚀 Starting API server..."
+	uv run uvicorn rag_agent.app:create_app --reload --factory --host 0.0.0.0 --port 8100
+
+worker:
+	@echo "⚡ Starting ARQ worker..."
+	uv run arq rag_agent.worker.settings.WorkerSettings
+
+migrate:
+	@echo "🔄 Running database migrations..."
+	uv run alembic revision --autogenerate -m "Auto-generated migration"
+	uv run alembic upgrade head
+
+wait:
+	@echo "⏳ Waiting for services to be ready..."
+	python scripts/wait_for_services.py
+
+.PHONY: run worker migrate wait
+
+# ── Dev-Fast (data infra in containers, app on host) ─────────────
+DEV_COMPOSE_SERVICES := postgres valkey milvus-etcd milvus-minio milvus
+
+dev-up:
+	@echo "🐳 Starting data infrastructure (Postgres, Valkey, Milvus)..."
+	$(COMPOSE) up -d $(DEV_COMPOSE_SERVICES)
+	@echo ""
+	@echo "⏳ Waiting for services to be ready..."
+	@$(MAKE) wait 2>/dev/null || echo "⚠️  wait script may fail if API not running; check 'make dev-logs'"
+	@echo ""
+	@echo "📋 Next steps:"
+	@echo "   make dev-setup    # create media dir + run migrations"
+	@echo "   make dev-fast     # start API with hot-reload"
+
+dev-down:
+	@echo "⏹️  Stopping data infrastructure..."
+	$(COMPOSE) down
+
+dev-logs:
+	@echo "📋 Showing data infra logs..."
+	$(COMPOSE) logs -f --tail=100 $(DEV_COMPOSE_SERVICES)
+
+# Helper: source .env.dev directly so variables are set in the current shell.
+# This avoids glob-expansion issues with values like CORS_ORIGINS=["*"].
+# (set -a ensures all sourced variables are exported to the environment)
+DEV_ENV := set -a; . .env.dev; set +a;
+
+dev-setup:
+	@echo "📁 Creating local media directory..."
+	@mkdir -p .dev-media
+	@echo "🔄 Running database migrations..."
+	-$(DEV_ENV) uv run alembic upgrade head
+	@echo "✅ Dev environment ready!"
+
+dev-migrate:
+	@echo "🔄 Running database migrations..."
+	$(DEV_ENV) uv run alembic revision --autogenerate -m "Auto-generated migration"
+	$(DEV_ENV) uv run alembic upgrade head
+
+dev-create-tables:
+	@echo "🏗️  Creating database tables directly..."
+	$(DEV_ENV) uv run python scripts/create_tables.py
+
+dev-fast:
+	@echo "🚀 Starting API server (hot-reload, no container)..."
+	$(DEV_ENV) uv run uvicorn rag_agent.app:create_app \
+		--reload \
+		--factory \
+		--host 0.0.0.0 \
+		--port 8100
+
+dev-fast-worker:
+	@echo "⚡ Starting ARQ worker (no container)..."
+	$(DEV_ENV) uv run arq rag_agent.worker.settings.WorkerSettings
+
+.PHONY: dev-up dev-down dev-logs dev-setup dev-migrate dev-create-tables dev-fast dev-fast-worker
+
+# ── Shell Access ─────────────────────────────────────────────────
+shell:
+	@echo "🐚 Opening shell in API container..."
+	$(COMPOSE) exec api bash
+
+db-shell:
+	@echo "🐘 Opening PostgreSQL shell..."
+	$(COMPOSE) exec postgres psql -U rag rag
+
+valkey-cli:
+	@echo "⚡ Opening Valkey CLI..."
+	$(COMPOSE) exec valkey valkey-cli
+
+milvus-cli:
+	@echo "🔍 Opening Milvus CLI..."
+	$(COMPOSE) exec milvus milvus_cli
+
+.PHONY: shell db-shell valkey-cli milvus-cli
+
+# ── Clean ────────────────────────────────────────────────────────
+clean:
+	@echo "🧹 Cleaning build artifacts..."
+	find . -type f -name "*.pyc" -delete
+	find . -type d -name "__pycache__" -exec rm -rf {} +
+	find . -type f -name "*.log" -delete
+	rm -rf .venv .pytest_cache .ruff_cache .mypy_cache
+
+.PHONY: clean
+
+# ── Default Target ────────────────────────────────────────────────
+.DEFAULT_GOAL := help