raglite-toolkit 1.0.2__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raglite_toolkit-1.2.0/.github/workflows/pr-verify.yml +36 -0
- raglite_toolkit-1.2.0/ARCHITECTURE.md +188 -0
- raglite_toolkit-1.2.0/CHANGELOG.md +46 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/PKG-INFO +124 -59
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/README.md +122 -58
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/basic.py +1 -1
- raglite_toolkit-1.2.0/examples/custom_store_example.py +103 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/multi_provider.py +2 -1
- raglite_toolkit-1.2.0/examples/qdrant_example.py +54 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/serve.py +1 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/pyproject.toml +13 -1
- raglite_toolkit-1.2.0/scripts/pre-commit.sh +23 -0
- raglite_toolkit-1.2.0/scripts/pre-release.sh +34 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/__init__.py +48 -49
- raglite_toolkit-1.2.0/src/raglite/api/__init__.py +4 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/api/schemas.py +2 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/api/server.py +14 -10
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/chunking/recursive.py +2 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/cli.py +43 -26
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/config.py +9 -5
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/constants.py +1 -1
- raglite_toolkit-1.2.0/src/raglite/core/collection.py +247 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/core/document.py +33 -14
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/factory.py +2 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/__init__.py +2 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/answer.py +4 -4
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/factory.py +2 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/prompt.py +1 -1
- raglite_toolkit-1.2.0/src/raglite/loaders/__init__.py +72 -0
- raglite_toolkit-1.2.0/src/raglite/loaders/directory.py +110 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/docx.py +2 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/json.py +2 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/pdf.py +2 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/txt.py +2 -2
- raglite_toolkit-1.2.0/src/raglite/loaders/web.py +66 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/retrieval/retriever.py +2 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/types.py +15 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/utils/logger.py +3 -1
- raglite_toolkit-1.2.0/src/raglite/vectordb/__init__.py +14 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/vectordb/base.py +2 -1
- raglite_toolkit-1.2.0/src/raglite/vectordb/factory.py +38 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/vectordb/memory.py +3 -3
- raglite_toolkit-1.2.0/src/raglite/vectordb/pinecone.py +168 -0
- raglite_toolkit-1.2.0/src/raglite/vectordb/qdrant.py +189 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/conftest.py +3 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/test_api.py +4 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/test_ask.py +3 -1
- raglite_toolkit-1.2.0/tests/integration/test_collection.py +106 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/test_document.py +3 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/test_ollama.py +170 -3
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_chunking.py +1 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_cli.py +3 -3
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_config.py +3 -4
- raglite_toolkit-1.2.0/tests/unit/test_directory_loader.py +53 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_errors.py +6 -7
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_hash.py +3 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_loaders.py +5 -3
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_prompt.py +1 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_retriever.py +2 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_vectordb.py +4 -1
- raglite_toolkit-1.2.0/tests/unit/test_web_loader.py +45 -0
- raglite_toolkit-1.0.2/src/raglite/api/__init__.py +0 -4
- raglite_toolkit-1.0.2/src/raglite/loaders/__init__.py +0 -34
- raglite_toolkit-1.0.2/src/raglite/vectordb/__init__.py +0 -4
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/.github/workflows/publish.yml +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/.gitignore +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/LICENSE +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/ollama_test.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/sample.txt +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/chunking/__init__.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/chunking/base.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/core/__init__.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/__init__.py +2 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/base.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/local.py +2 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/models.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/remote.py +2 -2
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/errors.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/models.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/base.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/markdown.py +1 -1
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/retrieval/__init__.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/utils/__init__.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/utils/hash.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/__init__.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/__init__.py +0 -0
- {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/__init__.py +0 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: PR Verification
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
- master
|
|
8
|
+
push:
|
|
9
|
+
branches:
|
|
10
|
+
- main
|
|
11
|
+
- master
|
|
12
|
+
workflow_dispatch:
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
verify:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- name: Checkout repository
|
|
20
|
+
uses: actions/checkout@v4
|
|
21
|
+
|
|
22
|
+
- name: Set up Python
|
|
23
|
+
uses: actions/setup-python@v5
|
|
24
|
+
with:
|
|
25
|
+
python-version: "3.12"
|
|
26
|
+
|
|
27
|
+
- name: Install dependencies
|
|
28
|
+
run: |
|
|
29
|
+
python -m pip install --upgrade pip
|
|
30
|
+
pip install -e ".[dev]"
|
|
31
|
+
|
|
32
|
+
- name: Check code style (ruff)
|
|
33
|
+
run: python -m ruff check .
|
|
34
|
+
|
|
35
|
+
- name: Run tests
|
|
36
|
+
run: python -m pytest tests/ -q
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# RAGLite Python (`raglite-toolkit`) Architecture
|
|
2
|
+
|
|
3
|
+
This document details the software architecture, component design, data flow, and APIs of the Python implementation of **RAGLite** (`raglite-toolkit`).
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Overview & Core Philosophy
|
|
8
|
+
|
|
9
|
+
`raglite-toolkit` is a Python-native Retrieval-Augmented Generation library built with Pydantic v2 and FastAPI. It provides zero-boilerplate semantic search, multi-provider LLM response synthesis, and self-hosted REST APIs over local files.
|
|
10
|
+
|
|
11
|
+
### Key Characteristics
|
|
12
|
+
* **Python-Native & Type-Annotated**: Modern Python 3.10+ typing with Pydantic v2 model validation and camelCase alias serialization.
|
|
13
|
+
* **1:1 Parity with TypeScript SDK**: Identical function signatures, option dictionaries, data models, and `.raglite/` persistence layout.
|
|
14
|
+
* **Pluggable & Extensible**: Modular Abstract Base Classes (ABCs) for Loaders, Chunkers, Embeddings, Vector Databases, and LLMs.
|
|
15
|
+
* **Per-Document Isolation**: Namespaced vector collections tied to source document path/identity.
|
|
16
|
+
* **SHA-256 Content Caching**: Automatically skips re-embedding if document content has not changed.
|
|
17
|
+
* **Offline-First Option**: Local offline embeddings via `sentence-transformers` (`all-MiniLM-L6-v2`) and local LLMs via Ollama.
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## 2. Directory & Module Structure
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
raglite-py/src/raglite/
|
|
25
|
+
├── api/ # FastAPI REST HTTP server & Pydantic schemas
|
|
26
|
+
│ ├── __init__.py
|
|
27
|
+
│ ├── schemas.py
|
|
28
|
+
│ └── server.py
|
|
29
|
+
├── chunking/ # Text splitting algorithms
|
|
30
|
+
│ ├── __init__.py
|
|
31
|
+
│ ├── base.py
|
|
32
|
+
│ └── recursive.py
|
|
33
|
+
├── core/ # Main facade orchestrator
|
|
34
|
+
│ ├── __init__.py
|
|
35
|
+
│ └── document.py
|
|
36
|
+
├── embeddings/ # Embedding provider factory & adapters
|
|
37
|
+
│ ├── __init__.py
|
|
38
|
+
│ ├── base.py
|
|
39
|
+
│ ├── factory.py
|
|
40
|
+
│ ├── local.py # sentence-transformers (offline)
|
|
41
|
+
│ ├── models.py
|
|
42
|
+
│ └── remote.py # OpenAI, Gemini, Mistral, Cohere, Voyage, Ollama
|
|
43
|
+
├── llm/ # LLM provider factory & prompt synthesis
|
|
44
|
+
│ ├── __init__.py
|
|
45
|
+
│ ├── answer.py
|
|
46
|
+
│ ├── factory.py
|
|
47
|
+
│ ├── models.py
|
|
48
|
+
│ └── prompt.py
|
|
49
|
+
├── loaders/ # Document parser implementations
|
|
50
|
+
│ ├── __init__.py
|
|
51
|
+
│ ├── base.py
|
|
52
|
+
│ ├── docx.py # python-docx reader
|
|
53
|
+
│ ├── json.py
|
|
54
|
+
│ ├── markdown.py
|
|
55
|
+
│ ├── pdf.py # pypdf / pymupdf reader
|
|
56
|
+
│ └── txt.py
|
|
57
|
+
├── retrieval/ # Retriever & ranking algorithms
|
|
58
|
+
│ ├── __init__.py
|
|
59
|
+
│ └── retriever.py
|
|
60
|
+
├── vectordb/ # Vector Database adapters
|
|
61
|
+
│ ├── __init__.py
|
|
62
|
+
│ ├── base.py # VectorStore Abstract Base Class
|
|
63
|
+
│ ├── factory.py
|
|
64
|
+
│ ├── memory.py # Local JSON persistence (.raglite/)
|
|
65
|
+
│ ├── pinecone.py # Pinecone Cloud
|
|
66
|
+
│ └── qdrant.py # Qdrant local/cloud
|
|
67
|
+
├── cli.py # Command-line interface tool (Typer/Argparse)
|
|
68
|
+
├── config.py # Environment & default configuration
|
|
69
|
+
├── constants.py # System constants & defaults
|
|
70
|
+
├── errors.py # Custom RAGLite exception definitions
|
|
71
|
+
├── types.py # Pydantic v2 data models & type aliases
|
|
72
|
+
└── utils/ # Utility functions (hashing, math, crypto)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## 3. High-Level System Diagram
|
|
78
|
+
|
|
79
|
+
```mermaid
|
|
80
|
+
graph TD
|
|
81
|
+
subgraph Client Application
|
|
82
|
+
App["Python Application"]
|
|
83
|
+
CLI["raglite CLI"]
|
|
84
|
+
HTTPClient["Curl / HTTPX Client"]
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
subgraph Entrypoints
|
|
88
|
+
Doc["Document Facade Class<br/>(src/raglite/core/document.py)"]
|
|
89
|
+
FastAPIServer["FastAPI REST Server<br/>(src/raglite/api/server.py)"]
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
subgraph Core Pipeline Modules
|
|
93
|
+
Loaders["Document Loaders<br/>(src/raglite/loaders/)"]
|
|
94
|
+
Chunker["Recursive Splitter<br/>(src/raglite/chunking/)"]
|
|
95
|
+
Embeddings["Embeddings Engine<br/>(src/raglite/embeddings/)"]
|
|
96
|
+
VectorStore["VectorStore Adapter<br/>(src/raglite/vectordb/)"]
|
|
97
|
+
Retriever["Retriever Engine<br/>(src/raglite/retrieval/)"]
|
|
98
|
+
LLM["LLM Synthesis Engine<br/>(src/raglite/llm/)"]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
subgraph Persistence Layer
|
|
102
|
+
JSONDisk["Disk Index Storage<br/>(.raglite/indexes/*.json)"]
|
|
103
|
+
ExternalVDB["External Vector DB<br/>(Qdrant / Pinecone)"]
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
App --> Doc
|
|
107
|
+
CLI --> Doc
|
|
108
|
+
HTTPClient --> FastAPIServer
|
|
109
|
+
FastAPIServer --> Doc
|
|
110
|
+
|
|
111
|
+
Doc --> Loaders
|
|
112
|
+
Doc --> Chunker
|
|
113
|
+
Doc --> Embeddings
|
|
114
|
+
Doc --> VectorStore
|
|
115
|
+
Doc --> Retriever
|
|
116
|
+
Doc --> LLM
|
|
117
|
+
|
|
118
|
+
VectorStore --> JSONDisk
|
|
119
|
+
VectorStore --> ExternalVDB
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## 4. End-to-End Data Pipeline
|
|
125
|
+
|
|
126
|
+
### 4.1 Ingestion & Indexing
|
|
127
|
+
1. `doc.build()` invokes the appropriate `BaseLoader` based on file extension (`.pdf`, `.txt`, `.json`, `.md`, `.docx`).
|
|
128
|
+
2. Calculates SHA-256 hash of raw document content.
|
|
129
|
+
3. Checks existing `IndexMetadata` in `VectorStore`. If hash matches, skips re-indexing.
|
|
130
|
+
4. If hash differs or force rebuild requested:
|
|
131
|
+
- `RecursiveCharacterTextSplitter` chunks text (default size 1000, overlap 200).
|
|
132
|
+
- `EmbeddingFactory` generates normalized vectors for each chunk.
|
|
133
|
+
- `VectorStore.add()` saves chunks and `VectorStore.save_index_metadata()` persists index metadata.
|
|
134
|
+
|
|
135
|
+
### 4.2 Retrieval & Answer Synthesis
|
|
136
|
+
1. `doc.search(query, top_k)`:
|
|
137
|
+
- Embeds query text.
|
|
138
|
+
- Executes vector search via `VectorStore.search()`, calculating cosine similarity over L2-normalized vectors.
|
|
139
|
+
- Returns top $K$ `SearchResult` objects.
|
|
140
|
+
2. `doc.ask(question, options)`:
|
|
141
|
+
- Runs `search(question)`.
|
|
142
|
+
- Constructs context-augmented system/user prompt via `build_prompt()`.
|
|
143
|
+
- Calls `generate_answer()` or `stream_answer()` with selected LLM adapter (OpenAI, Anthropic, Google, Groq, Ollama, etc.).
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## 5. REST API Architecture
|
|
148
|
+
|
|
149
|
+
Built using **FastAPI** framework for async capabilities, automatic OpenAPI docs, and Uvicorn serving.
|
|
150
|
+
|
|
151
|
+
| Method | Endpoint | Auth Required? | Description |
|
|
152
|
+
| :--- | :--- | :--- | :--- |
|
|
153
|
+
| `GET` | `/health` | No | Liveness status & total chunk count |
|
|
154
|
+
| `GET` | `/info` | Yes (Bearer token) | Configuration details & index state |
|
|
155
|
+
| `POST` | `/search` | Yes (Bearer token) | Semantic vector search |
|
|
156
|
+
| `POST` | `/ask` | Yes (Bearer token) | Context Q&A generation (supports streaming via StreamingResponse) |
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## 6. Vector Database Abstract Base Class (`src/raglite/vectordb/base.py`)
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from abc import ABC, abstractmethod
|
|
164
|
+
from typing import List, Optional
|
|
165
|
+
from raglite.types import IndexMetadata, SearchResult, StoredChunk
|
|
166
|
+
|
|
167
|
+
class VectorStore(ABC):
|
|
168
|
+
@abstractmethod
|
|
169
|
+
def load(self, doc_id: str) -> None: ...
|
|
170
|
+
|
|
171
|
+
@abstractmethod
|
|
172
|
+
def reset(self, doc_id: str) -> None: ...
|
|
173
|
+
|
|
174
|
+
@abstractmethod
|
|
175
|
+
def add(self, doc_id: str, chunks: List[StoredChunk]) -> None: ...
|
|
176
|
+
|
|
177
|
+
@abstractmethod
|
|
178
|
+
def search(self, doc_id: str, query_vector: List[float], top_k: int, min_score: Optional[float] = None) -> List[SearchResult]: ...
|
|
179
|
+
|
|
180
|
+
@abstractmethod
|
|
181
|
+
def count(self, doc_id: str) -> int: ...
|
|
182
|
+
|
|
183
|
+
@abstractmethod
|
|
184
|
+
def save_index_metadata(self, doc_id: str, meta: IndexMetadata) -> None: ...
|
|
185
|
+
|
|
186
|
+
@abstractmethod
|
|
187
|
+
def read_index_metadata(self, doc_id: str) -> Optional[IndexMetadata]: ...
|
|
188
|
+
```
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## [1.2.0] - 2026-08-02
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- **Multi-Document & Directory Ingestion (`DocumentCollection`):**
|
|
9
|
+
- Added `DocumentCollection` class to manage semantic indexing, multi-document retrieval, and Q&A across folders, glob patterns, web URLs, and mixed file lists.
|
|
10
|
+
- Parallel semantic search over collection vector stores with score-based top-$K$ merging and ranking.
|
|
11
|
+
- Contextual Q&A synthesis (`ask` and `ask_stream`) across multi-document collections.
|
|
12
|
+
- **Directory Loader (`DirectoryLoader`):**
|
|
13
|
+
- Recursive directory scanner (`recursive=True`) with glob pattern matching (e.g. `./docs/**/*.md`).
|
|
14
|
+
- Auto-detection of supported extensions (`.pdf`, `.txt`, `.md`, `.json`, `.docx`).
|
|
15
|
+
- Detailed error reporting and warning logs for unsupported/empty files.
|
|
16
|
+
- **Web Loader (`WebLoader`):**
|
|
17
|
+
- Native loader for fetching HTTP/HTTPS web URLs directly.
|
|
18
|
+
- Automatic HTML cleaning into formatted text/markdown with script, style, and SVG tag stripping.
|
|
19
|
+
- JSON and plain text content-type parsing.
|
|
20
|
+
- **CLI & FastAPI REST Server Support:**
|
|
21
|
+
- Upgraded `raglite index`, `search`, `ask`, and `serve` CLI commands to process directories, glob patterns, and URLs.
|
|
22
|
+
- Updated FastAPI REST server to support `DocumentCollection` and single `Document` targets.
|
|
23
|
+
- **Unit & Integration Tests:**
|
|
24
|
+
- Added unit and integration tests for `DirectoryLoader`, `WebLoader`, and `DocumentCollection`.
|
|
25
|
+
|
|
26
|
+
## [1.1.0] - 2026-07-19
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
- **Pluggable Vector Databases:** Added support for custom local and cloud vector database backends via a new `vectorStore` config option.
|
|
30
|
+
- **Memory Store (`"memory"`):** Default in-memory store persisting indexes locally to JSON (unchanged behaviour).
|
|
31
|
+
- **Qdrant Store (`"qdrant"`):** Wrapper for Qdrant local/cloud using stdlib `urllib` REST requests. Supports auto-collection creation, API key auth, and custom collection names.
|
|
32
|
+
- **Pinecone Store (`"pinecone"`):** Cloud database support using Pinecone Namespaces and stdlib `urllib` REST requests. Stores index metadata as a reserved `__metadata__` vector.
|
|
33
|
+
- **Custom Adapters:** Pass any class instance implementing the `VectorStore` ABC directly as `vectorStore` in `DocumentOptions`.
|
|
34
|
+
- **`VectorStoreProviderConfig` type:** New Pydantic model in `types.py` describing provider, URL, API key, index name, and store directory.
|
|
35
|
+
- **Factory:** `create_vector_store(config, namespace)` utility in `vectordb/factory.py` resolving the correct store from config.
|
|
36
|
+
- **Examples:**
|
|
37
|
+
- `examples/qdrant_example.py` — full index + search demo with Qdrant.
|
|
38
|
+
- `examples/custom_store_example.py` — implementing and using a custom VectorStore ABC subclass.
|
|
39
|
+
- **Automation Scripts:**
|
|
40
|
+
- `scripts/pre-commit.sh` — runs ruff lint, mypy type-check, and pytest before committing.
|
|
41
|
+
- `scripts/pre-release.sh` — cleans builds, runs full verification, and builds distribution packages.
|
|
42
|
+
- **GitHub Actions Workflow:** `pr-verify.yml` — automatically runs code style checks and the full test suite on every pull request and push to `main`/`master`.
|
|
43
|
+
|
|
44
|
+
### Changed
|
|
45
|
+
- **`DocumentOptions` / `ResolvedConfig`:** Added optional `vectorStore` field supporting `VectorStoreProviderConfig` or a custom `VectorStore` instance.
|
|
46
|
+
- **`Document.__init__`:** Constructor now resolves the appropriate vector store from config, accepting provider configs or custom instances.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: raglite-toolkit
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Build semantic search, multi-provider question answering, and REST APIs over your documents in a few lines of Python.
|
|
5
5
|
Project-URL: Homepage, https://github.com/creatorpiyush/raglite-py
|
|
6
6
|
Project-URL: Repository, https://github.com/creatorpiyush/raglite-py
|
|
@@ -40,20 +40,22 @@ Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
|
|
|
40
40
|
Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
|
|
41
41
|
Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
|
|
42
42
|
Requires-Dist: pytest>=7.0.0; extra == 'dev'
|
|
43
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
43
44
|
Requires-Dist: twine>=5.0.0; extra == 'dev'
|
|
44
45
|
Description-Content-Type: text/markdown
|
|
45
46
|
|
|
46
47
|
# raglite-toolkit
|
|
47
48
|
|
|
48
|
-
> Build semantic search, multi-provider question answering, and REST APIs over your documents in a few lines of Python.
|
|
49
|
+
> Build semantic search, multi-provider question answering, and REST APIs over your documents, directories, or web URLs in a few lines of Python.
|
|
49
50
|
|
|
50
|
-
**raglite-toolkit** is a Python port of the [`raglite-toolkit`](https://github.com/creatorpiyush/raglite) TypeScript package with full feature parity.
|
|
51
|
+
**raglite-toolkit** is a Python port of the [`raglite-toolkit`](https://github.com/creatorpiyush/raglite) TypeScript package with full 1:1 feature parity.
|
|
51
52
|
|
|
52
53
|
---
|
|
53
54
|
|
|
54
55
|
## Features
|
|
55
56
|
|
|
56
57
|
- 📄 **PDF, TXT, JSON, Markdown, DOCX** loaders out of the box
|
|
58
|
+
- 📁 **Multi-document, directory, & URL ingestion** — index folders, glob patterns, or web URLs with `DocumentCollection`
|
|
57
59
|
- 🤖 **Multi-provider LLMs** — OpenAI, Anthropic (Claude), Google (Gemini), Mistral, Cohere, Groq, xAI, Ollama
|
|
58
60
|
- 🔢 **Multi-provider embeddings** — OpenAI, Google, Mistral, Cohere, Voyage, Ollama, or a **local** offline sentence-transformer (no API key needed)
|
|
59
61
|
- 📐 **Cosine similarity** scoring with L2-normalized vectors
|
|
@@ -101,20 +103,44 @@ print(answer.text)
|
|
|
101
103
|
|
|
102
104
|
---
|
|
103
105
|
|
|
106
|
+
## Multi-Document & Directory Ingestion (`DocumentCollection`)
|
|
107
|
+
|
|
108
|
+
Index entire directories (`./docs`), glob patterns, web URLs, or mixed file arrays:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from raglite import DocumentCollection
|
|
112
|
+
|
|
113
|
+
collection = DocumentCollection(["./docs", "https://example.com"], {
|
|
114
|
+
"embeddings": {"provider": "local"},
|
|
115
|
+
"llm": {"provider": "openai", "apiKey": "sk-..."},
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
# Index all documents concurrently
|
|
119
|
+
result = collection.build()
|
|
120
|
+
print(f"Indexed {result.totalDocuments} document(s), {result.totalChunks} chunk(s).")
|
|
121
|
+
|
|
122
|
+
# Search across all collection documents simultaneously
|
|
123
|
+
hits = collection.search("refund policy", top_k=5)
|
|
124
|
+
|
|
125
|
+
# Contextual Q&A across the entire collection
|
|
126
|
+
answer = collection.ask("What is the refund policy?")
|
|
127
|
+
print(answer.text)
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
104
132
|
## Fully Offline — No API Key Needed
|
|
105
133
|
|
|
106
134
|
```python
|
|
107
135
|
from raglite import Document
|
|
108
136
|
|
|
109
|
-
doc = Document("./
|
|
110
|
-
"embeddings": {"provider": "local"},
|
|
111
|
-
"llm": {"provider": "ollama",
|
|
112
|
-
"model": "llama3.2",
|
|
113
|
-
"baseURL": "http://localhost:11434/api"},
|
|
137
|
+
doc = Document("./manual.txt", {
|
|
138
|
+
"embeddings": {"provider": "local"},
|
|
139
|
+
"llm": {"provider": "ollama", "model": "llama3.2"},
|
|
114
140
|
})
|
|
115
141
|
|
|
116
142
|
doc.build()
|
|
117
|
-
print(doc.ask("
|
|
143
|
+
print(doc.ask("How do I reset the device?").text)
|
|
118
144
|
```
|
|
119
145
|
|
|
120
146
|
---
|
|
@@ -122,24 +148,60 @@ print(doc.ask("What is the refund policy?").text)
|
|
|
122
148
|
## Choose Any LLM at Ask-Time
|
|
123
149
|
|
|
124
150
|
```python
|
|
125
|
-
#
|
|
126
|
-
|
|
127
|
-
{"provider": "openai",
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
print(f"[{llm['provider']}] {answer.text[:120]}")
|
|
151
|
+
# Pass an inline LLM override to ask()
|
|
152
|
+
gpt4 = doc.ask("Summarise this document", options={
|
|
153
|
+
"llm": {"provider": "openai", "model": "gpt-4o", "apiKey": "sk-..."}
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
claude = doc.ask("Summarise this document", options={
|
|
157
|
+
"llm": {"provider": "anthropic", "model": "claude-3-5-sonnet-20241022", "apiKey": "sk-ant-..."}
|
|
158
|
+
})
|
|
134
159
|
```
|
|
135
160
|
|
|
136
161
|
---
|
|
137
162
|
|
|
138
|
-
## Streaming
|
|
163
|
+
## Streaming Responses
|
|
139
164
|
|
|
140
165
|
```python
|
|
141
|
-
for chunk in doc.ask_stream("Explain
|
|
166
|
+
for chunk in doc.ask_stream("Explain section 3 in detail"):
|
|
142
167
|
print(chunk, end="", flush=True)
|
|
168
|
+
print()
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
---
|
|
172
|
+
|
|
173
|
+
## Pluggable Vector Databases
|
|
174
|
+
|
|
175
|
+
`raglite` supports pluggable vector stores (Memory, Qdrant, Pinecone, LanceDB, or custom subclasses):
|
|
176
|
+
|
|
177
|
+
### Memory Store (Default)
|
|
178
|
+
```python
|
|
179
|
+
doc = Document("./policy.pdf", {
|
|
180
|
+
"vectorStore": {"provider": "memory", "storeDir": ".raglite"}
|
|
181
|
+
})
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
### Qdrant Store
|
|
185
|
+
```python
|
|
186
|
+
doc = Document("./policy.pdf", {
|
|
187
|
+
"vectorStore": {
|
|
188
|
+
"provider": "qdrant",
|
|
189
|
+
"url": "http://localhost:6333",
|
|
190
|
+
"apiKey": "your-key",
|
|
191
|
+
"indexName": "my_collection"
|
|
192
|
+
}
|
|
193
|
+
})
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### Pinecone Store
|
|
197
|
+
```python
|
|
198
|
+
doc = Document("./policy.pdf", {
|
|
199
|
+
"vectorStore": {
|
|
200
|
+
"provider": "pinecone",
|
|
201
|
+
"url": "https://my-index.svc.pinecone.io",
|
|
202
|
+
"apiKey": "your-key"
|
|
203
|
+
}
|
|
204
|
+
})
|
|
143
205
|
```
|
|
144
206
|
|
|
145
207
|
---
|
|
@@ -147,25 +209,26 @@ for chunk in doc.ask_stream("Explain the introduction"):
|
|
|
147
209
|
## REST API
|
|
148
210
|
|
|
149
211
|
```python
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
"
|
|
212
|
+
from raglite import Document
|
|
213
|
+
|
|
214
|
+
doc = Document("./policy.pdf", {
|
|
215
|
+
"embeddings": {"provider": "local"},
|
|
216
|
+
"llm": {"provider": "openai", "apiKey": "sk-..."},
|
|
154
217
|
})
|
|
155
|
-
|
|
218
|
+
doc.build()
|
|
156
219
|
|
|
157
|
-
#
|
|
158
|
-
|
|
220
|
+
# Start background FastAPI server on port 8085
|
|
221
|
+
doc.serve(port=8085, bearer_token="secret-token")
|
|
159
222
|
```
|
|
160
223
|
|
|
161
|
-
|
|
224
|
+
Endpoints:
|
|
162
225
|
|
|
163
|
-
| Method | Path
|
|
164
|
-
|
|
165
|
-
| `GET` | `/health` | ❌
|
|
166
|
-
| `GET` | `/info` | ✅
|
|
167
|
-
| `POST` | `/search` | ✅
|
|
168
|
-
| `POST` | `/ask` | ✅
|
|
226
|
+
| Method | Path | Auth required? | Description |
|
|
227
|
+
| ------ | ---- | -------------- | ----------- |
|
|
228
|
+
| `GET` | `/health` | ❌ | Liveness + index stats |
|
|
229
|
+
| `GET` | `/info` | ✅ | Configuration snapshot |
|
|
230
|
+
| `POST` | `/search` | ✅ | Semantic search |
|
|
231
|
+
| `POST` | `/ask` | ✅ | Question answering (supports `stream: true`) |
|
|
169
232
|
|
|
170
233
|
### Example `curl` calls
|
|
171
234
|
|
|
@@ -175,19 +238,19 @@ curl http://127.0.0.1:8085/health
|
|
|
175
238
|
|
|
176
239
|
# Search
|
|
177
240
|
curl -X POST http://127.0.0.1:8085/search \
|
|
178
|
-
-H 'Authorization: Bearer
|
|
241
|
+
-H 'Authorization: Bearer secret-token' \
|
|
179
242
|
-H 'Content-Type: application/json' \
|
|
180
243
|
-d '{"query": "refund policy", "topK": 3}'
|
|
181
244
|
|
|
182
245
|
# Ask (non-streaming)
|
|
183
246
|
curl -X POST http://127.0.0.1:8085/ask \
|
|
184
|
-
-H 'Authorization: Bearer
|
|
247
|
+
-H 'Authorization: Bearer secret-token' \
|
|
185
248
|
-H 'Content-Type: application/json' \
|
|
186
249
|
-d '{"question": "What is the refund policy?"}'
|
|
187
250
|
|
|
188
251
|
# Ask (streaming)
|
|
189
252
|
curl -X POST http://127.0.0.1:8085/ask \
|
|
190
|
-
-H 'Authorization: Bearer
|
|
253
|
+
-H 'Authorization: Bearer secret-token' \
|
|
191
254
|
-H 'Content-Type: application/json' \
|
|
192
255
|
-d '{"question": "Summarize the document", "stream": true}'
|
|
193
256
|
```
|
|
@@ -197,18 +260,18 @@ curl -X POST http://127.0.0.1:8085/ask \
|
|
|
197
260
|
## CLI
|
|
198
261
|
|
|
199
262
|
```bash
|
|
200
|
-
# Index a document
|
|
263
|
+
# Index a document, directory, or URL
|
|
201
264
|
raglite index ./policy.pdf --embed-provider local
|
|
202
265
|
|
|
203
266
|
# Semantic search
|
|
204
|
-
raglite search ./
|
|
267
|
+
raglite search ./docs "refund policy" --top-k 5
|
|
205
268
|
|
|
206
269
|
# Ask a question (streaming)
|
|
207
|
-
raglite ask ./
|
|
208
|
-
--llm-provider
|
|
270
|
+
raglite ask ./docs "What is the refund policy?" \
|
|
271
|
+
--llm-provider anthropic --llm-key $ANTHROPIC_API_KEY --stream
|
|
209
272
|
|
|
210
273
|
# Serve a REST API
|
|
211
|
-
raglite serve
|
|
274
|
+
raglite serve https://example.com \
|
|
212
275
|
--llm-provider openai --llm-key $OPENAI_API_KEY \
|
|
213
276
|
--port 8085 --token $RAGLITE_TOKEN
|
|
214
277
|
```
|
|
@@ -339,7 +402,7 @@ query_vec = embedder.embed_query("refund policy")
|
|
|
339
402
|
|
|
340
403
|
```bash
|
|
341
404
|
# Clone and set up
|
|
342
|
-
git clone
|
|
405
|
+
git clone https://github.com/creatorpiyush/raglite-py.git
|
|
343
406
|
cd raglite-py
|
|
344
407
|
|
|
345
408
|
# Create virtual environment
|
|
@@ -349,15 +412,14 @@ source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
|
349
412
|
# Install in editable mode with dev dependencies
|
|
350
413
|
pip install -e ".[dev]"
|
|
351
414
|
|
|
352
|
-
# Run
|
|
415
|
+
# Run test suite
|
|
353
416
|
pytest
|
|
354
417
|
|
|
355
418
|
# Run with coverage
|
|
356
419
|
pytest --cov=raglite --cov-report=term-missing
|
|
357
420
|
|
|
358
|
-
# Run examples
|
|
421
|
+
# Run examples
|
|
359
422
|
python examples/basic.py
|
|
360
|
-
python examples/multi_provider.py # requires API keys in env
|
|
361
423
|
python examples/serve.py
|
|
362
424
|
```
|
|
363
425
|
|
|
@@ -366,23 +428,26 @@ python examples/serve.py
|
|
|
366
428
|
```
|
|
367
429
|
tests/
|
|
368
430
|
├── unit/
|
|
369
|
-
│ ├── test_chunking.py
|
|
370
|
-
│ ├── test_vectordb.py
|
|
371
|
-
│ ├── test_loaders.py
|
|
372
|
-
│ ├──
|
|
373
|
-
│ ├──
|
|
374
|
-
│ ├──
|
|
375
|
-
│ ├──
|
|
376
|
-
│ ├──
|
|
377
|
-
│
|
|
431
|
+
│ ├── test_chunking.py # RecursiveChunker algorithm
|
|
432
|
+
│ ├── test_vectordb.py # MemoryVectorStore (cosine, persistence, isolation)
|
|
433
|
+
│ ├── test_loaders.py # TxtLoader, MarkdownLoader, JsonLoader
|
|
434
|
+
│ ├── test_directory_loader.py # DirectoryLoader (recursive scanning, glob filtering)
|
|
435
|
+
│ ├── test_web_loader.py # WebLoader (HTML parsing, tag stripping)
|
|
436
|
+
│ ├── test_prompt.py # system/user prompt builders
|
|
437
|
+
│ ├── test_errors.py # exception hierarchy
|
|
438
|
+
│ ├── test_config.py # config defaults and overrides
|
|
439
|
+
│ ├── test_hash.py # SHA-256 file hashing + namespace generation
|
|
440
|
+
│ ├── test_retriever.py # Retriever with mocked embedder
|
|
441
|
+
│ └── test_cli.py # CLI commands and argument parsing
|
|
378
442
|
└── integration/
|
|
379
|
-
├── test_document.py
|
|
380
|
-
├──
|
|
381
|
-
|
|
443
|
+
├── test_document.py # build/cache/search lifecycle (mocked embeddings)
|
|
444
|
+
├── test_collection.py # DocumentCollection multi-document indexing & FastAPI server
|
|
445
|
+
├── test_ask.py # ask/stream with mocked LLM generation
|
|
446
|
+
└── test_api.py # FastAPI endpoints via TestClient
|
|
382
447
|
```
|
|
383
448
|
|
|
384
449
|
---
|
|
385
450
|
|
|
386
451
|
## License
|
|
387
452
|
|
|
388
|
-
MIT
|
|
453
|
+
MIT © [Piyush Anand](https://github.com/creatorpiyush)
|