raglite-toolkit 1.0.2__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. raglite_toolkit-1.2.0/.github/workflows/pr-verify.yml +36 -0
  2. raglite_toolkit-1.2.0/ARCHITECTURE.md +188 -0
  3. raglite_toolkit-1.2.0/CHANGELOG.md +46 -0
  4. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/PKG-INFO +124 -59
  5. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/README.md +122 -58
  6. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/basic.py +1 -1
  7. raglite_toolkit-1.2.0/examples/custom_store_example.py +103 -0
  8. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/multi_provider.py +2 -1
  9. raglite_toolkit-1.2.0/examples/qdrant_example.py +54 -0
  10. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/serve.py +1 -1
  11. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/pyproject.toml +13 -1
  12. raglite_toolkit-1.2.0/scripts/pre-commit.sh +23 -0
  13. raglite_toolkit-1.2.0/scripts/pre-release.sh +34 -0
  14. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/__init__.py +48 -49
  15. raglite_toolkit-1.2.0/src/raglite/api/__init__.py +4 -0
  16. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/api/schemas.py +2 -1
  17. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/api/server.py +14 -10
  18. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/chunking/recursive.py +2 -1
  19. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/cli.py +43 -26
  20. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/config.py +9 -5
  21. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/constants.py +1 -1
  22. raglite_toolkit-1.2.0/src/raglite/core/collection.py +247 -0
  23. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/core/document.py +33 -14
  24. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/factory.py +2 -1
  25. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/__init__.py +2 -2
  26. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/answer.py +4 -4
  27. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/factory.py +2 -2
  28. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/prompt.py +1 -1
  29. raglite_toolkit-1.2.0/src/raglite/loaders/__init__.py +72 -0
  30. raglite_toolkit-1.2.0/src/raglite/loaders/directory.py +110 -0
  31. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/docx.py +2 -1
  32. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/json.py +2 -1
  33. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/pdf.py +2 -1
  34. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/txt.py +2 -2
  35. raglite_toolkit-1.2.0/src/raglite/loaders/web.py +66 -0
  36. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/retrieval/retriever.py +2 -2
  37. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/types.py +15 -2
  38. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/utils/logger.py +3 -1
  39. raglite_toolkit-1.2.0/src/raglite/vectordb/__init__.py +14 -0
  40. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/vectordb/base.py +2 -1
  41. raglite_toolkit-1.2.0/src/raglite/vectordb/factory.py +38 -0
  42. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/vectordb/memory.py +3 -3
  43. raglite_toolkit-1.2.0/src/raglite/vectordb/pinecone.py +168 -0
  44. raglite_toolkit-1.2.0/src/raglite/vectordb/qdrant.py +189 -0
  45. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/conftest.py +3 -1
  46. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/test_api.py +4 -2
  47. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/test_ask.py +3 -1
  48. raglite_toolkit-1.2.0/tests/integration/test_collection.py +106 -0
  49. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/test_document.py +3 -2
  50. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/test_ollama.py +170 -3
  51. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_chunking.py +1 -0
  52. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_cli.py +3 -3
  53. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_config.py +3 -4
  54. raglite_toolkit-1.2.0/tests/unit/test_directory_loader.py +53 -0
  55. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_errors.py +6 -7
  56. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_hash.py +3 -1
  57. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_loaders.py +5 -3
  58. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_prompt.py +1 -2
  59. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_retriever.py +2 -2
  60. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/test_vectordb.py +4 -1
  61. raglite_toolkit-1.2.0/tests/unit/test_web_loader.py +45 -0
  62. raglite_toolkit-1.0.2/src/raglite/api/__init__.py +0 -4
  63. raglite_toolkit-1.0.2/src/raglite/loaders/__init__.py +0 -34
  64. raglite_toolkit-1.0.2/src/raglite/vectordb/__init__.py +0 -4
  65. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/.github/workflows/publish.yml +0 -0
  66. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/.gitignore +0 -0
  67. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/LICENSE +0 -0
  68. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/ollama_test.py +0 -0
  69. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/examples/sample.txt +0 -0
  70. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/chunking/__init__.py +0 -0
  71. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/chunking/base.py +0 -0
  72. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/core/__init__.py +0 -0
  73. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/__init__.py +2 -2
  74. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/base.py +0 -0
  75. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/local.py +2 -2
  76. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/models.py +0 -0
  77. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/embeddings/remote.py +2 -2
  78. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/errors.py +0 -0
  79. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/llm/models.py +0 -0
  80. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/base.py +0 -0
  81. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/loaders/markdown.py +1 -1
  82. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/retrieval/__init__.py +0 -0
  83. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/utils/__init__.py +0 -0
  84. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/src/raglite/utils/hash.py +0 -0
  85. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/__init__.py +0 -0
  86. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/integration/__init__.py +0 -0
  87. {raglite_toolkit-1.0.2 → raglite_toolkit-1.2.0}/tests/unit/__init__.py +0 -0
@@ -0,0 +1,36 @@
1
+ name: PR Verification
2
+
3
+ on:
4
+ pull_request:
5
+ branches:
6
+ - main
7
+ - master
8
+ push:
9
+ branches:
10
+ - main
11
+ - master
12
+ workflow_dispatch:
13
+
14
+ jobs:
15
+ verify:
16
+ runs-on: ubuntu-latest
17
+
18
+ steps:
19
+ - name: Checkout repository
20
+ uses: actions/checkout@v4
21
+
22
+ - name: Set up Python
23
+ uses: actions/setup-python@v5
24
+ with:
25
+ python-version: "3.12"
26
+
27
+ - name: Install dependencies
28
+ run: |
29
+ python -m pip install --upgrade pip
30
+ pip install -e ".[dev]"
31
+
32
+ - name: Check code style (ruff)
33
+ run: python -m ruff check .
34
+
35
+ - name: Run tests
36
+ run: python -m pytest tests/ -q
@@ -0,0 +1,188 @@
1
+ # RAGLite Python (`raglite-toolkit`) Architecture
2
+
3
+ This document details the software architecture, component design, data flow, and APIs of the Python implementation of **RAGLite** (`raglite-toolkit`).
4
+
5
+ ---
6
+
7
+ ## 1. Overview & Core Philosophy
8
+
9
+ `raglite-toolkit` is a Python-native Retrieval-Augmented Generation library built with Pydantic v2 and FastAPI. It provides zero-boilerplate semantic search, multi-provider LLM response synthesis, and self-hosted REST APIs over local files.
10
+
11
+ ### Key Characteristics
12
+ * **Python-Native & Type-Annotated**: Modern Python 3.10+ typing with Pydantic v2 model validation and camelCase alias serialization.
13
+ * **1:1 Parity with TypeScript SDK**: Identical function signatures, option dictionaries, data models, and `.raglite/` persistence layout.
14
+ * **Pluggable & Extensible**: Modular Abstract Base Classes (ABCs) for Loaders, Chunkers, Embeddings, Vector Databases, and LLMs.
15
+ * **Per-Document Isolation**: Namespaced vector collections tied to source document path/identity.
16
+ * **SHA-256 Content Caching**: Automatically skips re-embedding if document content has not changed.
17
+ * **Offline-First Option**: Local offline embeddings via `sentence-transformers` (`all-MiniLM-L6-v2`) and local LLMs via Ollama.
18
+
19
+ ---
20
+
21
+ ## 2. Directory & Module Structure
22
+
23
+ ```
24
+ raglite-py/src/raglite/
25
+ ├── api/ # FastAPI REST HTTP server & Pydantic schemas
26
+ │ ├── __init__.py
27
+ │ ├── schemas.py
28
+ │ └── server.py
29
+ ├── chunking/ # Text splitting algorithms
30
+ │ ├── __init__.py
31
+ │ ├── base.py
32
+ │ └── recursive.py
33
+ ├── core/ # Main facade orchestrator
34
+ │ ├── __init__.py
35
+ │ └── document.py
36
+ ├── embeddings/ # Embedding provider factory & adapters
37
+ │ ├── __init__.py
38
+ │ ├── base.py
39
+ │ ├── factory.py
40
+ │ ├── local.py # sentence-transformers (offline)
41
+ │ ├── models.py
42
+ │ └── remote.py # OpenAI, Gemini, Mistral, Cohere, Voyage, Ollama
43
+ ├── llm/ # LLM provider factory & prompt synthesis
44
+ │ ├── __init__.py
45
+ │ ├── answer.py
46
+ │ ├── factory.py
47
+ │ ├── models.py
48
+ │ └── prompt.py
49
+ ├── loaders/ # Document parser implementations
50
+ │ ├── __init__.py
51
+ │ ├── base.py
52
+ │ ├── docx.py # python-docx reader
53
+ │ ├── json.py
54
+ │ ├── markdown.py
55
+ │ ├── pdf.py # pypdf / pymupdf reader
56
+ │ └── txt.py
57
+ ├── retrieval/ # Retriever & ranking algorithms
58
+ │ ├── __init__.py
59
+ │ └── retriever.py
60
+ ├── vectordb/ # Vector Database adapters
61
+ │ ├── __init__.py
62
+ │ ├── base.py # VectorStore Abstract Base Class
63
+ │ ├── factory.py
64
+ │ ├── memory.py # Local JSON persistence (.raglite/)
65
+ │ ├── pinecone.py # Pinecone Cloud
66
+ │ └── qdrant.py # Qdrant local/cloud
67
+ ├── cli.py # Command-line interface tool (Typer/Argparse)
68
+ ├── config.py # Environment & default configuration
69
+ ├── constants.py # System constants & defaults
70
+ ├── errors.py # Custom RAGLite exception definitions
71
+ ├── types.py # Pydantic v2 data models & type aliases
72
+ └── utils/ # Utility functions (hashing, math, crypto)
73
+ ```
74
+
75
+ ---
76
+
77
+ ## 3. High-Level System Diagram
78
+
79
+ ```mermaid
80
+ graph TD
81
+ subgraph Client Application
82
+ App["Python Application"]
83
+ CLI["raglite CLI"]
84
+ HTTPClient["Curl / HTTPX Client"]
85
+ end
86
+
87
+ subgraph Entrypoints
88
+ Doc["Document Facade Class<br/>(src/raglite/core/document.py)"]
89
+ FastAPIServer["FastAPI REST Server<br/>(src/raglite/api/server.py)"]
90
+ end
91
+
92
+ subgraph Core Pipeline Modules
93
+ Loaders["Document Loaders<br/>(src/raglite/loaders/)"]
94
+ Chunker["Recursive Splitter<br/>(src/raglite/chunking/)"]
95
+ Embeddings["Embeddings Engine<br/>(src/raglite/embeddings/)"]
96
+ VectorStore["VectorStore Adapter<br/>(src/raglite/vectordb/)"]
97
+ Retriever["Retriever Engine<br/>(src/raglite/retrieval/)"]
98
+ LLM["LLM Synthesis Engine<br/>(src/raglite/llm/)"]
99
+ end
100
+
101
+ subgraph Persistence Layer
102
+ JSONDisk["Disk Index Storage<br/>(.raglite/indexes/*.json)"]
103
+ ExternalVDB["External Vector DB<br/>(Qdrant / Pinecone)"]
104
+ end
105
+
106
+ App --> Doc
107
+ CLI --> Doc
108
+ HTTPClient --> FastAPIServer
109
+ FastAPIServer --> Doc
110
+
111
+ Doc --> Loaders
112
+ Doc --> Chunker
113
+ Doc --> Embeddings
114
+ Doc --> VectorStore
115
+ Doc --> Retriever
116
+ Doc --> LLM
117
+
118
+ VectorStore --> JSONDisk
119
+ VectorStore --> ExternalVDB
120
+ ```
121
+
122
+ ---
123
+
124
+ ## 4. End-to-End Data Pipeline
125
+
126
+ ### 4.1 Ingestion & Indexing
127
+ 1. `doc.build()` invokes the appropriate `BaseLoader` based on file extension (`.pdf`, `.txt`, `.json`, `.md`, `.docx`).
128
+ 2. Calculates SHA-256 hash of raw document content.
129
+ 3. Checks existing `IndexMetadata` in `VectorStore`. If hash matches, skips re-indexing.
130
+ 4. If hash differs or force rebuild requested:
131
+ - `RecursiveCharacterTextSplitter` chunks text (default size 1000, overlap 200).
132
+ - `EmbeddingFactory` generates normalized vectors for each chunk.
133
+ - `VectorStore.add()` saves chunks and `VectorStore.save_index_metadata()` persists index metadata.
134
+
135
+ ### 4.2 Retrieval & Answer Synthesis
136
+ 1. `doc.search(query, top_k)`:
137
+ - Embeds query text.
138
+ - Executes vector search via `VectorStore.search()`, calculating cosine similarity over L2-normalized vectors.
139
+ - Returns top $K$ `SearchResult` objects.
140
+ 2. `doc.ask(question, options)`:
141
+ - Runs `search(question)`.
142
+ - Constructs context-augmented system/user prompt via `build_prompt()`.
143
+ - Calls `generate_answer()` or `stream_answer()` with selected LLM adapter (OpenAI, Anthropic, Google, Groq, Ollama, etc.).
144
+
145
+ ---
146
+
147
+ ## 5. REST API Architecture
148
+
149
+ Built using **FastAPI** framework for async capabilities, automatic OpenAPI docs, and Uvicorn serving.
150
+
151
+ | Method | Endpoint | Auth Required? | Description |
152
+ | :--- | :--- | :--- | :--- |
153
+ | `GET` | `/health` | No | Liveness status & total chunk count |
154
+ | `GET` | `/info` | Yes (Bearer token) | Configuration details & index state |
155
+ | `POST` | `/search` | Yes (Bearer token) | Semantic vector search |
156
+ | `POST` | `/ask` | Yes (Bearer token) | Context Q&A generation (supports streaming via StreamingResponse) |
157
+
158
+ ---
159
+
160
+ ## 6. Vector Database Abstract Base Class (`src/raglite/vectordb/base.py`)
161
+
162
+ ```python
163
+ from abc import ABC, abstractmethod
164
+ from typing import List, Optional
165
+ from raglite.types import IndexMetadata, SearchResult, StoredChunk
166
+
167
+ class VectorStore(ABC):
168
+ @abstractmethod
169
+ def load(self, doc_id: str) -> None: ...
170
+
171
+ @abstractmethod
172
+ def reset(self, doc_id: str) -> None: ...
173
+
174
+ @abstractmethod
175
+ def add(self, doc_id: str, chunks: List[StoredChunk]) -> None: ...
176
+
177
+ @abstractmethod
178
+ def search(self, doc_id: str, query_vector: List[float], top_k: int, min_score: Optional[float] = None) -> List[SearchResult]: ...
179
+
180
+ @abstractmethod
181
+ def count(self, doc_id: str) -> int: ...
182
+
183
+ @abstractmethod
184
+ def save_index_metadata(self, doc_id: str, meta: IndexMetadata) -> None: ...
185
+
186
+ @abstractmethod
187
+ def read_index_metadata(self, doc_id: str) -> Optional[IndexMetadata]: ...
188
+ ```
@@ -0,0 +1,46 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ ## [1.2.0] - 2026-08-02
6
+
7
+ ### Added
8
+ - **Multi-Document & Directory Ingestion (`DocumentCollection`):**
9
+ - Added `DocumentCollection` class to manage semantic indexing, multi-document retrieval, and Q&A across folders, glob patterns, web URLs, and mixed file lists.
10
+ - Parallel semantic search over collection vector stores with score-based top-$K$ merging and ranking.
11
+ - Contextual Q&A synthesis (`ask` and `ask_stream`) across multi-document collections.
12
+ - **Directory Loader (`DirectoryLoader`):**
13
+ - Recursive directory scanner (`recursive=True`) with glob pattern matching (e.g. `./docs/**/*.md`).
14
+ - Auto-detection of supported extensions (`.pdf`, `.txt`, `.md`, `.json`, `.docx`).
15
+ - Detailed error reporting and warning logs for unsupported/empty files.
16
+ - **Web Loader (`WebLoader`):**
17
+ - Native loader for fetching HTTP/HTTPS web URLs directly.
18
+ - Automatic HTML cleaning into formatted text/markdown with script, style, and SVG tag stripping.
19
+ - JSON and plain text content-type parsing.
20
+ - **CLI & FastAPI REST Server Support:**
21
+ - Upgraded `raglite index`, `search`, `ask`, and `serve` CLI commands to process directories, glob patterns, and URLs.
22
+ - Updated FastAPI REST server to support `DocumentCollection` and single `Document` targets.
23
+ - **Unit & Integration Tests:**
24
+ - Added unit and integration tests for `DirectoryLoader`, `WebLoader`, and `DocumentCollection`.
25
+
26
+ ## [1.1.0] - 2026-07-19
27
+
28
+ ### Added
29
+ - **Pluggable Vector Databases:** Added support for custom local and cloud vector database backends via a new `vectorStore` config option.
30
+ - **Memory Store (`"memory"`):** Default in-memory store persisting indexes locally to JSON (unchanged behaviour).
31
+ - **Qdrant Store (`"qdrant"`):** Wrapper for Qdrant local/cloud using stdlib `urllib` REST requests. Supports auto-collection creation, API key auth, and custom collection names.
32
+ - **Pinecone Store (`"pinecone"`):** Cloud database support using Pinecone Namespaces and stdlib `urllib` REST requests. Stores index metadata as a reserved `__metadata__` vector.
33
+ - **Custom Adapters:** Pass any class instance implementing the `VectorStore` ABC directly as `vectorStore` in `DocumentOptions`.
34
+ - **`VectorStoreProviderConfig` type:** New Pydantic model in `types.py` describing provider, URL, API key, index name, and store directory.
35
+ - **Factory:** `create_vector_store(config, namespace)` utility in `vectordb/factory.py` resolving the correct store from config.
36
+ - **Examples:**
37
+ - `examples/qdrant_example.py` — full index + search demo with Qdrant.
38
+ - `examples/custom_store_example.py` — implementing and using a custom VectorStore ABC subclass.
39
+ - **Automation Scripts:**
40
+ - `scripts/pre-commit.sh` — runs ruff lint, mypy type-check, and pytest before committing.
41
+ - `scripts/pre-release.sh` — cleans builds, runs full verification, and builds distribution packages.
42
+ - **GitHub Actions Workflow:** `pr-verify.yml` — automatically runs code style checks and the full test suite on every pull request and push to `main`/`master`.
43
+
44
+ ### Changed
45
+ - **`DocumentOptions` / `ResolvedConfig`:** Added optional `vectorStore` field supporting `VectorStoreProviderConfig` or a custom `VectorStore` instance.
46
+ - **`Document.__init__`:** Constructor now resolves the appropriate vector store from config, accepting provider configs or custom instances.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: raglite-toolkit
3
- Version: 1.0.2
3
+ Version: 1.2.0
4
4
  Summary: Build semantic search, multi-provider question answering, and REST APIs over your documents in a few lines of Python.
5
5
  Project-URL: Homepage, https://github.com/creatorpiyush/raglite-py
6
6
  Project-URL: Repository, https://github.com/creatorpiyush/raglite-py
@@ -40,20 +40,22 @@ Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
40
40
  Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
41
41
  Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
42
42
  Requires-Dist: pytest>=7.0.0; extra == 'dev'
43
+ Requires-Dist: ruff>=0.4.0; extra == 'dev'
43
44
  Requires-Dist: twine>=5.0.0; extra == 'dev'
44
45
  Description-Content-Type: text/markdown
45
46
 
46
47
  # raglite-toolkit
47
48
 
48
- > Build semantic search, multi-provider question answering, and REST APIs over your documents in a few lines of Python.
49
+ > Build semantic search, multi-provider question answering, and REST APIs over your documents, directories, or web URLs in a few lines of Python.
49
50
 
50
- **raglite-toolkit** is a Python port of the [`raglite-toolkit`](https://github.com/creatorpiyush/raglite) TypeScript package with full feature parity.
51
+ **raglite-toolkit** is a Python port of the [`raglite-toolkit`](https://github.com/creatorpiyush/raglite) TypeScript package with full 1:1 feature parity.
51
52
 
52
53
  ---
53
54
 
54
55
  ## Features
55
56
 
56
57
  - 📄 **PDF, TXT, JSON, Markdown, DOCX** loaders out of the box
58
+ - 📁 **Multi-document, directory, & URL ingestion** — index folders, glob patterns, or web URLs with `DocumentCollection`
57
59
  - 🤖 **Multi-provider LLMs** — OpenAI, Anthropic (Claude), Google (Gemini), Mistral, Cohere, Groq, xAI, Ollama
58
60
  - 🔢 **Multi-provider embeddings** — OpenAI, Google, Mistral, Cohere, Voyage, Ollama, or a **local** offline sentence-transformer (no API key needed)
59
61
  - 📐 **Cosine similarity** scoring with L2-normalized vectors
@@ -101,20 +103,44 @@ print(answer.text)
101
103
 
102
104
  ---
103
105
 
106
+ ## Multi-Document & Directory Ingestion (`DocumentCollection`)
107
+
108
+ Index entire directories (`./docs`), glob patterns, web URLs, or mixed file arrays:
109
+
110
+ ```python
111
+ from raglite import DocumentCollection
112
+
113
+ collection = DocumentCollection(["./docs", "https://example.com"], {
114
+ "embeddings": {"provider": "local"},
115
+ "llm": {"provider": "openai", "apiKey": "sk-..."},
116
+ })
117
+
118
+ # Index all documents concurrently
119
+ result = collection.build()
120
+ print(f"Indexed {result.totalDocuments} document(s), {result.totalChunks} chunk(s).")
121
+
122
+ # Search across all collection documents simultaneously
123
+ hits = collection.search("refund policy", top_k=5)
124
+
125
+ # Contextual Q&A across the entire collection
126
+ answer = collection.ask("What is the refund policy?")
127
+ print(answer.text)
128
+ ```
129
+
130
+ ---
131
+
104
132
  ## Fully Offline — No API Key Needed
105
133
 
106
134
  ```python
107
135
  from raglite import Document
108
136
 
109
- doc = Document("./policy.pdf", {
110
- "embeddings": {"provider": "local"}, # sentence-transformers offline
111
- "llm": {"provider": "ollama", # local Ollama instance
112
- "model": "llama3.2",
113
- "baseURL": "http://localhost:11434/api"},
137
+ doc = Document("./manual.txt", {
138
+ "embeddings": {"provider": "local"},
139
+ "llm": {"provider": "ollama", "model": "llama3.2"},
114
140
  })
115
141
 
116
142
  doc.build()
117
- print(doc.ask("What is the refund policy?").text)
143
+ print(doc.ask("How do I reset the device?").text)
118
144
  ```
119
145
 
120
146
  ---
@@ -122,24 +148,60 @@ print(doc.ask("What is the refund policy?").text)
122
148
  ## Choose Any LLM at Ask-Time
123
149
 
124
150
  ```python
125
- # Switch LLMs without rebuilding the index — embeddings are reused
126
- for llm in [
127
- {"provider": "openai", "model": "gpt-4o", "apiKey": "sk-..."},
128
- {"provider": "anthropic", "model": "claude-3-5-sonnet-20241022","apiKey": "sk-ant-..."},
129
- {"provider": "google", "model": "gemini-2.0-flash", "apiKey": "AI..."},
130
- {"provider": "groq", "model": "llama-3.3-70b-versatile", "apiKey": "gsk_..."},
131
- ]:
132
- answer = doc.ask("Summarize this document", {"llm": llm})
133
- print(f"[{llm['provider']}] {answer.text[:120]}")
151
+ # Pass an inline LLM override to ask()
152
+ gpt4 = doc.ask("Summarise this document", options={
153
+ "llm": {"provider": "openai", "model": "gpt-4o", "apiKey": "sk-..."}
154
+ })
155
+
156
+ claude = doc.ask("Summarise this document", options={
157
+ "llm": {"provider": "anthropic", "model": "claude-3-5-sonnet-20241022", "apiKey": "sk-ant-..."}
158
+ })
134
159
  ```
135
160
 
136
161
  ---
137
162
 
138
- ## Streaming
163
+ ## Streaming Responses
139
164
 
140
165
  ```python
141
- for chunk in doc.ask_stream("Explain the introduction"):
166
+ for chunk in doc.ask_stream("Explain section 3 in detail"):
142
167
  print(chunk, end="", flush=True)
168
+ print()
169
+ ```
170
+
171
+ ---
172
+
173
+ ## Pluggable Vector Databases
174
+
175
+ `raglite` supports pluggable vector stores (Memory, Qdrant, Pinecone, LanceDB, or custom subclasses):
176
+
177
+ ### Memory Store (Default)
178
+ ```python
179
+ doc = Document("./policy.pdf", {
180
+ "vectorStore": {"provider": "memory", "storeDir": ".raglite"}
181
+ })
182
+ ```
183
+
184
+ ### Qdrant Store
185
+ ```python
186
+ doc = Document("./policy.pdf", {
187
+ "vectorStore": {
188
+ "provider": "qdrant",
189
+ "url": "http://localhost:6333",
190
+ "apiKey": "your-key",
191
+ "indexName": "my_collection"
192
+ }
193
+ })
194
+ ```
195
+
196
+ ### Pinecone Store
197
+ ```python
198
+ doc = Document("./policy.pdf", {
199
+ "vectorStore": {
200
+ "provider": "pinecone",
201
+ "url": "https://my-index.svc.pinecone.io",
202
+ "apiKey": "your-key"
203
+ }
204
+ })
143
205
  ```
144
206
 
145
207
  ---
@@ -147,25 +209,26 @@ for chunk in doc.ask_stream("Explain the introduction"):
147
209
  ## REST API
148
210
 
149
211
  ```python
150
- handle = doc.serve({
151
- "port": 8085,
152
- "llm": {"provider": "openai", "apiKey": "sk-..."},
153
- "bearerToken": "my-secret-token",
212
+ from raglite import Document
213
+
214
+ doc = Document("./policy.pdf", {
215
+ "embeddings": {"provider": "local"},
216
+ "llm": {"provider": "openai", "apiKey": "sk-..."},
154
217
  })
155
- print(f"Serving on {handle.url}")
218
+ doc.build()
156
219
 
157
- # ... later
158
- handle.close()
220
+ # Start background FastAPI server on port 8085
221
+ doc.serve(port=8085, bearer_token="secret-token")
159
222
  ```
160
223
 
161
- ### Endpoints
224
+ Endpoints:
162
225
 
163
- | Method | Path | Auth? | Description |
164
- |--------|-----------|-------|-------------|
165
- | `GET` | `/health` | ❌ | Liveness + index stats |
166
- | `GET` | `/info` | ✅ | Configuration snapshot |
167
- | `POST` | `/search` | ✅ | Semantic search |
168
- | `POST` | `/ask` | ✅ | Question answering |
226
+ | Method | Path | Auth required? | Description |
227
+ | ------ | ---- | -------------- | ----------- |
228
+ | `GET` | `/health` | ❌ | Liveness + index stats |
229
+ | `GET` | `/info` | ✅ | Configuration snapshot |
230
+ | `POST` | `/search` | ✅ | Semantic search |
231
+ | `POST` | `/ask` | ✅ | Question answering (supports `stream: true`) |
169
232
 
170
233
  ### Example `curl` calls
171
234
 
@@ -175,19 +238,19 @@ curl http://127.0.0.1:8085/health
175
238
 
176
239
  # Search
177
240
  curl -X POST http://127.0.0.1:8085/search \
178
- -H 'Authorization: Bearer my-secret-token' \
241
+ -H 'Authorization: Bearer secret-token' \
179
242
  -H 'Content-Type: application/json' \
180
243
  -d '{"query": "refund policy", "topK": 3}'
181
244
 
182
245
  # Ask (non-streaming)
183
246
  curl -X POST http://127.0.0.1:8085/ask \
184
- -H 'Authorization: Bearer my-secret-token' \
247
+ -H 'Authorization: Bearer secret-token' \
185
248
  -H 'Content-Type: application/json' \
186
249
  -d '{"question": "What is the refund policy?"}'
187
250
 
188
251
  # Ask (streaming)
189
252
  curl -X POST http://127.0.0.1:8085/ask \
190
- -H 'Authorization: Bearer my-secret-token' \
253
+ -H 'Authorization: Bearer secret-token' \
191
254
  -H 'Content-Type: application/json' \
192
255
  -d '{"question": "Summarize the document", "stream": true}'
193
256
  ```
@@ -197,18 +260,18 @@ curl -X POST http://127.0.0.1:8085/ask \
197
260
  ## CLI
198
261
 
199
262
  ```bash
200
- # Index a document
263
+ # Index a document, directory, or URL
201
264
  raglite index ./policy.pdf --embed-provider local
202
265
 
203
266
  # Semantic search
204
- raglite search ./policy.pdf "refund policy" --top-k 5
267
+ raglite search ./docs "refund policy" --top-k 5
205
268
 
206
269
  # Ask a question (streaming)
207
- raglite ask ./policy.pdf "What is the refund policy?" \
208
- --llm-provider openai --llm-key $OPENAI_API_KEY --stream
270
+ raglite ask ./docs "What is the refund policy?" \
271
+ --llm-provider anthropic --llm-key $ANTHROPIC_API_KEY --stream
209
272
 
210
273
  # Serve a REST API
211
- raglite serve ./policy.pdf \
274
+ raglite serve https://example.com \
212
275
  --llm-provider openai --llm-key $OPENAI_API_KEY \
213
276
  --port 8085 --token $RAGLITE_TOKEN
214
277
  ```
@@ -339,7 +402,7 @@ query_vec = embedder.embed_query("refund policy")
339
402
 
340
403
  ```bash
341
404
  # Clone and set up
342
- git clone <repo>
405
+ git clone https://github.com/creatorpiyush/raglite-py.git
343
406
  cd raglite-py
344
407
 
345
408
  # Create virtual environment
@@ -349,15 +412,14 @@ source .venv/bin/activate # Windows: .venv\Scripts\activate
349
412
  # Install in editable mode with dev dependencies
350
413
  pip install -e ".[dev]"
351
414
 
352
- # Run tests (76 tests, ~5s, no network required)
415
+ # Run test suite
353
416
  pytest
354
417
 
355
418
  # Run with coverage
356
419
  pytest --cov=raglite --cov-report=term-missing
357
420
 
358
- # Run examples (uses local offline embeddings)
421
+ # Run examples
359
422
  python examples/basic.py
360
- python examples/multi_provider.py # requires API keys in env
361
423
  python examples/serve.py
362
424
  ```
363
425
 
@@ -366,23 +428,26 @@ python examples/serve.py
366
428
  ```
367
429
  tests/
368
430
  ├── unit/
369
- │ ├── test_chunking.py # RecursiveChunker algorithm
370
- │ ├── test_vectordb.py # MemoryVectorStore (cosine, persistence, isolation)
371
- │ ├── test_loaders.py # TxtLoader, MarkdownLoader, JsonLoader
372
- │ ├── test_prompt.py # system/user prompt builders
373
- │ ├── test_errors.py # exception hierarchy
374
- │ ├── test_config.py # config defaults and overrides
375
- │ ├── test_hash.py # SHA-256 file hashing + namespace generation
376
- │ ├── test_retriever.py # Retriever with mocked embedder
377
- │ └── test_cli.py # CLI commands and argument parsing
431
+ │ ├── test_chunking.py # RecursiveChunker algorithm
432
+ │ ├── test_vectordb.py # MemoryVectorStore (cosine, persistence, isolation)
433
+ │ ├── test_loaders.py # TxtLoader, MarkdownLoader, JsonLoader
434
+ │ ├── test_directory_loader.py # DirectoryLoader (recursive scanning, glob filtering)
435
+ │ ├── test_web_loader.py # WebLoader (HTML parsing, tag stripping)
436
+ │ ├── test_prompt.py # system/user prompt builders
437
+ │ ├── test_errors.py # exception hierarchy
438
+ │ ├── test_config.py # config defaults and overrides
439
+ │ ├── test_hash.py # SHA-256 file hashing + namespace generation
440
+ │ ├── test_retriever.py # Retriever with mocked embedder
441
+ │ └── test_cli.py # CLI commands and argument parsing
378
442
  └── integration/
379
- ├── test_document.py # build/cache/search lifecycle (mocked embeddings)
380
- ├── test_ask.py # ask/stream with mocked LLM generation
381
- └── test_api.py # FastAPI endpoints via TestClient
443
+ ├── test_document.py # build/cache/search lifecycle (mocked embeddings)
444
+ ├── test_collection.py # DocumentCollection multi-document indexing & FastAPI server
445
+ ├── test_ask.py # ask/stream with mocked LLM generation
446
+ └── test_api.py # FastAPI endpoints via TestClient
382
447
  ```
383
448
 
384
449
  ---
385
450
 
386
451
  ## License
387
452
 
388
- MIT
453
+ MIT © [Piyush Anand](https://github.com/creatorpiyush)