ragwire 1.0.0__tar.gz

ragwire-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 KGP Talkie Private Limited
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ragwire-1.0.0/PKG-INFO

@@ -0,0 +1,233 @@
+Metadata-Version: 2.4
+Name: ragwire
+Version: 1.0.0
+Summary: RAGWire — Production-grade RAG toolkit for document ingestion and retrieval with hybrid search support
+Author-email: KGP Talkie Private Limited <udemy@kgptalkie.com>
+Maintainer-email: KGP Talkie Private Limited <udemy@kgptalkie.com>
+License-Expression: MIT
+Project-URL: Homepage, https://kgptalkie.com
+Project-URL: Documentation, https://github.com/laxmimerit/ragwire#readme
+Project-URL: Repository, https://github.com/laxmimerit/ragwire.git
+Project-URL: Issues, https://github.com/laxmimerit/ragwire/issues
+Project-URL: YouTube, https://youtube.com/kgptalkie
+Keywords: rag,retrieval,vector-database,qdrant,embeddings,hybrid-search,nlp,document-processing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: langchain>=0.1.0
+Requires-Dist: langchain-core>=0.1.0
+Requires-Dist: langchain-community>=0.0.0
+Requires-Dist: langchain-text-splitters>=0.0.1
+Requires-Dist: qdrant-client>=1.6.0
+Requires-Dist: langchain-qdrant>=0.1.0
+Requires-Dist: markitdown[pdf]>=0.0.1
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: tqdm>=4.66.0
+Provides-Extra: openai
+Requires-Dist: langchain-openai>=0.0.0; extra == "openai"
+Provides-Extra: huggingface
+Requires-Dist: langchain-huggingface>=0.0.0; extra == "huggingface"
+Provides-Extra: ollama
+Requires-Dist: langchain-ollama>=0.0.0; extra == "ollama"
+Provides-Extra: google
+Requires-Dist: langchain-google-genai>=0.0.0; extra == "google"
+Provides-Extra: fastembed
+Requires-Dist: fastembed>=0.2.0; extra == "fastembed"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.0.0; extra == "dev"
+Requires-Dist: flake8>=6.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: ragwire[openai]; extra == "all"
+Requires-Dist: ragwire[huggingface]; extra == "all"
+Requires-Dist: ragwire[ollama]; extra == "all"
+Requires-Dist: ragwire[google]; extra == "all"
+Requires-Dist: ragwire[fastembed]; extra == "all"
+Requires-Dist: ragwire[dev]; extra == "all"
+Dynamic: license-file
+
+<p align="center">
+  <img src="assets/ragwire.png" alt="RAGWire logo" width="120"/>
+</p>
+
+<h1 align="center">RAGWire</h1>
+<p align="center">Production-grade RAG toolkit for document ingestion and retrieval</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/ragwire"><img src="https://img.shields.io/pypi/v/ragwire" alt="PyPI"/></a>
+  <a href="https://github.com/laxmimerit/ragwire/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License"/></a>
+  <a href="https://youtube.com/kgptalkie"><img src="https://img.shields.io/badge/YouTube-KGP%20Talkie-red" alt="YouTube"/></a>
+</p>
+
+---
+
+## Features
+
+- **Document Loading** — PDF, DOCX, XLSX, PPTX and more via MarkItDown
+- **LLM Metadata Extraction** — extracts company, doc type, fiscal period using your LLM
+- **Smart Text Splitting** — markdown-aware and recursive chunking strategies
+- **Multiple Embedding Providers** — Ollama, OpenAI, HuggingFace, Google, FastEmbed
+- **Qdrant Vector Store** — dense, sparse, and hybrid search
+- **Advanced Retrieval** — similarity, MMR, and hybrid search
+- **SHA256 Deduplication** — at both file and chunk level
+
+## Installation
+
+```bash
+pip install ragwire
+
+# With Ollama support (local, no API key)
+pip install "ragwire[ollama]"
+
+# With all providers
+pip install "ragwire[all]"
+```
+
+## Quick Start
+
+```python
+from ragwire import RAGPipeline
+
+pipeline = RAGPipeline("config.yaml")
+
+# Ingest documents
+stats = pipeline.ingest_documents(["data/Apple_10k_2025.pdf"])
+print(f"Chunks created: {stats['chunks_created']}")
+
+# Retrieve
+results = pipeline.retrieve("What is Apple's total revenue?", top_k=5)
+for doc in results:
+    print(doc.metadata.get("company_name"), doc.page_content[:200])
+```
+
+## Configuration
+
+Copy `config.example.yaml` to `config.yaml` and edit:
+
+```yaml
+embeddings:
+  provider: "ollama"
+  model: "qwen3-embedding:0.6b"
+  base_url: "http://localhost:11434"
+
+llm:
+  provider: "ollama"
+  model: "qwen3.5:9b"
+  temperature: 0.0
+  num_ctx: 16384
+
+vectorstore:
+  url: "http://localhost:6333"
+  collection_name: "my_docs"
+  use_sparse: true
+
+retriever:
+  search_type: "hybrid"
+  top_k: 5
+```
+
+## Embedding Providers
+
+```yaml
+# Ollama (local)
+embeddings:
+  provider: "ollama"
+  model: "qwen3-embedding:0.6b"
+
+# OpenAI
+embeddings:
+  provider: "openai"
+  model: "text-embedding-3-small"
+
+# HuggingFace (local)
+embeddings:
+  provider: "huggingface"
+  model_name: "sentence-transformers/all-MiniLM-L6-v2"
+
+# Google
+embeddings:
+  provider: "google"
+  model: "models/embedding-001"
+```
+
+## Component Usage
+
+```python
+from ragwire import (
+    MarkItDownLoader,
+    get_splitter,
+    get_markdown_splitter,
+    get_embedding,
+    QdrantStore,
+    MetadataExtractor,
+    hybrid_search,
+    mmr_search,
+)
+
+# Load a document
+loader = MarkItDownLoader()
+result = loader.load("document.pdf")
+
+# Split text
+splitter = get_markdown_splitter(chunk_size=10000, chunk_overlap=2000)
+chunks = splitter.split_text(result["text_content"])
+
+# Embeddings
+embedding = get_embedding({"provider": "ollama", "model": "qwen3-embedding:0.6b"})
+
+# Vector store
+store = QdrantStore(config={"url": "http://localhost:6333"}, embedding=embedding)
+store.set_collection("my_collection")
+vectorstore = store.get_store()
+```
+
+## Architecture
+
+```
+ragwire/
+├── core/          # Config loader + RAGPipeline orchestrator
+├── loaders/       # MarkItDown document converter
+├── processing/    # Text splitters + SHA256 hashing
+├── metadata/      # Pydantic schema + LLM extractor
+├── embeddings/    # Multi-provider embedding factory
+├── vectorstores/  # Qdrant wrapper with hybrid search
+├── retriever/     # Similarity, MMR, hybrid retrieval
+└── utils/         # Logging
+```
+
+## Troubleshooting
+
+| Error | Fix |
+|-------|-----|
+| Qdrant connection refused | `docker run -p 6333:6333 qdrant/qdrant` |
+| `markitdown[pdf]` missing | `pip install "markitdown[pdf]"` |
+| Ollama model not found | `ollama pull <model-name>` |
+| `fastembed` missing | `pip install fastembed` (needed for hybrid search) |
+| Embedding dimension mismatch | Set `force_recreate: true` in config once, then back to `false` |
+
+## License
+
+MIT © 2026 [KGP Talkie Private Limited](https://kgptalkie.com)
+
+## Links
+
+- Website: [kgptalkie.com](https://kgptalkie.com)
+- YouTube: [youtube.com/kgptalkie](https://youtube.com/kgptalkie)
+- GitHub: [github.com/laxmimerit/ragwire](https://github.com/laxmimerit/ragwire)
+- Email: udemy@kgptalkie.com

ragwire-1.0.0/README.md

@@ -0,0 +1,169 @@
+<p align="center">
+  <img src="assets/ragwire.png" alt="RAGWire logo" width="120"/>
+</p>
+
+<h1 align="center">RAGWire</h1>
+<p align="center">Production-grade RAG toolkit for document ingestion and retrieval</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/ragwire"><img src="https://img.shields.io/pypi/v/ragwire" alt="PyPI"/></a>
+  <a href="https://github.com/laxmimerit/ragwire/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License"/></a>
+  <a href="https://youtube.com/kgptalkie"><img src="https://img.shields.io/badge/YouTube-KGP%20Talkie-red" alt="YouTube"/></a>
+</p>
+
+---
+
+## Features
+
+- **Document Loading** — PDF, DOCX, XLSX, PPTX and more via MarkItDown
+- **LLM Metadata Extraction** — extracts company, doc type, fiscal period using your LLM
+- **Smart Text Splitting** — markdown-aware and recursive chunking strategies
+- **Multiple Embedding Providers** — Ollama, OpenAI, HuggingFace, Google, FastEmbed
+- **Qdrant Vector Store** — dense, sparse, and hybrid search
+- **Advanced Retrieval** — similarity, MMR, and hybrid search
+- **SHA256 Deduplication** — at both file and chunk level
+
+## Installation
+
+```bash
+pip install ragwire
+
+# With Ollama support (local, no API key)
+pip install "ragwire[ollama]"
+
+# With all providers
+pip install "ragwire[all]"
+```
+
+## Quick Start
+
+```python
+from ragwire import RAGPipeline
+
+pipeline = RAGPipeline("config.yaml")
+
+# Ingest documents
+stats = pipeline.ingest_documents(["data/Apple_10k_2025.pdf"])
+print(f"Chunks created: {stats['chunks_created']}")
+
+# Retrieve
+results = pipeline.retrieve("What is Apple's total revenue?", top_k=5)
+for doc in results:
+    print(doc.metadata.get("company_name"), doc.page_content[:200])
+```
+
+## Configuration
+
+Copy `config.example.yaml` to `config.yaml` and edit:
+
+```yaml
+embeddings:
+  provider: "ollama"
+  model: "qwen3-embedding:0.6b"
+  base_url: "http://localhost:11434"
+
+llm:
+  provider: "ollama"
+  model: "qwen3.5:9b"
+  temperature: 0.0
+  num_ctx: 16384
+
+vectorstore:
+  url: "http://localhost:6333"
+  collection_name: "my_docs"
+  use_sparse: true
+
+retriever:
+  search_type: "hybrid"
+  top_k: 5
+```
+
+## Embedding Providers
+
+```yaml
+# Ollama (local)
+embeddings:
+  provider: "ollama"
+  model: "qwen3-embedding:0.6b"
+
+# OpenAI
+embeddings:
+  provider: "openai"
+  model: "text-embedding-3-small"
+
+# HuggingFace (local)
+embeddings:
+  provider: "huggingface"
+  model_name: "sentence-transformers/all-MiniLM-L6-v2"
+
+# Google
+embeddings:
+  provider: "google"
+  model: "models/embedding-001"
+```
+
+## Component Usage
+
+```python
+from ragwire import (
+    MarkItDownLoader,
+    get_splitter,
+    get_markdown_splitter,
+    get_embedding,
+    QdrantStore,
+    MetadataExtractor,
+    hybrid_search,
+    mmr_search,
+)
+
+# Load a document
+loader = MarkItDownLoader()
+result = loader.load("document.pdf")
+
+# Split text
+splitter = get_markdown_splitter(chunk_size=10000, chunk_overlap=2000)
+chunks = splitter.split_text(result["text_content"])
+
+# Embeddings
+embedding = get_embedding({"provider": "ollama", "model": "qwen3-embedding:0.6b"})
+
+# Vector store
+store = QdrantStore(config={"url": "http://localhost:6333"}, embedding=embedding)
+store.set_collection("my_collection")
+vectorstore = store.get_store()
+```
+
+## Architecture
+
+```
+ragwire/
+├── core/          # Config loader + RAGPipeline orchestrator
+├── loaders/       # MarkItDown document converter
+├── processing/    # Text splitters + SHA256 hashing
+├── metadata/      # Pydantic schema + LLM extractor
+├── embeddings/    # Multi-provider embedding factory
+├── vectorstores/  # Qdrant wrapper with hybrid search
+├── retriever/     # Similarity, MMR, hybrid retrieval
+└── utils/         # Logging
+```
+
+## Troubleshooting
+
+| Error | Fix |
+|-------|-----|
+| Qdrant connection refused | `docker run -p 6333:6333 qdrant/qdrant` |
+| `markitdown[pdf]` missing | `pip install "markitdown[pdf]"` |
+| Ollama model not found | `ollama pull <model-name>` |
+| `fastembed` missing | `pip install fastembed` (needed for hybrid search) |
+| Embedding dimension mismatch | Set `force_recreate: true` in config once, then back to `false` |
+
+## License
+
+MIT © 2026 [KGP Talkie Private Limited](https://kgptalkie.com)
+
+## Links
+
+- Website: [kgptalkie.com](https://kgptalkie.com)
+- YouTube: [youtube.com/kgptalkie](https://youtube.com/kgptalkie)
+- GitHub: [github.com/laxmimerit/ragwire](https://github.com/laxmimerit/ragwire)
+- Email: udemy@kgptalkie.com

ragwire-1.0.0/pyproject.toml

@@ -0,0 +1,150 @@
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ragwire"
+version = "1.0.0"
+description = "RAGWire — Production-grade RAG toolkit for document ingestion and retrieval with hybrid search support"
+readme = "README.md"
+license = "MIT"
+authors = [
+    {name = "KGP Talkie Private Limited", email = "udemy@kgptalkie.com"}
+]
+maintainers = [
+    {name = "KGP Talkie Private Limited", email = "udemy@kgptalkie.com"}
+]
+keywords = [
+    "rag",
+    "retrieval",
+    "vector-database",
+    "qdrant",
+    "embeddings",
+    "hybrid-search",
+    "nlp",
+    "document-processing"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Text Processing :: Linguistic",
+]
+requires-python = ">=3.9"
+dependencies = [
+    # Core dependencies
+    "langchain>=0.1.0",
+    "langchain-core>=0.1.0",
+    "langchain-community>=0.0.0",
+    "langchain-text-splitters>=0.0.1",
+    
+    # Vector database
+    "qdrant-client>=1.6.0",
+    "langchain-qdrant>=0.1.0",
+    
+    # Document processing
+    "markitdown[pdf]>=0.0.1",
+    
+    # Configuration
+    "pyyaml>=6.0",
+    "python-dotenv>=1.0.0",
+    
+    # Data validation
+    "pydantic>=2.0.0",
+    
+    # Progress bars
+    "tqdm>=4.66.0",
+]
+
+[project.optional-dependencies]
+# OpenAI embeddings
+openai = [
+    "langchain-openai>=0.0.0",
+]
+
+# HuggingFace embeddings
+huggingface = [
+    "langchain-huggingface>=0.0.0",
+]
+
+# Ollama embeddings
+ollama = [
+    "langchain-ollama>=0.0.0",
+]
+
+# Google/Gemini embeddings and LLM
+google = [
+    "langchain-google-genai>=0.0.0",
+]
+
+# FastEmbed for sparse vectors
+fastembed = [
+    "fastembed>=0.2.0",
+]
+
+# Development dependencies
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "black>=23.0.0",
+    "isort>=5.0.0",
+    "flake8>=6.0.0",
+    "mypy>=1.0.0",
+    "pre-commit>=3.0.0",
+]
+
+# All dependencies
+all = [
+    "ragwire[openai]",
+    "ragwire[huggingface]",
+    "ragwire[ollama]",
+    "ragwire[google]",
+    "ragwire[fastembed]",
+    "ragwire[dev]",
+]
+
+[project.urls]
+Homepage = "https://kgptalkie.com"
+Documentation = "https://github.com/laxmimerit/ragwire#readme"
+Repository = "https://github.com/laxmimerit/ragwire.git"
+Issues = "https://github.com/laxmimerit/ragwire/issues"
+YouTube = "https://youtube.com/kgptalkie"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["ragwire*"]
+
+[tool.setuptools.package-data]
+ragwire = ["py.typed"]
+
+[tool.setuptools.data-files]
+"assets" = ["assets/ragwire.png"]
+
+[tool.black]
+line-length = 88
+target-version = ['py39', 'py310', 'py311', 'py312']
+include = '\.pyi?$'
+
+[tool.isort]
+profile = "black"
+line_length = 88
+multi_line_output = 3
+
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = false
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = "test_*.py"
+python_classes = "Test*"
+python_functions = "test_*"
+addopts = "-v --cov=ragwire --cov-report=term-missing"

ragwire-1.0.0/ragwire/__init__.py

@@ -0,0 +1,73 @@
+"""
+RAGWire — Production-grade RAG toolkit for document ingestion and retrieval.
+
+A clean, installable Python toolkit providing:
+- Document loading and conversion (PDF, DOCX, XLSX, etc.)
+- Text splitting and chunking
+- LLM-based metadata extraction
+- Multiple embedding provider support (OpenAI, HuggingFace, Ollama, Google)
+- Qdrant vector store with hybrid search
+- Advanced retrieval strategies (similarity, MMR, hybrid)
+
+Example:
+    >>> from ragwire import RAGPipeline
+    >>>
+    >>> pipeline = RAGPipeline("config.yaml")
+    >>> stats = pipeline.ingest_documents(["doc.pdf"])
+    >>> results = pipeline.retrieve("What is the revenue?")
+    >>> for doc in results:
+    ...     print(doc.page_content)
+
+Author: KGP Talkie Private Limited
+License: MIT
+"""
+
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("ragwire")
+except PackageNotFoundError:
+    __version__ = "1.0.0"
+
+__author__ = "KGP Talkie Private Limited"
+
+from .core.config import Config
+from .core.pipeline import RAGPipeline
+from .metadata.schema import DocumentMetadata
+from .metadata.extractor import MetadataExtractor
+from .loaders.markitdown_loader import MarkItDownLoader
+from .processing.splitter import get_splitter, get_markdown_splitter, get_code_splitter
+from .processing.hashing import sha256_text, sha256_file_from_path, sha256_chunk
+from .embeddings.factory import get_embedding
+from .vectorstores.qdrant_store import QdrantStore
+from .retriever.hybrid import get_retriever, hybrid_search, mmr_search
+from .utils.logging import setup_logging, get_logger
+
+__all__ = [
+    # Core
+    "Config",
+    "RAGPipeline",
+    # Metadata
+    "DocumentMetadata",
+    "MetadataExtractor",
+    # Loaders
+    "MarkItDownLoader",
+    # Processing
+    "get_splitter",
+    "get_markdown_splitter",
+    "get_code_splitter",
+    "sha256_text",
+    "sha256_file_from_path",
+    "sha256_chunk",
+    # Embeddings
+    "get_embedding",
+    # Vector Stores
+    "QdrantStore",
+    # Retrieval
+    "get_retriever",
+    "hybrid_search",
+    "mmr_search",
+    # Utilities
+    "setup_logging",
+    "get_logger",
+]

ragwire-1.0.0/ragwire/core/__init__.py

@@ -0,0 +1,6 @@
+"""Core module for RAG pipeline."""
+
+from .config import Config
+from .pipeline import RAGPipeline
+
+__all__ = ["Config", "RAGPipeline"]