easy-knowledge-retriever 0.1.0__tar.gz

easy_knowledge_retriever-0.1.0/LICENSE

@@ -0,0 +1,18 @@
+Easy Knowledge Retriever — License
+
+Copyright (c) 2025 Thomas Martinet
+
+This work is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License (CC BY-NC-SA 4.0).
+
+You are free to share and adapt the material under the following terms:
+- Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made.
+- NonCommercial — You may not use the material for commercial purposes.
+- ShareAlike — If you remix, transform, or build upon the material, you must distribute your contributions under the same license as the original.
+
+No additional restrictions — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.
+
+Full license text:
+https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode
+
+Plain‑language summary (EN):
+https://creativecommons.org/licenses/by-nc-sa/4.0/

easy_knowledge_retriever-0.1.0/MANIFEST.in

@@ -0,0 +1,3 @@
+include requirements.txt
+include README.md
+include LICENSE*

easy_knowledge_retriever-0.1.0/PKG-INFO

@@ -0,0 +1,255 @@
+Metadata-Version: 2.4
+Name: easy-knowledge-retriever
+Version: 0.1.0
+Summary: A simple and efficient RAG (Retrieval-Augmented Generation) library with Knowledge Graph support.
+Author: Thomas Martinet
+Author-email: your.email@example.com
+License: CC BY-NC-SA 4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: Free for non-commercial use
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: asyncpg<1.0.0,>=0.29.0
+Requires-Dist: httpx
+Requires-Dist: networkx
+Requires-Dist: numpy
+Requires-Dist: tenacity
+Requires-Dist: tiktoken
+Requires-Dist: openai<3.0.0,>=2.0.0
+Requires-Dist: neo4j<7.0.0,>=5.0.0
+Requires-Dist: pymilvus>=2.6.2
+Requires-Dist: nano-vectordb
+Requires-Dist: pypdf
+Requires-Dist: mineru
+Requires-Dist: doclayout_yolo
+Requires-Dist: ultralytics
+Requires-Dist: ftfy
+Requires-Dist: shapely
+Requires-Dist: pyclipper
+Requires-Dist: omegaconf
+Requires-Dist: pypinyin
+Requires-Dist: dill
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+<div align="center">
+<img src="docs/logo.png" alt="EKR Logo" width="400">
+</div>
+
+<div align="center">
+    <h1>Easy Knowledge Retriever - The easiest RAG lib ever</h1>
+</div>
+
+[![PyPI - Version](https://img.shields.io/pypi/v/easy-knowledge-retriever.svg)](https://pypi.org/project/easy-knowledge-retriever/) [![Python Versions](https://img.shields.io/pypi/pyversions/easy-knowledge-retriever.svg)](https://pypi.org/project/easy-knowledge-retriever/) [![License: CC BY-NC-SA 4.0](https://img.shields.io/badge/license-CC%20BY--NC--SA%204.0-lightgrey.svg)](https://creativecommons.org/licenses/by-nc-sa/4.0/) [![Docs](https://img.shields.io/badge/docs-GitHub%20Pages-blue.svg)](https://<YOUR_GITHUB_USERNAME>.github.io/EasyKnowledgeRetrieval/)
+
+**Easy Knowledge Retriever** is a powerful and flexible library for building Retrieval-Augmented Generation (RAG) systems with integrated Knowledge Graph support. It allows you to easily ingest documents, build a structured knowledge base (combining vector embeddings and graph relations), and perform advanced queries using Large Language Models (LLMs).
+
+Full documentation (GitHub Pages): https://hankerspace.github.io/EasyKnowledgeRetrieval/
+
+## Features~~~~
+
+- **Hybrid Retrieval**: Combines vector similarity search with knowledge graph exploration for more context-aware answers.
+- **Knowledge Graph Integration**: Automatically extracts entities and relationships from your text documents.
+- **Modular Storage**: Supports various backends for Key-Value pairs, Vector Stores, and Graph Storage (e.g., JSON, NanoVectorDB, NetworkX, Neo4j, Milvus).
+- **LLM Agnostic**: Designed to work with OpenAI-compatible LLM APIs (OpenAI, Gemini via OpenAI adapter, etc.).
+- **Async Support**: built with `asyncio` for high-performance ingestion and retrieval.
+
+## Installation
+
+You can install the library via pip:
+
+```bash
+pip install easy-knowledge-retriever
+```
+
+## Quick Start
+
+This guide will show you how to build a database from PDF documents and then query it.
+
+### 1. Build the Database (Ingestion)
+
+During this step, documents are processed, chunked, embedded, and entities/relations are extracted to build the Knowledge Graph options.
+
+```python
+import asyncio
+import os
+from easy_knowledge_retriever import EasyKnowledgeRetriever
+from easy_knowledge_retriever.llm.service import OpenAILLMService, OpenAIEmbeddingService
+from easy_knowledge_retriever.kg.json_kv_impl import JsonKVStorage
+from easy_knowledge_retriever.kg.nano_vector_db_impl import NanoVectorDBStorage
+from easy_knowledge_retriever.kg.networkx_impl import NetworkXStorage
+from easy_knowledge_retriever.kg.json_doc_status_impl import JsonDocStatusStorage
+
+async def build_database():
+    # 1. Configure Services
+    # Replace with your actual API keys and endpoints
+    embedding_service = OpenAIEmbeddingService(
+        api_key="your-embedding-api-key",
+        base_url="https://api.openai.com/v1", # or compatible
+        model="text-embedding-3-small",
+        embedding_dim=1536
+    )
+
+    llm_service = OpenAILLMService(
+        model="gpt-4o",
+        api_key="your-llm-api-key",
+        base_url="https://api.openai.com/v1"
+    )
+
+    # 2. Initialize Retriever with specific storage backends
+    working_dir = "./rag_data"
+    rag = EasyKnowledgeRetriever(
+        working_dir=working_dir,
+        llm_service=llm_service,
+        embedding_service=embedding_service,
+        kv_storage=JsonKVStorage(),
+        vector_storage=NanoVectorDBStorage(cosine_better_than_threshold=0.2),
+        graph_storage=NetworkXStorage(),
+        doc_status_storage=JsonDocStatusStorage(),
+    )
+
+    await rag.initialize_storages()
+    
+    try:
+        # 3. Ingest Documents
+        pdf_path = "./documents/example.pdf"
+        if os.path.exists(pdf_path):
+            print(f"Ingesting {pdf_path}...")
+            await rag.ingest(pdf_path)
+            print("Ingestion complete.")
+        else:
+            print("Please provide a valid PDF path.")
+            
+    finally:
+        # Always finalize to save state
+        await rag.finalize_storages()
+
+if __name__ == "__main__":
+    asyncio.run(build_database())
+```
+
+### 2. Retrieve Information (Querying)
+
+Once the database is built, you can query it.
+
+```python
+import asyncio
+from easy_knowledge_retriever import EasyKnowledgeRetriever, QueryParam
+from easy_knowledge_retriever.llm.service import OpenAILLMService, OpenAIEmbeddingService
+from easy_knowledge_retriever.kg.json_kv_impl import JsonKVStorage
+from easy_knowledge_retriever.kg.nano_vector_db_impl import NanoVectorDBStorage
+from easy_knowledge_retriever.kg.networkx_impl import NetworkXStorage
+from easy_knowledge_retriever.kg.json_doc_status_impl import JsonDocStatusStorage
+
+async def query_knowledge_base():
+    # 1. Re-initialize Services (same config as build)
+    embedding_service = OpenAIEmbeddingService(
+        api_key="your-embedding-api-key",
+        base_url="https://api.openai.com/v1",
+        model="text-embedding-3-small",
+        embedding_dim=1536
+    )
+    llm_service = OpenAILLMService(
+        model="gpt-4o",
+        api_key="your-llm-api-key",
+        base_url="https://api.openai.com/v1"
+    )
+
+    # 2. Load the existing Retriever
+    working_dir = "./rag_data"
+    rag = EasyKnowledgeRetriever(
+        working_dir=working_dir,
+        llm_service=llm_service,
+        embedding_service=embedding_service,
+        kv_storage=JsonKVStorage(),
+        vector_storage=NanoVectorDBStorage(cosine_better_than_threshold=0.2),
+        graph_storage=NetworkXStorage(),
+        doc_status_storage=JsonDocStatusStorage(),
+    )
+
+    await rag.initialize_storages()
+
+    try:
+        # 3. Perform a Query
+        query_text = "What does the document say about forest fires?"
+        
+        # 'mix' mode uses both vector search and knowledge graph
+        param = QueryParam(mode="mix") 
+        
+        print(f"Querying: {query_text}")
+        result = await rag.aquery(query_text, param=param)
+        
+        print("\nAnswer:")
+        print(result)
+        
+    finally:
+        await rag.finalize_storages()
+
+if __name__ == "__main__":
+    asyncio.run(query_knowledge_base())
+```
+
+## Advanced Configuration
+
+### Storage Options
+
+You can swap out storage implementations by creating instances of different classes from `easy_knowledge_retriever.kg.*`:
+
+*   **Vector Storage**: `NanoVectorDBStorage` (local, lightweight), `MilvusStorage` (scalable).
+*   **Graph Storage**: `NetworkXStorage` (in-memory/json, simple), `Neo4jStorage` (robust graph DB).
+*   **KV Storage**: `JsonKVStorage`, `RedisKVStorage` (if available), etc.
+
+Example for Neo4j:
+
+```python
+from easy_knowledge_retriever.kg.neo4j_impl import Neo4jStorage
+
+graph_storage = Neo4jStorage(
+    uri="bolt://localhost:7687",
+    user="neo4j",
+    password="password"
+)
+```
+
+## Service & Configuration Catalog
+
+For a complete, up-to-date list of all services (LLM, Vector/KV/Graph/Doc Status) and their configuration options, see:
+
+- docs/ServiceCatalog.md
+
+## Development
+
+To set up the project for development:
+
+1.  Clone the repository.
+2.  Install dependencies: `pip install -r requirements.txt`.
+3.  Install the package in editable mode: `pip install -e .`.
+
+### Running Tests
+
+(Instructions for running tests if applicable)
+```bash
+pytest
+```
+
+## License
+
+This project is licensed under the Creative Commons Attribution–NonCommercial–ShareAlike 4.0 International (CC BY‑NC‑SA 4.0).
+
+- You must give appropriate credit, provide a link to the license, and indicate if changes were made.
+- You may not use the material for commercial purposes.
+- If you remix, transform, or build upon the material, you must distribute your contributions under the same license as the original.
+
+Full legal text: https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode  
+Summary (EN): https://creativecommons.org/licenses/by-nc-sa/4.0/

easy_knowledge_retriever-0.1.0/README.md

@@ -0,0 +1,210 @@
+<div align="center">
+<img src="docs/logo.png" alt="EKR Logo" width="400">
+</div>
+
+<div align="center">
+    <h1>Easy Knowledge Retriever - The easiest RAG lib ever</h1>
+</div>
+
+[![PyPI - Version](https://img.shields.io/pypi/v/easy-knowledge-retriever.svg)](https://pypi.org/project/easy-knowledge-retriever/) [![Python Versions](https://img.shields.io/pypi/pyversions/easy-knowledge-retriever.svg)](https://pypi.org/project/easy-knowledge-retriever/) [![License: CC BY-NC-SA 4.0](https://img.shields.io/badge/license-CC%20BY--NC--SA%204.0-lightgrey.svg)](https://creativecommons.org/licenses/by-nc-sa/4.0/) [![Docs](https://img.shields.io/badge/docs-GitHub%20Pages-blue.svg)](https://<YOUR_GITHUB_USERNAME>.github.io/EasyKnowledgeRetrieval/)
+
+**Easy Knowledge Retriever** is a powerful and flexible library for building Retrieval-Augmented Generation (RAG) systems with integrated Knowledge Graph support. It allows you to easily ingest documents, build a structured knowledge base (combining vector embeddings and graph relations), and perform advanced queries using Large Language Models (LLMs).
+
+Full documentation (GitHub Pages): https://hankerspace.github.io/EasyKnowledgeRetrieval/
+
+## Features~~~~
+
+- **Hybrid Retrieval**: Combines vector similarity search with knowledge graph exploration for more context-aware answers.
+- **Knowledge Graph Integration**: Automatically extracts entities and relationships from your text documents.
+- **Modular Storage**: Supports various backends for Key-Value pairs, Vector Stores, and Graph Storage (e.g., JSON, NanoVectorDB, NetworkX, Neo4j, Milvus).
+- **LLM Agnostic**: Designed to work with OpenAI-compatible LLM APIs (OpenAI, Gemini via OpenAI adapter, etc.).
+- **Async Support**: built with `asyncio` for high-performance ingestion and retrieval.
+
+## Installation
+
+You can install the library via pip:
+
+```bash
+pip install easy-knowledge-retriever
+```
+
+## Quick Start
+
+This guide will show you how to build a database from PDF documents and then query it.
+
+### 1. Build the Database (Ingestion)
+
+During this step, documents are processed, chunked, embedded, and entities/relations are extracted to build the Knowledge Graph options.
+
+```python
+import asyncio
+import os
+from easy_knowledge_retriever import EasyKnowledgeRetriever
+from easy_knowledge_retriever.llm.service import OpenAILLMService, OpenAIEmbeddingService
+from easy_knowledge_retriever.kg.json_kv_impl import JsonKVStorage
+from easy_knowledge_retriever.kg.nano_vector_db_impl import NanoVectorDBStorage
+from easy_knowledge_retriever.kg.networkx_impl import NetworkXStorage
+from easy_knowledge_retriever.kg.json_doc_status_impl import JsonDocStatusStorage
+
+async def build_database():
+    # 1. Configure Services
+    # Replace with your actual API keys and endpoints
+    embedding_service = OpenAIEmbeddingService(
+        api_key="your-embedding-api-key",
+        base_url="https://api.openai.com/v1", # or compatible
+        model="text-embedding-3-small",
+        embedding_dim=1536
+    )
+
+    llm_service = OpenAILLMService(
+        model="gpt-4o",
+        api_key="your-llm-api-key",
+        base_url="https://api.openai.com/v1"
+    )
+
+    # 2. Initialize Retriever with specific storage backends
+    working_dir = "./rag_data"
+    rag = EasyKnowledgeRetriever(
+        working_dir=working_dir,
+        llm_service=llm_service,
+        embedding_service=embedding_service,
+        kv_storage=JsonKVStorage(),
+        vector_storage=NanoVectorDBStorage(cosine_better_than_threshold=0.2),
+        graph_storage=NetworkXStorage(),
+        doc_status_storage=JsonDocStatusStorage(),
+    )
+
+    await rag.initialize_storages()
+    
+    try:
+        # 3. Ingest Documents
+        pdf_path = "./documents/example.pdf"
+        if os.path.exists(pdf_path):
+            print(f"Ingesting {pdf_path}...")
+            await rag.ingest(pdf_path)
+            print("Ingestion complete.")
+        else:
+            print("Please provide a valid PDF path.")
+            
+    finally:
+        # Always finalize to save state
+        await rag.finalize_storages()
+
+if __name__ == "__main__":
+    asyncio.run(build_database())
+```
+
+### 2. Retrieve Information (Querying)
+
+Once the database is built, you can query it.
+
+```python
+import asyncio
+from easy_knowledge_retriever import EasyKnowledgeRetriever, QueryParam
+from easy_knowledge_retriever.llm.service import OpenAILLMService, OpenAIEmbeddingService
+from easy_knowledge_retriever.kg.json_kv_impl import JsonKVStorage
+from easy_knowledge_retriever.kg.nano_vector_db_impl import NanoVectorDBStorage
+from easy_knowledge_retriever.kg.networkx_impl import NetworkXStorage
+from easy_knowledge_retriever.kg.json_doc_status_impl import JsonDocStatusStorage
+
+async def query_knowledge_base():
+    # 1. Re-initialize Services (same config as build)
+    embedding_service = OpenAIEmbeddingService(
+        api_key="your-embedding-api-key",
+        base_url="https://api.openai.com/v1",
+        model="text-embedding-3-small",
+        embedding_dim=1536
+    )
+    llm_service = OpenAILLMService(
+        model="gpt-4o",
+        api_key="your-llm-api-key",
+        base_url="https://api.openai.com/v1"
+    )
+
+    # 2. Load the existing Retriever
+    working_dir = "./rag_data"
+    rag = EasyKnowledgeRetriever(
+        working_dir=working_dir,
+        llm_service=llm_service,
+        embedding_service=embedding_service,
+        kv_storage=JsonKVStorage(),
+        vector_storage=NanoVectorDBStorage(cosine_better_than_threshold=0.2),
+        graph_storage=NetworkXStorage(),
+        doc_status_storage=JsonDocStatusStorage(),
+    )
+
+    await rag.initialize_storages()
+
+    try:
+        # 3. Perform a Query
+        query_text = "What does the document say about forest fires?"
+        
+        # 'mix' mode uses both vector search and knowledge graph
+        param = QueryParam(mode="mix") 
+        
+        print(f"Querying: {query_text}")
+        result = await rag.aquery(query_text, param=param)
+        
+        print("\nAnswer:")
+        print(result)
+        
+    finally:
+        await rag.finalize_storages()
+
+if __name__ == "__main__":
+    asyncio.run(query_knowledge_base())
+```
+
+## Advanced Configuration
+
+### Storage Options
+
+You can swap out storage implementations by creating instances of different classes from `easy_knowledge_retriever.kg.*`:
+
+*   **Vector Storage**: `NanoVectorDBStorage` (local, lightweight), `MilvusStorage` (scalable).
+*   **Graph Storage**: `NetworkXStorage` (in-memory/json, simple), `Neo4jStorage` (robust graph DB).
+*   **KV Storage**: `JsonKVStorage`, `RedisKVStorage` (if available), etc.
+
+Example for Neo4j:
+
+```python
+from easy_knowledge_retriever.kg.neo4j_impl import Neo4jStorage
+
+graph_storage = Neo4jStorage(
+    uri="bolt://localhost:7687",
+    user="neo4j",
+    password="password"
+)
+```
+
+## Service & Configuration Catalog
+
+For a complete, up-to-date list of all services (LLM, Vector/KV/Graph/Doc Status) and their configuration options, see:
+
+- docs/ServiceCatalog.md
+
+## Development
+
+To set up the project for development:
+
+1.  Clone the repository.
+2.  Install dependencies: `pip install -r requirements.txt`.
+3.  Install the package in editable mode: `pip install -e .`.
+
+### Running Tests
+
+(Instructions for running tests if applicable)
+```bash
+pytest
+```
+
+## License
+
+This project is licensed under the Creative Commons Attribution–NonCommercial–ShareAlike 4.0 International (CC BY‑NC‑SA 4.0).
+
+- You must give appropriate credit, provide a link to the license, and indicate if changes were made.
+- You may not use the material for commercial purposes.
+- If you remix, transform, or build upon the material, you must distribute your contributions under the same license as the original.
+
+Full legal text: https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode  
+Summary (EN): https://creativecommons.org/licenses/by-nc-sa/4.0/

easy_knowledge_retriever-0.1.0/easy_knowledge_retriever/__init__.py

@@ -0,0 +1,4 @@
+from .retriever import EasyKnowledgeRetriever
+from easy_knowledge_retriever.kg.base import QueryParam, QueryResult
+
+__all__ = ["EasyKnowledgeRetriever", "QueryParam", "QueryResult"]

easy_knowledge_retriever-0.1.0/easy_knowledge_retriever/constants.py

@@ -0,0 +1,97 @@
+"""
+Centralized configuration constants for EasyKnowledgeRetriever.
+
+This module defines default values for configuration constants used across
+different parts of the EasyKnowledgeRetriever system. Centralizing these values ensures
+consistency and makes maintenance easier.
+"""
+
+# Default values for server settings
+DEFAULT_MAX_GRAPH_NODES = 1000
+
+# Default values for extraction settings
+DEFAULT_SUMMARY_LANGUAGE = "French"  # Default language for document processing
+DEFAULT_MAX_GLEANING = 1
+DEFAULT_ENTITY_NAME_MAX_LENGTH = 256
+
+# Number of description fragments to trigger LLM summary
+DEFAULT_FORCE_LLM_SUMMARY_ON_MERGE = 8
+# Max description token size to trigger LLM summary
+DEFAULT_SUMMARY_MAX_TOKENS = 1200
+# Recommended LLM summary output length in tokens
+DEFAULT_SUMMARY_LENGTH_RECOMMENDED = 600
+# Maximum token size sent to LLM for summary
+DEFAULT_SUMMARY_CONTEXT_SIZE = 12000
+# Default entities to extract if ENTITY_TYPES is not specified in .env
+DEFAULT_ENTITY_TYPES = [
+    "Person",
+    "Creature",
+    "Organization",
+    "Location",
+    "Event",
+    "Concept",
+    "Method",
+    "Content",
+    "Data",
+    "Artifact",
+    "NaturalObject",
+]
+
+# Separator for: description, source_id and relation-key fields(Can not be changed after data inserted)
+GRAPH_FIELD_SEP = "<SEP>"
+
+# Query and retrieval configuration defaults
+DEFAULT_TOP_K = 40
+DEFAULT_CHUNK_TOP_K = 20
+DEFAULT_MAX_ENTITY_TOKENS = 6000
+DEFAULT_MAX_RELATION_TOKENS = 8000
+DEFAULT_MAX_TOTAL_TOKENS = 30000
+DEFAULT_COSINE_THRESHOLD = 0.2
+DEFAULT_RELATED_CHUNK_NUMBER = 5
+DEFAULT_KG_CHUNK_PICK_METHOD = "VECTOR"
+
+
+# Rerank configuration defaults
+# Default source ids limit in meta data for entity and relation
+DEFAULT_MAX_SOURCE_IDS_PER_ENTITY = 300
+DEFAULT_MAX_SOURCE_IDS_PER_RELATION = 300
+### control chunk_ids limitation method: FIFO, FIFO
+###    FIFO: First in first out
+###    KEEP: Keep oldest (less merge action and faster)
+SOURCE_IDS_LIMIT_METHOD_KEEP = "KEEP"
+SOURCE_IDS_LIMIT_METHOD_FIFO = "FIFO"
+DEFAULT_SOURCE_IDS_LIMIT_METHOD = SOURCE_IDS_LIMIT_METHOD_FIFO
+VALID_SOURCE_IDS_LIMIT_METHODS = {
+    SOURCE_IDS_LIMIT_METHOD_KEEP,
+    SOURCE_IDS_LIMIT_METHOD_FIFO,
+}
+# Maximum number of file paths stored in entity/relation file_path field (For displayed only, does not affect query performance)
+DEFAULT_MAX_FILE_PATHS = 100
+
+# Field length of file_path in Milvus Schema for entity and relation (Should not be changed)
+# file_path must store all file paths up to the DEFAULT_MAX_FILE_PATHS limit within the metadata.
+DEFAULT_MAX_FILE_PATH_LENGTH = 32768
+# Placeholder for more file paths in meta data for entity and relation (Should not be changed)
+DEFAULT_FILE_PATH_MORE_PLACEHOLDER = "truncated"
+
+
+# Default maximum parallel insert operations
+DEFAULT_MAX_PARALLEL_INSERT = 1
+
+
+# Logging configuration defaults
+DEFAULT_LOG_MAX_BYTES = 10485760  # Default 10MB
+DEFAULT_LOG_BACKUP_COUNT = 5  # Default 5 backups
+DEFAULT_LOG_FILENAME = "easy_knowledge_retriever.log"  # Default log filename
+
+
+
+# LLM execution defaults
+DEFAULT_TEMPERATURE = 1.0
+DEFAULT_MAX_ASYNC = 1
+DEFAULT_LLM_TIMEOUT = 1000000
+
+# Embedding execution defaults
+DEFAULT_EMBEDDING_BATCH_NUM = 1
+DEFAULT_EMBEDDING_FUNC_MAX_ASYNC = 1
+DEFAULT_EMBEDDING_TIMEOUT = 1000000