hie-rag 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

hie_rag/hie_rag.py

@@ -16,7 +16,7 @@ class HieRag:
     def process_and_save_index_stream(self, file_name: str, uploaded_file: bytes, min_chunk_size, max_chunk_size):
         yield {"status": "🔍 Extracting text..."}
         print(f"Extracting text from {file_name}")
-        extracted_text = self.utils.extract_text(uploaded_file)
+        extracted_text = self.utils.extract_text(file_name=file_name, uploaded_bytes=uploaded_file)
 
         yield {"status": "✂️ Splitting into chunks..."}
         print(f"Splitting text into chunks with min size {min_chunk_size} and max size {max_chunk_size}")

hie_rag/utils.py

@@ -16,36 +16,26 @@ class Utils:
         # self.client = OpenAI(api_key=api_key)
         self.client = AiClient(base_url=base_url)
 
-    def extract_text(self, uploaded_file: bytes):
+    def extract_text(self, uploaded_bytes: bytes, file_name: str):
         """Extract text from an uploaded file using MarkItDown."""
-        # md = MarkItDown(llm_client=self.client, llm_model="gpt-4o")
         md = MarkItDown()
 
-        # Accept both raw bytes and file-like objects with `.read()`
-        if isinstance(uploaded_file, bytes):
-            file_bytes = uploaded_file
-            suffix = ".bin"  # fallback generic extension
-        elif hasattr(uploaded_file, "read"):
-            file_bytes = uploaded_file.read()
-            filename = getattr(uploaded_file, "name", None) or getattr(uploaded_file, "filename", None)
-            suffix = os.path.splitext(filename)[-1] if filename else ".bin"
-        else:
-            raise TypeError("Unsupported file type: must be bytes or file-like object")
-
-        # Write to temp file for MarkItDown to process
-        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
-            temp_file_path = temp_file.name
-            temp_file.write(file_bytes)
+        # derive a real suffix from the filename
+        suffix = os.path.splitext(file_name)[1].lower() or ".txt"
+
+        # write to temp file
+        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+            tmp.write(uploaded_bytes)
+            tmp_path = tmp.name
 
         try:
-            # Redirect stderr to suppress native print warnings like "CropBox missing"
             with contextlib.redirect_stderr(io.StringIO()):
-                extracted_text = md.convert(temp_file_path)
+                result = md.convert(tmp_path)
         finally:
-            # Clean up the temporary file
-            os.remove(temp_file_path)
+            os.remove(tmp_path)
 
-        return extracted_text.text_content
+        # depending on MarkItDown version this may return a str or an object
+        return getattr(result, "text_content", result)
 
     def count_tokens(self, text: str, encoding="cl100k_base") -> int:
         """Count tokens in text using tiktoken"""

hie_rag-0.2.4.dist-info/METADATA

@@ -0,0 +1,147 @@
+Metadata-Version: 2.4
+Name: hie_rag
+Version: 0.2.4
+Summary: A hierarchical RAG framework for chunks retrieval.
+License: Apache License
+        Version 2.0, January 2004
+        http://www.apache.org/licenses/
+        
+        Copyright [2025] [Chih-Han Chung]
+        
+        Licensed under the Apache License, Version 2.0 (the "License");
+        you may not use this file except in compliance with the License.
+        You may obtain a copy of the License at
+        
+            http://www.apache.org/licenses/LICENSE-2.0
+        
+        Unless required by applicable law or agreed to in writing, software
+        distributed under the License is distributed on an "AS IS" BASIS,
+        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        See the License for the specific language governing permissions and
+        limitations under the License.
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: werkzeug==3.1.3
+Requires-Dist: numpy
+Requires-Dist: markitdown==0.0.1a3
+Requires-Dist: openai==1.66.3
+Requires-Dist: scikit-learn
+Requires-Dist: tiktoken==0.8.0
+Requires-Dist: langchain==0.3.13
+Requires-Dist: langchain-ollama==0.3.3
+Requires-Dist: chromadb==0.6.2
+Dynamic: license-file
+
+# 📚 HieRAG – Hierarchical Retrieval-Augmented Generation
+
+`hie_rag` is a modular, extensible Python package designed for **Hierarchical Retrieval-Augmented Generation (Hie-RAG)**. It enables you to extract, split, embed, summarize, and query documents using both chunk- and tree-level semantics, all backed by a vector database.
+
+---
+
+## ✅ Features
+
+- PDF/DOCX/XLSX/CSV/PPT ingestion and intelligent semantic splitting
+- Hierarchical summarization tree building
+- Embedding-based similarity search
+- Vector DB indexing and querying (e.g., Qdrant)
+- Full streaming interface for frontend integration
+
+---
+
+## 📦 Components Used
+
+| Module      | Role                                                           |
+| ----------- | -------------------------------------------------------------- |
+| `HieRAG`    | Main interface for processing, querying, and managing indexes. |
+| `Split`     | Split raw text into chunks                                     |
+| `Process`   | Adds metadata and embeddings to chunks                         |
+| `TreeIndex` | Builds tree-based hierarchical summaries                       |
+| `Utils`     | Text extraction and token handling                             |
+| `Vectordb`  | Stores and queries summaries/chunks                            |
+| `AiClient`  | Handles embedding API (e.g., OpenAI, HuggingFace, Ollama)      |
+
+---
+
+## 🛠 Installation
+
+```bash
+pip install hie-rag
+```
+
+## ⏯︎ How to Use
+
+### Initialize HieRAG
+
+```python
+from hie_rag import HieRag
+
+hierag = HieRag(base_url="http://localhost:11434")
+```
+
+> [!NOTE]
+> Ensure you have set u an AI server. You should have a chatting model and a embedding model running.
+
+### Process and Index a File
+
+```python
+with open("sample.pdf", "rb") as f:
+    file_bytes = f.read()
+
+for status in hierag.process_and_save_index_stream(
+    file_name="sample.pdf",
+    uploaded_file=file_bytes,
+    min_chunk_size=300,
+    max_chunk_size=500
+):
+    print(status)
+```
+
+> ```JSON
+> {
+>   "status": "✅ Done",
+>   "file_id": "abc123",
+>   "summary_count": 5,
+>   "chunk_count": 22
+> }
+> ```
+
+### Query the Summaries or Chunks
+
+#### Query Summaries by text:
+
+```python
+results = hierag.query_summaries_by_text("What is the contract duration?")
+```
+
+#### Query Chunks by text:
+
+```python
+results = hierag.query_chunks_by_text("Explain clause 3.4", file_id="abc123")
+```
+
+### List & Manage Indexed Files
+
+#### List All Indexed Files
+
+```python
+hierag.list_summaries()
+```
+
+#### View Chunks of a File
+
+```python
+hierag.list_chunks(file_id="abc123")
+```
+
+#### Delete a File Index
+
+```python
+hierag.delete_index(file_id="abc123")
+```
+
+#### Get the Summary of a File
+
+```python
+hierag.get_summary(file_id="abc123")
+```

hie_rag-0.2.4.dist-info/RECORD

@@ -0,0 +1,13 @@
+hie_rag/__init__.py,sha256=p2glSTkCqGvMlcivcuKBStFh2C5adojaC9aGmF6nbhY,358
+hie_rag/ai_client.py,sha256=VbGQ0e3vZNn8W2YoR15Vvq2r-MUs-TBRNLGiImT4QxU,1000
+hie_rag/hie_rag.py,sha256=fprBRIKlbuDqSdBoeupFie-f8i1gkCz0VUPB5lB0mXc,2777
+hie_rag/process.py,sha256=Z4qpNmxSsxUJgnqJtw8cYWJTS6SxhRR7F7eX_akyVCU,2427
+hie_rag/split.py,sha256=gEQVt57xWruT5e1psgSOnwuBrQngzri3S4H6ZvKzsw4,5082
+hie_rag/tree_index.py,sha256=iTa25ohMv5O0HYc5JtzIzVAIhNdVklYiAIJvqyE8sbM,2722
+hie_rag/utils.py,sha256=_4TGiHuJ-Xo4JEEEdOjp4d1zxw6dNVsxROcom-vr7uU,4059
+hie_rag/vectordb.py,sha256=iI73ujrONjDaHU66RNdHnD2PZWSppnjm0isIHPJEGAY,11068
+hie_rag-0.2.4.dist-info/licenses/LICENSE,sha256=IwAxruLb1UG8F0KZtfnV6MJq10FRAxWM-XOTWkWsJt4,632
+hie_rag-0.2.4.dist-info/METADATA,sha256=-4V6CrFey8UwqnSdWGnM6WmhBRAIRhWNYPI07cX3hpY,3988
+hie_rag-0.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+hie_rag-0.2.4.dist-info/top_level.txt,sha256=tN2S3VpMUl6oLWL9sN4xIh4o2na_zjnW8rHiwPFf0T8,8
+hie_rag-0.2.4.dist-info/RECORD,,

hie_rag/split_and_process.py

@@ -1,17 +0,0 @@
-from hie_rag.process import Process
-from hie_rag.split import Split
-from hie_rag.utils import Utils
-
-
-class SplitAndProcess:
-    def __init__(self, base_url: str):
-        self.split = Split(base_url=base_url)
-        self.utils = Utils(base_url=base_url)
-        self.process = Process(base_url=base_url)
-
-    def split_and_process(self, uploaded_file):
-        extracted_text = self.utils.extract_text(uploaded_file)
-        result_split = self.split.split(extracted_text)
-        result_process = self.process.process_chunks(result_split)
-
-        return result_process

hie_rag-0.2.2.dist-info/METADATA

@@ -1,48 +0,0 @@
-Metadata-Version: 2.4
-Name: hie_rag
-Version: 0.2.2
-Summary: A hierarchical RAG framework for chunks retrieval.
-License: Apache License
-        Version 2.0, January 2004
-        http://www.apache.org/licenses/
-        
-        Copyright [2025] [Chih-Han Chung]
-        
-        Licensed under the Apache License, Version 2.0 (the "License");
-        you may not use this file except in compliance with the License.
-        You may obtain a copy of the License at
-        
-            http://www.apache.org/licenses/LICENSE-2.0
-        
-        Unless required by applicable law or agreed to in writing, software
-        distributed under the License is distributed on an "AS IS" BASIS,
-        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-        See the License for the specific language governing permissions and
-        limitations under the License.
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: python-dotenv==1.0.1
-Requires-Dist: werkzeug==3.1.3
-Requires-Dist: numpy==2.2.3
-Requires-Dist: markitdown==0.0.1a3
-Requires-Dist: openai==1.66.3
-Requires-Dist: scikit-learn
-Requires-Dist: tiktoken==0.8.0
-Requires-Dist: langchain==0.3.13
-Requires-Dist: langchain-ollama==0.3.3
-Requires-Dist: chromadb==0.6.2
-Dynamic: license-file
-
-# Hie-RAG
-
-Hie-RAG is a Hierarchical Retrieval-Augmented Generation framework for question answering.
-
-# Introduction
-
-It is designed to improve the retrieval quality and generation quality of the RAG model by introducing a hierarchical structure. The framework consists of two main parts: a higher-level summary retriever and a lower-level document retriever.
-
-# Installation
-
-```bash
-pip install hie-rag
-```

hie_rag-0.2.2.dist-info/RECORD

@@ -1,14 +0,0 @@
-hie_rag/__init__.py,sha256=p2glSTkCqGvMlcivcuKBStFh2C5adojaC9aGmF6nbhY,358
-hie_rag/ai_client.py,sha256=VbGQ0e3vZNn8W2YoR15Vvq2r-MUs-TBRNLGiImT4QxU,1000
-hie_rag/hie_rag.py,sha256=Nl_1WZM9IWhpNyZMvPzsae_u_xaCWEwrJgorZV-hp20,2741
-hie_rag/process.py,sha256=Z4qpNmxSsxUJgnqJtw8cYWJTS6SxhRR7F7eX_akyVCU,2427
-hie_rag/split.py,sha256=gEQVt57xWruT5e1psgSOnwuBrQngzri3S4H6ZvKzsw4,5082
-hie_rag/split_and_process.py,sha256=PkFlnOF7nW4Zs47JTsGF4AY9VDOXz1AtxG9Die8_mQk,572
-hie_rag/tree_index.py,sha256=iTa25ohMv5O0HYc5JtzIzVAIhNdVklYiAIJvqyE8sbM,2722
-hie_rag/utils.py,sha256=GwGiQj-zc8-U9UXOFHTKkjHWHx8YTYquR27gsXJgzCE,4687
-hie_rag/vectordb.py,sha256=iI73ujrONjDaHU66RNdHnD2PZWSppnjm0isIHPJEGAY,11068
-hie_rag-0.2.2.dist-info/licenses/LICENSE,sha256=IwAxruLb1UG8F0KZtfnV6MJq10FRAxWM-XOTWkWsJt4,632
-hie_rag-0.2.2.dist-info/METADATA,sha256=3vTI_zyvJxOOq8VrrchOAn0a7m8hwQPISnlholFi3u0,1698
-hie_rag-0.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-hie_rag-0.2.2.dist-info/top_level.txt,sha256=tN2S3VpMUl6oLWL9sN4xIh4o2na_zjnW8rHiwPFf0T8,8
-hie_rag-0.2.2.dist-info/RECORD,,