semantic-chunker-langchain 0.1.0__py3-none-any.whl

semantic_chunker_langchain/__init__.py

@@ -0,0 +1,10 @@
+from .chunker import SemanticChunker, SimpleSemanticChunker
+from .utils import estimate_token_count
+from .extractors.pdf import extract_pdf
+from .outputs.formatter import write_to_txt, write_to_json
+
+__all__ = [
+    "SemanticChunker", "SimpleSemanticChunker",
+    "estimate_token_count", "extract_pdf",
+    "write_to_txt", "write_to_json"
+]

semantic_chunker_langchain/chunker.py

@@ -0,0 +1,181 @@
+# === langchain_semantic_chunker/chunker.py ===
+from langchain_core.documents import Document
+from langchain_text_splitters import TextSplitter
+from langchain_semantic_chunker.utils import estimate_token_count
+import re
+
+class SemanticChunker(TextSplitter):
+    def __init__(self, max_tokens: int = None, overlap: int = 200, model_name: str = "gpt-3.5-turbo", chunking_type: str = "text"):
+        self.model_name = model_name
+        self.max_tokens = max_tokens or self._default_tokens_for_model(model_name)
+        self.overlap = overlap
+        self.chunking_type = chunking_type
+
+    def _default_tokens_for_model(self, model_name: str) -> int:
+        if "claude" in model_name:
+            return 8000
+        elif "gpt-4" in model_name:
+            return 4000
+        else:
+            return 1500
+
+    def score_chunk(self, text: str) -> float:
+        return estimate_token_count(text, model_name=self.model_name)
+
+    def _split_paragraphs(self, text: str) -> list[str]:
+        return [p.strip() for p in text.split("\n\n") if p.strip()]
+
+    def _split_markdown(self, text: str) -> list[str]:
+        # Split at headings (e.g., ## or ###)
+        return re.split(r"(?=^#{1,6}\s)", text, flags=re.MULTILINE)
+
+    def _split_code(self, text: str) -> list[str]:
+        return text.split("\n\n")  # Simple fallback chunker
+
+    def split_documents(self, documents: list[Document]) -> list[Document]:
+        chunks = []
+
+        for doc in documents:
+            text = doc.page_content
+            metadata = doc.metadata.copy()
+
+            if self.chunking_type == "markdown":
+                blocks = self._split_markdown(text)
+            elif self.chunking_type == "code":
+                blocks = self._split_code(text)
+            else:
+                blocks = self._split_paragraphs(text)
+
+            current_chunk = []
+            token_count = 0
+
+            for block in blocks:
+                block_tokens = estimate_token_count(block, model_name=self.model_name)
+
+                if token_count + block_tokens > self.max_tokens:
+                    chunk_text = "\n\n".join(current_chunk)
+                    chunk_metadata = metadata.copy()
+                    chunk_metadata["score"] = self.score_chunk(chunk_text)
+                    chunks.append(Document(page_content=chunk_text, metadata=chunk_metadata))
+
+                    if self.overlap and len(current_chunk) > 0:
+                        overlap_text = current_chunk[-1]
+                        overlap_tokens = estimate_token_count(overlap_text, model_name=self.model_name)
+                        current_chunk = [overlap_text]
+                        token_count = overlap_tokens
+                    else:
+                        current_chunk = []
+                        token_count = 0
+
+                current_chunk.append(block)
+                token_count += block_tokens
+
+            if current_chunk:
+                chunk_text = "\n\n".join(current_chunk)
+                chunk_metadata = metadata.copy()
+                chunk_metadata["score"] = self.score_chunk(chunk_text)
+                chunks.append(Document(page_content=chunk_text, metadata=chunk_metadata))
+
+        merged_chunks = []
+        i = 0
+        while i < len(chunks):
+            chunk = chunks[i]
+            token_count = estimate_token_count(chunk.page_content, model_name=self.model_name)
+            if token_count < 300 and i + 1 < len(chunks):
+                next_chunk = chunks[i + 1]
+                merged_text = chunk.page_content + "\n\n" + next_chunk.page_content
+                merged_meta = chunk.metadata.copy()
+                merged_meta.update(next_chunk.metadata)
+                merged_meta["score"] = self.score_chunk(merged_text)
+                merged_chunks.append(Document(page_content=merged_text, metadata=merged_meta))
+                i += 2
+            else:
+                merged_chunks.append(chunk)
+                i += 1
+
+        return merged_chunks
+
+    def split_text(self, text: str) -> list[str]:
+        return self._split_paragraphs(text)
+
+    def to_retriever(self, chunks: list[Document], embedding) -> object:
+        from langchain_community.vectorstores import FAISS
+        return FAISS.from_documents(chunks, embedding=embedding).as_retriever()
+
+    
+class SimpleSemanticChunker(SemanticChunker):
+    def split_text(self, text):
+        return text.split('\n\n')
+
+
+
+
+# from langchain_core.documents import Document
+# from langchain_text_splitters import TextSplitter
+# from langchain_semantic_chunker.utils import estimate_token_count
+
+
+# class SemanticChunker(TextSplitter):
+#     def __init__(self, max_tokens: int = 1500, overlap: int = 200, model_name: str = "gpt-3.5-turbo"):
+#         """
+#         Token-aware document chunker for LangChain.
+
+#         Args:
+#             max_tokens (int): Maximum tokens per chunk
+#             overlap (int): Optional overlap in tokens between chunks
+#             model_name (str): The model name for token estimation (used with tiktoken)
+#         """
+#         self.max_tokens = max_tokens
+#         self.overlap = overlap
+#         self.model_name = model_name
+
+#     def split_documents(self, documents: list[Document]) -> list[Document]:
+#         chunks = []
+
+#         for doc in documents:
+#             text = doc.page_content
+#             metadata = doc.metadata.copy()
+
+#             paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+#             current_chunk = []
+#             token_count = 0
+
+#             for para in paragraphs:
+#                 para_tokens = estimate_token_count(para, model_name=self.model_name)
+
+#                 if token_count + para_tokens > self.max_tokens:
+#                     # Commit current chunk
+#                     chunk_text = "\n\n".join(current_chunk)
+#                     chunks.append(Document(page_content=chunk_text, metadata=metadata))
+
+#                     # Start new chunk with overlap (if defined)
+#                     if self.overlap and len(current_chunk) > 0:
+#                         overlap_text = current_chunk[-1]
+#                         overlap_tokens = estimate_token_count(overlap_text, model_name=self.model_name)
+#                         current_chunk = [overlap_text]
+#                         token_count = overlap_tokens
+#                     else:
+#                         current_chunk = []
+#                         token_count = 0
+
+#                 current_chunk.append(para)
+#                 token_count += para_tokens
+
+#             if current_chunk:
+#                 chunk_text = "\n\n".join(current_chunk)
+#                 chunks.append(Document(page_content=chunk_text, metadata=metadata))
+
+#         return chunks
+
+#     def split_text(self, text: str) -> list[str]:
+#         """
+#         Dummy method to satisfy LangChain's abstract base class requirement.
+#         """
+#         return text.split('\n\n')
+
+
+
+# class SimpleSemanticChunker(SemanticChunker):
+#     def split_text(self, text):
+#         # Dummy implementation: split by paragraphs
+#         return text.split('\n\n')

semantic_chunker_langchain/extractors/pdf.py

@@ -0,0 +1,10 @@
+from langchain_core.documents import Document
+import pdfplumber
+
+def extract_pdf(path: str) -> list[Document]:
+    with pdfplumber.open(path) as pdf:
+        return [
+            Document(page_content=page.extract_text(), metadata={"page_number": i+1})
+            for i, page in enumerate(pdf.pages)
+            if page.extract_text()
+        ]

semantic_chunker_langchain/outputs/formatter.py

@@ -0,0 +1,23 @@
+import json
+
+def write_to_txt(docs, path="output.txt"):
+    with open(path, "w", encoding="utf-8") as f:
+        for i, doc in enumerate(docs):
+            chunk_header = f"# Chunk {i+1}  |  Source: {doc.metadata.get('source', 'N/A')} | Page: {doc.metadata.get('page_number', 'N/A')}\n"
+            
+            # Format content for markdown: spacing and bullets
+            content = doc.page_content.strip()
+            content = content.replace("•", "-")
+            content = content.replace("\n", "\n\n")  # Add blank line between lines
+
+            # Write formatted chunk
+            f.write(f"\n\n{chunk_header}{content}\n")
+
+
+def write_to_json(docs, path="output.json"):
+    formatted = [
+        {"chunk": i+1, "content": doc.page_content.strip(), "metadata": doc.metadata}
+        for i, doc in enumerate(docs)
+    ]
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(formatted, f, indent=2, ensure_ascii=False)

semantic_chunker_langchain/utils.py

@@ -0,0 +1,8 @@
+import tiktoken
+
+def estimate_token_count(text: str, model_name: str = "gpt-3.5-turbo") -> int:
+    try:
+        encoding = tiktoken.encoding_for_model(model_name)
+    except Exception:
+        encoding = tiktoken.get_encoding("cl100k_base")
+    return len(encoding.encode(text))

semantic_chunker_langchain-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Prajwal Mandale, Sudhnwa Ghorpade
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

semantic_chunker_langchain-0.1.0.dist-info/METADATA

@@ -0,0 +1,110 @@
+Metadata-Version: 2.3
+Name: semantic-chunker-langchain
+Version: 0.1.0
+Summary: Token-aware, LangChain-compatible semantic chunker with PDF and layout support
+License: MIT
+Author: Prajwal Shivaji Mandale
+Author-email: prajwal.mandale333@gmail.com
+Requires-Python: >=3.9,<3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: faiss-cpu (>=1.11.0,<2.0.0)
+Requires-Dist: langchain (>=0.3.25,<0.4.0)
+Requires-Dist: langchain-community (>=0.3.26,<0.4.0)
+Requires-Dist: openai (>=1.84.0,<2.0.0)
+Requires-Dist: pdfplumber (>=0.11.6,<0.12.0)
+Requires-Dist: tiktoken (>=0.9.0,<0.10.0)
+Description-Content-Type: text/markdown
+
+# Semantic Chunker for LangChain
+
+A **token-aware**, **LangChain-compatible** chunker that splits text (from PDF, markdown, or plain text) into semantically coherent chunks while respecting model token limits.
+
+---
+
+## 🚀 Features
+
+* 🔍 **Model-Aware Token Limits**: Automatically adjusts chunking size for GPT-3.5, GPT-4, Claude, and others.
+* 📄 **Multi-format Input Support**:
+
+  * PDF via `pdfplumber`
+  * Plain `.txt`
+  * Markdown
+  * (Extendable to `.docx` and `.html`)
+* 🔁 **Overlapping Chunks**: Smart overlap between paragraphs to preserve context.
+* 🧠 **Smart Merging**: Merges chunks smaller than 300 tokens.
+* 🧩 **Retriever-Ready**: Direct integration with `LangChain` retrievers via FAISS.
+* 🔧 **CLI Support**: Run from terminal with one command.
+
+---
+
+## 📦 Installation
+
+```bash
+pip install semantic-chunker-langchain
+```
+
+> Requires Python 3.9 - 3.12
+
+---
+
+## 🛠️ Usage
+
+### 🔸 Chunk a PDF and Save to JSON/TXT
+
+```bash
+semantic-chunker sample.pdf --txt chunks.txt --json chunks.json
+```
+
+### 🔸 From Code
+
+```python
+from langchain_semantic_chunker.chunker import SemanticChunker, SimpleSemanticChunker
+from langchain_semantic_chunker.extractors.pdf import extract_pdf
+from langchain_semantic_chunker.outputs.formatter import write_to_txt
+
+# Extract
+docs = extract_pdf("sample.pdf")
+chunker = SemanticChunker(model_name="gpt-3.5-turbo")
+chunks = chunker.split_documents(docs)
+
+# Save to file
+write_to_txt(chunks, "output.txt")
+
+# Using SimpleSemanticChunker
+simple_chunker = SimpleSemanticChunker(model_name="gpt-3.5-turbo")
+simple_chunks = simple_chunker.split_documents(docs)
+```
+
+### 🔸 Convert to Retriever
+
+```python
+from langchain_community.embeddings import OpenAIEmbeddings
+retriever = chunker.to_retriever(chunks, embedding=OpenAIEmbeddings())
+```
+
+---
+
+## 🧪 Testing
+
+```bash
+poetry run pytest tests/
+```
+
+---
+
+## 👨‍💻 Authors
+
+* Prajwal Shivaji Mandale
+* Sudhnwa Ghorpade
+
+---
+
+## 📜 License
+
+This project is licensed under the MIT License.
+

semantic_chunker_langchain-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+semantic_chunker_langchain/__init__.py,sha256=fg5lxVkyosKw8alIAQHzQFkysAcIdjUfQpFwnhlgqAQ,351
+semantic_chunker_langchain/chunker.py,sha256=KtdY3dHlJKCtux6G75GtinqoOCVzhVXfCCSmD11kOUM,7253
+semantic_chunker_langchain/extractors/pdf.py,sha256=8jRWBCMeIK3M_WgOyDqxxadEHQw678CzD5ryAJ0tvAA,356
+semantic_chunker_langchain/outputs/formatter.py,sha256=tYShwikgwIleV6Nz1ohmtGX6nQRVnY41NOkOT6v43Qk,964
+semantic_chunker_langchain/utils.py,sha256=E0Ajj2IBa6EFJJkGYZ8pyWUEKEAjiL9_Uof8KPnM8ew,288
+semantic_chunker_langchain-0.1.0.dist-info/entry_points.txt,sha256=Kve0GJQ5uzNSMBidDihM9sFuoUY90OeP5THfJWQLDVQ,45
+semantic_chunker_langchain-0.1.0.dist-info/LICENSE,sha256=vfqlCGc0OOjpze243uuSsBAAq1OFEoCLbmElHpljFWM,1111
+semantic_chunker_langchain-0.1.0.dist-info/METADATA,sha256=KnJVAnDvroFTjUlq6-J6ER2lGmGwI6WX1PwbIca1y2o,2894
+semantic_chunker_langchain-0.1.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+semantic_chunker_langchain-0.1.0.dist-info/RECORD,,

semantic_chunker_langchain-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.3
+Root-Is-Purelib: true
+Tag: py3-none-any

semantic_chunker_langchain-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+semantic-chunker=cli:main
+