semantic-chunker-langchain 0.1.1__tar.gz → 0.1.3__tar.gz

{semantic_chunker_langchain-0.1.1 → semantic_chunker_langchain-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: semantic-chunker-langchain
-Version: 0.1.1
+Version: 0.1.3
 Summary: Token-aware, LangChain-compatible semantic chunker with PDF and layout support
 License: MIT
 Author: Prajwal Shivaji Mandale
@@ -42,7 +42,7 @@ A **token-aware**, **LangChain-compatible** chunker that splits text (from PDF,
 
 ---
 
-## 📦 Installation
+## 📆 Installation
 
 ```bash
 pip install semantic-chunker-langchain
@@ -63,12 +63,14 @@ semantic-chunker sample.pdf --txt chunks.txt --json chunks.json
 ### 🔸 From Code
 
 ```python
-from langchain_semantic_chunker.chunker import SemanticChunker, SimpleSemanticChunker
-from langchain_semantic_chunker.extractors.pdf import extract_pdf
-from langchain_semantic_chunker.outputs.formatter import write_to_txt
+from semantic_chunker_langchain.chunker import SemanticChunker, SimpleSemanticChunker
+from semantic_chunker_langchain.extractors.pdf import extract_pdf
+from semantic_chunker_langchain.outputs.formatter import write_to_txt
 
 # Extract
 docs = extract_pdf("sample.pdf")
+
+# Using SemanticChunker
 chunker = SemanticChunker(model_name="gpt-3.5-turbo")
 chunks = chunker.split_documents(docs)
 
@@ -89,7 +91,7 @@ retriever = chunker.to_retriever(chunks, embedding=OpenAIEmbeddings())
 
 ---
 
-## 🧪 Testing
+## 📊 Testing
 
 ```bash
 poetry run pytest tests/

{semantic_chunker_langchain-0.1.1 → semantic_chunker_langchain-0.1.3}/README.md

@@ -20,7 +20,7 @@ A **token-aware**, **LangChain-compatible** chunker that splits text (from PDF,
 
 ---
 
-## 📦 Installation
+## 📆 Installation
 
 ```bash
 pip install semantic-chunker-langchain
@@ -41,12 +41,14 @@ semantic-chunker sample.pdf --txt chunks.txt --json chunks.json
 ### 🔸 From Code
 
 ```python
-from langchain_semantic_chunker.chunker import SemanticChunker, SimpleSemanticChunker
-from langchain_semantic_chunker.extractors.pdf import extract_pdf
-from langchain_semantic_chunker.outputs.formatter import write_to_txt
+from semantic_chunker_langchain.chunker import SemanticChunker, SimpleSemanticChunker
+from semantic_chunker_langchain.extractors.pdf import extract_pdf
+from semantic_chunker_langchain.outputs.formatter import write_to_txt
 
 # Extract
 docs = extract_pdf("sample.pdf")
+
+# Using SemanticChunker
 chunker = SemanticChunker(model_name="gpt-3.5-turbo")
 chunks = chunker.split_documents(docs)
 
@@ -67,7 +69,7 @@ retriever = chunker.to_retriever(chunks, embedding=OpenAIEmbeddings())
 
 ---
 
-## 🧪 Testing
+## 📊 Testing
 
 ```bash
 poetry run pytest tests/

{semantic_chunker_langchain-0.1.1 → semantic_chunker_langchain-0.1.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "semantic-chunker-langchain"
-version = "0.1.1"
+version = "0.1.3"
 description = "Token-aware, LangChain-compatible semantic chunker with PDF and layout support"
 authors = ["Prajwal Shivaji Mandale <prajwal.mandale333@gmail.com>","Sudhnwa Ghorpade <sudhnwa.ghorpade@gmail.com>"]
 license = "MIT"

{semantic_chunker_langchain-0.1.1 → semantic_chunker_langchain-0.1.3}/semantic_chunker_langchain/chunker.py

@@ -107,75 +107,3 @@ class SimpleSemanticChunker(SemanticChunker):
     def split_text(self, text):
         return text.split('\n\n')
 
-
-
-
-# from langchain_core.documents import Document
-# from langchain_text_splitters import TextSplitter
-# from langchain_semantic_chunker.utils import estimate_token_count
-
-
-# class SemanticChunker(TextSplitter):
-#     def __init__(self, max_tokens: int = 1500, overlap: int = 200, model_name: str = "gpt-3.5-turbo"):
-#         """
-#         Token-aware document chunker for LangChain.
-
-#         Args:
-#             max_tokens (int): Maximum tokens per chunk
-#             overlap (int): Optional overlap in tokens between chunks
-#             model_name (str): The model name for token estimation (used with tiktoken)
-#         """
-#         self.max_tokens = max_tokens
-#         self.overlap = overlap
-#         self.model_name = model_name
-
-#     def split_documents(self, documents: list[Document]) -> list[Document]:
-#         chunks = []
-
-#         for doc in documents:
-#             text = doc.page_content
-#             metadata = doc.metadata.copy()
-
-#             paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
-#             current_chunk = []
-#             token_count = 0
-
-#             for para in paragraphs:
-#                 para_tokens = estimate_token_count(para, model_name=self.model_name)
-
-#                 if token_count + para_tokens > self.max_tokens:
-#                     # Commit current chunk
-#                     chunk_text = "\n\n".join(current_chunk)
-#                     chunks.append(Document(page_content=chunk_text, metadata=metadata))
-
-#                     # Start new chunk with overlap (if defined)
-#                     if self.overlap and len(current_chunk) > 0:
-#                         overlap_text = current_chunk[-1]
-#                         overlap_tokens = estimate_token_count(overlap_text, model_name=self.model_name)
-#                         current_chunk = [overlap_text]
-#                         token_count = overlap_tokens
-#                     else:
-#                         current_chunk = []
-#                         token_count = 0
-
-#                 current_chunk.append(para)
-#                 token_count += para_tokens
-
-#             if current_chunk:
-#                 chunk_text = "\n\n".join(current_chunk)
-#                 chunks.append(Document(page_content=chunk_text, metadata=metadata))
-
-#         return chunks
-
-#     def split_text(self, text: str) -> list[str]:
-#         """
-#         Dummy method to satisfy LangChain's abstract base class requirement.
-#         """
-#         return text.split('\n\n')
-
-
-
-# class SimpleSemanticChunker(SemanticChunker):
-#     def split_text(self, text):
-#         # Dummy implementation: split by paragraphs
-#         return text.split('\n\n')