semantic-chunker-langchain 0.1.0__tar.gz → 0.1.2__tar.gz

{semantic_chunker_langchain-0.1.0 → semantic_chunker_langchain-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: semantic-chunker-langchain
-Version: 0.1.0
+Version: 0.1.2
 Summary: Token-aware, LangChain-compatible semantic chunker with PDF and layout support
 License: MIT
 Author: Prajwal Shivaji Mandale
@@ -62,13 +62,14 @@ semantic-chunker sample.pdf --txt chunks.txt --json chunks.json
 
 ### 🔸 From Code
 
-```python
-from langchain_semantic_chunker.chunker import SemanticChunker, SimpleSemanticChunker
-from langchain_semantic_chunker.extractors.pdf import extract_pdf
-from langchain_semantic_chunker.outputs.formatter import write_to_txt
+from semantic_chunker_langchain.chunker import SemanticChunker, SimpleSemanticChunker
+from semantic_chunker_langchain.extractors.pdf import extract_pdf
+from semantic_chunker_langchain.outputs.formatter import write_to_txt
 
 # Extract
 docs = extract_pdf("sample.pdf")
+
+# Using SemanticChunker
 chunker = SemanticChunker(model_name="gpt-3.5-turbo")
 chunks = chunker.split_documents(docs)
 
@@ -78,7 +79,7 @@ write_to_txt(chunks, "output.txt")
 # Using SimpleSemanticChunker
 simple_chunker = SimpleSemanticChunker(model_name="gpt-3.5-turbo")
 simple_chunks = simple_chunker.split_documents(docs)
-```
+
 
 ### 🔸 Convert to Retriever
 

{semantic_chunker_langchain-0.1.0 → semantic_chunker_langchain-0.1.2}/README.md

@@ -40,13 +40,14 @@ semantic-chunker sample.pdf --txt chunks.txt --json chunks.json
 
 ### 🔸 From Code
 
-```python
-from langchain_semantic_chunker.chunker import SemanticChunker, SimpleSemanticChunker
-from langchain_semantic_chunker.extractors.pdf import extract_pdf
-from langchain_semantic_chunker.outputs.formatter import write_to_txt
+from semantic_chunker_langchain.chunker import SemanticChunker, SimpleSemanticChunker
+from semantic_chunker_langchain.extractors.pdf import extract_pdf
+from semantic_chunker_langchain.outputs.formatter import write_to_txt
 
 # Extract
 docs = extract_pdf("sample.pdf")
+
+# Using SemanticChunker
 chunker = SemanticChunker(model_name="gpt-3.5-turbo")
 chunks = chunker.split_documents(docs)
 
@@ -56,7 +57,7 @@ write_to_txt(chunks, "output.txt")
 # Using SimpleSemanticChunker
 simple_chunker = SimpleSemanticChunker(model_name="gpt-3.5-turbo")
 simple_chunks = simple_chunker.split_documents(docs)
-```
+
 
 ### 🔸 Convert to Retriever
 

{semantic_chunker_langchain-0.1.0 → semantic_chunker_langchain-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "semantic-chunker-langchain"
-version = "0.1.0"
+version = "0.1.2"
 description = "Token-aware, LangChain-compatible semantic chunker with PDF and layout support"
 authors = ["Prajwal Shivaji Mandale <prajwal.mandale333@gmail.com>","Sudhnwa Ghorpade <sudhnwa.ghorpade@gmail.com>"]
 license = "MIT"

{semantic_chunker_langchain-0.1.0 → semantic_chunker_langchain-0.1.2}/semantic_chunker_langchain/chunker.py

@@ -1,7 +1,7 @@
 # === langchain_semantic_chunker/chunker.py ===
 from langchain_core.documents import Document
 from langchain_text_splitters import TextSplitter
-from langchain_semantic_chunker.utils import estimate_token_count
+from semantic_chunker_langchain.utils import estimate_token_count
 import re
 
 class SemanticChunker(TextSplitter):
@@ -107,75 +107,3 @@ class SimpleSemanticChunker(SemanticChunker):
     def split_text(self, text):
         return text.split('\n\n')
 
-
-
-
-# from langchain_core.documents import Document
-# from langchain_text_splitters import TextSplitter
-# from langchain_semantic_chunker.utils import estimate_token_count
-
-
-# class SemanticChunker(TextSplitter):
-#     def __init__(self, max_tokens: int = 1500, overlap: int = 200, model_name: str = "gpt-3.5-turbo"):
-#         """
-#         Token-aware document chunker for LangChain.
-
-#         Args:
-#             max_tokens (int): Maximum tokens per chunk
-#             overlap (int): Optional overlap in tokens between chunks
-#             model_name (str): The model name for token estimation (used with tiktoken)
-#         """
-#         self.max_tokens = max_tokens
-#         self.overlap = overlap
-#         self.model_name = model_name
-
-#     def split_documents(self, documents: list[Document]) -> list[Document]:
-#         chunks = []
-
-#         for doc in documents:
-#             text = doc.page_content
-#             metadata = doc.metadata.copy()
-
-#             paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
-#             current_chunk = []
-#             token_count = 0
-
-#             for para in paragraphs:
-#                 para_tokens = estimate_token_count(para, model_name=self.model_name)
-
-#                 if token_count + para_tokens > self.max_tokens:
-#                     # Commit current chunk
-#                     chunk_text = "\n\n".join(current_chunk)
-#                     chunks.append(Document(page_content=chunk_text, metadata=metadata))
-
-#                     # Start new chunk with overlap (if defined)
-#                     if self.overlap and len(current_chunk) > 0:
-#                         overlap_text = current_chunk[-1]
-#                         overlap_tokens = estimate_token_count(overlap_text, model_name=self.model_name)
-#                         current_chunk = [overlap_text]
-#                         token_count = overlap_tokens
-#                     else:
-#                         current_chunk = []
-#                         token_count = 0
-
-#                 current_chunk.append(para)
-#                 token_count += para_tokens
-
-#             if current_chunk:
-#                 chunk_text = "\n\n".join(current_chunk)
-#                 chunks.append(Document(page_content=chunk_text, metadata=metadata))
-
-#         return chunks
-
-#     def split_text(self, text: str) -> list[str]:
-#         """
-#         Dummy method to satisfy LangChain's abstract base class requirement.
-#         """
-#         return text.split('\n\n')
-
-
-
-# class SimpleSemanticChunker(SemanticChunker):
-#     def split_text(self, text):
-#         # Dummy implementation: split by paragraphs
-#         return text.split('\n\n')