langroid 0.1.64__py3-none-any.whl → 0.1.65__py3-none-any.whl

langroid/agent/special/doc_chat_agent.py

@@ -19,7 +19,7 @@ from langroid.embedding_models.models import OpenAIEmbeddingsConfig
 from langroid.language_models.base import StreamingIfAllowed
 from langroid.language_models.openai_gpt import OpenAIChatModel, OpenAIGPTConfig
 from langroid.mytypes import DocMetaData, Document, Entity
-from langroid.parsing.parser import ParsingConfig, Splitter
+from langroid.parsing.parser import Parser, ParsingConfig, Splitter
 from langroid.parsing.repo_loader import RepoLoader
 from langroid.parsing.url_loader import URLLoader
 from langroid.parsing.urls import get_urls_and_paths
@@ -86,7 +86,8 @@ class DocChatAgentConfig(ChatAgentConfig):
     ]
     parsing: ParsingConfig = ParsingConfig(  # modify as needed
         splitter=Splitter.TOKENS,
-        chunk_size=500,  # aim for this many tokens per chunk
+        chunk_size=800,  # aim for this many tokens per chunk
+        overlap=100,  # overlap between chunks
         max_chunks=10_000,
         # aim to have at least this many chars per chunk when
         # truncating due to punctuation
@@ -145,12 +146,13 @@ class DocChatAgent(ChatAgent):
             return
         urls, paths = get_urls_and_paths(self.config.doc_paths)
         docs: List[Document] = []
+        parser = Parser(self.config.parsing)
         if len(urls) > 0:
-            loader = URLLoader(urls=urls)
+            loader = URLLoader(urls=urls, parser=parser)
             docs = loader.load()
         if len(paths) > 0:
             for p in paths:
-                path_docs = RepoLoader.get_documents(p)
+                path_docs = RepoLoader.get_documents(p, parser=parser)
                 docs.extend(path_docs)
         n_docs = len(docs)
         n_splits = self.ingest_docs(docs)

langroid/mytypes.py

@@ -25,6 +25,7 @@ class DocMetaData(BaseModel):
     """Metadata for a document."""
 
     source: str = "context"
+    is_chunk: bool = False  # if it is a chunk, don't split
 
     class Config:
         extra = Extra.allow

langroid/parsing/parser.py

@@ -199,11 +199,18 @@ class Parser:
     def split(self, docs: List[Document]) -> List[Document]:
         if len(docs) == 0:
             return []
+        # some docs are already splits, so don't split them further!
+        chunked_docs = [d for d in docs if d.metadata.is_chunk]
+        big_docs = [d for d in docs if not d.metadata.is_chunk]
+        if len(big_docs) == 0:
+            return chunked_docs
         if self.config.splitter == Splitter.PARA_SENTENCE:
-            return self.split_para_sentence(docs)
+            big_doc_chunks = self.split_para_sentence(big_docs)
         elif self.config.splitter == Splitter.TOKENS:
-            return self.split_chunk_tokens(docs)
+            big_doc_chunks = self.split_chunk_tokens(big_docs)
         elif self.config.splitter == Splitter.SIMPLE:
-            return self.split_simple(docs)
+            big_doc_chunks = self.split_simple(big_docs)
         else:
             raise ValueError(f"Unknown splitter: {self.config.splitter}")
+
+        return chunked_docs + big_doc_chunks

langroid/parsing/pdf_parser.py

@@ -1,15 +1,20 @@
 from io import BytesIO
+from typing import List
 
 import requests
 from pypdf import PdfReader
 
 from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.parser import Parser, ParsingConfig
+from langroid.parsing.parser import Parser
 
 
 class PdfParser(Parser):
-    def __init__(self, config: ParsingConfig):
-        super().__init__(config)
+    def __init__(self, parser: Parser):
+        super().__init__(parser.config)
+
+    @classmethod
+    def from_Parser(cls, parser: Parser) -> "PdfParser":
+        return cls(parser)
 
     @staticmethod
     def _text_from_pdf_reader(reader: PdfReader) -> str:
@@ -25,7 +30,7 @@ class PdfParser(Parser):
             text += page.extract_text()
         return text
 
-    def _chunk_docs_from_pdf_reader(
+    def _doc_chunks_from_pdf_reader(
         self,
         reader: PdfReader,
         doc: str,
@@ -46,7 +51,8 @@ class PdfParser(Parser):
             overlap (int): number of tokens to overlap between chunks
 
         Returns:
-            List[Document]: a list of `Document` objects, each containing a chunk of text
+            List[Document]: a list of `Document` objects,
+                each containing a chunk of text
         """
 
         split = []  # tokens in curr split
@@ -64,7 +70,10 @@ class PdfParser(Parser):
                 docs.append(
                     Document(
                         content=self.tokenizer.decode(split[:chunk_tokens]),
-                        metadata=DocMetaData(source=f"{doc} pages {pg}"),
+                        metadata=DocMetaData(
+                            source=f"{doc} pages {pg}",
+                            is_chunk=True,
+                        ),
                     )
                 )
                 split = split[chunk_tokens - overlap :]
@@ -74,14 +83,45 @@ class PdfParser(Parser):
             docs.append(
                 Document(
                     content=self.tokenizer.decode(split[:chunk_tokens]),
-                    metadata=DocMetaData(source=f"{doc} pages {pg}"),
+                    metadata=DocMetaData(
+                        source=f"{doc} pages {pg}",
+                        is_chunk=True,
+                    ),
                 )
             )
         return docs
 
+    @staticmethod
+    def doc_chunks_from_pdf_url(url: str, parser: Parser) -> List[Document]:
+        """
+        Get chunks of text from pdf URL as a list of Document objects,
+        using the parser's chunk_size and overlap settings.
+
+        Args:
+            url (str): contains the URL to the PDF file
+        Returns:
+            a `Document` object containing the content of the pdf file,
+                and metadata containing url
+        """
+
+        pdfParser = PdfParser.from_Parser(parser)
+        response = requests.get(url)
+        response.raise_for_status()
+        with BytesIO(response.content) as f:
+            reader = PdfReader(f)
+            docs = pdfParser._doc_chunks_from_pdf_reader(
+                reader,
+                doc=url,
+                chunk_tokens=parser.config.chunk_size,
+                overlap=parser.config.overlap,
+            )
+        return docs
+
     @staticmethod
     def get_doc_from_pdf_url(url: str) -> Document:
         """
+        Get entire text from pdf URL as a single document.
+
         Args:
             url (str): contains the URL to the PDF file
         Returns:
@@ -92,9 +132,32 @@ class PdfParser(Parser):
         response.raise_for_status()
         with BytesIO(response.content) as f:
             reader = PdfReader(f)
-            text = _text_from_pdf_reader(reader)
+            text = PdfParser._text_from_pdf_reader(reader)
         return Document(content=text, metadata=DocMetaData(source=str(url)))
 
+    @staticmethod
+    def doc_chunks_from_pdf_path(path: str, parser: Parser) -> List[Document]:
+        """
+        Get chunks of text from pdf path as a list of Document objects,
+        using the parser's chunk_size and overlap settings.
+
+        Args:
+            url (str): contains the URL to the PDF file
+        Returns:
+            a `Document` object containing the content of the pdf file,
+                and metadata containing url
+        """
+
+        pdfParser = PdfParser.from_Parser(parser)
+        reader = PdfReader(path)
+        docs = pdfParser._doc_chunks_from_pdf_reader(
+            reader,
+            doc=path,
+            chunk_tokens=parser.config.chunk_size,
+            overlap=parser.config.overlap,
+        )
+        return docs
+
     @staticmethod
     def get_doc_from_pdf_file(path: str) -> Document:
         """
@@ -107,5 +170,5 @@ class PdfParser(Parser):
                 and metadata containing path/url
         """
         reader = PdfReader(path)
-        text = _text_from_pdf_reader(reader)
+        text = PdfParser._text_from_pdf_reader(reader)
         return Document(content=text, metadata=DocMetaData(source=str(path)))

langroid/parsing/repo_loader.py

@@ -18,7 +18,8 @@ from github.Repository import Repository
 from pydantic import BaseSettings
 
 from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.pdf_parser import get_doc_from_pdf_file
+from langroid.parsing.parser import Parser
+from langroid.parsing.pdf_parser import PdfParser
 
 logger = logging.getLogger(__name__)
 
@@ -443,6 +444,7 @@ class RepoLoader:
         exclude_dirs: Optional[List[str]] = None,
         depth: int = -1,
         lines: Optional[int] = None,
+        parser: Optional[Parser] = None,
     ) -> List[Document]:
         """
         Recursively get all files under a path as Document objects.
@@ -458,6 +460,7 @@ class RepoLoader:
                 which includes all depths.
             lines (int, optional): Number of lines to read from each file.
                 Defaults to None, which reads all lines.
+            parser (Parser, optional): Parser to use to parse files.
 
         Returns:
             List[Document]: List of Document objects representing files.
@@ -490,8 +493,11 @@ class RepoLoader:
 
         for file_path in file_paths:
             _, file_extension = os.path.splitext(file_path)
-            if file_extension == ".pdf":
-                docs.append(get_doc_from_pdf_file(file_path))
+            if file_extension.lower() == ".pdf":
+                if parser is None:
+                    docs.append(PdfParser.get_doc_from_pdf_file(file_path))
+                else:
+                    docs.extend(PdfParser.doc_chunks_from_pdf_path(file_path, parser))
             else:
                 with open(file_path, "r") as f:
                     if lines is not None:

langroid/parsing/url_loader.py

@@ -9,7 +9,8 @@ from trafilatura.downloads import (
 )
 
 from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.pdf_parser import get_doc_from_pdf_url
+from langroid.parsing.parser import Parser
+from langroid.parsing.pdf_parser import PdfParser
 
 logging.getLogger("trafilatura").setLevel(logging.ERROR)
 
@@ -26,8 +27,9 @@ class URLLoader:
      the "accept" button on the cookie dialog.
     """
 
-    def __init__(self, urls: List[str]):
+    def __init__(self, urls: List[str], parser: Parser | None = None):
         self.urls = urls
+        self.parser = parser
 
     @no_type_check
     def load(self) -> List[Document]:
@@ -43,7 +45,10 @@ class URLLoader:
             )
             for url, result in buffered_downloads(buffer, threads):
                 if url.lower().endswith(".pdf"):
-                    docs.append(get_doc_from_pdf_url(url))
+                    if self.parser is None:
+                        docs.append(PdfParser.get_doc_from_pdf_url(url))
+                    else:
+                        docs.extend(PdfParser.doc_chunks_from_pdf_url(url, self.parser))
                 else:
                     text = trafilatura.extract(
                         result,

{langroid-0.1.64.dist-info → langroid-0.1.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.64
+Version: 0.1.65
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani

{langroid-0.1.64.dist-info → langroid-0.1.65.dist-info}/RECORD

@@ -6,7 +6,7 @@ langroid/agent/chat_document.py,sha256=k7Klav3FIBTf2w95bQtxgqBrf2fMo1ydSlklQvv4R
 langroid/agent/helpers.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/junk,sha256=LxfuuW7Cijsg0szAzT81OjWWv1PMNI-6w_-DspVIO2s,339
 langroid/agent/special/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-langroid/agent/special/doc_chat_agent.py,sha256=lKenTsnbR91-NeZSvFyWJcmY_8_fKckCuR9oQQVnMXw,18097
+langroid/agent/special/doc_chat_agent.py,sha256=RmYRHclTz7D8dH3eIBFWINm8Cl5zvICt2Ri-6AHRPqc,18227
 langroid/agent/special/recipient_validator_agent.py,sha256=R3Rit93BNWQar_9stuDBGzmLr2W-IYOQ7oq-tlNNlps,6035
 langroid/agent/special/retriever_agent.py,sha256=DeOB5crFjXBvDEZT9k9ZVinOfFM2VgS6tQWWFyXSk9o,7204
 langroid/agent/special/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -40,18 +40,18 @@ langroid/language_models/prompt_formatter/__init__.py,sha256=47DEQpj8HBSa-_TImW-
 langroid/language_models/prompt_formatter/base.py,sha256=2y_GcwhstvB5ih3haS7l5Fv79jVnFJ_vEw1jqWJzB9k,1247
 langroid/language_models/prompt_formatter/llama2_formatter.py,sha256=YdcO88qyBeuMENVIVvVqSYuEpvYSTndUe_jd6hVTko4,2899
 langroid/language_models/utils.py,sha256=rmnSn-sJ3aKl_wBdeLPkck0Li4Ed6zkCxZYYl7n1V34,4668
-langroid/mytypes.py,sha256=_0TnRjIRFvJh1MThFGqtD8hUzq1W3LqzTS_WCFeodzw,1559
+langroid/mytypes.py,sha256=nJyTaX-nAe2dwRdF8NZKxUNAy_Hvxgtkn1c9buT-d14,1619
 langroid/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/parsing/agent_chats.py,sha256=sbZRV9ujdM5QXvvuHVjIi2ysYSYlap-uqfMMUKulrW0,1068
 langroid/parsing/code-parsing.md,sha256=--cyyNiSZSDlIwcjAV4-shKrSiRe2ytF3AdSoS_hD2g,3294
 langroid/parsing/code_parser.py,sha256=BbDAzp35wkYQ9U1dpf1ARL0lVyi0tfqEc6_eox2C090,3727
 langroid/parsing/json.py,sha256=MVqBUfInALQm1QKbcfEvLzWxBz_UztCIyGk7AK5uFPo,1650
 langroid/parsing/para_sentence_split.py,sha256=AJBzZojP3zpB-_IMiiHismhqcvkrVBQ3ZINoQyx_bE4,2000
-langroid/parsing/parser.py,sha256=GDYJSQAY-kHoGNpYll8HlSrsFpa_fJmlEfV1HWuXSJ0,7478
-langroid/parsing/pdf_parser.py,sha256=YRH4ZT0UsdMAaDi7vy0DBQTXOTxOF1xXERrs9nmoQ0c,3836
-langroid/parsing/repo_loader.py,sha256=ILlvBH-wSvfdLLQKHklAzuxU980_ajts_bz7_9IwtLY,27017
+langroid/parsing/parser.py,sha256=R1yvqjBvXKOcX1opCj5nTYuDK2HLAfkonM055DM5CP8,7826
+langroid/parsing/pdf_parser.py,sha256=0FHrxFXwJ5m9xLQlyKrlRGeApGjkJ3gRk1a3fs1h7AI,5890
+langroid/parsing/repo_loader.py,sha256=lQ_9tceOEiPWATf_SL43iubz05G1XXoF4vINsKJHmkY,27324
 langroid/parsing/table_loader.py,sha256=uqbupGr4y_7os18RtaY5GpD0hWcgzROoNy8dQIHB4kc,1767
-langroid/parsing/url_loader.py,sha256=ROXkdkzFeQ9lxJhaqYu95Eh9nWoZjwj2rlKWapj7p0Q,1839
+langroid/parsing/url_loader.py,sha256=MfYr2zK-1pOMEEc9y_vrMdtC052XMlr57X2ptYKpDX4,2103
 langroid/parsing/url_loader_cookies.py,sha256=Lg4sNpRz9MByWq2mde6T0hKv68VZSV3mtMjNEHuFeSU,2327
 langroid/parsing/urls.py,sha256=_Bcf1iRdT7cQrQ8hnbPX0Jtzxc0lVFaucTS5rJoKA14,3709
 langroid/parsing/utils.py,sha256=zqvZWpZktRJTKx_JAqxaIyoudMdKVdB1zzjnOhVYHS4,2196
@@ -82,7 +82,7 @@ langroid/vector_store/base.py,sha256=QZx3NUNwf2I0r3A7iuoUHIRGbqt_pFGD0hq1R-Yg8iM
 langroid/vector_store/chromadb.py,sha256=s5pQkKjaMP-Tt5A8M10EInFzttaALPbJAq7q4gf0TKg,5235
 langroid/vector_store/qdrant_cloud.py,sha256=3im4Mip0QXLkR6wiqVsjV1QvhSElfxdFSuDKddBDQ-4,188
 langroid/vector_store/qdrantdb.py,sha256=KRvIIj1IZG2zFqejofMnRs2hT86B-27LgBEnuczdqOU,9072
-langroid-0.1.64.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.1.64.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
-langroid-0.1.64.dist-info/METADATA,sha256=Zag_MPQ33R4KqBF3Uiptj39sRMAVT4o7PWoaiRYrvb4,35776
-langroid-0.1.64.dist-info/RECORD,,
+langroid-0.1.65.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.1.65.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
+langroid-0.1.65.dist-info/METADATA,sha256=B5O6tKm5lGdBFcm6Npepa4nYHHZ8zMq0VNkUxSOEo1g,35776
+langroid-0.1.65.dist-info/RECORD,,