langroid 0.1.63__py3-none-any.whl → 0.1.65__py3-none-any.whl

langroid/agent/special/doc_chat_agent.py

@@ -19,7 +19,7 @@ from langroid.embedding_models.models import OpenAIEmbeddingsConfig
 from langroid.language_models.base import StreamingIfAllowed
 from langroid.language_models.openai_gpt import OpenAIChatModel, OpenAIGPTConfig
 from langroid.mytypes import DocMetaData, Document, Entity
-from langroid.parsing.parser import ParsingConfig, Splitter
+from langroid.parsing.parser import Parser, ParsingConfig, Splitter
 from langroid.parsing.repo_loader import RepoLoader
 from langroid.parsing.url_loader import URLLoader
 from langroid.parsing.urls import get_urls_and_paths
@@ -86,7 +86,8 @@ class DocChatAgentConfig(ChatAgentConfig):
     ]
     parsing: ParsingConfig = ParsingConfig(  # modify as needed
         splitter=Splitter.TOKENS,
-        chunk_size=500,  # aim for this many tokens per chunk
+        chunk_size=800,  # aim for this many tokens per chunk
+        overlap=100,  # overlap between chunks
         max_chunks=10_000,
         # aim to have at least this many chars per chunk when
         # truncating due to punctuation
@@ -145,12 +146,13 @@ class DocChatAgent(ChatAgent):
             return
         urls, paths = get_urls_and_paths(self.config.doc_paths)
         docs: List[Document] = []
+        parser = Parser(self.config.parsing)
         if len(urls) > 0:
-            loader = URLLoader(urls=urls)
+            loader = URLLoader(urls=urls, parser=parser)
             docs = loader.load()
         if len(paths) > 0:
             for p in paths:
-                path_docs = RepoLoader.get_documents(p)
+                path_docs = RepoLoader.get_documents(p, parser=parser)
                 docs.extend(path_docs)
         n_docs = len(docs)
         n_splits = self.ingest_docs(docs)

langroid/agent/special/table_chat_agent.py

@@ -45,6 +45,10 @@ If you receive a null or other unexpected result, see if you have made an assump
 in your code, and try another way, or use `run_code` to explore the dataframe 
 before submitting your final code. 
 
+Once you have the answer to the question, say DONE and show me the answer.
+If you receive an error message, try using the `run_code` tool/function 
+again with the corrected code. 
+
 Start by asking me what I want to know about the data.
 """
 

langroid/mytypes.py

@@ -25,6 +25,7 @@ class DocMetaData(BaseModel):
     """Metadata for a document."""
 
     source: str = "context"
+    is_chunk: bool = False  # if it is a chunk, don't split
 
     class Config:
         extra = Extra.allow

langroid/parsing/parser.py

@@ -22,6 +22,7 @@ class Splitter(str, Enum):
 class ParsingConfig(BaseSettings):
     splitter: str = Splitter.TOKENS
     chunk_size: int = 200  # aim for this many tokens per chunk
+    overlap: int = 50  # overlap between chunks
     max_chunks: int = 10_000
     # aim to have at least this many chars per chunk when truncating due to punctuation
     min_chunk_chars: int = 350
@@ -198,11 +199,18 @@ class Parser:
     def split(self, docs: List[Document]) -> List[Document]:
         if len(docs) == 0:
             return []
+        # some docs are already splits, so don't split them further!
+        chunked_docs = [d for d in docs if d.metadata.is_chunk]
+        big_docs = [d for d in docs if not d.metadata.is_chunk]
+        if len(big_docs) == 0:
+            return chunked_docs
         if self.config.splitter == Splitter.PARA_SENTENCE:
-            return self.split_para_sentence(docs)
+            big_doc_chunks = self.split_para_sentence(big_docs)
         elif self.config.splitter == Splitter.TOKENS:
-            return self.split_chunk_tokens(docs)
+            big_doc_chunks = self.split_chunk_tokens(big_docs)
         elif self.config.splitter == Splitter.SIMPLE:
-            return self.split_simple(docs)
+            big_doc_chunks = self.split_simple(big_docs)
         else:
             raise ValueError(f"Unknown splitter: {self.config.splitter}")
+
+        return chunked_docs + big_doc_chunks

langroid/parsing/pdf_parser.py

@@ -1,51 +1,174 @@
 from io import BytesIO
+from typing import List
 
 import requests
 from pypdf import PdfReader
 
 from langroid.mytypes import DocMetaData, Document
+from langroid.parsing.parser import Parser
 
 
-def _text_from_pdf_reader(reader: PdfReader) -> str:
-    """
-    Extract text from a `PdfReader` object.
-    Args:
-        reader (PdfReader): a `PdfReader` object
-    Returns:
-        str: the extracted text
-    """
-    text = ""
-    for page in reader.pages:
-        text += page.extract_text()
-    return text
-
-
-def get_doc_from_pdf_url(url: str) -> Document:
-    """
-    Args:
-        url (str): contains the URL to the PDF file
-    Returns:
-        a `Document` object containing the content of the pdf file,
-            and metadata containing url
-    """
-    response = requests.get(url)
-    response.raise_for_status()
-    with BytesIO(response.content) as f:
-        reader = PdfReader(f)
-        text = _text_from_pdf_reader(reader)
-    return Document(content=text, metadata=DocMetaData(source=str(url)))
-
-
-def get_doc_from_pdf_file(path: str) -> Document:
-    """
-    Given local path to a PDF file, extract the text content.
-    Args:
-        path (str): full path to the PDF file
-            PDF file obtained via URL
-    Returns:
-        a `Document` object containing the content of the pdf file,
-            and metadata containing path/url
-    """
-    reader = PdfReader(path)
-    text = _text_from_pdf_reader(reader)
-    return Document(content=text, metadata=DocMetaData(source=str(path)))
+class PdfParser(Parser):
+    def __init__(self, parser: Parser):
+        super().__init__(parser.config)
+
+    @classmethod
+    def from_Parser(cls, parser: Parser) -> "PdfParser":
+        return cls(parser)
+
+    @staticmethod
+    def _text_from_pdf_reader(reader: PdfReader) -> str:
+        """
+        Extract text from a `PdfReader` object.
+        Args:
+            reader (PdfReader): a `PdfReader` object
+        Returns:
+            str: the extracted text
+        """
+        text = ""
+        for page in reader.pages:
+            text += page.extract_text()
+        return text
+
+    def _doc_chunks_from_pdf_reader(
+        self,
+        reader: PdfReader,
+        doc: str,
+        chunk_tokens: int,
+        overlap: int = 0,
+    ) -> List[Document]:
+        """
+        Get document chunks from a PdfReader object,
+        with page references in the document metadata.
+
+        Adapted from
+        https://github.com/whitead/paper-qa/blob/main/paperqa/readers.py
+
+        Args:
+            reader (PdfReader): a `PdfReader` object
+            doc: URL or filename of the PDF file
+            chunk_tokens (int): number of tokens in each chunk
+            overlap (int): number of tokens to overlap between chunks
+
+        Returns:
+            List[Document]: a list of `Document` objects,
+                each containing a chunk of text
+        """
+
+        split = []  # tokens in curr split
+        pages: List[str] = []
+        docs: List[Document] = []
+        for i, page in enumerate(reader.pages):
+            split += self.tokenizer.encode(page.extract_text())
+            pages.append(str(i + 1))
+            # split could be so long it needs to be split
+            # into multiple chunks. Or it could be so short
+            # that it needs to be combined with the next chunk.
+            while len(split) > chunk_tokens:
+                # pretty formatting of pages (e.g. 1-3, 4, 5-7)
+                pg = "-".join([pages[0], pages[-1]])
+                docs.append(
+                    Document(
+                        content=self.tokenizer.decode(split[:chunk_tokens]),
+                        metadata=DocMetaData(
+                            source=f"{doc} pages {pg}",
+                            is_chunk=True,
+                        ),
+                    )
+                )
+                split = split[chunk_tokens - overlap :]
+                pages = [str(i + 1)]
+        if len(split) > overlap:
+            pg = "-".join([pages[0], pages[-1]])
+            docs.append(
+                Document(
+                    content=self.tokenizer.decode(split[:chunk_tokens]),
+                    metadata=DocMetaData(
+                        source=f"{doc} pages {pg}",
+                        is_chunk=True,
+                    ),
+                )
+            )
+        return docs
+
+    @staticmethod
+    def doc_chunks_from_pdf_url(url: str, parser: Parser) -> List[Document]:
+        """
+        Get chunks of text from pdf URL as a list of Document objects,
+        using the parser's chunk_size and overlap settings.
+
+        Args:
+            url (str): contains the URL to the PDF file
+        Returns:
+            a `Document` object containing the content of the pdf file,
+                and metadata containing url
+        """
+
+        pdfParser = PdfParser.from_Parser(parser)
+        response = requests.get(url)
+        response.raise_for_status()
+        with BytesIO(response.content) as f:
+            reader = PdfReader(f)
+            docs = pdfParser._doc_chunks_from_pdf_reader(
+                reader,
+                doc=url,
+                chunk_tokens=parser.config.chunk_size,
+                overlap=parser.config.overlap,
+            )
+        return docs
+
+    @staticmethod
+    def get_doc_from_pdf_url(url: str) -> Document:
+        """
+        Get entire text from pdf URL as a single document.
+
+        Args:
+            url (str): contains the URL to the PDF file
+        Returns:
+            a `Document` object containing the content of the pdf file,
+                and metadata containing url
+        """
+        response = requests.get(url)
+        response.raise_for_status()
+        with BytesIO(response.content) as f:
+            reader = PdfReader(f)
+            text = PdfParser._text_from_pdf_reader(reader)
+        return Document(content=text, metadata=DocMetaData(source=str(url)))
+
+    @staticmethod
+    def doc_chunks_from_pdf_path(path: str, parser: Parser) -> List[Document]:
+        """
+        Get chunks of text from pdf path as a list of Document objects,
+        using the parser's chunk_size and overlap settings.
+
+        Args:
+            url (str): contains the URL to the PDF file
+        Returns:
+            a `Document` object containing the content of the pdf file,
+                and metadata containing url
+        """
+
+        pdfParser = PdfParser.from_Parser(parser)
+        reader = PdfReader(path)
+        docs = pdfParser._doc_chunks_from_pdf_reader(
+            reader,
+            doc=path,
+            chunk_tokens=parser.config.chunk_size,
+            overlap=parser.config.overlap,
+        )
+        return docs
+
+    @staticmethod
+    def get_doc_from_pdf_file(path: str) -> Document:
+        """
+        Given local path to a PDF file, extract the text content.
+        Args:
+            path (str): full path to the PDF file
+                PDF file obtained via URL
+        Returns:
+            a `Document` object containing the content of the pdf file,
+                and metadata containing path/url
+        """
+        reader = PdfReader(path)
+        text = PdfParser._text_from_pdf_reader(reader)
+        return Document(content=text, metadata=DocMetaData(source=str(path)))

langroid/parsing/repo_loader.py

@@ -18,7 +18,8 @@ from github.Repository import Repository
 from pydantic import BaseSettings
 
 from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.pdf_parser import get_doc_from_pdf_file
+from langroid.parsing.parser import Parser
+from langroid.parsing.pdf_parser import PdfParser
 
 logger = logging.getLogger(__name__)
 
@@ -443,6 +444,7 @@ class RepoLoader:
         exclude_dirs: Optional[List[str]] = None,
         depth: int = -1,
         lines: Optional[int] = None,
+        parser: Optional[Parser] = None,
     ) -> List[Document]:
         """
         Recursively get all files under a path as Document objects.
@@ -458,6 +460,7 @@ class RepoLoader:
                 which includes all depths.
             lines (int, optional): Number of lines to read from each file.
                 Defaults to None, which reads all lines.
+            parser (Parser, optional): Parser to use to parse files.
 
         Returns:
             List[Document]: List of Document objects representing files.
@@ -490,8 +493,11 @@ class RepoLoader:
 
         for file_path in file_paths:
             _, file_extension = os.path.splitext(file_path)
-            if file_extension == ".pdf":
-                docs.append(get_doc_from_pdf_file(file_path))
+            if file_extension.lower() == ".pdf":
+                if parser is None:
+                    docs.append(PdfParser.get_doc_from_pdf_file(file_path))
+                else:
+                    docs.extend(PdfParser.doc_chunks_from_pdf_path(file_path, parser))
             else:
                 with open(file_path, "r") as f:
                     if lines is not None:

langroid/parsing/url_loader.py

@@ -9,7 +9,8 @@ from trafilatura.downloads import (
 )
 
 from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.pdf_parser import get_doc_from_pdf_url
+from langroid.parsing.parser import Parser
+from langroid.parsing.pdf_parser import PdfParser
 
 logging.getLogger("trafilatura").setLevel(logging.ERROR)
 
@@ -26,8 +27,9 @@ class URLLoader:
      the "accept" button on the cookie dialog.
     """
 
-    def __init__(self, urls: List[str]):
+    def __init__(self, urls: List[str], parser: Parser | None = None):
         self.urls = urls
+        self.parser = parser
 
     @no_type_check
     def load(self) -> List[Document]:
@@ -43,7 +45,10 @@ class URLLoader:
             )
             for url, result in buffered_downloads(buffer, threads):
                 if url.lower().endswith(".pdf"):
-                    docs.append(get_doc_from_pdf_url(url))
+                    if self.parser is None:
+                        docs.append(PdfParser.get_doc_from_pdf_url(url))
+                    else:
+                        docs.extend(PdfParser.doc_chunks_from_pdf_url(url, self.parser))
                 else:
                     text = trafilatura.extract(
                         result,

{langroid-0.1.63.dist-info → langroid-0.1.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.63
+Version: 0.1.65
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani

{langroid-0.1.63.dist-info → langroid-0.1.65.dist-info}/RECORD

@@ -6,7 +6,7 @@ langroid/agent/chat_document.py,sha256=k7Klav3FIBTf2w95bQtxgqBrf2fMo1ydSlklQvv4R
 langroid/agent/helpers.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/junk,sha256=LxfuuW7Cijsg0szAzT81OjWWv1PMNI-6w_-DspVIO2s,339
 langroid/agent/special/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-langroid/agent/special/doc_chat_agent.py,sha256=lKenTsnbR91-NeZSvFyWJcmY_8_fKckCuR9oQQVnMXw,18097
+langroid/agent/special/doc_chat_agent.py,sha256=RmYRHclTz7D8dH3eIBFWINm8Cl5zvICt2Ri-6AHRPqc,18227
 langroid/agent/special/recipient_validator_agent.py,sha256=R3Rit93BNWQar_9stuDBGzmLr2W-IYOQ7oq-tlNNlps,6035
 langroid/agent/special/retriever_agent.py,sha256=DeOB5crFjXBvDEZT9k9ZVinOfFM2VgS6tQWWFyXSk9o,7204
 langroid/agent/special/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -16,7 +16,7 @@ langroid/agent/special/sql/utils/description_extractors.py,sha256=GcQ82IhKPInS_3
 langroid/agent/special/sql/utils/populate_metadata.py,sha256=zRjw31a1ZXvpx9bcmbtC2mngdHl-bp1ZNHStcPG8_Qk,2712
 langroid/agent/special/sql/utils/system_message.py,sha256=qKLHkvQWRQodTtPLPxr1GSLUYUFASZU8x-ybV67cB68,1885
 langroid/agent/special/sql/utils/tools.py,sha256=6uB2424SLtmapui9ggcEr0ZTiB6_dL1-JRGgN8RK9Js,1332
-langroid/agent/special/table_chat_agent.py,sha256=PTCE7MmunQj7tFiKAMIh7kvdIeQYU5ceXgBabwsxdg8,7244
+langroid/agent/special/table_chat_agent.py,sha256=2nRGW25WDEbR-ukQjeV3mzsC0qk2gOgl4MnLI6gejTs,7425
 langroid/agent/task.py,sha256=UqbjZP4hiG3yRrPWf-nqIyLtK8i0c3fWUEYKbcZ3n50,28275
 langroid/agent/tool_message.py,sha256=8I59BMkqfH_qpWazhv9_rpPjlaG826vVG5dyJGeOn3o,5936
 langroid/agent/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -40,18 +40,18 @@ langroid/language_models/prompt_formatter/__init__.py,sha256=47DEQpj8HBSa-_TImW-
 langroid/language_models/prompt_formatter/base.py,sha256=2y_GcwhstvB5ih3haS7l5Fv79jVnFJ_vEw1jqWJzB9k,1247
 langroid/language_models/prompt_formatter/llama2_formatter.py,sha256=YdcO88qyBeuMENVIVvVqSYuEpvYSTndUe_jd6hVTko4,2899
 langroid/language_models/utils.py,sha256=rmnSn-sJ3aKl_wBdeLPkck0Li4Ed6zkCxZYYl7n1V34,4668
-langroid/mytypes.py,sha256=_0TnRjIRFvJh1MThFGqtD8hUzq1W3LqzTS_WCFeodzw,1559
+langroid/mytypes.py,sha256=nJyTaX-nAe2dwRdF8NZKxUNAy_Hvxgtkn1c9buT-d14,1619
 langroid/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/parsing/agent_chats.py,sha256=sbZRV9ujdM5QXvvuHVjIi2ysYSYlap-uqfMMUKulrW0,1068
 langroid/parsing/code-parsing.md,sha256=--cyyNiSZSDlIwcjAV4-shKrSiRe2ytF3AdSoS_hD2g,3294
 langroid/parsing/code_parser.py,sha256=BbDAzp35wkYQ9U1dpf1ARL0lVyi0tfqEc6_eox2C090,3727
 langroid/parsing/json.py,sha256=MVqBUfInALQm1QKbcfEvLzWxBz_UztCIyGk7AK5uFPo,1650
 langroid/parsing/para_sentence_split.py,sha256=AJBzZojP3zpB-_IMiiHismhqcvkrVBQ3ZINoQyx_bE4,2000
-langroid/parsing/parser.py,sha256=6C9rpymc7R-FOILbTgS15IWwX7R4zLM5vU0UQLI-3C0,7430
-langroid/parsing/pdf_parser.py,sha256=RFrck50VvqYl37xzUp-cj4uSC4FDIvJqTwv100Dilgg,1432
-langroid/parsing/repo_loader.py,sha256=ILlvBH-wSvfdLLQKHklAzuxU980_ajts_bz7_9IwtLY,27017
+langroid/parsing/parser.py,sha256=R1yvqjBvXKOcX1opCj5nTYuDK2HLAfkonM055DM5CP8,7826
+langroid/parsing/pdf_parser.py,sha256=0FHrxFXwJ5m9xLQlyKrlRGeApGjkJ3gRk1a3fs1h7AI,5890
+langroid/parsing/repo_loader.py,sha256=lQ_9tceOEiPWATf_SL43iubz05G1XXoF4vINsKJHmkY,27324
 langroid/parsing/table_loader.py,sha256=uqbupGr4y_7os18RtaY5GpD0hWcgzROoNy8dQIHB4kc,1767
-langroid/parsing/url_loader.py,sha256=ROXkdkzFeQ9lxJhaqYu95Eh9nWoZjwj2rlKWapj7p0Q,1839
+langroid/parsing/url_loader.py,sha256=MfYr2zK-1pOMEEc9y_vrMdtC052XMlr57X2ptYKpDX4,2103
 langroid/parsing/url_loader_cookies.py,sha256=Lg4sNpRz9MByWq2mde6T0hKv68VZSV3mtMjNEHuFeSU,2327
 langroid/parsing/urls.py,sha256=_Bcf1iRdT7cQrQ8hnbPX0Jtzxc0lVFaucTS5rJoKA14,3709
 langroid/parsing/utils.py,sha256=zqvZWpZktRJTKx_JAqxaIyoudMdKVdB1zzjnOhVYHS4,2196
@@ -82,7 +82,7 @@ langroid/vector_store/base.py,sha256=QZx3NUNwf2I0r3A7iuoUHIRGbqt_pFGD0hq1R-Yg8iM
 langroid/vector_store/chromadb.py,sha256=s5pQkKjaMP-Tt5A8M10EInFzttaALPbJAq7q4gf0TKg,5235
 langroid/vector_store/qdrant_cloud.py,sha256=3im4Mip0QXLkR6wiqVsjV1QvhSElfxdFSuDKddBDQ-4,188
 langroid/vector_store/qdrantdb.py,sha256=KRvIIj1IZG2zFqejofMnRs2hT86B-27LgBEnuczdqOU,9072
-langroid-0.1.63.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.1.63.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
-langroid-0.1.63.dist-info/METADATA,sha256=5oka-JCHZrPQdVKsd7gIOXKlKDaDvZ6MWSnO4pmvWLc,35776
-langroid-0.1.63.dist-info/RECORD,,
+langroid-0.1.65.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.1.65.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
+langroid-0.1.65.dist-info/METADATA,sha256=B5O6tKm5lGdBFcm6Npepa4nYHHZ8zMq0VNkUxSOEo1g,35776
+langroid-0.1.65.dist-info/RECORD,,