langroid 0.1.134__tar.gz → 0.1.135__tar.gz

{langroid-0.1.134 → langroid-0.1.135}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.134
+Version: 0.1.135
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani

{langroid-0.1.134 → langroid-0.1.135}/langroid/parsing/parser.py

@@ -52,13 +52,14 @@ class Parser:
         return len(tokens)
 
     def add_window_ids(self, chunks: List[Document]) -> None:
-        """Chunks are consecutive parts of a single original document.
-        Add window_ids in metadata"""
+        """Chunks may belong to multiple docs, but for each doc,
+        they appear consecutively. Add window_ids in metadata"""
 
         # The original metadata.id (if any) is ignored since it will be same for all
         # chunks and is useless. We want a distinct id for each chunk.
         orig_ids = [c.metadata.id for c in chunks]
         ids = [Document.hash_id(str(c)) for c in chunks]
+        id2chunk = {id: c for id, c in zip(ids, chunks)}
 
         # group the ids by orig_id
         orig_id_to_ids: Dict[str, List[str]] = {}
@@ -71,9 +72,11 @@ class Parser:
 
         k = self.config.n_neighbor_ids
         for orig, ids in orig_id_to_ids.items():
+            # ids are consecutive chunks in a single doc
             n = len(ids)
             window_ids = [ids[max(0, i - k) : min(n, i + k + 1)] for i in range(n)]
-            for i, c in enumerate(chunks):
+            for i, _ in enumerate(ids):
+                c = id2chunk[ids[i]]
                 if c.content.strip() == "":
                     continue
                 c.metadata.window_ids = window_ids[i]
@@ -101,35 +104,35 @@ class Parser:
         return final_docs
 
     def split_para_sentence(self, docs: List[Document]) -> List[Document]:
-        final_chunks = []
         chunks = docs
         while True:
-            long_chunks = [
-                p
-                for p in chunks
-                if self.num_tokens(p.content) > 1.3 * self.config.chunk_size
-            ]
-            if len(long_chunks) == 0:
-                break
-            short_chunks = [
-                p
-                for p in chunks
-                if self.num_tokens(p.content) <= 1.3 * self.config.chunk_size
-            ]
-            final_chunks += short_chunks
-            chunks = self._split_para_sentence_once(long_chunks)
-            if len(chunks) == len(long_chunks):
-                max_len = max([self.num_tokens(p.content) for p in long_chunks])
-                logger.warning(
-                    f"""
-                    Unable to split {len(long_chunks)} long chunks
-                    using chunk_size = {self.config.chunk_size}.
-                    Max chunk size is {max_len} tokens.
-                    """
-                )
+            un_splittables = 0
+            split_chunks = []
+            for c in chunks:
+                if c.content.strip() == "":
+                    continue
+                if self.num_tokens(c.content) <= 1.3 * self.config.chunk_size:
+                    # small chunk: no need to split
+                    split_chunks.append(c)
+                    continue
+                splits = self._split_para_sentence_once([c])
+                un_splittables += len(splits) == 1
+                split_chunks += splits
+            if len(split_chunks) == len(chunks):
+                if un_splittables > 0:
+                    max_len = max([self.num_tokens(p.content) for p in chunks])
+                    logger.warning(
+                        f"""
+                        Unable to split {un_splittables} chunks
+                        using chunk_size = {self.config.chunk_size}.
+                        Max chunk size is {max_len} tokens.
+                        """
+                    )
                 break  # we won't be able to shorten them with current settings
+            chunks = split_chunks.copy()
 
-        return final_chunks + chunks
+        self.add_window_ids(chunks)
+        return chunks
 
     def _split_para_sentence_once(self, docs: List[Document]) -> List[Document]:
         final_chunks = []
@@ -144,7 +147,6 @@ class Parser:
                 for c in chunks
                 if c.strip() != ""
             ]
-            self.add_window_ids(chunk_docs)
             final_chunks += chunk_docs
 
         return final_chunks

{langroid-0.1.134 → langroid-0.1.135}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langroid"
-version = "0.1.134"
+version = "0.1.135"
 description = "Harness LLMs with Multi-Agent Programming"
 authors = ["Prasad Chalasani <pchalasani@gmail.com>"]
 readme = "README.md"