langroid 0.3.1__py3-none-any.whl → 0.5.1__py3-none-any.whl

langroid/vector_store/lancedb.py

@@ -32,13 +32,7 @@ from langroid.utils.configuration import settings
 from langroid.utils.pydantic_utils import (
     dataframe_to_document_model,
     dataframe_to_documents,
-    extend_document_class,
-    extra_metadata,
-    flatten_pydantic_instance,
-    flatten_pydantic_model,
-    nested_dict_from_flat,
 )
-from langroid.utils.system import pydantic_major_version
 from langroid.vector_store.base import VectorStore, VectorStoreConfig
 
 try:
@@ -58,10 +52,6 @@ class LanceDBConfig(VectorStoreConfig):
     storage_path: str = ".lancedb/data"
     embedding: EmbeddingModelsConfig = OpenAIEmbeddingsConfig()
     distance: str = "cosine"
-    # document_class is used to store in lancedb with right schema,
-    # and also to retrieve the right type of Documents when searching.
-    document_class: Type[Document] = Document
-    flatten: bool = False  # flatten Document class into LanceSchema ?
 
 
 class LanceDB(VectorStore):
@@ -78,7 +68,6 @@ class LanceDB(VectorStore):
         self.port = config.port
         self.is_from_dataframe = False  # were docs ingested from a dataframe?
         self.df_metadata_columns: List[str] = []  # metadata columns from dataframe
-        self._setup_schemas(config.document_class)
 
         load_dotenv()
         if self.config.cloud:
@@ -104,40 +93,6 @@ class LanceDB(VectorStore):
                     uri=new_storage_path,
                 )
 
-        # Note: Only create collection if a non-null collection name is provided.
-        # This is useful to delay creation of vecdb until we have a suitable
-        # collection name (e.g. we could get it from the url or folder path).
-        if config.collection_name is not None:
-            self.create_collection(
-                config.collection_name, replace=config.replace_collection
-            )
-
-    def _setup_schemas(self, doc_cls: Type[Document] | None) -> None:
-        try:
-            doc_cls = doc_cls or self.config.document_class
-            self.unflattened_schema = self._create_lance_schema(doc_cls)
-            self.schema = (
-                self._create_flat_lance_schema(doc_cls)
-                if self.config.flatten
-                else self.unflattened_schema
-            )
-        except (AttributeError, TypeError) as e:
-            pydantic_version = pydantic_major_version()
-            if pydantic_version > 1:
-                raise ValueError(
-                    f"""
-                    {e}
-                    ====
-                    You are using Pydantic v{pydantic_version},
-                    which is not yet compatible with Langroid's LanceDB integration.
-                    To use Lancedb with Langroid, please install the 
-                    latest pydantic 1.x instead of pydantic v2, e.g. 
-                    pip install "pydantic<2.0.0"
-                    """
-                )
-            else:
-                raise e
-
     def clear_empty_collections(self) -> int:
         coll_names = self.list_collections()
         n_deletes = 0
@@ -234,91 +189,8 @@ class LanceDB(VectorStore):
         )  # type: ignore
         return NewModel  # type: ignore
 
-    def _create_flat_lance_schema(self, doc_cls: Type[Document]) -> Type[BaseModel]:
-        """
-        Flat version of the lance_schema, as nested Pydantic schemas are not yet
-        supported by LanceDB.
-        """
-        if not has_lancedb:
-            raise LangroidImportError("lancedb", "lancedb")
-        lance_model = self._create_lance_schema(doc_cls)
-        FlatModel = flatten_pydantic_model(lance_model, base_model=LanceModel)
-        return FlatModel
-
     def create_collection(self, collection_name: str, replace: bool = False) -> None:
-        """
-        Create a collection with the given name, optionally replacing an existing
-            collection if `replace` is True.
-        Args:
-            collection_name (str): Name of the collection to create.
-            replace (bool): Whether to replace an existing collection
-                with the same name. Defaults to False.
-        """
-        self.config.collection_name = collection_name
-        collections = self.list_collections()
-        if collection_name in collections:
-            coll = self.client.open_table(collection_name)
-            if coll.head().shape[0] > 0:
-                logger.warning(f"Non-empty Collection {collection_name} already exists")
-                if not replace:
-                    logger.warning("Not replacing collection")
-                    return
-                else:
-                    logger.warning("Recreating fresh collection")
-        try:
-            self.client.create_table(
-                collection_name, schema=self.schema, mode="overwrite"
-            )
-        except (AttributeError, TypeError) as e:
-            pydantic_version = pydantic_major_version()
-            if pydantic_version > 1:
-                raise ValueError(
-                    f"""
-                    {e}
-                    ====
-                    You are using Pydantic v{pydantic_version},
-                    which is not yet compatible with Langroid's LanceDB integration.
-                    To use Lancedb with Langroid, please install the 
-                    latest pydantic 1.x instead of pydantic v2, e.g. 
-                    pip install "pydantic<2.0.0"
-                    """
-                )
-            else:
-                raise e
-
-        if settings.debug:
-            level = logger.getEffectiveLevel()
-            logger.setLevel(logging.INFO)
-            logger.setLevel(level)
-
-    def _maybe_set_doc_class_schema(self, doc: Document) -> None:
-        """
-        Set the config.document_class and self.schema based on doc if needed
-        Args:
-            doc: an instance of Document, to be added to a collection
-        """
-        extra_metadata_fields = extra_metadata(doc, self.config.document_class)
-        if len(extra_metadata_fields) > 0:
-            logger.warning(
-                f"""
-                    Added documents contain extra metadata fields:
-                    {extra_metadata_fields}
-                    which were not present in the original config.document_class.
-                    Trying to change document_class and corresponding schemas.
-                    Overriding LanceDBConfig.document_class with an auto-generated 
-                    Pydantic class that includes these extra fields.
-                    If this fails, or you see odd results, it is recommended that you 
-                    define a subclass of Document, with metadata of class derived from 
-                    DocMetaData, with extra fields defined via 
-                    `Field(..., description="...")` declarations,
-                    and set this document class as the value of the 
-                    LanceDBConfig.document_class attribute.
-                    """
-            )
-
-            doc_cls = extend_document_class(doc)
-            self.config.document_class = doc_cls
-            self._setup_schemas(doc_cls)
+        self.config.replace_collection = replace
 
     def add_documents(self, documents: Sequence[Document]) -> None:
         super().maybe_add_ids(documents)
@@ -329,39 +201,52 @@ class LanceDB(VectorStore):
         coll_name = self.config.collection_name
         if coll_name is None:
             raise ValueError("No collection name set, cannot ingest docs")
-        self._maybe_set_doc_class_schema(documents[0])
+        # self._maybe_set_doc_class_schema(documents[0])
+        table_exists = False
         if (
-            coll_name not in colls
-            or self.client.open_table(coll_name).head(1).shape[0] == 0
+            coll_name in colls
+            and self.client.open_table(coll_name).head(1).shape[0] > 0
         ):
-            # collection either doesn't exist or is empty, so replace it,
-            self.create_collection(coll_name, replace=True)
+            # collection exists and  is not empty:
+            # if replace_collection is True, we'll overwrite the existing collection,
+            # else we'll append to it.
+            if self.config.replace_collection:
+                self.client.drop_table(coll_name)
+            else:
+                table_exists = True
 
         ids = [str(d.id()) for d in documents]
         # don't insert all at once, batch in chunks of b,
         # else we get an API error
         b = self.config.batch_size
 
-        def make_batches() -> Generator[List[BaseModel], None, None]:
+        def make_batches() -> Generator[List[Dict[str, Any]], None, None]:
             for i in range(0, len(ids), b):
                 batch = [
-                    self.unflattened_schema(
+                    dict(
                         id=ids[i + j],
                         vector=embedding_vecs[i + j],
                         **doc.dict(),
                     )
                     for j, doc in enumerate(documents[i : i + b])
                 ]
-                if self.config.flatten:
-                    batch = [
-                        flatten_pydantic_instance(instance)  # type: ignore
-                        for instance in batch
-                    ]
                 yield batch
 
-        tbl = self.client.open_table(self.config.collection_name)
         try:
-            tbl.add(make_batches())
+            if table_exists:
+                tbl = self.client.open_table(coll_name)
+                tbl.add(make_batches())
+            else:
+                batch_gen = make_batches()
+                batch = next(batch_gen)
+                # use first batch to create table...
+                tbl = self.client.create_table(
+                    coll_name,
+                    data=batch,
+                    mode="create",
+                )
+                # ... and add the rest
+                tbl.add(batch_gen)
         except Exception as e:
             logger.error(
                 f"""
@@ -427,7 +312,6 @@ class LanceDB(VectorStore):
                 exclude=["vector"],
             )
             self.config.document_class = doc_cls  # type: ignore
-            self._setup_schemas(doc_cls)  # type: ignore
         else:
             # collection exists and is not empty, so append to it
             tbl = self.client.open_table(self.config.collection_name)
@@ -452,35 +336,19 @@ class LanceDB(VectorStore):
             return self._records_to_docs(records)
 
     def _records_to_docs(self, records: List[Dict[str, Any]]) -> List[Document]:
-        if self.config.flatten:
-            docs = [
-                self.unflattened_schema(**nested_dict_from_flat(rec)) for rec in records
-            ]
-        else:
-            try:
-                docs = [self.schema(**rec) for rec in records]
-            except ValidationError as e:
-                raise ValueError(
-                    f"""
-                Error validating LanceDB result: {e}
-                HINT: This could happen when you're re-using an 
-                existing LanceDB store with a different schema.
-                Try deleting your local lancedb storage at `{self.config.storage_path}`
-                re-ingesting your documents and/or replacing the collections.
-                """
-                )
-
-        doc_cls = self.config.document_class
-        doc_cls_field_names = doc_cls.__fields__.keys()
-        return [
-            doc_cls(
-                **{
-                    field_name: getattr(doc, field_name)
-                    for field_name in doc_cls_field_names
-                }
+        try:
+            docs = [self.config.document_class(**rec) for rec in records]
+        except ValidationError as e:
+            raise ValueError(
+                f"""
+            Error validating LanceDB result: {e}
+            HINT: This could happen when you're re-using an 
+            existing LanceDB store with a different schema.
+            Try deleting your local lancedb storage at `{self.config.storage_path}`
+            re-ingesting your documents and/or replacing the collections.
+            """
             )
-            for doc in docs
-        ]
+        return docs
 
     def get_all_documents(self, where: str = "") -> List[Document]:
         if self.config.collection_name is None:

langroid/vector_store/qdrantdb.py

@@ -380,7 +380,11 @@ class QdrantDB(VectorStore):
                 with_payload=True,
                 with_vectors=False,
             )
-            docs += [Document(**record.payload) for record in results]  # type: ignore
+            docs += [
+                self.config.document_class(**record.payload)  # type: ignore
+                for record in results
+            ]
+            # ignore
             if next_page_offset is None:
                 break
             offset = next_page_offset  # type: ignore
@@ -451,7 +455,7 @@ class QdrantDB(VectorStore):
         ]  # 2D list -> 1D list
         scores = [match.score for match in search_result if match is not None]
         docs = [
-            Document(**(match.payload))  # type: ignore
+            self.config.document_class(**(match.payload))  # type: ignore
             for match in search_result
             if match is not None
         ]

{langroid-0.3.1.dist-info → langroid-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.3.1
+Version: 0.5.1
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani

{langroid-0.3.1.dist-info → langroid-0.5.1.dist-info}/RECORD

@@ -1,25 +1,29 @@
 langroid/__init__.py,sha256=z_fCOLQJPOw3LLRPBlFB5-2HyCjpPgQa4m4iY5Fvb8Y,1800
 langroid/agent/__init__.py,sha256=ll0Cubd2DZ-fsCMl7e10hf9ZjFGKzphfBco396IKITY,786
-langroid/agent/base.py,sha256=eeYZ-NYbrepOjUVQS9K0nDhE8x2gKUNjgxFTA24mook,37560
+langroid/agent/base.py,sha256=x6SbInDGJUL_kusr-ligYsCwuaid2CmcRkzlucOXyw0,38999
 langroid/agent/batch.py,sha256=feRA_yRG768ElOQjrKEefcRv6Aefd_yY7qktuYUQDwc,10040
 langroid/agent/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/callbacks/chainlit.py,sha256=UKG2_v4ktfkEaGvdouVRHEqQejEYya2Rli8jrP65TmA,22055
-langroid/agent/chat_agent.py,sha256=bTQrIMbN8JxxtnVNC-xzODVLvH3SHmy5vijRjY3cCUE,41564
+langroid/agent/chat_agent.py,sha256=M5tdp1HuFthhMChLNd5XKBWxoiMSTkOuXlM8JoRLiUk,41586
 langroid/agent/chat_document.py,sha256=MwtNABK28tfSzqCeQlxoauT8uPn8oldU7dlnrX8aQ10,11232
 langroid/agent/helpers.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/junk,sha256=LxfuuW7Cijsg0szAzT81OjWWv1PMNI-6w_-DspVIO2s,339
 langroid/agent/openai_assistant.py,sha256=3saI9PwF8IZNJcjqyUy-rj73TInAzdlk14LiOvT_Dkc,33548
 langroid/agent/special/__init__.py,sha256=gik_Xtm_zV7U9s30Mn8UX3Gyuy4jTjQe9zjiE3HWmEo,1273
-langroid/agent/special/doc_chat_agent.py,sha256=CXFLfDMEabaBZwZwFgNOaG3E3S86xcBM4txrsMD_70I,54014
-langroid/agent/special/lance_doc_chat_agent.py,sha256=USp0U3eTaJzwF_3bdqE7CedSLbaqAi2tm-VzygcyLaA,10175
+langroid/agent/special/doc_chat_agent.py,sha256=8NPAhMnHkFUolQ8EHos40tz5Vwuz_m33NjUfjheXWXY,54569
+langroid/agent/special/lance_doc_chat_agent.py,sha256=Hjpu6u9UPAFMg5J6K97PRFaLbNrGhInC0N9oGi09CeY,10006
 langroid/agent/special/lance_rag/__init__.py,sha256=QTbs0IVE2ZgDg8JJy1zN97rUUg4uEPH7SLGctFNumk4,174
-langroid/agent/special/lance_rag/critic_agent.py,sha256=ufTdpHSeHgCzN85Q0sfWOrpBpsCjGVZdAg5yOH1ogU8,7296
+langroid/agent/special/lance_rag/critic_agent.py,sha256=S3NA3OAO7XaXjCrmwhKB7qCPlgRZFvDxiB5Qra65Zhs,7959
 langroid/agent/special/lance_rag/lance_rag_task.py,sha256=l_HQgrYY-CX2FwIsS961aEF3bYog3GDYo98fj0C0mSk,2889
-langroid/agent/special/lance_rag/query_planner_agent.py,sha256=M4RC_0f98_pwVL7ygrr1VI80LgJiFcmKjJFH0M4tccI,9830
-langroid/agent/special/lance_tools.py,sha256=BksGrrNgGgyYWP0HnfAuXMc0KzXooFOzY2l5rDDMtQ8,1467
+langroid/agent/special/lance_rag/query_planner_agent.py,sha256=QB8UYITUCkgSPturEwu_3i4kU8jXxW_jXNGSLlH5tMc,10109
+langroid/agent/special/lance_rag_new/__init__.py,sha256=QTbs0IVE2ZgDg8JJy1zN97rUUg4uEPH7SLGctFNumk4,174
+langroid/agent/special/lance_rag_new/critic_agent.py,sha256=ksUjI4Wd3SgXsEwMpEmCIvJrKd3k1k_AyUIYaNV5pBw,8341
+langroid/agent/special/lance_rag_new/lance_rag_task.py,sha256=mKo4lCC1ff5Jt9BlxwclQ_y3omw2S_f0MVIJfNmXM6w,5267
+langroid/agent/special/lance_rag_new/query_planner_agent.py,sha256=JqO_5fKW8HPn-zqKsZzX1sl05RgtcT63qpDMR9-q33A,10251
+langroid/agent/special/lance_tools.py,sha256=BznV_r3LAFyybvBRa9KQ0oU7mPM3uQVfri7PFp7M_qc,1894
 langroid/agent/special/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/special/neo4j/csv_kg_chat.py,sha256=dRsAgMBa1H_EMI2YYgJR2Xyv1D7e4o3G9M64mTewq_c,6409
-langroid/agent/special/neo4j/neo4j_chat_agent.py,sha256=Y4Zu-m8WKO1xjeBRarV_m4y00Y5n_NR2B-hepjZp_cY,13104
+langroid/agent/special/neo4j/neo4j_chat_agent.py,sha256=ryXFx2-ddlNzip8Zssrg0bCyNcFyIu4vax_z3WBXkyI,13106
 langroid/agent/special/neo4j/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/special/neo4j/utils/system_message.py,sha256=vRpz1P-OYLLiC6OGYYoK6x77yxVzDxMTCEJSsYUIuG4,2242
 langroid/agent/special/relevance_extractor_agent.py,sha256=zIx8GUdVo1aGW6ASla0NPQjYYIpmriK_TYMijqAx3F8,4796
@@ -34,7 +38,7 @@ langroid/agent/special/sql/utils/tools.py,sha256=vFYysk6Vi7HJjII8B4RitA3pt_z3gkS
 langroid/agent/special/table_chat_agent.py,sha256=d9v2wsblaRx7oMnKhLV7uO_ujvk9gh59pSGvBXyeyNc,9659
 langroid/agent/task.py,sha256=vKM2dmRYSH4i_VA0lf2axUtZcTGU44rVHz6EyxI4kG0,73990
 langroid/agent/team.py,sha256=88VNRSmK35WEl620GfBzuIrBASXYSeBZ8yDKX-nP_Bo,75778
-langroid/agent/tool_message.py,sha256=wIyZnUcZpxkiRPvM9O3MO3b5BBAdLEEan9kqPbvtApc,9743
+langroid/agent/tool_message.py,sha256=ggxmIZO_wi6x5uD-YWml07Bfgms-ohOSKHyQQdJFi4o,9571
 langroid/agent/tools/__init__.py,sha256=e-63cfwQNk_ftRKQwgDAJQK16QLbRVWDBILeXIc7wLk,402
 langroid/agent/tools/duckduckgo_search_tool.py,sha256=NhsCaGZkdv28nja7yveAhSK_w6l_Ftym8agbrdzqgfo,1935
 langroid/agent/tools/extract_tool.py,sha256=u5lL9rKBzaLBOrRyLnTAZ97pQ1uxyLP39XsWMnpaZpw,3789
@@ -118,20 +122,20 @@ langroid/utils/output/citations.py,sha256=PSY2cpti8W-ZGFMAgj1lYoEIZy0lsniLpCliMs
 langroid/utils/output/printing.py,sha256=yzPJZN-8_jyOJmI9N_oLwEDfjMwVgk3IDiwnZ4eK_AE,2962
 langroid/utils/output/status.py,sha256=rzbE7mDJcgNNvdtylCseQcPGCGghtJvVq3lB-OPJ49E,1049
 langroid/utils/pandas_utils.py,sha256=UctS986Jtl_MvU5rA7-GfrjEHXP7MNu8ePhepv0bTn0,755
-langroid/utils/pydantic_utils.py,sha256=FKC8VKXH2uBEpFjnnMgIcEsQn6hs31ftea8zv5pMK9g,21740
+langroid/utils/pydantic_utils.py,sha256=X35qxjE4sSIi-oBMkI1s9fiUIJbpXHLmJqcJ7zsy0jg,19914
 langroid/utils/system.py,sha256=nvKeeUAj4eviR4kYpcr9h-HYdhqUNMTRBTHBOhz0GdU,5182
 langroid/utils/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/utils/web/login.py,sha256=1iz9eUAHa87vpKIkzwkmFa00avwFWivDSAr7QUhK7U0,2528
 langroid/vector_store/__init__.py,sha256=6xBjb_z4QtUy4vz4RuFbcbSwmHrggHL8-q0DwCf3PMM,972
-langroid/vector_store/base.py,sha256=tuEPaxJcuU_39sRnUjjNd8D8n8IjP6jrbwQv_ecNpSw,13532
-langroid/vector_store/chromadb.py,sha256=bZ5HjwgKgfJj1PUHsatYsrHv-v0dpOfMR2l0tJ2H0_A,7890
-langroid/vector_store/lancedb.py,sha256=9x7e_5zo7nLhMbhjYby2ZpBJ-vyawcC0_XAuatfHJf8,20517
+langroid/vector_store/base.py,sha256=pkc4n0yWGVk7iRUOLFkU_ID5NiBFfAcA3lBlPNX79pU,13623
+langroid/vector_store/chromadb.py,sha256=KMfHrgovQEOeJR_LsMpGM8BteJ50wpisDu608RhU3SU,7940
+langroid/vector_store/lancedb.py,sha256=MLubJBhtNIFX6zY0qANqCoB6MlL-oZiJCg9gZp2H2rs,14620
 langroid/vector_store/meilisearch.py,sha256=6frB7GFWeWmeKzRfLZIvzRjllniZ1cYj3HmhHQICXLs,11663
 langroid/vector_store/momento.py,sha256=qR-zBF1RKVHQZPZQYW_7g-XpTwr46p8HJuYPCkfJbM4,10534
 langroid/vector_store/qdrant_cloud.py,sha256=3im4Mip0QXLkR6wiqVsjV1QvhSElfxdFSuDKddBDQ-4,188
-langroid/vector_store/qdrantdb.py,sha256=HkcK6jOf-FEDoOiG94MpsYDJr98T7vZkDyG__1BlnWI,17354
-pyproject.toml,sha256=x0YGXi9ennkubMYlFO-Eeyp6h2YE_aOBbeRJrUtTm34,7063
-langroid-0.3.1.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.3.1.dist-info/METADATA,sha256=9WLpuCfOtRfjB30PZa2jwGmnlotxXRZgHqt6UWiNh4E,54402
-langroid-0.3.1.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-langroid-0.3.1.dist-info/RECORD,,
+langroid/vector_store/qdrantdb.py,sha256=v88lqFkepADvlN6lByUj9I4NEKa9X9lWH16uTPPbYrE,17457
+pyproject.toml,sha256=1jCGCKPNCobDsOptcZnMKZIsI044DCR2YSnEPpEL4sY,7063
+langroid-0.5.1.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.5.1.dist-info/METADATA,sha256=TK7kU-GZZPw3-m2A1UJZwBH3kdL5wEHb0umIVGCcmeg,54402
+langroid-0.5.1.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+langroid-0.5.1.dist-info/RECORD,,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langroid"
-version = "0.3.1"
+version = "0.5.1"
 description = "Harness LLMs with Multi-Agent Programming"
 authors = ["Prasad Chalasani <pchalasani@gmail.com>"]
 readme = "README.md"