langroid 0.1.190__py3-none-any.whl → 0.1.192__py3-none-any.whl

langroid/agent/special/doc_chat_agent.py

@@ -34,7 +34,7 @@ from langroid.agent.task import Task
 from langroid.embedding_models.models import OpenAIEmbeddingsConfig
 from langroid.language_models.base import StreamingIfAllowed
 from langroid.language_models.openai_gpt import OpenAIChatModel, OpenAIGPTConfig
-from langroid.mytypes import Document, Entity
+from langroid.mytypes import DocMetaData, Document, Entity
 from langroid.parsing.parser import Parser, ParsingConfig, PdfParsingConfig, Splitter
 from langroid.parsing.repo_loader import RepoLoader
 from langroid.parsing.search import (
@@ -238,7 +238,10 @@ class DocChatAgent(ChatAgent):
     def ingest_doc_paths(
         self,
         paths: List[str],
-        metadata: List[Dict[str, Any]] | Dict[str, Any] = [],
+        metadata: List[Dict[str, Any]]
+        | Dict[str, Any]
+        | DocMetaData
+        | List[DocMetaData] = [],
     ) -> List[Document]:
         """Split, ingest docs from specified paths,
         do not add these to config.doc_paths.
@@ -250,14 +253,24 @@ class DocChatAgent(ChatAgent):
         Returns:
             List of Document objects
         """
+        all_paths = paths
         paths_meta: Dict[str, Any] = {}
         urls_meta: Dict[str, Any] = {}
         urls, paths = get_urls_and_paths(paths)
-        if len(metadata) > 0:
+        if (isinstance(metadata, list) and len(metadata) > 0) or not isinstance(
+            metadata, list
+        ):
             if isinstance(metadata, list):
-                path2meta = {p: m for p, m in zip(paths, metadata)}
+                path2meta = {
+                    p: m
+                    if isinstance(m, dict)
+                    else (isinstance(m, DocMetaData) and m.dict())  # appease mypy
+                    for p, m in zip(all_paths, metadata)
+                }
+            elif isinstance(metadata, dict):
+                path2meta = {p: metadata for p in all_paths}
             else:
-                path2meta = {p: metadata for p in paths}
+                path2meta = {p: metadata.dict() for p in all_paths}
             urls_meta = {u: path2meta[u] for u in urls}
             paths_meta = {p: path2meta[p] for p in paths}
         docs: List[Document] = []
@@ -298,7 +311,10 @@ class DocChatAgent(ChatAgent):
         self,
         docs: List[Document],
         split: bool = True,
-        metadata: List[Dict[str, Any]] | Dict[str, Any] = [],
+        metadata: List[Dict[str, Any]]
+        | Dict[str, Any]
+        | DocMetaData
+        | List[DocMetaData] = [],
     ) -> int:
         """
         Chunk docs into pieces, map each chunk to vec-embedding, store in vec-db
@@ -314,10 +330,15 @@ class DocChatAgent(ChatAgent):
         """
         if isinstance(metadata, list) and len(metadata) > 0:
             for d, m in zip(docs, metadata):
-                d.metadata = d.metadata.copy(update=m)
+                d.metadata = d.metadata.copy(
+                    update=m if isinstance(m, dict) else m.dict()  # type: ignore
+                )
         elif isinstance(metadata, dict):
             for d in docs:
                 d.metadata = d.metadata.copy(update=metadata)
+        elif isinstance(metadata, DocMetaData):
+            for d in docs:
+                d.metadata = d.metadata.copy(update=metadata.dict())
 
         self.original_docs.extend(docs)
         if self.parser is None:

langroid/agent/special/lance_doc_chat_agent.py

@@ -17,7 +17,7 @@ import pandas as pd
 
 from langroid.agent.special.doc_chat_agent import DocChatAgent, DocChatAgentConfig
 from langroid.agent.special.lance_rag.lance_tools import QueryPlanTool
-from langroid.mytypes import Document
+from langroid.mytypes import DocMetaData, Document
 from langroid.parsing.table_loader import describe_dataframe
 from langroid.utils.constants import DONE, NO_ANSWER
 from langroid.utils.pydantic_utils import (
@@ -38,6 +38,34 @@ class LanceDocChatAgent(DocChatAgent):
         self.enable_message(QueryPlanTool, use=False, handle=True)
 
     def _get_clean_vecdb_schema(self) -> str:
+        """Get a cleaned schema of the vector-db, to pass to the LLM
+        as part of instructions on how to generate a SQL filter."""
+        if len(self.config.filter_fields) == 0:
+            filterable_fields = (
+                self.vecdb.client.open_table(self.vecdb.config.collection_name)
+                .search()
+                .limit(1)
+                .to_pandas(flatten=True)
+                .columns.tolist()
+            )
+            # drop id, vector, metadata.id, metadata.window_ids, metadata.is_chunk
+            for fields in [
+                "id",
+                "vector",
+                "metadata.id",
+                "metadata.window_ids",
+                "metadata.is_chunk",
+            ]:
+                if fields in filterable_fields:
+                    filterable_fields.remove(fields)
+            logger.warning(
+                f"""
+            No filter_fields set in config, so using these fields as filterable fields:
+            {filterable_fields}
+            """
+            )
+            self.config.filter_fields = filterable_fields
+
         if self.from_dataframe:
             return self.df_description
         schema_dict = clean_schema(
@@ -50,6 +78,9 @@ class LanceDocChatAgent(DocChatAgent):
             self.config.filter_fields or schema_dict.keys()
         ).intersection(schema_dict.keys())
 
+        # remove 'content' from filter_fields_set, even if it's not in filter_fields_set
+        filter_fields_set.discard("content")
+
         # possible values of filterable fields
         filter_field_values = self.get_field_values(list(filter_fields_set))
 
@@ -63,18 +94,26 @@ class LanceDocChatAgent(DocChatAgent):
                 k: v for k, v in schema_dict.items() if k in self.config.filter_fields
             }
         schema = json.dumps(schema_dict, indent=4)
-        if len(fields := self.config.add_fields_to_content) > 0:
+
+        schema += f"""
+        NOTE when creating a filter for a query, 
+        ONLY the following fields are allowed:
+        {",".join(self.config.filter_fields)} 
+        """
+        if len(content_fields := self.config.add_fields_to_content) > 0:
             schema += f"""
             Additional fields added to `content` as key=value pairs:
-            NOTE That CAN Help with matching queries!
-            {fields}
+            NOTE that these CAN Help with matching queries!
+            {content_fields}
             """
         return schema
 
     def query_plan(self, msg: QueryPlanTool) -> str:
         """
         Handle the LLM's use of the FilterTool.
-        Temporarily set the config filter and invoke the DocChatAgent.llm_response()
+        Temporarily set the config filter and either return the final answer
+        in case there's a dataframe_calc, or return the rephrased query
+        so the LLM can handle it.
         """
         # create document-subset based on this filter
         plan = msg.plan
@@ -83,7 +122,14 @@ class LanceDocChatAgent(DocChatAgent):
         except Exception as e:
             logger.error(f"Error setting up documents: {e}")
             # say DONE with err msg so it goes back to LanceFilterAgent
-            return f"{DONE} Possible Filter Error:\n {e}"
+            return f"""
+            {DONE} Possible Filter Error:\n {e}
+            
+            Note that only the following fields are allowed in the filter
+            of a query plan: 
+            {", ".join(self.config.filter_fields)}
+            """
+
         # update the filter so it is used in the DocChatAgent
         self.config.filter = plan.filter or None
         if plan.dataframe_calc:
@@ -116,13 +162,16 @@ class LanceDocChatAgent(DocChatAgent):
         self,
         docs: List[Document],
         split: bool = True,
-        metadata: List[Dict[str, Any]] | Dict[str, Any] = [],
+        metadata: List[Dict[str, Any]]
+        | Dict[str, Any]
+        | DocMetaData
+        | List[DocMetaData] = [],
     ) -> int:
         n = super().ingest_docs(docs, split, metadata)
         tbl = self.vecdb.client.open_table(self.vecdb.config.collection_name)
         # We assume "content" is available as top-level field
         if "content" in tbl.schema.names:
-            tbl.create_fts_index("content")
+            tbl.create_fts_index("content", replace=True)
         return n
 
     def ingest_dataframe(
@@ -167,7 +216,7 @@ class LanceDocChatAgent(DocChatAgent):
         tbl = self.vecdb.client.open_table(self.vecdb.config.collection_name)
         # We assume "content" is available as top-level field
         if "content" in tbl.schema.names:
-            tbl.create_fts_index("content")
+            tbl.create_fts_index("content", replace=True)
         # We still need to do the below so that
         # other types of searches in DocChatAgent
         # can work, as they require Document objects

langroid/agent/special/lance_rag/query_planner_agent.py

@@ -159,7 +159,7 @@ class LanceQueryPlanAgent(ChatAgent):
         # then we know there was no tool, so we run below code
         if (
             isinstance(msg, ChatDocument)
-            and msg.metadata.sender_name == self.config.doc_agent_name
+            and self.curr_query_plan is not None
             and msg.metadata.parent is not None
         ):
             # save result, to be used in query_plan_feedback()
@@ -175,5 +175,6 @@ class LanceQueryPlanAgent(ChatAgent):
             response_tmpl.tool_messages = [query_plan_answer_tool]
             # set the recipient to the Critic so it can give feedback
             response_tmpl.metadata.recipient = self.config.critic_name
+            self.curr_query_plan = None  # reset
             return response_tmpl
         return None

langroid/language_models/openai_gpt.py

@@ -759,7 +759,11 @@ class OpenAIGPT(LanguageModel):
         )
 
     def _cache_store(self, k: str, v: Any) -> None:
-        self.cache.store(k, v)
+        try:
+            self.cache.store(k, v)
+        except Exception as e:
+            logging.error(f"Error in OpenAIGPT._cache_store: {e}")
+            pass
 
     def _cache_lookup(self, fn_name: str, **kwargs: Dict[str, Any]) -> Tuple[str, Any]:
         # Use the kwargs as the cache key
@@ -773,7 +777,12 @@ class OpenAIGPT(LanguageModel):
             # when caching disabled, return the hashed_key and none result
             return hashed_key, None
         # Try to get the result from the cache
-        return hashed_key, self.cache.retrieve(hashed_key)
+        try:
+            cached_val = self.cache.retrieve(hashed_key)
+        except Exception as e:
+            logging.error(f"Error in OpenAIGPT._cache_lookup: {e}")
+            return hashed_key, None
+        return hashed_key, cached_val
 
     def _cost_chat_model(self, prompt: int, completion: int) -> float:
         price = self.chat_cost()

langroid/utils/pydantic_utils.py

@@ -498,3 +498,78 @@ def dataframe_to_documents(
         for _, row in df.iterrows()
     ]
     return [m for m in docs if m is not None]
+
+
+def extra_metadata(document: Document, doc_cls: Type[Document] = Document) -> List[str]:
+    """
+    Checks for extra fields in a document's metadata that are not defined in the
+    original metadata schema.
+
+    Args:
+        document (Document): The document instance to check for extra fields.
+        doc_cls (Type[Document]): The class type derived from Document, used
+            as a reference to identify extra fields in the document's metadata.
+
+    Returns:
+        List[str]: A list of strings representing the keys of the extra fields found
+        in the document's metadata.
+    """
+    # Convert metadata to dict, including extra fields.
+    metadata_fields = set(document.metadata.dict().keys())
+
+    # Get defined fields in the metadata of doc_cls
+    defined_fields = set(doc_cls.__fields__["metadata"].type_.__fields__.keys())
+
+    # Identify extra fields not in defined fields.
+    extra_fields = list(metadata_fields - defined_fields)
+
+    return extra_fields
+
+
+def extend_document_class(d: Document) -> Type[Document]:
+    """Generates a new pydantic class based on a given document instance.
+
+    This function dynamically creates a new pydantic class with additional
+    fields based on the "extra" metadata fields present in the given document
+    instance. The new class is a subclass of the original Document class, with
+    the original metadata fields retained and extra fields added as normal
+    fields to the metadata.
+
+    Args:
+        d: An instance of the Document class.
+
+    Returns:
+        A new subclass of the Document class that includes the additional fields
+        found in the metadata of the given document instance.
+    """
+    # Extract the fields from the original metadata class, including types,
+    # correctly handling special types like List[str].
+    original_metadata_fields = {
+        k: (v.outer_type_ if v.shape != 1 else v.type_, ...)
+        for k, v in DocMetaData.__fields__.items()
+    }
+    # Extract extra fields from the metadata instance with their types
+    extra_fields = {
+        k: (type(v), ...)
+        for k, v in d.metadata.__dict__.items()
+        if k not in DocMetaData.__fields__
+    }
+
+    # Combine original and extra fields for the new metadata class
+    combined_fields = {**original_metadata_fields, **extra_fields}
+
+    # Create a new metadata class with combined fields
+    NewMetadataClass = create_model(  # type: ignore
+        "ExtendedDocMetadata", **combined_fields, __base__=DocMetaData
+    )
+    # NewMetadataClass.__config__.arbitrary_types_allowed = True
+
+    # Create a new document class using the new metadata class
+    NewDocumentClass = create_model(
+        "ExtendedDocument",
+        content=(str, ...),
+        metadata=(NewMetadataClass, ...),
+        __base__=Document,
+    )
+
+    return NewDocumentClass

langroid/vector_store/lancedb.py

@@ -18,6 +18,8 @@ from langroid.utils.configuration import settings
 from langroid.utils.pydantic_utils import (
     dataframe_to_document_model,
     dataframe_to_documents,
+    extend_document_class,
+    extra_metadata,
     flatten_pydantic_instance,
     flatten_pydantic_model,
     nested_dict_from_flat,
@@ -33,6 +35,8 @@ class LanceDBConfig(VectorStoreConfig):
     storage_path: str = ".lancedb/data"
     embedding: EmbeddingModelsConfig = OpenAIEmbeddingsConfig()
     distance: str = "cosine"
+    # document_class is used to store in lancedb with right schema,
+    # and also to retrieve the right type of Documents when searching.
     document_class: Type[Document] = Document
     flatten: bool = False  # flatten Document class into LanceSchema ?
 
@@ -231,8 +235,30 @@ class LanceDB(VectorStore):
         ):
             # collection either doesn't exist or is empty, so replace it,
             # possibly with a new schema
-            doc_cls = type(documents[0])
-            self.config.document_class = doc_cls
+            extra_metadata_fields = extra_metadata(
+                documents[0], self.config.document_class
+            )
+            if len(extra_metadata_fields) > 0:
+                logger.warning(
+                    f"""
+                    Added documents contain extra metadata fields:
+                    {extra_metadata_fields}
+                    Trying to recreate the collection {coll_name} with a new schema:
+                    Overriding LanceDBConfig.document_class with an auto-generated 
+                    Pydantic class that includes these extra fields.
+                    If this fails, or you see odd results, it is recommended that you 
+                    define a subclass of Document, with metadata of class derived from 
+                    DocMetaData, with extra fields defined via 
+                    `Field(..., description="...")` declarations,
+                    and set this document class as the value of the 
+                    LanceDBConfig.document_class attribute.
+                    """
+                )
+
+                doc_cls = extend_document_class(documents[0])
+                self.config.document_class = doc_cls
+
+            doc_cls = self.config.document_class
             self._setup_schemas(doc_cls)
             self.create_collection(coll_name, replace=True)
 

{langroid-0.1.190.dist-info → langroid-0.1.192.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.190
+Version: 0.1.192
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani

{langroid-0.1.190.dist-info → langroid-0.1.192.dist-info}/RECORD

@@ -10,13 +10,13 @@ langroid/agent/helpers.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/junk,sha256=LxfuuW7Cijsg0szAzT81OjWWv1PMNI-6w_-DspVIO2s,339
 langroid/agent/openai_assistant.py,sha256=yBtxis64XOnxtJzlkwUoTm-wCyvKr4DGo9-laXYMok0,32654
 langroid/agent/special/__init__.py,sha256=xj4TvQ_oQX_xYPySbhmQAi2CPhuy_3yQPqqwzb4wsc0,943
-langroid/agent/special/doc_chat_agent.py,sha256=0CpI9Pt4aoOoAH2TP2gS3gWj0yKJJPmFfBSOGMTBXgs,48231
-langroid/agent/special/lance_doc_chat_agent.py,sha256=tUpn6vnbHzlkkpKEHJym7dtxS-s39AwPfH2PqyNJuxY,8371
+langroid/agent/special/doc_chat_agent.py,sha256=XGKliuFv1KM58TQsvKeXpD1eAzsRdK1lpr1KVugDQ98,49015
+langroid/agent/special/lance_doc_chat_agent.py,sha256=pAIJchnBOVZnd2fxTteF0KSyZHMzTLKDj8vziTRuUUk,10184
 langroid/agent/special/lance_rag/__init__.py,sha256=-pq--upe-8vycYoTwxoomBnuUqrcRFUukmW3uBL1cFM,219
 langroid/agent/special/lance_rag/critic_agent.py,sha256=9izW4keCxVZEqrFOgyVUHD7N1vTXLkRynXYYd1Vpwzw,5785
 langroid/agent/special/lance_rag/lance_rag_task.py,sha256=l_HQgrYY-CX2FwIsS961aEF3bYog3GDYo98fj0C0mSk,2889
 langroid/agent/special/lance_rag/lance_tools.py,sha256=WypIS-3ZMDqY_PZEGB2K80-o4RfS43_OnER0dyFlsDY,1339
-langroid/agent/special/lance_rag/query_planner_agent.py,sha256=dZXVano2NbRZy91nBcEW6LrvedsHfxL1oNCgMQEHZ-U,8016
+langroid/agent/special/lance_rag/query_planner_agent.py,sha256=klWYyMSAaSwLwWBbTUHvZfni-nB7QWfni8vpo3QlfhQ,8043
 langroid/agent/special/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/special/neo4j/csv_kg_chat.py,sha256=YfuyblH9d307ggzN7fUD2mxLorIdexjVAlZ8uVDZqhY,6394
 langroid/agent/special/neo4j/neo4j_chat_agent.py,sha256=UGsnneFQVrzyNB5AhuY0BhnN-BH7me7dIpi7q8cw8ZA,12155
@@ -58,7 +58,7 @@ langroid/language_models/azure_openai.py,sha256=ncRCbKooqLVOY-PWQUIo9C3yTuKEFbAw
 langroid/language_models/base.py,sha256=RdXH-BnkFGS8xZTiukdxHTFxqELVSmf546itp9Fa8fs,21008
 langroid/language_models/config.py,sha256=5UF3DzO1a-Dfsc3vghE0XGq7g9t_xDsRCsuRiU4dgBg,366
 langroid/language_models/openai_assistants.py,sha256=9K-DEAL2aSWHeXj2hwCo2RAlK9_1oCPtqX2u1wISCj8,36
-langroid/language_models/openai_gpt.py,sha256=9gm6URAIh2-katOQGl8BTpDp0JtE20JJptPRR_-HfWY,48252
+langroid/language_models/openai_gpt.py,sha256=oBvrEjiCMPeHzZCSWZm9mmfAIWrcVqc_hsrJCbzLOGU,48568
 langroid/language_models/prompt_formatter/__init__.py,sha256=9JXFF22QNMmbQV1q4nrIeQVTtA3Tx8tEZABLtLBdFyc,352
 langroid/language_models/prompt_formatter/base.py,sha256=eDS1sgRNZVnoajwV_ZIha6cba5Dt8xjgzdRbPITwx3Q,1221
 langroid/language_models/prompt_formatter/hf_formatter.py,sha256=3MQhu8--p168qPWXqlp_nK4phi-SuAUMqahSVyLHIkA,4177
@@ -102,7 +102,7 @@ langroid/utils/logging.py,sha256=R8TN-FqVpwZ4Ajgls9TDMthLvPpQd0QVNXK-PJDj1Z8,391
 langroid/utils/output/__init__.py,sha256=Z58-2ZKnGpGNaKw_nEjHV_CHTzjMz-WRSRQnazTLrWU,289
 langroid/utils/output/printing.py,sha256=5EsYB1O4qKhocW19aebOUzK82RD9U5nygbY21yo8gfg,2872
 langroid/utils/pandas_utils.py,sha256=nSA1tIgOUTkRDn-IKq7HP8XGJcL6bA110LcPfRF7h8I,707
-langroid/utils/pydantic_utils.py,sha256=hx5TV_U_8tTXn8rRItPs0RfYYo4og3EcXz7cddsJMS8,17008
+langroid/utils/pydantic_utils.py,sha256=ctv5L6s94_F5CNnNsHybioQKu7Mp_upZcrMWxwmx57o,19827
 langroid/utils/system.py,sha256=x9204H1-6EubDe8-9yX87KZSmRCkf0puXQv91QOetF4,3326
 langroid/utils/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/utils/web/login.py,sha256=1iz9eUAHa87vpKIkzwkmFa00avwFWivDSAr7QUhK7U0,2528
@@ -110,12 +110,12 @@ langroid/utils/web/selenium_login.py,sha256=mYI6EvVmne34N9RajlsxxRqJQJvV-WG4LGp6
 langroid/vector_store/__init__.py,sha256=qOa3_BLvf8tjdUBT4Zq7pSLTY9TD2Fgw62UHHJWNu8w,557
 langroid/vector_store/base.py,sha256=JNk-2f6t_WCavizU332tOoZcXHP73RpobRk88Aus52w,13706
 langroid/vector_store/chromadb.py,sha256=Y80k6an5sN0cRWtcl78Xr-Ht87nd_hBjvkSU5OdCyY8,7312
-langroid/vector_store/lancedb.py,sha256=dsdZVHfnvOcs1BhVK99UknbIjbND0j6I6d2_QozuL7A,16671
+langroid/vector_store/lancedb.py,sha256=Vl0nWKqFyczgPRmWXLzof9UgOB0OhVZIuczY_rSAF10,17985
 langroid/vector_store/meilisearch.py,sha256=d2huA9P-NoYRuAQ9ZeXJmMKr7ry8u90RUSR28k2ecQg,11340
 langroid/vector_store/momento.py,sha256=j6Eo6oIDN2fe7lsBOlCXJn3uvvERHHTFL5QJfeREeOM,10044
 langroid/vector_store/qdrant_cloud.py,sha256=3im4Mip0QXLkR6wiqVsjV1QvhSElfxdFSuDKddBDQ-4,188
 langroid/vector_store/qdrantdb.py,sha256=_egbsP9SWBwmI827EDYSSOqfIQSmwNsmJfFTxrLpWYE,13457
-langroid-0.1.190.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.1.190.dist-info/METADATA,sha256=VX8PncumZaLEiukTyz2PmCiWDHIhJEcU-e5Ik7BDzZA,45671
-langroid-0.1.190.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-langroid-0.1.190.dist-info/RECORD,,
+langroid-0.1.192.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.1.192.dist-info/METADATA,sha256=-_fDgrzNhWdaDfOQBooJicN992gMfxUbn8Enut6Ymvo,45671
+langroid-0.1.192.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+langroid-0.1.192.dist-info/RECORD,,