langroid 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl

langroid/agent/chat_agent.py

@@ -416,7 +416,7 @@ class ChatAgent(Agent):
         ]
 
         if len(usable_tool_classes) == 0:
-            return "You can ask questions in natural language."
+            return ""
         format_instructions = "\n\n".join(
             [
                 msg_cls.format_instructions(tool=self.config.use_tools)
@@ -568,17 +568,14 @@ class ChatAgent(Agent):
         Returns:
             LLMMessage object
         """
-        content = textwrap.dedent(
-            f"""
-            {self.system_message}
-            
-            {self.system_tool_instructions}
-            
-            {self.system_tool_format_instructions}
+        content = self.system_message
+        if self.system_tool_instructions != "":
+            content += "\n\n" + self.system_tool_instructions
+        if self.system_tool_format_instructions != "":
+            content += "\n\n" + self.system_tool_format_instructions
+        if self.output_format_instructions != "":
+            content += "\n\n" + self.output_format_instructions
 
-            {self.output_format_instructions}
-            """
-        )
         # remove leading and trailing newlines and other whitespace
         return LLMMessage(role=Role.SYSTEM, content=content.strip())
 

langroid/vector_store/__init__.py

@@ -48,3 +48,14 @@ try:
     __all__.extend(["chromadb", "ChromaDBConfig", "ChromaDB"])
 except ImportError:
     pass
+
+try:
+    from . import weaviatedb
+    from .weaviatedb import WeaviateDBConfig, WeaviateDB
+
+    weaviatedb
+    WeaviateDB
+    WeaviateDBConfig
+    __all__.extend(["weaviatedb", "WeaviateDB", "WeaviateDBConfig"])
+except ImportError:
+    pass

langroid/vector_store/base.py

@@ -59,6 +59,7 @@ class VectorStore(ABC):
         from langroid.vector_store.meilisearch import MeiliSearch, MeiliSearchConfig
         from langroid.vector_store.momento import MomentoVI, MomentoVIConfig
         from langroid.vector_store.qdrantdb import QdrantDB, QdrantDBConfig
+        from langroid.vector_store.weaviatedb import WeaviateDB, WeaviateDBConfig
 
         if isinstance(config, QdrantDBConfig):
             return QdrantDB(config)
@@ -70,6 +71,8 @@ class VectorStore(ABC):
             return LanceDB(config)
         elif isinstance(config, MeiliSearchConfig):
             return MeiliSearch(config)
+        elif isinstance(config, WeaviateDBConfig):
+            return WeaviateDB(config)
 
         else:
             logger.warning(

langroid/vector_store/weaviatedb.py

@@ -0,0 +1,271 @@
+import logging
+import os
+import re
+from typing import Any, List, Optional, Sequence, Tuple
+
+from dotenv import load_dotenv
+
+from langroid.embedding_models.base import (
+    EmbeddingModelsConfig,
+)
+from langroid.embedding_models.models import OpenAIEmbeddingsConfig
+from langroid.exceptions import LangroidImportError
+from langroid.mytypes import DocMetaData, Document, EmbeddingFunction
+from langroid.utils.configuration import settings
+from langroid.vector_store.base import VectorStore, VectorStoreConfig
+
+logger = logging.getLogger(__name__)
+try:
+    import weaviate
+    from weaviate.classes.config import (
+        Configure,
+        VectorDistances,
+    )
+    from weaviate.classes.init import Auth
+    from weaviate.classes.query import Filter, MetadataQuery
+    from weaviate.util import generate_uuid5, get_valid_uuid
+except ImportError:
+    raise LangroidImportError("weaviate", "weaviate")
+
+
+class WeaviateDBConfig(VectorStoreConfig):
+    collection_name: str | None = "temp"
+    embedding: EmbeddingModelsConfig = OpenAIEmbeddingsConfig()
+    distance: str = VectorDistances.COSINE
+
+
+class WeaviateDB(VectorStore):
+    def __init__(self, config: WeaviateDBConfig = WeaviateDBConfig()):
+        super().__init__(config)
+        self.config: WeaviateDBConfig = config
+        self.embedding_fn: EmbeddingFunction = self.embedding_model.embedding_fn()
+        self.embedding_dim = self.embedding_model.embedding_dims
+        load_dotenv()
+        key = os.getenv("WEAVIATE_API_KEY")
+        url = os.getenv("WEAVIATE_API_URL")
+        if None in [key, url]:
+            logger.warning(
+                """WEAVIATE_API_KEY, WEAVIATE_API_URL env variable must be set to use
+                WeaviateDB in cloud mode. Please set these values
+                in your .env file.
+                """
+            )
+        self.client = weaviate.connect_to_weaviate_cloud(
+            cluster_url=url,
+            auth_credentials=Auth.api_key(key),
+        )
+        if config.collection_name is not None:
+            WeaviateDB.validate_and_format_collection_name(config.collection_name)
+
+    def clear_empty_collections(self) -> int:
+        colls = self.client.collections.list_all()
+        n_deletes = 0
+        for coll_name, _ in colls.items():
+            val = self.client.collections.get(coll_name)
+            if len(val) == 0:
+                n_deletes += 1
+                self.client.collections.delete(coll_name)
+        return n_deletes
+
+    def list_collections(self, empty: bool = False) -> List[str]:
+        colls = self.client.collections.list_all()
+        if empty:
+            return list(colls.keys())
+        non_empty_colls = [
+            coll_name
+            for coll_name in colls.keys()
+            if len(self.client.collections.get(coll_name)) > 0
+        ]
+
+        return non_empty_colls
+
+    def clear_all_collections(self, really: bool = False, prefix: str = "") -> int:
+        if not really:
+            logger.warning(
+                "Not really deleting all collections ,set really=True to confirm"
+            )
+            return 0
+        coll_names = [
+            c for c in self.list_collections(empty=True) if c.startswith(prefix)
+        ]
+        if len(coll_names) == 0:
+            logger.warning(f"No collections found with prefix {prefix}")
+            return 0
+        n_empty_deletes = 0
+        n_non_empty_deletes = 0
+        for name in coll_names:
+            info = self.client.collections.get(name)
+            points_count = len(info)
+
+            n_empty_deletes += points_count == 0
+            n_non_empty_deletes += points_count > 0
+            self.client.collections.delete(name)
+        logger.warning(
+            f"""
+            Deleted {n_empty_deletes} empty collections and
+            {n_non_empty_deletes} non-empty collections.
+            """
+        )
+        return n_empty_deletes + n_non_empty_deletes
+
+    def delete_collection(self, collection_name: str) -> None:
+        self.client.collections.delete(name=collection_name)
+
+    def create_collection(self, collection_name: str, replace: bool = False) -> None:
+        collection_name = WeaviateDB.validate_and_format_collection_name(
+            collection_name
+        )
+        self.config.collection_name = collection_name
+        if self.client.collections.exists(name=collection_name):
+            coll = self.client.collections.get(name=collection_name)
+            if len(coll) > 0:
+                logger.warning(f"Non-empty Collection {collection_name} already exists")
+                if not replace:
+                    logger.warning("Not replacing collection")
+                    return
+                else:
+                    logger.warning("Recreating fresh collection")
+            self.client.collections.delete(name=collection_name)
+
+        vector_index_config = Configure.VectorIndex.hnsw(
+            distance_metric=VectorDistances.COSINE,
+        )
+        if self.config.embedding == OpenAIEmbeddingsConfig:
+            vectorizer_config = Configure.Vectorizer.text2vec_openai(
+                model=self.embedding_model
+            )
+        else:
+            vectorizer_config = None
+
+        collection_info = self.client.collections.create(
+            name=collection_name,
+            vector_index_config=vector_index_config,
+            vectorizer_config=vectorizer_config,
+        )
+        collection_info = self.client.collections.get(name=collection_name)
+        assert len(collection_info) in [0, None]
+        if settings.debug:
+            level = logger.getEffectiveLevel()
+            logger.setLevel(logging.INFO)
+            logger.info(collection_info)
+            logger.setLevel(level)
+
+    def add_documents(self, documents: Sequence[Document]) -> None:
+        super().maybe_add_ids(documents)
+        colls = self.list_collections(empty=True)
+        for doc in documents:
+            doc.metadata.id = str(self._create_valid_uuid_id(doc.metadata.id))
+        if len(documents) == 0:
+            return
+
+        document_dicts = [doc.dict() for doc in documents]
+        embedding_vecs = self.embedding_fn([doc.content for doc in documents])
+        if self.config.collection_name is None:
+            raise ValueError("No collection name set, cannot ingest docs")
+        if self.config.collection_name not in colls:
+            self.create_collection(self.config.collection_name, replace=True)
+        coll_name = self.client.collections.get(self.config.collection_name)
+        with coll_name.batch.dynamic() as batch:
+            for i, doc_dict in enumerate(document_dicts):
+                id = doc_dict["metadata"].pop("id", None)
+                batch.add_object(properties=doc_dict, uuid=id, vector=embedding_vecs[i])
+
+    def get_all_documents(self, where: str = "") -> List[Document]:
+        if self.config.collection_name is None:
+            raise ValueError("No collection name set, cannot retrieve docs")
+        # cannot use filter as client does not support json type queries
+        coll = self.client.collections.get(self.config.collection_name)
+        return [self.weaviate_obj_to_doc(item) for item in coll.iterator()]
+
+    def get_documents_by_ids(self, ids: List[str]) -> List[Document]:
+        if self.config.collection_name is None:
+            raise ValueError("No collection name set, cannot retrieve docs")
+
+        docs = []
+        coll_name = self.client.collections.get(self.config.collection_name)
+
+        result = coll_name.query.fetch_objects(
+            filters=Filter.by_property("_id").contains_any(ids), limit=len(coll_name)
+        )
+
+        id_to_doc = {}
+        for item in result.objects:
+            doc = self.weaviate_obj_to_doc(item)
+            id_to_doc[doc.metadata.id] = doc
+
+        # Reconstruct the list of documents in the original order of input ids
+        docs = [id_to_doc[id] for id in ids if id in id_to_doc]
+
+        return docs
+
+    def similar_texts_with_scores(
+        self, text: str, k: int = 1, where: Optional[str] = None
+    ) -> List[Tuple[Document, float]]:
+        embedding = self.embedding_fn([text])[0]
+        if self.config.collection_name is None:
+            raise ValueError("No collections name set,cannot search")
+        coll = self.client.collections.get(self.config.collection_name)
+        response = coll.query.near_vector(
+            near_vector=embedding,
+            limit=k,
+            return_properties=True,
+            return_metadata=MetadataQuery(distance=True),
+        )
+        return [
+            (self.weaviate_obj_to_doc(item), 1 - item.metadata.distance)
+            for item in response.objects
+        ]
+
+    def _create_valid_uuid_id(self, id: str) -> Any:
+        try:
+            id = get_valid_uuid(id)
+            return id
+        except Exception:
+            return generate_uuid5(id)
+
+    def weaviate_obj_to_doc(self, input_object: Any) -> Document:
+        content = input_object.properties.get("content", "")
+        metadata_dict = input_object.properties.get("metadata", {})
+
+        window_ids = metadata_dict.pop("window_ids", [])
+        window_ids = [str(uuid) for uuid in window_ids]
+
+        # Ensure the id is a valid UUID string
+        id_value = get_valid_uuid(input_object.uuid)
+
+        metadata = DocMetaData(id=id_value, window_ids=window_ids, **metadata_dict)
+
+        return Document(content=content, metadata=metadata)
+
+    @staticmethod
+    def validate_and_format_collection_name(name: str) -> str:
+        """
+        Formats the collection name to comply with Weaviate's naming rules:
+        - Name must start with a capital letter.
+        - Name can only contain letters, numbers, and underscores.
+        - Replaces invalid characters with underscores.
+        """
+        if not name:
+            raise ValueError("Collection name cannot be empty.")
+
+        formatted_name = re.sub(r"[^a-zA-Z0-9_]", "_", name)
+
+        # Ensure the first letter is capitalized
+        if not formatted_name[0].isupper():
+            formatted_name = formatted_name.capitalize()
+
+        # Check if the name now meets the criteria
+        if not re.match(r"^[A-Z][A-Za-z0-9_]*$", formatted_name):
+            raise ValueError(
+                f"Invalid collection name '{name}'."
+                " Names must start with a capital letter "
+                "and contain only letters, numbers, and underscores."
+            )
+
+        if formatted_name != name:
+            logger.warning(
+                f"Collection name '{name}' was reformatted to '{formatted_name}' "
+                "to comply with Weaviate's rules."
+            )
+
+        return formatted_name

{langroid-0.35.0.dist-info → langroid-0.36.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.35.0
+Version: 0.36.0
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -75,6 +75,7 @@ Requires-Dist: sqlalchemy<3.0.0,>=2.0.19; extra == 'all'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'all'
 Requires-Dist: transformers<5.0.0,>=4.40.1; extra == 'all'
 Requires-Dist: unstructured[docx,pdf,pptx]<0.10.18,>=0.10.16; extra == 'all'
+Requires-Dist: weaviate-client>=4.9.6; extra == 'all'
 Provides-Extra: arango
 Requires-Dist: arango-datasets<2.0.0,>=1.2.2; extra == 'arango'
 Requires-Dist: python-arango<9.0.0,>=8.1.2; extra == 'arango'
@@ -148,6 +149,9 @@ Requires-Dist: chromadb<=0.4.23,>=0.4.21; extra == 'vecdbs'
 Requires-Dist: lancedb<0.9.0,>=0.8.2; extra == 'vecdbs'
 Requires-Dist: pyarrow<16.0.0,>=15.0.0; extra == 'vecdbs'
 Requires-Dist: tantivy<0.22.0,>=0.21.0; extra == 'vecdbs'
+Requires-Dist: weaviate-client>=4.9.6; extra == 'vecdbs'
+Provides-Extra: weaviate
+Requires-Dist: weaviate-client>=4.9.6; extra == 'weaviate'
 Description-Content-Type: text/markdown
 
 <div align="center">

{langroid-0.35.0.dist-info → langroid-0.36.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ langroid/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/__init__.py,sha256=ll0Cubd2DZ-fsCMl7e10hf9ZjFGKzphfBco396IKITY,786
 langroid/agent/base.py,sha256=oThlrYygKDu1-bKjAfygldJ511gMKT8Z0qCrD52DdDM,77834
 langroid/agent/batch.py,sha256=vi1r5i1-vN80WfqHDSwjEym_KfGsqPGUtwktmiK1nuk,20635
-langroid/agent/chat_agent.py,sha256=A-7Iiiw7jsoJNlWerljM29BidkiIbjPOQIkGZpZHmt0,82210
+langroid/agent/chat_agent.py,sha256=UvcZRoQ5jIYvlei8rku0T2Ul8tMpEhCJ2FGvr5_yc5Q,82275
 langroid/agent/chat_document.py,sha256=xzMtrPbaW-Y-BnF7kuhr2dorsD-D5rMWzfOqJ8HAoo8,17885
 langroid/agent/openai_assistant.py,sha256=JkAcs02bIrgPNVvUWVR06VCthc5-ulla2QMBzux_q6o,34340
 langroid/agent/task.py,sha256=XrXUbSoiFasvpIsZPn_cBpdWaTCKljJPRimtLMrSZrs,90347
@@ -114,14 +114,15 @@ langroid/utils/output/__init__.py,sha256=7P0f--4IZneNsTxXY5fd6d6iW-CeVe-KSsl-87s
 langroid/utils/output/citations.py,sha256=PSY2cpti8W-ZGFMAgj1lYoEIZy0lsniLpCliMsVkXtc,1425
 langroid/utils/output/printing.py,sha256=yzPJZN-8_jyOJmI9N_oLwEDfjMwVgk3IDiwnZ4eK_AE,2962
 langroid/utils/output/status.py,sha256=rzbE7mDJcgNNvdtylCseQcPGCGghtJvVq3lB-OPJ49E,1049
-langroid/vector_store/__init__.py,sha256=6xBjb_z4QtUy4vz4RuFbcbSwmHrggHL8-q0DwCf3PMM,972
-langroid/vector_store/base.py,sha256=L9_tIr8tghV09sbLZof6MhSp-wMxtdWxkAJsiU6A4-g,14038
+langroid/vector_store/__init__.py,sha256=BcoOm1tG3y0EqjkIGmMOHkY9iTUhDHgyruknWDKgqIg,1214
+langroid/vector_store/base.py,sha256=c9slwOcSWCG0SFGDuPLAQF9vBLDb4Eg8uaUol27Jf9c,14209
 langroid/vector_store/chromadb.py,sha256=9WXW9IoSnhOmGEtMruVhEtVWL_VO6NXnPIz-nzh0gIQ,8235
 langroid/vector_store/lancedb.py,sha256=b3_vWkTjG8mweZ7ZNlUD-NjmQP_rLBZfyKWcxt2vosA,14855
 langroid/vector_store/meilisearch.py,sha256=6frB7GFWeWmeKzRfLZIvzRjllniZ1cYj3HmhHQICXLs,11663
 langroid/vector_store/momento.py,sha256=UNHGT6jXuQtqY9f6MdqGU14bVnS0zHgIJUa30ULpUJo,10474
 langroid/vector_store/qdrantdb.py,sha256=HRLCt-FG8y4718omwpFaQZnWeYxPj0XCwS4tjokI1sU,18116
-langroid-0.35.0.dist-info/METADATA,sha256=57TZcMnBYqUH2CiqMsjLqJA2oclSLKY4igiXFlZwx_Y,59313
-langroid-0.35.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.35.0.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.35.0.dist-info/RECORD,,
+langroid/vector_store/weaviatedb.py,sha256=Jxe-cp2PyZdQ9NQVNZJ-CnsYsNxgUBdfAOoLZQEN650,10602
+langroid-0.36.0.dist-info/METADATA,sha256=aDLzYdeo80UbUZB0oEv9Rc1WMgWvG_jQtrBZnI2y5Bg,59508
+langroid-0.36.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.36.0.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.36.0.dist-info/RECORD,,