langroid 0.1.246__py3-none-any.whl → 0.1.248__py3-none-any.whl

langroid/agent/base.py

@@ -94,9 +94,16 @@ class Agent(ABC):
         self._indent = ""
         self.llm = LanguageModel.create(config.llm)
         self.vecdb = VectorStore.create(config.vecdb) if config.vecdb else None
+        # token_encoding_model is used to obtain the tokenizer,
+        # so we ensure that the tokenizer corresponding to the model is used.
+        if config.parsing is not None and self.config.llm is not None:
+            config.parsing.token_encoding_model = (
+                self.config.llm.chat_model or self.config.llm.completion_model
+            )
         self.parser: Optional[Parser] = (
             Parser(config.parsing) if config.parsing else None
         )
+
         self.callbacks = SimpleNamespace(
             start_llm_stream=lambda: noop_fn,
             cancel_llm_stream=noop_fn,

langroid/parsing/parser.py

@@ -56,7 +56,10 @@ class ParsingConfig(BaseSettings):
 class Parser:
     def __init__(self, config: ParsingConfig):
         self.config = config
-        self.tokenizer = tiktoken.encoding_for_model(config.token_encoding_model)
+        try:
+            self.tokenizer = tiktoken.encoding_for_model(config.token_encoding_model)
+        except Exception:
+            self.tokenizer = tiktoken.encoding_for_model("text-embedding-ada-002")
 
     def num_tokens(self, text: str) -> int:
         tokens = self.tokenizer.encode(text)

{langroid-0.1.246.dist-info → langroid-0.1.248.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.246
+Version: 0.1.248
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani
@@ -99,7 +99,7 @@ Requires-Dist: sentence-transformers (==2.2.2) ; extra == "hf-embeddings"
 Requires-Dist: sqlalchemy (>=2.0.19,<3.0.0)
 Requires-Dist: tantivy (>=0.21.0,<0.22.0)
 Requires-Dist: thefuzz (>=0.20.0,<0.21.0)
-Requires-Dist: tiktoken (>=0.5.1,<0.6.0)
+Requires-Dist: tiktoken (>=0.7.0,<0.8.0)
 Requires-Dist: torch (==2.0.0) ; extra == "hf-embeddings" or extra == "transformers"
 Requires-Dist: trafilatura (>=1.5.0,<2.0.0)
 Requires-Dist: transformers (>=4.40.1,<5.0.0) ; extra == "transformers"
@@ -236,6 +236,11 @@ teacher_task.run()
 <details>
 <summary> <b>Click to expand</b></summary>
 
+- **May 2024:** 
+  - `gpt-4o` is now the default LLM throughout; Update tests and examples to work 
+    with this LLM; use tokenizer corresponding to the LLM.
+  - `gemini 1.5 pro` support via `litellm`
+  - `QdrantDB:` update to support learned sparse embeddings.
 - **Apr 2024:**
   - **0.1.236**: Support for open LLMs hosted on Groq, e.g. specify 
     `chat_model="groq/llama3-8b-8192"`.

{langroid-0.1.246.dist-info → langroid-0.1.248.dist-info}/RECORD

@@ -1,6 +1,6 @@
 langroid/__init__.py,sha256=zsYpGiAUsvyzZzjm964NUamsJImrXSJPVGz9a2jE_uY,1679
 langroid/agent/__init__.py,sha256=_D8dxnfdr92ch1CIrUkKjrB5HVvsQdn62b1Fb2kBxV8,785
-langroid/agent/base.py,sha256=H1Fk9kJKr-7I1GgbmaflumG7so6yGmwl7AVTgU7PyeM,35210
+langroid/agent/base.py,sha256=Aa8e_dHibXfpOXnldGCrVvObFKhO0P05Is97AuVGFLo,35569
 langroid/agent/batch.py,sha256=feRA_yRG768ElOQjrKEefcRv6Aefd_yY7qktuYUQDwc,10040
 langroid/agent/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/callbacks/chainlit.py,sha256=uIfdmoOKIfzBVeM2ce0V5SCfHjcvYyaz13dfjFXCEN4,21020
@@ -79,7 +79,7 @@ langroid/parsing/document_parser.py,sha256=sfyNhkIdAQkufOHb-tTv63EF51pGNzrnAl0DN
 langroid/parsing/image_text.py,sha256=sbLIQ5nHe2UnYUksBaQsmZGaX-X0qgEpPd7CEzi_z5M,910
 langroid/parsing/para_sentence_split.py,sha256=AJBzZojP3zpB-_IMiiHismhqcvkrVBQ3ZINoQyx_bE4,2000
 langroid/parsing/parse_json.py,sha256=tgB_oatcrgt6L9ZplC-xBBXjLzL1gjSQf1L2_W5kwFA,4230
-langroid/parsing/parser.py,sha256=vE5j1LVDeFQPmLrXCWBfvuoPsjjvVIGHcsIWCBR8HDM,10617
+langroid/parsing/parser.py,sha256=2TT6YMgEe79Kz9bPIqI-1RIEK77V2H2gbpbX5DhNNrY,10743
 langroid/parsing/repo_loader.py,sha256=My5UIe-h1xr0I-6Icu0ZVwRHmGRRRW8SrJYMc9J1M9Q,29361
 langroid/parsing/search.py,sha256=plQtjarB9afGfJLB0CyPXPq3mM4m7kRsfd0_4brziEI,8846
 langroid/parsing/spider.py,sha256=w_mHR1B4KOmxsBLoVI8kMkMTEbwTzeK3ath9fOMJrTk,3043
@@ -121,7 +121,7 @@ langroid/vector_store/meilisearch.py,sha256=d2huA9P-NoYRuAQ9ZeXJmMKr7ry8u90RUSR2
 langroid/vector_store/momento.py,sha256=9cui31TTrILid2KIzUpBkN2Ey3g_CZWOQVdaFsA4Ors,10045
 langroid/vector_store/qdrant_cloud.py,sha256=3im4Mip0QXLkR6wiqVsjV1QvhSElfxdFSuDKddBDQ-4,188
 langroid/vector_store/qdrantdb.py,sha256=sk5Qb2ZNbooi0rorsMuqIMokF7WADw6PJ0D6goM2XBw,16802
-langroid-0.1.246.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.1.246.dist-info/METADATA,sha256=ZG2s9al3tSU4jkDPkjCySISHeNrThigZxy0jZ5vQlIs,49163
-langroid-0.1.246.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-langroid-0.1.246.dist-info/RECORD,,
+langroid-0.1.248.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.1.248.dist-info/METADATA,sha256=RkEwgd3nRozDym_Ey1aguztAAgRpTrzh6E4uOeHUDdo,49426
+langroid-0.1.248.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+langroid-0.1.248.dist-info/RECORD,,