langroid 0.1.262__py3-none-any.whl → 0.1.265__py3-none-any.whl

langroid/agent/base.py

@@ -20,7 +20,6 @@ from typing import (
     no_type_check,
 )
 
-from pydantic import BaseSettings, ValidationError, validator
 from rich import print
 from rich.console import Console
 from rich.markup import escape
@@ -41,6 +40,7 @@ from langroid.mytypes import Entity
 from langroid.parsing.parse_json import extract_top_level_json
 from langroid.parsing.parser import Parser, ParsingConfig
 from langroid.prompts.prompts_config import PromptsConfig
+from langroid.pydantic_v1 import BaseSettings, ValidationError, validator
 from langroid.utils.configuration import settings
 from langroid.utils.constants import NO_ANSWER
 from langroid.utils.output import status

langroid/agent/callbacks/chainlit.py

@@ -7,9 +7,8 @@ import logging
 import textwrap
 from typing import Any, Callable, Dict, List, Literal, Optional, no_type_check
 
-from pydantic import BaseSettings
-
 from langroid.exceptions import LangroidImportError
+from langroid.pydantic_v1 import BaseSettings
 
 try:
     import chainlit as cl

langroid/agent/chat_document.py

@@ -2,8 +2,6 @@ import json
 from enum import Enum
 from typing import List, Optional, Union
 
-from pydantic import BaseModel, Extra
-
 from langroid.agent.tool_message import ToolMessage
 from langroid.language_models.base import (
     LLMFunctionCall,
@@ -15,6 +13,7 @@ from langroid.language_models.base import (
 from langroid.mytypes import DocMetaData, Document, Entity
 from langroid.parsing.agent_chats import parse_message
 from langroid.parsing.parse_json import extract_top_level_json, top_level_json_field
+from langroid.pydantic_v1 import BaseModel, Extra
 from langroid.utils.output.printing import shorten_text
 
 
@@ -48,7 +47,7 @@ class ChatDocMetaData(DocMetaData):
     block: None | Entity = None
     sender_name: str = ""
     recipient: str = ""
-    usage: Optional[LLMTokenUsage]
+    usage: Optional[LLMTokenUsage] = None
     cached: bool = False
     displayed: bool = False
     has_citation: bool = False

langroid/agent/openai_assistant.py

@@ -15,7 +15,6 @@ from openai.types.beta.assistant_update_params import (
 )
 from openai.types.beta.threads import Message, Run
 from openai.types.beta.threads.runs import RunStep
-from pydantic import BaseModel
 from rich import print
 
 from langroid.agent.chat_agent import ChatAgent, ChatAgentConfig
@@ -27,6 +26,7 @@ from langroid.language_models.openai_gpt import (
     OpenAIGPT,
     OpenAIGPTConfig,
 )
+from langroid.pydantic_v1 import BaseModel
 from langroid.utils.configuration import settings
 from langroid.utils.system import generate_user_id, update_hash
 

langroid/agent/special/doc_chat_agent.py

@@ -1313,7 +1313,7 @@ class DocChatAgent(ChatAgent):
             meta.update(extracts[0].metadata)
             return ChatDocument(
                 content="\n\n".join([e.content for e in extracts]),
-                metadata=ChatDocMetaData(**meta),
+                metadata=ChatDocMetaData(**meta),  # type: ignore
             )
         response = self.get_summary_answer(query, extracts)
 

langroid/agent/special/lance_rag/query_planner_agent.py

@@ -191,7 +191,7 @@ class LanceQueryPlanAgent(ChatAgent):
             # save result, to be used in query_plan_feedback()
             self.result = msg.content
             # assemble QueryPlanAnswerTool...
-            query_plan_answer_tool = QueryPlanAnswerTool(
+            query_plan_answer_tool = QueryPlanAnswerTool(  # type: ignore
                 plan=self.curr_query_plan,
                 answer=self.result,
             )

langroid/agent/special/lance_tools.py

@@ -1,8 +1,7 @@
 import logging
 
-from pydantic import BaseModel
-
 from langroid.agent.tool_message import ToolMessage
+from langroid.pydantic_v1 import BaseModel
 
 logger = logging.getLogger(__name__)
 

langroid/agent/special/neo4j/neo4j_chat_agent.py

@@ -2,11 +2,11 @@ import json
 import logging
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-from pydantic import BaseModel, BaseSettings
 from rich import print
 from rich.console import Console
 
 from langroid.agent import ToolMessage
+from langroid.pydantic_v1 import BaseModel, BaseSettings
 
 if TYPE_CHECKING:
     import neo4j

langroid/agent/special/sql/utils/tools.py

@@ -29,7 +29,7 @@ class GetTableSchemaTool(ToolMessage):
     @classmethod
     def example(cls) -> "GetTableSchemaTool":
         return cls(
-            tables=("employees", "departments", "sales"),
+            tables=["employees", "departments", "sales"],
         )
 
 

langroid/agent/task.py

@@ -20,7 +20,6 @@ from typing import (
 )
 
 import numpy as np
-from pydantic import BaseModel
 from rich import print
 from rich.markup import escape
 
@@ -37,6 +36,7 @@ from langroid.exceptions import InfiniteLoopException
 from langroid.mytypes import Entity
 from langroid.parsing.parse_json import extract_top_level_json
 from langroid.parsing.routing import parse_addressed_message
+from langroid.pydantic_v1 import BaseModel
 from langroid.utils.configuration import settings
 from langroid.utils.constants import (
     DONE,
@@ -106,7 +106,7 @@ class Task:
     """
 
     # class variable called `cache` that is a RedisCache object
-    cache: RedisCache = RedisCache(RedisCacheConfig(fake=False))
+    _cache: RedisCache | None = None
 
     def __init__(
         self,
@@ -332,6 +332,12 @@ class Task:
             config=self.config,
         )
 
+    @classmethod
+    def cache(cls) -> RedisCache:
+        if cls._cache is None:
+            cls._cache = RedisCache(RedisCacheConfig(fake=False))
+        return cls._cache
+
     def __repr__(self) -> str:
         return f"{self.name}"
 
@@ -350,7 +356,7 @@ class Task:
         E.g. key = "kill", value = "1"
         """
         try:
-            self.cache.store(f"{self.session_id}:{key}", value)
+            self.cache().store(f"{self.session_id}:{key}", value)
         except Exception as e:
             logging.error(f"Error in Task._cache_session_store: {e}")
 
@@ -360,7 +366,7 @@ class Task:
         """
         session_id_key = f"{self.session_id}:{key}"
         try:
-            cached_val = self.cache.retrieve(session_id_key)
+            cached_val = self.cache().retrieve(session_id_key)
         except Exception as e:
             logging.error(f"Error in Task._cache_session_lookup: {e}")
             return None
@@ -384,7 +390,7 @@ class Task:
         Kill the session with the given session_id.
         """
         session_id_kill_key = f"{session_id}:kill"
-        cls.cache.store(session_id_kill_key, "1")
+        cls.cache().store(session_id_kill_key, "1")
 
     def kill(self) -> None:
         """
@@ -567,13 +573,15 @@ class Task:
         Runs asynchronously.
 
         Args:
-            msg (str|ChatDocument): initial message to process; if None,
+            msg (str|ChatDocument): initial *user-role* message to process; if None,
                 the LLM will respond to its initial `self.task_messages`
                 which set up and kick off the overall task.
                 The agent tries to achieve this goal by looping
                 over `self.step()` until the task is considered
                 done; this can involve a series of messages produced by Agent,
-                LLM or Human (User).
+                LLM or Human (User). Note that `msg`, if passed, is treated as
+                message with role `user`; a "system" role message should not be
+                passed here.
             turns (int): number of turns to run the task for;
                 default is -1, which means run until task is done.
             caller (Task|None): the calling task, if any
@@ -1059,6 +1067,9 @@ class Task:
         """
         Get result of task. This is the default behavior.
         Derived classes can override this.
+
+        Note the result of a task is returned as if it is from the User entity.
+
         Returns:
             ChatDocument: result of task
         """
@@ -1071,7 +1082,7 @@ class Task:
         fun_call = result_msg.function_call if result_msg else None
         tool_messages = result_msg.tool_messages if result_msg else []
         block = result_msg.metadata.block if result_msg else None
-        recipient = result_msg.metadata.recipient if result_msg else None
+        recipient = result_msg.metadata.recipient if result_msg else ""
         tool_ids = result_msg.metadata.tool_ids if result_msg else []
         status = result_msg.metadata.status if result_msg else None
 

langroid/agent/tool_message.py

@@ -13,9 +13,9 @@ from random import choice
 from typing import Any, Dict, List, Tuple, Type
 
 from docstring_parser import parse
-from pydantic import BaseModel
 
 from langroid.language_models.base import LLMFunctionSpec
+from langroid.pydantic_v1 import BaseModel
 from langroid.utils.pydantic_utils import (
     _recursive_purge_dict_key,
     generate_simple_schema,
@@ -73,7 +73,7 @@ class ToolMessage(ABC, BaseModel):
         - a tuple (description, ToolMessage instance), where the description is
             a natural language "thought" that leads to the tool usage,
             e.g. ("I want to find the square of 5",  SquareTool(num=5))
-            In some scenarios, ncluding such a description can significantly
+            In some scenarios, including such a description can significantly
             enhance reliability of tool use.
         Returns:
         """

langroid/cachedb/base.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List
 
-from pydantic import BaseSettings
+from langroid.pydantic_v1 import BaseSettings
 
 
 class CacheDBConfig(BaseSettings):

langroid/cachedb/redis_cachedb.py

@@ -23,6 +23,8 @@ class RedisCacheConfig(CacheDBConfig):
 class RedisCache(CacheDB):
     """Redis implementation of the CacheDB."""
 
+    _warned_password: bool = False
+
     def __init__(self, config: RedisCacheConfig):
         """
         Initialize a RedisCache with the given config.
@@ -40,10 +42,12 @@ class RedisCache(CacheDB):
             redis_host = os.getenv("REDIS_HOST")
             redis_port = os.getenv("REDIS_PORT")
             if None in [redis_password, redis_host, redis_port]:
-                logger.warning(
-                    """REDIS_PASSWORD, REDIS_HOST, REDIS_PORT not set in .env file,
-                    using fake redis client"""
-                )
+                if not RedisCache._warned_password:
+                    logger.warning(
+                        """REDIS_PASSWORD, REDIS_HOST, REDIS_PORT not set in .env file,
+                        using fake redis client"""
+                    )
+                    RedisCache._warned_password = True
                 self.pool = fakeredis.FakeStrictRedis()  # type: ignore
             else:
                 self.pool = redis.ConnectionPool(  # type: ignore

langroid/embedding_models/base.py

@@ -2,9 +2,9 @@ import logging
 from abc import ABC, abstractmethod
 
 import numpy as np
-from pydantic import BaseSettings
 
 from langroid.mytypes import EmbeddingFunction
+from langroid.pydantic_v1 import BaseSettings
 
 logging.getLogger("openai").setLevel(logging.ERROR)
 

langroid/language_models/base.py

@@ -1,5 +1,4 @@
 import ast
-import asyncio
 import json
 import logging
 from abc import ABC, abstractmethod
@@ -7,18 +6,11 @@ from datetime import datetime
 from enum import Enum
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
 
-import aiohttp
-from pydantic import BaseModel, BaseSettings, Field
-
 from langroid.cachedb.base import CacheDBConfig
-from langroid.mytypes import Document
 from langroid.parsing.agent_chats import parse_message
 from langroid.parsing.parse_json import top_level_json_field
 from langroid.prompts.dialog import collate_chat_history
-from langroid.prompts.templates import (
-    EXTRACTION_PROMPT_GPT4,
-    SUMMARY_ANSWER_PROMPT_GPT4,
-)
+from langroid.pydantic_v1 import BaseModel, BaseSettings, Field
 from langroid.utils.configuration import settings
 from langroid.utils.output.printing import show_if_debug
 
@@ -184,7 +176,7 @@ class LLMResponse(BaseModel):
     message: str
     tool_id: str = ""  # used by OpenAIAssistant
     function_call: Optional[LLMFunctionCall] = None
-    usage: Optional[LLMTokenUsage]
+    usage: Optional[LLMTokenUsage] = None
     cached: bool = False
 
     def __str__(self) -> str:
@@ -487,107 +479,6 @@ class LanguageModel(ABC):
         show_if_debug(prompt, "FOLLOWUP->STANDALONE-RESPONSE= ")
         return standalone
 
-    async def get_verbatim_extract_async(self, question: str, passage: Document) -> str:
-        """
-        Asynchronously, get verbatim extract from passage
-        that is relevant to a question.
-        Asynch allows parallel calls to the LLM API.
-        """
-        async with aiohttp.ClientSession():
-            templatized_prompt = EXTRACTION_PROMPT_GPT4
-            final_prompt = templatized_prompt.format(
-                question=question, content=passage.content
-            )
-            show_if_debug(final_prompt, "EXTRACT-PROMPT= ")
-            final_extract = await self.agenerate(prompt=final_prompt, max_tokens=1024)
-            show_if_debug(final_extract.message.strip(), "EXTRACT-RESPONSE= ")
-        return final_extract.message.strip()
-
-    async def _get_verbatim_extracts(
-        self,
-        question: str,
-        passages: List[Document],
-    ) -> List[Document]:
-        async with aiohttp.ClientSession():
-            verbatim_extracts = await asyncio.gather(
-                *(self.get_verbatim_extract_async(question, P) for P in passages)
-            )
-        metadatas = [P.metadata for P in passages]
-        # return with metadata so we can use it downstream, e.g. to cite sources
-        return [
-            Document(content=e, metadata=m)
-            for e, m in zip(verbatim_extracts, metadatas)
-        ]
-
-    def get_verbatim_extracts(
-        self, question: str, passages: List[Document]
-    ) -> List[Document]:
-        """
-        From each passage, extract verbatim text that is relevant to a question,
-        using concurrent API calls to the LLM.
-        Args:
-            question: question to be answered
-            passages: list of passages from which to extract relevant verbatim text
-            LLM: LanguageModel to use for generating the prompt and extract
-        Returns:
-            list of verbatim extracts from passages that are relevant to question
-        """
-        docs = asyncio.run(self._get_verbatim_extracts(question, passages))
-        return docs
-
-    def get_summary_answer(self, question: str, passages: List[Document]) -> Document:
-        """
-        Given a question and a list of (possibly) doc snippets,
-        generate an answer if possible
-        Args:
-            question: question to answer
-            passages: list of `Document` objects each containing a possibly relevant
-                snippet, and metadata
-        Returns:
-            a `Document` object containing the answer,
-            and metadata containing source citations
-
-        """
-
-        # Define an auxiliary function to transform the list of
-        # passages into a single string
-        def stringify_passages(passages: List[Document]) -> str:
-            return "\n".join(
-                [
-                    f"""
-                Extract: {p.content}
-                Source: {p.metadata.source}
-                """
-                    for p in passages
-                ]
-            )
-
-        passages_str = stringify_passages(passages)
-        # Substitute Q and P into the templatized prompt
-
-        final_prompt = SUMMARY_ANSWER_PROMPT_GPT4.format(
-            question=f"Question:{question}", extracts=passages_str
-        )
-        show_if_debug(final_prompt, "SUMMARIZE_PROMPT= ")
-        # Generate the final verbatim extract based on the final prompt
-        llm_response = self.generate(prompt=final_prompt, max_tokens=1024)
-        final_answer = llm_response.message.strip()
-        show_if_debug(final_answer, "SUMMARIZE_RESPONSE= ")
-        parts = final_answer.split("SOURCE:", maxsplit=1)
-        if len(parts) > 1:
-            content = parts[0].strip()
-            sources = parts[1].strip()
-        else:
-            content = final_answer
-            sources = ""
-        return Document(
-            content=content,
-            metadata={
-                "source": "SOURCE: " + sources,
-                "cached": llm_response.cached,
-            },
-        )
-
 
 class StreamingIfAllowed:
     """Context to temporarily enable or disable streaming, if allowed globally via

langroid/language_models/config.py

@@ -1,4 +1,4 @@
-from pydantic import BaseSettings
+from langroid.pydantic_v1 import BaseSettings
 
 
 class PromptFormatterConfig(BaseSettings):

langroid/language_models/openai_gpt.py

@@ -24,7 +24,6 @@ import openai
 from groq import AsyncGroq, Groq
 from httpx import Timeout
 from openai import AsyncOpenAI, OpenAI
-from pydantic import BaseModel
 from rich import print
 from rich.markup import escape
 
@@ -50,6 +49,7 @@ from langroid.language_models.utils import (
     async_retry_with_exponential_backoff,
     retry_with_exponential_backoff,
 )
+from langroid.pydantic_v1 import BaseModel
 from langroid.utils.configuration import settings
 from langroid.utils.constants import Colors
 from langroid.utils.system import friendly_error

langroid/mytypes.py

@@ -4,7 +4,7 @@ from enum import Enum
 from textwrap import dedent
 from typing import Any, Callable, Dict, List, Union
 
-from pydantic import BaseModel, Extra
+from langroid.pydantic_v1 import BaseModel, Extra
 
 Number = Union[int, float]
 Embedding = List[Number]

langroid/parsing/code_parser.py

@@ -2,12 +2,12 @@ from functools import reduce
 from typing import Callable, List
 
 import tiktoken
-from pydantic import BaseSettings
 from pygments import lex
 from pygments.lexers import get_lexer_by_name
 from pygments.token import Token
 
 from langroid.mytypes import Document
+from langroid.pydantic_v1 import BaseSettings
 
 
 def chunk_code(

langroid/parsing/parser.py

@@ -3,10 +3,10 @@ from enum import Enum
 from typing import Dict, List, Literal
 
 import tiktoken
-from pydantic import BaseSettings
 
 from langroid.mytypes import Document
 from langroid.parsing.para_sentence_split import create_chunks, remove_extra_whitespace
+from langroid.pydantic_v1 import BaseSettings
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.WARNING)

langroid/parsing/repo_loader.py

@@ -15,11 +15,11 @@ from github import Github
 from github.ContentFile import ContentFile
 from github.Label import Label
 from github.Repository import Repository
-from pydantic import BaseModel, BaseSettings, Field
 
 from langroid.mytypes import DocMetaData, Document
 from langroid.parsing.document_parser import DocumentParser, DocumentType
 from langroid.parsing.parser import Parser, ParsingConfig
+from langroid.pydantic_v1 import BaseModel, BaseSettings, Field
 
 logger = logging.getLogger(__name__)
 

langroid/parsing/urls.py

@@ -9,11 +9,12 @@ from urllib.parse import urldefrag, urljoin, urlparse
 import fire
 import requests
 from bs4 import BeautifulSoup
-from pydantic import BaseModel, HttpUrl, ValidationError, parse_obj_as
 from rich import print
 from rich.prompt import Prompt
 from trafilatura.spider import focused_crawler
 
+from langroid.pydantic_v1 import BaseModel, HttpUrl, ValidationError, parse_obj_as
+
 logger = logging.getLogger(__name__)
 
 

langroid/prompts/__init__.py

@@ -1,11 +1,9 @@
 from . import dialog
 from . import prompts_config
 from . import templates
-from . import transforms
 
 __all__ = [
     "dialog",
     "prompts_config",
     "templates",
-    "transforms",
 ]

langroid/prompts/prompts_config.py

@@ -1,4 +1,4 @@
-from pydantic import BaseSettings
+from langroid.pydantic_v1 import BaseSettings
 
 
 class PromptsConfig(BaseSettings):

langroid/pydantic_v1/__init__.py

@@ -0,0 +1,10 @@
+"""
+If we're on Pydantic v2, use the v1 namespace, else just use the main namespace.
+
+This allows compatibility with both Pydantic v1 and v2
+"""
+
+try:
+    from pydantic.v1 import *  # noqa: F403, F401
+except ImportError:
+    from pydantic import *  # type: ignore # noqa: F403, F401

langroid/pydantic_v1/main.py

@@ -0,0 +1,4 @@
+try:
+    from pydantic.v1.main import *  # noqa: F403, F401
+except ImportError:
+    from pydantic.main import *  # type: ignore # noqa: F403, F401

langroid/utils/configuration.py

@@ -4,7 +4,8 @@ from contextlib import contextmanager
 from typing import Iterator, List, Literal
 
 from dotenv import find_dotenv, load_dotenv
-from pydantic import BaseSettings
+
+from langroid.pydantic_v1 import BaseSettings
 
 
 class Settings(BaseSettings):

langroid/utils/constants.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 
 
 # Define the ANSI escape sequences for various colors and reset

langroid/utils/globals.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict, Optional, Type, TypeVar
 
-from pydantic import BaseModel
+from langroid.pydantic_v1 import BaseModel
 
 T = TypeVar("T", bound="GlobalState")
 

langroid/utils/pydantic_utils.py

@@ -16,9 +16,9 @@ from typing import (
 
 import numpy as np
 import pandas as pd
-from pydantic import BaseModel, ValidationError, create_model
 
 from langroid.mytypes import DocMetaData, Document
+from langroid.pydantic_v1 import BaseModel, ValidationError, create_model
 
 logger = logging.getLogger(__name__)
 

langroid/utils/system.py

@@ -1,6 +1,7 @@
 import getpass
 import hashlib
 import importlib
+import importlib.metadata
 import inspect
 import logging
 import shutil
@@ -18,6 +19,15 @@ DELETION_ALLOWED_PATHS = [
 ]
 
 
+def pydantic_major_version() -> int:
+    try:
+        pydantic_version = importlib.metadata.version("pydantic")
+        major_version = int(pydantic_version.split(".")[0])
+        return major_version
+    except importlib.metadata.PackageNotFoundError:
+        return -1
+
+
 class LazyLoad:
     """Lazy loading of modules or classes."""
 

langroid/vector_store/base.py

@@ -5,11 +5,11 @@ from typing import Dict, List, Optional, Sequence, Tuple
 
 import numpy as np
 import pandas as pd
-from pydantic import BaseSettings
 
 from langroid.embedding_models.base import EmbeddingModel, EmbeddingModelsConfig
 from langroid.embedding_models.models import OpenAIEmbeddingsConfig
 from langroid.mytypes import Document
+from langroid.pydantic_v1 import BaseSettings
 from langroid.utils.algorithms.graph import components, topological_sort
 from langroid.utils.configuration import settings
 from langroid.utils.output.printing import print_long_text

langroid/vector_store/lancedb.py

@@ -15,7 +15,8 @@ from typing import (
 
 import pandas as pd
 from dotenv import load_dotenv
-from pydantic import BaseModel, ValidationError, create_model
+
+from langroid.pydantic_v1 import BaseModel, ValidationError, create_model
 
 if TYPE_CHECKING:
     from lancedb.query import LanceVectorQueryBuilder
@@ -37,6 +38,7 @@ from langroid.utils.pydantic_utils import (
     flatten_pydantic_model,
     nested_dict_from_flat,
 )
+from langroid.utils.system import pydantic_major_version
 from langroid.vector_store.base import VectorStore, VectorStoreConfig
 
 try:
@@ -111,13 +113,30 @@ class LanceDB(VectorStore):
             )
 
     def _setup_schemas(self, doc_cls: Type[Document] | None) -> None:
-        doc_cls = doc_cls or self.config.document_class
-        self.unflattened_schema = self._create_lance_schema(doc_cls)
-        self.schema = (
-            self._create_flat_lance_schema(doc_cls)
-            if self.config.flatten
-            else self.unflattened_schema
-        )
+        try:
+            doc_cls = doc_cls or self.config.document_class
+            self.unflattened_schema = self._create_lance_schema(doc_cls)
+            self.schema = (
+                self._create_flat_lance_schema(doc_cls)
+                if self.config.flatten
+                else self.unflattened_schema
+            )
+        except (AttributeError, TypeError) as e:
+            pydantic_version = pydantic_major_version()
+            if pydantic_version > 1:
+                raise ValueError(
+                    f"""
+                    {e}
+                    ====
+                    You are using Pydantic v{pydantic_version},
+                    which is not yet compatible with Langroid's LanceDB integration.
+                    To use Lancedb with Langroid, please install the 
+                    latest pydantic 1.x instead of pydantic v2, e.g. 
+                    pip install "pydantic<2.0.0"
+                    """
+                )
+            else:
+                raise e
 
     def clear_empty_collections(self) -> int:
         coll_names = self.list_collections()
@@ -246,7 +265,27 @@ class LanceDB(VectorStore):
                     return
                 else:
                     logger.warning("Recreating fresh collection")
-        self.client.create_table(collection_name, schema=self.schema, mode="overwrite")
+        try:
+            self.client.create_table(
+                collection_name, schema=self.schema, mode="overwrite"
+            )
+        except (AttributeError, TypeError) as e:
+            pydantic_version = pydantic_major_version()
+            if pydantic_version > 1:
+                raise ValueError(
+                    f"""
+                    {e}
+                    ====
+                    You are using Pydantic v{pydantic_version},
+                    which is not yet compatible with Langroid's LanceDB integration.
+                    To use Lancedb with Langroid, please install the 
+                    latest pydantic 1.x instead of pydantic v2, e.g. 
+                    pip install "pydantic<2.0.0"
+                    """
+                )
+            else:
+                raise e
+
         if settings.debug:
             level = logger.getEffectiveLevel()
             logger.setLevel(logging.INFO)

{langroid-0.1.262.dist-info → langroid-0.1.265.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.262
+Version: 0.1.265
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani
@@ -51,10 +51,9 @@ Requires-Dist: grpcio (>=1.62.1,<2.0.0)
 Requires-Dist: halo (>=0.0.31,<0.0.32)
 Requires-Dist: huggingface-hub (>=0.21.2,<0.22.0) ; extra == "hf-transformers" or extra == "all" or extra == "transformers"
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
-Requires-Dist: lancedb (>=0.6.2,<0.7.0) ; extra == "vecdbs" or extra == "all" or extra == "lancedb"
+Requires-Dist: lancedb (>=0.8.2,<0.9.0) ; extra == "vecdbs" or extra == "lancedb"
 Requires-Dist: litellm (>=1.30.1,<2.0.0) ; extra == "all" or extra == "litellm"
 Requires-Dist: lxml (>=4.9.3,<5.0.0)
-Requires-Dist: meilisearch (>=0.28.3,<0.29.0) ; extra == "meilisearch"
 Requires-Dist: meilisearch-python-sdk (>=2.2.3,<3.0.0) ; extra == "meilisearch"
 Requires-Dist: metaphor-python (>=0.1.23,<0.2.0) ; extra == "all" or extra == "metaphor"
 Requires-Dist: momento (>=1.10.2,<2.0.0) ; extra == "momento"
@@ -68,8 +67,8 @@ Requires-Dist: pdf2image (>=1.17.0,<2.0.0) ; extra == "doc-chat" or extra == "al
 Requires-Dist: pdfplumber (>=0.10.2,<0.11.0) ; extra == "doc-chat" or extra == "all" or extra == "pdf-parsers"
 Requires-Dist: prettytable (>=3.8.0,<4.0.0)
 Requires-Dist: psycopg2 (>=2.9.7,<3.0.0) ; extra == "db" or extra == "all" or extra == "postgres" or extra == "sql"
-Requires-Dist: pyarrow (==15.0.0) ; extra == "vecdbs" or extra == "all" or extra == "lancedb"
-Requires-Dist: pydantic (==1.10.13)
+Requires-Dist: pyarrow (==15.0.0) ; extra == "vecdbs" or extra == "lancedb"
+Requires-Dist: pydantic (>=1,<3)
 Requires-Dist: pygithub (>=1.58.1,<2.0.0)
 Requires-Dist: pygments (>=2.15.1,<3.0.0)
 Requires-Dist: pymupdf (>=1.23.3,<2.0.0) ; extra == "doc-chat" or extra == "all" or extra == "pdf-parsers"
@@ -91,7 +90,7 @@ Requires-Dist: rich (>=13.3.4,<14.0.0)
 Requires-Dist: scrapy (>=2.11.0,<3.0.0) ; extra == "scrapy"
 Requires-Dist: sentence-transformers (==2.2.2) ; extra == "hf-transformers" or extra == "all" or extra == "hf-embeddings"
 Requires-Dist: sqlalchemy (>=2.0.19,<3.0.0) ; extra == "db" or extra == "all" or extra == "sql"
-Requires-Dist: tantivy (>=0.21.0,<0.22.0) ; extra == "vecdbs" or extra == "all" or extra == "lancedb"
+Requires-Dist: tantivy (>=0.21.0,<0.22.0) ; extra == "vecdbs" or extra == "lancedb"
 Requires-Dist: thefuzz (>=0.20.0,<0.21.0)
 Requires-Dist: tiktoken (>=0.7.0,<0.8.0)
 Requires-Dist: torch (==2.0.0) ; extra == "hf-transformers" or extra == "all" or extra == "hf-embeddings" or extra == "transformers"
@@ -515,9 +514,10 @@ with a postgres db, you will need to:
 ### Set up environment variables (API keys, etc)
 
 To get started, all you need is an OpenAI API Key.
-If you don't have one, see [this OpenAI Page](https://help.openai.com/en/collections/3675940-getting-started-with-openai-api).
-Currently only OpenAI models are supported. Others will be added later
-(Pull Requests welcome!).
+If you don't have one, see [this OpenAI Page](https://platform.openai.com/docs/quickstart).
+(Note that while this is the simplest way to get started, Langroid works with practically any LLM, not just those from OpenAI. 
+See the guides to using [Open/Local LLMs](https://langroid.github.io/langroid/tutorials/local-llm-setup/), 
+and other [non-OpenAI](https://langroid.github.io/langroid/tutorials/non-openai-llms/) proprietary LLMs.) 
 
 In the root of the repo, copy the `.env-template` file to a new file `.env`: 
 ```bash
@@ -545,9 +545,7 @@ All of the following environment variable settings are optional, and some are on
 to use specific features (as noted below).
 
 - **Qdrant** Vector Store API Key, URL. This is only required if you want to use Qdrant cloud.
-  The default vector store in our RAG agent (`DocChatAgent`) is LanceDB which uses file storage,
-  and you do not need to set up any environment variables for that.
-  Alternatively [Chroma](https://docs.trychroma.com/) is also currently supported. 
+  Alternatively [Chroma](https://docs.trychroma.com/) or [LanceDB](https://lancedb.com/) are also currently supported. 
   We use the local-storage version of Chroma, so there is no need for an API key.
 - **Redis** Password, host, port: This is optional, and only needed to cache LLM API responses
   using Redis Cloud. Redis [offers](https://redis.com/try-free/) a free 30MB Redis account
@@ -642,11 +640,12 @@ and they are **not** complete runnable examples! For that we encourage you to
 consult the [`langroid-examples`](https://github.com/langroid/langroid-examples) 
 repository.
 
-:information_source: The various LLM prompts and instructions in Langroid
-have been tested to work well with GPT4.
-Switching to GPT3.5-Turbo is easy via a config flag
-(e.g., `cfg = OpenAIGPTConfig(chat_model=OpenAIChatModel.GPT3_5_TURBO)`),
-and may suffice for some applications, but in general you may see inferior results.
+:information_source:
+The various LLM prompts and instructions in Langroid
+have been tested to work well with GPT-4 (and to some extent GPT-4o).
+Switching to other LLMs (local/open and proprietary) is easy (see guides mentioned above),
+and may suffice for some applications, but in general you may see inferior results
+unless you adjust the prompts and/or the multi-agent setup.
 
 
 :book: Also see the
@@ -921,7 +920,7 @@ config = DocChatAgentConfig(
     "https://en.wikipedia.org/wiki/N-gram_language_model",
     "/path/to/my/notes-on-language-models.txt",
   ],
-  vecdb=lr.vector_store.LanceDBConfig(),
+  vecdb=lr.vector_store.QdrantDBConfig(),
 )
 ```
 

{langroid-0.1.262.dist-info → langroid-0.1.265.dist-info}/RECORD

@@ -1,25 +1,25 @@
 langroid/__init__.py,sha256=z_fCOLQJPOw3LLRPBlFB5-2HyCjpPgQa4m4iY5Fvb8Y,1800
 langroid/agent/__init__.py,sha256=ll0Cubd2DZ-fsCMl7e10hf9ZjFGKzphfBco396IKITY,786
-langroid/agent/base.py,sha256=aSwWmOBg0d3QQHUSauscMNfnl8Wkv6nrk2nngKa9DjM,37183
+langroid/agent/base.py,sha256=CHFUZ4vnC1CSMMT5POnY4AWu96RQ-4uDUARUbhGAGo8,37195
 langroid/agent/batch.py,sha256=feRA_yRG768ElOQjrKEefcRv6Aefd_yY7qktuYUQDwc,10040
 langroid/agent/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-langroid/agent/callbacks/chainlit.py,sha256=6gkk9Qf_i4fOD13w8ZdUfMcgKYPzLMw30hzFUN60AIc,22044
+langroid/agent/callbacks/chainlit.py,sha256=UKG2_v4ktfkEaGvdouVRHEqQejEYya2Rli8jrP65TmA,22055
 langroid/agent/chat_agent.py,sha256=hnmeOxdi4i5w8WaL2kPjQOEpenoRW_hG5EfeMWuuVsQ,39478
-langroid/agent/chat_document.py,sha256=Xx4uFVI77YxDc99x5P4JGprDINT5umqvTzCgHsvJpDc,9200
+langroid/agent/chat_document.py,sha256=tXFuUw2fs8WVIhVt10Sne0IpOzPPEaFz_1yqQgU6V4U,9218
 langroid/agent/helpers.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/junk,sha256=LxfuuW7Cijsg0szAzT81OjWWv1PMNI-6w_-DspVIO2s,339
-langroid/agent/openai_assistant.py,sha256=kIVDI4r-xGvplLU5s0nShPVHs6Jq-wOsfWE0kcMhAdQ,33056
+langroid/agent/openai_assistant.py,sha256=rmGJD5n0eE7_O1EkPyXgHFMNGc3vb2GKweZMhzmRWvI,33068
 langroid/agent/special/__init__.py,sha256=gik_Xtm_zV7U9s30Mn8UX3Gyuy4jTjQe9zjiE3HWmEo,1273
-langroid/agent/special/doc_chat_agent.py,sha256=_jXEjxuymNjkDcwT2xEcpf9sWoUW0P-1UWHCuRQFt_w,54428
+langroid/agent/special/doc_chat_agent.py,sha256=dlXsY7brhrGOf_TA39EgkcL80L102Gkunoid7GvIZNY,54444
 langroid/agent/special/lance_doc_chat_agent.py,sha256=USp0U3eTaJzwF_3bdqE7CedSLbaqAi2tm-VzygcyLaA,10175
 langroid/agent/special/lance_rag/__init__.py,sha256=QTbs0IVE2ZgDg8JJy1zN97rUUg4uEPH7SLGctFNumk4,174
 langroid/agent/special/lance_rag/critic_agent.py,sha256=ufTdpHSeHgCzN85Q0sfWOrpBpsCjGVZdAg5yOH1ogU8,7296
 langroid/agent/special/lance_rag/lance_rag_task.py,sha256=l_HQgrYY-CX2FwIsS961aEF3bYog3GDYo98fj0C0mSk,2889
-langroid/agent/special/lance_rag/query_planner_agent.py,sha256=wSkrtY3Qz98KAqpVf0xMf4LRgKbHLASWVNUrbqwUAB0,9814
-langroid/agent/special/lance_tools.py,sha256=btMwKdcT8RdwAjmzbtN1xxm3s1H7ipO9GSpUamryYx8,1456
+langroid/agent/special/lance_rag/query_planner_agent.py,sha256=M4RC_0f98_pwVL7ygrr1VI80LgJiFcmKjJFH0M4tccI,9830
+langroid/agent/special/lance_tools.py,sha256=BksGrrNgGgyYWP0HnfAuXMc0KzXooFOzY2l5rDDMtQ8,1467
 langroid/agent/special/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/special/neo4j/csv_kg_chat.py,sha256=dRsAgMBa1H_EMI2YYgJR2Xyv1D7e4o3G9M64mTewq_c,6409
-langroid/agent/special/neo4j/neo4j_chat_agent.py,sha256=vBr6EQ_eJCYAtqDe-gTSvWHT-jRE_fZOPsGWxuDJe4w,13092
+langroid/agent/special/neo4j/neo4j_chat_agent.py,sha256=Y4Zu-m8WKO1xjeBRarV_m4y00Y5n_NR2B-hepjZp_cY,13104
 langroid/agent/special/neo4j/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/special/neo4j/utils/system_message.py,sha256=vRpz1P-OYLLiC6OGYYoK6x77yxVzDxMTCEJSsYUIuG4,2242
 langroid/agent/special/relevance_extractor_agent.py,sha256=zIx8GUdVo1aGW6ASla0NPQjYYIpmriK_TYMijqAx3F8,4796
@@ -30,10 +30,10 @@ langroid/agent/special/sql/utils/__init__.py,sha256=JFif6CRTrN-bc91uuAI4K9fe2ndI
 langroid/agent/special/sql/utils/description_extractors.py,sha256=cX8TIpmTPXZXQTMpIi3OUFwFsPywxFFdurpx717Kq0I,6529
 langroid/agent/special/sql/utils/populate_metadata.py,sha256=1J22UsyEPKzwK0XlJZtYn9r6kYc0FXIr8-lZrndYlhc,3131
 langroid/agent/special/sql/utils/system_message.py,sha256=qKLHkvQWRQodTtPLPxr1GSLUYUFASZU8x-ybV67cB68,1885
-langroid/agent/special/sql/utils/tools.py,sha256=6uB2424SLtmapui9ggcEr0ZTiB6_dL1-JRGgN8RK9Js,1332
+langroid/agent/special/sql/utils/tools.py,sha256=vFYysk6Vi7HJjII8B4RitA3pt_z3gkSglDNdhNVMiFc,1332
 langroid/agent/special/table_chat_agent.py,sha256=d9v2wsblaRx7oMnKhLV7uO_ujvk9gh59pSGvBXyeyNc,9659
-langroid/agent/task.py,sha256=DyQL-N8bpBhMI2dhNBXawVCs7HrtCDquYVomK_cQn6w,60603
-langroid/agent/tool_message.py,sha256=7t-UGEbykosKHAvaLI0Rm59sgxvN31IO3-P7bg7gLug,9730
+langroid/agent/task.py,sha256=AA8MKWVeZ6VeI3uAVhXzMgZWULGRtYoNXQ6HeAzS4oU,61011
+langroid/agent/tool_message.py,sha256=wIyZnUcZpxkiRPvM9O3MO3b5BBAdLEEan9kqPbvtApc,9743
 langroid/agent/tools/__init__.py,sha256=8Pc9BlGCB5FQ2IDGKS_WPpHCoWp5jblMU8EHJwwikAY,303
 langroid/agent/tools/duckduckgo_search_tool.py,sha256=NhsCaGZkdv28nja7yveAhSK_w6l_Ftym8agbrdzqgfo,1935
 langroid/agent/tools/extract_tool.py,sha256=u5lL9rKBzaLBOrRyLnTAZ97pQ1uxyLP39XsWMnpaZpw,3789
@@ -46,11 +46,11 @@ langroid/agent/tools/run_python_code.py,sha256=BvoxYzzHijU-p4703n2iVlt5BCieR1oMS
 langroid/agent/tools/segment_extract_tool.py,sha256=__srZ_VGYLVOdPrITUM8S0HpmX4q7r5FHWMDdHdEv8w,1440
 langroid/agent_config.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/cachedb/__init__.py,sha256=icAT2s7Vhf-ZGUeqpDQGNU6ob6o0aFEyjwcxxUGRFjg,225
-langroid/cachedb/base.py,sha256=LKiJyOFQUN1NRzPIynfbYKGFfSanA6auDfBNEedBK7Y,1342
+langroid/cachedb/base.py,sha256=ztVjB1DtN6pLCujCWnR6xruHxwVj3XkYniRTYAKKqk0,1354
 langroid/cachedb/momento_cachedb.py,sha256=YEOJ62hEcV6iIeMr5aGgRYgWQqFYaej9gEDEcY0sm7M,3172
-langroid/cachedb/redis_cachedb.py,sha256=NukuCWgdp1AWWNgguiZfuypbH9GHwiYe34ZZy866u54,4981
+langroid/cachedb/redis_cachedb.py,sha256=h12NxUeaCcQB06NQwmjm_NU-hc5HQw0fGg3f_MHAzcE,5140
 langroid/embedding_models/__init__.py,sha256=lsu8qxCjfGujXGueJWU-VI3LMZYGjLSYgqUKDd4F3Qo,715
-langroid/embedding_models/base.py,sha256=xY9QF01ilsMvaNH4JMDvkZgXY59AeYR4VAykgNd6Flg,1818
+langroid/embedding_models/base.py,sha256=MSjaTkFcfoMGY6SHPOqAsbZbKctj8-1N6zgaFYmOFTg,1830
 langroid/embedding_models/clustering.py,sha256=tZWElUqXl9Etqla0FAa7og96iDKgjqWjucZR_Egtp-A,6684
 langroid/embedding_models/models.py,sha256=-xeN0irBPc1tUgRFHGM1ki4NwOIHr6F3SKuEjD5nTOg,7144
 langroid/embedding_models/protoc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -62,50 +62,50 @@ langroid/embedding_models/remote_embeds.py,sha256=6_kjXByVbqhY9cGwl9R83ZcYC2km-n
 langroid/exceptions.py,sha256=w_Cr41nPAmsa6gW5nNFaO9yDcBCWdQqRspL1jYvZf5w,2209
 langroid/language_models/__init__.py,sha256=55602F5QA58MmRq-yRjoXK6xZOMRHQrR4QGaCnlX218,822
 langroid/language_models/azure_openai.py,sha256=ncRCbKooqLVOY-PWQUIo9C3yTuKEFbAwyngXT_M4P7k,5989
-langroid/language_models/base.py,sha256=8FTvWtOmIrz6K78kzyrVqf2uJk03dBc0AUnVY-l9ucg,21031
-langroid/language_models/config.py,sha256=5UF3DzO1a-Dfsc3vghE0XGq7g9t_xDsRCsuRiU4dgBg,366
+langroid/language_models/base.py,sha256=UbHcXSzQ5pTc_H5VSDh45c6dsyJwfY0H5_6ofEBXH-8,16917
+langroid/language_models/config.py,sha256=9Q8wk5a7RQr8LGMT_0WkpjY8S4ywK06SalVRjXlfCiI,378
 langroid/language_models/openai_assistants.py,sha256=9K-DEAL2aSWHeXj2hwCo2RAlK9_1oCPtqX2u1wISCj8,36
-langroid/language_models/openai_gpt.py,sha256=OZcFAtVI8JPGOPPRozWI1PfkdE450hS16RChaUc2uFM,50702
+langroid/language_models/openai_gpt.py,sha256=RXnLKULuCSeDeUPQvaZ4naqJgMKcMZogCtRDLycd4j8,50714
 langroid/language_models/prompt_formatter/__init__.py,sha256=2-5cdE24XoFDhifOLl8yiscohil1ogbP1ECkYdBlBsk,372
 langroid/language_models/prompt_formatter/base.py,sha256=eDS1sgRNZVnoajwV_ZIha6cba5Dt8xjgzdRbPITwx3Q,1221
 langroid/language_models/prompt_formatter/hf_formatter.py,sha256=TFL6ppmeQWnzr6CKQzRZFYY810zE1mr8DZnhw6i85ok,5217
 langroid/language_models/prompt_formatter/llama2_formatter.py,sha256=YdcO88qyBeuMENVIVvVqSYuEpvYSTndUe_jd6hVTko4,2899
 langroid/language_models/utils.py,sha256=j8xEEm__-2b9eql1oTiWQk5dHW59UwmrRKs5kMHaGGo,4803
-langroid/mytypes.py,sha256=qD3o2v1pccICz-xeei4cwkvJviVC2llJ3eIYgBP9RDE,3045
+langroid/mytypes.py,sha256=B5uKN4FMdHiJaAKozg8EmcPfle3sHlj11NcBnpXn7nE,3057
 langroid/parsing/__init__.py,sha256=ZgSAfgTC6VsTLFlRSWT-TwYco7SQeRMeZG-49MnKYGY,936
 langroid/parsing/agent_chats.py,sha256=sbZRV9ujdM5QXvvuHVjIi2ysYSYlap-uqfMMUKulrW0,1068
 langroid/parsing/code-parsing.md,sha256=--cyyNiSZSDlIwcjAV4-shKrSiRe2ytF3AdSoS_hD2g,3294
-langroid/parsing/code_parser.py,sha256=BbDAzp35wkYQ9U1dpf1ARL0lVyi0tfqEc6_eox2C090,3727
+langroid/parsing/code_parser.py,sha256=Fwa8MWY5EGk7Ekr8II5c-o9vBf4m1HfB5_K7e_EDYzo,3739
 langroid/parsing/config.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/parsing/document_parser.py,sha256=bN-D1kqx6qe1Sx-AMR8a8WbPJYPdoxLYPdgGl7dfW3I,24017
 langroid/parsing/image_text.py,sha256=sbLIQ5nHe2UnYUksBaQsmZGaX-X0qgEpPd7CEzi_z5M,910
 langroid/parsing/para_sentence_split.py,sha256=AJBzZojP3zpB-_IMiiHismhqcvkrVBQ3ZINoQyx_bE4,2000
 langroid/parsing/parse_json.py,sha256=tgB_oatcrgt6L9ZplC-xBBXjLzL1gjSQf1L2_W5kwFA,4230
-langroid/parsing/parser.py,sha256=2TT6YMgEe79Kz9bPIqI-1RIEK77V2H2gbpbX5DhNNrY,10743
-langroid/parsing/parser.pyi,sha256=4t55zlG_23hUFO7OvOttY1xzbLze2elpGjoVTUK9izM,1693
-langroid/parsing/repo_loader.py,sha256=My5UIe-h1xr0I-6Icu0ZVwRHmGRRRW8SrJYMc9J1M9Q,29361
+langroid/parsing/parser.py,sha256=VU3G2GXwho5vJ3tCDgw6_1W-guqAbPtml8RcJxvp-Xs,10755
+langroid/parsing/repo_loader.py,sha256=3GjvPJS6Vf5L6gV2zOU8s-Tf1oq_fZm-IB_RL_7CTsY,29373
 langroid/parsing/routing.py,sha256=_NFPe7wLjd5B6s47w3M8-5vldL8e2Sz51Gb5bwF5ooY,1072
 langroid/parsing/search.py,sha256=plQtjarB9afGfJLB0CyPXPq3mM4m7kRsfd0_4brziEI,8846
 langroid/parsing/spider.py,sha256=Y6y7b86Y2k770LdhxgjVlImBxuuy1V9n8-XQ3QPaG5s,3199
 langroid/parsing/table_loader.py,sha256=qNM4obT_0Y4tjrxNBCNUYjKQ9oETCZ7FbolKBTcz-GM,3410
 langroid/parsing/url_loader.py,sha256=Na2TBlKuQkloZzkE2d7xl6mh9olS3CbpgCsJbJ-xhIA,4472
 langroid/parsing/url_loader_cookies.py,sha256=Lg4sNpRz9MByWq2mde6T0hKv68VZSV3mtMjNEHuFeSU,2327
-langroid/parsing/urls.py,sha256=5B0-2MM4LoFC7jHUJ0rft7Mx5GUrnmz8oFioO0iaMt8,7975
+langroid/parsing/urls.py,sha256=XjpaV5onG7gKQ5iQeFTzHSw5P08Aqw0g-rMUu61lR6s,7988
 langroid/parsing/utils.py,sha256=pbSAbfwA28EBNESpQRJee_Kp1b44qze-2_2b9qJOKfM,12646
 langroid/parsing/web_search.py,sha256=XSiSHB4c1Wa8RjWkC4Yh-ac8S7a2WPPYj0n-Ma716RY,4759
-langroid/prompts/__init__.py,sha256=B0vpJzIJlMR3mFRtoQwyALsFzBHvLp9f92acD8xJA_0,185
+langroid/prompts/__init__.py,sha256=RW11vK6jiLPuaUh4GpeFvstti73gkm8_rDMtrbo2YsU,142
 langroid/prompts/chat-gpt4-system-prompt.md,sha256=Q3uLCJTPQvmUkZN2XDnkBC7M2K3X0F3C3GIQBaFvYvw,5329
 langroid/prompts/dialog.py,sha256=SpfiSyofSgy2pwD1YboHR_yHO3LEEMbv6j2sm874jKo,331
-langroid/prompts/prompts_config.py,sha256=XRQHzod7KBnoKn3B_V878jZiqBA7rcn-CtGPkuAe_yM,131
+langroid/prompts/prompts_config.py,sha256=p_lp9nbMuQwhhMwAZsOxveRw9C0ZFZvql7pdIfgVZYo,143
 langroid/prompts/templates.py,sha256=kz0rPiM6iLGhhpDonF3Y87OznSe9FRI6A0pHU0wgW4Q,6314
-langroid/prompts/transforms.py,sha256=GsQo1klGxUy0fACh6j0lTblk6XEl2erRnhRWlN2M4-c,2706
+langroid/pydantic_v1/__init__.py,sha256=HxPGVERapVueRUORgSpj2JX_vTZxVlVbWvhpQlpjygE,283
+langroid/pydantic_v1/main.py,sha256=p_k7kDY9eDrsA5dxNNqXusKLgx7mS_icGnS7fu4goqY,147
 langroid/utils/__init__.py,sha256=ARx5To4Hsv1K5QAzK4uUqdEoB_iq5HK797vae1AcMBI,300
 langroid/utils/algorithms/__init__.py,sha256=WylYoZymA0fnzpB4vrsH_0n7WsoLhmuZq8qxsOCjUpM,41
 langroid/utils/algorithms/graph.py,sha256=JbdpPnUOhw4-D6O7ou101JLA3xPCD0Lr3qaPoFCaRfo,2866
-langroid/utils/configuration.py,sha256=FvkbWf0A5iNdmtORfjlY6ZAHp4Fov_OTL6A8U4C3y-A,3282
-langroid/utils/constants.py,sha256=Y_8p7CyLF5b3xsEV5O3wuutLHQCtegsaxWgr8yNTlIE,563
+langroid/utils/configuration.py,sha256=A70LdvdMuunlLSGI1gBmBL5j6Jhz-1syNP8R4AdjqDc,3295
+langroid/utils/constants.py,sha256=eTiXfx8Nq2kmq0WChVLqV9C58UWju0NCIuW28sMgd5g,575
 langroid/utils/docker.py,sha256=kJQOLTgM0x9j9pgIIqp0dZNZCTvoUDhp6i8tYBq1Jr0,1105
-langroid/utils/globals.py,sha256=VkTHhlqSz86oOPq65sjul0XU8I52UNaFC5vwybMQ74w,1343
+langroid/utils/globals.py,sha256=Az9dOFqR6n9CoTYSqa2kLikQWS0oCQ9DFQIQAnG-2q8,1355
 langroid/utils/llms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/utils/llms/strings.py,sha256=CSAX9Z6FQOLXOzbLMe_Opqtc3ruDAKTTk7cPqc6Blh0,263
 langroid/utils/logging.py,sha256=WN180zjxhlozwtyTcLWmbVkXylBs5EvQj85dBPeVUwc,3985
@@ -114,20 +114,20 @@ langroid/utils/output/citations.py,sha256=PSY2cpti8W-ZGFMAgj1lYoEIZy0lsniLpCliMs
 langroid/utils/output/printing.py,sha256=yzPJZN-8_jyOJmI9N_oLwEDfjMwVgk3IDiwnZ4eK_AE,2962
 langroid/utils/output/status.py,sha256=rzbE7mDJcgNNvdtylCseQcPGCGghtJvVq3lB-OPJ49E,1049
 langroid/utils/pandas_utils.py,sha256=UctS986Jtl_MvU5rA7-GfrjEHXP7MNu8ePhepv0bTn0,755
-langroid/utils/pydantic_utils.py,sha256=yb-ghaQYL7EIYeiZ0tailvZvAuJZNF7UBXkd3z35OYc,21728
-langroid/utils/system.py,sha256=RfAcQODu4tjl-pAO8zZ65yKB9-6WsvzSz2dEPkJdSdw,4909
+langroid/utils/pydantic_utils.py,sha256=FKC8VKXH2uBEpFjnnMgIcEsQn6hs31ftea8zv5pMK9g,21740
+langroid/utils/system.py,sha256=ClIQOpJOeCgfi9n5c-bOS62B2cuqrZFGkWPRbjCpSqk,5208
 langroid/utils/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/utils/web/login.py,sha256=1iz9eUAHa87vpKIkzwkmFa00avwFWivDSAr7QUhK7U0,2528
 langroid/vector_store/__init__.py,sha256=6xBjb_z4QtUy4vz4RuFbcbSwmHrggHL8-q0DwCf3PMM,972
-langroid/vector_store/base.py,sha256=VZl-pvGs6K-ruTT8SQmDthsCp-VARYaf6OuzKmcXN58,13469
+langroid/vector_store/base.py,sha256=1bzFEDJcbKIaZnTPhBjnQ260c6QYs5SpZwGMtwt0-6Y,13481
 langroid/vector_store/chromadb.py,sha256=bZ5HjwgKgfJj1PUHsatYsrHv-v0dpOfMR2l0tJ2H0_A,7890
-langroid/vector_store/lancedb.py,sha256=nC5pcrFoUOOO941Y7XiPZONUO4LuoZIAR1aR4PecKto,19014
+langroid/vector_store/lancedb.py,sha256=9x7e_5zo7nLhMbhjYby2ZpBJ-vyawcC0_XAuatfHJf8,20517
 langroid/vector_store/meilisearch.py,sha256=6frB7GFWeWmeKzRfLZIvzRjllniZ1cYj3HmhHQICXLs,11663
 langroid/vector_store/momento.py,sha256=QaPzUnTwlswoawGB-paLtUPyLRvckFXLfLDfvbTzjNQ,10505
 langroid/vector_store/qdrant_cloud.py,sha256=3im4Mip0QXLkR6wiqVsjV1QvhSElfxdFSuDKddBDQ-4,188
 langroid/vector_store/qdrantdb.py,sha256=wYOuu5c2vIKn9ZgvTXcAiZXMpV8AOXEWFAzI8S8UP-0,16828
-pyproject.toml,sha256=sgBXhHt5XwaR1LhnRYDFrDzRlWJ3VGbpKU7uNThWdu4,7026
-langroid-0.1.262.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.1.262.dist-info/METADATA,sha256=LF2uL1n0o-o_kXzU2LE_KPNmfhRc0T3EeXKv-4YQslw,52506
-langroid-0.1.262.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-langroid-0.1.262.dist-info/RECORD,,
+pyproject.toml,sha256=zJRntm67SKuTYmuGM9SuBHq3t8_YCES7QJZrUnOBhEs,6966
+langroid-0.1.265.dist-info/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.1.265.dist-info/METADATA,sha256=k8pBX7RAwBzJqfOxHE0HOezRNV1HGgxhRFEu9r9BpWQ,52514
+langroid-0.1.265.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+langroid-0.1.265.dist-info/RECORD,,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langroid"
-version = "0.1.262"
+version = "0.1.265"
 description = "Harness LLMs with Multi-Agent Programming"
 authors = ["Prasad Chalasani <pchalasani@gmail.com>"]
 readme = "README.md"
@@ -20,7 +20,6 @@ sentence-transformers = {version="2.2.2", optional=true}
 torch = {version="2.0.0", optional=true}
 psycopg2 = {version="^2.9.7", optional=true}
 pymysql = {version = "^1.1.0", optional = true}
-meilisearch = {version="^0.28.3", optional=true}
 meilisearch-python-sdk = {version="^2.2.3", optional=true}
 litellm = {version = "^1.30.1", optional = true}
 metaphor-python = {version = "^0.1.23", optional = true}
@@ -29,7 +28,7 @@ python-socketio = {version="^5.11.0", optional=true}
 neo4j = {version = "^5.14.1", optional = true}
 huggingface-hub = {version="^0.21.2", optional=true}
 transformers = {version="^4.40.1", optional=true}
-lancedb = {version="^0.6.2", optional=true}
+lancedb = {version="^0.8.2", optional=true}
 tantivy = {version="^0.21.0", optional=true}
 pypdf = {version="^3.12.2", optional=true}
 pymupdf = {version="^1.23.3", optional=true}
@@ -65,7 +64,7 @@ requests = "^2.31.0"
 pyparsing = "^3.0.9"
 nltk = "^3.8.1"
 qdrant-client = "^1.8.0"
-pydantic = "1.10.13"
+pydantic = ">=1,<3"
 pandas = "^2.0.3"
 prettytable = "^3.8.0"
 
@@ -107,7 +106,8 @@ all = [
     "pymupdf", "pdf2image", "pytesseract",
     "postgres", "mysql", "sqlalchemy", "psycopg2", "pymysql",
     "sentence-transformers", "torch", "transformers", "huggingface-hub",
-    "lancedb", "tantivy", "pyarrow", "chromadb",
+#    "lancedb", "tantivy", "pyarrow",
+    "chromadb",
     "metaphor-python", "neo4j",
     "litellm",
     "chainlit", "python-socketio",
@@ -133,7 +133,7 @@ mkdocs = [
     "mkdocs-gen-files", "mkdocs-literate-nav",
     "mkdocs-section-index", "mkdocs-jupyter", "mkdocs-rss-plugin"
 ]
-meilisearch = ["meilisearch", "meilisearch-python-sdk"]
+meilisearch = ["meilisearch-python-sdk"]
 momento = ["momento"]
 
 

langroid/parsing/parser.pyi

@@ -1,56 +0,0 @@
-from enum import Enum
-from typing import Literal
-
-from _typeshed import Incomplete
-from pydantic import BaseSettings
-
-from langroid.mytypes import Document as Document
-from langroid.parsing.para_sentence_split import (
-    create_chunks as create_chunks,
-)
-from langroid.parsing.para_sentence_split import (
-    remove_extra_whitespace as remove_extra_whitespace,
-)
-
-logger: Incomplete
-
-class Splitter(str, Enum):
-    TOKENS: str
-    PARA_SENTENCE: str
-    SIMPLE: str
-
-class PdfParsingConfig(BaseSettings):
-    library: Literal["fitz", "pdfplumber", "pypdf", "unstructured", "pdf2image"]
-
-class DocxParsingConfig(BaseSettings):
-    library: Literal["python-docx", "unstructured"]
-
-class DocParsingConfig(BaseSettings):
-    library: Literal["unstructured"]
-
-class ParsingConfig(BaseSettings):
-    splitter: str
-    chunk_size: int
-    overlap: int
-    max_chunks: int
-    min_chunk_chars: int
-    discard_chunk_chars: int
-    n_similar_docs: int
-    n_neighbor_ids: int
-    separators: list[str]
-    token_encoding_model: str
-    pdf: PdfParsingConfig
-    docx: DocxParsingConfig
-    doc: DocParsingConfig
-
-class Parser:
-    config: Incomplete
-    tokenizer: Incomplete
-    def __init__(self, config: ParsingConfig) -> None: ...
-    def num_tokens(self, text: str) -> int: ...
-    def add_window_ids(self, chunks: list[Document]) -> None: ...
-    def split_simple(self, docs: list[Document]) -> list[Document]: ...
-    def split_para_sentence(self, docs: list[Document]) -> list[Document]: ...
-    def split_chunk_tokens(self, docs: list[Document]) -> list[Document]: ...
-    def chunk_tokens(self, text: str) -> list[str]: ...
-    def split(self, docs: list[Document]) -> list[Document]: ...

langroid/prompts/transforms.py

@@ -1,84 +0,0 @@
-import asyncio
-from typing import List, Tuple
-
-import aiohttp
-
-from langroid.language_models.base import LanguageModel
-from langroid.mytypes import Document
-from langroid.prompts.dialog import collate_chat_history
-from langroid.prompts.templates import EXTRACTION_PROMPT
-
-
-async def get_verbatim_extract_async(
-    question: str,
-    passage: Document,
-    LLM: LanguageModel,
-) -> str:
-    """
-    Asynchronously, get verbatim extract from passage that is relevant to a question.
-    """
-    async with aiohttp.ClientSession():
-        templatized_prompt = EXTRACTION_PROMPT
-        final_prompt = templatized_prompt.format(question=question, content=passage)
-        final_extract = await LLM.agenerate(prompt=final_prompt, max_tokens=1024)
-
-    return final_extract.message.strip()
-
-
-async def _get_verbatim_extracts(
-    question: str,
-    passages: List[Document],
-    LLM: LanguageModel,
-) -> List[Document]:
-    async with aiohttp.ClientSession():
-        verbatim_extracts = await asyncio.gather(
-            *(get_verbatim_extract_async(question, P, LLM) for P in passages)
-        )
-    metadatas = [P.metadata for P in passages]
-    # return with metadata so we can use it downstream, e.g. to cite sources
-    return [
-        Document(content=e, metadata=m) for e, m in zip(verbatim_extracts, metadatas)
-    ]
-
-
-def get_verbatim_extracts(
-    question: str,
-    passages: List[Document],
-    LLM: LanguageModel,
-) -> List[Document]:
-    """
-    From each passage, extract verbatim text that is relevant to a question,
-    using concurrent API calls to the LLM.
-    Args:
-        question: question to be answered
-        passages: list of passages from which to extract relevant verbatim text
-        LLM: LanguageModel to use for generating the prompt and extract
-    Returns:
-        list of verbatim extracts (Documents) from passages that are relevant to
-        question
-    """
-    return asyncio.run(_get_verbatim_extracts(question, passages, LLM))
-
-
-def followup_to_standalone(
-    LLM: LanguageModel, chat_history: List[Tuple[str, str]], question: str
-) -> str:
-    """
-    Given a chat history and a question, convert it to a standalone question.
-    Args:
-        chat_history: list of tuples of (question, answer)
-        query: follow-up question
-
-    Returns: standalone version of the question
-    """
-    history = collate_chat_history(chat_history)
-
-    prompt = f"""
-    Given the conversationn below, and a follow-up question, rephrase the follow-up 
-    question as a standalone question.
-    
-    Chat history: {history}
-    Follow-up question: {question} 
-    """.strip()
-    standalone = LLM.generate(prompt=prompt, max_tokens=1024).message.strip()
-    return standalone