beswarm 0.1.36__tar.gz → 0.1.38__tar.gz

{beswarm-0.1.36 → beswarm-0.1.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: beswarm
-Version: 0.1.36
+Version: 0.1.38
 Summary: MAS
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -27,7 +27,6 @@ Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: pytz>=2025.2
 Requires-Dist: requests>=2.32.3
 Requires-Dist: scipy>=1.15.2
-Requires-Dist: tiktoken==0.6.0
 Requires-Dist: tqdm>=4.67.1
 
 # beswarm

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm/aient/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="aient",
-    version="1.0.93",
+    version="1.0.94",
     description="Aient: The Awakening of Agent.",
     long_description=Path.open(Path("README.md"), encoding="utf-8").read(),
     long_description_content_type="text/markdown",

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm/aient/src/aient/models/claude.py

@@ -2,7 +2,6 @@ import os
 import re
 import json
 import copy
-import tiktoken
 import requests
 
 from .base import BaseLLM
@@ -65,39 +64,6 @@ class claude(BaseLLM):
         self.conversation[convo_id] = claudeConversation()
         self.system_prompt = system_prompt or self.system_prompt
 
-    def __truncate_conversation(self, convo_id: str = "default") -> None:
-        """
-        Truncate the conversation
-        """
-        while True:
-            if (
-                self.get_token_count(convo_id) > self.truncate_limit
-                and len(self.conversation[convo_id]) > 1
-            ):
-                # Don't remove the first message
-                self.conversation[convo_id].pop(1)
-            else:
-                break
-
-    def get_token_count(self, convo_id: str = "default") -> int:
-        """
-        Get token count
-        """
-        tiktoken.model.MODEL_TO_ENCODING["claude-2.1"] = "cl100k_base"
-        encoding = tiktoken.encoding_for_model(self.engine)
-
-        num_tokens = 0
-        for message in self.conversation[convo_id]:
-            # every message follows <im_start>{role/name}\n{content}<im_end>\n
-            num_tokens += 5
-            for key, value in message.items():
-                if value:
-                    num_tokens += len(encoding.encode(value))
-                if key == "name":  # if there's a name, the role is omitted
-                    num_tokens += 5  # role is always required and always 1 token
-        num_tokens += 5  # every reply is primed with <im_start>assistant
-        return num_tokens
-
     def ask_stream(
         self,
         prompt: str,
@@ -267,39 +233,6 @@ class claude3(BaseLLM):
         self.conversation[convo_id] = list()
         self.system_prompt = system_prompt or self.system_prompt
 
-    def __truncate_conversation(self, convo_id: str = "default") -> None:
-        """
-        Truncate the conversation
-        """
-        while True:
-            if (
-                self.get_token_count(convo_id) > self.truncate_limit
-                and len(self.conversation[convo_id]) > 1
-            ):
-                # Don't remove the first message
-                self.conversation[convo_id].pop(1)
-            else:
-                break
-
-    def get_token_count(self, convo_id: str = "default") -> int:
-        """
-        Get token count
-        """
-        tiktoken.model.MODEL_TO_ENCODING["claude-2.1"] = "cl100k_base"
-        encoding = tiktoken.encoding_for_model(self.engine)
-
-        num_tokens = 0
-        for message in self.conversation[convo_id]:
-            # every message follows <im_start>{role/name}\n{content}<im_end>\n
-            num_tokens += 5
-            for key, value in message.items():
-                if value:
-                    num_tokens += len(encoding.encode(value))
-                if key == "name":  # if there's a name, the role is omitted
-                    num_tokens += 5  # role is always required and always 1 token
-        num_tokens += 5  # every reply is primed with <im_start>assistant
-        return num_tokens
-
     def ask_stream(
         self,
         prompt: str,

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm/aient/src/aient/models/groq.py

@@ -1,7 +1,6 @@
 import os
 import json
 import requests
-import tiktoken
 
 from .base import BaseLLM
 
@@ -52,39 +51,6 @@ class groq(BaseLLM):
         self.conversation[convo_id] = list()
         self.system_prompt = system_prompt or self.system_prompt
 
-    def __truncate_conversation(self, convo_id: str = "default") -> None:
-        """
-        Truncate the conversation
-        """
-        while True:
-            if (
-                self.get_token_count(convo_id) > self.truncate_limit
-                and len(self.conversation[convo_id]) > 1
-            ):
-                # Don't remove the first message
-                self.conversation[convo_id].pop(1)
-            else:
-                break
-
-    def get_token_count(self, convo_id: str = "default") -> int:
-        """
-        Get token count
-        """
-        # tiktoken.model.MODEL_TO_ENCODING["mixtral-8x7b-32768"] = "cl100k_base"
-        encoding = tiktoken.get_encoding("cl100k_base")
-
-        num_tokens = 0
-        for message in self.conversation[convo_id]:
-            # every message follows <im_start>{role/name}\n{content}<im_end>\n
-            num_tokens += 5
-            for key, value in message.items():
-                if value:
-                    num_tokens += len(encoding.encode(value))
-                if key == "name":  # if there's a name, the role is omitted
-                    num_tokens += 5  # role is always required and always 1 token
-        num_tokens += 5  # every reply is primed with <im_start>assistant
-        return num_tokens
-
     def ask_stream(
         self,
         prompt: str,

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm/aient/src/aient/plugins/config.py

@@ -3,8 +3,7 @@ import json
 import inspect
 
 from .registry import registry
-from ..utils.scripts import cut_message
-from ..utils.prompt import search_key_word_prompt, arxiv_doc_user_prompt
+from ..utils.prompt import search_key_word_prompt
 
 async def get_tools_result_async(function_call_name, function_full_response, function_call_max_tokens, engine, robot, api_key, api_url, use_plugins, model, add_message, convo_id, language):
     function_response = ""
@@ -26,10 +25,7 @@ async def get_tools_result_async(function_call_name, function_full_response, fun
                 yield chunk
             else:
                 function_response = "\n\n".join(chunk)
-            # function_response = yield chunk
-        # function_response = yield from eval(function_call_name)(prompt, keywords)
-        function_call_max_tokens = 32000
-        function_response, text_len = cut_message(function_response, function_call_max_tokens, engine)
+
         if function_response:
             function_response = (
                 f"You need to response the following question: {prompt}. Search results is provided inside <Search_results></Search_results> XML tags. Your task is to think about the question step by step and then answer the above question in {language} based on the Search results provided. Please response in {language} and adopt a style that is logical, in-depth, and detailed. Note: In order to make the answer appear highly professional, you should be an expert in textual analysis, aiming to make the answer precise and comprehensive. Directly response markdown format, without using markdown code blocks. For each sentence quoting search results, a markdown ordered superscript number url link must be used to indicate the source, e.g., [¹](https://www.example.com)"
@@ -40,18 +36,13 @@ async def get_tools_result_async(function_call_name, function_full_response, fun
             ).format(function_response)
         else:
             function_response = "无法找到相关信息，停止使用 tools"
-        # user_prompt = f"You need to response the following question: {prompt}. Search results is provided inside <Search_results></Search_results> XML tags. Your task is to think about the question step by step and then answer the above question in {config.language} based on the Search results provided. Please response in {config.language} and adopt a style that is logical, in-depth, and detailed. Note: In order to make the answer appear highly professional, you should be an expert in textual analysis, aiming to make the answer precise and comprehensive. Directly response markdown format, without using markdown code blocks"
-        # self.add_to_conversation(user_prompt, "user", convo_id=convo_id)
+
     elif function_to_call:
         prompt = json.loads(function_full_response)
         if inspect.iscoroutinefunction(function_to_call):
             function_response = await function_to_call(**prompt)
         else:
             function_response = function_to_call(**prompt)
-        function_response, text_len = cut_message(function_response, function_call_max_tokens, engine)
-
-    # if function_call_name == "download_read_arxiv_pdf":
-    #     add_message(arxiv_doc_user_prompt, "user", convo_id=convo_id)
 
     function_response = (
         f"function_response:{function_response}"

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm/aient/src/aient/utils/scripts.py

@@ -1,33 +1,10 @@
 import os
 import json
-import base64
-import tiktoken
 import requests
 import urllib.parse
 
 from ..core.utils import get_image_message
 
-def get_encode_text(text, model_name):
-    tiktoken.get_encoding("cl100k_base")
-    model_name = "gpt-3.5-turbo"
-    encoding = tiktoken.encoding_for_model(model_name)
-    encode_text = encoding.encode(text, disallowed_special=())
-    return encoding, encode_text
-
-def get_text_token_len(text, model_name):
-    encoding, encode_text = get_encode_text(text, model_name)
-    return len(encode_text)
-
-def cut_message(message: str, max_tokens: int, model_name: str):
-    if type(message) != str:
-        message = str(message)
-    encoding, encode_text = get_encode_text(message, model_name)
-    if len(encode_text) > max_tokens:
-        encode_text = encode_text[:max_tokens]
-        message = encoding.decode(encode_text)
-    encode_text = encoding.encode(message, disallowed_special=())
-    return message, len(encode_text)
-
 def get_doc_from_url(url):
     filename = urllib.parse.unquote(url.split("/")[-1])
     response = requests.get(url, stream=True)

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm/tools/UIworker.py

@@ -3,7 +3,6 @@ import io
 import copy
 import base64
 import platform
-import pyautogui
 from datetime import datetime
 from ..aient.src.aient.plugins import register_tool, get_function_call_list
 
@@ -16,6 +15,7 @@ from ..utils import extract_xml_content
 async def get_current_screen_image_message(prompt):
     print("instruction agent 正在截取当前屏幕...")
     try:
+        import pyautogui
         # 使用 pyautogui 截取屏幕，返回 PIL Image 对象
         screenshot = pyautogui.screenshot()
         # img_width, img_height = screenshot.size # 获取截图尺寸

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: beswarm
-Version: 0.1.36
+Version: 0.1.38
 Summary: MAS
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -27,7 +27,6 @@ Requires-Dist: pyperclip>=1.9.0
 Requires-Dist: pytz>=2025.2
 Requires-Dist: requests>=2.32.3
 Requires-Dist: scipy>=1.15.2
-Requires-Dist: tiktoken==0.6.0
 Requires-Dist: tqdm>=4.67.1
 
 # beswarm

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm.egg-info/SOURCES.txt

@@ -66,7 +66,6 @@ beswarm/aient/test/test_get_token_dict.py
 beswarm/aient/test/test_google_search.py
 beswarm/aient/test/test_jieba.py
 beswarm/aient/test/test_json.py
-beswarm/aient/test/test_langchain_search_old.py
 beswarm/aient/test/test_logging.py
 beswarm/aient/test/test_ollama.py
 beswarm/aient/test/test_plugin.py

{beswarm-0.1.36 → beswarm-0.1.38}/beswarm.egg-info/requires.txt

@@ -21,5 +21,4 @@ pyperclip>=1.9.0
 pytz>=2025.2
 requests>=2.32.3
 scipy>=1.15.2
-tiktoken==0.6.0
 tqdm>=4.67.1

{beswarm-0.1.36 → beswarm-0.1.38}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "beswarm"
-version = "0.1.36"
+version = "0.1.38"
 description = "MAS"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -28,7 +28,6 @@ dependencies = [
     "pytz>=2025.2",
     "requests>=2.32.3",
     "scipy>=1.15.2",
-    "tiktoken==0.6.0",
     "tqdm>=4.67.1",
 ]
 

beswarm-0.1.36/beswarm/aient/test/test_langchain_search_old.py

@@ -1,235 +0,0 @@
-import os
-import re
-
-import sys
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import config
-
-from langchain.chat_models import ChatOpenAI
-
-
-from langchain.chains import RetrievalQA, RetrievalQAWithSourcesChain
-
-from langchain.prompts.chat import (
-    ChatPromptTemplate,
-    SystemMessagePromptTemplate,
-    HumanMessagePromptTemplate,
-)
-from langchain.embeddings.openai import OpenAIEmbeddings
-from langchain.vectorstores import Chroma
-from langchain.text_splitter import CharacterTextSplitter
-
-from langchain.document_loaders import UnstructuredPDFLoader
-
-def getmd5(string):
-    import hashlib
-    md5_hash = hashlib.md5()
-    md5_hash.update(string.encode('utf-8'))
-    md5_hex = md5_hash.hexdigest()
-    return md5_hex
-
-from utils.sitemap import SitemapLoader
-async def get_doc_from_sitemap(url):
-    # https://www.langchain.asia/modules/indexes/document_loaders/examples/sitemap#%E8%BF%87%E6%BB%A4%E7%AB%99%E7%82%B9%E5%9C%B0%E5%9B%BE-url-
-    sitemap_loader = SitemapLoader(web_path=url)
-    docs = await sitemap_loader.load()
-    return docs
-
-async def get_doc_from_local(docpath, doctype="md"):
-    from langchain.document_loaders import DirectoryLoader
-    # 加载文件夹中的所有txt类型的文件
-    loader = DirectoryLoader(docpath, glob='**/*.' + doctype)
-    # 将数据转成 document 对象，每个文件会作为一个 document
-    documents = loader.load()
-    return documents
-
-system_template="""Use the following pieces of context to answer the users question. 
-If you don't know the answer, just say "Hmm..., I'm not sure.", don't try to make up an answer.
-ALWAYS return a "Sources" part in your answer.
-The "Sources" part should be a reference to the source of the document from which you got your answer.
-
-Example of your response should be:
-
-```
-The answer is foo
-
-Sources:
-1. abc
-2. xyz
-```
-Begin!
-----------------
-{summaries}
-"""
-messages = [
-    SystemMessagePromptTemplate.from_template(system_template),
-    HumanMessagePromptTemplate.from_template("{question}")
-]
-prompt = ChatPromptTemplate.from_messages(messages)
-
-def get_chain(store, llm):
-    chain_type_kwargs = {"prompt": prompt}
-    chain = RetrievalQAWithSourcesChain.from_chain_type(
-        llm, 
-        chain_type="stuff", 
-        retriever=store.as_retriever(),
-        chain_type_kwargs=chain_type_kwargs,
-        reduce_k_below_max_tokens=True
-    )
-    return chain
-
-async def docQA(docpath, query_message, persist_db_path="db", model = "gpt-3.5-turbo"):
-    chatllm = ChatOpenAI(temperature=0.5, openai_api_base=config.bot_api_url.v1_url, model_name=model, openai_api_key=config.API)
-    embeddings = OpenAIEmbeddings(openai_api_base=config.bot_api_url.v1_url, openai_api_key=config.API)
-
-    sitemap = "sitemap.xml"
-    match = re.match(r'^(https?|ftp)://[^\s/$.?#].[^\s]*$', docpath)
-    if match:
-        doc_method = get_doc_from_sitemap
-        docpath = os.path.join(docpath, sitemap)
-    else:
-        doc_method = get_doc_from_local
-
-    persist_db_path = getmd5(docpath)
-    if not os.path.exists(persist_db_path):
-        documents = await doc_method(docpath)
-        # 初始化加载器
-        text_splitter = CharacterTextSplitter(chunk_size=100, chunk_overlap=50)
-        # 持久化数据
-        split_docs = text_splitter.split_documents(documents)
-        vector_store = Chroma.from_documents(split_docs, embeddings, persist_directory=persist_db_path)
-        vector_store.persist()
-    else:
-        # 加载数据
-        vector_store = Chroma(persist_directory=persist_db_path, embedding_function=embeddings)
-
-    # 创建问答对象
-    qa = get_chain(vector_store, chatllm)
-    # qa = RetrievalQA.from_chain_type(llm=chatllm, chain_type="stuff", retriever=vector_store.as_retriever(), return_source_documents=True)
-    # 进行问答
-    result = qa({"question": query_message})
-    return result
-
-
-def persist_emdedding_pdf(docurl, persist_db_path):
-    embeddings = OpenAIEmbeddings(openai_api_base=config.bot_api_url.v1_url, openai_api_key=os.environ.get('API', None))
-    filename = get_doc_from_url(docurl)
-    docpath = os.getcwd() + "/" + filename
-    loader = UnstructuredPDFLoader(docpath)
-    documents = loader.load()
-    # 初始化加载器
-    text_splitter = CharacterTextSplitter(chunk_size=100, chunk_overlap=25)
-    # 切割加载的 document
-    split_docs = text_splitter.split_documents(documents)
-    vector_store = Chroma.from_documents(split_docs, embeddings, persist_directory=persist_db_path)
-    vector_store.persist()
-    os.remove(docpath)
-    return vector_store
-
-async def pdfQA(docurl, docpath, query_message, model="gpt-3.5-turbo"):
-    chatllm = ChatOpenAI(temperature=0.5, openai_api_base=config.bot_api_url.v1_url, model_name=model, openai_api_key=os.environ.get('API', None))
-    embeddings = OpenAIEmbeddings(openai_api_base=config.bot_api_url.v1_url, openai_api_key=os.environ.get('API', None))
-    persist_db_path = getmd5(docpath)
-    if not os.path.exists(persist_db_path):
-        vector_store = persist_emdedding_pdf(docurl, persist_db_path)
-    else:
-        vector_store = Chroma(persist_directory=persist_db_path, embedding_function=embeddings)
-    qa = RetrievalQA.from_chain_type(llm=chatllm, chain_type="stuff", retriever=vector_store.as_retriever(), return_source_documents=True)
-    result = qa({"query": query_message})
-    return result['result']
-
-
-def pdf_search(docurl, query_message, model="gpt-3.5-turbo"):
-    chatllm = ChatOpenAI(temperature=0.5, openai_api_base=config.bot_api_url.v1_url, model_name=model, openai_api_key=os.environ.get('API', None))
-    embeddings = OpenAIEmbeddings(openai_api_base=config.bot_api_url.v1_url, openai_api_key=os.environ.get('API', None))
-    filename = get_doc_from_url(docurl)
-    docpath = os.getcwd() + "/" + filename
-    loader = UnstructuredPDFLoader(docpath)
-    try:
-        documents = loader.load()
-    except:
-        print("pdf load error! docpath:", docpath)
-        return ""
-    os.remove(docpath)
-    # 初始化加载器
-    text_splitter = CharacterTextSplitter(chunk_size=100, chunk_overlap=25)
-    # 切割加载的 document
-    split_docs = text_splitter.split_documents(documents)
-    vector_store = Chroma.from_documents(split_docs, embeddings)
-    # 创建问答对象
-    qa = RetrievalQA.from_chain_type(llm=chatllm, chain_type="stuff", retriever=vector_store.as_retriever(),return_source_documents=True)
-    # 进行问答
-    result = qa({"query": query_message})
-    return result['result']
-
-def summary_each_url(threads, chainllm, prompt):
-    summary_prompt = PromptTemplate(
-        input_variables=["web_summary", "question", "language"],
-        template=(
-            "You need to response the following question: {question}."
-            "Your task is answer the above question in {language} based on the Search results provided. Provide a detailed and in-depth response"
-            "If there is no relevant content in the search results, just answer None, do not make any explanations."
-            "Search results: {web_summary}."
-        ),
-    )
-    summary_threads = []
-
-    for t in threads:
-        tmp = t.join()
-        print(tmp)
-        chain = LLMChain(llm=chainllm, prompt=summary_prompt)
-        chain_thread = ThreadWithReturnValue(target=chain.run, args=({"web_summary": tmp, "question": prompt, "language": config.LANGUAGE},))
-        chain_thread.start()
-        summary_threads.append(chain_thread)
-
-    url_result = ""
-    for t in summary_threads:
-        tmp = t.join()
-        print("summary", tmp)
-        if tmp != "None":
-            url_result += "\n\n" + tmp
-    return url_result
-
-def get_search_results(prompt: str, context_max_tokens: int):
-
-    url_text_list = get_url_text_list(prompt)
-    useful_source_text = "\n\n".join(url_text_list)
-    # useful_source_text = summary_each_url(threads, chainllm, prompt)
-
-    useful_source_text, search_tokens_len = cut_message(useful_source_text, context_max_tokens)
-    print("search tokens len", search_tokens_len, "\n\n")
-
-    return useful_source_text
-
-from typing import Any
-from langchain.schema.output import LLMResult
-from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
-class ChainStreamHandler(StreamingStdOutCallbackHandler):
-    def __init__(self):
-        self.tokens = []
-        # 记得结束后这里置true
-        self.finish = False
-        self.answer = ""
-
-    def on_llm_new_token(self, token: str, **kwargs):
-        # print(token)
-        self.tokens.append(token)
-        # yield ''.join(self.tokens)
-        # print(''.join(self.tokens))
-
-    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
-        self.finish = 1
-
-    def on_llm_error(self, error: Exception, **kwargs: Any) -> None:
-        print(str(error))
-        self.tokens.append(str(error))
-
-    def generate_tokens(self):
-        while not self.finish or self.tokens:
-            if self.tokens:
-                data = self.tokens.pop(0)
-                self.answer += data
-                yield data
-            else:
-                pass
-        return self.answer