hjxdl 0.1.38__py3-none-any.whl → 0.1.40__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.38'
-__version_tuple__ = version_tuple = (0, 1, 38)
+__version__ = version = '0.1.40'
+__version_tuple__ = version_tuple = (0, 1, 40)

hdl/utils/llm/chat.py

@@ -1,6 +1,8 @@
 import typing as t
+import asyncio
+from concurrent.futures import ProcessPoolExecutor
+
 
-from llama_cpp import Llama
 from openai import OpenAI
 from ..desc.template import FN_TEMPLATE
 from ..desc.func_desc import FN_DESC
@@ -85,126 +87,6 @@ def chat_oai_invoke(
     return response.choices[0].message.content
 
 
-class GGUF_M(Llama):
-    def __init__(
-        self,
-        model_path :str,
-        device: str='gpu',
-        generation_kwargs: dict = {},
-        server_ip: str = "127.0.0.1",
-        server_port: int = 8000,
-        *args,
-        **kwargs
-    ):
-        """Initialize the model with the specified parameters.
-
-        Args:
-            model_path (str): The path to the model.
-            device (str, optional): The device to use, either 'gpu' or 'cpu'. Defaults to 'gpu'.
-            generation_kwargs (dict, optional): Additional generation keyword arguments. Defaults to {}.
-            server_ip (str, optional): The IP address of the server. Defaults to "127.0.0.1".
-            server_port (int, optional): The port of the server. Defaults to 8000.
-            *args: Variable length argument list.
-            **kwargs: Arbitrary keyword arguments.
-
-        Raises:
-            KeyError: If 'num_threads' or 'max_context_length' is missing in generation_kwargs.
-        """
-        print("正在从本地加载模型...")
-        if device.lower() == 'cpu':
-            super().__init__(
-                model_path=model_path,
-                n_threads=generation_kwargs['num_threads'],
-                n_ctx=generation_kwargs['max_context_length'],
-                *args,
-                **kwargs
-            )
-        else:
-            super().__init__(
-                model_path=model_path,
-                n_threads=generation_kwargs['num_threads'],
-                n_ctx=generation_kwargs['max_context_length'],
-                n_gpu_layers=-1,
-                flash_attn=True,
-                *args,
-                **kwargs
-            )
-        self.generation_kwargs = generation_kwargs
-
-    def invoke(
-        self,
-        prompt : str,
-        stop: list[str] | None = ["USER:", "ASSISTANT:"],
-        # history: list = [],
-        **kwargs: t.Any,
-    ) -> str:
-        """Invoke the model to generate a response based on the given prompt.
-
-        Args:
-            prompt (str): The prompt to be used for generating the response.
-            stop (list[str], optional): List of strings that indicate when the model should stop generating the response. Defaults to ["USER:", "ASSISTANT:"].
-            **kwargs: Additional keyword arguments to be passed to the model.
-
-        Returns:
-            str: The generated response based on the prompt.
-        """
-        prompt_final = f"USER:\n{prompt}\nASSISTANT:\n"
-
-        result = self.create_completion(
-            prompt_final,
-            repeat_penalty=self.generation_kwargs["repetition_penalty"],
-            max_tokens=self.generation_kwargs["max_new_tokens"],
-            stop=stop,
-            echo=False,
-            temperature=self.generation_kwargs["temperature"],
-            mirostat_mode = 2,
-            mirostat_tau=4.0,
-            mirostat_eta=1.1
-        )
-        resp = result['choices'][0]['text']
-        # history.append(
-        #     [prompt, resp]
-        # )
-        return resp
-
-    def stream(
-        self,
-        prompt: str,
-        stop: list[str] | None = ["USER:", "ASSISTANT:"],
-        # history: list = [],
-        **kwargs: t.Any,
-    ):
-        """Generate text responses based on the given prompt using the model.
-
-        Args:
-            prompt (str): The prompt to generate text responses.
-            stop (list[str], optional): List of strings to stop the generation. Defaults to ["USER:", "ASSISTANT:"].
-            **kwargs: Additional keyword arguments for the model.
-
-        Yields:
-            str: Text responses generated by the model based on the prompt.
-        """
-        prompt = f"USER:\n{prompt}\nASSISTANT:\n"
-        output = self.create_completion(
-            prompt,
-            stream=True,
-            repeat_penalty=self.generation_kwargs["repetition_penalty"],
-            max_tokens=self.generation_kwargs["max_new_tokens"],
-            stop=stop,
-            echo=False,
-            temperature=self.generation_kwargs["temperature"],
-            mirostat_mode = 2,
-            mirostat_tau=4.0,
-            mirostat_eta=1.1
-        )
-        # history.append([])
-        for chunk in output:
-            item = chunk['choices'][0]['text']
-            # self.resps[-1].append(item)
-            yield chunk['choices'][0]['text']
-        # self.resps[-1] = "".join(self.resps[-1])
-
-
 class OpenAI_M():
     def __init__(
         self,
@@ -316,47 +198,47 @@ class OpenAI_M():
             if content:
                 yield content
 
-    def get_decision(
+    def agent_response(
         self,
-        prompt: str,
+        prompt : str,
+        stream = True,
         **kwargs: t.Any
     ):
-        """Get decision based on the provided prompt and additional keyword arguments.
+        """'''Generate agent response based on the given prompt.
 
         Args:
-            prompt (str): The prompt for decision making.
+            prompt (str): The prompt for which agent response is generated.
+            stream (bool, optional): Flag to determine if the response should be streamed. Defaults to True.
             **kwargs: Additional keyword arguments.
 
         Returns:
-            dict: A dictionary containing the decision.
-
-        Example:
-            decision = get_decision("Should I buy this product?", option1="yes", option2="no")
+            str: The agent response based on the prompt.
+        '''
         """
+        decision_dict = self.get_decision(prompt, **kwargs)
+        if decision_dict.get("function_name", None) is None:
+            return self.stream(prompt, **kwargs)
+        else:
+            tool_result = str(self.get_tool_result(prompt, **kwargs))
+            prompt_final = "根据上下文回答最后的用户问题：\n上下文信息：\n"
+            prompt_final += tool_result
+            prompt_final += f"\n用户的问题：\n{prompt}"
+            if stream:
+                return self.stream(prompt_final, **kwargs)
+            else:
+                return self.invoke(prompt_final, **kwargs)
+
+    def get_decision(self, prompt: str, **kwargs: t.Any):
+        # 该方法与之前一致...
         prompt_final = FN_TEMPLATE
         for tool in self.tools:
             prompt_final += self.tool_desc.get(tool.__name__, "")
         prompt_final += f"\n用户的问题：\n{prompt}"
-        # print(prompt_final)
-        decision_dict_str = self.invoke(prompt_final ,**kwargs)
-        print(decision_dict_str)
+        decision_dict_str = self.invoke(prompt_final, **kwargs)
         return decision_dict_str
-        # return json.loads(decision_dict)
 
-    def get_tool_result(
-        self,
-        prompt: str,
-        **kwargs: t.Any
-    ):
-        """Get the result from a tool based on the provided prompt and keyword arguments.
-
-            Args:
-                prompt (str): The prompt to get the decision for.
-                **kwargs: Additional keyword arguments to pass to the decision function.
-
-            Returns:
-                str: The result from the selected tool based on the decision made.
-        """
+    def get_tool_result(self, prompt: str, **kwargs: t.Any):
+        # 同步方法与之前一致...
         decision_dict_str = self.get_decision(prompt, **kwargs)
         try:
             decision_dict = json.loads(decision_dict_str)
@@ -377,33 +259,41 @@ class OpenAI_M():
                 print(e)
                 return ""
 
-
-    def agent_response(
-        self,
-        prompt : str,
-        stream = True,
-        **kwargs: t.Any
-    ):
-        """'''Generate agent response based on the given prompt.
+    async def get_tool_result_async(self, prompt: str, **kwargs: t.Any):
+        """
+        Asynchronous version of the get_tool_result function that can run in parallel using multiprocessing.
 
         Args:
-            prompt (str): The prompt for which agent response is generated.
-            stream (bool, optional): Flag to determine if the response should be streamed. Defaults to True.
-            **kwargs: Additional keyword arguments.
+            prompt (str): The prompt to get the decision for.
+            **kwargs: Additional keyword arguments to pass to the decision function.
 
         Returns:
-            str: The agent response based on the prompt.
-        '''
+            str: The result from the selected tool based on the decision made.
         """
-        decision_dict = self.get_decision(prompt, **kwargs)
-        if decision_dict.get("function_name", None) is None:
-            return self.stream(prompt, **kwargs)
+
+        def run_tool_with_kwargs(tool, **kwargs):
+            return tool(**kwargs)
+
+        decision_dict_str = await asyncio.to_thread(self.get_decision, prompt, **kwargs)
+        try:
+            decision_dict = json.loads(decision_dict_str)
+        except Exception as e:
+            print(e)
+            return ""
+        func_name = decision_dict.get("function_name", None)
+        if func_name is None:
+            return ""
         else:
-            tool_result = str(self.get_tool_result(prompt, **kwargs))
-            prompt_final = "根据上下文回答最后的用户问题：\n上下文信息：\n"
-            prompt_final += tool_result
-            prompt_final += f"\n用户的问题：\n{prompt}"
-            if stream:
-                return self.stream(prompt_final, **kwargs)
-            else:
-                return self.invoke(prompt_final, **kwargs)
+            try:
+                for tool in self.tools:
+                    if tool.__name__ == func_name:
+                        tool_final = tool
+                func_kwargs = decision_dict.get("params")
+
+                loop = asyncio.get_running_loop()
+                with ProcessPoolExecutor() as pool:
+                    result = await loop.run_in_executor(pool, run_tool_with_kwargs, tool_final, **func_kwargs)
+                return result
+            except Exception as e:
+                print(e)
+                return ""

hdl/utils/llm/llama_chat.py

@@ -0,0 +1,122 @@
+import typing as t
+
+from llama_cpp import Llama
+
+class GGUF_M(Llama):
+    def __init__(
+        self,
+        model_path :str,
+        device: str='gpu',
+        generation_kwargs: dict = {},
+        server_ip: str = "127.0.0.1",
+        server_port: int = 8000,
+        *args,
+        **kwargs
+    ):
+        """Initialize the model with the specified parameters.
+
+        Args:
+            model_path (str): The path to the model.
+            device (str, optional): The device to use, either 'gpu' or 'cpu'. Defaults to 'gpu'.
+            generation_kwargs (dict, optional): Additional generation keyword arguments. Defaults to {}.
+            server_ip (str, optional): The IP address of the server. Defaults to "127.0.0.1".
+            server_port (int, optional): The port of the server. Defaults to 8000.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Raises:
+            KeyError: If 'num_threads' or 'max_context_length' is missing in generation_kwargs.
+        """
+        print("正在从本地加载模型...")
+        if device.lower() == 'cpu':
+            super().__init__(
+                model_path=model_path,
+                n_threads=generation_kwargs['num_threads'],
+                n_ctx=generation_kwargs['max_context_length'],
+                *args,
+                **kwargs
+            )
+        else:
+            super().__init__(
+                model_path=model_path,
+                n_threads=generation_kwargs['num_threads'],
+                n_ctx=generation_kwargs['max_context_length'],
+                n_gpu_layers=-1,
+                flash_attn=True,
+                *args,
+                **kwargs
+            )
+        self.generation_kwargs = generation_kwargs
+
+    def invoke(
+        self,
+        prompt : str,
+        stop: list[str] | None = ["USER:", "ASSISTANT:"],
+        # history: list = [],
+        **kwargs: t.Any,
+    ) -> str:
+        """Invoke the model to generate a response based on the given prompt.
+
+        Args:
+            prompt (str): The prompt to be used for generating the response.
+            stop (list[str], optional): List of strings that indicate when the model should stop generating the response. Defaults to ["USER:", "ASSISTANT:"].
+            **kwargs: Additional keyword arguments to be passed to the model.
+
+        Returns:
+            str: The generated response based on the prompt.
+        """
+        prompt_final = f"USER:\n{prompt}\nASSISTANT:\n"
+
+        result = self.create_completion(
+            prompt_final,
+            repeat_penalty=self.generation_kwargs["repetition_penalty"],
+            max_tokens=self.generation_kwargs["max_new_tokens"],
+            stop=stop,
+            echo=False,
+            temperature=self.generation_kwargs["temperature"],
+            mirostat_mode = 2,
+            mirostat_tau=4.0,
+            mirostat_eta=1.1
+        )
+        resp = result['choices'][0]['text']
+        # history.append(
+        #     [prompt, resp]
+        # )
+        return resp
+
+    def stream(
+        self,
+        prompt: str,
+        stop: list[str] | None = ["USER:", "ASSISTANT:"],
+        # history: list = [],
+        **kwargs: t.Any,
+    ):
+        """Generate text responses based on the given prompt using the model.
+
+        Args:
+            prompt (str): The prompt to generate text responses.
+            stop (list[str], optional): List of strings to stop the generation. Defaults to ["USER:", "ASSISTANT:"].
+            **kwargs: Additional keyword arguments for the model.
+
+        Yields:
+            str: Text responses generated by the model based on the prompt.
+        """
+        prompt = f"USER:\n{prompt}\nASSISTANT:\n"
+        output = self.create_completion(
+            prompt,
+            stream=True,
+            repeat_penalty=self.generation_kwargs["repetition_penalty"],
+            max_tokens=self.generation_kwargs["max_new_tokens"],
+            stop=stop,
+            echo=False,
+            temperature=self.generation_kwargs["temperature"],
+            mirostat_mode = 2,
+            mirostat_tau=4.0,
+            mirostat_eta=1.1
+        )
+        # history.append([])
+        for chunk in output:
+            item = chunk['choices'][0]['text']
+            # self.resps[-1].append(item)
+            yield chunk['choices'][0]['text']
+        # self.resps[-1] = "".join(self.resps[-1])

{hjxdl-0.1.38.dist-info → hjxdl-0.1.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.1.38
+Version: 0.1.40
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.1.38.dist-info → hjxdl-0.1.40.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=_2BzHyrUcTFR5ix_ca-uPHbXU_rcLiE194BnEhwOnRU,413
+hdl/_version.py,sha256=3IhTfHIu7NeAPXyE10KzRtPGq9MVXGPwZ6ggQQd3ZPs,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -127,14 +127,15 @@ hdl/utils/desc/template.py,sha256=a3NcSihzZMm9Bk76iDVe54_xBDceGmLebS0XMONE3nk,11
 hdl/utils/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/general/glob.py,sha256=8-RCnt6L297wMIfn34ZAMCsGCZUjHG3MGglGZI1cX0g,491
 hdl/utils/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdl/utils/llm/chat.py,sha256=6ZlIhmnTc6FrJEhDfzMVsILFmpHWzNMtrmDrrPoMJYg,13700
+hdl/utils/llm/chat.py,sha256=esvDM5CPOMVexmQAcjgRf3G_2Dn0fHVfb5eCzktewno,9969
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
+hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.1.38.dist-info/METADATA,sha256=esqj1qYfOApxJmmvuZouXa8ylxI9v4PDPEkH5aS8HYE,737
-hjxdl-0.1.38.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-hjxdl-0.1.38.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.1.38.dist-info/RECORD,,
+hjxdl-0.1.40.dist-info/METADATA,sha256=mgTKZ6PKpPjlDHrrFVdwbLnaxc-LRoNhwJykiT3aQpg,737
+hjxdl-0.1.40.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+hjxdl-0.1.40.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.1.40.dist-info/RECORD,,