hjxdl 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.2'
-__version_tuple__ = version_tuple = (0, 0, 2)
+__version__ = version = '0.0.3'
+__version_tuple__ = version_tuple = (0, 0, 3)

hdl/utils/llm/chat.py

@@ -0,0 +1,164 @@
+import typing as t
+
+from llama_cpp import Llama
+from jupyfuncs.llm.openapi import (
+    chat_oai_invoke,
+    chat_oai_stream
+)
+
+
+class GGUF_M():
+    def __init__(
+        self,
+        model_path :str,
+        device: str='gpu',
+        generation_kwargs: dict = {},
+        server_ip: str = "127.0.0.1",
+        server_port: int = 8000,
+    ):
+        """Initialize the model from a local path.
+        Here it does not manage chat histories.
+        
+        Args:
+            model_path (str): The path to the model.
+            device (str, optional): The device to use for model initialization. Defaults to 'gpu'.
+            generation_kwargs (dict, optional): Additional keyword arguments for model generation. Defaults to {}.
+        """
+        # 从本地初始化模型
+        super().__init__()
+        self.generation_kwargs = generation_kwargs
+        self.questions = []
+        self.resps = []
+        print("正在从本地加载模型...")
+        if device == 'cpu':
+            self.model = Llama(
+                model_path=model_path,
+                n_threads=self.generation_kwargs['num_threads'],
+                n_ctx=self.generation_kwargs['max_context_length'],
+            )
+        else:
+            self.model = Llama(
+                model_path=model_path,
+                n_threads=self.generation_kwargs['num_threads'],
+                n_ctx=self.generation_kwargs['max_context_length'],
+                n_gpu_layers=-1
+            )
+            
+        print("完成本地模型的加载")
+
+    def invoke(
+        self, 
+        prompt : str,
+        stop: list[str] | None = ["USER:", "ASSISTANT:"],
+        # history: list = [],
+        **kwargs: t.Any,
+    ) -> str:
+        """Invoke the model to generate a response based on the given prompt.
+        
+        Args:
+            prompt (str): The prompt to be used for generating the response.
+            stop (list[str], optional): List of strings that indicate when the model should stop generating the response. Defaults to ["USER:", "ASSISTANT:"].
+            **kwargs: Additional keyword arguments to be passed to the model.
+        
+        Returns:
+            str: The generated response based on the prompt.
+        """
+        prompt_final = f"USER:\n{prompt}\nASSISTANT:\n" 
+
+        result = self.model.create_completion(
+            prompt_final,
+            repeat_penalty=self.generation_kwargs["repetition_penalty"],
+            max_tokens=self.generation_kwargs["max_new_tokens"],
+            stop=stop,
+            echo=False,
+            temperature=self.generation_kwargs["temperature"],
+            mirostat_mode = 2,
+            mirostat_tau=4.0,
+            mirostat_eta=1.1
+        )
+        resp = result['choices'][0]['text']
+        # history.append(
+        #     [prompt, resp]
+        # )
+        return resp
+ 
+    def stream(
+        self,
+        prompt: str,
+        stop: list[str] | None = ["USER:", "ASSISTANT:"],
+        # history: list = [],
+        **kwargs: t.Any,
+    ):
+        """Generate text responses based on the given prompt using the model.
+        
+        Args:
+            prompt (str): The prompt to generate text responses.
+            stop (list[str], optional): List of strings to stop the generation. Defaults to ["USER:", "ASSISTANT:"].
+            **kwargs: Additional keyword arguments for the model.
+        
+        Yields:
+            str: Text responses generated by the model based on the prompt.
+        """
+        self.questions.append(prompt)
+        prompt = f"USER:\n{prompt}\nASSISTANT:\n" 
+        output = self.model.create_completion(
+            prompt,
+            stream=True,
+            repeat_penalty=self.generation_kwargs["repetition_penalty"], 
+            max_tokens=self.generation_kwargs["max_new_tokens"],
+            stop=stop,
+            echo=False,
+            temperature=self.generation_kwargs["temperature"],
+            mirostat_mode = 2,
+            mirostat_tau=4.0,
+            mirostat_eta=1.1
+        )
+        # history.append([])
+        for chunk in output:
+            item = chunk['choices'][0]['text']
+            # self.resps[-1].append(item)
+            yield chunk['choices'][0]['text']
+        # self.resps[-1] = "".join(self.resps[-1])
+
+
+class OpenAI_M():
+    def __init__(
+        self,
+        model_path: str = None,
+        device: str='gpu',
+        generation_kwargs: dict = {},
+        server_ip: str = "172.28.1.2",
+        server_port: int = 8000,
+    ):
+        self.model_path = model_path
+        self.server_ip = server_ip
+        self.server_port = server_port
+        self.base_url = "http://{self.server_ip}:{str(self.server_port)}/v1"
+    
+    def invoke(
+        self, 
+        prompt : str,
+        stop: list[str] | None = ["USER:", "ASSISTANT:"],
+        # history: list = [],
+        **kwargs: t.Any,
+    ) -> str:
+        resp = chat_oai_invoke(
+            base_url=self.base_url,
+            model=self.model_path,
+            prompt=prompt
+        )
+        return resp
+    
+    def stream(
+        self, 
+        prompt : str,
+        stop: list[str] | None = ["USER:", "ASSISTANT:"],
+        # history: list = [],
+        **kwargs: t.Any,
+    ) -> str:
+        resp = chat_oai_stream(
+            base_url=self.base_url,
+            model=self.model_path,
+            prompt=prompt
+        )
+        return resp

{hjxdl-0.0.2.dist-info → hjxdl-0.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.0.2
+Version: 0.0.3
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.0.2.dist-info → hjxdl-0.0.3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdl/_version.py,sha256=NDHlyIcJZjLz8wKlmD1-pr6me5FHBAYwO_ynLG-37N8,411
+hdl/_version.py,sha256=hB095avW4HuDZxn8qPHRG1UMzSSonb8ZDAsLxt9hmk8,411
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -83,9 +83,11 @@ hdl/utils/database_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 hdl/utils/database_tools/connect.py,sha256=KUnVG-8raifEJ_N0b3c8LkTTIfn9NIyw8LX6qvpA3YU,723
 hdl/utils/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/general/glob.py,sha256=8-RCnt6L297wMIfn34ZAMCsGCZUjHG3MGglGZI1cX0g,491
+hdl/utils/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hdl/utils/llm/chat.py,sha256=SoODp3Sf6FNYdajoI39swvPJm64fSd6bFYFdta4xD28,5344
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
-hjxdl-0.0.2.dist-info/METADATA,sha256=_ZhNiflTqXZrDBYdt5v1uG6REZc3wRpmW_76BjYgvP0,525
-hjxdl-0.0.2.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-hjxdl-0.0.2.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.0.2.dist-info/RECORD,,
+hjxdl-0.0.3.dist-info/METADATA,sha256=IMDR9eTW1PeEGxs6eZESV2ys8LsI0j7aZp6qO9eXZWg,525
+hjxdl-0.0.3.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+hjxdl-0.0.3.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.0.3.dist-info/RECORD,,