hjxdl 0.2.29__py3-none-any.whl → 0.2.31__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.29'
-__version_tuple__ = version_tuple = (0, 2, 29)
+__version__ = version = '0.2.31'
+__version_tuple__ = version_tuple = (0, 2, 31)

hdl/utils/llm/chatgr.py

@@ -4,6 +4,19 @@ from .chat import OpenAI_M
 
 # 定义流式输出的生成函数
 def chat_with_llm(user_input, chat_history=[]):
+    """
+    Facilitates a chat interaction with a language model (LLM).
+    This function takes user input and maintains a chat history. It streams the response from the LLM and updates the chat history in real-time.
+    Args:
+        user_input (str): The input message from the user.
+        chat_history (list, optional): A list of tuples representing the chat history. Each tuple contains two strings: the user's message and the bot's response. Defaults to an empty list.
+    Yields:
+        tuple: A tuple containing three elements:
+            - An empty string (for compatibility with certain frameworks).
+            - The updated chat history including the latest user message and the bot's response.
+            - The same updated chat history.
+    """
+
     chat_history.append(("User: " + user_input, "Bot: "))  # 初始先追加用户消息
     yield "", chat_history, chat_history  # 返回用户消息
 

hdl/utils/llm/vis.py

@@ -10,8 +10,10 @@ import torch
 import numpy as np
 from PIL import Image
 # from transformers import ChineseCLIPProcessor, ChineseCLIPModel
+from transformers import AutoModel
+from transformers import AutoTokenizer
 import open_clip
-import natsort
+# import natsort
 from redis.commands.search.field import VectorField
 from redis.commands.search.indexDefinition import IndexDefinition, IndexType
 from hdl.jupyfuncs.show.pbar import tqdm
@@ -96,7 +98,7 @@ def imgbase64_to_pilimg(img_base64: str):
         BytesIO(
             base64.b64decode(img_base64.split(",")[-1])
         )
-    )
+    ).convert('RGB')
     return img_pil
 
 
@@ -119,34 +121,72 @@ def pilimg_to_base64(pilimg):
 
 
 class ImgHandler:
+    """
+    ImgHandler is a class for handling image processing tasks using pretrained models.
+    Attributes:
+        device_str (str): The device string (e.g., "cpu" or "cuda").
+        device (torch.device): The device to run the model on.
+        model_path (str): The path to the pretrained model.
+        model_name (str): The name of the model.
+        model_type (str): The type of the model (e.g., "openclip" or "cpm").
+        db_conn: The database connection object.
+        num_vec_dim (int): The number of vector dimensions.
+        pic_idx_name (str): The name of the picture index.
+        open_clip_cfg (dict): The configuration for the OpenCLIP model.
+        model: The pretrained model.
+        preprocess_train: The preprocessing function for training data.
+        preprocess_val: The preprocessing function for validation data.
+        tokenizer: The tokenizer for the model.
+    Methods:
+        __init__(self, model_path, conn=None, model_name: str = None, model_type: str = "openclip", device: str = "cpu", num_vec_dim: int = None, load_model: bool = True) -> None:
+            Initializes the ImgHandler class with the specified parameters.
+        load_model(self):
+            Loads the pretrained model and related configurations.
+        get_img_features(self, images, to_numpy=False, **kwargs):
+            Gets image features using a pretrained model.
+        get_text_features(self, texts, to_numpy=False, **kwargs):
+            Gets text features from the input texts.
+        get_text_img_probs(self, texts, images, probs=False, to_numpy=False, **kwargs):
+            Gets the probabilities of text-image associations.
+        get_pics_sims(self, images1, images2, to_numpy=False, **kwargs):
+            Calculates similarity scores between two sets of images.
+        vec_pics_todb(self, images: list[str], conn=None, print_idx_info=False):
+            Saves image features to a Redis database, avoiding duplicates.
+        get_pic_idx_info(self, conn=None):
+            Gets information about the picture index in the Redis database.
+        emb_search(self, emb_query, num_max=3, extra_params=None, conn=None):
+            Searches for similar embeddings in the database.
+        img_search(self, img, num_max=3, extra_params=None, conn=None):
+            Searches for similar images in the database based on the input image.
+        """
     def __init__(
         self,
         model_path,
         conn=None,
         model_name: str = None,
+        model_type: str = "openclip",
         device: str = "cpu",
         num_vec_dim: int = None,
         load_model: bool = True,
     ) -> None:
-        """Initializes the class with the provided parameters.
-
+        """
+        Initializes the visualization utility.
         Args:
-            model_path (str): Path to the model file.
-            db_host (str): Hostname of the database.
-            db_port (int): Port number of the database.
+            model_path (str): Path to the model.
+            conn (optional): Database connection object. Defaults to None.
             model_name (str, optional): Name of the model. Defaults to None.
+            model_type (str, optional): Type of the model. Defaults to "openclip".
             device (str, optional): Device to run the model on. Defaults to "cpu".
             num_vec_dim (int, optional): Number of vector dimensions. Defaults to None.
-            load_model (bool, optional): Whether to load the model. Defaults to True.
-
+            load_model (bool, optional): Flag to load the model immediately. Defaults to True.
         Returns:
             None
         """
-
         self.device_str = device
         self.device = torch.device(device)
         self.model_path = model_path
         self.model_name = model_name
+        self.model_type = model_type
 
         self.db_conn = conn
         self.num_vec_dim = num_vec_dim
@@ -155,41 +195,72 @@ class ImgHandler:
             self.load_model()
 
     def load_model(self):
-        """Load the OpenCLIP model and related configurations.
-
-        This function loads the OpenCLIP model from the specified model path
-        and initializes the necessary components such as the model,
-        preprocessors for training and validation data, tokenizer, etc.
-
-        Returns:
-            None
         """
-        ckpt_file = (
-            Path(self.model_path) / Path("open_clip_pytorch_model.bin")
-        ).as_posix()
-        self.open_clip_cfg = json.load(
-            open(Path(self.model_path) / Path("open_clip_config.json"))
-        )
+        Loads the model and tokenizer based on the specified model type.
+        This method supports loading two types of models: "cpm" and "openclip".
+        For "cpm":
+            - Loads the tokenizer and model using `AutoTokenizer` and `AutoModel` from the Hugging Face library.
+            - Sets the model to the specified device.
+            - Sets the number of vector dimensions to 2304.
+        For "openclip":
+            - Loads the model checkpoint and configuration from the specified path.
+            - Sets the model name if not already specified.
+            - Creates the model and preprocessing transforms using `open_clip.create_model_and_transforms`.
+            - Sets the number of vector dimensions based on the configuration if not already specified.
+            - Loads the tokenizer using `open_clip.get_tokenizer`.
+        Attributes:
+            model_type (str): The type of the model to load ("cpm" or "openclip").
+            model_path (str): The path to the model files.
+            device (str): The device to load the model onto (e.g., "cpu" or "cuda").
+            model_name (str, optional): The name of the model (used for "openclip" type).
+            num_vec_dim (int, optional): The number of vector dimensions (used for "openclip" type).
+            tokenizer: The tokenizer for the model.
+            model: The loaded model.
+            preprocess_train: The preprocessing transform for training (used for "openclip" type).
+            preprocess_val: The preprocessing transform for validation (used for "openclip" type).
+            open_clip_cfg (dict): The configuration for the "openclip" model.
+        """
+
 
-        if self.model_name is None:
-            self.model_name = (
-                self.open_clip_cfg['model_cfg']['text_cfg']['hf_tokenizer_name']
-                .split('/')[-1]
+        if self.model_type == "cpm":
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                self.model_path,
+                trust_remote_code=True
+            )
+            self.model = AutoModel.from_pretrained(
+                self.model_path,
+                trust_remote_code=True
             )
+            self.model.to(self.device)
+            self.num_vec_dim = 2304
+
+        elif self.model_type == "openclip":
+            ckpt_file = (
+                Path(self.model_path) / Path("open_clip_pytorch_model.bin")
+            ).as_posix()
+            self.open_clip_cfg = json.load(
+                open(Path(self.model_path) / Path("open_clip_config.json"))
+            )
+
+            if self.model_name is None:
+                self.model_name = (
+                    self.open_clip_cfg['model_cfg']['text_cfg']['hf_tokenizer_name']
+                    .split('/')[-1]
+                )
 
-        self.model, self.preprocess_train, self.preprocess_val = (
-            open_clip.create_model_and_transforms(
-                model_name=self.model_name,
-                pretrained=ckpt_file,
-                device=self.device,
-                # precision=precision
+            self.model, self.preprocess_train, self.preprocess_val = (
+                open_clip.create_model_and_transforms(
+                    model_name=self.model_name,
+                    pretrained=ckpt_file,
+                    device=self.device,
+                    # precision=precision
+                )
+            )
+            if self.num_vec_dim is None:
+                self.num_vec_dim = self.open_clip_cfg["model_cfg"]["embed_dim"]
+            self.tokenizer = open_clip.get_tokenizer(
+                HF_HUB_PREFIX + self.model_path
             )
-        )
-        if self.num_vec_dim is None:
-            self.num_vec_dim = self.open_clip_cfg["model_cfg"]["embed_dim"]
-        self.tokenizer = open_clip.get_tokenizer(
-            HF_HUB_PREFIX + self.model_path
-        )
 
     def get_img_features(
         self,
@@ -222,16 +293,24 @@ class ImgHandler:
                     f"Not supported image type for {type(img)}"
                 )
 
-
-        with torch.no_grad(), torch.amp.autocast(self.device_str):
-            imgs = torch.stack([
-                self.preprocess_val(image).to(self.device)
-                for image in images_fixed
-            ])
-            img_features = self.model.encode_image(imgs, **kwargs)
-            img_features /= img_features.norm(dim=-1, keepdim=True)
-            if to_numpy:
-                img_features = img_features.cpu().numpy()
+        if self.model_type == "cpm":
+            with torch.no_grad():
+                img_features = self.model(
+                    text=[""] * len(images_fixed),
+                    image=images_fixed,
+                    tokenizer=self.tokenizer
+                ).reps
+
+        if self.model_type == "openclip":
+            with torch.no_grad(), torch.amp.autocast(self.device_str):
+                imgs = torch.stack([
+                    self.preprocess_val(image).to(self.device)
+                    for image in images_fixed
+                ])
+                img_features = self.model.encode_image(imgs, **kwargs)
+                img_features /= img_features.norm(dim=-1, keepdim=True)
+        if to_numpy:
+            img_features = img_features.cpu().numpy()
         return img_features
 
     def get_text_features(
@@ -253,17 +332,25 @@ class ImgHandler:
         Example:
             get_text_features(["text1", "text2"], to_numpy=True)
         """
-        with torch.no_grad(), torch.amp.autocast(self.device_str):
-            txts = self.tokenizer(
-                texts,
-                context_length=self.model.context_length
-            ).to(self.device)
-            txt_features = self.model.encode_text(txts, **kwargs)
-            txt_features /= txt_features.norm(dim=-1, keepdim=True)
-            if to_numpy:
-                txt_features = txt_features.cpu().numpy()
-        return txt_features
 
+        if self.model_type == "cpm":
+            with torch.no_grad():
+                txt_features = self.model(
+                    text=texts,
+                    image=[None] * len(texts),
+                    tokenizer=self.tokenizer
+                ).reps
+        elif self.model_type == "openclip":
+            with torch.no_grad(), torch.amp.autocast(self.device_str):
+                txts = self.tokenizer(
+                    texts,
+                    context_length=self.model.context_length
+                ).to(self.device)
+                txt_features = self.model.encode_text(txts, **kwargs)
+                txt_features /= txt_features.norm(dim=-1, keepdim=True)
+        if to_numpy:
+            txt_features = txt_features.cpu().numpy()
+        return txt_features
 
     def get_text_img_probs(
         self,

{hjxdl-0.2.29.dist-info → hjxdl-0.2.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.2.29
+Version: 0.2.31
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.2.29.dist-info → hjxdl-0.2.31.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=AtwvoTC96AXOg97Emp1_Wmo7L-xUzOR7aFTUto-9JfA,413
+hdl/_version.py,sha256=DftfWlt0Q-ezfk5YMbK9lW9QTADqDhJKW_mOmhx7dwY,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -128,17 +128,17 @@ hdl/utils/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 hdl/utils/general/glob.py,sha256=8-RCnt6L297wMIfn34ZAMCsGCZUjHG3MGglGZI1cX0g,491
 hdl/utils/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/llm/chat.py,sha256=OzyY9xACOOocx9zZigtq9YAPvHtDUo8v2fvf1Tyjg_U,14891
-hdl/utils/llm/chatgr.py,sha256=GO2G7g6YybduA5VCUuGjvEsJfC_6L7rycSnPeHMcxyM,2820
+hdl/utils/llm/chatgr.py,sha256=8L7RLpi3tU_9HfP1qSiqH1BQDhBilSEj6Rn93lZOdDc,3584
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
-hdl/utils/llm/vis.py,sha256=Kixrhc3eByHjdxiAcB2BnthsS31dHMavp6qc2JX46Dc,16292
+hdl/utils/llm/vis.py,sha256=BsGAfy5X8sMFnX5A3vHpTPDRe_-IDdhs6YVQ-efvyQ0,21424
 hdl/utils/llm/visrag.py,sha256=_PuKtmQIXD5bnmXwDWhTLdzOhgC42JiqdMNb1uKA7n8,9190
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.2.29.dist-info/METADATA,sha256=7qiDTPY06pouX5FgbV4ekWvdDdXKae-Q9m1iGrIO1GA,836
-hjxdl-0.2.29.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-hjxdl-0.2.29.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.2.29.dist-info/RECORD,,
+hjxdl-0.2.31.dist-info/METADATA,sha256=T9COT4TfRlNcqpIOdr4tbU2bC_JhNIbRpbOW9855RKo,836
+hjxdl-0.2.31.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+hjxdl-0.2.31.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.2.31.dist-info/RECORD,,

{hjxdl-0.2.29.dist-info → hjxdl-0.2.31.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.2.0)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any