hjxdl 0.1.57__py3-none-any.whl → 0.1.58__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.57'
-__version_tuple__ = version_tuple = (0, 1, 57)
+__version__ = version = '0.1.58'
+__version_tuple__ = version_tuple = (0, 1, 58)

hdl/utils/llm/vis.py

@@ -1,19 +1,20 @@
-import requests
+from pathlib import Path
+import json
 
 import torch
 import numpy as np
 from PIL import Image
-from transformers import ChineseCLIPProcessor, ChineseCLIPModel
+# from transformers import ChineseCLIPProcessor, ChineseCLIPModel
+import open_clip
 
 from ..database_tools.connect import conn_redis
 
 
-# url = "https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg"
-# image = Image.open(requests.get(url, stream=True).raw)
 __all__ = [
     "ImgHandler"
 ]
 
+HF_HUB_PREFIX = "hf-hub:"
 
 class ImgHandler:
     def __init__(
@@ -29,9 +30,31 @@ class ImgHandler:
                 else torch.device("cpu")
         else:
             self.device = device
+        ckpt_file = (
+            Path(model_path) / Path("open_clip_pytorch_model.bin")
+        ).as_posix()
 
-        self.model = ChineseCLIPModel.from_pretrained(model_path).to(self.device)
-        self.processor = ChineseCLIPProcessor.from_pretrained(model_path)
+        self.open_clip_cfg = json.load(
+            open(Path(model_path) / Path("open_clip_config.json"))
+        )
+        self.model_name = (
+            self.open_clip_cfg['model_cfg']['text_cfg']['hf_tokenizer_name']
+            .split('/')[-1]
+        )
+
+        self.model, self.preprocess_train, self.preprocess_val = (
+            open_clip.create_model_and_transforms(
+                model_name=self.model_name,
+                pretrained=ckpt_file,
+                device=self.device,
+                # precision=precision
+            )
+        )
+        self.tokenizer = open_clip.get_tokenizer(
+            HF_HUB_PREFIX + model_path
+        )
+        # self.model = ChineseCLIPModel.from_pretrained(model_path).to(self.device)
+        # self.processor = ChineseCLIPProcessor.from_pretrained(model_path)
         self.redis_host = redis_host
         self.redis_port = redis_port
         self._redis_conn = None
@@ -43,48 +66,45 @@ class ImgHandler:
             self._redis_conn = conn_redis(self.redis_host, self.redis_port)
         return self._redis_conn
 
-    def get_img_features(self, images, **kwargs):
-        inputs = self.processor(
-            images=images,
-            return_tensors="pt",
-            **kwargs
-        ).to(self.device)
-        image_features = self.model.get_image_features(**inputs)
-        image_features = image_features / \
-            image_features.norm(p=2, dim=-1, keepdim=True)
-        return image_features
+    def get_img_features(
+        self,
+        images,
+        to_numpy = False,
+        **kwargs
+    ):
+        imgs = [
+            self.preprocess_val(Image.open(image)).unsqueeze(0).to(self.device)
+            for image in images
+        ]
+        img_features = self.model.encode_image(imgs, **kwargs)
+        img_features /= img_features.norm(dim=-1, keepdim=True)
+        if to_numpy:
+            img_features = img_features.cpu().numpy()
+        return img_features
 
     def get_text_features(
         self,
         texts,
+        to_numpy = False,
         **kwargs
     ):
-        inputs = self.processor(
-            text=texts,
-            padding=True,
-            return_tensors="pt",
-            **kwargs
-        ).to(self.device)
-        text_features = self.model.get_text_features(**inputs)
-        text_features = text_features / \
-            text_features.norm(p=2, dim=-1, keepdim=True)
-        return text_features
-
-    def get_text_img_sims(
+        txts = self.tokenizer(texts).to(self.device)
+        txt_features = self.model.encode_text(txts, **kwargs)
+        txt_features /= txt_features.norm(dim=-1, keepdim=True)
+        if to_numpy:
+            txt_features = txt_features.cpu().numpy()
+        return txt_features
+
+
+    def get_text_img_probs(
         self,
         texts,
         images,
         **kwargs
     ):
-        inputs = self.processor(
-            text=texts,
-            images=images,
-            return_tensors="pt",
-            padding=True,
-            **kwargs
-        ).to(self.device)
-        outputs = self.model(**inputs)
-        logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
-        probs = logits_per_image.softmax(dim=1)
-        return probs
+        image_features = self.get_img_features(images, **kwargs)
+        text_features = self.get_text_features(texts, **kwargs)
+        text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
+        return text_probs
+
 

{hjxdl-0.1.57.dist-info → hjxdl-0.1.58.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.1.57
+Version: 0.1.58
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu
@@ -22,6 +22,7 @@ Requires-Dist: opencv-python
 Requires-Dist: redis[hiredis]
 Requires-Dist: psycopg[binary]
 Requires-Dist: Pillow
+Requires-Dist: open-clip-torch
 
 # DL framework by Jianxing
 

{hjxdl-0.1.57.dist-info → hjxdl-0.1.58.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=1L2CAEYH8rRBQ01naMSlAdbRLOkKJXOpIY9i96QXS-s,413
+hdl/_version.py,sha256=0coHK1MSWGlBW3NiGGuClrVbkHMpMYKfQkg34r9FVSU,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -131,12 +131,12 @@ hdl/utils/llm/chat.py,sha256=sk7Lw5Oa30k-l2fnJknkMmTc5zkBeEKsR981aeFhH5s,11907
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
-hdl/utils/llm/vis.py,sha256=dzXpv9xtm9qxZSj1zPTIwq2sskzMPsPgh30_LjAcDgU,2480
+hdl/utils/llm/vis.py,sha256=mbtSG76h8PjWCZ4Pp6k5rlfTONM-K8el6f3D8kF0U0c,3071
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.1.57.dist-info/METADATA,sha256=cLXP4zGr3OY_Xcx0mjMLyj6jyRJBH3vp2PxmkmS-hYA,849
-hjxdl-0.1.57.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-hjxdl-0.1.57.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.1.57.dist-info/RECORD,,
+hjxdl-0.1.58.dist-info/METADATA,sha256=NP2vFvS46Yv4PIoXOLassCxLHwq0L3WkuguoY7aiLcQ,880
+hjxdl-0.1.58.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+hjxdl-0.1.58.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.1.58.dist-info/RECORD,,