hjxdl 0.2.24__py3-none-any.whl → 0.2.26__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.24'
-__version_tuple__ = version_tuple = (0, 2, 24)
+__version__ = version = '0.2.26'
+__version_tuple__ = version_tuple = (0, 2, 26)

hdl/utils/llm/vis.py

@@ -4,6 +4,7 @@ import base64
 from io import BytesIO
 import requests
 import uuid
+import hashlib
 
 import torch
 import numpy as np
@@ -132,8 +133,6 @@ class ImgHandler:
     def __init__(
         self,
         model_path,
-        db_host,
-        db_port,
         conn=None,
         model_name: str = None,
         device: str = "cpu",
@@ -159,9 +158,7 @@ class ImgHandler:
         self.model_path = model_path
         self.model_name = model_name
 
-        self.db_host = db_host
-        self.db_port = db_port
-        self._db_conn = None
+        self.db_conn = conn
         self.num_vec_dim = num_vec_dim
         self.pic_idx_name = "idx:pic_idx"
         if load_model:
@@ -204,17 +201,6 @@ class ImgHandler:
             HF_HUB_PREFIX + self.model_path
         )
 
-    @property
-    def db_conn(self):
-        """Establishes a connection to Redis server if not already connected.
-
-            Returns:
-                Redis connection object: A connection to the Redis server.
-        """
-        if self._db_conn is None:
-            self._db_conn = conn_redis(self.db_host, self.db_port)
-        return self._db_conn
-
     def get_img_features(
         self,
         images,
@@ -353,7 +339,7 @@ class ImgHandler:
         conn=None,
         print_idx_info: bool = False,
     ):
-        """Save image features to a Redis database.
+        """Save image features to a Redis database, avoiding duplicates.
 
         Args:
             images (list): A list of image file paths.
@@ -370,6 +356,7 @@ class ImgHandler:
         if conn is None:
             conn = self.db_conn
         pipeline = conn.pipeline()
+
         for img_file, emb in tqdm(zip(sorted_imgs, img_feats)):
             if img_file.startswith("data:"):
                 img_data = img_file
@@ -378,13 +365,26 @@ class ImgHandler:
                 img_data = imgfile_to_base64(img_file)
                 img_idx = f"pic-{img_file}"
 
+            # 使用图片特征生成唯一哈希值
+            emb_hash = hashlib.sha256(emb.tobytes()).hexdigest()
+
+            # 检查该哈希值是否已存在，避免重复存储
+            if conn.exists(f"pic-hash-{emb_hash}"):
+                print(f"Image {img_file} already exists, skipping.")
+                continue
+
+            # 存储新图片的特征和数据
             emb = emb.astype(np.float32).tolist()
             emb_json = {
                 "emb": emb,
                 "data": img_data
             }
             pipeline.json().set(img_idx, "$", emb_json)
-            res = pipeline.execute()
+
+            # 将哈希值作为键存储，以便后续检查
+            pipeline.set(f"pic-hash-{emb_hash}", img_idx)
+
+        res = pipeline.execute()
 
         # 定义向量索引的schema
         schema = (

{hjxdl-0.2.24.dist-info → hjxdl-0.2.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.2.24
+Version: 0.2.26
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.2.24.dist-info → hjxdl-0.2.26.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=60O-fkBLileJecASMvt-pFLFn9U7Uv1i95gYM4vcTpM,413
+hdl/_version.py,sha256=-Bz-F1hr7wbrrnyo6EqAHpJHRnGCyOCHoNlwDcFCT1M,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -132,13 +132,13 @@ hdl/utils/llm/chatgr.py,sha256=GO2G7g6YybduA5VCUuGjvEsJfC_6L7rycSnPeHMcxyM,2820
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
-hdl/utils/llm/vis.py,sha256=QT1rr49EXyAR9iH1j4_eyiDMLRQSwpJOFaIBGycrOhA,16293
+hdl/utils/llm/vis.py,sha256=M2Xk-2LnOcRge9Aej7ejGZzzLTV82PfI_wo9VX9h-3E,16381
 hdl/utils/llm/visrag.py,sha256=_PuKtmQIXD5bnmXwDWhTLdzOhgC42JiqdMNb1uKA7n8,9190
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.2.24.dist-info/METADATA,sha256=ekKkJLBvfpN2pDDTrQmQRCaWud-UsGsn0FCrC6fzUMo,836
-hjxdl-0.2.24.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-hjxdl-0.2.24.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.2.24.dist-info/RECORD,,
+hjxdl-0.2.26.dist-info/METADATA,sha256=tHMyD2XRvWwLO7NlyzBujX2Bjdx1lnQRxxVWnPwtR_w,836
+hjxdl-0.2.26.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+hjxdl-0.2.26.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.2.26.dist-info/RECORD,,