hjxdl 0.2.23__py3-none-any.whl → 0.2.25__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.23'
-__version_tuple__ = version_tuple = (0, 2, 23)
+__version__ = version = '0.2.25'
+__version_tuple__ = version_tuple = (0, 2, 25)

hdl/utils/llm/vis.py

@@ -4,6 +4,7 @@ import base64
 from io import BytesIO
 import requests
 import uuid
+import hashlib
 
 import torch
 import numpy as np
@@ -132,8 +133,8 @@ class ImgHandler:
     def __init__(
         self,
         model_path,
-        db_host,
-        db_port,
+        db_host = "127.0.0.1",
+        db_port = 8888,
         conn=None,
         model_name: str = None,
         device: str = "cpu",
@@ -353,7 +354,7 @@ class ImgHandler:
         conn=None,
         print_idx_info: bool = False,
     ):
-        """Save image features to a Redis database.
+        """Save image features to a Redis database, avoiding duplicates.
 
         Args:
             images (list): A list of image file paths.
@@ -370,10 +371,8 @@ class ImgHandler:
         if conn is None:
             conn = self.db_conn
         pipeline = conn.pipeline()
-        for img_file, emb in tqdm(zip(sorted_imgs, img_feats)):
-            # 初始化 Redis，先使用 img 文件名作为 Key 和 Value，后续再更新为图片特征向量
-            # pipeline.json().set(img_file, "$", img_file)
 
+        for img_file, emb in tqdm(zip(sorted_imgs, img_feats)):
             if img_file.startswith("data:"):
                 img_data = img_file
                 img_idx = f"pic-{str(uuid.uuid4())}"
@@ -381,39 +380,58 @@ class ImgHandler:
                 img_data = imgfile_to_base64(img_file)
                 img_idx = f"pic-{img_file}"
 
+            # 使用图片特征生成唯一哈希值
+            emb_hash = hashlib.sha256(emb.tobytes()).hexdigest()
+
+            # 检查该哈希值是否已存在，避免重复存储
+            if conn.exists(f"pic-hash-{emb_hash}"):
+                print(f"Image {img_file} already exists, skipping.")
+                continue
+
+            # 存储新图片的特征和数据
             emb = emb.astype(np.float32).tolist()
             emb_json = {
                 "emb": emb,
                 "data": img_data
             }
             pipeline.json().set(img_idx, "$", emb_json)
-            res = pipeline.execute()
-            # print('redis set:', res)
 
+            # 将哈希值作为键存储，以便后续检查
+            pipeline.set(f"pic-hash-{emb_hash}", img_idx)
+
+        res = pipeline.execute()
+
+        # 定义向量索引的schema
         schema = (
             VectorField(
-                "$.emb",  # 这是 JSON 中存储向量的路径
-                "FLAT",  # 使用 FLAT 索引类型
+                "$.emb",
+                "FLAT",
                 {
-                    "TYPE": "FLOAT32",  # 向量类型
-                    "DIM": self.num_vec_dim,  # 向量维度，必须与实际数据的维度一致
-                    "DISTANCE_METRIC": "COSINE",  # 余弦相似度作为距离度量
+                    "TYPE": "FLOAT32",
+                    "DIM": self.num_vec_dim,
+                    "DISTANCE_METRIC": "COSINE",
                 },
-                as_name="vector",  # 给这个字段定义一个别名，后续可以使用
+                as_name="vector",
             ),
         )
-        # vector_idx_name = "idx:pic_idx"
+        # 定义索引的配置
         definition = IndexDefinition(
             prefix=["pic-"],
             index_type=IndexType.JSON
         )
-        res = conn.ft(
-            self.pic_idx_name
-        ).create_index(
-            fields=schema,
-            definition=definition
-        )
-        print("create_index:", res)
+
+        # 检查索引是否已经存在
+        try:
+            conn.ft(self.pic_idx_name).info()  # 检查索引信息
+            print("Index already exists, skipping creation.")
+        except Exception:
+            # 如果索引不存在，创建新的索引
+            res = conn.ft(self.pic_idx_name).create_index(
+                fields=schema,
+                definition=definition
+            )
+            print("create_index:", res)
+
         if print_idx_info:
             print(self.pic_idx_info)
 

{hjxdl-0.2.23.dist-info → hjxdl-0.2.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.2.23
+Version: 0.2.25
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.2.23.dist-info → hjxdl-0.2.25.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=-hljAU21MXFeuL9fV1MtjsvUKcUWxJn4DlS0PdRjDcs,413
+hdl/_version.py,sha256=UmsAAHFuqYBzGCBQpGNQPt7zCgBErX1ViQbYB7gdPPg,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -132,13 +132,13 @@ hdl/utils/llm/chatgr.py,sha256=GO2G7g6YybduA5VCUuGjvEsJfC_6L7rycSnPeHMcxyM,2820
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
-hdl/utils/llm/vis.py,sha256=aeTVX-jKXJBnvQXaOWiHpI9Nc7IneSCqm_g7N4spJUc,16446
+hdl/utils/llm/vis.py,sha256=skeoBgFNes8swPBeWIsn3oKZ4Jv4Cl5GGrxsRDNupac,16854
 hdl/utils/llm/visrag.py,sha256=_PuKtmQIXD5bnmXwDWhTLdzOhgC42JiqdMNb1uKA7n8,9190
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.2.23.dist-info/METADATA,sha256=xFu32son2_PThYfRcnKAh5-Kc1bUoRgyAhoo9hsHixc,836
-hjxdl-0.2.23.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-hjxdl-0.2.23.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.2.23.dist-info/RECORD,,
+hjxdl-0.2.25.dist-info/METADATA,sha256=i8tbHzSQf9nal7CU2P7Whe3AChbtkBHJPUIIKQaVqQo,836
+hjxdl-0.2.25.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+hjxdl-0.2.25.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.2.25.dist-info/RECORD,,