hjxdl 0.1.51__py3-none-any.whl → 0.1.53__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.51'
-__version_tuple__ = version_tuple = (0, 1, 51)
+__version__ = version = '0.1.53'
+__version_tuple__ = version_tuple = (0, 1, 53)

hdl/jupyfuncs/path/glob.py

@@ -12,6 +12,7 @@ import sys
 import importlib
 import subprocess
 import re
+from pathlib import Path
 
 import multiprocess as mp
 
@@ -21,7 +22,7 @@ import json
 
 def in_jupyter():
     """Check if the code is running in a Jupyter notebook.
-    
+
         Returns:
             bool: True if running in Jupyter notebook, False otherwise.
     """
@@ -32,7 +33,7 @@ def in_jupyter():
 
 def in_docker():
     """Check if the code is running inside a Docker container.
-    
+
         Returns:
             bool: True if running inside a Docker container, False otherwise.
     """
@@ -44,10 +45,10 @@ def get_files(
     file_types: list = ["txt"]
 ):
     """Get a list of files with specific file extensions in the given directory path.
-    
+
     Args:
         dir_path (str): The path to the target directory.
-    
+
     Returns:
         list: A list of absolute file paths that have file extensions such as .md, .doc, .docx, .pdf, .csv, or .txt.
     """
@@ -56,7 +57,7 @@ def get_files(
     for filepath, dirnames, filenames in os.walk(dir_path):
         # os.walk 函数将递归遍历指定文件夹
         filenames = [f for f in filenames if not f[0] == '.']
-        dirnames[:] = [d for d in dirnames if not d[0] == '.'] 
+        dirnames[:] = [d for d in dirnames if not d[0] == '.']
         for filename in filenames:
             # 通过后缀名判断文件类型是否满足要求
             if filename.endswith(file_types):
@@ -67,10 +68,10 @@ def get_files(
 
 def get_dataset_file(filename):
     """Get dataset file.
-    
+
     Args:
         filename (str): The name of the dataset file.
-    
+
     Returns:
         dict: The data loaded from the dataset file.
     """
@@ -82,11 +83,11 @@ def get_dataset_file(filename):
 
 def recursive_glob(treeroot, pattern):
     """Recursively searches for files matching a specified pattern starting from the given directory.
-    
+
     Args:
         treeroot (str): The root directory to start the search from.
         pattern (str): The pattern to match the files against.
-    
+
     Returns:
         list: A list of file paths that match the specified pattern.
     """
@@ -104,9 +105,9 @@ def makedirs(path: str, isfile: bool = False) -> None:
 
 
     Args:
-        path (str): Path to a directory or file. 
+        path (str): Path to a directory or file.
         isfile (bool, optional): Whether the provided path is a directory or file.Defaults to False.
-    """    
+    """
     if isfile:
         path = os.path.dirname(path)
     if path != '':
@@ -120,10 +121,10 @@ def get_current_dir():
 
 def get_num_lines(file):
     """Get the number of lines in a file.
-    
+
     Args:
         file (str): The path to the file.
-    
+
     Returns:
         int: The number of lines in the file.
     """
@@ -143,11 +144,11 @@ def chunkify_file(
     """
     function to divide a large text file into chunks each having size ~= size so that the chunks are line aligned
 
-    Params : 
+    Params :
         fname : path to the file to be chunked
         size : size of each chink is ~> this
         skiplines : number of lines in the begining to skip, -1 means don't skip any lines
-    Returns : 
+    Returns :
         start and end position of chunks in Bytes
     """
     chunks = []
@@ -177,7 +178,7 @@ def parallel_apply_line_by_line_chunk(chunk_data):
     function to apply a function to each line in a chunk
 
     Params :
-        chunk_data : the data for this chunk 
+        chunk_data : the data for this chunk
     Returns :
         list of the non-None results for this chunk
     """
@@ -262,10 +263,10 @@ def parallel_apply_line_by_line(
 
 def get_func_from_dir(score_dir: str) -> t.Tuple[t.Callable, str]:
     """Get function and mode from directory.
-    
+
     Args:
         score_dir (str): The directory path containing the function file.
-        
+
     Returns:
         Tuple[Callable, str]: A tuple containing the main function and the mode.
     """
@@ -277,9 +278,17 @@ def get_func_from_dir(score_dir: str) -> t.Tuple[t.Callable, str]:
         file_name = "main"
 
     sys.path.append(func_dir)
-    module = importlib.import_module(file_name) 
+    module = importlib.import_module(file_name)
     try:
         mode = module.MODE
     except Exception as _:
         mode = 'batch'
-    return module.main, mode 
+    return module.main, mode
+
+
+def find_images_recursive(
+    directory,
+    extensions=(".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff")
+):
+    path = Path(directory)
+    return [str(file) for file in path.rglob("*") if file.suffix.lower() in extensions]

hdl/utils/database_tools/connect.py

@@ -1,16 +1,17 @@
 import psycopg
+import redis
 
 
 def connect_by_infofile(info_file: str) -> psycopg.Connection:
     """Create a postgres connection
 
     Args:
-        info_file (str): 
+        info_file (str):
             the path of the connection info like
             host=127.0.0.1 dbname=dbname port=5432 user=postgres password=lala
 
     Returns:
-        psycopg.Connection: 
+        psycopg.Connection:
             the connection instance should be closed after committing.
     """
     conn = psycopg.connect(
@@ -25,4 +26,17 @@ def connect_by_infofile(info_file: str) -> psycopg.Connection:
 #     for record in cur:
 #         print(record)
 #     conn.commit()
-#     conn.close()
+#     conn.close()
+
+def conn_redis(
+    host: str,
+    port: int
+):
+    client = redis.Redis(
+        host=host,
+        port=port,
+        decode_responses=True
+    )
+    res = client.ping()
+    print(res)
+    return client

hdl/utils/llm/vis.py

@@ -0,0 +1,79 @@
+import requests
+
+import torch
+import numpy as np
+from PIL import Image
+import redis
+from transformers import ChineseCLIPProcessor, ChineseCLIPModel
+
+from ..database_tools.connect import conn_redis
+
+
+# url = "https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg"
+# image = Image.open(requests.get(url, stream=True).raw)
+
+
+class ImgHandler:
+    def __init__(
+        self,
+        model_path,
+        redis_host,
+        redis_port,
+    ) -> None:
+        self.model = ChineseCLIPModel.from_pretrained(model_path)
+        self.processor = ChineseCLIPProcessor.from_pretrained(model_path)
+        self.redis_host = redis_host
+        self.redis_port = redis_port
+        self._redis_conn = None
+
+    @property
+    def redis_conn(self):
+        if self._redis_conn is None:
+            self._redis_conn = conn_redis(self.redis_host, self.redis_port)
+        return self._redis_conn
+
+    def get_img_features(self, images, **kwargs):
+        inputs = self.processor(
+            images=images,
+            return_tensors="pt",
+            **kwargs
+        )
+        image_features = self.model.get_image_features(**inputs)
+        image_features = image_features / \
+            image_features.norm(p=2, dim=-1, keepdim=True)
+        return image_features
+
+    def get_text_features(
+        self,
+        texts,
+        **kwargs
+    ):
+        inputs = self.processor(
+            text=texts,
+            padding=True,
+            return_tensors="pt",
+            **kwargs
+        )
+        text_features = self.model.get_text_features(**inputs)
+        text_features = text_features / \
+            text_features.norm(p=2, dim=-1, keepdim=True)
+        return text_features
+
+    def get_text_img_sims(
+        self,
+        texts,
+        images,
+        **kwargs
+    ):
+        inputs = self.processor(
+            text=texts,
+            images=images,
+            return_tensors="pt",
+            padding=True,
+            **kwargs
+        )
+        outputs = self.model(**inputs)
+        logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
+        probs = logits_per_image.softmax(dim=1)
+        return probs
+

{hjxdl-0.1.51.dist-info → hjxdl-0.1.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.1.51
+Version: 0.1.53
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu
@@ -18,6 +18,9 @@ Requires-Dist: geopy
 Requires-Dist: timezonefinder
 Requires-Dist: pytz
 Requires-Dist: duckduckgo-search[lxml]
+Requires-Dist: opencv-python
+Requires-Dist: redis[hiredis]
+Requires-Dist: Pillow
 
 # DL framework by Jianxing
 

{hjxdl-0.1.51.dist-info → hjxdl-0.1.53.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=f9ICF0HnULXyn5fGbc4yAmBNgRL7m8zPXBZHmrIl7ZQ,413
+hdl/_version.py,sha256=bPb2Oc5wemZHZQ36bFj8MSRaDTXfjYMCh-GfQfIz_eE,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -76,7 +76,7 @@ hdl/jupyfuncs/llm/openapi.py,sha256=pNBW0Jzt0JAZP8ZexgoQZVF118jrhKnBYheClcMu9bU,
 hdl/jupyfuncs/network/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/jupyfuncs/network/proxy.py,sha256=foZm3gGFTPLeMNRfWs4QKNUEmlhtNTr_1GQvn0IgDVw,545
 hdl/jupyfuncs/path/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdl/jupyfuncs/path/glob.py,sha256=fl0YDLDS9QI2WgDlBzDGlALbvkmPGcXp4UnbbQ17BOM,8300
+hdl/jupyfuncs/path/glob.py,sha256=W4sxFmCLgUUfvYJR7axlEKfwOohDS0fEggLQoTI2Xi0,8491
 hdl/jupyfuncs/path/strings.py,sha256=eZCXElh7pT0xwy6ZBqSu3frq3Xx8CN5TMuQzsxb0Sbw,2009
 hdl/jupyfuncs/show/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/jupyfuncs/show/pbar.py,sha256=QzyHV9XEyk9U5oyonxLSwYb5pD09iVUw_atVlzxYBNQ,1005
@@ -118,7 +118,7 @@ hdl/utils/chemical_tools/__init__.py,sha256=_QRNtVx0ieZZgSEsHndPFKm6XU3WXfRb7GYq
 hdl/utils/chemical_tools/query_info.py,sha256=wyQXwKSY_gBGVUNvYggHpYBtOLAtpYKq3PN5wqDb7Co,4204
 hdl/utils/chemical_tools/sdf.py,sha256=71PEqU0H885L6IeGHEa6n7ZLZThvMsZOVLuFG2wnoyM,542
 hdl/utils/database_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdl/utils/database_tools/connect.py,sha256=KUnVG-8raifEJ_N0b3c8LkTTIfn9NIyw8LX6qvpA3YU,723
+hdl/utils/database_tools/connect.py,sha256=1CKK1VbgbvYMlA6iBSPpQM6bDt0fxwKwC7cxoppIHoU,940
 hdl/utils/database_tools/datetime.py,sha256=xqE2xNiOpADzX-R8_bM0bioJRF3Ay9Jp1CAG6dy6uVI,1202
 hdl/utils/database_tools/web.py,sha256=vZJYWA02QbyUUMGLDX710cexn7RlfWeHa8YwK3UsDZ0,1934
 hdl/utils/desc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -131,11 +131,12 @@ hdl/utils/llm/chat.py,sha256=sk7Lw5Oa30k-l2fnJknkMmTc5zkBeEKsR981aeFhH5s,11907
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
+hdl/utils/llm/vis.py,sha256=J7wSvizcsAOdGi6MyzzyQ1K5iINYhCXShDw1yG7QriI,2158
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.1.51.dist-info/METADATA,sha256=syqwDtpatUKLAZTxw0RqV4GFDHAqP-Hh840LZtLgfZE,737
-hjxdl-0.1.51.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-hjxdl-0.1.51.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.1.51.dist-info/RECORD,,
+hjxdl-0.1.53.dist-info/METADATA,sha256=uGSeHnUI-OHFWnechzweGcSlxeRTEWACTLZObCHQrI8,818
+hjxdl-0.1.53.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+hjxdl-0.1.53.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.1.53.dist-info/RECORD,,