hjxdl 0.2.82__py3-none-any.whl → 0.2.84__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.82'
-__version_tuple__ = version_tuple = (0, 2, 82)
+__version__ = version = '0.2.84'
+__version_tuple__ = version_tuple = (0, 2, 84)

hdl/utils/desc/template.py

@@ -30,3 +30,14 @@ COT_TEMPLATE = """
 你的回答中应只能是 Markdown 格式，且不能包含其他多余文字或格式错误。
 以下是可用的工具：
 """
+
+OD_TEMPLATE = """
+Detect all the objects in the image, return bounding boxes for all of them using the following format (DO NOT INCLUDE ANY OTHER WORDS IN YOUR ANSWER BUT ONLY THE LIST ITSELF!):
+[
+    {
+        "object": "object_name",
+        "bboxes": [[xmin, ymin, xmax, ymax], [xmin, ymin, xmax, ymax], ...]
+    },
+    ...
+]
+"""

hdl/utils/llm/chat.py

@@ -8,8 +8,9 @@ import re
 
 
 from openai import OpenAI
-from ..desc.template import FN_TEMPLATE, COT_TEMPLATE
+from ..desc.template import FN_TEMPLATE, COT_TEMPLATE, OD_TEMPLATE
 from ..desc.func_desc import TOOL_DESC
+from .vis import draw_and_plot_boxes_from_json
 import json
 # import traceback
 
@@ -132,6 +133,7 @@ class OpenAI_M():
         tools: list = None,
         tool_desc: dict = None,
         cot_desc: str = None,
+        od_desc: str = None,
         *args,
         **kwargs
     ):
@@ -189,9 +191,8 @@ class OpenAI_M():
         self.tool_info = "\n".join(self.tool_descs)
         self.tool_desc_str = "\n".join(self.tool_descs_verbose)
 
-        self.cot_desc = cot_desc
-        if not self.cot_desc:
-            self.cot_desc = COT_TEMPLATE
+        self.cot_desc = cot_desc if cot_desc else COT_TEMPLATE
+        self.od_desc = od_desc if od_desc else OD_TEMPLATE
 
     def cot(
         self,
@@ -573,6 +574,26 @@ class OpenAI_M():
                 print(e)
                 return ""
 
+    def od(
+        self,
+        image_path,
+    ):
+        json_str = self.invoke(
+            prompt=self.od_desc,
+            images=image_path,
+        )
+        return json_str
+
+    def od_v(
+        self,
+        image_path,
+        save_path,
+    ):
+        json_str = self.od(image_path)
+        img = draw_and_plot_boxes_from_json(json_str, image_path, save_path)
+        return img, save_path
+
+
 
 class MMChatter():
     def __init__(
@@ -644,4 +665,4 @@ class MMChatter():
         # Process the model's response by parsing the output
         response = output.splitlines()[-1].strip('<assistant>')  # Assuming the last line is the model's response
 
-        return response
+        return response

hdl/utils/llm/vis.py

@@ -8,11 +8,15 @@ import hashlib
 
 import torch
 import numpy as np
-from PIL import Image
 # from transformers import ChineseCLIPProcessor, ChineseCLIPModel
 from transformers import AutoModel
 from transformers import AutoTokenizer
 import open_clip
+
+from PIL import Image, ImageDraw, ImageFont
+import json
+import re
+import matplotlib.pyplot as plt
 # import natsort
 from redis.commands.search.field import VectorField
 from redis.commands.search.indexDefinition import IndexDefinition, IndexType
@@ -120,6 +124,87 @@ def pilimg_to_base64(pilimg):
     return img_base64
 
 
+
+def draw_and_plot_boxes_from_json(
+    json_data,
+    image_path,
+    save_path=None
+):
+    """
+    Parses the JSON data to extract bounding box coordinates,
+    scales them according to the image size, draws the boxes on the image,
+    and returns the image as a PIL object.
+
+    Args:
+        json_data (str or list): The JSON data as a string or already parsed list.
+        image_path (str): The path to the image file on which boxes are to be drawn.
+        save_path (str or None): The path to save the resulting image. If None, the image won't be saved.
+
+    Returns:
+        PIL.Image.Image: The processed image with boxes drawn on it.
+    """
+    # If json_data is a string, parse it into a Python object
+    if isinstance(json_data, str):
+        json_data = json_data.strip()
+        json_data = re.sub(r"^```json\s*", "", json_data)
+        json_data = re.sub(r"```$", "", json_data)
+        try:
+            data = json.loads(json_data)
+        except json.JSONDecodeError as e:
+            print("Failed to parse JSON data:", e)
+            return None
+    else:
+        data = json_data
+
+    # Open the image
+    try:
+        img = Image.open(image_path)
+    except FileNotFoundError:
+        print(f"Image file not found at {image_path}. Please check the path.")
+        return None
+
+    draw = ImageDraw.Draw(img)
+    width, height = img.size
+
+    # Use a commonly available font
+    try:
+        font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", size=25)
+    except IOError:
+        print("Default font not found. Using a basic PIL font.")
+        font = ImageFont.load_default()
+
+    # Process and draw boxes
+    for item in data:
+        object_type = item.get("object", "unknown")
+        for bbox in item.get("bboxes", []):
+            x1, y1, x2, y2 = bbox
+            x1 = x1 * width / 1000
+            y1 = y1 * height / 1000
+            x2 = x2 * width / 1000
+            y2 = y2 * height / 1000
+            draw.rectangle([(x1, y1), (x2, y2)], outline="blue", width=5)
+            draw.text((x1, y1), object_type, fill="red", font=font)
+
+    # Plot the image using matplotlib and save it as a PIL Image
+    buf = BytesIO()
+    plt.figure(figsize=(8, 8))
+    plt.imshow(img)
+    plt.axis("off")  # Hide axes ticks
+    plt.savefig(buf, format='png', bbox_inches='tight', pad_inches=0)
+    buf.seek(0)
+
+    # Load the buffer into a PIL Image and ensure full loading into memory
+    pil_image = Image.open(buf)
+    pil_image.load()  # Ensure full data is loaded from the buffer
+
+    # Save the image if save_path is provided
+    if save_path:
+        pil_image.save(save_path)
+
+    buf.close()  # Close the buffer after use
+
+    return pil_image
+
 class ImgHandler:
     """
     ImgHandler is a class for handling image processing tasks using pretrained models.

{hjxdl-0.2.82.dist-info → hjxdl-0.2.84.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.2.82
+Version: 0.2.84
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.2.82.dist-info → hjxdl-0.2.84.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=e3IzU75tUXwJv1hSzAqotnR7QIz2cy3BvcTNm40NY9c,413
+hdl/_version.py,sha256=6bOdkZwhZPgKGVmGjUF-gPGdfXzkUfQKQI9fPPKf_UQ,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -123,23 +123,23 @@ hdl/utils/database_tools/datetime.py,sha256=xqE2xNiOpADzX-R8_bM0bioJRF3Ay9Jp1CAG
 hdl/utils/database_tools/web.py,sha256=awJ8lafL-2KRjf3V1uuij8JIvX9U5fI8fLZKOkOvqtk,5771
 hdl/utils/desc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/desc/func_desc.py,sha256=VCqjvZs7qCwBq3NR3ZRknl4oiO5-JP7xm-Rx85W2exg,3365
-hdl/utils/desc/template.py,sha256=GJSXkVzdTAQoNT3j7YTLhz8-4CmvCMt2gLr7YBYPRWw,2192
+hdl/utils/desc/template.py,sha256=Kf_tbL-XkDCKNQ3UncbCuYEeUgXEa7kRVCf9TD2b8og,2526
 hdl/utils/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/general/glob.py,sha256=Zuf7WHU0UdUPOs9UrhxmrCiMC8GrHxQU6n3mTThv6yc,1120
 hdl/utils/general/runners.py,sha256=x7QBolp3MrqNV6L4rB6Ueybr26bqkRFZTuXhY0SwyLk,3061
 hdl/utils/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdl/utils/llm/chat.py,sha256=JnegglANgyp-avF4BdnhGji1w-x2wtonPPgjllsquV0,23283
+hdl/utils/llm/chat.py,sha256=nEQMWqbRQe4J0cHMZZ0lGYW8HMBsKUTqS9-bHV452mY,23797
 hdl/utils/llm/chatgr.py,sha256=5F5PJHe8vz3iCfi4TT54DCLRi1UeJshECdVtgvvvao0,3696
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
-hdl/utils/llm/vis.py,sha256=BsGAfy5X8sMFnX5A3vHpTPDRe_-IDdhs6YVQ-efvyQ0,21424
+hdl/utils/llm/vis.py,sha256=JPil4gJ-n-awv_EjNIjOwG9Gc89lsqTojV58_66U0_A,24204
 hdl/utils/llm/visrag.py,sha256=0i-VrxqgiV-J7R3VPshu9oc7-rKjFJOldYik3HDXj6M,10176
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.2.82.dist-info/METADATA,sha256=o5ShJ2TtbtQjRWiWt_laol2kYbOys3NYLJsJdvffplo,836
-hjxdl-0.2.82.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-hjxdl-0.2.82.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.2.82.dist-info/RECORD,,
+hjxdl-0.2.84.dist-info/METADATA,sha256=tuKn0Ql6hjCplkEJLhtI3CgA2Z4hC7MGzR9kgce6x3Q,836
+hjxdl-0.2.84.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+hjxdl-0.2.84.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.2.84.dist-info/RECORD,,