hjxdl 0.2.82__py3-none-any.whl → 0.2.88__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.82'
-__version_tuple__ = version_tuple = (0, 2, 82)
+__version__ = version = '0.2.88'
+__version_tuple__ = version_tuple = (0, 2, 88)

hdl/utils/desc/template.py

@@ -30,3 +30,14 @@ COT_TEMPLATE = """
 你的回答中应只能是 Markdown 格式，且不能包含其他多余文字或格式错误。
 以下是可用的工具：
 """
+
+OD_TEMPLATE = """
+Detect all the objects in the image, return bounding boxes for all of them using the following format (DO NOT INCLUDE ANY OTHER WORDS IN YOUR ANSWER BUT ONLY THE LIST ITSELF!):
+[
+    {
+        "object": "object_name",
+        "bboxes": [[xmin, ymin, xmax, ymax], [xmin, ymin, xmax, ymax], ...]
+    },
+    ...
+]
+"""

hdl/utils/llm/chat.py

@@ -8,9 +8,10 @@ import re
 
 
 from openai import OpenAI
-from ..desc.template import FN_TEMPLATE, COT_TEMPLATE
+from ..desc.template import FN_TEMPLATE, COT_TEMPLATE, OD_TEMPLATE
 from ..desc.func_desc import TOOL_DESC
-import json
+from .vis import draw_and_plot_boxes_from_json, to_img
+# import json
 # import traceback
 
 def parse_fn_markdown(markdown_text, params_key="params"):
@@ -132,6 +133,7 @@ class OpenAI_M():
         tools: list = None,
         tool_desc: dict = None,
         cot_desc: str = None,
+        od_desc: str = None,
         *args,
         **kwargs
     ):
@@ -189,9 +191,8 @@ class OpenAI_M():
         self.tool_info = "\n".join(self.tool_descs)
         self.tool_desc_str = "\n".join(self.tool_descs_verbose)
 
-        self.cot_desc = cot_desc
-        if not self.cot_desc:
-            self.cot_desc = COT_TEMPLATE
+        self.cot_desc = cot_desc if cot_desc else COT_TEMPLATE
+        self.od_desc = od_desc if od_desc else OD_TEMPLATE
 
     def cot(
         self,
@@ -573,6 +574,42 @@ class OpenAI_M():
                 print(e)
                 return ""
 
+    def od(
+        self,
+        image,
+    ):
+        """
+        Perform object detection on the given image.
+        Args:
+            image_path (str): The path to the image file on which to perform object detection.
+        Returns:
+            str: A JSON string containing the results of the object detection.
+        """
+        if isinstance(image, str):
+            image = to_img(image)
+        json_str = self.invoke(
+            prompt=self.od_desc,
+            images=[image]
+        )
+        return json_str
+
+    def od_v(
+        self,
+        image,
+        save_path,
+    ):
+        """
+        Perform object detection on an image and save the result.
+        Args:
+            image_path (str): The path to the input image.
+            save_path (str): The path to save the output image with detected objects.
+        Returns:
+            tuple: A tuple containing the processed image and the save path.
+        """
+        json_str = self.od(image)
+        img = draw_and_plot_boxes_from_json(json_str, image, save_path)
+        return img, save_path
+
 
 class MMChatter():
     def __init__(
@@ -644,4 +681,4 @@ class MMChatter():
         # Process the model's response by parsing the output
         response = output.splitlines()[-1].strip('<assistant>')  # Assuming the last line is the model's response
 
-        return response
+        return response

hdl/utils/llm/vis.py

@@ -8,11 +8,15 @@ import hashlib
 
 import torch
 import numpy as np
-from PIL import Image
 # from transformers import ChineseCLIPProcessor, ChineseCLIPModel
 from transformers import AutoModel
 from transformers import AutoTokenizer
 import open_clip
+
+from PIL import Image, ImageDraw, ImageFont
+import json
+import re
+import matplotlib.pyplot as plt
 # import natsort
 from redis.commands.search.field import VectorField
 from redis.commands.search.indexDefinition import IndexDefinition, IndexType
@@ -25,6 +29,21 @@ from redis.commands.search.query import Query
 
 HF_HUB_PREFIX = "hf-hub:"
 
+def to_img(img_str):
+    if img_str.startswith("data:image"):
+        img = imgbase64_to_pilimg(img_str)
+    elif img_str.startswith("http"):
+        response = requests.get(img_str)
+        if response.status_code == 200:
+            # Read the image content from the response
+            img_data = response.content
+
+            # Load the image using PIL to determine its format
+            img = Image.open(BytesIO(img_data))
+    elif Path(img_str).is_file():
+        img = Image.open(img_str)
+    return img
+
 
 def imgurl_to_base64(image_url: str):
     """Converts an image from a URL to base64 format.
@@ -120,6 +139,88 @@ def pilimg_to_base64(pilimg):
     return img_base64
 
 
+
+def draw_and_plot_boxes_from_json(
+    json_data,
+    image,
+    save_path=None
+):
+    """
+    Parses the JSON data to extract bounding box coordinates,
+    scales them according to the image size, draws the boxes on the image,
+    and returns the image as a PIL object.
+
+    Args:
+        json_data (str or list): The JSON data as a string or already parsed list.
+        image_path (str): The path to the image file on which boxes are to be drawn.
+        save_path (str or None): The path to save the resulting image. If None, the image won't be saved.
+
+    Returns:
+        PIL.Image.Image: The processed image with boxes drawn on it.
+    """
+    # If json_data is a string, parse it into a Python object
+    if isinstance(json_data, str):
+        json_data = json_data.strip()
+        json_data = re.sub(r"^```json\s*", "", json_data)
+        json_data = re.sub(r"```$", "", json_data)
+        try:
+            data = json.loads(json_data)
+        except json.JSONDecodeError as e:
+            print("Failed to parse JSON data:", e)
+            return None
+    else:
+        data = json_data
+
+    # Open the image
+    # try:
+    #     img = Image.open(image_path)
+    # except FileNotFoundError:
+    #     print(f"Image file not found at {image_path}. Please check the path.")
+    #     return None
+    img = image
+
+    draw = ImageDraw.Draw(img)
+    width, height = img.size
+
+    # Use a commonly available font
+    try:
+        font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", size=25)
+    except IOError:
+        print("Default font not found. Using a basic PIL font.")
+        font = ImageFont.load_default()
+
+    # Process and draw boxes
+    for item in data:
+        object_type = item.get("object", "unknown")
+        for bbox in item.get("bboxes", []):
+            x1, y1, x2, y2 = bbox
+            x1 = x1 * width / 1000
+            y1 = y1 * height / 1000
+            x2 = x2 * width / 1000
+            y2 = y2 * height / 1000
+            draw.rectangle([(x1, y1), (x2, y2)], outline="blue", width=5)
+            draw.text((x1, y1), object_type, fill="red", font=font)
+
+    # Plot the image using matplotlib and save it as a PIL Image
+    buf = BytesIO()
+    plt.figure(figsize=(8, 8))
+    plt.imshow(img)
+    plt.axis("off")  # Hide axes ticks
+    plt.savefig(buf, format='png', bbox_inches='tight', pad_inches=0)
+    buf.seek(0)
+
+    # Load the buffer into a PIL Image and ensure full loading into memory
+    pil_image = Image.open(buf)
+    pil_image.load()  # Ensure full data is loaded from the buffer
+
+    # Save the image if save_path is provided
+    if save_path:
+        pil_image.save(save_path)
+
+    buf.close()  # Close the buffer after use
+
+    return pil_image
+
 class ImgHandler:
     """
     ImgHandler is a class for handling image processing tasks using pretrained models.

{hjxdl-0.2.82.dist-info → hjxdl-0.2.88.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.2.82
+Version: 0.2.88
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.2.82.dist-info → hjxdl-0.2.88.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=e3IzU75tUXwJv1hSzAqotnR7QIz2cy3BvcTNm40NY9c,413
+hdl/_version.py,sha256=gmIfdIg5gEvfMxffRlk3KF_G7qsO5gyiwsJ6wEhCrTk,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -123,23 +123,23 @@ hdl/utils/database_tools/datetime.py,sha256=xqE2xNiOpADzX-R8_bM0bioJRF3Ay9Jp1CAG
 hdl/utils/database_tools/web.py,sha256=awJ8lafL-2KRjf3V1uuij8JIvX9U5fI8fLZKOkOvqtk,5771
 hdl/utils/desc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/desc/func_desc.py,sha256=VCqjvZs7qCwBq3NR3ZRknl4oiO5-JP7xm-Rx85W2exg,3365
-hdl/utils/desc/template.py,sha256=GJSXkVzdTAQoNT3j7YTLhz8-4CmvCMt2gLr7YBYPRWw,2192
+hdl/utils/desc/template.py,sha256=Kf_tbL-XkDCKNQ3UncbCuYEeUgXEa7kRVCf9TD2b8og,2526
 hdl/utils/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/general/glob.py,sha256=Zuf7WHU0UdUPOs9UrhxmrCiMC8GrHxQU6n3mTThv6yc,1120
 hdl/utils/general/runners.py,sha256=x7QBolp3MrqNV6L4rB6Ueybr26bqkRFZTuXhY0SwyLk,3061
 hdl/utils/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdl/utils/llm/chat.py,sha256=JnegglANgyp-avF4BdnhGji1w-x2wtonPPgjllsquV0,23283
+hdl/utils/llm/chat.py,sha256=B0OSX2Yc4OZUwRu-aHnZOW-knTeDqlzCP_UqlgnhMKA,24476
 hdl/utils/llm/chatgr.py,sha256=5F5PJHe8vz3iCfi4TT54DCLRi1UeJshECdVtgvvvao0,3696
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
-hdl/utils/llm/vis.py,sha256=BsGAfy5X8sMFnX5A3vHpTPDRe_-IDdhs6YVQ-efvyQ0,21424
+hdl/utils/llm/vis.py,sha256=fuXdEFfUpW4KR7TKmIBEuMmo3ZdZxfd79a_qfTM0tPk,24739
 hdl/utils/llm/visrag.py,sha256=0i-VrxqgiV-J7R3VPshu9oc7-rKjFJOldYik3HDXj6M,10176
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.2.82.dist-info/METADATA,sha256=o5ShJ2TtbtQjRWiWt_laol2kYbOys3NYLJsJdvffplo,836
-hjxdl-0.2.82.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-hjxdl-0.2.82.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.2.82.dist-info/RECORD,,
+hjxdl-0.2.88.dist-info/METADATA,sha256=jAJxvM6YypO5nHnLRSe8Zhi8QwOlvmsfMFi8H9IjeVg,836
+hjxdl-0.2.88.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+hjxdl-0.2.88.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.2.88.dist-info/RECORD,,