hjxdl 0.2.97__py3-none-any.whl → 0.2.99__py3-none-any.whl

hdl/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.97'
-__version_tuple__ = version_tuple = (0, 2, 97)
+__version__ = version = '0.2.99'
+__version_tuple__ = version_tuple = (0, 2, 99)

hdl/utils/desc/func_desc.py

@@ -109,6 +109,21 @@ TOOL_DESC = {
 """
     },
 
+    "object_detect": {
+        "desc": """
+## 函数名：object_detect
+描述：当用户要求识别图片中的目标位置时，调用此工具，给出具体位置信息。
+参数 ：
+- image (str): 图片的路径或 url
+返回值 (str): 图片中目标的位置信息（也可能含有带有标记的渲染图片路径）
+""",
+        "md": """
+需要返回的 markdown:
+- function_name: object_detect
+- image: <image_path>
+"""
+    },
+
     "default": None
 
 }

hdl/utils/llm/chat.py

@@ -450,12 +450,12 @@ class OpenAI_M():
             str: The agent response based on the prompt.
         '''
         """
-        decision_dict = self.get_decision(prompt, **kwargs)
-        decision_dict = parse_fn_markdown(decision_dict)
+        decision_dict_str = self.get_decision(prompt, **kwargs)
+        decision_dict = parse_fn_markdown(decision_dict_str)
         if decision_dict.get("function_name", None) is None:
             return self.chat(prompt=prompt, stream=stream, **kwargs)
         else:
-            tool_result = str(self.get_tool_result(decision_dict))
+            tool_result = str(self.get_tool_result(decision_dict_str))
             prompt_final = "根据上下文回答最后的用户问题：\n上下文信息：\n"
             prompt_final += tool_result
             # prompt_final += f"\n用户的问题：\n{prompt}"
@@ -502,7 +502,7 @@ class OpenAI_M():
 
     def get_tool_result(
         self,
-        decision_dict: dict,
+        decision_dict_str: str,
     ):
         """Get the result of a tool based on the decision made.
 
@@ -513,16 +513,7 @@ class OpenAI_M():
         Returns:
             str: The result of the tool.
         """
-        # decision_dict_str = self.get_decision(
-        #     prompt,
-        #     **kwargs
-        # )
-        # try:
-        #     decision_dict = parse_fn_markdown(decision_dict_str)
-        #     # decision_dict = json.loads(decision_dict_str)
-        # except Exception as e:
-        #     print(e)
-        #     return ""
+        decision_dict = parse_fn_markdown(decision_dict_str)
         func_name = decision_dict.get("function_name", None)
         if func_name is None:
             return ""
@@ -532,6 +523,8 @@ class OpenAI_M():
                     if tool.__name__ == func_name:
                         tool_final = tool
                 func_kwargs = decision_dict.get("params")
+                if tool_final.__name__ == "object_detect":
+                    func_kwargs["llm"] = self
                 return tool_final(**func_kwargs)
             except Exception as e:
                 print(e)
@@ -695,4 +688,59 @@ class MMChatter():
         else:
             response = output.strip()
 
-        return response
+        return response
+
+
+def object_detect(
+    image,
+    prompt: str = OD_TEMPLATE,
+    llm = None,
+    cli_dir: str = None,
+    model_dir: str = None,
+    mmproj_dir: str = None,
+    save_dir: str = None,
+):
+    resp = ""
+    cli_dir = cli_dir if cli_dir is not None else os.getenv(
+        "MM_CLI_DIR",
+        None
+    )
+    if model_dir is None:
+        model_dir = os.getenv(
+            "MM_MODEL_DIR",
+            "/home/jhu/dev/models/MiniCPM-V-2_6-gguf/ggml-model-Q4_K_M.gguf"
+        )
+    if mmproj_dir is None:
+        mmproj_dir = os.getenv(
+            "MM_PROJ_DIR",
+            "/home/jhu/dev/models/MiniCPM-V-2_6-gguf/mmproj-model-f16.gguf"
+        )
+    if llm is None and cli_dir is None:
+        raise ValueError("Either 'llm' or 'cli_dir' must be provided.")
+    # mm_server = os.getenv("LLM_SERVER", "192.168.1.232")
+    # mm_port = int(os.getevn("LLM_PORT", 22299))
+
+    if cli_dir:
+        mm = MMChatter(
+            cli_dir=cli_dir,
+            model_dir=model_dir,
+            mmproj_dir=mmproj_dir
+        )
+        json_data = mm.get_resp(
+            prompt=prompt,
+            image=image
+        )
+    else:
+        json_data = llm.od(image)
+    _, save_dir = draw_and_plot_boxes_from_json(
+        json_data=json_data,
+        image=image,
+        save_path=save_dir
+    )
+    resp += "Detected objects are:\n"
+    resp += json_data
+    resp =+ "\n"
+    if save_dir:
+        resp += "Picture with marks were saved at:\n"
+        resp += save_dir
+    return resp

hdl/utils/llm/vis.py

@@ -257,7 +257,7 @@ def draw_and_plot_boxes_from_json(
 
     buf.close()  # Close the buffer after use
 
-    return pil_image
+    return pil_image, save_path
 
 class ImgHandler:
     """

{hjxdl-0.2.97.dist-info → hjxdl-0.2.99.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hjxdl
-Version: 0.2.97
+Version: 0.2.99
 Summary: A collection of functions for Jupyter notebooks
 Home-page: https://github.com/huluxiaohuowa/hdl
 Author: Jianxing Hu

{hjxdl-0.2.97.dist-info → hjxdl-0.2.99.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hdl/__init__.py,sha256=GffnD0jLJdhkd-vo989v40N90sQbofkayRBwxc6TVhQ,72
-hdl/_version.py,sha256=T0AjOUctLG_1wdj-g5Ni8vW0LU2eRCNwp1hO3EJB9gg,413
+hdl/_version.py,sha256=eh_O0mMXOD0hXwM9sRjgMdeWbi6snGlGab0dp1qqp9g,413
 hdl/args/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/args/loss_args.py,sha256=s7YzSdd7IjD24rZvvOrxLLFqMZQb9YylxKeyelSdrTk,70
 hdl/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -122,24 +122,24 @@ hdl/utils/database_tools/connect.py,sha256=xCacGucKxlQUXs6AsNddpeECvdqT1180V1ZWq
 hdl/utils/database_tools/datetime.py,sha256=xqE2xNiOpADzX-R8_bM0bioJRF3Ay9Jp1CAG6dy6uVI,1202
 hdl/utils/database_tools/web.py,sha256=awJ8lafL-2KRjf3V1uuij8JIvX9U5fI8fLZKOkOvqtk,5771
 hdl/utils/desc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdl/utils/desc/func_desc.py,sha256=VCqjvZs7qCwBq3NR3ZRknl4oiO5-JP7xm-Rx85W2exg,3365
+hdl/utils/desc/func_desc.py,sha256=sHmVZZmV7Zgii--gnHqMs6fTb7HVkqTOf8Pl_0F6qlI,3808
 hdl/utils/desc/template.py,sha256=Kf_tbL-XkDCKNQ3UncbCuYEeUgXEa7kRVCf9TD2b8og,2526
 hdl/utils/general/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/general/glob.py,sha256=Zuf7WHU0UdUPOs9UrhxmrCiMC8GrHxQU6n3mTThv6yc,1120
 hdl/utils/general/runners.py,sha256=x7QBolp3MrqNV6L4rB6Ueybr26bqkRFZTuXhY0SwyLk,3061
 hdl/utils/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hdl/utils/llm/chat.py,sha256=1ijgGyD25dyy2XeQe5Fm3xYAIuev4kbyuFhX9iNZbEs,24936
+hdl/utils/llm/chat.py,sha256=0MXmw9FP10uR_NRJrSUhcrK88zE9oIdgGRbZp5l4N74,26263
 hdl/utils/llm/chatgr.py,sha256=5F5PJHe8vz3iCfi4TT54DCLRi1UeJshECdVtgvvvao0,3696
 hdl/utils/llm/embs.py,sha256=Tf0FOYrOFZp7qQpEPiSCXzlgyHH0X9HVTUtsup74a9E,7174
 hdl/utils/llm/extract.py,sha256=2sK_WJzmYIc8iuWaM9DA6Nw3_6q1O4lJ5pKpcZo-bBA,6512
 hdl/utils/llm/llama_chat.py,sha256=watcHGOaz-bv3x-yDucYlGk5f8FiqfFhwWogrl334fk,4387
-hdl/utils/llm/vis.py,sha256=TM365x-W5pZt4VFFmpICt3TuWkENYR0uWoRVxu9xW4s,26044
+hdl/utils/llm/vis.py,sha256=SSP6tOwKLq0hWcpM3twI9TitqzBmKjlcGrnXEWYlCzM,26055
 hdl/utils/llm/visrag.py,sha256=0i-VrxqgiV-J7R3VPshu9oc7-rKjFJOldYik3HDXj6M,10176
 hdl/utils/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/schedulers/norm_lr.py,sha256=bDwCmdEK-WkgxQMFBiMuchv8Mm7C0-GZJ6usm-PQk14,4461
 hdl/utils/weather/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hdl/utils/weather/weather.py,sha256=k11o6wM15kF8b9NMlEfrg68ak-SfSYLN3nOOflFUv-I,4381
-hjxdl-0.2.97.dist-info/METADATA,sha256=fCQFw9wQ6dIP77VUwvW2IEcmarz1CG7-EalYK9HEr2g,862
-hjxdl-0.2.97.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-hjxdl-0.2.97.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
-hjxdl-0.2.97.dist-info/RECORD,,
+hjxdl-0.2.99.dist-info/METADATA,sha256=pEj-TvHeI8XvXpXfCVACTTk5bNd-hj4116dIYje6zCI,862
+hjxdl-0.2.99.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+hjxdl-0.2.99.dist-info/top_level.txt,sha256=-kxwTM5JPhylp06z3zAVO3w6_h7wtBfBo2zgM6YZoTk,4
+hjxdl-0.2.99.dist-info/RECORD,,