MeUtils 2025.3.3.18.41.24__py3-none-any.whl → 2025.3.5.19.55.22__py3-none-any.whl

meutils/io/parsers/fileparser/filetype.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : filetype
+# @Time         : 2023/7/15 15:05
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 文件或扩展名被更改
+
+from meutils.pipe import *
+
+import filetype
+import magic
+import mimetypes
+
+import mimetypes
+
+# mimetype, _ = mimetypes.guess_type('example.txt')
+# print(mimetype)
+#
+# import magic
+#
+# file_type = magic.from_file('example.pdf', mime=True)
+# print(file_type)
+#
+# import filetype
+#
+# kind = filetype.guess('example.jpg')
+# if kind is None:
+#     print('Cannot guess file type!')
+# else:
+#     print('File extension: %s' % kind.extension)
+#     print('File MIME type: %s' % kind.mime)
+
+if __name__ == '__main__':
+    # mimetype, _ = mimetypes.guess_type('example.mp3')
+    mimetype, _ = mimetypes.guess_type('https://www.baidu.com/img/flexible/logo/pc/result@2.png')
+
+    print(mimetype)
+    print(_)

meutils/io/parsers/fileparser/mineru.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : mineru
+# @Time         : 2025/1/23 13:13
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+
+import requests
+
+token = """
+eyJ0eXBlIjoiSldUIiwiYWxnIjoiSFM1MTIifQ.eyJqdGkiOiI4MDAwNDQ3NyIsInJvbCI6IlJPTEVfUkVHSVNURVIiLCJpc3MiOiJPcGVuWExhYiIsImlhdCI6MTczNzYxMTQ5MiwiY2xpZW50SWQiOiJsa3pkeDU3bnZ5MjJqa3BxOXgydyIsInBob25lIjoiIiwidXVpZCI6IjczNTZjNjY1LTU4MTMtNGQxNC04ZjFiLWM0NWIyZmFhYTBhMCIsImVtYWlsIjoiMzEzMzAzMzAzQHFxLmNvbSIsImV4cCI6MTczODgyMTA5Mn0.i8CwWoRE6j5wAC_hD9z9amkWT56HdOewgXMFV4jMpg17JHB0HOY-K4o9zp06Puav2vxkuC3Lnqm_8ip7-QdxsQ
+"""
+url = 'https://mineru.net/api/v4/extract/task'
+header = {
+    'Content-Type': 'application/json',
+    "Authorization": f"Bearer {token.strip()}"
+}
+data = {
+    'url': 'https://cdn-mineru.openxlab.org.cn/demo/example.pdf',
+    'is_ocr': True,
+    'enable_formula': False,
+}
+
+res = requests.post(url, headers=header, json=data)
+print(res.status_code)
+print(res.json())
+print(res.json()["data"])
+
+# {'task_id': 'adb223f6-794b-4950-8d60-d766ebd0bf14'}
+
+task_id = 'adb223f6-794b-4950-8d60-d766ebd0bf14'
+import requests
+
+url = f'https://mineru.net/api/v4/extract/task/{task_id}'
+header = {
+    'Content-Type':'application/json',
+    "Authorization": f"Bearer {token.strip()}"
+}
+
+res = requests.get(url, headers=header)
+print(res.status_code)
+print(res.json())
+print(res.json()["data"])

meutils/io/parsers/fileparser/pdf.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : pdf
+# @Time         : 2023/5/18 16:39
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+
+
+class PageWord(BaseModel):
+    x0 = 153.5
+    x1 = 441.72032
+    top = 76.19035999999994
+    doctop = 76.19035999999994
+    bottom = 92.15035999999998
+    upright = True
+    direction = 1
+    text = "国投瑞银基金管理有限公司基金相关参数"
+
+
+class PageWords(BaseModel):  # page.extract_words()
+    pagewords: List[PageWord]
+
+
+class Part(BaseModel):
+    pass

meutils/io/parsers/fileparser//350/241/250/346/240/274/346/212/275/345/217/226.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : 表格抽取
+# @Time         : 2023/7/17 18:14
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+import os
+from dataclasses import dataclass
+from typing import List
+from collections import Counter
+
+import pandas as pd
+import pdfplumber
+import tabulate
+
+
+def extract_tables_with_text(pdf) -> List[str]:
+    """抽取表格并嵌入文本"""
+
+    def clean_cell_text(text):
+        """
+        清除文本中的换行符和多余空格
+        """
+        if text is None:
+            return ""
+        text = text.replace("\n", "")
+        # 去除字符串开头和结尾的空白字符
+        text = text.strip()
+        return text
+
+    def check_bboxes(word, table_bbox):
+        left = word['x0'], word['top'], word['x1'], word['bottom']
+        r = table_bbox
+        return left[0] > r[0] and left[1] > r[1] and left[2] < r[2] and left[3] < r[3]
+
+    def keep_visible_lines(obj):
+
+        if obj['object_type'] == 'rect':
+            return obj['non_stroking_color'] == 0
+        return True
+
+    lines = []
+    page_counter = 0
+    # 用来缓存没完整展示的表
+    pending_table = []
+    tp = None
+    for page_number, page in enumerate(pdf.pages, start=0):
+        p = page.filter(keep_visible_lines)
+        combine_flag = len(pending_table) != 0
+        page_counter = page_number
+        tables = p.find_tables(
+            table_settings={
+                "vertical_strategy": "lines",
+                "horizontal_strategy": "lines",
+                # "explicit_vertical_lines": self.curves_to_edges(p.curves) + p.edges,
+                # "explicit_horizontal_lines": self.curves_to_edges(p.curves) + p.edges,
+            }
+        )
+        if len(tables) != 0:
+            is_over_footer = p.chars[-1].get('y0') >= p.bbox[3] - p.find_tables()[-1].bbox[3]
+        else:
+            is_over_footer = False
+        bboxes = [table.bbox for table in tables]
+        tables = [{'table': i.extract(), 'top': i.bbox[1]} for i in tables]
+        if is_over_footer:
+            tp = pd.DataFrame(tables[-1]['table']).applymap(clean_cell_text)
+            tables = tables[0:len(tables) - 1]
+        non_table_words = [word for word in p.extract_words() if
+                           not any([check_bboxes(word, table_bbox) for table_bbox in bboxes])]
+
+        for cluster in pdfplumber.utils.cluster_objects(non_table_words + tables, 'top', tolerance=5):
+            for c in cluster:
+                if 'text' in c and c['text'] != str(page_number) and c['text'] != non_table_words[-1]:
+                    lines.append("".join(c['text']))
+                elif 'table' in c:
+                    if combine_flag:
+                        dt = pd.concat([pd.concat(pending_table, axis=0),
+                                        pd.DataFrame(c['table']).applymap(clean_cell_text)], axis=0)
+                        combine_flag = False
+                        pending_table = []
+                    else:
+                        dt = pd.DataFrame(c['table']).applymap(clean_cell_text)
+                    tb = tabulate.tabulate(dt, tablefmt="pipe", showindex=False, headers="keys")
+                    lines.append(tb)
+        if is_over_footer:
+            pending_table.append(tp)
+    content_counter = Counter(lines)
+    return [word for word in content_counter if content_counter[word] != page_counter]
+
+
+@dataclass
+class Document:
+    """文档结构"""
+    text: List[str]
+    source: str
+
+
+def extract_text(filepath) -> Document:
+    with pdfplumber.open(filepath) as pdf:
+        text = extract_tables_with_text(pdf)
+    return Document(text=text, source=os.path.basename(filepath))
+
+
+if __name__ == '__main__':
+    """
+    pdfplumber==0.9.0
+    tabulate[widechars]==0.9.0
+    numpy==1.21.5
+    pandas==2.0.3
+    """
+    doc = extract_text("/Users/betterme/Downloads/H2_AN202303301584686196_1.pdf")
+    for line in doc.text:
+        print(line)

meutils/llm/check_utils.py

@@ -92,18 +92,49 @@ async def check_token_for_jina(api_key, threshold=1000):
         return False
 
 
+@retrying()
+async def check_token_for_moonshot(api_key, threshold: float=0):
+    if not isinstance(api_key, str):
+        return await check_tokens(api_key, check_token_for_jina)
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Accept": "application/json"
+    }
+
+    try:
+        async with httpx.AsyncClient(base_url="https://api.moonshot.cn/v1", headers=headers, timeout=60) as client:
+            response: httpx.Response = await client.get("/users/me/balance")
+            response.raise_for_status()
+
+            logger.debug(response.text)
+            logger.debug(response.status_code)
+
+            if response.is_success:
+                data = response.json()
+                logger.debug(data)
+                balance = data['data']['available_balance']
+                return float(balance) >= threshold
+
+    except Exception as e:
+        logger.error(f"Error: {e}\n{api_key}")
+        return False
+
+
 if __name__ == '__main__':
     from meutils.config_utils.lark_utils import get_next_token_for_polling
 
     check_valid_token = partial(check_token_for_siliconflow, threshold=-1)
 
     pass
-    arun(check_valid_token("sk-LlB4W38z9kv5Wy1c3ceeu4PHeIWs6bbWsjr8Om31jYvsucRv", threshold=0.1))
+    # arun(check_valid_token("sk-LlB4W38z9kv5Wy1c3ceeu4PHeIWs6bbWsjr8Om31jYvsucRv", threshold=0.1))
 
-    FEISHU_URL = "https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=KVClcs"
+    # FEISHU_URL = "https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=KVClcs"
 
     # b = arun(check_token_for_openai(os.getenv("STEP_API_KEY")))
 
     # arun(get_next_token_for_polling(check_token=check_token_for_openai, feishu_url=FEISHU_URL))
 
     # arun(check_token_for_jina(["jina_c8da77fed9704d558c8def39837960edplTLkNYrsPTJHBF1HcYg_RkRVh0X"]*10))
+
+    arun(check_token_for_moonshot("sk-fWqLGmUtoGgoK9gx5IefO1mWrRF9QHaV7uVRrTcFv1lrJVvJ"))

meutils/llm/clients.py

@@ -18,6 +18,7 @@ chatfire_client = AsyncOpenAI()
 
 moonshot_client = AsyncOpenAI(
     api_key=os.getenv("MOONSHOT_API_KEY"),
+    # api_key="sk-fWqLGmUtoGgoK9gx5IefO1mWrRF9QHaV7uVRrTcFv1lrJVvJ",
     base_url=os.getenv("MOONSHOT_BASE_URL")
 )
 zhipuai_client = AsyncOpenAI(

meutils/llm/completions/chat_gemini.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : gemini
+# @Time         : 2025/2/14 17:36
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+
+from meutils.pipe import *
+from meutils.llm.openai_utils import to_openai_params
+from meutils.llm.clients import AsyncOpenAI
+
+from meutils.schemas.openai_types import chat_completion, chat_completion_chunk, CompletionRequest, CompletionUsage
+
+"""
+image => file
+
+      "type": "image_url",
+      "image_url": {
+          
+
+"""
+
+
+class Completions(object):
+
+    def __init__(self,
+                 base_url: Optional[str] = None,
+                 api_key: Optional[str] = None
+                 ):
+        self.client = AsyncOpenAI(
+            base_url=base_url,
+            api_key=api_key,
+        )
+
+    async def create(self, request: CompletionRequest):
+        urls = sum(request.last_urls.values(), [])
+        for url in urls:
+            request.messages[-1]["content"].append({"type": "image_url", "image_url": {"url": url}})
+
+        data = to_openai_params(request)
+        return await self.client.chat.completions.create(**data)
+
+
+if __name__ == '__main__':
+    url = "https://oss.ffire.cc/files/lipsync.mp3"
+    content = [
+        {"type": "text", "text": "总结下"},
+        # {"type": "image_url", "image_url": {"url": url}},
+
+        {"type": "video_url", "video_url": {"url": url}}
+
+    ]
+    request = CompletionRequest(
+        # model="qwen-turbo-2024-11-01",
+        model="gemini-all",
+        # model="qwen-plus-latest",
+
+        messages=[
+            {
+                'role': 'user',
+
+                'content': content
+            },
+
+        ],
+        stream=False,
+    )
+    arun(Completions().create(request))

meutils/llm/completions/chat_plus.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : chat_all
+# @Time         : 2025/3/4 13:25
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+
+from openai import AsyncOpenAI
+
+from meutils.pipe import *
+from meutils.decorators.retry import retrying
+from meutils.io.files_utils import to_bytes
+from meutils.io.openai_files import file_extract
+
+from meutils.llm.clients import qwen_client, chatfire_client
+from meutils.llm.openai_utils import to_openai_params
+
+from meutils.config_utils.lark_utils import get_next_token_for_polling
+from meutils.schemas.openai_types import chat_completion, chat_completion_chunk, CompletionRequest, CompletionUsage
+
+FEISHU_URL = "https://xchatllm.feishu.cn/sheets/Bmjtst2f6hfMqFttbhLcdfRJnNf?sheet=PP1PGr"
+
+base_url = "https://chat.qwenlm.ai/api"
+
+from fake_useragent import UserAgent
+
+ua = UserAgent()
+
+"""
+# vision_model="doubao-1.5-vision-pro-32k"
+# glm-4v-flash 
+
+1. 文件解析
+
+"""
+
+
+class Completions(object):
+
+    def __init__(self,
+                 vision_model: str = "doubao-1.5-vision-pro-32k",
+                 base_url: Optional[str] = None,
+                 api_key: Optional[str] = None,
+                 ):
+        self.vision_model = vision_model
+        self.client = AsyncOpenAI(
+            base_url=base_url,
+            api_key=api_key,
+        )
+
+    async def create(self, request: CompletionRequest):
+        """适配任意客户端"""
+
+        last_message = request.last_message
+        file_urls = request.last_urls.get("file_url", [])
+        file_contents = await file_extract(file_urls)
+
+
+        # if 'r1' in request.model:  # vl
+        #     data = to_openai_params(request)
+        #     data['stream'] = False
+        #     data['model'] = self.vision_model
+        #
+        #     data['messages'] = [
+        #         {"type": "text", "text": "图片描述："}
+        #     ]
+        #
+        #     # await chatfire_client.create(**data)
+        #
+        #     return await chatfire_client.create(**data)
+        #
+        # request.last_message
+
+# "image_url": "http://ai.chatfire.cn/files/images/image-1725418399272-d7b71012f.png"

meutils/llm/completions/{agents/file.py → chat_spark.py}

@@ -17,14 +17,12 @@ todo: 记录上下文日志
 """
 
 from meutils.pipe import *
-from meutils.parsers.file_parsers import file_extract
+from meutils.io.openai_files import file_extract
 
 from meutils.llm.clients import AsyncOpenAI
 from meutils.llm.openai_utils import to_openai_params
 
-from meutils.str_utils.regular_expression import parse_url
-
-from meutils.schemas.openai_types import chat_completion, chat_completion_chunk, ChatCompletionRequest, CompletionUsage
+from meutils.schemas.openai_types import chat_completion, chat_completion_chunk, CompletionRequest, CompletionUsage
 
 
 class Completions(object):
@@ -32,21 +30,36 @@ class Completions(object):
     def __init__(self, api_key: Optional[str] = None):
         self.api_key = api_key
 
-    async def create(self, request: ChatCompletionRequest):
-        """[{'role': 'user', 'content': 'hi'}]
-        {"type": "file_url", "file_url": {"url": "https://oss.ffire.cc/files/招标文件备案表（第二次）.pdf", "detai": "auto"}}"""
-
-        if urls := parse_url(str(request.messages)):
-            logger.debug(urls)
-
-            file_reponse = await file_extract(urls[-1])
-            if file_reponse:  # 仅关注最后一个
-                request.system_messages.append({
-                    "role": "system",
-                    "content": json.dumps(file_reponse, ensure_ascii=False),
-                })
-
-            request.messages = request.system_messages + request.messages
+    async def create(self, request: CompletionRequest):
+
+        if image_urls := request.last_urls.get("image_url"):
+            request.model = "doubao-1.5-vision-pro-32k"  # 6月过期
+
+            request.messages = [
+                {
+                    'role': 'user',
+                    "content": [
+                        {"type": "text", "text": request.last_user_content},
+                        {"type": "image_url", "image_url": {"url": image_urls[-1]}}
+                    ]
+                }
+            ]
+        else:
+            for i, message in enumerate(request.messages[::-1], 1):
+                if message.get("role") == "user":  # 每一轮还要处理
+                    content = message.get("content")
+
+                    if content.startswith("http"):
+                        file_url, content = content.split(maxsplit=1)
+                        file_content = await file_extract(file_url)
+
+                        request.messages[-i] = {
+                            'role': 'user',
+                            'content': f"""{json.dumps(file_content, ensure_ascii=False)}\n\n
+                            {content}
+                            """
+                        }
+                        break
 
         logger.debug(request)
 
@@ -58,16 +71,16 @@ class Completions(object):
 if __name__ == '__main__':
     c = Completions()
 
-    request = ChatCompletionRequest(
+    request = CompletionRequest(
         # model="qwen-turbo-2024-11-01",
         # model="claude-3-5-sonnet-20241022",
         model="gpt-4o-mini",
 
         messages=[
-            {
-                'role': 'system',
-                'content': '你是一个文件问答助手'
-            },
+            # {
+            #     'role': 'system',
+            #     'content': '你是一个文件问答助手'
+            # },
             # {
             #     'role': 'user',
             #     # 'content': {
@@ -89,6 +102,13 @@ if __name__ == '__main__':
 
             {
                 'role': 'user',
+                # "content": '你好',
+                # "content": [
+                #     {"type": "text", "text": "描述"},
+                #
+                #     {"type": "image_url", "image_url": "https://oss.ffire.cc/files/kling_watermark.png"}
+                # ],
+
                 # 'content': {
                 #     "type": "file_url",
                 #     "file_url": {"url": "https://mj101-1317487292.cos.ap-shanghai.myqcloud.com/ai/test.pdf", "detai": "auto"}
@@ -96,10 +116,10 @@ if __name__ == '__main__':
                 # 'content': "https://oss.ffire.cc/files/%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6%E5%A4%87%E6%A1%88%E8%A1%A8%EF%BC%88%E7%AC%AC%E4%BA%8C%E6%AC%A1%EF%BC%89.pdf 这个文件讲了什么？",
                 # 'content': "https://translate.google.com/?sl=zh-CN&tl=en&text=%E6%8F%90%E4%BE%9B%E6%96%B9&op=tr1anslate 这个文件讲了什么？",
 
-                # "content": "总结下 https://oss.ffire.cc/files/百炼系列手机产品介绍.docx"
+                "content": "https://oss.ffire.cc/files/百炼系列手机产品介绍.docx 总结下"
                 # "content": "https://mj101-1317487292.cos.ap-shanghai.myqcloud.com/ai/test.pdf\n\n总结下"
 
-                "content": "https://admin.ilovechatgpt.top/file/lunIMYAIzhinengzhushouduishenghuodocx_14905733.docx 总结"
+                # "content": "https://admin.ilovechatgpt.top/file/lunIMYAIzhinengzhushouduishenghuodocx_14905733.docx 总结"
 
             },