MeUtils 2024.12.5.13.7.4__tar.gz → 2024.12.5.17.36.18__tar.gz

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18/MeUtils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: MeUtils
-Version: 2024.12.5.13.7.4
+Version: 2024.12.5.17.36.18
 Summary: description
 Home-page: https://github.com/yuanjie-ai/MeUtils
 Author: yuanjie
@@ -106,42 +106,42 @@ Requires-Dist: pandas-profiling[notebook]; extra == "pd"
 Requires-Dist: pandas_summary; extra == "pd"
 Requires-Dist: polars; extra == "pd"
 Provides-Extra: all
-Requires-Dist: thefuck; extra == "all"
-Requires-Dist: sqlalchemy; extra == "all"
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: fastapi; extra == "all"
-Requires-Dist: gunicorn; extra == "all"
-Requires-Dist: redis-py-cluster; extra == "all"
-Requires-Dist: pymysql; extra == "all"
-Requires-Dist: pymupd; extra == "all"
-Requires-Dist: pymongo; extra == "all"
-Requires-Dist: pandas-profiling[notebook]; extra == "all"
-Requires-Dist: fastapi[all]; extra == "all"
-Requires-Dist: thriftpy2; extra == "all"
-Requires-Dist: pymilvus; extra == "all"
-Requires-Dist: pandas_summary; extra == "all"
-Requires-Dist: jieba; extra == "all"
-Requires-Dist: jinja2; extra == "all"
-Requires-Dist: langchain; extra == "all"
-Requires-Dist: polars; extra == "all"
-Requires-Dist: streamlit; extra == "all"
 Requires-Dist: openai; extra == "all"
 Requires-Dist: geopy; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: faiss-cpu; extra == "all"
+Requires-Dist: pandas-profiling[notebook]; extra == "all"
+Requires-Dist: pymilvus; extra == "all"
 Requires-Dist: iteration_utilities; extra == "all"
+Requires-Dist: pymupd; extra == "all"
+Requires-Dist: thefuck; extra == "all"
+Requires-Dist: uvicorn; extra == "all"
+Requires-Dist: faiss-cpu; extra == "all"
+Requires-Dist: jmespath; extra == "all"
+Requires-Dist: langchain; extra == "all"
+Requires-Dist: sqlalchemy; extra == "all"
 Requires-Dist: missingno; extra == "all"
+Requires-Dist: gunicorn; extra == "all"
+Requires-Dist: polars; extra == "all"
+Requires-Dist: jieba; extra == "all"
+Requires-Dist: fastapi[all]; extra == "all"
 Requires-Dist: asyncmy; extra == "all"
-Requires-Dist: cachetools; extra == "all"
-Requires-Dist: sse_starlette; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: streamlit; extra == "all"
 Requires-Dist: reportlab; extra == "all"
+Requires-Dist: fastapi; extra == "all"
 Requires-Dist: dataframe_image; extra == "all"
+Requires-Dist: pandas_summary; extra == "all"
+Requires-Dist: pymysql; extra == "all"
 Requires-Dist: simplejson; extra == "all"
-Requires-Dist: schedule; extra == "all"
-Requires-Dist: pretty_errors; extra == "all"
-Requires-Dist: jmespath; extra == "all"
-Requires-Dist: uvicorn; extra == "all"
 Requires-Dist: seaborn; extra == "all"
+Requires-Dist: thriftpy2; extra == "all"
+Requires-Dist: redis-py-cluster; extra == "all"
+Requires-Dist: pymongo; extra == "all"
+Requires-Dist: pretty_errors; extra == "all"
+Requires-Dist: sse_starlette; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: cachetools; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: schedule; extra == "all"
 
 [![Downloads](http://pepy.tech/badge/meutils)](http://pepy.tech/project/meutils)
 

{meutils-2024.12.5.13.7.4/MeUtils.egg-info → meutils-2024.12.5.17.36.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: MeUtils
-Version: 2024.12.5.13.7.4
+Version: 2024.12.5.17.36.18
 Summary: description
 Home-page: https://github.com/yuanjie-ai/MeUtils
 Author: yuanjie
@@ -106,42 +106,42 @@ Requires-Dist: pandas-profiling[notebook]; extra == "pd"
 Requires-Dist: pandas_summary; extra == "pd"
 Requires-Dist: polars; extra == "pd"
 Provides-Extra: all
-Requires-Dist: thefuck; extra == "all"
-Requires-Dist: sqlalchemy; extra == "all"
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: fastapi; extra == "all"
-Requires-Dist: gunicorn; extra == "all"
-Requires-Dist: redis-py-cluster; extra == "all"
-Requires-Dist: pymysql; extra == "all"
-Requires-Dist: pymupd; extra == "all"
-Requires-Dist: pymongo; extra == "all"
-Requires-Dist: pandas-profiling[notebook]; extra == "all"
-Requires-Dist: fastapi[all]; extra == "all"
-Requires-Dist: thriftpy2; extra == "all"
-Requires-Dist: pymilvus; extra == "all"
-Requires-Dist: pandas_summary; extra == "all"
-Requires-Dist: jieba; extra == "all"
-Requires-Dist: jinja2; extra == "all"
-Requires-Dist: langchain; extra == "all"
-Requires-Dist: polars; extra == "all"
-Requires-Dist: streamlit; extra == "all"
 Requires-Dist: openai; extra == "all"
 Requires-Dist: geopy; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: faiss-cpu; extra == "all"
+Requires-Dist: pandas-profiling[notebook]; extra == "all"
+Requires-Dist: pymilvus; extra == "all"
 Requires-Dist: iteration_utilities; extra == "all"
+Requires-Dist: pymupd; extra == "all"
+Requires-Dist: thefuck; extra == "all"
+Requires-Dist: uvicorn; extra == "all"
+Requires-Dist: faiss-cpu; extra == "all"
+Requires-Dist: jmespath; extra == "all"
+Requires-Dist: langchain; extra == "all"
+Requires-Dist: sqlalchemy; extra == "all"
 Requires-Dist: missingno; extra == "all"
+Requires-Dist: gunicorn; extra == "all"
+Requires-Dist: polars; extra == "all"
+Requires-Dist: jieba; extra == "all"
+Requires-Dist: fastapi[all]; extra == "all"
 Requires-Dist: asyncmy; extra == "all"
-Requires-Dist: cachetools; extra == "all"
-Requires-Dist: sse_starlette; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: streamlit; extra == "all"
 Requires-Dist: reportlab; extra == "all"
+Requires-Dist: fastapi; extra == "all"
 Requires-Dist: dataframe_image; extra == "all"
+Requires-Dist: pandas_summary; extra == "all"
+Requires-Dist: pymysql; extra == "all"
 Requires-Dist: simplejson; extra == "all"
-Requires-Dist: schedule; extra == "all"
-Requires-Dist: pretty_errors; extra == "all"
-Requires-Dist: jmespath; extra == "all"
-Requires-Dist: uvicorn; extra == "all"
 Requires-Dist: seaborn; extra == "all"
+Requires-Dist: thriftpy2; extra == "all"
+Requires-Dist: redis-py-cluster; extra == "all"
+Requires-Dist: pymongo; extra == "all"
+Requires-Dist: pretty_errors; extra == "all"
+Requires-Dist: sse_starlette; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: cachetools; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: schedule; extra == "all"
 
 [![Downloads](http://pepy.tech/badge/meutils)](http://pepy.tech/project/meutils)
 

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18}/meutils/apis/hunyuan/image_tools.py

@@ -10,7 +10,7 @@
 import json_repair
 
 from meutils.pipe import *
-from meutils.decorators.retry import retrying
+from meutils.decorators.retry import retrying, IgnoredRetryException
 from meutils.schemas.yuanbao_types import FEISHU_URL, YUANBAO_BASE_URL
 from meutils.schemas.image_types import HunyuanImageProcessRequest
 
@@ -18,7 +18,7 @@ from meutils.config_utils.lark_utils import get_next_token_for_polling
 from meutils.io.files_utils import to_url
 
 
-@retrying(min=3)
+@retrying(min=3, ignored_exception_types=(IgnoredRetryException,))
 async def image_process(request: HunyuanImageProcessRequest, token: Optional[str] = None):
     token = token or await get_next_token_for_polling(FEISHU_URL)
 
@@ -39,10 +39,15 @@ async def image_process(request: HunyuanImageProcessRequest, token: Optional[str
         response.raise_for_status()
         logger.debug(response.text)
 
-        data: dict = json_repair.repair_json(response.text.replace(r'\u0026', '&'), return_objects=True)[-2]
-        if any(data["code"] == code for code in {"429"}):
+        data = json_repair.repair_json(response.text.replace(r'\u0026', '&'), return_objects=True)[-2]
+
+        # todo: 错误处理
+        if isinstance(data, dict) and any(data["code"] == code for code in {"429"}):
             Exception("重试")
 
+        elif any(i in response.text for i in {"当前图片没有检测到水印"}):  # 跳过重试并返回原始错误
+            raise IgnoredRetryException(data)
+
         data = {
             "data": [
                 {
@@ -61,6 +66,7 @@ if __name__ == '__main__':
 
     with timer():
         image = "https://sfile.chatglm.cn/chatglm4/3dcb1cc2-22ad-420b-9d16-dc71dffc02b2.png"
-        request = ImageProcessRequest(image=image, task='clarity')
+        image = "https://oss.ffire.cc/files/kling_watermark.png"
+        request = HunyuanImageProcessRequest(image=image, task='removewatermark')
 
         arun(image_process(request))

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18}/meutils/apis/proxy/ips.py

@@ -36,12 +36,15 @@ async def get_one_proxy(token: Optional[str] = None, feishu_url: str = "", from_
     else:
         # 随机获取ip: 如果从redis就是循环队列 check_proxy
         ip = await get_next_token_for_polling(feishu_url, from_redis=from_redis)
+    # https://github.com/encode/httpx/pull/2879
+    # proxies = {
+    #     "http://": f"http://{ip}",  # 自建 38443
+    #     "https://": f"http://{ip}",
+    # }
 
-    proxies = {
-        "http://": f"http://{ip}",  # 自建 38443
-        "https://": f"http://{ip}",
-    }
-    return proxies
+    proxy = f"http://{ip}"
+
+    return proxy
 
 
 def checkip():  # 获取本地ip
@@ -154,12 +157,11 @@ if __name__ == '__main__':
     #
     # logger.debug(resp.text)
     ip = "http://proxy.chatfire.cn"
-    ip="http://chatfire:chatfire@110.42.51.201:38443"
-    ip="http://154.9.253.9:38443"
+    ip = "http://chatfire:chatfire@110.42.51.201:38443"
+    ip = "http://154.9.253.9:38443"
     ip = "http://154.9.252.28:38443"
-    ip="http://110.42.51.223:38443"
-    ip="http://154.40.54.76:38443"
-
+    ip = "http://110.42.51.223:38443"
+    ip = "http://154.40.54.76:38443"
 
     proxies = {'http://': ip, 'https://': ip}
 

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18}/meutils/apis/textin.py

@@ -9,6 +9,8 @@
 # @Description  : todo: 重构 https://tools.textin.com/
 # https://www.textin.com/console/recognition/robot_enhance?service=watermark-remove
 
+# proxies 取消了 https://github.com/encode/httpx/pull/2879
+
 from meutils.pipe import *
 from meutils.decorators.retry import retrying
 from meutils.notice.feishu import send_message
@@ -21,14 +23,7 @@ ua = UserAgent()
 
 BASE_URL = "https://api.textin.com/home"
 
-ip = "110.42.51.201"
-ip = "http://proxy.chatfire.cn/"
-proxies = {
-    "http://": ip,  # 自建 8443
-    "https://": ip,
-}
-
-
+proxy = "http://110.42.51.201:38443"
 
 
 @alru_cache(ttl=3600 * 24)
@@ -72,27 +67,32 @@ async def document_process(data: bytes,
                 logger.debug(response.status_code)
                 # logger.info(response.json())  # {'msg': '今日请求超过限制次数', 'code': 431}
 
-                if response.is_success:
-                    _ = response.json()
+                _ = response.json()
+
+                raise Exception("更新请求头或者加代理")
+
+                # if '今日请求超过限制次数' in str(_):
+                #     # logger.info(response.json())
+                #     raise Exception("更新请求头或者加代理")
+                # else:
+                #     break
 
-                    if '今日请求超过限制次数' in str(_):
-                        # logger.info(response.json())
-                        raise Exception("更新请求头或者加代理")
-                    else:
-                        break
 
         except Exception as e:  # 加代理
             logger.error(e)
 
             request_kwargs = {
-                "proxies": await get_one_proxy(),
-                # "proxies": proxies,
+                # "proxy": await get_one_proxy(),
+                "proxy": proxy,
+
             }
 
-    if response_format == "url" and service in {"watermark-remove", "dewarp"}:
+    if response_format == "url" and service in {"watermark-remove", "crop_enhance_image"}:
         if "data" not in _:
             send_message(_, title=__name__)
 
+        # _ => {'msg': '今日请求超过限制次数', 'code': 431}
+
         url = await to_url(_['data']['result']['image'])
         _ = {"data": [{"url": url}]}
 
@@ -119,15 +119,16 @@ if __name__ == '__main__':
 
     service = Purpose.watermark_remove.value
 
-
     # service = "pdf_to_markdown"
 
-    service = 'dewarp'
+    service = 'bill_recognize_v2'
+
 
     async def main(n=1):
         for i in tqdm(range(n)):
             try:
                 _ = await document_process(data, service=service)
+                logger.debug(bjson(_))
             except Exception as e:
                 logger.error(f"{i}: {e}")
                 break

meutils-2024.12.5.17.36.18/meutils/data/VERSION

@@ -0,0 +1 @@
+2024.12.05.17.36.18

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18}/meutils/data/oneapi/NOTICE.md

@@ -2,6 +2,10 @@
 
 ---
 
+## 2024-12-05
+- 新增模型gpt-4-plus/gpt-4o-plus按倍率计算
+  > OpenAI-plus会员 逆向工程
+  
 ## 2024-11-29
 - 新增推理模型 
   - Qwen/QwQ-32B-Preview

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18}/meutils/decorators/retry.py

@@ -11,12 +11,17 @@
 from meutils.pipe import *
 from tenacity import retry, stop_after_attempt, before_sleep_log
 from tenacity import wait_fixed, wait_exponential, wait_incrementing, wait_exponential_jitter  # Wait strategy
-from tenacity import retry_if_result, retry_if_exception, retry_if_exception_type, RetryCallState  # 重试策略
+from tenacity import retry_if_result, RetryCallState  # 重试策略
+from tenacity import retry_if_exception, retry_if_exception_type, retry_if_not_exception_type
 
 # 不兼容异步
 from meutils.notice.feishu import send_message
 
 
+class IgnoredRetryException(Exception):
+    pass
+
+
 def default_retry_error_callback(retry_state: RetryCallState, title: Optional[str] = None):
     """return the result of the last call attempt"""
     # logger.debug(f"最后一次重试仍然失败调用的函数: {retry_state.outcome}")
@@ -50,7 +55,11 @@ def retrying(
         max: int = 100000,
         retry_error_callback: Optional[Callable[[RetryCallState], Any]] = None,
         predicate: Callable[[Any], bool] = lambda r: False,
-        title: Optional[str] = None
+        title: Optional[str] = None,
+        ignored_exception_types: Optional[typing.Union[
+            typing.Type[BaseException],
+            typing.Tuple[typing.Type[BaseException], ...],
+        ]] = None,  # 该错误类型不重试
 ):
     import logging
 
@@ -65,8 +74,9 @@ def retrying(
                     & retry_if_exception(not_in_status_codes)
             )
             | retry_if_result(predicate)
-
     )
+    if ignored_exception_types:
+        retry_reasons = retry_reasons & retry_if_not_exception_type(ignored_exception_types)
 
     _retry_decorator = retry(
         reraise=True,
@@ -166,13 +176,12 @@ if __name__ == '__main__':
     #     # print(wait_incrementing(100, -10)(retry_state))
     #     print(wait_exponential(1, exp_base=2)(retry_state))
 
-    @retrying(3)
-    async def f():
-        logger.debug("retry")
-        pass
-        # 1 / 0
-        raise Exception('xx')
-
+    # @retrying(3)
+    # async def f():
+    #     logger.debug("retry")
+    #     pass
+    #     # 1 / 0
+    #     raise Exception('xx')
 
     # try:
     #     f()
@@ -181,17 +190,43 @@ if __name__ == '__main__':
     #     print(e)
     #     print(type(e))
 
-    @retrying(3)
-    def f():
-    # async def f():
-        logger.debug("retry")
-        for i in range(10):
-            yield ""
-            if i == 8:
-                raise Exception('xx')
+    # @retrying(3)
+    # def f():
+    #     # async def f():
+    #     logger.debug("retry")
+    #     for i in range(10):
+    #         yield ""
+    #         if i == 8:
+    #             raise Exception('xx')
 
+    # arun(f())
 
+    # f()
+
+    # 示例使用
+    class CustomError1(Exception):
+        pass
+
+
+    class CustomError2(Exception):
+        pass
+
+
+    @retrying(
+        max_retries=3,
+        ignored_exception_types=(CustomError1, IgnoredRetryException)
+    )
+    def my_function():
+        logger.debug(1)
+        # 模拟抛出异常
+        raise IgnoredRetryException("This is a custom error")
+
+
+    def main():
+        try:
+            my_function()
+        except Exception as e:
+            print(f"Operation failed after all retries: {e}")
 
-    # arun(f())
 
-    f()
+    main()

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18}/meutils/schemas/image_types.py

@@ -385,11 +385,24 @@ class HunyuanImageProcessRequest(ImageProcessRequest):
 
 
 TEXTIN_TASKS = {
+    # 文本解析
     "image-to-markdown": "pdf_to_markdown",
     "pdf-to-markdown": "pdf_to_markdown",
+    "textin-pdf-to-markdown": "pdf_to_markdown",
+
+    # 去水印
     'textin-removewatermark': 'watermark-remove',
+
+    'textin-ocr': 'text_recognize_3d1',
     'text-recognize': 'text_recognize_3d1',
-    'dewarp': 'dewarp'
+
+    # 图片提取表格
+    "textin-table": "table",
+
+    "textin-crop-enhance": "crop_enhance_image",
+
+    # 票据识别
+    "textin-bill-recognize": "bill_recognize_v2",
 }
 
 

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18}/meutils/schemas/oneapi/common.py

@@ -176,7 +176,6 @@ MODEL_PRICE = {
     "gpt-4-all": 0.1,
     "gpt-4o-all": 0.1,
 
-    "gpt-4-mobile": 0.1,
     "gpt-4-gizmo-*": 0.1,
     "advanced-voice": 1,
 
@@ -236,6 +235,10 @@ MODEL_PRICE = {
 }
 
 MODEL_RATIO = {
+    # 智能体
+    "gpt-4-plus": 2.5,
+    "gpt-4o-plus": 2.5,
+
     # embedding & rerank
     "rerank-multilingual-v2.0": 0.1,
     "rerank-multilingual-v3.0": 0.1,
@@ -550,6 +553,10 @@ MODEL_RATIO = {
 }
 
 COMPLETION_RATIO = {
+    # 智能体
+    "gpt-4-plus": 5,
+    "gpt-4o-plus": 5,
+
     "grok": 3,
     "grok-beta": 3,
     "x-ai/grok-beta": 3,

meutils-2024.12.5.13.7.4/meutils/data/VERSION

@@ -1 +0,0 @@
-2024.12.05.13.07.04

{meutils-2024.12.5.13.7.4 → meutils-2024.12.5.17.36.18}/MeUtils.egg-info/requires.txt

@@ -43,42 +43,42 @@ openai
 langchain
 
 [all]
-thefuck
-sqlalchemy
-filetype
-fastapi
-gunicorn
-redis-py-cluster
-pymysql
-pymupd
-pymongo
-pandas-profiling[notebook]
-fastapi[all]
-thriftpy2
-pymilvus
-pandas_summary
-jieba
-jinja2
-langchain
-polars
-streamlit
 openai
 geopy
-pyarrow
-faiss-cpu
+pandas-profiling[notebook]
+pymilvus
 iteration_utilities
+pymupd
+thefuck
+uvicorn
+faiss-cpu
+jmespath
+langchain
+sqlalchemy
 missingno
+gunicorn
+polars
+jieba
+fastapi[all]
 asyncmy
-cachetools
-sse_starlette
+pyarrow
+streamlit
 reportlab
+fastapi
 dataframe_image
+pandas_summary
+pymysql
 simplejson
-schedule
-pretty_errors
-jmespath
-uvicorn
 seaborn
+thriftpy2
+redis-py-cluster
+pymongo
+pretty_errors
+sse_starlette
+jinja2
+cachetools
+filetype
+schedule
 
 [ann]
 pymilvus