MeUtils 2024.3.4.13.4.45__py3-none-any.whl → 2025.1.16.17.15.52__py3-none-any.whl

meutils/playwright_utils/common.py

@@ -23,16 +23,19 @@ async def get_new_browser(headless: bool = True):
 
 
 @alru_cache(ttl=3600)
-async def get_new_page(headless: bool = True, iphone: Optional[str] = None):
+async def get_new_page(headless: bool = True, iphone: Optional[str] = None, browser_type: str = "webkit"):
     """
 
     :param headless:
     :param iphone: 'iPhone 13'
-    :return:
+    :return: WebKit
     """
 
     playwright = await async_playwright().start()  # 打开未关闭
-    browser = await playwright.chromium.launch(headless=headless)
+    if browser_type == "webkit":
+        browser = await playwright.webkit.launch(headless=headless)
+    else:
+        browser = await playwright.chromium.launch(headless=headless)  # todo: close
 
     logger.info("初始化浏览器-页面")
 
@@ -44,7 +47,7 @@ async def get_new_page(headless: bool = True, iphone: Optional[str] = None):
 
 
 # 监听所有的请求和响应
-async def open():
+async def open(headless):
     async with async_playwright() as p:
         browser = await p.chromium.launch(headless=headless, args=["--start-maximized"])
         context = await browser.new_context(no_viewport=True)
@@ -66,18 +69,23 @@ async def my_request(request):
     print(await request.all_headers())
 
 
-async def monitor_requests(request: Request):
-    if request.method == "POST":
+async def _monitor_requests(request: Request, method):
+    if request.method == method:
         # _ = await request.post_data()
 
         _ = request.post_data_json
 
-        logger.debug(_)
+        logger.debug(f"{method}: \n {request.url}")
         return _
 
 
-async def monitor_responses(response: Response):
-    if response:
-        _ = await response.all_headers()
-        logger.debug(_)
-        return _
+async def monitor_requests_get(request: Request, method='GET'):
+    return _monitor_requests(request, method)
+
+
+async def monitor_requests_put(request: Request, method='PUT'):
+    return _monitor_requests(request, method)
+
+
+async def monitor_requests_post(request: Request, method='POST'):
+    return _monitor_requests(request, method)

meutils/plots/common.py

@@ -10,39 +10,40 @@
 
 from meutils.pipe import *
 
-
-def plot_lift_curve(predictions, labels, threshold_list, cut_point=100):
-    base = len([x for x in labels if x == 1]) / len(labels)
-    predictions_labels = list(zip(predictions, labels))
-    lift_values = []
-
-    x_axis_range = np.linspace(0, 1, cut_point)
-    x_axis_valid = []
-    for i in x_axis_range:
-        hit_data = [x[1] for x in predictions_labels if x[0] > i]
-        if hit_data:  # 避免为空
-            bad_hit = [x for x in hit_data if x == 1]
-            precision = len(bad_hit) / len(hit_data)
-            lift_value = precision / base
-            lift_values.append(lift_value)
-            x_axis_valid.append(i)
-
-    plt.plot(x_axis_valid, lift_values, color="blue")  # 提升线
-    plt.plot([0, 1], [1, 1], linestyle="-", color="darkorange", alpha=0.5, linewidth=2)  # base线
-
-    for threshold in threshold_list:
-        threshold_hit_data = [x[1] for x in predictions_labels if x[0] > threshold]
-        if threshold_hit_data:
-            threshold_bad_hit = [x for x in threshold_hit_data if x == 1]
-            threshold_precision = len(threshold_bad_hit) / len(threshold_hit_data)
-            threshold_lift_value = threshold_precision / base
-            plt.scatter([threshold], [threshold_lift_value], color="white", edgecolors="blue", s=20,
-                        label="threshold:{} lift:{}".format(threshold, round(threshold_lift_)), value, 2)))  # 阈值点
-            plt.plot([threshold, threshold], [0, 20], linestyle="--", color="black", alpha=0.2, linewidth=1)  # 阈值的纵轴
-            plt.text(threshold - 0.02, threshold_lift_value + 1, round(threshold_lift_value, 2))
-            plt.title("Lift plot")
-            plt.legend(loc=2, prop={"size": 9})
-            plt.grid()
-            plt.show()
+import matplotlib.pyplot as plt  #######
+#
+# def plot_lift_curve(predictions, labels, threshold_list, cut_point=100):
+#     base = len([x for x in labels if x == 1]) / len(labels)
+#     predictions_labels = list(zip(predictions, labels))
+#     lift_values = []
+#
+#     x_axis_range = np.linspace(0, 1, cut_point)
+#     x_axis_valid = []
+#     for i in x_axis_range:
+#         hit_data = [x[1] for x in predictions_labels if x[0] > i]
+#         if hit_data:  # 避免为空
+#             bad_hit = [x for x in hit_data if x == 1]
+#             precision = len(bad_hit) / len(hit_data)
+#             lift_value = precision / base
+#             lift_values.append(lift_value)
+#             x_axis_valid.append(i)
+#
+#     plt.plot(x_axis_valid, lift_values, color="blue")  # 提升线
+#     plt.plot([0, 1], [1, 1], linestyle="-", color="darkorange", alpha=0.5, linewidth=2)  # base线
+#
+#     for threshold in threshold_list:
+#         threshold_hit_data = [x[1] for x in predictions_labels if x[0] > threshold]
+#         if threshold_hit_data:
+#             threshold_bad_hit = [x for x in threshold_hit_data if x == 1]
+#             threshold_precision = len(threshold_bad_hit) / len(threshold_hit_data)
+#             threshold_lift_value = threshold_precision / base
+#             plt.scatter([threshold], [threshold_lift_value], color="white", edgecolors="blue", s=20,
+#                         label="threshold:{} lift:{}".format(threshold, round(threshold_lift_value)), value, 2)))  # 阈值点
+#             plt.plot([threshold, threshold], [0, 20], linestyle="--", color="black", alpha=0.2, linewidth=1)  # 阈值的纵轴
+#             plt.text(threshold - 0.02, threshold_lift_value + 1, round(threshold_lift_value, 2))
+#             plt.title("Lift plot")
+#             plt.legend(loc=2, prop={"size": 9})
+#             plt.grid()
+#             plt.show()
 
 

meutils/queues/demo.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : demo
+# @Time         : 2024/3/22 15:55
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+import asyncio
+import random
+import time
+
+
+async def producer(queue: asyncio.Queue, name: str):
+    for i in range(10):
+        # 模拟数据生成需要的时间
+        await asyncio.sleep(random.random())
+        item = (name, i)
+        # 将数据放入队列
+        await queue.put(item)
+        print(f'生产者 {name} 生产了 {item}')
+    # 生产完毕后发送结束信号
+    await queue.put(None)
+
+
+async def consumer(queue: asyncio.Queue, name: str):
+    while True:
+        # 从队列中获取数据
+        item = await queue.get()
+        if item is None:
+            # 生产者发送了结束信号，终止循环
+            break
+        # 模拟数据处理需要的时间
+        await asyncio.sleep(random.random())
+        print(f'消费者 {name} 消费了 {item}')
+        queue.task_done()
+
+
+async def main():
+    queue = asyncio.Queue()
+
+    # 创建生产者和消费者协程
+    producers = [producer(queue, f'P{i}') for i in range(2)]  # 两个生产者
+    consumers = [consumer(queue, f'C{i}') for i in range(3)]  # 三个消费者
+
+    # 启动所有协程
+    await asyncio.gather(*producers, *consumers)
+
+    # 等待队列被处理完毕
+    await queue.join()
+
+
+asyncio.run(main())

meutils/queues/smooth_queue.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : uniform_queue
+# @Time         : 2023/12/6 17:59
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import time
+
+from meutils.pipe import *
+
+
+class SmoothQueue(object):
+    def __init__(self, end=None):
+        self.queue = asyncio.Queue()
+        self.end = end
+
+    async def producer(self, generator):  # 生产者将任务放入队列
+        if inspect.isasyncgen(generator) or hasattr(generator, '__anext__'):  # if hasattr(future, '__anext__'):
+            async for i in generator:
+                await self.queue.put(i)
+        else:
+            for i in generator:
+                await self.queue.put(i)
+
+        # 生产结束
+        await self.queue.put(self.end)  # 使用 None 作为生产结束的标志
+
+    async def consumer(self, generator, delay: float = 1, debug: bool = False):
+        producer_task = asyncio.create_task(self.producer(generator))  # 是否合理？
+
+        for i in range(100000):
+            item = await self.queue.get()  # await asyncio.wait_for(self.queue.get(), timeout=2)
+
+            if debug: logger.debug(item)
+
+            if item == self.end: break  # 生产者已经结束生产
+
+            yield item  # 不包含【结束值】
+
+            # 生成器等待，模拟处理速度，这里处理速度是均匀的
+            if i > 5: await asyncio.sleep(delay)  # 调整此处的等待时间来控制消费的速度
+
+        await producer_task
+
+    async def main(self, generator):
+        producer_task = asyncio.create_task(self.producer(generator))
+
+        async for item in self.consumer(generator):
+            print(f'Consumed {item}')
+
+        await producer_task
+
+
+if __name__ == '__main__':
+    async def gen():
+        await asyncio.sleep(10)
+        for i in range(10):
+            logger.debug(time.ctime())
+            # 模拟不均匀的生产速度
+            await asyncio.sleep(random.random())
+            print(f'{time.ctime()}: Produced {i}')
+            yield i
+
+
+    sq = SmoothQueue()
+
+
+    async def main():
+        iters = sq.consumer(gen(), delay=2)
+        for i in range(10):
+            await asyncio.sleep(1)
+            logger.debug(f"{time.ctime()}: {i}")
+
+        async for item in iters:
+            print(f'Consumed {item}')
+
+        # await producer_task
+
+
+    arun(main())
+
+    # import asyncio
+    # import random
+    #
+    #
+    # async def producer(queue):
+    #     for i in range(10):
+    #         # 模拟不均匀的生产速度
+    #         await asyncio.sleep(random.random())
+    #         await queue.put(i)
+    #         print(f'Produced {i}')
+    #     # 生产结束
+    #     await queue.put(None)  # 使用 None 作为生产结束的标志
+    #
+    #
+    # async def consumer(queue):
+    #     while True:
+    #         item = await queue.get()
+    #         if item is None:
+    #             # 生产者已经结束生产
+    #             break
+    #         # 生成器等待，模拟处理速度，这里处理速度是均匀的
+    #         await asyncio.sleep(1)  # 调整此处的等待时间来控制消费的速度
+    #         yield item
+    #
+    #
+    # async def main():
+    #     queue = asyncio.Queue()
+    #     producer_task = asyncio.create_task(producer(queue))
+    #
+    #     async for item in consumer(queue):
+    #         print(f'Consumed {item}')
+    #
+    #     await producer_task
+    #
+    #
+    # asyncio.run(main())

meutils/queues/uniform_queue.py

@@ -7,6 +7,7 @@
 # @WeChat       : meutils
 # @Software     : PyCharm
 # @Description  :
+import time
 
 from meutils.pipe import *
 from queue import Queue, Empty
@@ -16,12 +17,13 @@ class UniformQueue(object):
 
     def __init__(self, generator: Generator):
         self.queue = Queue()
+
         self.producer(generator)
 
     def smooth(self, interval: float = 0.05, timeout: float = 30, break_fn: Callable = lambda item: item is None):
         return self.consumer(interval, timeout, break_fn)
 
-    def consumer(self, interval: float = 0.05, timeout: float = 30, break_fn: Callable = lambda item: item is None):
+    def consumer(self, interval: float = 0.066, timeout: float = 10, break_fn: Callable = lambda item: item is None):
         """
 
         :param interval:

meutils/request_utils/__init__.py

@@ -20,7 +20,7 @@ from meutils.pipe import *
        reraise=True)
 def request4retry(url, method='get', return_json=True, timeout=3, encoding=None, **kwargs):
     """None or {}"""
-    logger.info("Try")
+    # logger.info("Try")
 
     _kwargs = {}
     _kwargs['timeout'] = timeout
@@ -32,6 +32,8 @@ def request4retry(url, method='get', return_json=True, timeout=3, encoding=None,
     r.encoding = encoding if encoding else r.apparent_encoding
 
     return r.json() if return_json else r
+
+
 # def _create_retry_decorator(embeddings: OpenAIEmbeddings) -> Callable[[Any], Any]:
 #     import openai
 #
@@ -89,5 +91,27 @@ def is_url_alive(url):
     return True if request.status_code == 200 else False
 
 
+async def create_request(
+        url,
+        request,
+        token: str = None,
+        headers: Optional[dict] = None,
+        post_kv: Optional[dict] = None
+):
+    token = token or 'This is a token'
+    payload = request if isinstance(request, dict) else request.model_dump(exclude_none=True)
+
+    headers = {
+        'Authorization': f'Bearer {token}',
+        'Cookie': token,
+        **(headers or {})
+    }
+    async with httpx.AsyncClient(headers=headers, timeout=60) as client:
+        response = await client.post(url=url, json=payload, **(post_kv or {}))
+        if response.is_success:
+            return response.json()
+        response.raise_for_status()
+
+
 if __name__ == '__main__':
-    print(request4retry('xx'))
+    print(request4retry('xx'))

meutils/request_utils/ark.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : ark
+# @Time         : 2024/12/31 13:44
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  :
+import json
+import os
+
+from meutils.pipe import *
+import volcenginesdkcore
+from volcenginesdkcore.signv4 import SignerV4
+import volcenginesdkark
+
+# sign(path, method, headers, body, query, ak, sk, region, service)
+
+headers = {}
+payload = {
+    "Prompt": "关于星空的歌",
+    "Genre": "R&B/Soul",
+    "Mood": "Dynamic/Energetic",
+    "Gender": "Male",
+    "ModelVersion": "v4.0"
+}
+params = {"Action": "GenLyrics", "Version": "2024-08-12"}
+SignerV4.sign(
+    path="/",
+    method="POST",
+    query=params,
+    body=json.dumps(payload),
+    headers=headers,
+    ak=os.getenv("ARK_ACCESS_KEY"),
+    sk=os.getenv("ARK_SECRET_ACCESS_KEY"),
+    region="cn-beijing",
+    service="imagination"
+)
+
+print(headers)
+
+s = ''
+for k, v in headers.items():
+    s += f"{k}:{v}\n"
+
+print(s)

meutils/request_utils/crawler.py

@@ -33,14 +33,43 @@ if __name__ == '__main__':
     from meutils.pipe import *
     url = "https://top.baidu.com/board?tab=realtime"
 
-    _ = Crawler(url).xpath('//*[@id="sanRoot"]/main/div[2]/div/div[2]/div[*]/div[2]/a/div[1]//text()')
-    print("\n".join(_))
+    #
+    #
+    # _ = Crawler(url).xpath('//*[@id="sanRoot"]/main/div[2]/div/div[2]/div[*]/div[2]/a/div[1]//text()')
+    # print("\n".join(_))
+
+
 
     # from meutils.request_utils.crawler import Crawler
     #
     # url = 'http://10.2.41.63:8004/bert_models/bert-base-uncased/'
     # Crawler(url).xpath('//@href')
 
-    c = Crawler('https://www.maoyan.com/board?offset=0') # 猫眼top10
-    _ = c.xpath('//*[@id="app"]/div/div/div/dl/dd[*]/div/div/div[1]//text()') | xgroup(7)
-    top10 = pd.DataFrame(_)
+    # c = Crawler('https://www.maoyan.com/board?offset=0') # 猫眼top10
+    # _ = c.xpath('//*[@id="app"]/div/div/div/dl/dd[*]/div/div/div[1]//text()') | xgroup(7)
+    # top10 = pd.DataFrame(_)
+
+    # from meutils.request_utils.crawler import Crawler
+    # w = "宝能公馆1288"
+    # url = f"https://m.fang.com/fangjia/sh_list_pinggu/?keyword={w}&city=sh&r=0.9648271001521231"
+    #
+    # print(Crawler(url).xpath('//*[@id="houselist"]/li/a//text()'))
+
+    url  = "https://chat.tune.app/?id=7f268d94-d2d4-4bd4-a732-f196aa20dceb"
+
+    # print(Crawler(url).xpath('//script//text()'))
+
+    html_content = httpx.get(url).text
+
+
+    # 正则表达式匹配以 "/_next/static/chunks/7116-" 开头的 JS 文件
+    pattern = r'(/_next/static/chunks/7116-[^"]+\.js)'
+
+    # 使用 re.findall() 找到所有匹配项
+    matches = re.findall(pattern, html_content)
+
+    "/_next/static/chunks/7116-aed224a0caaab94c.js"
+
+    # 打印结果
+    for match in matches:
+        print(match)

meutils/request_utils/jwt_utils/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : __init__.py
+# @Time         : 2024/10/28 20:57
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *

meutils/request_utils/jwt_utils/common.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Project      : AI.  @by PyCharm
+# @File         : common
+# @Time         : 2024/10/28 20:57
+# @Author       : betterme
+# @WeChat       : meutils
+# @Software     : PyCharm
+# @Description  : 
+
+from meutils.pipe import *
+
+import jwt
+import time
+import datetime
+
+# Header and payload
+header = {
+    "alg": "HS256",
+    "typ": "JWT"
+}
+
+payload = {
+    "exp": int(time.time()) + 3600,  # 1 hour from now
+    "user": {
+        "id": "302833867771949058",
+        "name": "me better",
+        "avatar": "https://lh3.googleusercontent.com/a/ACg8ocIgSSChs1D4sTj1STk7PsTm7y53JDX99o8BxpZcV6560AJbRg=s96-c",
+        "deviceID": ""
+    }
+}
+
+# Your secret key
+secret = ""
+
+
+# Create the JWT
+token = jwt.encode(payload, secret, algorithm="HS256", headers=header)
+
+print(token)
+
+jwt.decode(token, secret, algorithms=["HS256"])