pymecli 0.2.5__py3-none-any.whl

utils/logger.py

@@ -0,0 +1,26 @@
+import logging
+
+
+def get_logger(name, level=logging.DEBUG, fmt=logging.Formatter("%(message)s")):
+    root_logger = logging.getLogger()
+    root_level = root_logger.level if root_logger.level != 0 else logging.NOTSET
+
+    # 如果 root 的 level 已设置且与传入 level 相同，则直接返回，不做本地修改
+    if root_level != logging.NOTSET and level == root_level:
+        return logging.getLogger(name)
+
+    # 否则按传入 level 配置当前模块的 logger
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+
+    # 如果没有 StreamHandler，则添加一个，保证有输出目的地
+    if not any(isinstance(h, logging.StreamHandler) for h in logger.handlers):
+        handler = logging.StreamHandler()
+        handler.setLevel(level)
+        # fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        handler.setFormatter(fmt)
+        logger.addHandler(handler)
+
+    # 禁止传播到父 logger，避免被父 handler 过滤或重复输出
+    logger.propagate = False
+    return logger

utils/mysql.py

@@ -0,0 +1,79 @@
+import os
+
+import pandas as pd
+from dotenv import load_dotenv
+from sqlalchemy import Engine, create_engine, text
+
+from utils.pd import deduplicated
+
+
+def get_database_engine(env_path: str) -> Engine:
+    """创建数据库引擎"""
+    load_dotenv(env_path)
+    host = os.getenv("UNI_CLI_MYSQL_HOST")
+    port = os.getenv("UNI_CLI_MYSQL_PORT")
+    user = os.getenv("UNI_CLI_MYSQL_USER")
+    password = os.getenv("UNI_CLI_MYSQL_PASSWORD")
+    database = os.getenv("UNI_CLI_MYSQL_DATABASE")
+
+    engine = create_engine(
+        f"mysql+pymysql://{user}:{password}@{host}:{port}/{database}"
+    )
+
+    try:
+        with engine.connect() as connection:
+            connection.execute(text("SELECT 1"))
+    except Exception as e:
+        print(f"数据库连接失败: {str(e)}")
+        raise
+
+    return engine
+
+
+def mysql_to_csv(
+    engine: Engine,
+    csv_path: str,
+    table: str,
+    query: str,
+    update_status: int,
+    d_column_names: list[str],
+    pd_dtype: dict | None = None,
+    del_column_names: list[str] = ["id"],
+) -> int:
+    # 查询数据
+    data_frame = pd.read_sql(query, engine, dtype=pd_dtype)
+    # 提取 'id' 列
+    ids = data_frame["id"].tolist()
+    # 删除 'id' 列
+    data_frame = data_frame.drop(columns=del_column_names)
+
+    # 根据 'open_at' 列降序排序
+    # data_frame = data_frame.sort_values(by="open_at", ascending=False)
+
+    # 将数据追加写入 CSV 文件
+    data_frame.to_csv(
+        csv_path,
+        mode="a",
+        header=not os.path.exists(csv_path),
+        index=False,
+        encoding="utf-8",
+    )
+    # csv去重,保留最后加入的数据
+    deduplicated(csv_path, d_column_names, "last", pd_dtype)
+
+    # 根据提取的 'id' 列更新数据库中 up_status 字段
+    if ids:
+        # 使用 text() 构建查询时，确保 :ids 是一个列表
+        update_query = text(
+            f"UPDATE {table} SET up_status = :status WHERE id IN ({','.join(map(str, ids))});"
+        )
+        with engine.connect() as connection:
+            with connection.begin():
+                result = connection.execute(
+                    update_query,
+                    {"status": update_status},
+                )
+
+                return result.rowcount
+
+    return 0

utils/pd.py

@@ -0,0 +1,20 @@
+import os
+from typing import Literal
+
+import pandas as pd
+from pandas import DataFrame
+
+
+def deduplicated(
+    file_path: str,
+    column_names: list[str],
+    keep: Literal["first", "last"] = "last",
+    pd_dtype: dict | None = None,
+):
+    if os.path.exists(file_path):
+        existing_df: DataFrame = pd.read_csv(
+            file_path, encoding="utf-8", dtype=pd_dtype
+        )
+        existing_df.drop_duplicates(subset=column_names, keep=keep, inplace=True)
+        existing_df.to_csv(file_path, index=False, encoding="utf-8")
+        return existing_df

utils/sleep.py

@@ -0,0 +1,16 @@
+import time
+from datetime import datetime, timedelta
+
+
+def interruptible_sleep(seconds: float) -> bool:
+    """
+    可中断的休眠函数
+    :param seconds: 休眠秒数
+    """
+    end_time = datetime.now() + timedelta(seconds=seconds)
+    try:
+        while datetime.now() < end_time:
+            time.sleep(1)  # 每1秒检查一次，便于中断
+    except KeyboardInterrupt:
+        return False
+    return True

utils/text.py

@@ -0,0 +1,33 @@
+import unicodedata
+
+
+def normalize_text(text):
+    """
+    将全角字符转换为半角字符
+    """
+    normalized = ""
+    for char in text:
+        # 将全角字符转换为半角字符
+        normalized_char = unicodedata.normalize("NFKC", char)
+        normalized += normalized_char
+    return normalized
+
+
+def match_text(text: str, target_list: list[str]):
+    text = text.strip()
+    # text = str.replace(text, "I", "")
+    remaining_text = ""
+
+    if not text:
+        return None, remaining_text
+
+    for target_text in target_list:
+        if text.startswith(target_text):
+            len_target_text = len(target_text)
+            # if target_text == "JO":
+            #     len_target_text = 5
+            remaining_text = text[len_target_text:]
+
+            return target_text, remaining_text
+
+    return None, remaining_text

utils/toml.py

@@ -0,0 +1,6 @@
+import toml
+
+
+def read_toml(fp: str):
+    with open(fp, "r", encoding="utf-8") as f:
+        return toml.load(f)