qrpa 1.0.2__py3-none-any.whl → 1.0.8__py3-none-any.whl

qrpa/fun_file.py

@@ -0,0 +1,173 @@
+import os
+import threading
+import json
+
+file_lock = threading.Lock()  # 线程锁
+
+from datetime import date, datetime, timedelta, timezone
+
+from .fun_base import log
+
+def read_dict_from_file(file_path, cache_interval=3600 * 24 * 365 * 10):
+    """
+    从文件中读取字典。
+    如果文件的修改时间未超过一个小时，则返回字典；否则返回 None。
+
+    :param file_path: 文件路径
+    :return: 字典或 None
+    """
+    with file_lock:  # 使用锁保护文件操作
+        # 检查文件是否存在
+        if not os.path.exists(file_path):
+            return {}
+
+        # 获取文件的最后修改时间
+        modification_time = os.path.getmtime(file_path)
+        modification_time = datetime.fromtimestamp(modification_time)
+
+        # 获取当前时间
+        current_time = datetime.now()
+
+        interval = current_time - modification_time
+        log(f'缓存文件 {file_path} 缓存时长 {timedelta(seconds=int(cache_interval))} 已过时长 {interval}')
+
+        # 判断文件的修改时间是否超过一个小时
+        if interval <= timedelta(seconds=int(cache_interval)):
+            # 如果未超过一个小时，则读取文件内容
+            with open(file_path, "r", encoding='utf-8') as file:
+                return json.load(file)
+        else:
+            # 如果超过一个小时，则返回 None
+            return {}
+
+def write_dict_to_file(file_path, data):
+    """
+    将字典写入文件。
+
+    :param file_path: 文件路径
+    :param data: 要写入的字典
+    """
+    with file_lock:  # 使用锁保护文件操作
+        # 确保目标文件夹存在
+        dir_name = os.path.dirname(file_path)
+        if dir_name and not os.path.exists(dir_name):
+            os.makedirs(dir_name, exist_ok=True)  # 递归创建目录
+
+        with open(file_path, 'w', encoding='utf-8') as f:
+            # 使用 json.dump() 并设置 ensure_ascii=False
+            json.dump(data, f, ensure_ascii=False, indent=4)
+
+def read_dict_from_file_ex(file_path, key, cache_interval=3600 * 24 * 365 * 10, default='dict'):
+    """
+    从 JSON 文件中读取指定键的值。
+
+    :param file_path: JSON 文件路径
+    :param key: 要读取的键
+    :param default: 如果文件不存在、解析失败或键不存在时返回的默认值
+    :return: 对应键的值，或 default
+    """
+    with file_lock:  # 使用锁保护文件操作
+        if not os.path.exists(file_path):
+            return {} if default == 'dict' else []
+
+        # 获取文件的最后修改时间
+        modification_time = os.path.getmtime(file_path)
+        modification_time = datetime.fromtimestamp(modification_time)
+
+        # 获取当前时间
+        current_time = datetime.now()
+
+        interval = current_time - modification_time
+        log(f'缓存文件 {file_path} 缓存时长 {timedelta(seconds=cache_interval)} 已过时长 {interval}')
+
+        # 判断文件的修改时间是否超过一个小时
+        if interval <= timedelta(seconds=cache_interval):
+            # 如果未超过一个小时，则读取文件内容
+            with open(file_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+                return data.get(key, {})
+        else:
+            # 如果超过一个小时，则返回 None
+            return {} if default == 'dict' else []
+
+def write_dict_to_file_ex(file_path, data, update_keys=None):
+    """
+    将字典写入文件，可选择性地只更新指定键。
+
+    :param file_path: 文件路径
+    :param data: 要写入的字典数据
+    :param update_keys: 可选，需要更新的键列表。如果为None，则替换整个文件内容
+    """
+    with file_lock:  # 使用锁保护文件操作
+        # 确保目标文件夹存在
+        dir_name = os.path.dirname(file_path)
+        if dir_name and not os.path.exists(dir_name):
+            os.makedirs(dir_name, exist_ok=True)  # 递归创建目录
+
+        # 如果指定了update_keys，先读取现有数据然后合并
+        if update_keys is not None:
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    existing_data = json.load(f)
+            except (FileNotFoundError, json.JSONDecodeError):
+                existing_data = {}
+
+            # 只更新指定的键
+            for key in update_keys:
+                if key in data:
+                    existing_data[key] = data[key]
+            data = existing_data
+
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=4)
+
+######################################################################################################
+def getTaskStoreKey(key_id, store_name):
+    return f'{key_id}_{store_name}'
+
+def generate_progress_file(config, key_id):
+    return f'{config.auto_dir}/progress/progress_{key_id}.json'
+
+def get_progress_index_ex(config, task_key, store_name):
+    task_store_key = getTaskStoreKey(task_key, store_name)
+    progress_file = generate_progress_file(config, task_key)
+    dict = read_dict_from_file(progress_file)
+    if len(dict) > 0:
+        count = 0
+        for key, value in dict.items():
+            if key == task_store_key:
+                return count
+            count += 1
+    return len(dict)
+
+def get_progress_json_ex(config, task_key, store_name):
+    task_store_key = getTaskStoreKey(task_key, store_name)
+    progress_file = generate_progress_file(config, task_key)
+    dict = read_dict_from_file_ex(progress_file, task_store_key)
+    if len(dict) > 0:
+        return dict[0] == 1
+    else:
+        length = get_progress_index_ex(config, task_key, store_name)
+        write_dict_to_file_ex(progress_file, {task_store_key: [0, length + 1, datetime.now().strftime('%Y-%m-%d %H:%M:%S')]}, [task_store_key])
+    return False
+
+def done_progress_json_ex(config, task_key, store_name):
+    task_store_key = getTaskStoreKey(task_key, store_name)
+    progress_file = generate_progress_file(config, task_key)
+    length = get_progress_index_ex(config, task_key, store_name)
+    write_dict_to_file_ex(progress_file, {task_store_key: [1, length + 1, datetime.now().strftime('%Y-%m-%d %H:%M:%S')]}, [task_store_key])
+
+def check_progress_json_ex(config, task_key, just_store_username=None):
+    progress_file = generate_progress_file(config, task_key)
+    dict = read_dict_from_file(progress_file)
+    if len(dict) > 0:
+        for task_store_key, data_list in dict.items():
+            if just_store_username and len(just_store_username) > 0:
+                if all([store_username not in task_store_key for store_username in just_store_username]):
+                    continue
+            if 'run_' not in task_store_key and int(data_list[0]) == 0:
+                log(task_store_key, just_store_username)
+                return False
+    else:
+        log(f"进度文件不存在或为空: {progress_file}")
+    return True

qrpa/fun_win.py

@@ -0,0 +1,31 @@
+import os
+import win32com.client
+
+from .fun_base import log
+
+def find_software_install_path(app_keyword: str):
+    """从开始菜单或桌面查找指定软件的安装路径"""
+    possible_dirs = [
+        os.environ.get('PROGRAMDATA', '') + r'\Microsoft\Windows\Start Menu\Programs',
+        os.environ.get('APPDATA', '') + r'\Microsoft\Windows\Start Menu\Programs',
+        os.environ.get('USERPROFILE', '') + r'\Desktop',
+        os.environ.get('PUBLIC', '') + r'\Desktop'
+    ]
+
+    shell = win32com.client.Dispatch("WScript.Shell")
+
+    for base_dir in possible_dirs:
+        for root, _, files in os.walk(base_dir):
+            for file in files:
+                if file.lower().endswith('.lnk') and app_keyword.lower() in file.lower():
+                    lnk_path = os.path.join(root, file)
+                    try:
+                        shortcut = shell.CreateShortcut(lnk_path)
+                        target_path = shortcut.Targetpath
+                        if os.path.exists(target_path):
+                            return target_path
+                    except Exception as e:
+                        continue
+
+    log(f'未能查找到{str}安装位置')
+    return None

qrpa/shein_ziniao.py

@@ -0,0 +1,441 @@
+"""
+# 适用环境python3
+# 紫鸟浏览器自动化操作 - 面向对象重构版本
+"""
+import os
+import platform
+import shutil
+import time
+import traceback
+import uuid
+import json
+from concurrent.futures import ThreadPoolExecutor
+from typing import Literal, Optional, Callable, List, Dict, Any
+
+import requests
+import subprocess
+from playwright import sync_api
+from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
+# 使用 partial 或 lambda 固定部分参数
+from functools import partial
+
+from .fun_win import find_software_install_path
+from .fun_base import log, hostname, send_exception
+from .fun_file import check_progress_json_ex, get_progress_json_ex, done_progress_json_ex, write_dict_to_file_ex
+
+class ZiniaoClient:
+    """紫鸟客户端管理类"""
+
+    def __init__(self, config):
+        self.config = config
+        self.version = "v5"
+        self.socket_port = 16851
+        self.is_windows = platform.system() == 'Windows'
+        self.is_mac = platform.system() == 'Darwin'
+        self.client_path = self._get_client_path()
+        self.user_info = self._get_user_info()
+
+        if not self.is_windows and not self.is_mac:
+            raise RuntimeError("webdriver/cdp只支持windows和mac操作系统")
+
+    def _get_client_path(self) -> str:
+        """获取客户端路径"""
+        if self.is_windows:
+            ziniao = find_software_install_path('SuperBrowser')
+            if ziniao is None:
+                raise RuntimeError('未找到SuperBrowser安装路径')
+            return ziniao
+        else:
+            return 'ziniao'
+
+    def _get_user_info(self) -> Dict[str, str]:
+        """获取用户登录信息"""
+        return {
+            "company" : self.config.ziniao.company,
+            "username": self.config.ziniao.username,
+            "password": self.config.ziniao.password
+        }
+
+    def kill_process(self):
+        """杀紫鸟客户端进程"""
+        if self.version == "v5":
+            process_name = 'SuperBrowser.exe'
+        else:
+            process_name = 'ziniao.exe'
+
+        if self.is_windows:
+            os.system('taskkill /f /t /im ' + process_name)
+        elif self.is_mac:
+            os.system('killall ziniao')
+            time.sleep(3)
+
+    def start_browser(self):
+        """启动客户端"""
+        try:
+            if self.is_windows:
+                cmd = [self.client_path, '--run_type=web_driver', '--ipc_type=http', '--port=' + str(self.socket_port)]
+            elif self.is_mac:
+                cmd = ['open', '-a', self.client_path, '--args', '--run_type=web_driver', '--ipc_type=http',
+                       '--port=' + str(self.socket_port)]
+            else:
+                raise RuntimeError("不支持的操作系统")
+
+            subprocess.Popen(cmd)
+            time.sleep(5)
+        except Exception:
+            raise RuntimeError('start browser process failed')
+
+    def update_core(self):
+        """下载所有内核，打开店铺前调用，需客户端版本5.285.7以上"""
+        data = {
+            "action"   : "updateCore",
+            "requestId": str(uuid.uuid4()),
+        }
+        data.update(self.user_info)
+
+        while True:
+            result = self.send_http(data)
+            print(result)
+            if result is None:
+                print("等待客户端启动...")
+                time.sleep(2)
+                continue
+            if result.get("statusCode") is None or result.get("statusCode") == -10003:
+                print("当前版本不支持此接口，请升级客户端")
+                return
+            elif result.get("statusCode") == 0:
+                print("更新内核完成")
+                return
+            else:
+                print(f"等待更新内核: {json.dumps(result)}")
+                time.sleep(2)
+
+    def send_http(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        """HTTP通讯方式"""
+        try:
+            url = f'http://127.0.0.1:{self.socket_port}'
+            response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
+            return json.loads(response.text)
+        except Exception as err:
+            print(err)
+            return None
+
+    def delete_all_cache(self):
+        """删除所有店铺缓存"""
+        if not self.is_windows:
+            return
+        local_appdata = os.getenv('LOCALAPPDATA')
+        cache_path = os.path.join(local_appdata, 'SuperBrowser')
+        if os.path.exists(cache_path):
+            shutil.rmtree(cache_path)
+
+    def delete_all_cache_with_path(self, path: str):
+        """删除指定路径的店铺缓存"""
+        if not self.is_windows:
+            return
+        cache_path = os.path.join(path, 'SuperBrowser')
+        if os.path.exists(cache_path):
+            shutil.rmtree(cache_path)
+
+    def exit(self):
+        """关闭客户端"""
+        data = {"action": "exit", "requestId": str(uuid.uuid4())}
+        data.update(self.user_info)
+        print('@@ get_exit...' + json.dumps(data))
+        self.send_http(data)
+
+class ZiniaoBrowser:
+    """紫鸟浏览器操作类"""
+
+    def __init__(self, client: ZiniaoClient, config):
+        self.client = client
+        self.config = config
+
+    def open_store(self, store_info: str, isWebDriverReadOnlyMode: int = 0,
+            isprivacy: int = 0, isHeadless: int = 0,
+            cookieTypeSave: int = 0, jsInfo: str = "") -> Dict[str, Any]:
+        """打开店铺"""
+        request_id = str(uuid.uuid4())
+        data = {
+            "action"                 : "startBrowser",
+            "isWaitPluginUpdate"     : 0,
+            "isHeadless"             : isHeadless,
+            "requestId"              : request_id,
+            "isWebDriverReadOnlyMode": isWebDriverReadOnlyMode,
+            "cookieTypeLoad"         : 0,
+            "cookieTypeSave"         : cookieTypeSave,
+            "runMode"                : "1",
+            "isLoadUserPlugin"       : False,
+            "pluginIdType"           : 1,
+            "privacyMode"            : isprivacy
+        }
+        data.update(self.client.user_info)
+
+        if store_info.isdigit():
+            data["browserId"] = store_info
+        else:
+            data["browserOauth"] = store_info
+
+        if len(str(jsInfo)) > 2:
+            data["injectJsInfo"] = json.dumps(jsInfo)
+
+        r = self.client.send_http(data)
+        if str(r.get("statusCode")) == "0":
+            return r
+        elif str(r.get("statusCode")) == "-10003":
+            print(f"login Err {json.dumps(r, ensure_ascii=False)}")
+            raise RuntimeError("登录错误")
+        else:
+            print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
+            raise RuntimeError("打开店铺失败")
+
+    def close_store(self, browser_oauth: str):
+        """关闭店铺"""
+        request_id = str(uuid.uuid4())
+        data = {
+            "action"      : "stopBrowser",
+            "requestId"   : request_id,
+            "duplicate"   : 0,
+            "browserOauth": browser_oauth
+        }
+        data.update(self.client.user_info)
+
+        r = self.client.send_http(data)
+        if str(r.get("statusCode")) == "0":
+            return r
+        elif str(r.get("statusCode")) == "-10003":
+            print(f"login Err {json.dumps(r, ensure_ascii=False)}")
+            raise RuntimeError("登录错误")
+        else:
+            print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
+            raise RuntimeError("关闭店铺失败")
+
+    def get_browser_list(self) -> List[Dict[str, Any]]:
+        """获取浏览器列表"""
+        request_id = str(uuid.uuid4())
+        data = {
+            "action"   : "getBrowserList",
+            "requestId": request_id
+        }
+        data.update(self.client.user_info)
+
+        r = self.client.send_http(data)
+        if str(r.get("statusCode")) == "0":
+            print(r)
+            return r.get("browserList", [])
+        elif str(r.get("statusCode")) == "-10003":
+            print(f"login Err {json.dumps(r, ensure_ascii=False)}")
+            raise RuntimeError("登录错误")
+        else:
+            print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
+            raise RuntimeError("获取浏览器列表失败")
+
+    def get_browser_context(self, playwright, port: int):
+        """获取playwright浏览器会话"""
+        browser = playwright.chromium.connect_over_cdp(f"http://127.0.0.1:{port}")
+        context = browser.contexts[0]
+        return context
+
+    def open_ip_check(self, browser_context, ip_check_url: str) -> bool:
+        """打开ip检测页检测ip是否正常"""
+        try:
+            page = browser_context.pages[0]
+            page.goto(ip_check_url)
+            success_button = page.locator('//button[contains(@class, "styles_btn--success")]')
+            success_button.wait_for(timeout=60000)  # 等待查找元素60秒
+            print("ip检测成功")
+            return True
+        except PlaywrightTimeoutError:
+            print("ip检测超时")
+            return False
+        except Exception as e:
+            print("ip检测异常:" + traceback.format_exc())
+            return False
+
+    def open_launcher_page(self, browser_context, launcher_page: str, store_username: str, store_name: str, run_func: Callable, task_key: str):
+        """打开启动页面并执行业务逻辑"""
+        page = browser_context.pages[0]
+        page.goto(launcher_page)
+        page.wait_for_load_state('load')
+
+        # 业务逻辑
+        run_func(page, store_username, store_name, task_key)
+
+        # 标记完成
+        done_progress_json_ex(self.config, task_key, store_name)
+
+class ZiniaoTaskManager:
+    """紫鸟任务管理类"""
+
+    def __init__(self, browser: ZiniaoBrowser, config):
+        self.browser = browser
+        self.config = config
+
+    def run_single_store_task(self, browser_info: Dict[str, Any],
+            run_func: Callable, task_key: str,
+            just_store_username: Optional[List[str]] = None,
+            is_skip_store: Optional[Callable] = None
+    ):
+        """运行单个店铺的任务"""
+        retry_count = 0
+        while True:
+            try:
+                retry_count += 1
+
+                store_id = browser_info.get('browserOauth')
+                store_name = browser_info.get("browserName")
+                store_username = browser_info.get("store_username")
+
+                # 记录店铺账号与店铺别名对应关系
+                write_dict_to_file_ex(self.config.shein_store_alias, {store_username: store_name}, [store_username])
+
+                if is_skip_store and is_skip_store(store_username, store_name):
+                    return
+
+                if just_store_username is not None:
+                    if store_username not in just_store_username:
+                        log(f'=================================跳过 just_store_username: {store_name},{store_username}, {just_store_username}======================================')
+                        return
+                    else:
+                        log(f'---------------------------------命中 just_store_username: {store_name},{store_username}, {just_store_username}-------------------------------------')
+
+                if get_progress_json_ex(self.config, task_key, store_name):
+                    log(f'=================================跳过 进度已完成: {task_key},{store_name},{store_username}=================================')
+                    return
+
+                # 打开店铺
+                print(f"=====打开店铺：{store_name}=====")
+                ret_json = self.browser.open_store(store_id)
+                print(ret_json)
+                store_id = ret_json.get("browserOauth") or ret_json.get("browserId")
+
+                # 获取playwright浏览器会话
+                with sync_api.sync_playwright() as playwright:
+                    try:
+                        browser_context = self.browser.get_browser_context(playwright, ret_json.get('debuggingPort'))
+                        if browser_context is None:
+                            print(f"=====关闭店铺：{store_name}=====")
+                            self.browser.close_store(store_id)
+                            return
+
+                        # 获取ip检测页地址
+                        ip_check_url = ret_json.get("ipDetectionPage")
+                        if not ip_check_url:
+                            print("ip检测页地址为空，请升级紫鸟浏览器到最新版")
+                            print(f"=====关闭店铺：{store_name}=====")
+                            self.browser.close_store(store_id)
+                            raise RuntimeError("ip检测页地址为空")
+
+                        ip_usable = self.browser.open_ip_check(browser_context, ip_check_url)
+                        if ip_usable:
+                            print("ip检测通过，打开店铺平台主页")
+                            self.browser.open_launcher_page(browser_context, ret_json.get("launcherPage"), store_username, store_name, run_func, task_key)
+                        else:
+                            print("ip检测不通过，请检查")
+                    except:
+                        print("脚本运行异常:" + traceback.format_exc())
+                        raise
+                    finally:
+                        print(f"=====关闭店铺：{store_name}=====")
+                        self.browser.close_store(store_id)
+                    break
+            except:
+                send_exception(self.config, f'第{retry_count}次运行失败，准备重新打开店铺')
+                if retry_count > 5:
+                    break
+
+    def run_all_stores_task(self, browser_list: List[Dict[str, Any]],
+            run_func: Callable, task_key: str,
+            just_store_username: Optional[List[str]] = None,
+            is_skip_store: Optional[Callable] = None
+    ):
+        """循环运行所有店铺的任务"""
+        for browser_info in browser_list:
+            self.run_single_store_task(browser_info, run_func, task_key, just_store_username, is_skip_store)
+
+    def run_with_thread_pool(self, browser_list: List[Dict[str, Any]],
+            max_threads: int = 3, run_func: Callable = None,
+            task_key: str = None,
+            just_store_username: Optional[List[str]] = None,
+            is_skip_store: Optional[Callable] = None
+    ):
+        """使用线程池控制最大并发线程数运行任务"""
+        with ThreadPoolExecutor(max_workers=max_threads) as executor:
+            task = partial(self.run_single_store_task,
+                           run_func=run_func, task_key=task_key,
+                           just_store_username=just_store_username,
+                           is_skip_store=is_skip_store)
+            log(f'店铺总数: {len(browser_list)}')
+            executor.map(task, browser_list)
+
+class ZiniaoRunner:
+    """紫鸟主运行器类"""
+
+    def __init__(self, config):
+        self.config = config
+        self.client = ZiniaoClient(config)
+        self.browser = ZiniaoBrowser(self.client, config)
+        self.task_manager = ZiniaoTaskManager(self.browser, config)
+
+    def execute(self, run_prepare: Optional[Callable] = None,
+            run: Optional[Callable] = None,
+            run_summary: Optional[Callable] = None,
+            run_notify: Optional[Callable] = None,
+            task_key: Optional[str] = None,
+            just_store_username: Optional[List[str]] = None,
+            is_skip_store: Optional[Callable] = None,
+    ):
+        """主执行入口"""
+        # 前置执行 if run_prepare:
+        run_prepare()
+
+        # 终止紫鸟客户端已启动的进程
+        self.client.kill_process()
+
+        print("=====启动客户端=====")
+        self.client.start_browser()
+        print("=====更新内核=====")
+        self.client.update_core()
+
+        # 获取店铺列表
+        print("=====获取店铺列表=====")
+        browser_list = self.browser.get_browser_list()
+        if not browser_list:
+            print("browser list is empty")
+            raise RuntimeError("店铺列表为空")
+
+        # 多线程并发执行任务
+        max_threads = 5 if (hostname().lower() == 'krrpa') else 3
+        log(f'当前启用线程数: {max_threads}')
+        self.task_manager.run_with_thread_pool(browser_list, max_threads, run, task_key, just_store_username, is_skip_store)
+
+        # 任务重试逻辑
+        try_times = 0
+        while not check_progress_json_ex(self.config, task_key, just_store_username):
+            try_times += 1
+            send_exception(self.config, f'检测到任务未全部完成,再次执行: {try_times}')
+            self.task_manager.run_with_thread_pool(browser_list, max_threads, run, task_key, just_store_username, is_skip_store)
+            if try_times >= 4:
+                send_exception(self.config, f'检测到任务未全部完成,再次执行: {try_times}')
+                break
+
+        # 数据汇总
+        if not get_progress_json_ex(self.config, task_key, 'run_summary'):
+            if run_summary:
+                run_summary()
+            done_progress_json_ex(self.config, task_key, 'run_summary')
+        log('run_summary 完成')
+
+        # 发送通知
+        if not get_progress_json_ex(self.config, task_key, 'run_notify'):
+            if run_notify:
+                run_notify()
+            done_progress_json_ex(self.config, task_key, 'run_notify')
+        log('run_notify 完成')
+
+        # 关闭客户端
+        self.client.exit()
+
+if __name__ == "__main__":
+    pass