python-playwright-helper 0.2.8__py3-none-any.whl

playwright_helper/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     __init__.py
+# Description:  源码包
+# Author:       ASUS
+# CreateDate:   2025/12/13
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""

playwright_helper/libs/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     __init__.py
+# Description:  核心包
+# Author:       ASUS
+# CreateDate:   2025/12/13
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""

playwright_helper/libs/base_po.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     base_po.py
+# Description:  po对象基础类
+# Author:       ASUS
+# CreateDate:   2025/12/13
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""
+import asyncio
+from logging import Logger
+from typing import List, Any, cast
+from playwright.async_api import Page, Locator, TimeoutError as PlaywrightTimeoutError
+
+
+class BasePo(object):
+    __page: Page
+
+    def __init__(self, page: Page, url: str):
+        self.url = url
+        self.__page = page
+
+    def get_page(self) -> Page:
+        return self.__page
+
+    def is_current_page(self) -> bool:
+        return self.iss_current_page(self.__page, self.url)
+
+    def get_url_domain(self) -> str:
+        if isinstance(self.__page, Page):
+            page_slice: List[str] = self.__page.url.split("/")
+            return f"{page_slice[0]}://{page_slice[2]}"
+        else:
+            raise AttributeError("PO对象中的page属性未被初始化")
+
+    def get_url(self) -> str:
+        if self.__page.url.find("://") != -1:
+            return self.__page.url.split("?")[0]
+        else:
+            return self.__page.url
+
+    @staticmethod
+    def iss_current_page(page: Page, url: str) -> bool:
+        if isinstance(page, Page):
+            page_url_prefix = page.url.split("?")[0]
+            url_prefix = url.split("?")[0]
+            if page_url_prefix.endswith(url_prefix):
+                return True
+            else:
+                return False
+        else:
+            return False
+
+    @staticmethod
+    async def exists(locator):
+        return await locator.count() > 0
+
+    @staticmethod
+    async def exists_one(locator):
+        return await locator.count() == 1
+
+    async def get_locator(self, selector: str, timeout: float = 3.0) -> Locator:
+        """
+        获取页面元素locator
+        :param selector: 选择器表达式
+        :param timeout: 超时时间（秒）
+        :return: 元素对象
+        :return:
+        """
+        locator = self.__page.locator(selector)
+        try:
+            await locator.first.wait_for(state='visible', timeout=timeout * 1000)
+            return locator
+        except (PlaywrightTimeoutError,):
+            raise PlaywrightTimeoutError(f"元素 '{selector}' 未在 {timeout} 秒内找到")
+        except Exception as e:
+            raise RuntimeError(f"检查元素时发生错误: {str(e)}")
+
+    @staticmethod
+    async def get_sub_locator(locator: Locator, selector: str, timeout: float = 3.0) -> Locator:
+        """
+        获取页面locator的子locator
+        :param locator: 页面Locator对象
+        :param selector: 选择器表达式
+        :param timeout: 超时时间（秒）
+        :return: 元素对象
+        :return:
+        """
+        locator_inner = locator.locator(selector)
+        try:
+            await locator_inner.first.wait_for(state='visible', timeout=timeout * 1000)
+            return locator_inner
+        except (PlaywrightTimeoutError,):
+            raise PlaywrightTimeoutError(f"元素 '{selector}' 未在 {timeout} 秒内找到")
+        except Exception as e:
+            raise RuntimeError(f"检查元素时发生错误: {str(e)}")
+
+    @classmethod
+    async def handle_po_cookie_tip(cls, page: Any, logger: Logger, timeout: float = 3.0,
+                                   selectors: List[str] = None) -> None:
+        selectors_inner: List[str] = [
+            '//div[@id="isReadedCookie"]/button',
+            '//button[@id="continue-btn"]/span[normalize-space(text())="同意"]'
+        ]
+        if selectors:
+            selectors_inner.extend(selectors)
+        for selector in selectors_inner:
+            try:
+                page_inner = cast(cls, page)
+                cookie: Locator = await cls.get_locator(self=page_inner, selector=selector, timeout=timeout)
+                logger.info(
+                    f'找到页面中存在cookie提示：[本网站使用cookie，用于在您的电脑中储存信息。这些cookie可以使网站正常运行，以及帮助我们改进用户体验。使用本网站，即表示您接受放置这些cookie。]')
+                await cookie.click(button="left")
+                logger.info("【同意】按钮点击完成")
+                await asyncio.sleep(1)
+                return
+            except (Exception,):
+                pass
+
+    async def url_wait_for(self, url: str, timeout: float = 3.0) -> None:
+        """
+        url_suffix格式：
+            /shopping/oneway/SHA,PVG-URC/2026-01-08
+            https://www.ceair.com/shopping/oneway/SHA,PVG-URC/2026-01-08
+        :param url:
+        :param timeout:
+        :return:
+        """
+        for _ in range(int(timeout) * 10):
+            if self.iss_current_page(page=self.__page, url=url):
+                return
+            await asyncio.sleep(delay=0.1)
+        if url.find("://") == -1:
+            url = self.get_url_domain() + url
+        raise RuntimeError(f"无法打开/加载页面<{url}>")

playwright_helper/libs/browser_pool.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     browser_pool.py
+# Description:  浏览器池，一次起 Chrome，并发复用
+# Author:       ASUS
+# CreateDate:   2025/12/13
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""
+import asyncio
+from logging import Logger
+from typing import Any, Optional
+from playwright.async_api import Browser, async_playwright, Playwright
+
+
+class BrowserPool:
+    def __init__(
+            self,
+            *,
+            size: int,
+            logger: Logger,
+            **launch_config: Any,
+    ):
+        self.size = size
+        self.logger = logger
+        self.launch_config = launch_config
+        self._queue: asyncio.Queue[Browser] = asyncio.Queue()
+        self._started: bool = False
+        self._playwright: Optional[Playwright] = None
+
+    async def start(self, playwright: Playwright = None):
+        if self._started:
+            return
+
+        if playwright:
+            self._playwright = playwright
+
+        if not self._playwright:
+            self._playwright = await async_playwright().start()
+
+        self.logger.debug(f"[BrowserPool] start size={self.size}")
+
+        for i in range(self.size):
+            self.logger.debug(f"[BrowserPool] launching browser {i}")
+            browser = await self._playwright.chromium.launch(
+                **self.launch_config
+            )
+            await self._queue.put(browser)
+
+        self._started = True
+        self.logger.debug("[BrowserPool] started")
+
+    async def acquire(self) -> Browser:
+        self.logger.debug("[BrowserPool] acquire waiting...")
+        browser = await self._queue.get()
+        self.logger.debug("[BrowserPool] acquire ok")
+        return browser
+
+    async def release(self, browser: Browser):
+        self.logger.debug("[BrowserPool] release")
+        await self._queue.put(browser)
+
+    async def stop(self):
+        self.logger.debug("[BrowserPool] stopping")
+        while not self._queue.empty():
+            browser = await self._queue.get()
+            await browser.close()
+        if self._playwright:
+            await self._playwright.stop()
+        self.logger.debug("[BrowserPool] stopped")

playwright_helper/libs/executor.py

@@ -0,0 +1,243 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     executor.py
+# Description:  执行器模块
+# Author:       ASUS
+# CreateDate:   2025/12/13
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""
+import os
+import time
+import uuid
+import asyncio
+from logging import Logger
+from contextlib import asynccontextmanager
+from playwright_helper.utils.type_utils import RunResult
+from playwright._impl._api_structures import ViewportSize
+from playwright_helper.utils.log_utils import logger as log
+from playwright_helper.libs.browser_pool import BrowserPool
+from playwright_helper.utils.file_handle import get_caller_dir
+from typing import Any, List, Optional, cast, Callable, Literal, Dict
+from playwright.async_api import Page, Browser, BrowserContext, TimeoutError as PlaywrightTimeoutError, \
+    Error as PlaywrightError, async_playwright
+
+
+class PlaywrightBrowserExecutor:
+    def __init__(
+            self,
+            *,
+            logger: Logger = log,
+            browser_pool: Optional[BrowserPool] = None,
+            mode: Literal["persistent", "storage"] = "storage",
+            middlewares: Optional[List[Callable]] = None,
+            retries: int = 1,
+            record_video: bool = False,
+            record_trace: bool = False,
+            video_dir: str = None,
+            trace_dir: str = None,
+            screenshot_dir: str = None,
+            viewport: Optional[ViewportSize] = None,
+            user_agent: Optional[str] = None,
+            storage_state: Optional[Dict[str, Any]] = None,
+            **browser_config: Any,
+    ):
+        self.mode = mode
+        self.logger = logger
+        self.browser_pool = browser_pool
+        self.middlewares = middlewares or []
+        self.retries = retries
+        self.user_agent = user_agent
+        self.viewport = viewport
+
+        self.record_video = record_video
+        self.record_trace = record_trace
+        self.video_dir = video_dir or get_caller_dir()
+        self.trace_dir = trace_dir or get_caller_dir()
+        self.screenshot_dir = screenshot_dir or get_caller_dir()
+        self.storage_state = storage_state
+        self.browser_config = browser_config
+
+        self._playwright = None
+
+        if self.mode == "storage" and not self.browser_pool:
+            raise ValueError("storage 模式必须提供 browser_pool")
+
+    async def _safe_screenshot(self, page: Page, name: str = None):
+        try:
+            os.makedirs(self.screenshot_dir, exist_ok=True)
+            if name is None or "unknown" in name:
+                name = f"error_{int(time.time())}"
+            path = os.path.join(self.screenshot_dir, f"{name}.png")
+            await page.screenshot(path=path)
+            self.logger.debug(f"[Screenshot Saved] {path}")
+        except Exception as e:
+            self.logger.error(f"[Screenshot Failed] {e}")
+
+    async def start(self):
+        """Executor 生命周期开始（进程级调用一次）"""
+        if not self._playwright:
+            self._playwright = await async_playwright().start()
+
+            # storage 模式：BrowserPool 需要 playwright
+            if self.mode == "storage" and self.browser_pool:
+                await self.browser_pool.start(self._playwright)
+
+    async def stop(self):
+        """Executor 生命周期结束"""
+        if self.mode == "persistent" and self._playwright:
+            await self._playwright.stop()
+            self._playwright = None
+
+    @asynccontextmanager
+    async def session(self, storage_state: Optional[Dict[str, Any]] = None):
+        context: BrowserContext = cast(BrowserContext, None)
+        try:
+            context = await self._create_context(storage_state=storage_state or self.storage_state)
+            page: Page = await context.new_page()
+            yield page
+        finally:
+            if context:
+                await self._cleanup_context(context)
+
+    async def _create_context(self, storage_state: Optional[Dict[str, Any]] = None) -> BrowserContext:
+        task_id = str(uuid.uuid4())
+
+        if self.mode == "persistent":
+            self.logger.debug("[Executor] mode=persistent")
+            # persistent 模式：不走 pool
+            context = await self._playwright.chromium.launch_persistent_context(
+                record_video_dir=self.video_dir if self.record_video else None,
+                **self.browser_config
+            )
+        else:
+            # 并发模式：BrowserPool
+            self.logger.debug("[Executor] mode=storage")
+
+            browser: Browser = await self.browser_pool.acquire()
+
+            context = await browser.new_context(
+                storage_state=storage_state or self.storage_state,
+                viewport=self.viewport,
+                user_agent=self.user_agent,
+                record_video_dir=self.video_dir if self.record_video else None,
+            )
+
+            # 关键：挂载资源，供 cleanup 使用
+            context._browser = browser  # type: ignore[attr-defined]
+
+        # task_id 给 trace / video 用
+        context._task_id = task_id  # type: ignore[attr-defined]
+
+        if self.record_trace:
+            await context.tracing.start(
+                screenshots=True,
+                snapshots=True,
+                sources=True,
+            )
+
+        return context
+
+    async def _cleanup_context(self, context: BrowserContext):
+        if getattr(context, "_closed", False):
+            return
+
+        context._closed = True  # type: ignore
+
+        try:
+            await context.close()
+        except Exception as e:
+            self.logger.warning(f"[Cleanup] ignore close error: {e}")
+
+        browser = getattr(context, "_browser", None)
+        if browser:
+            try:
+                self.logger.debug("[Executor] Release browser to pool")
+                await self.browser_pool.release(browser)
+            except Exception as e:
+                self.logger.error(f"[Cleanup] ignore release error: {e}")
+
+    async def _run_callback_chain(self, *, callback: Callable, page: Page, context: BrowserContext, **kwargs) -> Any:
+        # Run middlewares before callback
+        for mw in self.middlewares:
+            await mw(page=page, logger=self.logger, context=context, **kwargs)
+
+        # Main callback
+        return await callback(page=page, logger=self.logger, context=context, **kwargs)
+
+    async def run(
+            self, *, callback: Callable, storage_state: Optional[Dict[str, Any]] = None, **kwargs: Any
+    ) -> RunResult:
+        attempt = 0
+        last_error: Optional[Exception] = None
+        task_id: Optional[str] = "unknown"
+        result: Any = None
+
+        while attempt <= self.retries:
+            page = None
+            context: BrowserContext = cast(BrowserContext, None)
+            try:
+                context = await self._create_context(storage_state=storage_state)
+                task_id = getattr(context, "_task_id", "unknown")
+                page = await context.new_page()
+
+                result = await self._run_callback_chain(
+                    callback=callback, page=page, context=context, **kwargs
+                )
+                self.logger.info(f"[Task<{task_id}> Success]")
+                return RunResult(
+                    success=True,
+                    attempts=attempt + 1,
+                    task_id=task_id,
+                    error=last_error,
+                    result=result
+                )
+            except (asyncio.CancelledError, KeyboardInterrupt):
+                # ⚠️ 不要当成错误
+                self.logger.warning(f"[Task<{task_id}> Cancelled]")
+
+                # 清理资源
+                if context and self.record_trace:
+                    os.makedirs(self.trace_dir, exist_ok=True)
+                    trace_path = os.path.join(self.trace_dir, f"{task_id}.zip")
+                    await context.tracing.stop(path=trace_path)
+                    self.logger.debug(f"[Trace Saved] {trace_path}")
+
+                # ❗关键：不要 raise
+                return RunResult(
+                    success=False,
+                    attempts=attempt + 1,
+                    error=asyncio.CancelledError(),
+                    task_id=task_id,
+                    result=result
+                )
+            except (PlaywrightTimeoutError, PlaywrightError, Exception) as e:
+                last_error = e
+                self.logger.error(f"[Task<{task_id}> Attempt {attempt + 1} Failed] {e}")
+                if page:
+                    await self._safe_screenshot(page=page, name=task_id)
+
+                if context and self.record_trace:
+                    os.makedirs(self.trace_dir, exist_ok=True)
+                    trace_path = os.path.join(self.trace_dir, f"{task_id}.zip")
+                    await context.tracing.stop(path=trace_path)
+                    self.logger.debug(f"[Trace Saved] {trace_path}")
+
+                attempt += 1
+                if attempt <= self.retries:
+                    await asyncio.sleep(1)
+            finally:
+                if context:
+                    await self._cleanup_context(context)
+        # 所有重试结束，仍然失败
+        self.logger.error(f"[Task<{task_id}> Final Failure]")
+
+        return RunResult(
+            success=False,
+            attempts=attempt,
+            error=last_error,
+            task_id=task_id,
+            result=result
+        )

playwright_helper/middlewares/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     __init__.py
+# Description:  中间件包
+# Author:       ASUS
+# CreateDate:   2025/12/13
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""

playwright_helper/middlewares/stealth.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     stealth.py
+# Description:  安全代理中间件
+# Author:       ASUS
+# CreateDate:   2025/12/13
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""
+from typing import Any
+from logging import Logger
+from playwright.async_api import Page
+from playwright_stealth import Stealth
+
+CHROME_STEALTH_ARGS = [
+    # 核心防检测
+    '--disable-blink-features=AutomationControlled',
+    '--disable-automation-controlled-blink-features',
+
+    # 隐藏"Chrome正受到自动测试软件控制"提示
+    '--disable-infobars',
+    '--disable-popup-blocking',
+
+    # 性能优化
+    '--no-first-run',
+    '--no-default-browser-check',
+    '--disable-default-apps',
+    '--disable-translate',
+
+    # 禁用自动化标志
+    '--disable-background-timer-throttling',
+    '--disable-backgrounding-occluded-windows',
+    '--disable-renderer-backgrounding',
+
+    # 网络和安全
+    '--disable-web-security',
+    '--disable-features=VizDisplayCompositor',
+    '--disable-features=RendererCodeIntegrity',
+    '--remote-debugging-port=0',  # 随机端口
+
+    # 硬件相关（减少特征）
+    '--disable-gpu',
+    '--disable-software-rasterizer',
+    '--disable-dev-shm-usage',
+]
+
+IGNORE_ARGS = [
+    '--enable-automation',
+    '--enable-automation-controlled-blink-features',
+    '--password-store=basic',  # 避免密码存储提示
+]
+
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"
+
+viewport = {'width': 1920, 'height': 1080}
+
+
+async def _setup_stealth_page(page: Page):
+    """设置页面为隐身模式"""
+    # 修改 navigator.webdriver
+    await page.add_init_script("""
+        // 进一步修改 navigator 属性
+        Object.defineProperty(navigator, 'webdriver', {
+            get: () => undefined,
+        });
+        Object.defineProperty(navigator, 'plugins', {
+            get: () => [1, 2, 3, 4, 5],
+        });
+        Object.defineProperty(navigator, 'languages', {
+            get: () => ['zh-CN', 'zh', 'en'],
+        });
+
+        // 删除 webdriver 属性
+        delete navigator.__proto__.webdriver;
+
+        // 修改 plugins
+        Object.defineProperty(navigator, 'plugins', {
+            get: () => [{
+                name: 'Chrome PDF Plugin',
+                filename: 'internal-pdf-viewer'
+            }],
+        });
+
+        // 修改 languages
+        Object.defineProperty(navigator, 'languages', {
+            get: () => ['zh-CN', 'zh', 'en-US', 'en'],
+        });
+
+        // 修改 platform
+        Object.defineProperty(navigator, 'platform', {
+            get: () => 'Win32',
+        });
+
+        // 隐藏 chrome 对象
+        window.chrome = {
+            runtime: {},
+            loadTimes: function(){},
+            csi: function(){},
+            app: {}
+        };
+    """)
+
+
+# 创建 stealth 配置
+stealth = Stealth(
+    navigator_webdriver=True,  # 隐藏 webdriver
+    navigator_plugins=True,  # 修改插件
+    navigator_languages=True,  # 修改语言
+    navigator_platform=True,  # 修改平台
+    navigator_user_agent=False,  # 修改 UA
+    script_logging=False,  # 生产环境关闭日志
+)
+
+
+async def stealth_middleware(
+        *,
+        page: Page,
+        logger: Logger,
+        **kwargs: Any
+):
+    await stealth.apply_stealth_async(page_or_context=page)
+    await _setup_stealth_page(page)
+    logger.debug("[Middleware] Stealth applied")

playwright_helper/utils/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     __init__.py
+# Description:  工具包
+# Author:       ASUS
+# CreateDate:   2025/12/08
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""

playwright_helper/utils/browser_utils.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     browser_utils.py
+# Description:  浏览器工具模块
+# Author:       ASUS
+# CreateDate:   2025/12/18
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""
+import asyncio
+from playwright.async_api import BrowserContext, Page
+
+
+async def switch_for_table_window(browser: BrowserContext, url_keyword: str, wait_time: float = 10.0) -> Page:
+    # 最多等待 wait_time 秒
+    for _ in range(int(wait_time) * 10):
+        await asyncio.sleep(delay=0.1)
+        for page in browser.pages:
+            if url_keyword.lower() in page.url.lower():
+                await page.bring_to_front()
+                return page
+    raise RuntimeError(f"根据关键信息<{url_keyword}>，没有找到浏览器的page对象")

playwright_helper/utils/file_handle.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+# ---------------------------------------------------------------------------------------------------------
+# ProjectName:  playwright-helper
+# FileName:     file_handle.py
+# Description:  文件处理工具模块
+# Author:       ASUS
+# CreateDate:   2025/12/08
+# Copyright ©2011-2025. Hunan xxxxxxx Company limited. All rights reserved.
+# ---------------------------------------------------------------------------------------------------------
+"""
+import os
+import inspect
+
+
+def get_caller_dir() -> str:
+    # 获取调用者的 frame
+    frame = inspect.stack()[1]
+    caller_file = frame.filename  # 调用者文件的完整路径
+    return os.path.dirname(os.path.abspath(caller_file))
+
+
+def save_image(file_name: str, img_bytes: bytes) -> None:
+    """
+    保存验证码图片到本地。
+    若文件已存在，会自动覆盖。
+    """
+    # 确保目录存在
+    os.makedirs(os.path.dirname(file_name), exist_ok=True)
+
+    # "wb" 会覆盖已有文件
+    with open(file_name, "wb") as f:
+        f.write(img_bytes)