aukeys-opscli 0.0.6__py3-none-any.whl

opscli/amazon/scraping/scraper.py

@@ -0,0 +1,315 @@
+"""amazon 模块抓取器。"""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime
+from urllib.parse import quote_plus
+
+from opscli.amazon.domain.exceptions import InvalidAsinError, ScraperDependencyError
+from opscli.amazon.domain.models import AmazonProductSnapshot, AmazonSearchResult
+from opscli.amazon.scraping.parser import normalize_text, parse_price, parse_rating, parse_review_count
+
+
+class AmazonScraper:
+    """基于 Playwright 的 Amazon 抓取器。"""
+
+    PRODUCT_URL_TEMPLATE = "https://www.amazon.com/dp/{asin}"
+    SEARCH_URL_TEMPLATE = "https://www.amazon.com/s?k={keyword}&s=review-rank"
+    DEFAULT_TIMEOUT_MS = 120000
+    DEFAULT_WAIT_MS = 3000
+
+    def __init__(self, *, headless: bool = True, timeout_ms: int = DEFAULT_TIMEOUT_MS) -> None:
+        self.headless = headless
+        self.timeout_ms = timeout_ms
+
+    def scrape_product(self, asin: str, zip_code: str = "10001") -> AmazonProductSnapshot:
+        """同步抓取单个商品。"""
+        return asyncio.run(self.scrape_product_async(asin, zip_code))
+
+    def search_products(
+        self,
+        keyword: str,
+        *,
+        max_results: int = 10,
+        zip_code: str = "10001",
+    ) -> list[AmazonSearchResult]:
+        """同步抓取搜索结果。"""
+        return asyncio.run(self.search_products_async(keyword, max_results=max_results, zip_code=zip_code))
+
+    async def scrape_product_async(self, asin: str, zip_code: str = "10001") -> AmazonProductSnapshot:
+        """异步抓取单个商品。"""
+        self._validate_asin(asin)
+        async with self._browser_page() as page:
+            page_url = self.PRODUCT_URL_TEMPLATE.format(asin=asin)
+            await page.goto(page_url, wait_until="domcontentloaded", timeout=self.timeout_ms)
+            await page.wait_for_timeout(self.DEFAULT_WAIT_MS)
+            await self._apply_zip_code(page, zip_code)
+
+            page_title = await page.title()
+            if self._looks_invalid(page_title):
+                return AmazonProductSnapshot(
+                    asin=asin,
+                    zip_code=zip_code,
+                    marketplace="amazon.com",
+                    page_url=page_url,
+                    page_title=page_title,
+                    product_name="",
+                    price_text="",
+                    price_amount=None,
+                    currency=None,
+                    rating_text="",
+                    rating_value=None,
+                    review_count_text="",
+                    review_count_value=None,
+                    location="",
+                    collected_at=self._now(),
+                    valid=False,
+                    error="链接失效或页面不存在",
+                )
+
+            product_name = await self._get_text(page, ["#productTitle", "#title"])
+            if len(product_name) < 50 and page_title:
+                product_name = self._choose_better_title(product_name, page_title)
+
+            price_text = await self._get_text(
+                page,
+                [
+                    "#corePriceDisplay_desktop_feature_div .a-price .a-offscreen",
+                    ".a-price .a-offscreen",
+                    ".reinventPricePriceToPayMargin .a-offscreen",
+                ],
+            )
+            rating_text = await self._get_text(
+                page,
+                [
+                    "#acrPopover .a-icon-alt",
+                    "[data-hook='rating-out-of-text']",
+                ],
+            )
+            review_count_text = await self._get_text(
+                page,
+                [
+                    "#acrCustomerReviewText",
+                    "[data-hook='total-review-count']",
+                    "#acrCustomerReviewLink",
+                ],
+            )
+            location = await self._get_text(page, ["#glow-ingress-line2"])
+
+            price_amount, currency = parse_price(price_text)
+            rating_value = parse_rating(rating_text)
+            review_count_value = parse_review_count(review_count_text)
+
+            return AmazonProductSnapshot(
+                asin=asin,
+                zip_code=zip_code,
+                marketplace="amazon.com",
+                page_url=page_url,
+                page_title=page_title,
+                product_name=product_name,
+                price_text=price_text,
+                price_amount=price_amount,
+                currency=currency,
+                rating_text=rating_text,
+                rating_value=rating_value,
+                review_count_text=review_count_text,
+                review_count_value=review_count_value,
+                location=location,
+                collected_at=self._now(),
+                valid=True,
+                raw={
+                    "productName": product_name,
+                    "price": price_text,
+                    "rating": rating_text,
+                    "reviewCount": review_count_text,
+                    "location": location,
+                },
+            )
+
+    async def search_products_async(
+        self,
+        keyword: str,
+        *,
+        max_results: int = 10,
+        zip_code: str = "10001",
+    ) -> list[AmazonSearchResult]:
+        """异步抓取搜索结果。"""
+        async with self._browser_page() as page:
+            search_url = self.SEARCH_URL_TEMPLATE.format(keyword=quote_plus(keyword))
+            await page.goto(search_url, wait_until="domcontentloaded", timeout=self.timeout_ms)
+            await page.wait_for_timeout(self.DEFAULT_WAIT_MS)
+            await self._apply_zip_code(page, zip_code)
+
+            items = await page.query_selector_all('[data-component-type="s-search-result"], .s-result-item')
+            results: list[AmazonSearchResult] = []
+            for item in items:
+                asin = normalize_text(await item.get_attribute("data-asin"))
+                if not asin:
+                    continue
+
+                title = await self._get_text_from_scope(
+                    item,
+                    [
+                        "h2 .a-link-normal span",
+                        ".a-size-medium.a-color-base.a-text-normal",
+                    ],
+                )
+                price_text = await self._get_text_from_scope(item, [".a-price .a-offscreen", ".a-price-whole"])
+                rating_text = await self._get_text_from_scope(item, [".a-icon-alt"])
+                review_count_text = await self._get_review_count_from_scope(item)
+                badge = await item.query_selector("[aria-label*='Best Seller']")
+
+                price_amount, _ = parse_price(price_text)
+                rating_value = parse_rating(rating_text)
+                review_count_value = parse_review_count(review_count_text)
+
+                results.append(
+                    AmazonSearchResult(
+                        asin=asin,
+                        keyword=keyword,
+                        zip_code=zip_code,
+                        rank=len(results) + 1,
+                        title=title,
+                        price_text=price_text,
+                        price_amount=price_amount,
+                        rating_text=rating_text,
+                        rating_value=rating_value,
+                        review_count_text=review_count_text,
+                        review_count_value=review_count_value,
+                        is_best_seller=badge is not None,
+                    )
+                )
+                if len(results) >= max_results:
+                    break
+
+            return results
+
+    async def _apply_zip_code(self, page, zip_code: str) -> None:
+        """设置邮编以稳定价格口径。"""
+        if not zip_code:
+            return
+        try:
+            location_btn = await page.query_selector("#glow-ingress-line2")
+            if location_btn:
+                await location_btn.click()
+                await page.wait_for_timeout(1500)
+
+            zip_input = await page.query_selector("#GLUXZipUpdateInput")
+            if not zip_input:
+                return
+
+            await zip_input.fill(zip_code)
+            await page.wait_for_timeout(600)
+
+            submit_btn = await page.query_selector("#GLUXZipUpdate")
+            if submit_btn:
+                await submit_btn.click()
+                await page.wait_for_timeout(2500)
+                await page.reload(wait_until="domcontentloaded", timeout=self.timeout_ms)
+                await page.wait_for_timeout(1500)
+        except Exception:
+            return
+
+    async def _get_text(self, page, selectors: list[str]) -> str:
+        """从 page 上按优先级读取文本。"""
+        for selector in selectors:
+            handle = await page.query_selector(selector)
+            if handle:
+                value = normalize_text(await handle.inner_text())
+                if value:
+                    return value
+        return ""
+
+    async def _get_text_from_scope(self, scope, selectors: list[str]) -> str:
+        """从局部作用域按优先级读取文本。"""
+        for selector in selectors:
+            handle = await scope.query_selector(selector)
+            if handle:
+                value = normalize_text(await handle.inner_text())
+                if value:
+                    return value
+        return ""
+
+    async def _get_review_count_from_scope(self, scope) -> str:
+        """优先从评论链接中读取评论数，避免误取评分文本。"""
+        selectors = [
+            "a[href*='customerReviews'] span.a-size-base",
+            "a[href*='customerReviews'] .a-size-base",
+            "a[href*='customerReviews']",
+            "span.a-size-base.s-underline-text",
+        ]
+        for selector in selectors:
+            handles = await scope.query_selector_all(selector)
+            for handle in handles:
+                for value in (
+                    normalize_text(await handle.inner_text()),
+                    normalize_text(await handle.get_attribute("aria-label")),
+                ):
+                    if parse_review_count(value) is not None:
+                        return value
+        return ""
+
+    def _validate_asin(self, asin: str) -> None:
+        """校验 ASIN。"""
+        normalized = normalize_text(asin).upper()
+        if len(normalized) != 10 or not normalized.isalnum():
+            raise InvalidAsinError("ASIN 必须是 10 位字母或数字")
+
+    def _looks_invalid(self, page_title: str) -> bool:
+        """根据标题粗略判断页面是否失效。"""
+        title = normalize_text(page_title).lower()
+        return "page not found" in title or "sorry" in title and "amazon" in title
+
+    def _choose_better_title(self, product_name: str, page_title: str) -> str:
+        """当 DOM 标题过短时，回退使用页面标题。"""
+        candidate = normalize_text(page_title.split(":", 1)[0])
+        if len(candidate) > len(product_name):
+            return candidate
+        return product_name
+
+    def _now(self) -> str:
+        """返回统一的抓取时间。"""
+        return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    def _load_playwright(self):
+        """延迟导入 Playwright，避免未安装时影响其他模块。"""
+        try:
+            from playwright.async_api import async_playwright
+        except ModuleNotFoundError as exc:
+            raise ScraperDependencyError(
+                "缺少 playwright 依赖，请安装 `pip install 'opscli[amazon]'` 并执行 `playwright install chromium`"
+            ) from exc
+        return async_playwright
+
+    def _browser_page(self):
+        """创建 page 上下文。"""
+        async_playwright = self._load_playwright()
+
+        class _PageContext:
+            def __init__(self, outer: AmazonScraper):
+                self.outer = outer
+                self.playwright = None
+                self.browser = None
+                self.context = None
+                self.page = None
+
+            async def __aenter__(self):
+                self.playwright = await async_playwright().start()
+                self.browser = await self.playwright.chromium.launch(headless=self.outer.headless)
+                self.context = await self.browser.new_context(
+                    locale="en-US",
+                    extra_http_headers={"Accept-Language": "en-US,en;q=0.9"},
+                )
+                self.page = await self.context.new_page()
+                return self.page
+
+            async def __aexit__(self, exc_type, exc, tb):
+                if self.context is not None:
+                    await self.context.close()
+                if self.browser is not None:
+                    await self.browser.close()
+                if self.playwright is not None:
+                    await self.playwright.stop()
+
+        return _PageContext(self)

opscli/amazon/services/__init__.py

@@ -0,0 +1,5 @@
+"""amazon 服务层。"""
+
+from opscli.amazon.services.manager import AmazonManager
+
+__all__ = ["AmazonManager"]

opscli/amazon/services/manager.py

@@ -0,0 +1,105 @@
+"""amazon 模块业务编排层。"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from opscli.amazon.domain.models import AmazonCollectResult, AmazonProductSnapshot, AmazonSearchResult
+from opscli.amazon.scraping.scraper import AmazonScraper
+from opscli.amazon.transport.client import AmazonOpsClient
+from opscli.config import CONFIG_DIR
+
+
+class AmazonManager:
+    """协调抓取、本地落盘和远端提交。"""
+
+    def __init__(
+        self,
+        *,
+        base_dir: Path | None = None,
+        scraper: AmazonScraper | None = None,
+        client: AmazonOpsClient | None = None,
+    ) -> None:
+        self.base_dir = Path(base_dir or CONFIG_DIR)
+        self.data_dir = self.base_dir / "amazon"
+        self.history_dir = self.data_dir / "history"
+        self.scraper = scraper or AmazonScraper()
+        self.client = client or AmazonOpsClient()
+
+    def scrape_product(
+        self,
+        *,
+        asin: str,
+        zip_code: str = "10001",
+        save_history: bool = True,
+    ) -> AmazonCollectResult:
+        """抓取单个商品并可选写历史。"""
+        snapshot = self.scraper.scrape_product(asin, zip_code)
+        history_path = self._append_history(snapshot) if save_history else None
+        return AmazonCollectResult(snapshot=snapshot, history_path=history_path)
+
+    def scrape_and_submit(
+        self,
+        *,
+        asin: str,
+        zip_code: str = "10001",
+        endpoint: str | None = None,
+        save_history: bool = True,
+    ) -> AmazonCollectResult:
+        """抓取后直接提交到 ops。"""
+        result = self.scrape_product(asin=asin, zip_code=zip_code, save_history=save_history)
+        result.submit_result = self.client.submit_snapshot(result.snapshot, endpoint=endpoint)
+        return result
+
+    def build_submit_payload(self, snapshot: AmazonProductSnapshot) -> dict:
+        """构造未来提交到 ops 的标准 payload。"""
+        return {
+            "source": "opscli.amazon",
+            "snapshot": snapshot.to_dict(include_raw=True),
+        }
+
+    def scrape_payload(
+        self,
+        *,
+        asin: str,
+        zip_code: str = "10001",
+        save_history: bool = True,
+    ) -> dict:
+        """抓取商品并输出未来用于提交 ops 的 payload。"""
+        result = self.scrape_product(asin=asin, zip_code=zip_code, save_history=save_history)
+        return {
+            "payload": self.build_submit_payload(result.snapshot),
+            "history_path": str(result.history_path) if result.history_path else None,
+        }
+
+    def search_products(
+        self,
+        *,
+        keyword: str,
+        zip_code: str = "10001",
+        limit: int = 10,
+    ) -> list[AmazonSearchResult]:
+        """搜索竞品列表。"""
+        return self.scraper.search_products(keyword, max_results=limit, zip_code=zip_code)
+
+    def load_history(self, asin: str) -> list[dict]:
+        """读取本地历史快照。"""
+        history_path = self.history_dir / f"{asin.upper()}.jsonl"
+        if not history_path.exists():
+            return []
+        records: list[dict] = []
+        for line in history_path.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            records.append(json.loads(line))
+        return records
+
+    def _append_history(self, snapshot: AmazonProductSnapshot) -> Path:
+        """将快照附加写入本地历史文件。"""
+        self.history_dir.mkdir(parents=True, exist_ok=True)
+        history_path = self.history_dir / f"{snapshot.asin.upper()}.jsonl"
+        with history_path.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(snapshot.to_dict(include_raw=True), ensure_ascii=False))
+            fh.write("\n")
+        return history_path

opscli/amazon/transport/__init__.py

@@ -0,0 +1,5 @@
+"""amazon 远端传输层。"""
+
+from opscli.amazon.transport.client import AmazonOpsClient
+
+__all__ = ["AmazonOpsClient"]

opscli/amazon/transport/client.py

@@ -0,0 +1,77 @@
+"""amazon 模块远端提交客户端。
+
+当前阶段仅做 API 预留，默认不在 CLI 主流程中使用。
+"""
+
+from __future__ import annotations
+
+import httpx
+
+from opscli.amazon.domain.exceptions import BadRemoteJsonError, RemoteBusinessError, RemoteHttpError, SubmissionConfigError
+from opscli.amazon.domain.models import AmazonProductSnapshot
+from opscli.auth import AuthClient, OPS_URL
+from opscli.auth.config import get_amazon_submit_endpoint
+
+
+class AmazonOpsClient:
+    """统一封装 Amazon 数据提交预留能力。"""
+
+    def __init__(self, auth_client: AuthClient | None = None) -> None:
+        self.auth_client = auth_client or AuthClient()
+        self.ops_url = OPS_URL.rstrip("/")
+        self.default_submit_endpoint = get_amazon_submit_endpoint()
+
+    def submit_snapshot(self, snapshot: AmazonProductSnapshot, *, endpoint: str | None = None) -> dict:
+        """将抓取结果提交到 ops。
+
+        该能力已就绪，但本期默认仅保留作为接口预留。
+        """
+        submit_endpoint = (endpoint or self.default_submit_endpoint or "").strip()
+        if not submit_endpoint:
+            raise SubmissionConfigError(
+                "未配置 Amazon 提交 endpoint，请在 config.ini 的 [systems] 段配置 amazon_submit_endpoint，或通过 --endpoint 指定"
+            )
+        if not submit_endpoint.startswith("/"):
+            raise SubmissionConfigError("提交 endpoint 必须以 / 开头，并相对 OPS_URL 解析")
+
+        headers, cookies = self.auth_client.build_request_auth("ops")
+        response = httpx.post(
+            f"{self.ops_url}{submit_endpoint}",
+            json={
+                "source": "opscli.amazon",
+                "snapshot": snapshot.to_dict(include_raw=True),
+            },
+            headers=headers,
+            cookies=cookies,
+            timeout=10,
+        )
+        return self._parse_response(response)
+
+    def _parse_response(self, response: httpx.Response) -> dict:
+        """统一解析远端响应。"""
+        try:
+            payload = response.json()
+        except Exception as exc:
+            raise BadRemoteJsonError("远端返回了无法解析的 JSON") from exc
+
+        if response.status_code >= 400:
+            message = self._extract_message(payload) or f"远端请求失败，HTTP {response.status_code}"
+            raise RemoteHttpError(response.status_code, message)
+
+        if isinstance(payload, dict):
+            business_code = payload.get("code")
+            if business_code not in (None, 0, 200):
+                message = self._extract_message(payload) or "远端业务执行失败"
+                raise RemoteBusinessError(business_code, message)
+
+        if not isinstance(payload, dict):
+            raise BadRemoteJsonError("远端返回结构不是 JSON 对象")
+        return payload
+
+    def _extract_message(self, payload: dict) -> str | None:
+        """从远端返回中提取错误信息。"""
+        for key in ("msg", "message", "error"):
+            value = payload.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+        return None

opscli/auth/__init__.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from opscli.auth.storage.credential_store import CredentialStore
+from opscli.auth.core.system_registry import SystemRegistry
+from opscli.auth.core.token_manager import TokenManager
+from opscli.auth.config import get_builtin_systems, get_ops_url
+
+# 从配置文件读取，不存在则使用默认值（生产环境）
+BUILTIN_SYSTEMS = get_builtin_systems()
+OPS_URL = get_ops_url()
+
+
+class AuthClient:
+    """供 Skill/Python 代码调用的 SDK 入口"""
+
+    def __init__(self, base_dir: Path | None = None):
+        self._store = CredentialStore(base_dir=base_dir)
+        self._registry = SystemRegistry(
+            base_dir=base_dir,
+            builtin_systems=BUILTIN_SYSTEMS,
+        )
+        self._tm = TokenManager(store=self._store, registry=self._registry)
+
+    def get_token(self, alias: str) -> str:
+        """获取指定系统的有效 JWT（自动刷新）"""
+        return self._tm.get_token(alias)
+
+    def get_session(self, alias: str | None = None) -> str:
+        """获取当前登录态对应的 session_id。
+
+        参数 `alias` 当前仅用于保持调用语义与 `get_token(alias)` 一致，
+        session 本身为全局登录态，不区分具体业务系统。
+        """
+        _ = alias
+        return self._tm.get_session_id()
+
+    def get_device_code(self) -> str | None:
+        """获取当前登录态对应的 device_code。"""
+        data = self._store.load() or {}
+        return data.get("device_code")
+
+    def build_request_auth(self, alias: str) -> tuple[dict[str, str], dict[str, str]]:
+        """构造统一请求认证参数。
+
+        当前统一返回：
+        - Authorization Bearer JWT
+        - polarisUserToken session cookie
+        - opscliDeviceCode cookie（若本地已保存）
+        """
+        token = self.get_token(alias)
+        session_id = self.get_session(alias)
+        headers = {"Authorization": f"Bearer {token}"}
+        cookies = {"polarisUserToken": session_id}
+        device_code = self.get_device_code()
+        if device_code:
+            cookies["opscliDeviceCode"] = device_code
+        return headers, cookies
+
+    def build_session_headers(self, alias: str | None = None) -> dict[str, str]:
+        """构造基于 session 的请求头。"""
+        return {"X-Session-Id": self.get_session(alias)}
+
+    def check_token(self, alias: str) -> dict:
+        """检测指定系统 token 是否有效，返回 {valid, expires_in}"""
+        return self._tm.check_token(alias)
+
+    def is_authenticated(self) -> bool:
+        """是否已登录（session_id 存在且未过期）"""
+        try:
+            self._tm.get_session_id()
+            return True
+        except Exception:
+            return False
+
+    def refresh_token(self, alias: str) -> str:
+        """强制刷新指定系统 JWT"""
+        return self._tm.refresh_token(alias)
+
+
+__all__ = ["AuthClient", "BUILTIN_SYSTEMS", "OPS_URL"]