scraper2-hj3415 2.4.1__py3-none-any.whl → 2.7.0__py3-none-any.whl

scraper2_hj3415/app/adapters/out/playwright/browser.py

@@ -0,0 +1,26 @@
+# scraper2_hj3415/app/adapters/out/playwright/browser.py
+from __future__ import annotations
+
+from playwright.async_api import Page
+
+from .capabilities import (
+    _PlaywrightBase,
+    PlaywrightNavigation,
+    PlaywrightWait,
+    PlaywrightInteraction,
+    PlaywrightText,
+    PlaywrightScope,
+    PlaywrightTable,
+)
+
+
+class PlaywrightBrowser(
+    PlaywrightNavigation,
+    PlaywrightWait,
+    PlaywrightInteraction,
+    PlaywrightText,
+    PlaywrightScope,
+    PlaywrightTable,
+):
+    def __init__(self, page: Page):
+        _PlaywrightBase.__init__(self, page)

{scraper2 → scraper2_hj3415/app}/adapters/out/playwright/browser_factory.py

@@ -1,15 +1,15 @@
-# scraper2/adapters/out/playwright/browser_factory.py
+# scraper2_hj3415/app/adapters/out/playwright/browser_factory.py
 from __future__ import annotations
 
 import asyncio
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
-from typing import AsyncIterator
+from typing import AsyncIterator, cast
 
-from scraper2.app.ports.browser.browser_factory_port import BrowserFactoryPort
-from scraper2.app.ports.browser.browser_port import BrowserPort
-from scraper2.adapters.out.playwright.session import PlaywrightPageSession
-from scraper2.adapters.out.playwright.browser import PlaywrightBrowser
+from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.adapters.out.playwright.session import PlaywrightPageSession
+from scraper2_hj3415.app.adapters.out.playwright.browser import PlaywrightBrowser
 
 
 @dataclass
@@ -54,7 +54,7 @@ class PlaywrightBrowserFactory(BrowserFactoryPort):
             for _ in range(self.max_concurrency):
                 session = PlaywrightPageSession(headless=self.headless, timeout_ms=self.timeout_ms)
                 page = await session.start()
-                browser = PlaywrightBrowser(page)
+                browser:BrowserPort = cast(BrowserPort, PlaywrightBrowser(page))
 
                 item = _LeaseItem(session=session, browser=browser)
                 self._items.append(item)

scraper2_hj3415/app/adapters/out/playwright/capabilities/__init__.py

@@ -0,0 +1,18 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/__init__.py
+from ._base import _PlaywrightBase
+from .navigation import PlaywrightNavigation
+from .wait import PlaywrightWait
+from .interaction import PlaywrightInteraction
+from .text import PlaywrightText
+from .scope import PlaywrightScope
+from .table import PlaywrightTable
+
+__all__ = [
+    "_PlaywrightBase",
+    "PlaywrightNavigation",
+    "PlaywrightWait",
+    "PlaywrightInteraction",
+    "PlaywrightText",
+    "PlaywrightScope",
+    "PlaywrightTable",
+]

scraper2_hj3415/app/adapters/out/playwright/capabilities/_base.py

@@ -0,0 +1,19 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/_base.py
+from __future__ import annotations
+
+from playwright.async_api import Page
+from logging_hj3415 import logger
+
+
+class _PlaywrightBase:
+    def __init__(self, page: Page):
+        self._page = page
+
+    async def _wait_for_network_quiet(self, *, timeout_ms: int = 10_000) -> None:
+        # networkidle은 사이트에 따라 영원히 안 올 수도 있어서 try로 감싸는 게 안전
+        logger.debug("wait for network quiet")
+        try:
+            await self._page.wait_for_load_state("networkidle", timeout=timeout_ms)
+        except Exception:
+            # networkidle이 안 와도 다음 단계(앵커 wait)가 더 중요함
+            return

scraper2_hj3415/app/adapters/out/playwright/capabilities/interaction.py

@@ -0,0 +1,37 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/interaction.py
+from __future__ import annotations
+
+from playwright.async_api import TimeoutError as PwTimeoutError
+from ._base import _PlaywrightBase
+
+
+class PlaywrightInteraction(_PlaywrightBase):
+
+    async def click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 4_000,
+        force: bool = False,
+    ) -> None:
+        loc = self._page.locator(selector).nth(index)
+        await loc.click(timeout=timeout_ms, force=force)
+
+    async def try_click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 1_500,
+        force: bool = False,
+    ) -> bool:
+        loc = self._page.locator(selector).nth(index)
+        try:
+            await loc.click(timeout=timeout_ms, trial=True, force=force)
+            return True
+        except PwTimeoutError:
+            return False
+
+    async def scroll_into_view(self, selector: str, *, index: int = 0) -> None:
+        await self._page.locator(selector).nth(index).scroll_into_view_if_needed()

scraper2_hj3415/app/adapters/out/playwright/capabilities/navigation.py

@@ -0,0 +1,24 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/navigation.py
+from __future__ import annotations
+
+from logging_hj3415 import logger
+from ._base import _PlaywrightBase
+
+
+class PlaywrightNavigation(_PlaywrightBase):
+    async def title(self) -> str:
+        return await self._page.title()
+
+    async def current_url(self) -> str:
+        return self._page.url
+
+    async def goto_and_wait_for_stable(
+        self, url: str, timeout_ms: int = 10_000
+    ) -> None:
+        logger.info(f"goto: {url}")
+        await self._page.goto(url, timeout=timeout_ms, wait_until="domcontentloaded")
+        await self._wait_for_network_quiet(timeout_ms=timeout_ms // 2)
+
+    async def reload(self, *, timeout_ms: int = 10_000) -> None:
+        logger.info("reload")
+        await self._page.reload(timeout=timeout_ms, wait_until="domcontentloaded")

scraper2_hj3415/app/adapters/out/playwright/capabilities/scope.py

@@ -0,0 +1,84 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/scope.py
+from __future__ import annotations
+
+from typing import Any
+from ._base import _PlaywrightBase
+
+
+class PlaywrightScope(_PlaywrightBase):
+    async def is_attached(self, selector: str, *, index: int = 0) -> bool:
+        try:
+            loc = self._page.locator(selector).nth(index)
+            return await loc.count() > 0
+        except Exception:
+            return False
+
+    async def computed_style(self, selector: str, *, index: int = 0, prop: str) -> str:
+        loc = self._page.locator(selector).nth(index)
+        return await loc.evaluate(
+            "(el, prop) => getComputedStyle(el)[prop] || ''", prop
+        )
+
+    async def count_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+    ) -> int:
+        scope = self._page.locator(scope_selector).nth(scope_index)
+        return await scope.locator(inner_selector).count()
+
+    async def eval_in_nth_first(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        expression: str,
+    ) -> Any:
+        scope = self._page.locator(scope_selector).nth(scope_index)
+        loc = scope.locator(inner_selector).first
+
+        if await loc.count() == 0:
+            return None
+
+        return await loc.evaluate(expression)
+
+    async def inner_text_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        scope = self._page.locator(scope_selector).nth(scope_index)
+        inner = scope.locator(inner_selector).nth(inner_index)
+
+        await inner.wait_for(state="attached", timeout=timeout_ms)
+
+        try:
+            return (await inner.inner_text()) or ""
+        except Exception:
+            return ""
+
+    async def text_content_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        scope = self._page.locator(scope_selector).nth(scope_index)
+        inner = scope.locator(inner_selector).nth(inner_index)
+
+        await inner.wait_for(state="attached", timeout=timeout_ms)
+
+        try:
+            return (await inner.text_content()) or ""
+        except Exception:
+            return ""

scraper2_hj3415/app/adapters/out/playwright/capabilities/table.py

@@ -0,0 +1,90 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/table.py
+from __future__ import annotations
+
+import re
+from io import StringIO
+from typing import Any
+import pandas as pd
+from ._base import _PlaywrightBase
+
+_PERIOD_MM_RE = re.compile(r"\b(19|20)\d{2}/(03|06|09|12)\b")
+
+
+class PlaywrightTable(_PlaywrightBase):
+    async def table_records(
+        self,
+        table_selector: str,
+        *,
+        header: int | list[int] | None = 0,
+    ) -> list[dict[str, Any]]:
+        await self.wait_attached(table_selector)
+
+        table = self._page.locator(table_selector).first
+        html = await table.evaluate("el => el.outerHTML")
+
+        try:
+            df = pd.read_html(StringIO(html), header=header)[0]
+        except Exception as e:
+            raise RuntimeError(f"pd.read_html failed: {type(e).__name__}: {e}") from e
+
+        # 문자열 컬럼일 때만 정규화
+        if all(isinstance(c, str) for c in df.columns):
+            if "항목" in df.columns:
+                df["항목"] = (
+                    df["항목"].astype(str).str.replace("펼치기", "").str.strip()
+                )
+
+            df.columns = (
+                df.columns.astype(str)
+                .str.replace("연간컨센서스보기", "", regex=False)
+                .str.replace("연간컨센서스닫기", "", regex=False)
+                .str.replace("(IFRS연결)", "", regex=False)
+                .str.replace("(IFRS별도)", "", regex=False)
+                .str.replace("(GAAP개별)", "", regex=False)
+                .str.replace("(YoY)", "", regex=False)
+                .str.replace("(QoQ)", "", regex=False)
+                .str.replace("(E)", "", regex=False)
+                .str.replace(".", "", regex=False)
+                .str.strip()
+            )
+
+        return df.where(pd.notnull(df), None).to_dict(orient="records")
+
+    async def table_header_texts_nth(
+        self, table_selector: str, *, index: int
+    ) -> list[str]:
+        table = self._page.locator(table_selector).nth(index)
+        thead = table.locator("thead")
+        if await thead.count() == 0:
+            return []
+
+        ths = thead.locator("th")
+        try:
+            texts = await ths.all_inner_texts()
+        except Exception:
+            texts = []
+
+        out: list[str] = []
+        for t in texts:
+            t = " ".join((t or "").split())
+            if t:
+                out.append(t)
+        return out
+
+    async def table_header_periods_mm_nth(
+        self, table_selector: str, *, index: int
+    ) -> list[str]:
+        texts = await self.table_header_texts_nth(table_selector, index=index)
+        periods: list[str] = []
+        for t in texts:
+            for m in _PERIOD_MM_RE.finditer(t):
+                periods.append(m.group(0))
+
+        seen: set[str] = set()
+        uniq: list[str] = []
+        for p in periods:
+            if p in seen:
+                continue
+            seen.add(p)
+            uniq.append(p)
+        return uniq

scraper2_hj3415/app/adapters/out/playwright/capabilities/text.py

@@ -0,0 +1,25 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/text.py
+from __future__ import annotations
+from ._base import _PlaywrightBase
+
+
+class PlaywrightText(_PlaywrightBase):
+    async def count(self, selector: str) -> int:
+        return await self._page.locator(selector).count()
+
+    async def text_content_first(self, selector: str) -> str:
+        return (await self._page.locator(selector).first.text_content()) or ""
+
+    async def all_texts(self, selector: str) -> list[str]:
+        loc = self._page.locator(selector)
+        return await loc.all_text_contents()
+
+    async def get_text_by_text(self, needle: str) -> str:
+        return (await self._page.get_by_text(needle).first.text_content()) or ""
+
+    async def inner_text(self, selector: str) -> str:
+        return await self._page.locator(selector).first.inner_text()
+
+    async def outer_html_nth(self, selector: str, index: int) -> str:
+        loc = self._page.locator(selector).nth(index)
+        return await loc.evaluate("el => el.outerHTML")

scraper2_hj3415/app/adapters/out/playwright/capabilities/wait.py

@@ -0,0 +1,96 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/wait.py
+from __future__ import annotations
+
+import asyncio
+import time
+from logging_hj3415 import logger
+from ._base import _PlaywrightBase
+
+class PlaywrightWait(_PlaywrightBase):
+    """
+    wait 관련은 _page만 필요하므로 _PlaywrightBase 상속 안 해도 되지만,
+    여기서는 self._page가 있다고 가정(PlaywrightBrowser가 베이스 제공).
+    """
+
+    async def sleep_ms(self, ms: int) -> None:
+        await asyncio.sleep(ms / 1000)
+
+    async def wait_attached(self, selector: str, *, timeout_ms: int = 10_000) -> None:
+        await self._page.locator(selector).first.wait_for(state="attached", timeout=timeout_ms)
+
+    async def wait_visible(self, selector: str, *, timeout_ms: int = 10_000) -> None:
+        await self._page.locator(selector).first.wait_for(state="visible", timeout=timeout_ms)
+
+    async def wait_table_nth_ready(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+        min_rows: int = 1,
+        timeout_ms: int = 20_000,
+        poll_ms: int = 200,
+    ) -> None:
+        logger.debug("wait for table nth_ready")
+        table = self._page.locator(table_selector).nth(index)
+        await table.wait_for(state="attached", timeout=timeout_ms)
+
+        rows = table.locator("tbody tr")
+        deadline = time.monotonic() + timeout_ms / 1000
+
+        cnt = 0
+        while time.monotonic() < deadline:
+            try:
+                cnt = await rows.count()
+            except Exception:
+                cnt = 0
+
+            if cnt >= min_rows:
+                return
+
+            await asyncio.sleep(poll_ms / 1000)
+
+        logger.warning(f"table rows timeout: last_cnt={cnt}, need>={min_rows}")
+        raise TimeoutError(f"nth table not ready: index={index}, rows<{min_rows}")
+
+    async def wait_table_text_changed(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+        prev_text: str | None,
+        min_rows: int = 1,
+        min_lines: int = 50,
+        timeout_sec: float = 12.0,
+        poll_sec: float = 0.2,
+    ) -> str:
+        # 0) row 기준 ready
+        await self.wait_table_nth_ready(
+            table_selector,
+            index=index,
+            min_rows=min_rows,
+            timeout_ms=int(timeout_sec * 1000),
+            poll_ms=int(poll_sec * 1000),
+        )
+
+        start = time.monotonic()
+        last_text = ""
+
+        while True:
+            loc = self._page.locator(table_selector).nth(index)
+            try:
+                text = await loc.inner_text()
+            except Exception:
+                text = ""
+
+            lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
+            is_valid = len(lines) >= min_lines
+
+            if is_valid:
+                last_text = text
+                if prev_text is None or text != prev_text:
+                    return text
+
+            if time.monotonic() - start >= timeout_sec:
+                return last_text
+
+            await asyncio.sleep(poll_sec)

{scraper2 → scraper2_hj3415/app}/adapters/out/playwright/session.py

@@ -1,4 +1,4 @@
-# src/scraper2/adapters/out/playwright/session.py
+# scraper2_hj3415/app/adapters/out/playwright/session.py
 from __future__ import annotations
 
 import os

scraper2_hj3415/app/adapters/out/sinks/memory_sink.py

@@ -0,0 +1,25 @@
+# scraper2_hj3415/app/adapters/out/sinks/memory_sink.py
+from __future__ import annotations
+
+from typing import Iterable
+
+from contracts_hj3415.nfs.nfs_dto import NfsDTO
+from contracts_hj3415.nfs.types import Endpoint
+
+from scraper2_hj3415.app.adapters.out.sinks.store import InMemoryStore
+
+
+class MemorySink:
+    def __init__(self, store: InMemoryStore[NfsDTO]):
+        self._store = store
+
+    async def write(self, dto: NfsDTO, *, endpoint: Endpoint) -> None:
+        await self._store.put(endpoint, dto.code, dto)
+
+    async def write_many(
+        self,
+        dtos: Iterable[NfsDTO],
+        *,
+        endpoint: Endpoint,
+    ) -> None:
+        await self._store.put_many(endpoint, ((d.code, d) for d in dtos))

scraper2_hj3415/app/adapters/out/sinks/mongo_sink.py

@@ -0,0 +1,63 @@
+# scraper2_hj3415/app/adapters/out/sinks/mongo_sink.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable
+
+from pymongo.asynchronous.database import AsyncDatabase
+
+from contracts_hj3415.nfs.nfs_dto import NfsDTO
+from contracts_hj3415.nfs.types import Endpoint
+
+from db2_hj3415.nfs.repo import (
+    upsert_latest,
+    upsert_latest_many,
+    insert_snapshot,
+    insert_snapshots_many,
+)
+
+
+class MongoSink:
+    def __init__(self, db: AsyncDatabase):
+        self._db = db
+
+    async def write(self, dto: NfsDTO, *, endpoint: Endpoint) -> None:
+        code = str(dto.code).strip()
+        if not code:
+            return
+
+        payload = dict(dto.payload)  # Mapping 방어
+
+        await upsert_latest(
+            self._db, endpoint=endpoint, code=code, payload=payload, asof=dto.asof
+        )
+        await insert_snapshot(
+            self._db, endpoint=endpoint, code=code, payload=payload, asof=dto.asof
+        )
+
+    async def write_many(
+        self,
+        dtos: Iterable[NfsDTO],
+        *,
+        endpoint: Endpoint,
+    ) -> None:
+        items: dict[str, dict] = {}
+        ts: datetime | None = None
+
+        for dto in dtos:
+            code = str(dto.code).strip()
+            if not code:
+                continue
+            items[code] = dict(dto.payload)
+            if ts is None:
+                ts = dto.asof  # 첫 dto의 asof를 배치 기준으로
+
+        if not items:
+            return
+
+        await upsert_latest_many(
+            self._db, endpoint=endpoint, items=items, asof=ts
+        )
+        await insert_snapshots_many(
+            self._db, endpoint=endpoint, items=items, asof=ts
+        )

{scraper2/adapters/out/sinks/memory → scraper2_hj3415/app/adapters/out/sinks}/store.py

@@ -1,29 +1,34 @@
-# scraper2/adapters/out/sinks/memory/store.py
+# scraper2_hj3415/app/adapters/out/sinks/store.py
 from __future__ import annotations
 
 import asyncio
 from collections import defaultdict, deque
 from dataclasses import dataclass
-from typing import Any, Deque, Dict, Generic, Iterable, List, Optional, Tuple, TypeVar
+from typing import Deque, Dict, Generic, Iterable, List, Optional, Tuple, TypeVar
 
 T = TypeVar("T")  # DTO 타입
 
+
 @dataclass(frozen=True)
 class StoreStats:
     endpoint: str
     latest_count: int
     history_count: int
 
+
 class InMemoryStore(Generic[T]):
     """
     endpoint별로 DTO를 저장한다.
     - latest: endpoint -> key(보통 코드) -> dto
     - history: endpoint -> deque[dto] (최근 max_history개)
     """
+
     def __init__(self, *, max_history: int = 2000):
         self._lock = asyncio.Lock()
         self._max_history = max_history
-        self._history: Dict[str, Deque[T]] = defaultdict(lambda: deque(maxlen=max_history))
+        self._history: Dict[str, Deque[T]] = defaultdict(
+            lambda: deque(maxlen=max_history)
+        )
         self._latest: Dict[str, Dict[str, T]] = defaultdict(dict)
 
     # ---------- write ----------
@@ -62,7 +67,11 @@ class InMemoryStore(Generic[T]):
         async with self._lock:
             latest_count = len(self._latest.get(endpoint, {}))
             history_count = len(self._history.get(endpoint, []))
-            return StoreStats(endpoint=endpoint, latest_count=latest_count, history_count=history_count)
+            return StoreStats(
+                endpoint=endpoint,
+                latest_count=latest_count,
+                history_count=history_count,
+            )
 
     async def clear(self, endpoint: str | None = None) -> None:
         async with self._lock:
@@ -71,4 +80,4 @@ class InMemoryStore(Generic[T]):
                 self._latest.clear()
             else:
                 self._history.pop(endpoint, None)
-                self._latest.pop(endpoint, None)
+                self._latest.pop(endpoint, None)