scraper2-hj3415 2.6.0__tar.gz → 2.7.0__tar.gz

{scraper2_hj3415-2.6.0 → scraper2_hj3415-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scraper2-hj3415
-Version: 2.6.0
+Version: 2.7.0
 Summary: Naver WiseReport scraper
 Keywords: example,demo
 Author-email: Hyungjin Kim <hj3415@gmail.com>

{scraper2_hj3415-2.6.0 → scraper2_hj3415-2.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "scraper2-hj3415"                          # PyPI 이름 (하이픈 허용)
-version = "2.6.0"
+version = "2.7.0"
 description = "Naver WiseReport scraper"
 readme = "README.md"
 requires-python = ">=3.11"

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/browser.py

@@ -0,0 +1,26 @@
+# scraper2_hj3415/app/adapters/out/playwright/browser.py
+from __future__ import annotations
+
+from playwright.async_api import Page
+
+from .capabilities import (
+    _PlaywrightBase,
+    PlaywrightNavigation,
+    PlaywrightWait,
+    PlaywrightInteraction,
+    PlaywrightText,
+    PlaywrightScope,
+    PlaywrightTable,
+)
+
+
+class PlaywrightBrowser(
+    PlaywrightNavigation,
+    PlaywrightWait,
+    PlaywrightInteraction,
+    PlaywrightText,
+    PlaywrightScope,
+    PlaywrightTable,
+):
+    def __init__(self, page: Page):
+        _PlaywrightBase.__init__(self, page)

{scraper2_hj3415-2.6.0 → scraper2_hj3415-2.7.0}/src/scraper2_hj3415/app/adapters/out/playwright/browser_factory.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import asyncio
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
-from typing import AsyncIterator
+from typing import AsyncIterator, cast
 
 from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
 from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
@@ -54,7 +54,7 @@ class PlaywrightBrowserFactory(BrowserFactoryPort):
             for _ in range(self.max_concurrency):
                 session = PlaywrightPageSession(headless=self.headless, timeout_ms=self.timeout_ms)
                 page = await session.start()
-                browser = PlaywrightBrowser(page)
+                browser:BrowserPort = cast(BrowserPort, PlaywrightBrowser(page))
 
                 item = _LeaseItem(session=session, browser=browser)
                 self._items.append(item)

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/capabilities/__init__.py

@@ -0,0 +1,18 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/__init__.py
+from ._base import _PlaywrightBase
+from .navigation import PlaywrightNavigation
+from .wait import PlaywrightWait
+from .interaction import PlaywrightInteraction
+from .text import PlaywrightText
+from .scope import PlaywrightScope
+from .table import PlaywrightTable
+
+__all__ = [
+    "_PlaywrightBase",
+    "PlaywrightNavigation",
+    "PlaywrightWait",
+    "PlaywrightInteraction",
+    "PlaywrightText",
+    "PlaywrightScope",
+    "PlaywrightTable",
+]

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/capabilities/_base.py

@@ -0,0 +1,19 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/_base.py
+from __future__ import annotations
+
+from playwright.async_api import Page
+from logging_hj3415 import logger
+
+
+class _PlaywrightBase:
+    def __init__(self, page: Page):
+        self._page = page
+
+    async def _wait_for_network_quiet(self, *, timeout_ms: int = 10_000) -> None:
+        # networkidle은 사이트에 따라 영원히 안 올 수도 있어서 try로 감싸는 게 안전
+        logger.debug("wait for network quiet")
+        try:
+            await self._page.wait_for_load_state("networkidle", timeout=timeout_ms)
+        except Exception:
+            # networkidle이 안 와도 다음 단계(앵커 wait)가 더 중요함
+            return

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/capabilities/interaction.py

@@ -0,0 +1,37 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/interaction.py
+from __future__ import annotations
+
+from playwright.async_api import TimeoutError as PwTimeoutError
+from ._base import _PlaywrightBase
+
+
+class PlaywrightInteraction(_PlaywrightBase):
+
+    async def click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 4_000,
+        force: bool = False,
+    ) -> None:
+        loc = self._page.locator(selector).nth(index)
+        await loc.click(timeout=timeout_ms, force=force)
+
+    async def try_click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 1_500,
+        force: bool = False,
+    ) -> bool:
+        loc = self._page.locator(selector).nth(index)
+        try:
+            await loc.click(timeout=timeout_ms, trial=True, force=force)
+            return True
+        except PwTimeoutError:
+            return False
+
+    async def scroll_into_view(self, selector: str, *, index: int = 0) -> None:
+        await self._page.locator(selector).nth(index).scroll_into_view_if_needed()

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/capabilities/navigation.py

@@ -0,0 +1,24 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/navigation.py
+from __future__ import annotations
+
+from logging_hj3415 import logger
+from ._base import _PlaywrightBase
+
+
+class PlaywrightNavigation(_PlaywrightBase):
+    async def title(self) -> str:
+        return await self._page.title()
+
+    async def current_url(self) -> str:
+        return self._page.url
+
+    async def goto_and_wait_for_stable(
+        self, url: str, timeout_ms: int = 10_000
+    ) -> None:
+        logger.info(f"goto: {url}")
+        await self._page.goto(url, timeout=timeout_ms, wait_until="domcontentloaded")
+        await self._wait_for_network_quiet(timeout_ms=timeout_ms // 2)
+
+    async def reload(self, *, timeout_ms: int = 10_000) -> None:
+        logger.info("reload")
+        await self._page.reload(timeout=timeout_ms, wait_until="domcontentloaded")

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/capabilities/scope.py

@@ -0,0 +1,84 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/scope.py
+from __future__ import annotations
+
+from typing import Any
+from ._base import _PlaywrightBase
+
+
+class PlaywrightScope(_PlaywrightBase):
+    async def is_attached(self, selector: str, *, index: int = 0) -> bool:
+        try:
+            loc = self._page.locator(selector).nth(index)
+            return await loc.count() > 0
+        except Exception:
+            return False
+
+    async def computed_style(self, selector: str, *, index: int = 0, prop: str) -> str:
+        loc = self._page.locator(selector).nth(index)
+        return await loc.evaluate(
+            "(el, prop) => getComputedStyle(el)[prop] || ''", prop
+        )
+
+    async def count_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+    ) -> int:
+        scope = self._page.locator(scope_selector).nth(scope_index)
+        return await scope.locator(inner_selector).count()
+
+    async def eval_in_nth_first(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        expression: str,
+    ) -> Any:
+        scope = self._page.locator(scope_selector).nth(scope_index)
+        loc = scope.locator(inner_selector).first
+
+        if await loc.count() == 0:
+            return None
+
+        return await loc.evaluate(expression)
+
+    async def inner_text_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        scope = self._page.locator(scope_selector).nth(scope_index)
+        inner = scope.locator(inner_selector).nth(inner_index)
+
+        await inner.wait_for(state="attached", timeout=timeout_ms)
+
+        try:
+            return (await inner.inner_text()) or ""
+        except Exception:
+            return ""
+
+    async def text_content_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        scope = self._page.locator(scope_selector).nth(scope_index)
+        inner = scope.locator(inner_selector).nth(inner_index)
+
+        await inner.wait_for(state="attached", timeout=timeout_ms)
+
+        try:
+            return (await inner.text_content()) or ""
+        except Exception:
+            return ""

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/capabilities/table.py

@@ -0,0 +1,90 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/table.py
+from __future__ import annotations
+
+import re
+from io import StringIO
+from typing import Any
+import pandas as pd
+from ._base import _PlaywrightBase
+
+_PERIOD_MM_RE = re.compile(r"\b(19|20)\d{2}/(03|06|09|12)\b")
+
+
+class PlaywrightTable(_PlaywrightBase):
+    async def table_records(
+        self,
+        table_selector: str,
+        *,
+        header: int | list[int] | None = 0,
+    ) -> list[dict[str, Any]]:
+        await self.wait_attached(table_selector)
+
+        table = self._page.locator(table_selector).first
+        html = await table.evaluate("el => el.outerHTML")
+
+        try:
+            df = pd.read_html(StringIO(html), header=header)[0]
+        except Exception as e:
+            raise RuntimeError(f"pd.read_html failed: {type(e).__name__}: {e}") from e
+
+        # 문자열 컬럼일 때만 정규화
+        if all(isinstance(c, str) for c in df.columns):
+            if "항목" in df.columns:
+                df["항목"] = (
+                    df["항목"].astype(str).str.replace("펼치기", "").str.strip()
+                )
+
+            df.columns = (
+                df.columns.astype(str)
+                .str.replace("연간컨센서스보기", "", regex=False)
+                .str.replace("연간컨센서스닫기", "", regex=False)
+                .str.replace("(IFRS연결)", "", regex=False)
+                .str.replace("(IFRS별도)", "", regex=False)
+                .str.replace("(GAAP개별)", "", regex=False)
+                .str.replace("(YoY)", "", regex=False)
+                .str.replace("(QoQ)", "", regex=False)
+                .str.replace("(E)", "", regex=False)
+                .str.replace(".", "", regex=False)
+                .str.strip()
+            )
+
+        return df.where(pd.notnull(df), None).to_dict(orient="records")
+
+    async def table_header_texts_nth(
+        self, table_selector: str, *, index: int
+    ) -> list[str]:
+        table = self._page.locator(table_selector).nth(index)
+        thead = table.locator("thead")
+        if await thead.count() == 0:
+            return []
+
+        ths = thead.locator("th")
+        try:
+            texts = await ths.all_inner_texts()
+        except Exception:
+            texts = []
+
+        out: list[str] = []
+        for t in texts:
+            t = " ".join((t or "").split())
+            if t:
+                out.append(t)
+        return out
+
+    async def table_header_periods_mm_nth(
+        self, table_selector: str, *, index: int
+    ) -> list[str]:
+        texts = await self.table_header_texts_nth(table_selector, index=index)
+        periods: list[str] = []
+        for t in texts:
+            for m in _PERIOD_MM_RE.finditer(t):
+                periods.append(m.group(0))
+
+        seen: set[str] = set()
+        uniq: list[str] = []
+        for p in periods:
+            if p in seen:
+                continue
+            seen.add(p)
+            uniq.append(p)
+        return uniq

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/capabilities/text.py

@@ -0,0 +1,25 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/text.py
+from __future__ import annotations
+from ._base import _PlaywrightBase
+
+
+class PlaywrightText(_PlaywrightBase):
+    async def count(self, selector: str) -> int:
+        return await self._page.locator(selector).count()
+
+    async def text_content_first(self, selector: str) -> str:
+        return (await self._page.locator(selector).first.text_content()) or ""
+
+    async def all_texts(self, selector: str) -> list[str]:
+        loc = self._page.locator(selector)
+        return await loc.all_text_contents()
+
+    async def get_text_by_text(self, needle: str) -> str:
+        return (await self._page.get_by_text(needle).first.text_content()) or ""
+
+    async def inner_text(self, selector: str) -> str:
+        return await self._page.locator(selector).first.inner_text()
+
+    async def outer_html_nth(self, selector: str, index: int) -> str:
+        loc = self._page.locator(selector).nth(index)
+        return await loc.evaluate("el => el.outerHTML")

scraper2_hj3415-2.7.0/src/scraper2_hj3415/app/adapters/out/playwright/capabilities/wait.py

@@ -0,0 +1,96 @@
+# scraper2_hj3415/app/adapters/out/playwright/capabilities/wait.py
+from __future__ import annotations
+
+import asyncio
+import time
+from logging_hj3415 import logger
+from ._base import _PlaywrightBase
+
+class PlaywrightWait(_PlaywrightBase):
+    """
+    wait 관련은 _page만 필요하므로 _PlaywrightBase 상속 안 해도 되지만,
+    여기서는 self._page가 있다고 가정(PlaywrightBrowser가 베이스 제공).
+    """
+
+    async def sleep_ms(self, ms: int) -> None:
+        await asyncio.sleep(ms / 1000)
+
+    async def wait_attached(self, selector: str, *, timeout_ms: int = 10_000) -> None:
+        await self._page.locator(selector).first.wait_for(state="attached", timeout=timeout_ms)
+
+    async def wait_visible(self, selector: str, *, timeout_ms: int = 10_000) -> None:
+        await self._page.locator(selector).first.wait_for(state="visible", timeout=timeout_ms)
+
+    async def wait_table_nth_ready(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+        min_rows: int = 1,
+        timeout_ms: int = 20_000,
+        poll_ms: int = 200,
+    ) -> None:
+        logger.debug("wait for table nth_ready")
+        table = self._page.locator(table_selector).nth(index)
+        await table.wait_for(state="attached", timeout=timeout_ms)
+
+        rows = table.locator("tbody tr")
+        deadline = time.monotonic() + timeout_ms / 1000
+
+        cnt = 0
+        while time.monotonic() < deadline:
+            try:
+                cnt = await rows.count()
+            except Exception:
+                cnt = 0
+
+            if cnt >= min_rows:
+                return
+
+            await asyncio.sleep(poll_ms / 1000)
+
+        logger.warning(f"table rows timeout: last_cnt={cnt}, need>={min_rows}")
+        raise TimeoutError(f"nth table not ready: index={index}, rows<{min_rows}")
+
+    async def wait_table_text_changed(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+        prev_text: str | None,
+        min_rows: int = 1,
+        min_lines: int = 50,
+        timeout_sec: float = 12.0,
+        poll_sec: float = 0.2,
+    ) -> str:
+        # 0) row 기준 ready
+        await self.wait_table_nth_ready(
+            table_selector,
+            index=index,
+            min_rows=min_rows,
+            timeout_ms=int(timeout_sec * 1000),
+            poll_ms=int(poll_sec * 1000),
+        )
+
+        start = time.monotonic()
+        last_text = ""
+
+        while True:
+            loc = self._page.locator(table_selector).nth(index)
+            try:
+                text = await loc.inner_text()
+            except Exception:
+                text = ""
+
+            lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
+            is_valid = len(lines) >= min_lines
+
+            if is_valid:
+                last_text = text
+                if prev_text is None or text != prev_text:
+                    return text
+
+            if time.monotonic() - start >= timeout_sec:
+                return last_text
+
+            await asyncio.sleep(poll_sec)

{scraper2_hj3415-2.6.0 → scraper2_hj3415-2.7.0}/src/scraper2_hj3415/app/adapters/out/sinks/memory_sink.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 from typing import Iterable
 
 from contracts_hj3415.nfs.nfs_dto import NfsDTO
-from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.types import Endpoint
 
 from scraper2_hj3415.app.adapters.out.sinks.store import InMemoryStore
 
@@ -13,13 +13,13 @@ class MemorySink:
     def __init__(self, store: InMemoryStore[NfsDTO]):
         self._store = store
 
-    async def write(self, dto: NfsDTO, *, endpoint: Endpoints) -> None:
+    async def write(self, dto: NfsDTO, *, endpoint: Endpoint) -> None:
         await self._store.put(endpoint, dto.code, dto)
 
     async def write_many(
         self,
         dtos: Iterable[NfsDTO],
         *,
-        endpoint: Endpoints,
+        endpoint: Endpoint,
     ) -> None:
         await self._store.put_many(endpoint, ((d.code, d) for d in dtos))

{scraper2_hj3415-2.6.0 → scraper2_hj3415-2.7.0}/src/scraper2_hj3415/app/adapters/out/sinks/mongo_sink.py

@@ -7,13 +7,13 @@ from typing import Iterable
 from pymongo.asynchronous.database import AsyncDatabase
 
 from contracts_hj3415.nfs.nfs_dto import NfsDTO
-from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.types import Endpoint
 
 from db2_hj3415.nfs.repo import (
-    upsert_latest_payload,
-    upsert_latest_payload_many,
-    insert_snapshot_payload,
-    insert_snapshots_payload_many,
+    upsert_latest,
+    upsert_latest_many,
+    insert_snapshot,
+    insert_snapshots_many,
 )
 
 
@@ -21,17 +21,17 @@ class MongoSink:
     def __init__(self, db: AsyncDatabase):
         self._db = db
 
-    async def write(self, dto: NfsDTO, *, endpoint: Endpoints) -> None:
+    async def write(self, dto: NfsDTO, *, endpoint: Endpoint) -> None:
         code = str(dto.code).strip()
         if not code:
             return
 
         payload = dict(dto.payload)  # Mapping 방어
 
-        await upsert_latest_payload(
+        await upsert_latest(
             self._db, endpoint=endpoint, code=code, payload=payload, asof=dto.asof
         )
-        await insert_snapshot_payload(
+        await insert_snapshot(
             self._db, endpoint=endpoint, code=code, payload=payload, asof=dto.asof
         )
 
@@ -39,7 +39,7 @@ class MongoSink:
         self,
         dtos: Iterable[NfsDTO],
         *,
-        endpoint: Endpoints,
+        endpoint: Endpoint,
     ) -> None:
         items: dict[str, dict] = {}
         ts: datetime | None = None
@@ -55,9 +55,9 @@ class MongoSink:
         if not items:
             return
 
-        await upsert_latest_payload_many(
+        await upsert_latest_many(
             self._db, endpoint=endpoint, items=items, asof=ts
         )
-        await insert_snapshots_payload_many(
+        await insert_snapshots_many(
             self._db, endpoint=endpoint, items=items, asof=ts
         )