scraper2-hj3415 1.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

scraper2/adapters/out/playwright/browser.py

@@ -0,0 +1,103 @@
+# src/scraper2/adapters/out/playwright/session.py
+from __future__ import annotations
+from typing import Any
+from io import StringIO
+import pandas as pd
+from playwright.async_api import Page
+
+class PlaywrightBrowser:
+    def __init__(self, page: Page):
+        self.page = page
+
+    async def goto(self, url: str, timeout_ms: int = 10_000) -> None:
+        self.page.set_default_timeout(10_000)
+        await self.page.goto(url, timeout=timeout_ms)
+
+    async def title(self) -> str:
+        return await self.page.title()
+
+    async def current_url(self) -> str:
+        return self.page.url
+
+    async def wait(self, selector: str, timeout_ms: int = 10_000) -> None:
+        await self.page.wait_for_selector(selector, timeout=timeout_ms, state="attached")
+
+    async def text(self, selector: str) -> str:
+        await self.wait(selector)
+        return (await self.page.locator(selector).first.text_content()) or ""
+
+    async def texts(self, selector: str) -> list[str]:
+        await self.wait(selector)
+        loc = self.page.locator(selector)
+        items = await loc.all()
+        out: list[str] = []
+        for it in items:
+            out.append((await it.text_content()) or "")
+        return out
+
+    async def text_first_by_text(self, needle: str) -> str:
+        return (await self.page.get_by_text(needle).first.text_content()) or ""
+
+    async def inner_text(self, selector: str) -> str:
+        await self.wait(selector)
+        return await self.page.locator(selector).first.inner_text()
+
+    async def click(self, selector: str) -> None:
+        await self.wait(selector)
+        await self.page.locator(selector).click()
+
+    async def table_records(
+            self,
+            table_selector: str,
+            *,
+            header: int | list[int] = 0
+    ) -> list[dict[str, Any]]:
+        await self.wait(table_selector)
+
+        table = self.page.locator(table_selector).first
+        html = await table.evaluate("el => el.outerHTML")  # <table> 포함
+        #print(html)
+
+        try:
+            df = pd.read_html(StringIO(html), header=header)[0]
+            #print(df.head(3))
+        except Exception as e:
+            # ImportError(lxml 없음), ValueError 등 모두 여기서 잡아서 원인 노출
+            raise RuntimeError(f"pd.read_html failed: {type(e).__name__}: {e}") from e
+
+        if header == 0:
+            if "항목" in df.columns:
+                df["항목"] = df["항목"].astype(str).str.replace("펼치기", "").str.strip()
+
+            df.columns = (
+                df.columns.astype(str)
+                .str.replace("연간컨센서스보기", "", regex=False)
+                .str.replace("연간컨센서스닫기", "", regex=False)
+                .str.replace("(IFRS연결)", "", regex=False)
+                .str.replace("(IFRS별도)", "", regex=False)
+                .str.replace("(GAAP개별)", "", regex=False)
+                .str.replace("(YoY)", "", regex=False)
+                .str.replace("(QoQ)", "", regex=False)
+                .str.replace("(E)", "", regex=False)
+                .str.replace(".", "", regex=False)
+                .str.strip()
+            )
+
+        # NaN -> None 처리
+        records: list[dict[str, Any]] = df.where(pd.notnull(df), None).to_dict(orient="records")
+        return records
+
+    async def outer_html(self, selector: str) -> str:
+        loc = self.page.locator(selector).first
+        return await loc.evaluate("el => el.outerHTML")
+
+    async def all_texts(self, selector: str) -> list[str]:
+        # selector는 css도 되고, "xpath=..." 도 됨
+        loc = self.page.locator(selector)
+        return await loc.all_text_contents()
+
+    async def outer_html_nth(self, selector: str, index: int) -> str:
+        loc = self.page.locator(selector).nth(index)
+        # index가 범위를 벗어나면 playwright가 에러를 내는데,
+        # 필요하면 여기서 더 친절한 에러로 감싸도 됨.
+        return await loc.evaluate("el => el.outerHTML")

scraper2/adapters/out/playwright/browser_factory.py

@@ -0,0 +1,112 @@
+# scraper2/adapters/out/playwright/browser_factory.py
+from __future__ import annotations
+
+import asyncio
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import AsyncIterator
+
+from scraper2.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2.app.ports.browser.browser_port import BrowserPort
+from scraper2.adapters.out.playwright.session import PlaywrightPageSession
+from scraper2.adapters.out.playwright.browser import PlaywrightBrowser
+
+
+@dataclass
+class _LeaseItem:
+    session: PlaywrightPageSession
+    browser: BrowserPort
+
+
+class PlaywrightBrowserFactory(BrowserFactoryPort):
+    """
+    풀링 방식:
+    - astart()에서 max_concurrency 만큼 세션/페이지/브라우저를 미리 생성
+    - lease()는 큐에서 하나 빌려주고 반납받음
+    - aclose()에서 모두 종료
+    """
+
+    def __init__(self, *, headless: bool, timeout_ms: int, max_concurrency: int = 2):
+        self.headless = headless
+        self.timeout_ms = timeout_ms
+        self.max_concurrency = max_concurrency
+
+        self._pool: asyncio.Queue[_LeaseItem] = asyncio.Queue(maxsize=max_concurrency)
+        self._items: list[_LeaseItem] = []  # 종료용 레퍼런스
+        self._started = False
+        self._start_lock = asyncio.Lock()
+        self._closed = False
+
+    async def astart(self) -> None:
+        """
+        풀을 미리 채움.
+        여러 번 호출돼도 1회만 초기화되도록 방어.
+        """
+        if self._started:
+            return
+
+        async with self._start_lock:
+            if self._started:
+                return
+            if self._closed:
+                raise RuntimeError("Factory is closed; cannot start again.")
+
+            for _ in range(self.max_concurrency):
+                session = PlaywrightPageSession(headless=self.headless, timeout_ms=self.timeout_ms)
+                page = await session.start()
+                browser = PlaywrightBrowser(page)
+
+                item = _LeaseItem(session=session, browser=browser)
+                self._items.append(item)
+                await self._pool.put(item)
+
+            self._started = True
+
+    @asynccontextmanager
+    async def lease(self) -> AsyncIterator[BrowserPort]:
+        """
+        브라우저 하나를 풀에서 빌려줌.
+        사용 후 반드시 풀에 반납.
+        """
+        if self._closed:
+            raise RuntimeError("Factory is closed; cannot lease.")
+        if not self._started:
+            await self.astart()
+
+        item = await self._pool.get()
+        try:
+            yield item.browser
+        finally:
+            # close 중이면 반납하지 말고 그냥 종료 플로우에 맡김
+            if not self._closed:
+                await self._pool.put(item)
+
+    async def aclose(self) -> None:
+        """
+        풀에 있는 모든 세션 종료.
+        - 실행 중인 lease가 끝나기 전에 닫으면: 남아있는 세션만 닫히고,
+          나중에 lease가 반납하려 할 때 _closed=True라 put이 안 되도록 처리.
+        """
+        if self._closed:
+            return
+        self._closed = True
+
+        # 전체 세션 종료
+        # (이미 대여 중인 애도 결국 같은 session 객체이므로 close 시도됨)
+        for item in self._items:
+            try:
+                await item.session.close()
+            except Exception:
+                # 종료 단계에서는 예외 삼키는 게 보통 안전
+                pass
+
+        self._items.clear()
+
+        # 큐 비우기 (참조 제거)
+        try:
+            while True:
+                self._pool.get_nowait()
+        except asyncio.QueueEmpty:
+            pass
+
+

scraper2/adapters/out/playwright/session.py

@@ -0,0 +1,121 @@
+# src/scraper2/adapters/out/playwright/session.py
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from typing import Optional
+
+from playwright.async_api import (
+    async_playwright,
+    Browser,
+    BrowserContext,
+    Page,
+    Error as PWError,
+)
+
+
+def _install_playwright_browsers(*names: str) -> None:
+    """python -m playwright install [names...] 를 코드에서 실행"""
+    subprocess.run([sys.executable, "-m", "playwright", "install", *names], check=True)
+
+    if sys.platform.startswith("linux"):
+        # deps는 실패해도 그냥 진행 (환경에 따라 불필요/권한 문제)
+        try:
+            subprocess.run(
+                [sys.executable, "-m", "playwright", "install-deps"], check=True
+            )
+        except Exception:
+            pass
+
+
+def _need_install(e: Exception) -> bool:
+    msg = str(e)
+    return (
+        "Executable doesn't exist" in msg
+        or "download new browsers" in msg
+        or "playwright install" in msg
+        or "Please run the following command" in msg
+    )
+
+
+@dataclass
+class PlaywrightPageSession:
+    """
+    main에서 쓰기 쉬운 세션:
+        s = PlaywrightPageSession(headless=True)
+        page = await s.start()
+        ...
+        await s.close()
+    """
+
+    headless: bool = True
+    browser_name: str = "chromium"
+    timeout_ms: int = 10_000
+    auto_install: bool = True  # env PW_SKIP_AUTO_INSTALL=1이면 자동으로 꺼짐
+
+    # runtime resources
+    pw: Optional[object] = None
+    browser: Optional[Browser] = None
+    context: Optional[BrowserContext] = None
+    page: Optional[Page] = None
+
+    async def start(self) -> Page:
+        if self.page is not None:
+            return self.page  # 이미 시작된 경우 재사용(원치 않으면 제거)
+
+        self.pw = await async_playwright().start()
+        try:
+            browser_type = getattr(self.pw, self.browser_name)
+
+            try:
+                self.browser = await browser_type.launch(headless=self.headless)
+            except PWError as e:
+                should_auto = self.auto_install and os.getenv("PW_SKIP_AUTO_INSTALL") != "1"
+                if should_auto and _need_install(e):
+                    # pw 종료 -> 설치 -> pw 재시작
+                    await self.pw.stop()
+                    _install_playwright_browsers(self.browser_name)
+                    self.pw = await async_playwright().start()
+                    browser_type = getattr(self.pw, self.browser_name)
+                    self.browser = await browser_type.launch(headless=self.headless)
+                else:
+                    raise
+
+            self.context = await self.browser.new_context()
+            self.page = await self.context.new_page()
+            self.page.set_default_timeout(self.timeout_ms)
+            return self.page
+
+        except Exception:
+            # start 중간에 터지면 자원 정리
+            await self.close()
+            raise
+
+    async def close(self) -> None:
+        # 역순 정리 (page는 context close 시 같이 정리됨)
+        if self.context is not None:
+            try:
+                await self.context.close()
+            except Exception:
+                pass
+            finally:
+                self.context = None
+                self.page = None
+
+        if self.browser is not None:
+            try:
+                await self.browser.close()
+            except Exception:
+                pass
+            finally:
+                self.browser = None
+
+        if self.pw is not None:
+            try:
+                await self.pw.stop()
+            except Exception:
+                pass
+            finally:
+                self.pw = None

scraper2/adapters/out/sinks/memory/__init__.py

@@ -0,0 +1,15 @@
+# scraper2/adapters/out/sinks/memory/__init__.py
+from .c101_memory_sink import MemoryC101Sink
+from .c103_memory_sink import MemoryC103Sink
+from .c104_memory_sink import MemoryC104Sink
+from .c106_memory_sink import MemoryC106Sink
+from .c108_memory_sink import MemoryC108Sink
+
+__all__ = [
+    "MemoryC101Sink",
+    "MemoryC103Sink",
+    "MemoryC104Sink",
+    "MemoryC106Sink",
+    "MemoryC108Sink",
+]
+

scraper2/adapters/out/sinks/memory/c101_memory_sink.py

@@ -0,0 +1,20 @@
+#scraper2/adapters/out/sinks/memory/c101_memory_sink.py
+from __future__ import annotations
+
+from typing import Iterable
+
+from contracts.nfs.c101 import C101DTO
+from scraper2.adapters.out.sinks.memory.store import InMemoryStore
+from scraper2.app.ports.sinks.c101_sink_port import C101SinkPort
+
+_ENDPOINT = "c101"
+
+class MemoryC101Sink(C101SinkPort):
+    def __init__(self, store: InMemoryStore[C101DTO]):
+        self._store = store
+
+    async def write(self, dto: C101DTO) -> None:
+        await self._store.put(_ENDPOINT, dto.코드, dto)
+
+    async def write_many(self, dtos: Iterable[C101DTO]) -> None:
+        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))

scraper2/adapters/out/sinks/memory/c103_memory_sink.py

@@ -0,0 +1,20 @@
+# scraper2/adapters/out/sinks/memory/c103_memory_sink.py
+from __future__ import annotations
+
+from typing import Iterable
+
+from contracts.nfs.c103 import C103DTO
+from scraper2.adapters.out.sinks.memory.store import InMemoryStore
+from scraper2.app.ports.sinks.c103_sink_port import C103SinkPort
+
+_ENDPOINT = "c103"
+
+class MemoryC103Sink(C103SinkPort):
+    def __init__(self, store: InMemoryStore[C103DTO]):
+        self._store = store
+
+    async def write(self, dto: C103DTO) -> None:
+        await self._store.put(_ENDPOINT, dto.코드, dto)
+
+    async def write_many(self, dtos: Iterable[C103DTO]) -> None:
+        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))

scraper2/adapters/out/sinks/memory/c104_memory_sink.py

@@ -0,0 +1,20 @@
+# scraper2/adapters/out/sinks/memory/c104_memory_sink.py
+from __future__ import annotations
+
+from typing import Iterable
+
+from contracts.nfs.c104 import C104DTO
+from scraper2.adapters.out.sinks.memory.store import InMemoryStore
+from scraper2.app.ports.sinks.c104_sink_port import C104SinkPort
+
+_ENDPOINT = "c104"
+
+class MemoryC104Sink(C104SinkPort):
+    def __init__(self, store: InMemoryStore[C104DTO]):
+        self._store = store
+
+    async def write(self, dto: C104DTO) -> None:
+        await self._store.put(_ENDPOINT, dto.코드, dto)
+
+    async def write_many(self, dtos: Iterable[C104DTO]) -> None:
+        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))

scraper2/adapters/out/sinks/memory/c106_memory_sink.py

@@ -0,0 +1,20 @@
+#scraper2/adapters/out/sinks/memory/c106_memory_sink.py
+from __future__ import annotations
+
+from typing import Iterable
+
+from contracts.nfs.c106 import C106DTO
+from scraper2.adapters.out.sinks.memory.store import InMemoryStore
+from scraper2.app.ports.sinks.c106_sink_port import C106SinkPort
+
+_ENDPOINT = "c106"
+
+class MemoryC106Sink(C106SinkPort):
+    def __init__(self, store: InMemoryStore[C106DTO]):
+        self._store = store
+
+    async def write(self, dto: C106DTO) -> None:
+        await self._store.put(_ENDPOINT, dto.코드, dto)
+
+    async def write_many(self, dtos: Iterable[C106DTO]) -> None:
+        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))

scraper2/adapters/out/sinks/memory/c108_memory_sink.py

@@ -0,0 +1,20 @@
+#scraper2/adapters/out/sinks/memory/c108_memory_sink.py
+from __future__ import annotations
+
+from typing import Iterable
+
+from contracts.nfs.c108 import C108DTO
+from scraper2.adapters.out.sinks.memory.store import InMemoryStore
+from scraper2.app.ports.sinks.c108_sink_port import C108SinkPort
+
+_ENDPOINT = "c108"
+
+class MemoryC108Sink(C108SinkPort):
+    def __init__(self, store: InMemoryStore[C108DTO]):
+        self._store = store
+
+    async def write(self, dto: C108DTO) -> None:
+        await self._store.put(_ENDPOINT, dto.코드, dto)
+
+    async def write_many(self, dtos: Iterable[C108DTO]) -> None:
+        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))

scraper2/adapters/out/sinks/memory/store.py

@@ -0,0 +1,74 @@
+# scraper2/adapters/out/sinks/memory/store.py
+from __future__ import annotations
+
+import asyncio
+from collections import defaultdict, deque
+from dataclasses import dataclass
+from typing import Any, Deque, Dict, Generic, Iterable, List, Optional, Tuple, TypeVar
+
+T = TypeVar("T")  # DTO 타입
+
+@dataclass(frozen=True)
+class StoreStats:
+    endpoint: str
+    latest_count: int
+    history_count: int
+
+class InMemoryStore(Generic[T]):
+    """
+    endpoint별로 DTO를 저장한다.
+    - latest: endpoint -> key(보통 코드) -> dto
+    - history: endpoint -> deque[dto] (최근 max_history개)
+    """
+    def __init__(self, *, max_history: int = 2000):
+        self._lock = asyncio.Lock()
+        self._max_history = max_history
+        self._history: Dict[str, Deque[T]] = defaultdict(lambda: deque(maxlen=max_history))
+        self._latest: Dict[str, Dict[str, T]] = defaultdict(dict)
+
+    # ---------- write ----------
+
+    async def put(self, endpoint: str, key: str, dto: T) -> None:
+        async with self._lock:
+            self._history[endpoint].append(dto)
+            self._latest[endpoint][key] = dto
+
+    async def put_many(self, endpoint: str, items: Iterable[Tuple[str, T]]) -> None:
+        async with self._lock:
+            for key, dto in items:
+                self._history[endpoint].append(dto)
+                self._latest[endpoint][key] = dto
+
+    # ---------- read ----------
+
+    async def get(self, endpoint: str, key: str) -> Optional[T]:
+        async with self._lock:
+            return self._latest.get(endpoint, {}).get(key)
+
+    async def all_latest(self, endpoint: str) -> Dict[str, T]:
+        async with self._lock:
+            return dict(self._latest.get(endpoint, {}))
+
+    async def list_keys(self, endpoint: str) -> List[str]:
+        async with self._lock:
+            # 정렬은 취향인데, CLI 출력은 정렬이 편해서 기본 정렬
+            return sorted(self._latest.get(endpoint, {}).keys())
+
+    async def history(self, endpoint: str) -> List[T]:
+        async with self._lock:
+            return list(self._history.get(endpoint, []))
+
+    async def stats(self, endpoint: str) -> StoreStats:
+        async with self._lock:
+            latest_count = len(self._latest.get(endpoint, {}))
+            history_count = len(self._history.get(endpoint, []))
+            return StoreStats(endpoint=endpoint, latest_count=latest_count, history_count=history_count)
+
+    async def clear(self, endpoint: str | None = None) -> None:
+        async with self._lock:
+            if endpoint is None:
+                self._history.clear()
+                self._latest.clear()
+            else:
+                self._history.pop(endpoint, None)
+                self._latest.pop(endpoint, None)

scraper2/adapters/out/sinks/mongo/__init__.py

@@ -0,0 +1,14 @@
+# scraper2/adapters/out/sinks/mongo/__init__.py
+from .c101_mongo_sink import MongoC101Sink
+from .c103_mongo_sink import MongoC103Sink
+from .c104_mongo_sink import MongoC104Sink
+from .c106_mongo_sink import MongoC106Sink
+from .c108_mongo_sink import MongoC108Sink
+
+__all__ = [
+    "MongoC101Sink",
+    "MongoC103Sink",
+    "MongoC104Sink",
+    "MongoC106Sink",
+    "MongoC108Sink",
+]

scraper2/adapters/out/sinks/mongo/c101_mongo_sink.py

@@ -0,0 +1,43 @@
+# scraper2/adapters/out/sinks/mongo/c101_mongo_sink.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional
+
+from pymongo.asynchronous.database import AsyncDatabase
+
+from contracts.nfs.c101 import C101DTO
+from scraper2.app.ports.sinks.c101_sink_port import C101SinkPort
+
+from db2.nfs import (
+    upsert_latest,
+    upsert_latest_many,
+    insert_snapshot,
+    insert_snapshots_many,
+)
+
+_ENDPOINT = "c101"
+
+
+class MongoC101Sink(C101SinkPort):
+    def __init__(self, db: AsyncDatabase):
+        self._db = db
+
+    async def write(self, dto: C101DTO, *, asof: Optional[datetime] = None) -> None:
+        # 최신 상태는 upsert
+        await upsert_latest(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+        # 히스토리는 insert
+        await insert_snapshot(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+
+    async def write_many(
+        self,
+        dtos: Iterable[C101DTO],
+        *,
+        asof: Optional[datetime] = None,
+    ) -> None:
+        dtos_list = list(dtos)
+        if not dtos_list:
+            return
+
+        await upsert_latest_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)
+        await insert_snapshots_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)

scraper2/adapters/out/sinks/mongo/c103_mongo_sink.py

@@ -0,0 +1,41 @@
+# scraper2/adapters/out/sinks/mongo/c103_mongo_sink.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional
+
+from pymongo.asynchronous.database import AsyncDatabase
+
+from contracts.nfs.c103 import C103DTO
+from scraper2.app.ports.sinks.c103_sink_port import C103SinkPort
+
+from db2.nfs import (
+    upsert_latest,
+    upsert_latest_many,
+    insert_snapshot,
+    insert_snapshots_many,
+)
+
+_ENDPOINT = "c103"
+
+
+class MongoC103Sink(C103SinkPort):
+    def __init__(self, db: AsyncDatabase):
+        self._db = db
+
+    async def write(self, dto: C103DTO, *, asof: Optional[datetime] = None) -> None:
+        await upsert_latest(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+        await insert_snapshot(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+
+    async def write_many(
+        self,
+        dtos: Iterable[C103DTO],
+        *,
+        asof: Optional[datetime] = None,
+    ) -> None:
+        dtos_list = list(dtos)
+        if not dtos_list:
+            return
+
+        await upsert_latest_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)
+        await insert_snapshots_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)

scraper2/adapters/out/sinks/mongo/c104_mongo_sink.py

@@ -0,0 +1,41 @@
+# scraper2/adapters/out/sinks/mongo/c104_mongo_sink.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional
+
+from pymongo.asynchronous.database import AsyncDatabase
+
+from contracts.nfs.c104 import C104DTO
+from scraper2.app.ports.sinks.c104_sink_port import C104SinkPort
+
+from db2.nfs import (
+    upsert_latest,
+    upsert_latest_many,
+    insert_snapshot,
+    insert_snapshots_many,
+)
+
+_ENDPOINT = "c104"
+
+
+class MongoC104Sink(C104SinkPort):
+    def __init__(self, db: AsyncDatabase):
+        self._db = db
+
+    async def write(self, dto: C104DTO, *, asof: Optional[datetime] = None) -> None:
+        await upsert_latest(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+        await insert_snapshot(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+
+    async def write_many(
+        self,
+        dtos: Iterable[C104DTO],
+        *,
+        asof: Optional[datetime] = None,
+    ) -> None:
+        dtos_list = list(dtos)
+        if not dtos_list:
+            return
+
+        await upsert_latest_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)
+        await insert_snapshots_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)