scraper2-hj3415 2.4.1__py3-none-any.whl → 2.6.0__py3-none-any.whl

scraper2_hj3415/app/parsing/c108_parser.py

@@ -0,0 +1,254 @@
+# scraper2_hj3415/app/parsing/c108_parser.py
+from __future__ import annotations
+
+import re
+from html import unescape
+from typing import Any
+from common_hj3415.utils import clean_text
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+
+_TAGS = re.compile(r"<[^>]+>")
+_WS = re.compile(r"\s+")
+
+_TD_ID_RE = re.compile(r"^td(\d+)$")   # td0, td1, ...
+_C_ID_RE = re.compile(r"^c(\d+)$")     # c0, c1, ...
+
+
+def _clean_text(x: Any) -> str:
+    """
+    경계/로깅/파싱 단계에서 Any를 안전하게 사람이 읽을 문자열로 만든다.
+    - Any → str
+    - html entity unescape
+    - 이후 normalize_text 적용
+    """
+    if x is None:
+        return ""
+    s = unescape(str(x))   # ❗ x or "" 대신 None만 처리 (falsy 보존)
+    return clean_text(s)
+
+
+def _clean_html_to_text(html: str) -> str:
+    s = unescape(html or "")
+    s = s.replace("<br/>", "\n").replace("<br>", "\n").replace("<br />", "\n")
+    s = _TAGS.sub("", s)
+    s = s.replace("\r", "")
+    lines = [ln.strip() for ln in s.split("\n")]
+    lines = [ln for ln in lines if ln]
+    return "\n".join(lines).strip()
+
+
+_UI_LINES = {"요약정보닫기"}
+_UI_PREFIXES = ("요약정보 :", "요약정보:")
+_BULLET_RE = re.compile(r"^\s*▶\s*")
+_MULTI_NL = re.compile(r"\n{3,}")
+
+
+def _prettify_report_text(
+    text: str,
+    *,
+    bullet: str = "- ",
+) -> str:
+    if not text:
+        return ""
+
+    lines = [ln.strip() for ln in text.split("\n")]
+    out: list[str] = []
+
+    for ln in lines:
+        if not ln:
+            continue
+
+        # UI 잔재 제거 (prefix)
+        for p in _UI_PREFIXES:
+            if ln.startswith(p):
+                ln = ln[len(p) :].strip()
+                break
+        if not ln:
+            continue
+
+        if ln in _UI_LINES:
+            continue
+
+        # 불릿 정리
+        if _BULLET_RE.match(ln):
+            ln = _BULLET_RE.sub(bullet, ln)
+
+        out.append(ln)
+
+    s = "\n".join(out)
+    s = _MULTI_NL.sub("\n\n", s).strip()
+    return s
+
+
+def _parse_target_price(x: Any) -> int | None:
+    s = _clean_text(x)
+    if not s:
+        return None
+    s2 = re.sub(r"[^0-9]", "", s)
+    if not s2:
+        return None
+    try:
+        return int(s2)
+    except Exception:
+        return None
+
+
+def _parse_pages(x: Any) -> int | None:
+    s = _clean_text(x)
+    m = re.search(r"(\d+)", s)
+    return int(m.group(1)) if m else None
+
+
+async def parse_c108_recent_reports_dom(
+    browser: BrowserPort,
+    *,
+    table_selector: str = "#tableCmpDetail",
+) -> list[dict[str, Any]]:
+    """
+    pandas(read_html) 없이 DOM 기반으로 안정적으로 추출.
+
+    전제:
+    - "정상 행"에는 td[id^='td'] 가 있고, 그 id가 tdN 형태다.
+    - "상세 요약(숨김)"은 td[id='cN'] data-content로 붙어있다.
+    - summary는 td[id='tdN'] data-content에, comment는 td[id='cN'] data-content에 들어있다.
+
+    BrowserPort 요구 기능:
+    - wait_attached(selector)
+    - count_in_nth(scope_selector, scope_index, inner_selector) -> int
+    - eval_in_nth_first(scope_selector, scope_index, inner_selector, expression) -> Any
+      (이미 네가 추가해둔 형태 그대로 사용)
+    """
+
+    await browser.wait_attached(table_selector)
+
+    # tbody tr 개수
+    tr_count = await browser.count_in_nth(
+        table_selector, scope_index=0, inner_selector="tbody tr"
+    )
+    if tr_count <= 0:
+        return []
+
+    out: list[dict[str, Any]] = []
+
+    for tr_idx in range(tr_count):
+        # row scope: table_selector >> tbody tr (nth=tr_idx)
+        row_scope = f"{table_selector} >> tbody tr >> nth={tr_idx}"
+
+        # 1) 이 행이 "정상 행"인지 판정: td[id^=td]가 있어야 함
+        td_id = await browser.eval_in_nth_first(
+            row_scope,
+            scope_index=0,
+            inner_selector="td[id^='td']",
+            expression="el => el.id",
+        )
+        td_id = _clean_text(td_id)
+        m = _TD_ID_RE.match(td_id)
+        if not m:
+            # 숨김 상세행(cN) 같은 건 스킵
+            continue
+
+        n = m.group(1)  # row_id
+        # 2) 컬럼 텍스트 추출 (C108 테이블 구조에 맞게 td 순서 기준)
+        #    보통: 1=일자, 2=제목, 3=작성자, 4=제공처, 5=투자의견, 6=목표가, 7=분량 ...
+        date = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(1)",
+                expression="el => el.innerText",
+            )
+        )
+        title = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(2)",
+                expression="el => el.innerText",
+            )
+        )
+
+        # 최소 필터
+        if not date or not title:
+            continue
+
+        authors = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(3)",
+                expression="el => el.innerText",
+            )
+        ) or None
+
+        provider = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(4)",
+                expression="el => el.innerText",
+            )
+        ) or None
+
+        rating = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(5)",
+                expression="el => el.innerText",
+            )
+        ) or None
+
+        target_price_raw = await browser.eval_in_nth_first(
+            row_scope,
+            scope_index=0,
+            inner_selector="td:nth-child(6)",
+            expression="el => el.innerText",
+        )
+        target_price = _parse_target_price(target_price_raw)
+
+        pages_raw = await browser.eval_in_nth_first(
+            row_scope,
+            scope_index=0,
+            inner_selector="td:nth-child(7)",
+            expression="el => el.innerText",
+        )
+        pages = _parse_pages(pages_raw)
+
+        # 3) summary/comment: N으로 tdN / cN의 data-content를 직접 읽기
+        #    (DOM에 존재하지만 display:none인 경우도 data-content는 읽을 수 있음)
+        summary_html = await browser.eval_in_nth_first(
+            table_selector,
+            scope_index=0,
+            inner_selector=f"td#td{n}",
+            expression="el => el.getAttribute('data-content') || ''",
+        )
+        comment_html = await browser.eval_in_nth_first(
+            table_selector,
+            scope_index=0,
+            inner_selector=f"td#c{n}",
+            expression="el => el.getAttribute('data-content') || ''",
+        )
+
+        summary = _prettify_report_text(_clean_html_to_text(_clean_text(summary_html)))
+        comment = _prettify_report_text(_clean_html_to_text(_clean_text(comment_html)))
+
+        out.append(
+            {
+                "row_id": n,
+                "date": date,
+                "title": title,
+                "authors": authors,
+                "provider": provider,
+                "rating": rating,
+                "target_price": target_price,
+                "pages": pages,
+                "summary": summary or None,
+                "comment": comment or None,
+            }
+        )
+
+    return out
+
+
+async def parse_c108_to_dict(browser: BrowserPort) -> dict[str, list[dict[str, Any]]]:
+    return {"리포트": await parse_c108_recent_reports_dom(browser)}

scraper2_hj3415/app/ports/browser/browser_factory_port.py

@@ -0,0 +1,9 @@
+# scraper2_hj3415/app/ports/browser/browser_factory_port.py
+from __future__ import annotations
+from typing import Protocol, AsyncContextManager
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+
+class BrowserFactoryPort(Protocol):
+    def lease(self) -> AsyncContextManager[BrowserPort]: ...
+    async def aclose(self) -> None: ...

scraper2_hj3415/app/ports/browser/browser_port.py

@@ -0,0 +1,115 @@
+# scraper2_hj3415/app/ports/browser/browser_port.py
+from __future__ import annotations
+
+from typing import Protocol, Any
+
+
+class BrowserPort(Protocol):
+    async def wait_table_nth_ready(
+            self,
+            table_selector: str,
+            *,
+            index: int,
+            min_rows: int = 1,
+            timeout_ms: int = 20_000,
+            poll_ms: int = 200,
+    ) -> None: ...
+    async def title(self) -> str: ...
+    async def current_url(self) -> str: ...
+    async def goto_and_wait_for_stable(self, url: str, timeout_ms: int = 10_000) -> None: ...
+    async def reload(self, *, timeout_ms: int = 10_000) -> None: ...
+    async def sleep_ms(self, ms: int) -> None: ...
+    async def wait_attached(
+        self, selector: str, *, timeout_ms: int = 10_000
+    ) -> None: ...
+    async def wait_visible(
+        self, selector: str, *, timeout_ms: int = 10_000
+    ) -> None: ...
+    async def click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 4_000,
+        force: bool = False,
+    ) -> None: ...
+    async def try_click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 1_500,
+        force: bool = False,
+    ) -> bool: ...
+    async def count(self, selector: str) -> int: ...
+    async def scroll_into_view(self, selector: str, *, index: int = 0) -> None: ...
+    async def text_content_first(self, selector: str) -> str: ...
+    async def all_texts(self, selector: str) -> list[str]: ...
+    async def get_text_by_text(self, needle: str) -> str: ...
+    async def inner_text(self, selector: str) -> str: ...
+    async def outer_html_nth(self, selector: str, index: int) -> str: ...
+    async def wait_table_text_changed(
+            self,
+            table_selector: str,
+            *,
+            index: int,
+            prev_text: str | None,
+            min_rows: int = 1,
+            min_lines: int = 50,
+            timeout_sec: float = 12.0,
+            poll_sec: float = 0.2,
+    ) -> str: ...
+    async def is_attached(self, selector: str, *, index: int = 0) -> bool: ...
+    async def computed_style(
+        self, selector: str, *, index: int = 0, prop: str
+    ) -> str: ...
+    async def count_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+    ) -> int: ...
+    async def eval_in_nth_first(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        expression: str,
+    ) -> Any: ...
+    async def inner_text_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        """
+        scope_selector의 nth(scope_index) 요소 안에서
+        inner_selector의 nth(inner_index) 요소의 innerText를 반환.
+        (렌더링 기준 텍스트: 줄바꿈/스타일 영향 반영)
+        """
+        ...
+
+    async def text_content_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        """
+        scope_selector의 nth(scope_index) 요소 안에서
+        inner_selector의 nth(inner_index) 요소의 textContent를 반환.
+        (DOM 기준 텍스트: 숨김 텍스트도 포함될 수 있음)
+        """
+        ...
+
+    async def table_records(
+        self, table_selector: str, *, header: int | list[int] | None = 0
+    ) -> list[dict[str, Any]]: ...

scraper2_hj3415/app/ports/ingest/nfs_ingest_port.py

@@ -0,0 +1,28 @@
+# scraper2_hj3415/app/ports/ingest/nfs_ingest_port.py
+from __future__ import annotations
+from typing import Protocol, Iterable, Optional, TypeVar
+from datetime import datetime
+
+from contracts_hj3415.nfs.nfs_dto import NfsDTO
+
+TDto = TypeVar("TDto", bound=NfsDTO)
+
+
+class NfsIngestPort(Protocol[TDto]):
+    async def execute(
+        self,
+        code: str,
+        *,
+        sleep_sec: float = ...,
+        asof: Optional[datetime] = None,
+    ) -> TDto:
+        ...
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = ...,
+        asof: Optional[datetime] = None,
+    ) -> list[TDto]:
+        ...

scraper2_hj3415/app/ports/sinks/nfs_sink_port.py

@@ -0,0 +1,20 @@
+# scraper2_hj3415/app/ports/sinks/nfs_sink_port.py
+from __future__ import annotations
+
+from typing import Protocol, Iterable, TypeVar
+from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.nfs_dto import NfsDTO
+
+TDto = TypeVar("TDto", bound=NfsDTO)
+
+class NfsSinkPort(Protocol[TDto]):
+    async def write(
+        self, dto: TDto, *, endpoint: Endpoints
+    ) -> None: ...
+
+    async def write_many(
+        self,
+        dtos: Iterable[TDto],
+        *,
+        endpoint: Endpoints
+    ) -> None: ...

scraper2_hj3415/app/ports/site/wisereport_port.py

@@ -0,0 +1,20 @@
+# scraper2_hj3415/app/ports/site/wisereport_port.py
+from __future__ import annotations
+from typing import Protocol
+
+class WiseReportPort(Protocol):
+    async def ensure_yearly_consensus_open_in_table_nth(
+        self,
+        *,
+        table_selector: str,   # 예: TABLE_XPATH ("xpath=//div[@id='wrapper']//div//table")
+        table_index: int,      # 예: TABLE_INDEX (2)
+        after_click_sleep_ms: int = 150,
+        max_rounds: int = 6,
+        wait_timeout_sec: float = 12.0,
+    ) -> bool: ...
+    async def click_steps(
+            self,
+            steps: list[tuple[str, str]],
+            *,
+            jitter_sec: float = 0.6,
+    ) -> None: ...

scraper2_hj3415/app/services/fetch/fetch_c101.py

@@ -0,0 +1,59 @@
+# scraper2_hj3415/app/usecases/fetch/fetch_c101.py
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable
+
+from logging_hj3415 import logger
+from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2_hj3415.app.parsing.c101_parser import parse_c101_to_dict
+
+from scraper2_hj3415.app.services.nfs_doc_builders import build_c101_doc_from_parsed
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import BLOCK_KEYS_BY_ENDPOINT
+
+
+class FetchC101:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> NfsDoc | None:
+        async with self.factory.lease() as browser:
+            url = f"https://navercomp.wisereport.co.kr/v2/company/c1010001.aspx?cmp_cd={code}"
+            await browser.goto_and_wait_for_stable(url, timeout_ms=10_000)
+
+            if sleep_sec > 0:
+                await asyncio.sleep(sleep_sec + random.uniform(0, 1.0))
+
+            parsed = await parse_c101_to_dict(browser)
+
+            logger.debug(f"parsed data: {parsed}")
+            block_keys = BLOCK_KEYS_BY_ENDPOINT[EndpointKind.C101]
+            if not parsed or all(not (parsed.get(str(bk)) or []) for bk in block_keys):
+                logger.warning(
+                    f"c101 fetch: parsed result empty; return None | code={code}"
+                )
+                return None
+
+            doc = build_c101_doc_from_parsed(
+                code=code, parsed=parsed, keep_empty_blocks=True
+            )
+            logger.debug(f"c101 doc: {doc}")
+            return doc
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> NfsDoc | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+    ) -> list[NfsDoc]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
+            return_exceptions=False,
+        )
+        return [r for r in results if r is not None]

scraper2_hj3415/app/services/fetch/fetch_c103.py

@@ -0,0 +1,135 @@
+# scraper2_hj3415/app/usecases/fetch/fetch_c103.py
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable, Any
+
+from logging_hj3415 import logger
+from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2_hj3415.app.ports.site.wisereport_port import WiseReportPort
+
+from scraper2_hj3415.app.adapters.site.wisereport_playwright import WiseReportPlaywright
+from scraper2_hj3415.app.parsing.c103_parser import parse_c103_current_table
+from scraper2_hj3415.app.services.nfs_doc_builders import build_metrics_doc_from_parsed
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.blocks import BLOCK_KEYS_BY_ENDPOINT
+from scraper2_hj3415.app.domain.doc import NfsDoc
+
+BTN_SETS: dict[str, list[tuple[str, str]]] = {
+    "손익계산서y": [
+        ("손익계산서", 'xpath=//*[@id="rpt_tab1"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "재무상태표y": [
+        ("재무상태표", 'xpath=//*[@id="rpt_tab2"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "현금흐름표y": [
+        ("현금흐름표", 'xpath=//*[@id="rpt_tab3"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "손익계산서q": [
+        ("손익계산서", 'xpath=//*[@id="rpt_tab1"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "재무상태표q": [
+        ("재무상태표", 'xpath=//*[@id="rpt_tab2"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "현금흐름표q": [
+        ("현금흐름표", 'xpath=//*[@id="rpt_tab3"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+}
+
+
+class FetchC103:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> NfsDoc | None:
+        async with self.factory.lease() as browser:
+            wr: WiseReportPort = WiseReportPlaywright(browser)
+
+            url = (
+                "https://navercomp.wisereport.co.kr/v2/company/c1030001.aspx"
+                f"?cn=&cmp_cd={code}"
+            )
+            await browser.goto_and_wait_for_stable(url, timeout_ms=10_000)
+
+            if sleep_sec > 0:
+                await asyncio.sleep(sleep_sec + random.uniform(0, 1.0))
+
+            parsed: dict[str, list[dict[str, Any]]] = {}
+            prev_text: str | None = None
+
+            # 최초 기준 텍스트 확보(없어도 동작하게)
+            prev_text = await browser.wait_table_text_changed(
+                "xpath=//div[@id='wrapper']//div//table",
+                index=2,
+                prev_text=None,
+                min_rows=5,
+                min_lines=50,
+                timeout_sec=10.0,
+            )
+
+            for key, steps in BTN_SETS.items():
+                # ✅ 상태 전환 (행동)
+                await wr.click_steps(steps, jitter_sec=0.6)  # 포트/어댑터로 이동 권장
+                await wr.ensure_yearly_consensus_open_in_table_nth(
+                    table_selector="xpath=//div[@id='wrapper']//div//table",
+                    table_index=2,
+                )
+
+                # ✅ 데이터 변경 대기 (행동)
+                prev_text = await browser.wait_table_text_changed(
+                    "xpath=//div[@id='wrapper']//div//table",
+                    index=2,
+                    prev_text=prev_text,
+                    min_rows=5,
+                    min_lines=50,
+                    timeout_sec=12.0,
+                )
+
+                # ✅ 파싱은 “현재 화면 테이블”만
+                try:
+                    parsed[key] = await parse_c103_current_table(browser)
+                except Exception:
+                    parsed[key] = []
+
+            block_keys = BLOCK_KEYS_BY_ENDPOINT[EndpointKind.C103]
+            if not parsed or all(not (parsed.get(str(bk)) or []) for bk in block_keys):
+                logger.warning(
+                    f"c103 fetch: parsed result empty; return None | code={code}"
+                )
+                return None
+
+            doc = build_metrics_doc_from_parsed(
+                code=code,
+                endpoint_kind=EndpointKind.C103,
+                parsed=parsed,
+                block_keys=block_keys,
+                item_key="항목",
+                raw_label_key="항목_raw",
+                keep_empty_blocks=True,
+            )
+            return doc
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> NfsDoc | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(
+        self, codes: Iterable[str], *, sleep_sec: float = 2.0
+    ) -> list[NfsDoc]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes)
+        )
+        return [r for r in results if r is not None]