scraper2-hj3415 2.4.1__py3-none-any.whl → 2.7.0__py3-none-any.whl

scraper2_hj3415/app/parsing/c101_parser.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from typing import Any
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from logging_hj3415 import logger
+
+from .c101.sise import parse_c101_sise_table
+from .c101.earning_surprise import parse_c101_earnings_surprise_table
+from .c101.fundamentals import parse_c101_fundamentals_table
+from .c101.major_shareholders import parse_c101_major_shareholders
+from .c101.company_overview import parse_c101_company_overview
+from .c101.summary_cmp import parse_c101_summary_cmp_table
+from .c101.yearly_consensus import parse_c101_yearly_consensus_table
+
+async def parse_c101_to_dict(browser: BrowserPort) -> dict[str, list[dict[str, Any]]]:
+    parsed_summary_cmp = await parse_c101_summary_cmp_table(browser)
+    logger.debug(f"parsed_summary_cmp data: {parsed_summary_cmp}")
+
+    parsed_sise = await parse_c101_sise_table(browser)
+    logger.debug(f"parsed_sise data: {parsed_sise}")
+
+    parsed_company_overview = await parse_c101_company_overview(browser)
+    logger.debug(f"parsed_company_overview data: {parsed_company_overview}")
+
+    parsed_major_shareholders = await parse_c101_major_shareholders(browser)
+    logger.debug(f"parsed_major_shareholders data: {parsed_major_shareholders}")
+
+    parsed_fundamentals = await parse_c101_fundamentals_table(browser)
+    logger.debug(f"parsed_fundamentals data: {parsed_fundamentals}")
+
+    parsed_earnings_surprise = await parse_c101_earnings_surprise_table(browser)
+    logger.debug(f"parsed_earnings_surprise data: {parsed_earnings_surprise}")
+
+    parsed_yearly_consensus = await parse_c101_yearly_consensus_table(browser)
+    logger.debug(f"parsed_yearly_consensus data: {parsed_yearly_consensus}")
+
+    return {
+        "요약": parsed_summary_cmp,
+        "시세": parsed_sise,
+        "주주현황": parsed_major_shareholders,
+        "기업개요": parsed_company_overview,
+        "펀더멘털": parsed_fundamentals,
+        "어닝서프라이즈": parsed_earnings_surprise,
+        "연간컨센서스": parsed_yearly_consensus,
+    }

scraper2_hj3415/app/parsing/c103_parser.py

@@ -0,0 +1,22 @@
+# scraper2_hj3415/app/parsing/c103_parser.py
+from __future__ import annotations
+from typing import Any
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.parsing._tables.html_table import (
+    try_html_table_to_df,
+    df_to_c1034_metric_list,
+)
+
+TABLE_XPATH = "xpath=//div[@id='wrapper']//div//table"
+TABLE_INDEX = 2
+
+
+async def parse_c103_current_table(browser: BrowserPort) -> list[dict[str, Any]]:
+    """
+    ✅ 현재 화면 상태(탭/연간/분기/검색 결과)가 이미 준비되었다는 전제.
+    이 상태에서 TABLE_INDEX 테이블만 읽어서 rows로 변환한다.
+    """
+    html = await browser.outer_html_nth(TABLE_XPATH, TABLE_INDEX)
+    df = try_html_table_to_df(html)
+    return df_to_c1034_metric_list(df)

scraper2_hj3415/app/parsing/c104_parser.py

@@ -0,0 +1,26 @@
+# scraper2_hj3415/app/parsing/c104_parser.py
+from __future__ import annotations
+
+from typing import Any
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.parsing._tables.html_table import (
+    try_html_table_to_df,
+    df_to_c1034_metric_list,
+)
+
+TABLE_XPATH = 'xpath=//table[@class="gHead01 all-width data-list"]'
+
+
+async def parse_c104_current_table(
+    browser: BrowserPort,
+    *,
+    table_index: int,
+) -> list[dict[str, Any]]:
+    """
+    ✅ 현재 화면 상태(탭/연간/분기/검색 결과)가 이미 준비되었다는 전제.
+    이 상태에서 지정된 table_index 테이블만 읽어서 rows로 변환한다.
+    """
+    html = await browser.outer_html_nth(TABLE_XPATH, table_index)
+    df = try_html_table_to_df(html)
+    return df_to_c1034_metric_list(df)

scraper2_hj3415/app/parsing/c106_parser.py

@@ -0,0 +1,137 @@
+# scraper2_hj3415/app/parsing/c106_parser.py
+from __future__ import annotations
+
+from io import StringIO
+import re
+import numpy as np
+import pandas as pd
+from typing import Any
+
+from common_hj3415.utils import clean_text
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.parsing._normalize.label import (
+    normalize_metric_label,
+    sanitize_label,
+)
+from logging_hj3415 import logger
+
+_CODE_RE = re.compile(r"\b\d{6}\b")
+
+
+async def parse_c106_header_codes(browser: BrowserPort) -> list[str]:
+    """
+    현재 페이지에서 '기업간비교자료' 헤더(회사명들)에서 종목코드(6자리)만 추출한다.
+    (goto/sleep 없음)
+    """
+    selector = (
+        'xpath=//caption[contains(normalize-space(.), "기업간비교자료")]'
+        "/following-sibling::thead//th[not(@colspan)]"
+    )
+    await browser.wait_attached(selector)
+    th_texts = await browser.all_texts(selector)
+
+    codes: list[str] = []
+    for i, t in enumerate(th_texts):
+        text = (t or "").strip()
+        if not text:
+            continue
+        m = _CODE_RE.search(text)
+        if not m:
+            continue
+        codes.append(m.group(0))
+
+    # 중복 제거(순서 유지)
+    seen: set[str] = set()
+    uniq: list[str] = []
+    for c in codes:
+        if c not in seen:
+            seen.add(c)
+            uniq.append(c)
+    logger.debug(f"c106 header codes: {uniq}")
+    return uniq
+
+
+def html_table_to_df(html: str, codes: list[str]) -> pd.DataFrame:
+    df = pd.read_html(StringIO(html), header=None)[0]
+    if df is None or df.empty:
+        return pd.DataFrame()
+
+    df.columns = ["항목_group", "항목"] + codes
+    df["항목_group"] = df["항목_group"].ffill()
+
+    # 첫 두 줄 주가데이터 주입(기존 로직 유지)
+    for i in range(min(2, len(df))):
+        row = df.loc[i].tolist()
+        new_row = ["주가데이터"] + row
+        df.loc[i] = new_row[: len(df.columns)]
+
+    df = df[df["항목"].notna()].reset_index(drop=True)
+    df.loc[df["항목"].isin(["투자의견", "목표주가(원)"]), "항목_group"] = "기타지표"
+    df = df[df["항목"] != "재무연월"].reset_index(drop=True)
+
+    for col in df.columns[2:]:
+        df[col] = df[col].replace("-", "0")
+        df[col] = pd.to_numeric(df[col], errors="coerce")
+
+    df["항목_group"] = df["항목_group"].astype("string").map(clean_text)
+    df["항목"] = df["항목"].astype("string").map(clean_text)
+
+    return df.replace({np.nan: None})
+
+
+def df_to_c106_metric_list(df: pd.DataFrame) -> list[dict[str, Any]]:
+    """
+    C106 DataFrame -> records(list[dict])
+
+    A안 적용:
+    - 항목(key)은 normalize_c1034_item으로 강하게 정규화(괄호/별표 등 제거)
+    - 항목_raw는 정규화 전(단 UI 노이즈만 제거된) 원라벨을 저장
+    - 항목_group은 그대로 두되, 필요 없으면 caller에서 삭제하면 됨
+    """
+    if df is None or df.empty:
+        return []
+
+    df = df.copy()
+
+    # raw 보존(정규화 전, UI 노이즈만 제거)
+    raw = df["항목"].where(df["항목"].notna(), None)
+    df["항목_raw"] = raw.map(
+        lambda x: sanitize_label(str(x)) if x is not None else None
+    )
+
+    # 항목_group 컬럼들은 제거(있을 때만)
+    drop_cols = [c for c in ("항목_group", "항목_group_raw") if c in df.columns]
+    if drop_cols:
+        df = df.drop(columns=drop_cols)
+
+    # key 정규화(A안)
+    df["항목"] = df["항목"].map(
+        lambda x: normalize_metric_label(str(x)) if x is not None else ""
+    )
+
+    # 유효 행만
+    df = df[df["항목"].astype(str).str.strip() != ""].reset_index(drop=True)
+
+    # NaN -> None
+    df = df.where(pd.notnull(df), None)
+
+    return df.to_dict(orient="records")
+
+
+async def parse_c106_current_table(
+    browser: BrowserPort,
+    *,
+    columns: list[str],
+    table_selector: str = "#cTB611",
+    table_index: int = 0,
+    timeout_ms: int = 10_000,
+) -> list[dict[str, Any]]:
+    """
+    ✅ 현재 화면(이미 goto/대기 완료된 상태)에서 비교테이블만 파싱한다.
+    """
+    await browser.wait_table_nth_ready(
+        table_selector, index=table_index, min_rows=3, timeout_ms=timeout_ms
+    )
+    html = await browser.outer_html_nth(table_selector, table_index)
+    df = html_table_to_df(html, columns)
+    return df_to_c106_metric_list(df)

scraper2_hj3415/app/parsing/c108_parser.py

@@ -0,0 +1,254 @@
+# scraper2_hj3415/app/parsing/c108_parser.py
+from __future__ import annotations
+
+import re
+from html import unescape
+from typing import Any
+from common_hj3415.utils import clean_text
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+
+_TAGS = re.compile(r"<[^>]+>")
+_WS = re.compile(r"\s+")
+
+_TD_ID_RE = re.compile(r"^td(\d+)$")   # td0, td1, ...
+_C_ID_RE = re.compile(r"^c(\d+)$")     # c0, c1, ...
+
+
+def _clean_text(x: Any) -> str:
+    """
+    경계/로깅/파싱 단계에서 Any를 안전하게 사람이 읽을 문자열로 만든다.
+    - Any → str
+    - html entity unescape
+    - 이후 normalize_text 적용
+    """
+    if x is None:
+        return ""
+    s = unescape(str(x))   # ❗ x or "" 대신 None만 처리 (falsy 보존)
+    return clean_text(s)
+
+
+def _clean_html_to_text(html: str) -> str:
+    s = unescape(html or "")
+    s = s.replace("<br/>", "\n").replace("<br>", "\n").replace("<br />", "\n")
+    s = _TAGS.sub("", s)
+    s = s.replace("\r", "")
+    lines = [ln.strip() for ln in s.split("\n")]
+    lines = [ln for ln in lines if ln]
+    return "\n".join(lines).strip()
+
+
+_UI_LINES = {"요약정보닫기"}
+_UI_PREFIXES = ("요약정보 :", "요약정보:")
+_BULLET_RE = re.compile(r"^\s*▶\s*")
+_MULTI_NL = re.compile(r"\n{3,}")
+
+
+def _prettify_report_text(
+    text: str,
+    *,
+    bullet: str = "- ",
+) -> str:
+    if not text:
+        return ""
+
+    lines = [ln.strip() for ln in text.split("\n")]
+    out: list[str] = []
+
+    for ln in lines:
+        if not ln:
+            continue
+
+        # UI 잔재 제거 (prefix)
+        for p in _UI_PREFIXES:
+            if ln.startswith(p):
+                ln = ln[len(p) :].strip()
+                break
+        if not ln:
+            continue
+
+        if ln in _UI_LINES:
+            continue
+
+        # 불릿 정리
+        if _BULLET_RE.match(ln):
+            ln = _BULLET_RE.sub(bullet, ln)
+
+        out.append(ln)
+
+    s = "\n".join(out)
+    s = _MULTI_NL.sub("\n\n", s).strip()
+    return s
+
+
+def _parse_target_price(x: Any) -> int | None:
+    s = _clean_text(x)
+    if not s:
+        return None
+    s2 = re.sub(r"[^0-9]", "", s)
+    if not s2:
+        return None
+    try:
+        return int(s2)
+    except Exception:
+        return None
+
+
+def _parse_pages(x: Any) -> int | None:
+    s = _clean_text(x)
+    m = re.search(r"(\d+)", s)
+    return int(m.group(1)) if m else None
+
+
+async def parse_c108_recent_reports_dom(
+    browser: BrowserPort,
+    *,
+    table_selector: str = "#tableCmpDetail",
+) -> list[dict[str, Any]]:
+    """
+    pandas(read_html) 없이 DOM 기반으로 안정적으로 추출.
+
+    전제:
+    - "정상 행"에는 td[id^='td'] 가 있고, 그 id가 tdN 형태다.
+    - "상세 요약(숨김)"은 td[id='cN'] data-content로 붙어있다.
+    - summary는 td[id='tdN'] data-content에, comment는 td[id='cN'] data-content에 들어있다.
+
+    BrowserPort 요구 기능:
+    - wait_attached(selector)
+    - count_in_nth(scope_selector, scope_index, inner_selector) -> int
+    - eval_in_nth_first(scope_selector, scope_index, inner_selector, expression) -> Any
+      (이미 네가 추가해둔 형태 그대로 사용)
+    """
+
+    await browser.wait_attached(table_selector)
+
+    # tbody tr 개수
+    tr_count = await browser.count_in_nth(
+        table_selector, scope_index=0, inner_selector="tbody tr"
+    )
+    if tr_count <= 0:
+        return []
+
+    out: list[dict[str, Any]] = []
+
+    for tr_idx in range(tr_count):
+        # row scope: table_selector >> tbody tr (nth=tr_idx)
+        row_scope = f"{table_selector} >> tbody tr >> nth={tr_idx}"
+
+        # 1) 이 행이 "정상 행"인지 판정: td[id^=td]가 있어야 함
+        td_id = await browser.eval_in_nth_first(
+            row_scope,
+            scope_index=0,
+            inner_selector="td[id^='td']",
+            expression="el => el.id",
+        )
+        td_id = _clean_text(td_id)
+        m = _TD_ID_RE.match(td_id)
+        if not m:
+            # 숨김 상세행(cN) 같은 건 스킵
+            continue
+
+        n = m.group(1)  # row_id
+        # 2) 컬럼 텍스트 추출 (C108 테이블 구조에 맞게 td 순서 기준)
+        #    보통: 1=일자, 2=제목, 3=작성자, 4=제공처, 5=투자의견, 6=목표가, 7=분량 ...
+        date = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(1)",
+                expression="el => el.innerText",
+            )
+        )
+        title = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(2)",
+                expression="el => el.innerText",
+            )
+        )
+
+        # 최소 필터
+        if not date or not title:
+            continue
+
+        authors = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(3)",
+                expression="el => el.innerText",
+            )
+        ) or None
+
+        provider = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(4)",
+                expression="el => el.innerText",
+            )
+        ) or None
+
+        rating = _clean_text(
+            await browser.eval_in_nth_first(
+                row_scope,
+                scope_index=0,
+                inner_selector="td:nth-child(5)",
+                expression="el => el.innerText",
+            )
+        ) or None
+
+        target_price_raw = await browser.eval_in_nth_first(
+            row_scope,
+            scope_index=0,
+            inner_selector="td:nth-child(6)",
+            expression="el => el.innerText",
+        )
+        target_price = _parse_target_price(target_price_raw)
+
+        pages_raw = await browser.eval_in_nth_first(
+            row_scope,
+            scope_index=0,
+            inner_selector="td:nth-child(7)",
+            expression="el => el.innerText",
+        )
+        pages = _parse_pages(pages_raw)
+
+        # 3) summary/comment: N으로 tdN / cN의 data-content를 직접 읽기
+        #    (DOM에 존재하지만 display:none인 경우도 data-content는 읽을 수 있음)
+        summary_html = await browser.eval_in_nth_first(
+            table_selector,
+            scope_index=0,
+            inner_selector=f"td#td{n}",
+            expression="el => el.getAttribute('data-content') || ''",
+        )
+        comment_html = await browser.eval_in_nth_first(
+            table_selector,
+            scope_index=0,
+            inner_selector=f"td#c{n}",
+            expression="el => el.getAttribute('data-content') || ''",
+        )
+
+        summary = _prettify_report_text(_clean_html_to_text(_clean_text(summary_html)))
+        comment = _prettify_report_text(_clean_html_to_text(_clean_text(comment_html)))
+
+        out.append(
+            {
+                "row_id": n,
+                "date": date,
+                "title": title,
+                "authors": authors,
+                "provider": provider,
+                "rating": rating,
+                "target_price": target_price,
+                "pages": pages,
+                "summary": summary or None,
+                "comment": comment or None,
+            }
+        )
+
+    return out
+
+
+async def parse_c108_to_dict(browser: BrowserPort) -> dict[str, list[dict[str, Any]]]:
+    return {"리포트": await parse_c108_recent_reports_dom(browser)}

scraper2_hj3415/app/ports/browser/browser_factory_port.py

@@ -0,0 +1,9 @@
+# scraper2_hj3415/app/ports/browser/browser_factory_port.py
+from __future__ import annotations
+from typing import Protocol, AsyncContextManager
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+
+class BrowserFactoryPort(Protocol):
+    def lease(self) -> AsyncContextManager[BrowserPort]: ...
+    async def aclose(self) -> None: ...

scraper2_hj3415/app/ports/browser/browser_port.py

@@ -0,0 +1,32 @@
+# scraper2_hj3415/app/ports/browser/browser_port.py
+from __future__ import annotations
+
+from typing import Protocol
+
+from .capabilities import (
+    BrowserInteractionPort,
+    BrowserNavigationPort,
+    BrowserScopePort,
+    BrowserTablePort,
+    BrowserTextPort,
+    BrowserWaitPort,
+)
+
+
+class BrowserPort(
+    BrowserNavigationPort,
+    BrowserWaitPort,
+    BrowserInteractionPort,
+    BrowserTextPort,
+    BrowserScopePort,
+    BrowserTablePort,
+    Protocol,
+):
+    """
+    프로젝트에서 사용하는 최종 BrowserPort.
+
+    - 내부는 capability 단위로 분리되어 있으며,
+      필요하면 파서/유스케이스가 BrowserPort 대신
+      특정 capability 포트만 의존하도록 바꿀 수 있다.
+    """
+    ...

scraper2_hj3415/app/ports/browser/capabilities/__init__.py

@@ -0,0 +1,15 @@
+from .navigation import BrowserNavigationPort
+from .wait import BrowserWaitPort
+from .interaction import BrowserInteractionPort
+from .text import BrowserTextPort
+from .scope import BrowserScopePort
+from .table import BrowserTablePort
+
+__all__ = [
+    "BrowserNavigationPort",
+    "BrowserWaitPort",
+    "BrowserInteractionPort",
+    "BrowserTextPort",
+    "BrowserScopePort",
+    "BrowserTablePort",
+]

scraper2_hj3415/app/ports/browser/capabilities/interaction.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class BrowserInteractionPort(Protocol):
+    """클릭/스크롤 등 상호작용"""
+
+    async def click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 4_000,
+        force: bool = False,
+    ) -> None: ...
+
+    async def try_click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 1_500,
+        force: bool = False,
+    ) -> bool: ...
+
+    async def scroll_into_view(self, selector: str, *, index: int = 0) -> None: ...

scraper2_hj3415/app/ports/browser/capabilities/navigation.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class BrowserNavigationPort(Protocol):
+    """페이지 이동/기본 정보"""
+
+    async def title(self) -> str: ...
+    async def current_url(self) -> str: ...
+
+    async def goto_and_wait_for_stable(
+        self,
+        url: str,
+        timeout_ms: int = 10_000,
+    ) -> None: ...
+
+    async def reload(self, *, timeout_ms: int = 10_000) -> None: ...

scraper2_hj3415/app/ports/browser/capabilities/scope.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+
+class BrowserScopePort(Protocol):
+    """scope/nth 컨텍스트 기반 조회 (현재는 기존 API 유지)"""
+
+    async def is_attached(self, selector: str, *, index: int = 0) -> bool: ...
+
+    async def computed_style(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        prop: str,
+    ) -> str: ...
+
+    async def count_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+    ) -> int: ...
+
+    async def eval_in_nth_first(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        expression: str,
+    ) -> Any: ...
+
+    async def inner_text_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        """
+        scope_selector의 nth(scope_index) 요소 안에서
+        inner_selector의 nth(inner_index) 요소의 innerText를 반환.
+        (렌더링 기준 텍스트: 줄바꿈/스타일 영향 반영)
+        """
+        ...
+
+    async def text_content_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        """
+        scope_selector의 nth(scope_index) 요소 안에서
+        inner_selector의 nth(inner_index) 요소의 textContent를 반환.
+        (DOM 기준 텍스트: 숨김 텍스트도 포함될 수 있음)
+        """
+        ...

scraper2_hj3415/app/ports/browser/capabilities/table.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+
+class BrowserTablePort(Protocol):
+    """테이블 파싱/헤더 추출"""
+
+    async def table_records(
+        self,
+        table_selector: str,
+        *,
+        header: int | list[int] | None = 0,
+    ) -> list[dict[str, Any]]: ...
+
+    async def table_header_texts_nth(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+    ) -> list[str]: ...
+
+    async def table_header_periods_mm_nth(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+    ) -> list[str]: ...