scraper2-hj3415 2.4.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

scraper2_hj3415/app/parsing/c101/major_shareholders.py

@@ -0,0 +1,57 @@
+# scraper2_hj3415/app/parsing/c101/major_shareholders.py
+from __future__ import annotations
+
+from typing import Any
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.parsing._normalize.text import normalize_text
+from scraper2_hj3415.app.parsing._normalize.label import normalize_key_label
+from scraper2_hj3415.app.parsing._normalize.values import to_int, to_float
+
+def _pick_value_by_norm_key(row: dict[str, Any], candidates: list[str]) -> Any:
+    # row의 키들을 정규화 맵으로 만든 뒤 후보를 정규화해서 조회
+    norm_map: dict[str, str] = {
+        normalize_key_label(k): k for k in row.keys()
+    }
+    for cand in candidates:
+        rk = norm_map.get(normalize_key_label(cand))
+        if rk is None:
+            continue
+        v = row.get(rk)
+        # "키만 있고 값이 비어있는" 경우 다음 후보 탐색
+        if v is None:
+            continue
+        if isinstance(v, str) and not v.strip():
+            continue
+        return v
+    return None
+
+
+async def parse_c101_major_shareholders(browser: BrowserPort) -> list[dict[str, Any]]:
+    table_sel = "#cTB13"
+    await browser.wait_attached(table_sel)
+
+    records = await browser.table_records(table_sel, header=0)
+
+    if not records:
+        return []
+
+    out: list[dict[str, Any]] = []
+    for r in records:
+        name = normalize_text(_pick_value_by_norm_key(r, ["주요주주", "주요주주명"]))
+        if not name:
+            continue
+
+        shares_raw = _pick_value_by_norm_key(
+            r, ["보유주식수(보통)", "보유주식수", "보유주식수(보통주)"]
+        )
+        ratio_raw = _pick_value_by_norm_key(r, ["보유지분(%)", "보유지분", "보유지분%"])
+
+        out.append(
+            {
+                "주요주주": name,
+                "보유주식수": to_int(shares_raw),  # 파싱 실패 시 None 가능
+                "보유지분": to_float(ratio_raw),  # "0.91%"도 처리되게 파서 보장 필요
+            }
+        )
+
+    return out

scraper2_hj3415/app/parsing/c101/sise.py

@@ -0,0 +1,47 @@
+# scraper2_hj3415/app/parsing/c101/sise.py
+from __future__ import annotations
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from common_hj3415.utils import clean_text
+from ._sise_normalizer import normalize_sise_kv_map
+
+_SISE_TABLE = "#cTB11"
+
+async def parse_c101_sise_table(browser: BrowserPort) -> dict[str, str]:
+    """
+    #cTB11 시세정보 테이블을 th(항목명) -> td(값) dict로 추출한다.
+    - 화면에 보이는 텍스트 기준(innerText)
+    """
+    await browser.wait_attached(_SISE_TABLE)
+
+    row_cnt = await browser.count_in_nth(
+        _SISE_TABLE,
+        scope_index=0,
+        inner_selector="tbody tr",
+    )
+
+    out: dict[str, str] = {}
+
+    for i in range(1, row_cnt + 1):  # nth-child는 1-based
+        row_sel = f"tbody tr:nth-child({i})"
+
+        key = await browser.inner_text_in_nth(
+            _SISE_TABLE,
+            scope_index=0,
+            inner_selector=f"{row_sel} th",
+            inner_index=0,
+        )
+        val = await browser.inner_text_in_nth(
+            _SISE_TABLE,
+            scope_index=0,
+            inner_selector=f"{row_sel} td",
+            inner_index=0,
+        )
+
+        k = clean_text(key)
+        v = clean_text(val)
+        if k:
+            out[k] = v
+    raw = out
+    return normalize_sise_kv_map(raw)
+

scraper2_hj3415/app/parsing/c101/summary_cmp.py

@@ -0,0 +1,87 @@
+# scraper2_hj3415/app/parsing/c101/summary_cmp.py
+from __future__ import annotations
+
+from typing import Any
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from common_hj3415.utils import clean_text
+from scraper2_hj3415.app.parsing._normalize.values import to_number
+
+
+async def parse_c101_summary_cmp_table(browser: BrowserPort) -> dict[str, Any]:
+    """
+    <table class="cmp-table"> (회사 요약 테이블)에서 종목 기본 + EPS/BPS/PER... 등을 추출한다.
+
+    반환 예:
+    {
+      "종목명": "삼성전자",
+      "코드": "005930",
+      "영문명": "SamsungElec",
+      "시장": "KOSPI : 코스피 전기·전자",
+      "WICS": "WICS : 반도체와반도체장비",
+      "EPS": 4816,
+      "BPS": 60632,
+      "PER": 31.58,
+      "업종PER": 21.93,
+      "PBR": 2.51,
+      "현금배당수익률": 0.95,
+      "결산": "12월 결산",
+    }
+    """
+    out: dict[str, Any] = {}
+
+    # 테이블 존재 확인
+    await browser.wait_attached("table.cmp-table")
+
+    # --- 1) td0101: 종목명/코드/영문/시장/WICS ---
+    out["종목명"] = clean_text(
+        await browser.text_content_first("table.cmp-table td.td0101 span.name")
+    )
+    out["코드"] = clean_text(
+        await browser.text_content_first("table.cmp-table td.td0101 b.num")
+    )
+
+    # td0101의 dt 텍스트들을 읽어 분류
+    dt0101 = await browser.all_texts("table.cmp-table td.td0101 dl > dt")
+    for t in dt0101[1:] if dt0101 else []:
+        t = clean_text(t)
+        if not t:
+            continue
+        if t.startswith("KOSPI") or t.startswith("KOSDAQ"):
+            out["시장"] = t
+        elif t.startswith("WICS"):
+            out["WICS"] = t
+        else:
+            if "영문명" not in out:
+                out["영문명"] = t
+
+    # --- 2) td0301: EPS/BPS/PER/업종PER/PBR/현금배당수익률/결산 ---
+    base_dl = "table.cmp-table td.td0301 dl"
+    dt_sel = f"{base_dl} > dt"
+
+    dt_texts = await browser.all_texts(dt_sel)  # dt 전체 텍스트(숫자 포함)
+    if not dt_texts:
+        return out
+
+    # dt는 DOM 상에서 1..N 순서
+    for i, raw_dt in enumerate(dt_texts, start=1):
+        dt_text = clean_text(raw_dt)
+        if not dt_text:
+            continue
+
+        num_sel = f"{base_dl} > dt:nth-child({i}) b.num"
+
+        # 숫자 없는 라인: 예) "12월 결산"
+        if not await browser.is_attached(num_sel):
+            if "결산" in dt_text:
+                out["결산"] = dt_text
+            continue
+
+        num_text = clean_text(await browser.text_content_first(num_sel))
+        if not num_text:
+            continue
+
+        label = clean_text(dt_text.replace(num_text, "")).replace(":", "")
+        if label:
+            out[label] = to_number(num_text)
+
+    return out

scraper2_hj3415/app/parsing/c101/yearly_consensus.py

@@ -0,0 +1,197 @@
+# scraper2_hj3415/app/parsing/c101/yearly_consensus.py
+from __future__ import annotations
+
+from io import StringIO
+import re
+from typing import Any
+
+import pandas as pd
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.parsing._normalize.values import to_float
+from scraper2_hj3415.app.parsing._normalize.text import normalize_text
+from common_hj3415.utils import clean_text
+from logging_hj3415 import logger
+
+_YEARLY_CONSENSUS_TABLE = "#cTB25"
+
+
+# -----------------------------
+# column / period normalize
+# -----------------------------
+_COL_UNIT_RE = re.compile(r"\([^)]*\)")  # (억원, %), (원), (배) ... 제거용
+_PERIOD_RE = re.compile(r"^\s*(\d{4})\s*\(?([A-Za-z])?\)?\s*$")  # 2022(A), 2025(E)
+
+
+def _flatten_col(col: Any) -> str:
+    """
+    pd.read_html(header=[0,1])로 생긴 MultiIndex 컬럼을 '매출액_금액' 같은 단일 키로 만든다.
+    - ('매출액(억원, %)', '금액') -> '매출액_금액'
+    - ('매출액(억원, %)', 'YoY') -> '매출액_YoY'
+    - 단위 괄호 제거
+    """
+    if isinstance(col, tuple):
+        parts = [clean_text(str(p)) for p in col if clean_text(str(p))]
+        if len(parts) == 2 and parts[0] == parts[1]:
+            s = parts[0]
+        else:
+            s = "_".join(parts) if parts else ""
+    else:
+        s = clean_text(str(col))
+
+    # 단위 괄호 제거
+    s = _COL_UNIT_RE.sub("", s)
+    s = clean_text(s)
+
+    # 컬럼 표기 깨짐 보정
+    s = s.replace("주재 무제표", "주재무제표")
+
+    # 공백 제거(키 안정화)
+    s = s.replace(" ", "")
+    return s
+
+
+def _normalize_period(
+    s: Any,
+    *,
+    keep_suffix: bool = False,
+) -> str | None:
+    """
+    기간 문자열을 표준 period key로 정규화한다.
+
+    - "2022(A)", "2026(E)", "2022" 등을 처리
+    - 기본 정책: 연간 = YYYY/12
+    """
+    t = normalize_text(s)
+    if not t:
+        return None
+
+    # 헤더 방어
+    if t == "재무년월":
+        return None
+
+    # 이미 표준 포맷이면 그대로
+    if re.fullmatch(r"\d{4}/\d{2}", t):
+        return t
+
+    m = _PERIOD_RE.match(t)
+    if not m:
+        return None
+
+    year, suffix = m.groups()  # suffix: "A" | "E" | None
+
+    if keep_suffix and suffix:
+        return f"{year}{suffix}"
+
+    return f"{year}/{12}"
+
+
+def _normalize_metric_key(col_key: str) -> str:
+    """
+    최종 metric key를 사람이 쓰기 좋은 형태로 정리.
+    """
+    k = col_key
+
+    # 매출액은 '금액'/'YoY'가 분리되어 있으니 명시적으로 이름을 고정
+    if k.startswith("매출액_금액"):
+        return "매출액"
+    if k.startswith("매출액_YoY"):
+        return "매출액YoY"
+
+    # 나머지는 그대로(단위/공백은 _flatten_col에서 제거됨)
+    # 예: "영업이익", "당기순이익", "EPS", "PER", "PBR", "ROE", "EV/EBITDA", "순부채비율"
+    return k
+
+
+def _html_to_df(html: str) -> pd.DataFrame | None:
+    """
+    yearly consensus 테이블은 2줄 헤더이므로 header=[0,1]로 읽고 flatten한다.
+    """
+    try:
+        dfs = pd.read_html(StringIO(html), header=[0, 1])
+    except Exception as e:
+        logger.exception("pd.read_html failed: {}", e)
+        return None
+    if not dfs:
+        return None
+    df = dfs[0]
+    if df is None or df.empty:
+        return None
+
+    df = df.copy()
+    df.columns = [_flatten_col(c) for c in df.columns]
+    return df
+
+
+def _df_to_metric_map(df: pd.DataFrame) -> dict[str, dict[str, Any]]:
+    """
+    DataFrame(row: period, col: metric) -> {metric: {period: value}} 로 pivot
+    """
+    if df is None or df.empty:
+        return {}
+
+    # NaN -> None
+    df = df.where(pd.notnull(df), None)
+
+    # '재무년월' 컬럼 찾기(안정)
+    # 보통 "재무년월"로 flatten 되지만, 혹시 깨지는 경우 대비
+    period_col = None
+    for c in df.columns:
+        if "재무년월" == c or c.endswith("재무년월") or "재무년월" in c:
+            period_col = c
+            break
+    if not period_col:
+        logger.warning("[cTB25] period column not found")
+        return {}
+
+    out: dict[str, dict[str, Any]] = {}
+
+    for _, row in df.iterrows():
+        period = _normalize_period(row.get(period_col), keep_suffix=True)
+        if not period:
+            continue
+
+        for col, raw_val in row.items():
+            if col == period_col:
+                continue
+            # 주재무제표는 metric-map에서 제외(원하면 따로 meta로 빼도 됨)
+            if "주재무제표" in str(col):
+                continue
+
+            metric = _normalize_metric_key(str(col))
+
+            num = to_float(raw_val)
+            val: Any = num if num is not None else (normalize_text(raw_val) or None)
+
+            out.setdefault(metric, {})[period] = val
+
+    return out
+
+
+async def parse_c101_yearly_consensus_table(
+    browser: BrowserPort,
+) -> dict[str, dict[str, Any]]:
+    """
+    #cTB25 (3년 실적 + 2년 추정) 테이블을
+    {metric: {period: value}} 형태로 반환한다.
+    """
+    await browser.wait_attached(_YEARLY_CONSENSUS_TABLE)
+    await browser.wait_table_nth_ready(
+        _YEARLY_CONSENSUS_TABLE,
+        index=0,
+        min_rows=5,
+        timeout_ms=30_000,
+        poll_ms=200,
+    )
+
+    html = await browser.outer_html_nth(_YEARLY_CONSENSUS_TABLE, 0)
+    if not html or "<table" not in html:
+        logger.warning("[cTB25] outerHTML invalid or empty")
+        return {}
+
+    df = _html_to_df(html)
+    if df is None:
+        logger.warning("[cTB25] df is empty/invalid")
+        return {}
+
+    return _df_to_metric_map(df)

scraper2_hj3415/app/parsing/c101_parser.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from typing import Any
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from logging_hj3415 import logger
+
+from .c101.sise import parse_c101_sise_table
+from .c101.earning_surprise import parse_c101_earnings_surprise_table
+from .c101.fundamentals import parse_c101_fundamentals_table
+from .c101.major_shareholders import parse_c101_major_shareholders
+from .c101.company_overview import parse_c101_company_overview
+from .c101.summary_cmp import parse_c101_summary_cmp_table
+from .c101.yearly_consensus import parse_c101_yearly_consensus_table
+
+async def parse_c101_to_dict(browser: BrowserPort) -> dict[str, list[dict[str, Any]]]:
+    parsed_summary_cmp = await parse_c101_summary_cmp_table(browser)
+    logger.debug(f"parsed_summary_cmp data: {parsed_summary_cmp}")
+
+    parsed_sise = await parse_c101_sise_table(browser)
+    logger.debug(f"parsed_sise data: {parsed_sise}")
+
+    parsed_company_overview = await parse_c101_company_overview(browser)
+    logger.debug(f"parsed_company_overview data: {parsed_company_overview}")
+
+    parsed_major_shareholders = await parse_c101_major_shareholders(browser)
+    logger.debug(f"parsed_major_shareholders data: {parsed_major_shareholders}")
+
+    parsed_fundamentals = await parse_c101_fundamentals_table(browser)
+    logger.debug(f"parsed_fundamentals data: {parsed_fundamentals}")
+
+    parsed_earnings_surprise = await parse_c101_earnings_surprise_table(browser)
+    logger.debug(f"parsed_earnings_surprise data: {parsed_earnings_surprise}")
+
+    parsed_yearly_consensus = await parse_c101_yearly_consensus_table(browser)
+    logger.debug(f"parsed_yearly_consensus data: {parsed_yearly_consensus}")
+
+    return {
+        "요약": parsed_summary_cmp,
+        "시세": parsed_sise,
+        "주주현황": parsed_major_shareholders,
+        "기업개요": parsed_company_overview,
+        "펀더멘털": parsed_fundamentals,
+        "어닝서프라이즈": parsed_earnings_surprise,
+        "연간컨센서스": parsed_yearly_consensus,
+    }

scraper2_hj3415/app/parsing/c103_parser.py

@@ -0,0 +1,19 @@
+# scraper2_hj3415/app/parsing/c103_parser.py
+from __future__ import annotations
+from typing import Any
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.parsing._tables.html_table import try_html_table_to_df, df_to_c1034_metric_list
+
+TABLE_XPATH = "xpath=//div[@id='wrapper']//div//table"
+TABLE_INDEX = 2
+
+
+async def parse_c103_current_table(browser: BrowserPort) -> list[dict[str, Any]]:
+    """
+    ✅ 현재 화면 상태(탭/연간/분기/검색 결과)가 이미 준비되었다는 전제.
+    이 상태에서 TABLE_INDEX 테이블만 읽어서 rows로 변환한다.
+    """
+    html = await browser.outer_html_nth(TABLE_XPATH, TABLE_INDEX)
+    df = try_html_table_to_df(html)
+    return df_to_c1034_metric_list(df)

scraper2_hj3415/app/parsing/c104_parser.py

@@ -0,0 +1,23 @@
+# scraper2_hj3415/app/parsing/c104_parser.py
+from __future__ import annotations
+
+from typing import Any
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.parsing._tables.html_table import try_html_table_to_df, df_to_c1034_metric_list
+
+TABLE_XPATH = 'xpath=//table[@class="gHead01 all-width data-list"]'
+
+
+async def parse_c104_current_table(
+    browser: BrowserPort,
+    *,
+    table_index: int,
+) -> list[dict[str, Any]]:
+    """
+    ✅ 현재 화면 상태(탭/연간/분기/검색 결과)가 이미 준비되었다는 전제.
+    이 상태에서 지정된 table_index 테이블만 읽어서 rows로 변환한다.
+    """
+    html = await browser.outer_html_nth(TABLE_XPATH, table_index)
+    df = try_html_table_to_df(html)
+    return df_to_c1034_metric_list(df)

scraper2_hj3415/app/parsing/c106_parser.py

@@ -0,0 +1,137 @@
+# scraper2_hj3415/app/parsing/c106_parser.py
+from __future__ import annotations
+
+from io import StringIO
+import re
+import numpy as np
+import pandas as pd
+from typing import Any
+
+from common_hj3415.utils import clean_text
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from scraper2_hj3415.app.parsing._normalize.label import (
+    normalize_metric_label,
+    sanitize_label,
+)
+from logging_hj3415 import logger
+
+_CODE_RE = re.compile(r"\b\d{6}\b")
+
+
+async def parse_c106_header_codes(browser: BrowserPort) -> list[str]:
+    """
+    현재 페이지에서 '기업간비교자료' 헤더(회사명들)에서 종목코드(6자리)만 추출한다.
+    (goto/sleep 없음)
+    """
+    selector = (
+        'xpath=//caption[contains(normalize-space(.), "기업간비교자료")]'
+        "/following-sibling::thead//th[not(@colspan)]"
+    )
+    await browser.wait_attached(selector)
+    th_texts = await browser.all_texts(selector)
+
+    codes: list[str] = []
+    for i, t in enumerate(th_texts):
+        text = (t or "").strip()
+        if not text:
+            continue
+        m = _CODE_RE.search(text)
+        if not m:
+            continue
+        codes.append(m.group(0))
+
+    # 중복 제거(순서 유지)
+    seen: set[str] = set()
+    uniq: list[str] = []
+    for c in codes:
+        if c not in seen:
+            seen.add(c)
+            uniq.append(c)
+    logger.debug(f"c106 header codes: {uniq}")
+    return uniq
+
+
+def html_table_to_df(html: str, codes: list[str]) -> pd.DataFrame:
+    df = pd.read_html(StringIO(html), header=None)[0]
+    if df is None or df.empty:
+        return pd.DataFrame()
+
+    df.columns = ["항목_group", "항목"] + codes
+    df["항목_group"] = df["항목_group"].ffill()
+
+    # 첫 두 줄 주가데이터 주입(기존 로직 유지)
+    for i in range(min(2, len(df))):
+        row = df.loc[i].tolist()
+        new_row = ["주가데이터"] + row
+        df.loc[i] = new_row[: len(df.columns)]
+
+    df = df[df["항목"].notna()].reset_index(drop=True)
+    df.loc[df["항목"].isin(["투자의견", "목표주가(원)"]), "항목_group"] = "기타지표"
+    df = df[df["항목"] != "재무연월"].reset_index(drop=True)
+
+    for col in df.columns[2:]:
+        df[col] = df[col].replace("-", "0")
+        df[col] = pd.to_numeric(df[col], errors="coerce")
+
+    df["항목_group"] = df["항목_group"].astype("string").map(clean_text)
+    df["항목"] = df["항목"].astype("string").map(clean_text)
+
+    return df.replace({np.nan: None})
+
+
+def df_to_c106_metric_list(df: pd.DataFrame) -> list[dict[str, Any]]:
+    """
+    C106 DataFrame -> records(list[dict])
+
+    A안 적용:
+    - 항목(key)은 normalize_c1034_item으로 강하게 정규화(괄호/별표 등 제거)
+    - 항목_raw는 정규화 전(단 UI 노이즈만 제거된) 원라벨을 저장
+    - 항목_group은 그대로 두되, 필요 없으면 caller에서 삭제하면 됨
+    """
+    if df is None or df.empty:
+        return []
+
+    df = df.copy()
+
+    # raw 보존(정규화 전, UI 노이즈만 제거)
+    raw = df["항목"].where(df["항목"].notna(), None)
+    df["항목_raw"] = raw.map(
+        lambda x: sanitize_label(str(x)) if x is not None else None
+    )
+
+    # 항목_group 컬럼들은 제거(있을 때만)
+    drop_cols = [c for c in ("항목_group", "항목_group_raw") if c in df.columns]
+    if drop_cols:
+        df = df.drop(columns=drop_cols)
+
+    # key 정규화(A안)
+    df["항목"] = df["항목"].map(
+        lambda x: normalize_metric_label(str(x)) if x is not None else ""
+    )
+
+    # 유효 행만
+    df = df[df["항목"].astype(str).str.strip() != ""].reset_index(drop=True)
+
+    # NaN -> None
+    df = df.where(pd.notnull(df), None)
+
+    return df.to_dict(orient="records")
+
+
+async def parse_c106_current_table(
+    browser: BrowserPort,
+    *,
+    columns: list[str],
+    table_selector: str = "#cTB611",
+    table_index: int = 0,
+    timeout_ms: int = 10_000,
+) -> list[dict[str, Any]]:
+    """
+    ✅ 현재 화면(이미 goto/대기 완료된 상태)에서 비교테이블만 파싱한다.
+    """
+    await browser.wait_table_nth_ready(
+        table_selector, index=table_index, min_rows=3, timeout_ms=timeout_ms
+    )
+    html = await browser.outer_html_nth(table_selector, table_index)
+    df = html_table_to_df(html, columns)
+    return df_to_c106_metric_list(df)