scraper2-hj3415 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

scraper2/app/parsing/c103_parser.py

@@ -0,0 +1,128 @@
+# scraper2/app/parsing/c103_parser.py
+from __future__ import annotations
+
+import asyncio
+from io import StringIO
+from typing import Any
+
+import pandas as pd
+
+from scraper2.app.ports.browser.browser_port import BrowserPort
+from scraper2.app.parsing._normalize import normalize_c1034_df
+
+# ---- constants ----
+
+TABLE_XPATH = "xpath=//div[@id='wrapper']//div//table"   # 기존 selector 의도 유지
+TABLE_INDEX = 2  # 너 코드의 nth(2)
+
+
+BTN_SETS: dict[str, list[tuple[str, str]]] = {
+    "손익계산서y": [
+        ("손익계산서", 'xpath=//*[@id="rpt_tab1"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "재무상태표y": [
+        ("재무상태표", 'xpath=//*[@id="rpt_tab2"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "현금흐름표y": [
+        ("현금흐름표", 'xpath=//*[@id="rpt_tab3"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "손익계산서q": [
+        ("손익계산서", 'xpath=//*[@id="rpt_tab1"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "재무상태표q": [
+        ("재무상태표", 'xpath=//*[@id="rpt_tab2"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "현금흐름표q": [
+        ("현금흐름표", 'xpath=//*[@id="rpt_tab3"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+}
+
+
+# ---- small helpers ----
+
+async def _click_steps(
+    browser: BrowserPort,
+    steps: list[tuple[str, str]],
+    *,
+    jitter_sec: float = 0.6,
+) -> None:
+    """
+    (goto 없음) 현재 페이지에서 탭/라디오/검색 버튼 클릭만 수행.
+    """
+    for _name, selector in steps:
+        await browser.wait(selector)
+        await browser.click(selector)
+        # 서버/클라이언트 부담 줄이기: 작은 지터
+        await asyncio.sleep(0.2 + (jitter_sec * 0.5))
+
+
+def _html_table_to_df(html: str) -> pd.DataFrame:
+    """
+    table outerHTML -> DataFrame
+    """
+    dfs = pd.read_html(StringIO(html), header=0)
+    if not dfs:
+        raise ValueError("pd.read_html() 테이블 파싱 실패")
+    return dfs[0]
+
+
+async def _nth_table_outer_html(browser: BrowserPort, nth: int) -> str:
+    return await browser.outer_html_nth(TABLE_XPATH, nth)
+
+def df_to_c103_records(df: pd.DataFrame) -> list[dict[str, Any]]:
+    """
+    C103 테이블 DataFrame -> 정규화된 records(list[dict])
+    - 항목이 비면 제거
+    """
+    df = normalize_c1034_df(df)
+    if df is None or df.empty:
+        return []
+
+    records: list[dict[str, Any]] = []
+    for r in df.to_dict(orient="records"):
+        item = r.get("항목")
+        if not item:
+            continue
+        records.append(r)
+    return records
+
+# ---- public parser ----
+
+async def parse_c103_to_dict(browser: BrowserPort) -> dict[str, list[dict[str, Any]]]:
+    """
+    C103 파서: dict만 반환
+    {
+      "손익계산서y": [ {...}, ... ],
+      "손익계산서q": [ {...}, ... ],
+      ...
+    }
+    """
+    out: dict[str, list[dict[str, Any]]] = {}
+
+    for key, steps in BTN_SETS.items():
+        # 클릭 → 테이블 로딩 → nth table outerHTML → df → records
+        await _click_steps(browser, steps)
+        await browser.wait(TABLE_XPATH)
+
+        try:
+            html = await _nth_table_outer_html(browser, TABLE_INDEX)
+            df = _html_table_to_df(html)
+            out[key] = df_to_c103_records(df)
+        except Exception:
+            out[key] = []  # 실패는 빈 리스트
+
+    return out
+
+

scraper2/app/parsing/c104_parser.py

@@ -0,0 +1,143 @@
+# scraper2/app/parsing/c104_parser.py
+from __future__ import annotations
+
+import asyncio
+from io import StringIO
+from typing import Any
+
+import pandas as pd
+
+from scraper2.app.ports.browser.browser_port import BrowserPort
+from scraper2.app.parsing._normalize import normalize_c1034_df
+
+
+# ---- constants ----
+
+TABLE_SELECTOR = 'xpath=//table[@class="gHead01 all-width data-list"]'
+
+BTN_SETS: dict[str, list[tuple[str, str]]] = {
+    "수익성y": [
+        ("수익성", 'xpath=//*[ @id="val_tab1"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "성장성y": [
+        ("성장성", 'xpath=//*[ @id="val_tab2"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "안정성y": [
+        ("안정성", 'xpath=//*[ @id="val_tab3"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "활동성y": [
+        ("활동성", 'xpath=//*[ @id="val_tab4"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "가치분석y": [
+        ("가치분석연간", 'xpath=//*[@id="frqTyp0_2"]'),
+        ("가치분석검색", 'xpath=//*[@id="hfinGubun2"]'),
+    ],
+    "수익성q": [
+        ("수익성", 'xpath=//*[ @id="val_tab1"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "성장성q": [
+        ("성장성", 'xpath=//*[ @id="val_tab2"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "안정성q": [
+        ("안정성", 'xpath=//*[ @id="val_tab3"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "활동성q": [
+        ("활동성", 'xpath=//*[ @id="val_tab4"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "가치분석q": [
+        ("가치분석분기", 'xpath=//*[@id="frqTyp1_2"]'),
+        ("가치분석검색", 'xpath=//*[@id="hfinGubun2"]'),
+    ],
+}
+
+
+# ---- small helpers ----
+
+async def _click_steps(
+    browser: BrowserPort,
+    steps: list[tuple[str, str]],
+    *,
+    jitter_sec: float = 0.6,
+) -> None:
+    """
+    (goto 없음) 현재 페이지에서 탭/라디오/검색 버튼 클릭만 수행.
+    """
+    for _name, selector in steps:
+        await browser.wait(selector)
+        await browser.click(selector)
+        await asyncio.sleep(0.2 + (jitter_sec * 0.5))
+
+
+def _html_table_to_df(html: str) -> pd.DataFrame:
+    dfs = pd.read_html(StringIO(html), header=0)
+    if not dfs:
+        raise ValueError("pd.read_html() 테이블 파싱 실패")
+    return dfs[0]
+
+
+async def _table_outer_html(browser: BrowserPort, *, is_value_analysis: bool) -> str:
+    """
+    c104 원본 로직 유지:
+      - 가치분석*: 두 번째 테이블(nth=1)
+      - 그 외: 첫 번째 테이블(nth=0)
+    """
+    idx = 1 if is_value_analysis else 0
+    return await browser.outer_html_nth(TABLE_SELECTOR, idx)
+
+
+def df_to_c104_records(df: pd.DataFrame) -> list[dict[str, Any]]:
+    """
+    C104 테이블 DataFrame -> records(list[dict])
+    - NaN -> None
+    - (원하면 normalize_c104_df(df) 추가)
+    """
+    df = normalize_c1034_df(df)
+    if df is None or df.empty:
+        return []
+
+    df = df.where(pd.notnull(df), None)
+
+    return df.to_dict(orient="records")
+
+
+# ---- public parser ----
+
+async def parse_c104_to_dict(browser: BrowserPort) -> dict[str, list[dict[str, Any]]]:
+    """
+    C104 파서: dict만 반환
+    {
+      "수익성y": [ {...}, ... ],
+      "가치분석q": [ {...}, ... ],
+      ...
+    }
+    """
+    out: dict[str, list[dict[str, Any]]] = {}
+
+    for key, steps in BTN_SETS.items():
+        await _click_steps(browser, steps)
+        await browser.wait(TABLE_SELECTOR)
+
+        try:
+            html = await _table_outer_html(browser, is_value_analysis=key.startswith("가치분석"))
+            df = _html_table_to_df(html)
+            out[key] = df_to_c104_records(df)
+        except Exception:
+            out[key] = []
+
+    return out

scraper2/app/parsing/c106_parser.py

@@ -0,0 +1,153 @@
+# scraper2/app/parsing/c106_parser.py
+from __future__ import annotations
+
+from io import StringIO
+from typing import Dict
+
+import numpy as np
+import pandas as pd
+
+from scraper2.app.ports.browser.browser_port import BrowserPort
+
+
+# ---------- pure helpers (sync) ----------
+
+def df_to_metrics(df: pd.DataFrame) -> dict[str, dict[str, float | None]]:
+    """
+    DataFrame ->
+      { "<항목2>": { "<회사명>": float|None, ... }, ... }
+    """
+    out: dict[str, dict[str, float | None]] = {}
+    if df is None or df.empty:
+        return out
+
+    rows = df.replace({np.nan: None}).to_dict(orient="records")
+    for r in rows:
+        metric = r.get("항목2")
+        if not metric:
+            continue
+
+        bucket = out.setdefault(str(metric).strip(), {})
+        for k, v in r.items():
+            if k in ("항목", "항목2"):
+                continue
+            bucket[str(k)] = None if v is None else float(v)
+
+    return out
+
+
+def df_from_comparison_html(html: str, company_names: list[str]) -> pd.DataFrame:
+    """
+    #cTB611 테이블 HTML -> 정제된 DataFrame
+    """
+    df = pd.read_html(StringIO(html), header=None)[0]
+
+    # 첫 번째 열은 '항목', 두 번째 열은 '항목2', 나머지는 회사명
+    df.columns = ["항목", "항목2"] + company_names
+
+    # 대분류(항목) NaN은 이전 값으로 채움
+    df["항목"] = df["항목"].ffill()
+
+    # 첫 두 줄만 '주가데이터' 삽입(기존 로직 유지)
+    for i in range(min(2, len(df))):
+        row = df.loc[i].tolist()
+        new_row = ["주가데이터"] + row
+        df.loc[i] = new_row[: len(df.columns)]
+
+    # 항목2 없는 행 제거
+    df = df[df["항목2"].notna()].reset_index(drop=True)
+
+    # 특정 항목은 대분류 수동 지정
+    df.loc[df["항목2"].isin(["투자의견", "목표주가(원)"]), "항목"] = "기타지표"
+
+    # 필요없는 행 제거
+    df = df[df["항목2"] != "재무연월"].reset_index(drop=True)
+
+    # 숫자 정리 (회사 컬럼만)
+    for col in df.columns[2:]:
+        df[col] = df[col].replace("-", "0")
+        df[col] = pd.to_numeric(df[col], errors="coerce")
+
+    # 문자열 정리
+    df["항목"] = df["항목"].astype(str).str.replace("펼치기", "").str.strip()
+    df["항목2"] = df["항목2"].astype(str).str.strip()
+
+    return df
+
+
+# ---------- parsers (async, BrowserPort only) ----------
+
+async def parse_c106_header(browser: BrowserPort) -> list[str]:
+    """
+    현재 페이지에서 '기업간비교자료' 헤더(회사명들)만 뽑는다.
+    (goto/sleep 없음)
+    """
+    await browser.wait("#cTB611_h")
+
+    selector = (
+        'xpath=//caption[contains(text(), "기업간비교자료")]/following-sibling::thead//th[not(@colspan)]'
+    )
+    th_texts = await browser.all_texts(selector)
+
+    names: list[str] = []
+    for t in th_texts:
+        name = t.strip().split("\n")[0]
+        if name:
+            names.append(name)
+    return names
+
+
+async def parse_c106_table_to_metrics(
+    browser: BrowserPort,
+    company_names: list[str],
+) -> dict[str, dict[str, float | None]]:
+    """
+    현재 페이지에서 #cTB611 테이블을 읽어 metrics dict로 변환한다.
+    """
+    await browser.wait("#cTB611")
+    html = await browser.outer_html("#cTB611")
+
+    df = df_from_comparison_html(html, company_names)
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError(f"c106 df invalid: type={type(df)}")
+    return df_to_metrics(df)
+
+
+METRIC_NAME_MAP = {
+    "전일종가(원)": "전일종가",
+    "시가총액(억원)": "시가총액",
+
+    "자산총계(억원)": "자산총계",
+    "부채총계(억원)": "부채총계",
+
+    "매출액(억원)": "매출액",
+    "영업이익(억원)": "영업이익",
+    "당기순이익(억원)": "당기순이익",
+    "당기순이익(지배)(억원)": "당기순이익_지배",
+
+    "영업이익률(%)": "영업이익률",
+    "순이익률(%)": "순이익률",
+    "ROE(%)": "ROE",
+    "부채비율(%)": "부채비율",
+
+    "PER": "PER",
+    "PBR": "PBR",
+
+    "투자의견": "투자의견",
+    "목표주가(원)": "목표주가",
+}
+
+def normalize_c106_metrics(
+    raw: Dict[str, Dict[str, float | None]],
+) -> Dict[str, Dict[str, float | None]]:
+    out: Dict[str, Dict[str, float | None]] = {}
+
+    for raw_name, company_map in raw.items():
+        field = METRIC_NAME_MAP.get(raw_name)
+        if not field:
+            # 모르는 항목은 버리거나 로그만 남김
+            continue
+
+        out[field] = company_map
+
+    return out

scraper2/app/parsing/c108_parser.py

@@ -0,0 +1,65 @@
+# scraper2/app/parsing/c108_parser.py
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+from scraper2.app.ports.browser.browser_port import BrowserPort
+
+
+def extract_bullets(text: str | None) -> list[str]:
+    if not text:
+        return []
+    return [
+        line.replace("▶", "").strip()
+        for line in text.splitlines()
+        if line.strip().startswith("▶")
+    ]
+
+
+async def parse_c108_to_dicts(browser: BrowserPort) -> list[dict[str, Any]]:
+    title = await browser.title()
+    url = await browser.current_url()
+
+    if "접속장애" in title:
+        print(
+            "[C108][ACCESS_ERROR] "
+            f"title={title!r} url={url}"
+        )
+        return []
+
+    # table → records (adapter가 처리)
+    try:
+        records = await browser.table_records("#tableCmpDetail", header=0)
+    except Exception as e:
+        print("table_records failed:", type(e).__name__, e)
+        return []
+
+    # 날짜 변환(파서에서 처리)
+    # records의 key가 '일자'일 가능성이 있으니 통일
+    for r in records:
+        if "일자" in r and "날짜" not in r:
+            r["날짜"] = r.pop("일자")
+
+        # "%y/%m/%d" -> "%Y.%m.%d"
+        if r.get("날짜"):
+            dt = pd.to_datetime(r["날짜"], format="%y/%m/%d", errors="coerce")
+            r["날짜"] = None if pd.isna(dt) else dt.strftime("%Y.%m.%d")
+
+    # 내용 추출
+    contents: list[list[str]] = []
+    row_count = len(records)
+
+    for i in range(row_count):
+        try:
+            await browser.click(f"#a{i}")
+            await browser.wait(f"#c{i} > div > div.comment-body")
+            text = await browser.inner_text(f"#c{i} > div > div.comment-body")
+            contents.append(extract_bullets(text))
+        except Exception:
+            contents.append([])
+
+    # records에 내용/코드 주입
+    for r, content in zip(records, contents, strict=False):
+        r["내용"] = content
+    return records

scraper2/app/ports/browser/browser_factory_port.py

@@ -0,0 +1,11 @@
+# scraper2/app/ports/browser/browser_factory_port.py
+from __future__ import annotations
+from typing import AsyncIterator, Protocol
+from contextlib import asynccontextmanager
+
+from scraper2.app.ports.browser.browser_port import BrowserPort
+
+class BrowserFactoryPort(Protocol):
+    @asynccontextmanager
+    async def lease(self) -> AsyncIterator[BrowserPort]: ...
+    async def aclose(self) -> None: ...

scraper2/app/ports/browser/browser_port.py

@@ -0,0 +1,22 @@
+# scraper2/app/ports/browser/browser_port.py
+from __future__ import annotations
+from typing import Protocol, Any
+
+class BrowserPort(Protocol):
+    async def goto(self, url: str, timeout_ms: int = 10_000) -> None: ...
+    async def title(self) -> str: ...
+    async def current_url(self) -> str: ...
+    async def text(self, selector: str) -> str: ...
+    async def texts(self, selector: str) -> list[str]: ...
+    async def text_first_by_text(self, needle: str) -> str: ...
+    async def inner_text(self, selector: str) -> str: ...
+    async def click(self, selector: str) -> None: ...
+    async def wait(self, selector: str, timeout_ms: int = 10_000) -> None: ...
+    # 핵심 추가: table → records
+    async def table_records(self, table_selector: str, *, header: int | list[int] = 0) -> list[dict[str, Any]]: ...
+    # c106에서 사용
+    async def outer_html(self, selector: str) -> str: ...
+    async def all_texts(self, selector: str) -> list[str]: ...
+
+    # c1034 추가: selector로 여러 개 잡히는 것 중 nth(0-based)의 outerHTML 반환
+    async def outer_html_nth(self, selector: str, index: int) -> str: ...

scraper2/app/ports/ingest_port.py

@@ -0,0 +1,13 @@
+# scraper2/app/ports/ingest_port.py
+from typing import Protocol, Iterable
+from datetime import datetime
+
+class IngestPort(Protocol):
+    async def execute_many(
+        self,
+        codes: list[str],
+        *,
+        sleep_sec: float = ...,
+        asof: datetime | None = None,
+    ) -> list[object]:
+        ...

scraper2/app/ports/sinks/base_sink_port.py

@@ -0,0 +1,14 @@
+# scraper2/app/ports/sinks/base_sink_port.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Protocol, TypeVar, Iterable, Optional
+
+T = TypeVar("T")
+
+class SinkPort(Protocol[T]):
+    async def write(self, dto: T, *, asof: Optional[datetime] = None) -> None:
+        ...
+
+    async def write_many(self, dtos: Iterable[T], *, asof: Optional[datetime] = None) -> None:
+        ...

scraper2/app/ports/sinks/c101_sink_port.py

@@ -0,0 +1,9 @@
+# scraper2/app/ports/sinks/c101_sink_port.py
+from __future__ import annotations
+from typing import Protocol
+from scraper2.app.ports.sinks.base_sink_port import SinkPort
+from contracts.nfs.c101 import C101DTO
+
+class C101SinkPort(SinkPort[C101DTO], Protocol):
+    """C101DTO 저장 싱크"""
+    ...

scraper2/app/ports/sinks/c103_sink_port.py

@@ -0,0 +1,9 @@
+# scraper2/app/ports/sinks/c103_sink_port.py
+from __future__ import annotations
+from typing import Protocol
+from scraper2.app.ports.sinks.base_sink_port import SinkPort
+from contracts.nfs.c103 import C103DTO
+
+class C103SinkPort(SinkPort[C103DTO], Protocol):
+    """C103DTO 저장 싱크"""
+    ...

scraper2/app/ports/sinks/c104_sink_port.py

@@ -0,0 +1,9 @@
+# scraper2/app/ports/sinks/c104_sink_port.py
+from __future__ import annotations
+from typing import Protocol
+from scraper2.app.ports.sinks.base_sink_port import SinkPort
+from contracts.nfs.c104 import C104DTO
+
+class C104SinkPort(SinkPort[C104DTO], Protocol):
+    """C104DTO 저장 싱크"""
+    ...

scraper2/app/ports/sinks/c106_sink_port.py

@@ -0,0 +1,9 @@
+# scraper2/app/ports/sinks/c106_sink_port.py
+from __future__ import annotations
+from typing import Protocol
+from scraper2.app.ports.sinks.base_sink_port import SinkPort
+from contracts.nfs.c106 import C106DTO
+
+class C106SinkPort(SinkPort[C106DTO], Protocol):
+    """C106DTO 저장 싱크"""
+    ...

scraper2/app/ports/sinks/c108_sink_port.py

@@ -0,0 +1,9 @@
+# scraper2/app/ports/sinks/c108_sink_port.py
+from __future__ import annotations
+from typing import Protocol
+from scraper2.app.ports.sinks.base_sink_port import SinkPort
+from contracts.nfs.c108 import C108DTO
+
+class C108SinkPort(SinkPort[C108DTO], Protocol):
+    """C108DTO 저장 싱크"""
+    ...

scraper2/app/usecases/fetch/fetch_c101.py

@@ -0,0 +1,43 @@
+# scraper2/app/usecases/fetch/fetch_c101.py
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable
+from contracts.nfs.c101 import C101DTO
+from scraper2.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2.app.parsing.c101_parser import parse_c101_to_dict
+
+
+class FetchC101:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> C101DTO | None:
+        async with self.factory.lease() as browser:
+            url = f"https://navercomp.wisereport.co.kr/v2/company/c1010001.aspx?cmp_cd={code}"
+            await browser.goto(url, timeout_ms=10_000)
+
+            jitter_sec = 1.0
+            if sleep_sec > 0:
+                delay = sleep_sec + random.uniform(0, jitter_sec)
+                await asyncio.sleep(delay)
+
+            d = await parse_c101_to_dict(browser)
+            return None if not d else C101DTO(**d)
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C101DTO | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+    ) -> list[C101DTO]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
+            return_exceptions=False,
+        )
+        return [r for r in results if r is not None]
+