scraper2-hj3415 2.4.1__py3-none-any.whl → 2.6.0__py3-none-any.whl

scraper2_hj3415/app/services/fetch/fetch_c104.py

@@ -0,0 +1,183 @@
+# scraper2_hj3415/app/usecases/fetch/fetch_c104.py
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable, Any
+
+from logging_hj3415 import logger
+from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2_hj3415.app.ports.site.wisereport_port import WiseReportPort
+from scraper2_hj3415.app.adapters.site.wisereport_playwright import WiseReportPlaywright
+
+from scraper2_hj3415.app.parsing.c104_parser import (
+    parse_c104_current_table,
+    TABLE_XPATH,
+)
+from scraper2_hj3415.app.services.nfs_doc_builders import build_metrics_doc_from_parsed
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import BLOCK_KEYS_BY_ENDPOINT
+
+
+BTN_SETS: dict[str, list[tuple[str, str]]] = {
+    "수익성y": [
+        ("수익성", 'xpath=//*[ @id="val_tab1"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "성장성y": [
+        ("성장성", 'xpath=//*[ @id="val_tab2"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "안정성y": [
+        ("안정성", 'xpath=//*[ @id="val_tab3"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "활동성y": [
+        ("활동성", 'xpath=//*[ @id="val_tab4"]'),
+        ("연간", 'xpath=//*[@id="frqTyp0"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "가치분석y": [
+        ("가치분석연간", 'xpath=//*[@id="frqTyp0_2"]'),
+        ("가치분석검색", 'xpath=//*[@id="hfinGubun2"]'),
+    ],
+    "수익성q": [
+        ("수익성", 'xpath=//*[ @id="val_tab1"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "성장성q": [
+        ("성장성", 'xpath=//*[ @id="val_tab2"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "안정성q": [
+        ("안정성", 'xpath=//*[ @id="val_tab3"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "활동성q": [
+        ("활동성", 'xpath=//*[ @id="val_tab4"]'),
+        ("분기", 'xpath=//*[@id="frqTyp1"]'),
+        ("검색", 'xpath=//*[@id="hfinGubun"]'),
+    ],
+    "가치분석q": [
+        ("가치분석분기", 'xpath=//*[@id="frqTyp1_2"]'),
+        ("가치분석검색", 'xpath=//*[@id="hfinGubun2"]'),
+    ],
+}
+
+
+def _is_value_analysis(key: str) -> bool:
+    return key.startswith("가치분석")
+
+
+def _table_index_for_key(key: str) -> int:
+    # ✅ 네 주석대로 가치분석만 1, 나머지 0
+    return 1 if _is_value_analysis(key) else 0
+
+
+class FetchC104:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> NfsDoc | None:
+        async with self.factory.lease() as browser:
+            wr: WiseReportPort = WiseReportPlaywright(browser)
+
+            url = (
+                "https://navercomp.wisereport.co.kr/v2/company/c1040001.aspx"
+                f"?cn=&cmp_cd={code}"
+            )
+            await browser.goto_and_wait_for_stable(url, timeout_ms=10_000)
+
+            if sleep_sec > 0:
+                await asyncio.sleep(sleep_sec + random.uniform(0, 1.0))
+
+            parsed: dict[str, list[dict[str, Any]]] = {}
+
+            # ✅ table index별로 prev_text를 따로 들고가야 안정적
+            prev_text_by_idx: dict[int, str | None] = {0: None, 1: None}
+
+            # ✅ 최초 baseline 확보(둘 다 시도)
+            for idx in (0, 1):
+                try:
+                    prev_text_by_idx[idx] = await browser.wait_table_text_changed(
+                        TABLE_XPATH,
+                        index=idx,
+                        prev_text=None,
+                        min_rows=5,
+                        min_lines=50,
+                        timeout_sec=10.0,
+                    )
+                except Exception:
+                    prev_text_by_idx[idx] = None
+
+            for key, steps in BTN_SETS.items():
+                idx = _table_index_for_key(key)
+
+                # ✅ 상태 전환(행동)
+                await wr.click_steps(steps, jitter_sec=0.6)
+                await wr.ensure_yearly_consensus_open_in_table_nth(
+                    table_selector=TABLE_XPATH,
+                    table_index=idx,
+                )
+
+                # ✅ 데이터 변경 대기(행동) - idx별로 추적
+                prev_text_by_idx[idx] = await browser.wait_table_text_changed(
+                    TABLE_XPATH,
+                    index=idx,
+                    prev_text=prev_text_by_idx[idx],
+                    min_rows=5,
+                    min_lines=50,
+                    timeout_sec=12.0,
+                )
+
+                # ✅ 파싱은 “현재 화면의 idx 테이블 1개”만
+                try:
+                    parsed[key] = await parse_c104_current_table(
+                        browser,
+                        table_index=idx,
+                    )
+                except Exception:
+                    parsed[key] = []
+
+            logger.debug(f"parsed:\n{parsed}")
+
+            block_keys = BLOCK_KEYS_BY_ENDPOINT[EndpointKind.C104]
+            if not parsed or all(not (parsed.get(str(bk)) or []) for bk in block_keys):
+                logger.warning(
+                    f"c104 fetch: parsed result empty; return None | code={code}"
+                )
+                return None
+
+            doc = build_metrics_doc_from_parsed(
+                code=code,
+                endpoint_kind=EndpointKind.C104,
+                parsed=parsed,
+                block_keys=block_keys,
+                item_key="항목",
+                raw_label_key="항목_raw",
+                keep_empty_blocks=True,
+            )
+            logger.debug(f"c104 doc: {doc}")
+            return doc
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> NfsDoc | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+    ) -> list[NfsDoc]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
+            return_exceptions=False,
+        )
+        return [r for r in results if r is not None]

scraper2_hj3415/app/services/fetch/fetch_c106.py

@@ -0,0 +1,90 @@
+# scraper2_hj3415/app/usecases/fetch/fetch_c106.py
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable, Any
+
+from logging_hj3415 import logger
+from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2_hj3415.app.parsing.c106_parser import (
+    parse_c106_header_codes,
+    parse_c106_current_table,
+)
+from scraper2_hj3415.app.services.nfs_doc_builders import build_metrics_doc_from_parsed
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import BLOCK_KEYS_BY_ENDPOINT
+
+
+class FetchC106:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> NfsDoc | None:
+        async with self.factory.lease() as browser:
+            # 1) 헤더 코드 추출용(기준 페이지)
+            url0 = (
+                "https://navercomp.wisereport.co.kr/v2/company/c1060001.aspx"
+                f"?cn=&cmp_cd={code}"
+            )
+            await browser.goto_and_wait_for_stable(url0, timeout_ms=10_000)
+
+            if sleep_sec > 0:
+                await asyncio.sleep(sleep_sec + random.uniform(0, 1.0))
+
+            header_codes = await parse_c106_header_codes(browser)
+            if not header_codes:
+                logger.warning(f"c106 fetch: header codes empty; code={code}")
+                return None
+
+            base_url = (
+                "https://navercomp.wisereport.co.kr/v2/company/cF6002.aspx"
+                f"?cmp_cd={code}&finGubun=MAIN&sec_cd=FG000&frq="
+            )
+
+            parsed: dict[str, list[dict[str, Any]]] = {}
+
+            for frq in ("q", "y"):
+                url = base_url + frq
+                await browser.goto_and_wait_for_stable(url, timeout_ms=10_000)
+
+                # 기존 지터 유지(필요하면 정책화)
+                await asyncio.sleep(0.5 + random.uniform(0, 0.3))
+
+                parsed[frq] = await parse_c106_current_table(
+                    browser,
+                    columns=header_codes,
+                    table_selector="#cTB611",
+                    table_index=0,
+                    timeout_ms=10_000,
+                )
+
+            logger.debug(f"parsed:\n{parsed}")
+
+            block_keys = BLOCK_KEYS_BY_ENDPOINT[EndpointKind.C106]
+            if not parsed or all(not (parsed.get(str(bk)) or []) for bk in block_keys):
+                logger.warning(f"c106 fetch: parsed result empty; return None | code={code}")
+                return None
+
+            doc = build_metrics_doc_from_parsed(
+                code=code,
+                endpoint_kind=EndpointKind.C106,
+                parsed=parsed,
+                block_keys=block_keys,
+                item_key="항목",
+                raw_label_key="항목_raw",
+                keep_empty_blocks=True,
+            )
+            logger.debug(f"c106 doc: {doc}")
+            return doc
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> NfsDoc | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(self, codes: Iterable[str], *, sleep_sec: float = 2.0) -> list[NfsDoc]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
+            return_exceptions=False,
+        )
+        return [r for r in results if r is not None]

scraper2_hj3415/app/services/fetch/fetch_c108.py

@@ -0,0 +1,59 @@
+# scraper2_hj3415/app/usecases/fetch/fetch_c108.py
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable
+
+from logging_hj3415 import logger
+from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2_hj3415.app.parsing.c108_parser import parse_c108_to_dict
+from scraper2_hj3415.app.services.nfs_doc_builders import build_c108_doc_from_parsed
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import BLOCK_KEYS_BY_ENDPOINT
+
+
+class FetchC108:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> NfsDoc | None:
+        async with self.factory.lease() as browser:
+            url = f"https://navercomp.wisereport.co.kr/v2/company/c1080001.aspx?cn=&cmp_cd={code}"
+            await browser.goto_and_wait_for_stable(url, timeout_ms=10_000)
+
+            if sleep_sec > 0:
+                await asyncio.sleep(sleep_sec + random.uniform(0, 1.0))
+
+            parsed = await parse_c108_to_dict(browser=browser)
+
+            logger.debug(f"parsed:\n{parsed}")
+            block_keys = BLOCK_KEYS_BY_ENDPOINT[EndpointKind.C108]
+            if not parsed or all(not (parsed.get(str(bk)) or []) for bk in block_keys):
+                logger.warning(
+                    f"c108 fetch: parsed result empty; return None | code={code}"
+                )
+                return None
+
+            doc = build_c108_doc_from_parsed(
+                code=code, parsed=parsed, keep_empty_blocks=True
+            )
+            logger.debug(f"c108 doc: {doc}")
+            return doc
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> NfsDoc | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+    ) -> list[NfsDoc]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
+            return_exceptions=False,
+        )
+        return [r for r in results if r is not None]

scraper2_hj3415/app/services/nfs_doc_builders.py

@@ -0,0 +1,290 @@
+# scraper2_hj3415/app/services/nfs_doc_builders.py
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Mapping, Iterable, Any
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import BLOCK_KEYS_BY_ENDPOINT
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import MetricsBlock, RecordsBlock, KvBlock, BlockData
+from scraper2_hj3415.app.domain.series import MetricSeries
+from scraper2_hj3415.app.domain.types import LabelsMap, MetricKey, Period, Num, BlockKey, Records, Record
+
+from common_hj3415.utils import nan_to_none
+
+
+def is_all_none(row: dict[str, Any]) -> bool:
+    return all(v is None for v in row.values())
+
+
+ParsedBlocks = Mapping[
+    str, Any
+]  # parser가 반환한 "block_key(str) -> rows(list[dict])"
+
+
+def build_metrics_block_and_labels_from_rows(
+    *,
+    endpoint_kind: EndpointKind,
+    block_key: BlockKey,
+    rows: Records,
+    item_key: str = "항목",
+    raw_label_key: str = "항목_raw",
+) -> tuple[MetricsBlock, LabelsMap]:
+    """
+    rows(list[dict]) -> (MetricsBlock, LabelsMap)
+    c103/c104/c106 공통 빌더.
+
+    - Metric key는 item_key(보통 '항목')에서 만들고,
+      기간 컬럼들은 {Period: Num}으로 유지한다.
+    - LabelsMap은 dto_key -> raw_label(정제된 원라벨)
+    """
+    grouped: dict[str, list[tuple[dict[Period, Num], str]]] = defaultdict(list)
+
+    for r in rows:
+        item = r.get(item_key)
+        if not item:
+            continue
+
+        raw_label = r.get(raw_label_key)
+        if raw_label is None:
+            raw_label = item
+
+        per_map: dict[Period, Num] = {
+            str(k): nan_to_none(v)
+            for k, v in r.items()
+            if k not in (item_key, raw_label_key)
+        }
+
+        grouped[item].append((per_map, raw_label))
+
+    series_map: dict[MetricKey, MetricSeries] = {}
+    labels_map: LabelsMap = {}
+
+    for item, pairs in grouped.items():
+        if len(pairs) == 1:
+            per_map, raw_label = pairs[0]
+            series_map[item] = MetricSeries(key=item, values=per_map)
+            labels_map[item] = raw_label
+            continue
+
+        kept = [(per_map, raw) for (per_map, raw) in pairs if not is_all_none(per_map)]
+        if not kept:
+            continue
+
+        for idx, (per_map, raw_label) in enumerate(kept, start=1):
+            mk = item if idx == 1 else f"{item}_{idx}"
+            series_map[mk] = MetricSeries(key=mk, values=per_map)
+            labels_map[mk] = raw_label
+
+    block = MetricsBlock(
+        endpoint_kind=endpoint_kind, block_key=block_key, metrics=series_map
+    )
+    return block, labels_map
+
+
+def build_metrics_doc_from_parsed(
+    *,
+    code: str,
+    endpoint_kind: EndpointKind,
+    parsed: ParsedBlocks,
+    block_keys: Iterable[BlockKey] | None = None,
+    item_key: str = "항목",
+    raw_label_key: str = "항목_raw",
+    keep_empty_blocks: bool = True,
+) -> NfsDoc:
+    """
+    parser가 만든 dict(블록키 -> rows)를 받아서 NfsDoc(=MetricsBlock들)로 조립.
+    - c103/c104/c106 공용으로 사용 가능.
+
+    keep_empty_blocks:
+      - True: block은 항상 생성 (metrics 비어도 block 존재)
+      - False: rows가 없거나 metrics가 비면 blocks에서 제외
+    """
+    if block_keys is None:
+        block_keys = BLOCK_KEYS_BY_ENDPOINT[endpoint_kind]
+
+    blocks: dict[BlockKey, MetricsBlock] = {}
+    labels: dict[BlockKey, LabelsMap] = {}
+
+    for bk in block_keys:
+        rows = parsed.get(str(bk), []) or []
+        block, lm = build_metrics_block_and_labels_from_rows(
+            endpoint_kind=endpoint_kind,
+            block_key=bk,
+            rows=rows,
+            item_key=item_key,
+            raw_label_key=raw_label_key,
+        )
+
+        if not keep_empty_blocks and not block.metrics:
+            continue
+
+        blocks[bk] = block
+        labels[bk] = lm  # 비어있어도 {}로 유지
+
+    return NfsDoc(code=code, endpoint_kind=endpoint_kind, blocks=blocks, labels=labels)
+
+
+def _as_records(x: Any) -> Records:
+    """
+    안전하게 rows를 Records(=Sequence[Record])로 캐스팅/정리.
+    - None/비정상 값이면 빈 리스트
+    - list[dict] 형태만 통과시키고 나머지는 필터
+    """
+    if not x:
+        return []
+    if not isinstance(x, list):
+        return []
+
+    out: list[Record] = []
+    for it in x:
+        if isinstance(it, dict):
+            out.append(it)
+    return out
+
+
+def build_records_block_from_rows(
+    *,
+    endpoint_kind: EndpointKind,
+    block_key: BlockKey,
+    rows: Records,
+) -> RecordsBlock:
+    """
+    rows(list[dict]) -> RecordsBlock
+    - c108 같은 레코드성 블록(리포트 목록 등)에 사용
+    """
+    # RecordsBlock 쪽에서도 __post_init__로 block_key 검증이 수행된다는 전제(네가 정돈한 도메인)
+    return RecordsBlock(endpoint_kind=endpoint_kind, block_key=block_key, rows=list(rows))
+
+
+def build_c108_doc_from_parsed(
+    *,
+    code: str,
+    parsed: ParsedBlocks,
+    block_keys: Iterable[BlockKey] | None = None,
+    keep_empty_blocks: bool = True,
+) -> NfsDoc:
+    """
+    c108 parser 결과(dict)를 받아서 NfsDoc(=RecordsBlock들)로 조립.
+
+    규칙(너가 정한 원칙):
+      - labels는 항상 존재(빈 dict라도)
+      - c108은 labels를 비우는 것을 정상으로 간주
+
+    keep_empty_blocks:
+      - True: block은 항상 생성(rows 비어도 block 존재)
+      - False: rows가 비면 blocks에서 제외
+    """
+    endpoint_kind = EndpointKind.C108
+
+    if block_keys is None:
+        # 보통 ("리포트",) 같은 튜플
+        block_keys = BLOCK_KEYS_BY_ENDPOINT[endpoint_kind]
+
+    blocks: dict[BlockKey, RecordsBlock] = {}
+    labels: dict[BlockKey, LabelsMap] = {}
+
+    for bk in block_keys:
+        rows = _as_records(parsed.get(str(bk)))
+        block = build_records_block_from_rows(
+            endpoint_kind=endpoint_kind,
+            block_key=bk,
+            rows=rows,
+        )
+
+        if not keep_empty_blocks and not block.rows:
+            continue
+
+        blocks[bk] = block
+        labels[bk] = {}  # c108은 labels 비우는 것이 정상
+
+    return NfsDoc(
+        code=code,
+        endpoint_kind=endpoint_kind,
+        blocks=blocks,
+        labels=labels,
+    )
+
+def build_kv_block_from_mapping(
+    *,
+    endpoint_kind: EndpointKind,
+    block_key: BlockKey,
+    data: Mapping[str, Any] | None,
+    keep_empty: bool = True,
+) -> KvBlock | None:
+    """
+    dict 형태 블록을 KvBlock으로 감싼다.
+    - c101 요약/시세/기업개요/펀더멘털/어닝서프라이즈 같은 "구조 dict"에 사용
+    """
+    if not data:
+        if not keep_empty:
+            return None
+        data = {}
+
+    return KvBlock(endpoint_kind=endpoint_kind, block_key=block_key, values=data)
+
+
+ParsedC101 = Mapping[str, Any]  # c101은 dict/list/dict(중첩) 섞여서 Any가 현실적
+
+
+def build_c101_doc_from_parsed(
+    *,
+    code: str,
+    parsed: ParsedC101,
+    block_keys: Iterable[BlockKey] | None = None,
+    keep_empty_blocks: bool = True,
+) -> NfsDoc:
+    """
+    c101 parser 결과(블록별 다양한 타입)를 NfsDoc으로 조립.
+    labels는 c101은 '비어도 정상' 규칙을 따르므로 항상 {}로 둔다.
+    """
+    endpoint_kind = EndpointKind.C101
+
+    if block_keys is None:
+        block_keys = BLOCK_KEYS_BY_ENDPOINT[endpoint_kind]
+
+    blocks: dict[BlockKey, BlockData] = {}
+    labels: dict[BlockKey, LabelsMap] = {}
+
+    for bk in block_keys:
+        v = parsed.get(str(bk))
+
+        # c101 규칙: labels는 비어도 정상, 있으면 넣는 게 아니라 "기본 비움" 추천
+        labels[bk] = {}
+
+        # list -> RecordsBlock
+        if isinstance(v, list):
+            # v: list[dict[str, Any]] 가정 (파서가 그렇게 만들고 있음)
+            rb = build_records_block_from_rows(
+                endpoint_kind=endpoint_kind,
+                block_key=bk,
+                rows=v,               # type: ignore[arg-type]  (rows 타입 맞추면 제거 가능)
+            )
+            if rb is not None:
+                blocks[bk] = rb
+            continue
+
+        # dict(중첩 포함) -> KvBlock
+        if isinstance(v, dict):
+            kb = build_kv_block_from_mapping(
+                endpoint_kind=endpoint_kind,
+                block_key=bk,
+                data=v,
+                keep_empty=keep_empty_blocks,
+            )
+            if kb is not None:
+                blocks[bk] = kb
+            continue
+
+        # None/기타 -> empty policy
+        if keep_empty_blocks:
+            kb = build_kv_block_from_mapping(
+                endpoint_kind=endpoint_kind,
+                block_key=bk,
+                data={},
+                keep_empty=True,
+            )
+            blocks[bk] = kb
+
+    return NfsDoc(code=code, endpoint_kind=endpoint_kind, blocks=blocks, labels=labels)