scraper2-hj3415 2.4.1__py3-none-any.whl → 2.6.0__py3-none-any.whl

scraper2_hj3415/app/adapters/site/wisereport_playwright.py

@@ -0,0 +1,168 @@
+# scraper2_hj3415/app/adapters/site/wisereport_playwright.py
+from __future__ import annotations
+
+from scraper2_hj3415.app.ports.browser.browser_port import BrowserPort
+from logging_hj3415 import logger
+
+
+class WiseReportPlaywright:
+    def __init__(self, browser: BrowserPort):
+        self.browser = browser
+
+    async def ensure_yearly_consensus_open_in_table_nth(
+        self,
+        *,
+        table_selector: str,  # 예: TABLE_XPATH ("xpath=//div[@id='wrapper']//div//table")
+        table_index: int,  # 예: TABLE_INDEX (2)
+        after_click_sleep_ms: int = 150,
+        max_rounds: int = 6,
+        wait_timeout_sec: float = 12.0,
+    ) -> bool:
+        """
+        목표: 연간 컨센서스 컬럼이 '반드시 펼쳐진 상태'가 되게 한다.
+        전략:
+          - TABLE_NTH 스코프 안에서
+          - btn_moreY 또는 btn_moreQQ 이면서
+          - '연간컨센서스보기' 텍스트를 가진 a 토글들 중
+          - computedStyle(display) != 'none' 인 것들을 전부 클릭
+          - 클릭마다 테이블 텍스트 변경을 기다림
+        """
+
+        table_scoped = f"{table_selector} >> nth={table_index}"
+
+        # table 내부의 토글(a)만 잡기 (btn_moreY / btn_moreQQ 둘 다)
+        VIEW_ALL = (
+            f"{table_scoped} >> xpath=.//a["
+            "("
+            "contains(@class,'btn_moreY') or contains(@class,'btn_moreQQ')"
+            ")"
+            " and .//span[contains(normalize-space(.),'연간컨센서스보기')]"
+            "]"
+        )
+
+        CLOSE_ALL = (
+            f"{table_scoped} >> xpath=.//a["
+            "("
+            "contains(@class,'btn_moreY') or contains(@class,'btn_moreQQ')"
+            ")"
+            " and .//span[contains(normalize-space(.),'연간컨센서스닫기')]"
+            "]"
+        )
+
+        # 테이블 텍스트 변화 감지용 “prev_text”
+        prev_text = await self.browser.wait_table_text_changed(
+            table_selector,
+            index=table_index,
+            prev_text=None,
+            timeout_sec=wait_timeout_sec,
+            min_lines=10,
+        )
+
+        logger.debug("ensure_yearly_consensus_open_in_table_nth: start")
+
+        # round를 두는 이유:
+        # - 보기 토글이 여러 개고, 클릭할 때 DOM이 재배치될 수 있음
+        # - 1번에 다 못 누르면 다음 라운드에서 다시 스캔
+        for round_no in range(1, max_rounds + 1):
+            view_cnt = await self.browser.count(VIEW_ALL)
+            close_cnt = await self.browser.count(CLOSE_ALL)
+            logger.debug(
+                f"round={round_no} toggle exists: view={view_cnt}, close={close_cnt}"
+            )
+
+            # "보기" 토글이 아예 없으면 -> 이미 다 펼쳐져 있거나(닫기만 존재),
+            # 혹은 페이지 구조가 달라서 못 찾는 것. 여기서는 '성공'으로 간주.
+            if view_cnt == 0:
+                logger.debug("no VIEW toggles found in-table -> treat as OPEN")
+                return True
+
+            clicked_any = False
+
+            # i를 0..view_cnt-1로 돌면서 display != none 인 것만 클릭
+            # (중간에 DOM 바뀌면 count/순서가 바뀔 수 있으니, 실패해도 계속)
+            for i in range(view_cnt):
+                try:
+                    # 혹시 DOM이 바뀌어 index가 사라졌으면 skip
+                    if not await self.browser.is_attached(VIEW_ALL, index=i):
+                        continue
+
+                    disp = await self.browser.computed_style(
+                        VIEW_ALL, index=i, prop="display"
+                    )
+                    if disp.strip().lower() == "none":
+                        continue
+
+                    # 화면 밖이면 클릭 실패할 수 있으니 스크롤
+                    await self.browser.scroll_into_view(VIEW_ALL, index=i)
+
+                    # trial(실패해도 진행)
+                    _ = await self.browser.try_click(
+                        VIEW_ALL, index=i, timeout_ms=1500, force=False
+                    )
+
+                    # 실제 클릭
+                    try:
+                        await self.browser.click(
+                            VIEW_ALL, index=i, timeout_ms=4000, force=False
+                        )
+                    except Exception:
+                        await self.browser.click(
+                            VIEW_ALL, index=i, timeout_ms=4000, force=True
+                        )
+
+                    await self.browser.sleep_ms(after_click_sleep_ms)
+
+                    # 클릭 후 테이블 텍스트 변경 대기
+                    prev_text = await self.browser.wait_table_text_changed(
+                        table_selector,
+                        index=table_index,
+                        prev_text=prev_text,
+                        timeout_sec=wait_timeout_sec,
+                        min_lines=10,
+                    )
+
+                    clicked_any = True
+                    logger.debug(f"clicked VIEW toggle: idx={i}, display={disp}")
+
+                except Exception as e:
+                    logger.debug(
+                        f"click VIEW toggle failed: idx={i}, err={type(e).__name__}: {e}"
+                    )
+                    continue
+
+            # 이번 라운드에서 클릭을 하나도 못 했으면:
+            # - 모든 VIEW가 display:none 이었거나
+            # - 클릭이 막혔거나
+            # => VIEW(display!=none)가 남아있는지 다시 검사
+            if not clicked_any:
+                remain = await self.browser.count(VIEW_ALL)
+                logger.debug(f"no clicks in round; remain VIEW count={remain}")
+                # VIEW는 있는데 전부 display:none 이면 사실상 '열림' 상태로 볼 수 있음
+                # (닫기만 보이는 케이스)
+                # 여기서는 “성공” 처리
+                return True
+
+            # 다음 라운드에서 다시 스캔해서 VIEW(display!=none)가 남아있으면 또 클릭
+            # (다 눌렀으면 결국 클릭할 게 없어짐)
+
+        # 라운드 다 돌았는데도 여기까지 왔다면,
+        # “보기 토글이 계속 display!=none으로 남는다” = 열리지 않는 구조/권한/오버레이 등
+        logger.warning("ensure_yearly_consensus_open_in_table_nth: exceeded max_rounds")
+        return False
+
+    async def click_steps(
+        self,
+        steps: list[tuple[str, str]],
+        *,
+        jitter_sec: float = 0.6,
+    ) -> None:
+        """
+        현재 페이지에서 탭/라디오/검색 버튼 클릭만 수행.
+        """
+        for _name, selector in steps:
+            await self.browser.wait_attached(selector)
+            logger.info(f"click step: {_name}")
+            await self.browser.click(selector)
+            # 서버/클라이언트 부담 줄이기: 작은 지터
+            wait = int((0.5 + (jitter_sec * 0.5))*1000)
+            await self.browser.sleep_ms(wait)

scraper2_hj3415/app/composition.py

@@ -0,0 +1,225 @@
+# scraper2_hj3415/app/composition.py
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+from pymongo.asynchronous.database import AsyncDatabase
+
+from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2_hj3415.app.adapters.out.playwright.browser_factory import (
+    PlaywrightBrowserFactory,
+)
+
+from scraper2_hj3415.app.services.fetch.fetch_c101 import FetchC101
+from scraper2_hj3415.app.services.fetch.fetch_c103 import FetchC103
+from scraper2_hj3415.app.services.fetch.fetch_c104 import FetchC104
+from scraper2_hj3415.app.services.fetch.fetch_c106 import FetchC106
+from scraper2_hj3415.app.services.fetch.fetch_c108 import FetchC108
+
+from scraper2_hj3415.app.usecases.ingest.ingest_c101 import IngestC101
+from scraper2_hj3415.app.usecases.ingest.ingest_c103 import IngestC103
+from scraper2_hj3415.app.usecases.ingest.ingest_c104 import IngestC104
+from scraper2_hj3415.app.usecases.ingest.ingest_c106 import IngestC106
+from scraper2_hj3415.app.usecases.ingest.ingest_c108 import IngestC108
+
+
+from scraper2_hj3415.app.ports.sinks.nfs_sink_port import NfsSinkPort
+from contracts_hj3415.nfs.c101_dto import C101DTO
+from contracts_hj3415.nfs.c103_dto import C103DTO
+from contracts_hj3415.nfs.c104_dto import C104DTO
+from contracts_hj3415.nfs.c106_dto import C106DTO
+from contracts_hj3415.nfs.c108_dto import C108DTO
+
+from scraper2_hj3415.app.adapters.out.sinks.mongo_sink import MongoSink
+from scraper2_hj3415.app.adapters.out.sinks.memory_sink import MemorySink
+
+from scraper2_hj3415.app.adapters.out.sinks.store import InMemoryStore
+
+from db2_hj3415.mongo import Mongo
+
+from scraper2_hj3415.app.domain.types import Sink
+
+
+def _env_bool(key: str, default: bool) -> bool:
+    v = os.getenv(key)
+    return (
+        default if v is None else v.strip().lower() in {"1", "true", "yes", "y", "on"}
+    )
+
+
+def _env_int(key: str, default: int) -> int:
+    v = os.getenv(key)
+    if v is None:
+        return default
+    try:
+        return int(v)
+    except ValueError:
+        return default
+
+
+def build_browser_factory() -> BrowserFactoryPort:
+    return PlaywrightBrowserFactory(
+        headless=_env_bool("SCRAPER_HEADLESS", True),
+        timeout_ms=_env_int("SCRAPER_TIMEOUT_MS", 20_000),
+        max_concurrency=_env_int("SCRAPER_MAX_CONCURRENCY", 2),
+    )
+
+
+# -------------------------
+# Bundles
+# -------------------------
+
+
+@dataclass(frozen=True)
+class FetchUsecases:
+    c101: FetchC101
+    c103: FetchC103
+    c104: FetchC104
+    c106: FetchC106
+    c108: FetchC108
+
+
+@dataclass(frozen=True)
+class Sinks:
+    c101: NfsSinkPort[C101DTO]
+    c103: NfsSinkPort[C103DTO]
+    c104: NfsSinkPort[C104DTO]
+    c106: NfsSinkPort[C106DTO]
+    c108: NfsSinkPort[C108DTO]
+
+
+@dataclass(frozen=True)
+class IngestUsecases:
+    c101: IngestC101
+    c103: IngestC103
+    c104: IngestC104
+    c106: IngestC106
+    c108: IngestC108
+
+
+@dataclass(frozen=True)
+class Usecases:
+    fetch: FetchUsecases
+    ingest: IngestUsecases
+    sinks: Sinks
+    store: InMemoryStore | None = None  # ✅ memory일 때만
+    mongo: Mongo | None = None  # ✅ mongo일 때만
+    db: AsyncDatabase | None = None  # ✅ mongo일 때만
+    browser_factory: Optional[BrowserFactoryPort] = None
+
+    async def aclose(self) -> None:
+        if self.browser_factory is not None:
+            await self.browser_factory.aclose()
+
+        if self.mongo is not None:
+            await self.mongo.close()
+
+
+# -------------------------
+# builders
+# -------------------------
+
+
+def build_fetch_usecases(*, factory: BrowserFactoryPort) -> FetchUsecases:
+    return FetchUsecases(
+        c101=FetchC101(factory=factory),
+        c103=FetchC103(factory=factory),
+        c104=FetchC104(factory=factory),
+        c106=FetchC106(factory=factory),
+        c108=FetchC108(factory=factory),
+    )
+
+
+@dataclass(frozen=True)
+class MemoryBundle:
+    store: InMemoryStore
+    sinks: Sinks
+
+
+def build_memory_bundle() -> MemoryBundle:
+    store = InMemoryStore()
+    c101_sink: NfsSinkPort[C101DTO] = MemorySink(store)
+    c103_sink: NfsSinkPort[C103DTO] = MemorySink(store)
+    c104_sink: NfsSinkPort[C104DTO] = MemorySink(store)
+    c106_sink: NfsSinkPort[C106DTO] = MemorySink(store)
+    c108_sink: NfsSinkPort[C108DTO] = MemorySink(store)
+    sinks = Sinks(
+        c101=c101_sink,
+        c103=c103_sink,
+        c104=c104_sink,
+        c106=c106_sink,
+        c108=c108_sink,
+    )
+    return MemoryBundle(store=store, sinks=sinks)
+
+
+# ---- mongo bundle ----
+
+
+@dataclass(frozen=True)
+class MongoBundle:
+    mongo: Mongo
+    db: AsyncDatabase
+    sinks: Sinks
+
+
+def build_mongo_bundle() -> MongoBundle:
+    mongo = Mongo()  # settings는 db2가 env로 읽음 (DB2_MONGO_URI 등)
+    db = mongo.get_db()
+    c101_sink: NfsSinkPort[C101DTO] = MongoSink(db)
+    c103_sink: NfsSinkPort[C103DTO] = MongoSink(db)
+    c104_sink: NfsSinkPort[C104DTO] = MongoSink(db)
+    c106_sink: NfsSinkPort[C106DTO] = MongoSink(db)
+    c108_sink: NfsSinkPort[C108DTO] = MongoSink(db)
+    sinks = Sinks(
+        c101=c101_sink,
+        c103=c103_sink,
+        c104=c104_sink,
+        c106=c106_sink,
+        c108=c108_sink,
+    )
+    return MongoBundle(mongo=mongo, db=db, sinks=sinks)
+
+
+def build_ingest_usecases(*, fetch: FetchUsecases, sinks: Sinks) -> IngestUsecases:
+    return IngestUsecases(
+        c101=IngestC101(fetch=fetch.c101, sink=sinks.c101),
+        c103=IngestC103(fetch=fetch.c103, sink=sinks.c103),
+        c104=IngestC104(fetch=fetch.c104, sink=sinks.c104),
+        c106=IngestC106(fetch=fetch.c106, sink=sinks.c106),
+        c108=IngestC108(fetch=fetch.c108, sink=sinks.c108),
+    )
+
+
+def build_usecases(
+    *, factory: BrowserFactoryPort | None = None, sink: Sink = "memory"
+) -> Usecases:
+    factory = factory or build_browser_factory()
+    fetch = build_fetch_usecases(factory=factory)
+
+    if sink == "memory":
+        bundle = build_memory_bundle()
+        ingest = build_ingest_usecases(fetch=fetch, sinks=bundle.sinks)
+        return Usecases(
+            fetch=fetch,
+            ingest=ingest,
+            sinks=bundle.sinks,
+            store=bundle.store,
+            browser_factory=factory,
+        )
+
+    if sink == "mongo":
+        bundle = build_mongo_bundle()
+        ingest = build_ingest_usecases(fetch=fetch, sinks=bundle.sinks)
+        return Usecases(
+            fetch=fetch,
+            ingest=ingest,
+            sinks=bundle.sinks,
+            mongo=bundle.mongo,
+            db=bundle.db,
+            browser_factory=factory,
+        )
+
+    raise ValueError(f"Unknown sink_kind: {sink}")

scraper2_hj3415/app/domain/blocks.py

@@ -0,0 +1,61 @@
+# scraper2_hj3415/app/domain/blocks.py
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Mapping, Sequence
+from scraper2_hj3415.app.domain.constants import BLOCK_KEYS_BY_ENDPOINT
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.types import BlockKey, MetricKey, Record
+from scraper2_hj3415.app.domain.series import MetricSeries
+
+
+def _validate_block_key(endpoint_kind: EndpointKind, block_key: str) -> None:
+    allowed = BLOCK_KEYS_BY_ENDPOINT.get(endpoint_kind)
+    if allowed is not None and block_key not in allowed:
+        raise ValueError(f"Invalid block key for {endpoint_kind}: {block_key!r}")
+
+
+@dataclass(frozen=True)
+class MetricsBlock:
+    endpoint_kind: EndpointKind
+    block_key: BlockKey
+    metrics: Mapping[MetricKey, MetricSeries]
+
+    def __post_init__(self) -> None:
+        _validate_block_key(self.endpoint_kind, self.block_key)
+
+        # 컨테이너 키와 엔티티 키 불일치 방지(선택)
+        for k, m in self.metrics.items():
+            if m.key != k:
+                raise ValueError(
+                    f"Metric key mismatch: map key={k!r} != series key={m.key!r}"
+                )
+
+    def get(self, key: MetricKey) -> MetricSeries | None:
+        m = self.metrics.get(key)
+        if m and m.key != key:
+            raise ValueError("Metric key mismatch")
+        return m
+
+
+# 다양한 블록형태 구성 추후 수정필요
+
+
+@dataclass(frozen=True)
+class RecordsBlock:
+    endpoint_kind: EndpointKind
+    block_key: BlockKey
+    rows: Sequence[Record]
+
+    def __post_init__(self) -> None:
+        _validate_block_key(self.endpoint_kind, self.block_key)
+
+
+@dataclass(frozen=True)
+class KvBlock:
+    endpoint_kind: EndpointKind
+    block_key: BlockKey
+    values: Mapping[str, Any]
+
+
+BlockData = MetricsBlock | RecordsBlock | KvBlock

scraper2_hj3415/app/domain/constants.py

@@ -0,0 +1,33 @@
+# scraper2_hj3415/app/domain/constants.py
+from __future__ import annotations
+
+from typing import Mapping
+
+from contracts_hj3415.nfs.types import BlockKeys
+from contracts_hj3415.nfs.constants import C101_BLOCK_KEYS, C103_BLOCK_KEYS, C104_BLOCK_KEYS, C106_BLOCK_KEYS, C108_BLOCK_KEYS
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+
+
+BLOCK_KEYS_BY_ENDPOINT: Mapping[EndpointKind, tuple[str, ...]] = {
+    EndpointKind.C101: C101_BLOCK_KEYS,
+    EndpointKind.C103: C103_BLOCK_KEYS,
+    EndpointKind.C104: C104_BLOCK_KEYS,
+    EndpointKind.C106: C106_BLOCK_KEYS,
+    EndpointKind.C108: C108_BLOCK_KEYS,
+}
+
+
+def get_block_keys(endpoint: EndpointKind) -> tuple[str, ...]:
+    """
+    엔드포인트의 "공식" 블록 키 목록.
+    - 도메인 레이어에 두되, selector/table index 같은 구현 디테일은 넣지 않는다.
+    """
+    return BLOCK_KEYS_BY_ENDPOINT.get(endpoint, ())
+
+
+def is_known_block(endpoint: EndpointKind, key: BlockKeys) -> bool:
+    """
+    블록 키가 해당 endpoint의 공식 목록에 포함되는지 여부.
+    (검증/필터링/동적 payload merge 등에 사용)
+    """
+    return key in BLOCK_KEYS_BY_ENDPOINT.get(endpoint, ())

scraper2_hj3415/app/domain/doc.py

@@ -0,0 +1,16 @@
+# scraper2_hj3415/app/domain/doc.py
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Mapping
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.types import BlockKey, LabelsMap
+from scraper2_hj3415.app.domain.blocks import BlockData
+
+
+@dataclass(frozen=True)
+class NfsDoc:
+    code: str
+    endpoint_kind: EndpointKind
+    blocks: Mapping[BlockKey, BlockData]
+    labels: Mapping[BlockKey, LabelsMap]

scraper2_hj3415/app/domain/endpoint.py

@@ -0,0 +1,11 @@
+# scraper2_hj3415/app/domain/endpoint.py
+from __future__ import annotations
+
+from enum import StrEnum
+
+class EndpointKind(StrEnum):
+    C101 = "c101"
+    C103 = "c103"
+    C104 = "c104"
+    C106 = "c106"
+    C108 = "c108"

scraper2_hj3415/app/domain/series.py

@@ -0,0 +1,11 @@
+# scraper2_hj3415/app/domain/series.py
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Mapping
+from scraper2_hj3415.app.domain.types import MetricKey, Period, Num
+
+@dataclass(frozen=True)
+class MetricSeries:
+    key: MetricKey
+    values: Mapping[Period, Num]

scraper2_hj3415/app/domain/types.py

@@ -0,0 +1,19 @@
+# scraper2_hj3415/app/domain/types.py
+from __future__ import annotations
+
+from typing import Mapping, Any, Sequence, TypeAlias, Literal
+
+BlockKey = str
+MetricKey = str
+Period = str
+Num = float | int | None
+
+Record: TypeAlias = Mapping[str, Any]
+Records: TypeAlias = Sequence[Record]
+RawLabel = str
+LabelsMap = dict[MetricKey, RawLabel]
+
+Sink = Literal["memory", "mongo"]
+
+
+

scraper2_hj3415/app/parsing/_normalize/label.py

@@ -0,0 +1,92 @@
+# scraper2_hj3415/app/parsing/_normalize/label.py
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from common_hj3415.utils import clean_text
+from scraper2_hj3415.app.parsing._normalize.text import normalize_text
+
+# -----------------------------
+# 일반적인 라벨 정규화
+# -----------------------------
+
+UI_LABEL_NOISE = (
+    "펼치기",
+    "접기",
+    "더보기",
+)
+
+
+def sanitize_label(x: Any) -> str:
+    """
+    raw label(항목_raw 포함)에서:
+      - '펼치기' 같은 UI 텍스트 제거
+      - 과도한 공백 정리
+      - 양끝 공백 제거
+    """
+    s = normalize_text(x)
+
+    # UI 노이즈 단어 제거
+    for w in UI_LABEL_NOISE:
+        s = s.replace(w, " ")
+
+    return clean_text(s)
+
+
+# -----------------------------
+# metric 라벨 정규화
+# -----------------------------
+_BRACKET_PATTERN = re.compile(r"\[[^\]]*\]")  # [구K-IFRS] 등
+_EXTRA_WORDS_PATTERN = re.compile(r"(펼치기|연간컨센서스보기|연간컨센서스닫기)")
+_ALL_PAREN_PATTERN = re.compile(r"\([^)]*\)")  # ★ 모든 괄호 제거
+
+
+def normalize_label_base(text: str | None) -> str:
+    s = sanitize_label(text)
+    s = _EXTRA_WORDS_PATTERN.sub("", s)
+    s = _BRACKET_PATTERN.sub("", s)
+    s = _ALL_PAREN_PATTERN.sub("", s)
+    s = s.replace("*", "")
+    return clean_text(s)
+
+
+def normalize_metric_label(text: str | None) -> str:
+    # "보유 지분 (%)" → "보유 지분"   (공백 유지)
+    return normalize_label_base(text)
+
+
+def normalize_key_label(text: str | None) -> str:
+    # "보유 지분 (%)" → "보유지분"
+    s = normalize_label_base(text)
+    return s.replace(" ", "").replace("\xa0", "").replace("%", "").strip()
+
+
+# -----------------------------
+# 컬럼명 정규화
+# -----------------------------
+_COL_PAREN_PATTERN = re.compile(r"\((IFRS[^)]*|E|YoY|QoQ)[^)]*\)")
+_COL_EXTRA_WORDS = re.compile(r"(연간컨센서스보기|연간컨센서스닫기)")
+_COL_DOTNUM = re.compile(r"\.\d+$")  # pandas 중복 컬럼 suffix 제거용 (.1, .2 ...)
+
+
+def normalize_col_label(col: str | None) -> str:
+    """
+    컬럼명 정규화
+    예)
+      "2024/12 (IFRS연결)  연간컨센서스보기" -> "2024/12"
+      "2025/12(E) (IFRS연결)  연간컨센서스닫기" -> "2025/12"
+      "전년대비 (YoY)" -> "전년대비"
+      "전년대비 (YoY).1" -> "전년대비"  (중복은 후처리에서 _2/_3로 자동 분리)
+    """
+    s = normalize_text(col)
+    # 1) pandas가 붙인 .1 같은 suffix 제거 (정규화 충돌은 후단에서 처리)
+    s = _COL_DOTNUM.sub("", s)
+
+    # 2) 컨센서스 문구 제거
+    s = _COL_EXTRA_WORDS.sub("", s)
+
+    # 3) 괄호 주석 제거: (IFRS...), (E), (YoY), (QoQ)
+    s = _COL_PAREN_PATTERN.sub("", s)
+
+    return clean_text(s)

scraper2_hj3415/app/parsing/_normalize/table.py

@@ -0,0 +1,53 @@
+# scraper2_hj3415/app/parsing/_normalize/table.py
+from __future__ import annotations
+
+from collections import Counter
+
+import numpy as np
+import pandas as pd
+from .label import normalize_col_label, normalize_metric_label
+
+
+def _dedupe_columns(cols: list[str]) -> list[str]:
+    """
+    정규화 후 중복 컬럼명이 생기면 자동으로 _2, _3 ... 붙여서 유니크하게 만든다.
+    예) ["전년대비", "전년대비"] -> ["전년대비", "전년대비_2"]
+    """
+    seen: Counter[str] = Counter()
+    out: list[str] = []
+    for c in cols:
+        c = c or ""
+        seen[c] += 1
+        if seen[c] == 1:
+            out.append(c)
+        else:
+            out.append(f"{c}_{seen[c]}")
+    return out
+
+
+# -----------------------------
+# 3) DataFrame 전체 정규화 + records 변환
+# -----------------------------
+def normalize_metrics_df(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    - 컬럼명 전체 정규화
+    - '항목' 값 정규화
+    - NaN -> None
+    - 중복 컬럼명 자동 분리(_2/_3)
+    """
+    if df is None or df.empty:
+        return df
+
+    df = df.copy()
+
+    # 컬럼명 정규화 + 중복 방지
+    norm_cols = [normalize_col_label(c) for c in df.columns.astype(str).tolist()]
+    df.columns = _dedupe_columns(norm_cols)
+
+    # 항목 값 정규화
+    if "항목" in df.columns:
+        df["항목"] = df["항목"].map(normalize_metric_label)
+
+    # NaN -> None
+    df = df.replace({np.nan: None})
+    return df