scraper2-hj3415 2.6.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scraper2_hj3415/app/ports/browser/capabilities/__init__.py

@@ -0,0 +1,15 @@
+from .navigation import BrowserNavigationPort
+from .wait import BrowserWaitPort
+from .interaction import BrowserInteractionPort
+from .text import BrowserTextPort
+from .scope import BrowserScopePort
+from .table import BrowserTablePort
+
+__all__ = [
+    "BrowserNavigationPort",
+    "BrowserWaitPort",
+    "BrowserInteractionPort",
+    "BrowserTextPort",
+    "BrowserScopePort",
+    "BrowserTablePort",
+]

scraper2_hj3415/app/ports/browser/capabilities/interaction.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class BrowserInteractionPort(Protocol):
+    """클릭/스크롤 등 상호작용"""
+
+    async def click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 4_000,
+        force: bool = False,
+    ) -> None: ...
+
+    async def try_click(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        timeout_ms: int = 1_500,
+        force: bool = False,
+    ) -> bool: ...
+
+    async def scroll_into_view(self, selector: str, *, index: int = 0) -> None: ...

scraper2_hj3415/app/ports/browser/capabilities/navigation.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class BrowserNavigationPort(Protocol):
+    """페이지 이동/기본 정보"""
+
+    async def title(self) -> str: ...
+    async def current_url(self) -> str: ...
+
+    async def goto_and_wait_for_stable(
+        self,
+        url: str,
+        timeout_ms: int = 10_000,
+    ) -> None: ...
+
+    async def reload(self, *, timeout_ms: int = 10_000) -> None: ...

scraper2_hj3415/app/ports/browser/capabilities/scope.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+
+class BrowserScopePort(Protocol):
+    """scope/nth 컨텍스트 기반 조회 (현재는 기존 API 유지)"""
+
+    async def is_attached(self, selector: str, *, index: int = 0) -> bool: ...
+
+    async def computed_style(
+        self,
+        selector: str,
+        *,
+        index: int = 0,
+        prop: str,
+    ) -> str: ...
+
+    async def count_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+    ) -> int: ...
+
+    async def eval_in_nth_first(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        expression: str,
+    ) -> Any: ...
+
+    async def inner_text_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        """
+        scope_selector의 nth(scope_index) 요소 안에서
+        inner_selector의 nth(inner_index) 요소의 innerText를 반환.
+        (렌더링 기준 텍스트: 줄바꿈/스타일 영향 반영)
+        """
+        ...
+
+    async def text_content_in_nth(
+        self,
+        scope_selector: str,
+        *,
+        scope_index: int,
+        inner_selector: str,
+        inner_index: int = 0,
+        timeout_ms: int = 10_000,
+    ) -> str:
+        """
+        scope_selector의 nth(scope_index) 요소 안에서
+        inner_selector의 nth(inner_index) 요소의 textContent를 반환.
+        (DOM 기준 텍스트: 숨김 텍스트도 포함될 수 있음)
+        """
+        ...

scraper2_hj3415/app/ports/browser/capabilities/table.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+
+class BrowserTablePort(Protocol):
+    """테이블 파싱/헤더 추출"""
+
+    async def table_records(
+        self,
+        table_selector: str,
+        *,
+        header: int | list[int] | None = 0,
+    ) -> list[dict[str, Any]]: ...
+
+    async def table_header_texts_nth(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+    ) -> list[str]: ...
+
+    async def table_header_periods_mm_nth(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+    ) -> list[str]: ...

scraper2_hj3415/app/ports/browser/capabilities/text.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class BrowserTextPort(Protocol):
+    """DOM 조회/텍스트 추출"""
+
+    async def count(self, selector: str) -> int: ...
+
+    async def text_content_first(self, selector: str) -> str: ...
+    async def all_texts(self, selector: str) -> list[str]: ...
+    async def get_text_by_text(self, needle: str) -> str: ...
+
+    async def inner_text(self, selector: str) -> str: ...
+    async def outer_html_nth(self, selector: str, index: int) -> str: ...

scraper2_hj3415/app/ports/browser/capabilities/wait.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Protocol
+
+
+class BrowserWaitPort(Protocol):
+    """대기/동기화 유틸"""
+
+    async def sleep_ms(self, ms: int) -> None:
+        """
+        ms에 1000을 넣으면 1초 쉼.
+        구현체가 없을 때를 대비해 기본 구현 제공.
+        """
+        await asyncio.sleep(ms / 1000)
+
+    async def wait_attached(
+        self,
+        selector: str,
+        *,
+        timeout_ms: int = 10_000,
+    ) -> None: ...
+
+    async def wait_visible(
+        self,
+        selector: str,
+        *,
+        timeout_ms: int = 10_000,
+    ) -> None: ...
+
+    async def wait_table_nth_ready(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+        min_rows: int = 1,
+        timeout_ms: int = 20_000,
+        poll_ms: int = 200,
+    ) -> None: ...
+
+    async def wait_table_text_changed(
+        self,
+        table_selector: str,
+        *,
+        index: int,
+        prev_text: str | None,
+        min_rows: int = 1,
+        min_lines: int = 50,
+        timeout_sec: float = 12.0,
+        poll_sec: float = 0.2,
+    ) -> str: ...

scraper2_hj3415/app/ports/sinks/nfs_sink_port.py

@@ -2,19 +2,19 @@
 from __future__ import annotations
 
 from typing import Protocol, Iterable, TypeVar
-from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.types import Endpoint
 from contracts_hj3415.nfs.nfs_dto import NfsDTO
 
 TDto = TypeVar("TDto", bound=NfsDTO)
 
 class NfsSinkPort(Protocol[TDto]):
     async def write(
-        self, dto: TDto, *, endpoint: Endpoints
+        self, dto: TDto, *, endpoint: Endpoint
     ) -> None: ...
 
     async def write_many(
         self,
         dtos: Iterable[TDto],
         *,
-        endpoint: Endpoints
+        endpoint: Endpoint
     ) -> None: ...

scraper2_hj3415/app/ports/site/wisereport_port.py

@@ -3,18 +3,28 @@ from __future__ import annotations
 from typing import Protocol
 
 class WiseReportPort(Protocol):
-    async def ensure_yearly_consensus_open_in_table_nth(
+    async def set_view_c103(
         self,
         *,
-        table_selector: str,   # 예: TABLE_XPATH ("xpath=//div[@id='wrapper']//div//table")
-        table_index: int,      # 예: TABLE_INDEX (2)
-        after_click_sleep_ms: int = 150,
-        max_rounds: int = 6,
-        wait_timeout_sec: float = 12.0,
-    ) -> bool: ...
-    async def click_steps(
+        key: str,
+        steps: list[tuple[str, str]],
+        table_selector: str,
+        table_index: int,
+        max_attempts: int = 5,
+        stabilize_timeout_sec: float = 10.0,
+    ) -> None: ...
+
+    async def set_view_c104(
             self,
-            steps: list[tuple[str, str]],
             *,
-            jitter_sec: float = 0.6,
+            key: str,
+            steps: list[tuple[str, str]],
+            table_selector: str,
+            table_index: int,
+            prev_text_by_idx: dict[int, str | None],
+            max_attempts: int = 5,
+            stabilize_timeout_sec: float = 10.0,
+            min_rows: int = 5,
+            min_lines: int = 30,
+            open_consensus: bool = True,
     ) -> None: ...

scraper2_hj3415/app/services/fetch/fetch_c103.py

@@ -8,9 +8,9 @@ from typing import Iterable, Any
 from logging_hj3415 import logger
 from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
 from scraper2_hj3415.app.ports.site.wisereport_port import WiseReportPort
-
 from scraper2_hj3415.app.adapters.site.wisereport_playwright import WiseReportPlaywright
-from scraper2_hj3415.app.parsing.c103_parser import parse_c103_current_table
+
+from scraper2_hj3415.app.parsing.c103_parser import parse_c103_current_table, TABLE_XPATH
 from scraper2_hj3415.app.services.nfs_doc_builders import build_metrics_doc_from_parsed
 
 from scraper2_hj3415.app.domain.endpoint import EndpointKind
@@ -50,6 +50,8 @@ BTN_SETS: dict[str, list[tuple[str, str]]] = {
     ],
 }
 
+TABLE_INDEX = 2
+
 
 class FetchC103:
     def __init__(self, factory: BrowserFactoryPort):
@@ -69,40 +71,24 @@ class FetchC103:
                 await asyncio.sleep(sleep_sec + random.uniform(0, 1.0))
 
             parsed: dict[str, list[dict[str, Any]]] = {}
-            prev_text: str | None = None
-
-            # 최초 기준 텍스트 확보(없어도 동작하게)
-            prev_text = await browser.wait_table_text_changed(
-                "xpath=//div[@id='wrapper']//div//table",
-                index=2,
-                prev_text=None,
-                min_rows=5,
-                min_lines=50,
-                timeout_sec=10.0,
-            )
 
             for key, steps in BTN_SETS.items():
-                # ✅ 상태 전환 (행동)
-                await wr.click_steps(steps, jitter_sec=0.6)  # 포트/어댑터로 이동 권장
-                await wr.ensure_yearly_consensus_open_in_table_nth(
-                    table_selector="xpath=//div[@id='wrapper']//div//table",
-                    table_index=2,
-                )
-
-                # ✅ 데이터 변경 대기 (행동)
-                prev_text = await browser.wait_table_text_changed(
-                    "xpath=//div[@id='wrapper']//div//table",
-                    index=2,
-                    prev_text=prev_text,
-                    min_rows=5,
-                    min_lines=50,
-                    timeout_sec=12.0,
-                )
-
-                # ✅ 파싱은 “현재 화면 테이블”만
                 try:
+                    # 1) 상태 확정 (분기/연간이 맞는지 헤더로 검증)
+                    await wr.set_view_c103(
+                        key=key,
+                        steps=steps,
+                        table_selector=TABLE_XPATH,
+                        table_index=TABLE_INDEX,
+                    )
+
+                    # 2) 파싱은 “현재 화면 테이블”만
                     parsed[key] = await parse_c103_current_table(browser)
-                except Exception:
+
+                except Exception as e:
+                    logger.warning(
+                        f"c103 view/parse failed: key={key} err={type(e).__name__}: {e}"
+                    )
                     parsed[key] = []
 
             block_keys = BLOCK_KEYS_BY_ENDPOINT[EndpointKind.C103]

scraper2_hj3415/app/services/fetch/fetch_c104.py

@@ -3,22 +3,19 @@ from __future__ import annotations
 
 import asyncio
 import random
-from typing import Iterable, Any
+from typing import Any, Iterable
 
 from logging_hj3415 import logger
 from scraper2_hj3415.app.ports.browser.browser_factory_port import BrowserFactoryPort
 from scraper2_hj3415.app.ports.site.wisereport_port import WiseReportPort
 from scraper2_hj3415.app.adapters.site.wisereport_playwright import WiseReportPlaywright
 
-from scraper2_hj3415.app.parsing.c104_parser import (
-    parse_c104_current_table,
-    TABLE_XPATH,
-)
+from scraper2_hj3415.app.parsing.c104_parser import parse_c104_current_table, TABLE_XPATH
 from scraper2_hj3415.app.services.nfs_doc_builders import build_metrics_doc_from_parsed
+
 from scraper2_hj3415.app.domain.endpoint import EndpointKind
-from scraper2_hj3415.app.domain.doc import NfsDoc
 from scraper2_hj3415.app.domain.blocks import BLOCK_KEYS_BY_ENDPOINT
-
+from scraper2_hj3415.app.domain.doc import NfsDoc
 
 BTN_SETS: dict[str, list[tuple[str, str]]] = {
     "수익성y": [
@@ -72,13 +69,9 @@ BTN_SETS: dict[str, list[tuple[str, str]]] = {
 }
 
 
-def _is_value_analysis(key: str) -> bool:
-    return key.startswith("가치분석")
-
-
 def _table_index_for_key(key: str) -> int:
-    # ✅ 네 주석대로 가치분석만 1, 나머지 0
-    return 1 if _is_value_analysis(key) else 0
+    # 가치분석은 별도 테이블(보통 index=1)
+    return 1 if key.startswith("가치분석") else 0
 
 
 class FetchC104:
@@ -100,53 +93,38 @@ class FetchC104:
 
             parsed: dict[str, list[dict[str, Any]]] = {}
 
-            # ✅ table index별로 prev_text를 따로 들고가야 안정적
+            # idx별 안정화 상태 추적
             prev_text_by_idx: dict[int, str | None] = {0: None, 1: None}
 
-            # ✅ 최초 baseline 확보(둘 다 시도)
-            for idx in (0, 1):
-                try:
-                    prev_text_by_idx[idx] = await browser.wait_table_text_changed(
-                        TABLE_XPATH,
-                        index=idx,
-                        prev_text=None,
-                        min_rows=5,
-                        min_lines=50,
-                        timeout_sec=10.0,
-                    )
-                except Exception:
-                    prev_text_by_idx[idx] = None
-
             for key, steps in BTN_SETS.items():
                 idx = _table_index_for_key(key)
 
-                # ✅ 상태 전환(행동)
-                await wr.click_steps(steps, jitter_sec=0.6)
-                await wr.ensure_yearly_consensus_open_in_table_nth(
-                    table_selector=TABLE_XPATH,
-                    table_index=idx,
-                )
-
-                # ✅ 데이터 변경 대기(행동) - idx별로 추적
-                prev_text_by_idx[idx] = await browser.wait_table_text_changed(
-                    TABLE_XPATH,
-                    index=idx,
-                    prev_text=prev_text_by_idx[idx],
-                    min_rows=5,
-                    min_lines=50,
-                    timeout_sec=12.0,
-                )
-
-                # ✅ 파싱은 “현재 화면의 idx 테이블 1개”만
                 try:
+                    # 1) 상태 확정 (분기/연간이 맞는지 헤더로 검증)
+                    await wr.set_view_c104(
+                        key=key,
+                        steps=steps,
+                        table_selector=TABLE_XPATH,
+                        table_index=idx,
+                        prev_text_by_idx=prev_text_by_idx,
+                        max_attempts=5,
+                        stabilize_timeout_sec=10.0,
+                        min_rows=5,
+                        min_lines=30,
+                        open_consensus=True,
+                    )
+
+                    # 2) 파싱은 “현재 화면의 idx 테이블 1개”만
                     parsed[key] = await parse_c104_current_table(
                         browser,
                         table_index=idx,
                     )
-                except Exception:
-                    parsed[key] = []
 
-            logger.debug(f"parsed:\n{parsed}")
+                except Exception as e:
+                    logger.warning(
+                        f"c104 view/parse failed: key={key} idx={idx} err={type(e).__name__}: {e}"
+                    )
+                    parsed[key] = []
 
             block_keys = BLOCK_KEYS_BY_ENDPOINT[EndpointKind.C104]
             if not parsed or all(not (parsed.get(str(bk)) or []) for bk in block_keys):
@@ -164,7 +142,6 @@ class FetchC104:
                 raw_label_key="항목_raw",
                 keep_empty_blocks=True,
             )
-            logger.debug(f"c104 doc: {doc}")
             return doc
 
     async def execute(self, code: str, *, sleep_sec: float = 2.0) -> NfsDoc | None:
@@ -180,4 +157,4 @@ class FetchC104:
             *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
             return_exceptions=False,
         )
-        return [r for r in results if r is not None]
+        return [r for r in results if r is not None]

scraper2_hj3415/app/services/nfs_doc_builders.py

@@ -7,9 +7,22 @@ from typing import Mapping, Iterable, Any
 from scraper2_hj3415.app.domain.endpoint import EndpointKind
 from scraper2_hj3415.app.domain.constants import BLOCK_KEYS_BY_ENDPOINT
 from scraper2_hj3415.app.domain.doc import NfsDoc
-from scraper2_hj3415.app.domain.blocks import MetricsBlock, RecordsBlock, KvBlock, BlockData
+from scraper2_hj3415.app.domain.blocks import (
+    MetricsBlock,
+    RecordsBlock,
+    KvBlock,
+    BlockData,
+)
 from scraper2_hj3415.app.domain.series import MetricSeries
-from scraper2_hj3415.app.domain.types import LabelsMap, MetricKey, Period, Num, BlockKey, Records, Record
+from scraper2_hj3415.app.domain.types import (
+    LabelsMap,
+    MetricKey,
+    Period,
+    Num,
+    BlockKey,
+    Records,
+    Record,
+)
 
 from common_hj3415.utils import nan_to_none
 
@@ -18,9 +31,7 @@ def is_all_none(row: dict[str, Any]) -> bool:
     return all(v is None for v in row.values())
 
 
-ParsedBlocks = Mapping[
-    str, Any
-]  # parser가 반환한 "block_key(str) -> rows(list[dict])"
+ParsedBlocks = Mapping[str, Any]  # parser가 반환한 "block_key(str) -> rows(list[dict])"
 
 
 def build_metrics_block_and_labels_from_rows(
@@ -155,7 +166,9 @@ def build_records_block_from_rows(
     - c108 같은 레코드성 블록(리포트 목록 등)에 사용
     """
     # RecordsBlock 쪽에서도 __post_init__로 block_key 검증이 수행된다는 전제(네가 정돈한 도메인)
-    return RecordsBlock(endpoint_kind=endpoint_kind, block_key=block_key, rows=list(rows))
+    return RecordsBlock(
+        endpoint_kind=endpoint_kind, block_key=block_key, rows=list(rows)
+    )
 
 
 def build_c108_doc_from_parsed(
@@ -206,6 +219,7 @@ def build_c108_doc_from_parsed(
         labels=labels,
     )
 
+
 def build_kv_block_from_mapping(
     *,
     endpoint_kind: EndpointKind,
@@ -259,7 +273,7 @@ def build_c101_doc_from_parsed(
             rb = build_records_block_from_rows(
                 endpoint_kind=endpoint_kind,
                 block_key=bk,
-                rows=v,               # type: ignore[arg-type]  (rows 타입 맞추면 제거 가능)
+                rows=v,  # type: ignore[arg-type]  (rows 타입 맞추면 제거 가능)
             )
             if rb is not None:
                 blocks[bk] = rb

scraper2_hj3415/app/usecases/ingest/ingest_c101.py

@@ -13,13 +13,13 @@ from scraper2_hj3415.app.domain.constants import get_block_keys
 from scraper2_hj3415.app.domain.doc import NfsDoc
 from scraper2_hj3415.app.domain.blocks import KvBlock, RecordsBlock, MetricsBlock
 
-from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.types import Endpoint
 from contracts_hj3415.nfs.c101_dto import C101DTO, C101Payload, C101Blocks
 
 from logging_hj3415 import logger
 
 endpoint_kind = EndpointKind.C101
-endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+endpoint: Endpoint = cast(Endpoint, endpoint_kind.value)
 
 
 def _unwrap_c101_block(block: Any) -> Any:

scraper2_hj3415/app/usecases/ingest/ingest_c103.py

@@ -13,7 +13,7 @@ from scraper2_hj3415.app.domain.constants import get_block_keys
 from scraper2_hj3415.app.domain.doc import NfsDoc
 from scraper2_hj3415.app.domain.blocks import MetricsBlock
 
-from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.types import Endpoint
 
 from contracts_hj3415.nfs.c103_dto import (
     C103DTO,
@@ -27,7 +27,7 @@ from logging_hj3415 import logger
 
 
 endpoint_kind = EndpointKind.C103
-endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+endpoint: Endpoint = cast(Endpoint, endpoint_kind.value)
 
 
 def _metricsblock_to_c103_metric_map(block: MetricsBlock) -> dict[str, C103ValuesMap]:

scraper2_hj3415/app/usecases/ingest/ingest_c104.py

@@ -13,7 +13,7 @@ from scraper2_hj3415.app.domain.constants import get_block_keys
 from scraper2_hj3415.app.domain.doc import NfsDoc
 from scraper2_hj3415.app.domain.blocks import MetricsBlock
 
-from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.types import Endpoint
 
 from contracts_hj3415.nfs.c104_dto import (
     C104DTO,
@@ -27,7 +27,7 @@ from logging_hj3415 import logger
 
 
 endpoint_kind = EndpointKind.C104
-endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+endpoint: Endpoint = cast(Endpoint, endpoint_kind.value)
 
 
 def _metricsblock_to_c104_metric_map(block: MetricsBlock) -> dict[str, C104ValuesMap]:

scraper2_hj3415/app/usecases/ingest/ingest_c106.py

@@ -13,7 +13,7 @@ from scraper2_hj3415.app.domain.constants import get_block_keys
 from scraper2_hj3415.app.domain.doc import NfsDoc
 from scraper2_hj3415.app.domain.blocks import MetricsBlock
 
-from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.types import Endpoint
 
 from contracts_hj3415.nfs.c106_dto import (
     C106DTO,
@@ -27,7 +27,7 @@ from logging_hj3415 import logger
 
 
 endpoint_kind = EndpointKind.C106
-endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+endpoint: Endpoint = cast(Endpoint, endpoint_kind.value)
 
 
 def _metricsblock_to_c106_metric_map(block: MetricsBlock) -> dict[str, C106ValuesMap]:

scraper2_hj3415/app/usecases/ingest/ingest_c108.py

@@ -13,14 +13,14 @@ from scraper2_hj3415.app.domain.constants import get_block_keys
 from scraper2_hj3415.app.domain.doc import NfsDoc
 from scraper2_hj3415.app.domain.blocks import RecordsBlock
 
-from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.types import Endpoint
 
 from contracts_hj3415.nfs.c108_dto import C108DTO, C108Payload, C108Blocks
 
 from logging_hj3415 import logger
 
 endpoint_kind = EndpointKind.C108
-endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+endpoint: Endpoint = cast(Endpoint, endpoint_kind.value)
 
 
 def _to_list_of_dict(rows: object) -> list[dict]: