scraper2-hj3415 2.4.1__py3-none-any.whl → 2.7.0__py3-none-any.whl

scraper2_hj3415/app/services/nfs_doc_builders.py

@@ -0,0 +1,304 @@
+# scraper2_hj3415/app/services/nfs_doc_builders.py
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Mapping, Iterable, Any
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import BLOCK_KEYS_BY_ENDPOINT
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import (
+    MetricsBlock,
+    RecordsBlock,
+    KvBlock,
+    BlockData,
+)
+from scraper2_hj3415.app.domain.series import MetricSeries
+from scraper2_hj3415.app.domain.types import (
+    LabelsMap,
+    MetricKey,
+    Period,
+    Num,
+    BlockKey,
+    Records,
+    Record,
+)
+
+from common_hj3415.utils import nan_to_none
+
+
+def is_all_none(row: dict[str, Any]) -> bool:
+    return all(v is None for v in row.values())
+
+
+ParsedBlocks = Mapping[str, Any]  # parser가 반환한 "block_key(str) -> rows(list[dict])"
+
+
+def build_metrics_block_and_labels_from_rows(
+    *,
+    endpoint_kind: EndpointKind,
+    block_key: BlockKey,
+    rows: Records,
+    item_key: str = "항목",
+    raw_label_key: str = "항목_raw",
+) -> tuple[MetricsBlock, LabelsMap]:
+    """
+    rows(list[dict]) -> (MetricsBlock, LabelsMap)
+    c103/c104/c106 공통 빌더.
+
+    - Metric key는 item_key(보통 '항목')에서 만들고,
+      기간 컬럼들은 {Period: Num}으로 유지한다.
+    - LabelsMap은 dto_key -> raw_label(정제된 원라벨)
+    """
+    grouped: dict[str, list[tuple[dict[Period, Num], str]]] = defaultdict(list)
+
+    for r in rows:
+        item = r.get(item_key)
+        if not item:
+            continue
+
+        raw_label = r.get(raw_label_key)
+        if raw_label is None:
+            raw_label = item
+
+        per_map: dict[Period, Num] = {
+            str(k): nan_to_none(v)
+            for k, v in r.items()
+            if k not in (item_key, raw_label_key)
+        }
+
+        grouped[item].append((per_map, raw_label))
+
+    series_map: dict[MetricKey, MetricSeries] = {}
+    labels_map: LabelsMap = {}
+
+    for item, pairs in grouped.items():
+        if len(pairs) == 1:
+            per_map, raw_label = pairs[0]
+            series_map[item] = MetricSeries(key=item, values=per_map)
+            labels_map[item] = raw_label
+            continue
+
+        kept = [(per_map, raw) for (per_map, raw) in pairs if not is_all_none(per_map)]
+        if not kept:
+            continue
+
+        for idx, (per_map, raw_label) in enumerate(kept, start=1):
+            mk = item if idx == 1 else f"{item}_{idx}"
+            series_map[mk] = MetricSeries(key=mk, values=per_map)
+            labels_map[mk] = raw_label
+
+    block = MetricsBlock(
+        endpoint_kind=endpoint_kind, block_key=block_key, metrics=series_map
+    )
+    return block, labels_map
+
+
+def build_metrics_doc_from_parsed(
+    *,
+    code: str,
+    endpoint_kind: EndpointKind,
+    parsed: ParsedBlocks,
+    block_keys: Iterable[BlockKey] | None = None,
+    item_key: str = "항목",
+    raw_label_key: str = "항목_raw",
+    keep_empty_blocks: bool = True,
+) -> NfsDoc:
+    """
+    parser가 만든 dict(블록키 -> rows)를 받아서 NfsDoc(=MetricsBlock들)로 조립.
+    - c103/c104/c106 공용으로 사용 가능.
+
+    keep_empty_blocks:
+      - True: block은 항상 생성 (metrics 비어도 block 존재)
+      - False: rows가 없거나 metrics가 비면 blocks에서 제외
+    """
+    if block_keys is None:
+        block_keys = BLOCK_KEYS_BY_ENDPOINT[endpoint_kind]
+
+    blocks: dict[BlockKey, MetricsBlock] = {}
+    labels: dict[BlockKey, LabelsMap] = {}
+
+    for bk in block_keys:
+        rows = parsed.get(str(bk), []) or []
+        block, lm = build_metrics_block_and_labels_from_rows(
+            endpoint_kind=endpoint_kind,
+            block_key=bk,
+            rows=rows,
+            item_key=item_key,
+            raw_label_key=raw_label_key,
+        )
+
+        if not keep_empty_blocks and not block.metrics:
+            continue
+
+        blocks[bk] = block
+        labels[bk] = lm  # 비어있어도 {}로 유지
+
+    return NfsDoc(code=code, endpoint_kind=endpoint_kind, blocks=blocks, labels=labels)
+
+
+def _as_records(x: Any) -> Records:
+    """
+    안전하게 rows를 Records(=Sequence[Record])로 캐스팅/정리.
+    - None/비정상 값이면 빈 리스트
+    - list[dict] 형태만 통과시키고 나머지는 필터
+    """
+    if not x:
+        return []
+    if not isinstance(x, list):
+        return []
+
+    out: list[Record] = []
+    for it in x:
+        if isinstance(it, dict):
+            out.append(it)
+    return out
+
+
+def build_records_block_from_rows(
+    *,
+    endpoint_kind: EndpointKind,
+    block_key: BlockKey,
+    rows: Records,
+) -> RecordsBlock:
+    """
+    rows(list[dict]) -> RecordsBlock
+    - c108 같은 레코드성 블록(리포트 목록 등)에 사용
+    """
+    # RecordsBlock 쪽에서도 __post_init__로 block_key 검증이 수행된다는 전제(네가 정돈한 도메인)
+    return RecordsBlock(
+        endpoint_kind=endpoint_kind, block_key=block_key, rows=list(rows)
+    )
+
+
+def build_c108_doc_from_parsed(
+    *,
+    code: str,
+    parsed: ParsedBlocks,
+    block_keys: Iterable[BlockKey] | None = None,
+    keep_empty_blocks: bool = True,
+) -> NfsDoc:
+    """
+    c108 parser 결과(dict)를 받아서 NfsDoc(=RecordsBlock들)로 조립.
+
+    규칙(너가 정한 원칙):
+      - labels는 항상 존재(빈 dict라도)
+      - c108은 labels를 비우는 것을 정상으로 간주
+
+    keep_empty_blocks:
+      - True: block은 항상 생성(rows 비어도 block 존재)
+      - False: rows가 비면 blocks에서 제외
+    """
+    endpoint_kind = EndpointKind.C108
+
+    if block_keys is None:
+        # 보통 ("리포트",) 같은 튜플
+        block_keys = BLOCK_KEYS_BY_ENDPOINT[endpoint_kind]
+
+    blocks: dict[BlockKey, RecordsBlock] = {}
+    labels: dict[BlockKey, LabelsMap] = {}
+
+    for bk in block_keys:
+        rows = _as_records(parsed.get(str(bk)))
+        block = build_records_block_from_rows(
+            endpoint_kind=endpoint_kind,
+            block_key=bk,
+            rows=rows,
+        )
+
+        if not keep_empty_blocks and not block.rows:
+            continue
+
+        blocks[bk] = block
+        labels[bk] = {}  # c108은 labels 비우는 것이 정상
+
+    return NfsDoc(
+        code=code,
+        endpoint_kind=endpoint_kind,
+        blocks=blocks,
+        labels=labels,
+    )
+
+
+def build_kv_block_from_mapping(
+    *,
+    endpoint_kind: EndpointKind,
+    block_key: BlockKey,
+    data: Mapping[str, Any] | None,
+    keep_empty: bool = True,
+) -> KvBlock | None:
+    """
+    dict 형태 블록을 KvBlock으로 감싼다.
+    - c101 요약/시세/기업개요/펀더멘털/어닝서프라이즈 같은 "구조 dict"에 사용
+    """
+    if not data:
+        if not keep_empty:
+            return None
+        data = {}
+
+    return KvBlock(endpoint_kind=endpoint_kind, block_key=block_key, values=data)
+
+
+ParsedC101 = Mapping[str, Any]  # c101은 dict/list/dict(중첩) 섞여서 Any가 현실적
+
+
+def build_c101_doc_from_parsed(
+    *,
+    code: str,
+    parsed: ParsedC101,
+    block_keys: Iterable[BlockKey] | None = None,
+    keep_empty_blocks: bool = True,
+) -> NfsDoc:
+    """
+    c101 parser 결과(블록별 다양한 타입)를 NfsDoc으로 조립.
+    labels는 c101은 '비어도 정상' 규칙을 따르므로 항상 {}로 둔다.
+    """
+    endpoint_kind = EndpointKind.C101
+
+    if block_keys is None:
+        block_keys = BLOCK_KEYS_BY_ENDPOINT[endpoint_kind]
+
+    blocks: dict[BlockKey, BlockData] = {}
+    labels: dict[BlockKey, LabelsMap] = {}
+
+    for bk in block_keys:
+        v = parsed.get(str(bk))
+
+        # c101 규칙: labels는 비어도 정상, 있으면 넣는 게 아니라 "기본 비움" 추천
+        labels[bk] = {}
+
+        # list -> RecordsBlock
+        if isinstance(v, list):
+            # v: list[dict[str, Any]] 가정 (파서가 그렇게 만들고 있음)
+            rb = build_records_block_from_rows(
+                endpoint_kind=endpoint_kind,
+                block_key=bk,
+                rows=v,  # type: ignore[arg-type]  (rows 타입 맞추면 제거 가능)
+            )
+            if rb is not None:
+                blocks[bk] = rb
+            continue
+
+        # dict(중첩 포함) -> KvBlock
+        if isinstance(v, dict):
+            kb = build_kv_block_from_mapping(
+                endpoint_kind=endpoint_kind,
+                block_key=bk,
+                data=v,
+                keep_empty=keep_empty_blocks,
+            )
+            if kb is not None:
+                blocks[bk] = kb
+            continue
+
+        # None/기타 -> empty policy
+        if keep_empty_blocks:
+            kb = build_kv_block_from_mapping(
+                endpoint_kind=endpoint_kind,
+                block_key=bk,
+                data={},
+                keep_empty=True,
+            )
+            blocks[bk] = kb
+
+    return NfsDoc(code=code, endpoint_kind=endpoint_kind, blocks=blocks, labels=labels)

scraper2_hj3415/app/usecases/ingest/ingest_c101.py

@@ -0,0 +1,111 @@
+# scraper2_hj3415/app/usecases/ingest/ingest_c101.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional, Any, cast
+
+from scraper2_hj3415.app.services.fetch.fetch_c101 import FetchC101
+from scraper2_hj3415.app.ports.sinks.nfs_sink_port import NfsSinkPort
+from common_hj3415.utils.time import utcnow
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import get_block_keys
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import KvBlock, RecordsBlock, MetricsBlock
+
+from contracts_hj3415.nfs.types import Endpoint
+from contracts_hj3415.nfs.c101_dto import C101DTO, C101Payload, C101Blocks
+
+from logging_hj3415 import logger
+
+endpoint_kind = EndpointKind.C101
+endpoint: Endpoint = cast(Endpoint, endpoint_kind.value)
+
+
+def _unwrap_c101_block(block: Any) -> Any:
+    """
+    domain BlockData -> DTO로 들어갈 순수 python 구조(dict/list/...)
+    - C101은 KvBlock/RecordsBlock 위주
+    - (혹시 MetricsBlock이 섞여도 안전하게 처리)
+    """
+    if isinstance(block, KvBlock):
+        return dict(block.values)
+
+    if isinstance(block, RecordsBlock):
+        # rows: Sequence[Record] -> list[dict]
+        return [dict(r) for r in block.rows]
+
+    if isinstance(block, MetricsBlock):
+        # C101에서 MetricsBlock 쓸 일은 거의 없겠지만, 안전망
+        # metrics: Mapping[MetricKey, MetricSeries(values: Mapping[Period, Num])]
+        out: dict[str, dict[str, Any]] = {}
+        for mk, series in block.metrics.items():
+            out[str(mk)] = dict(series.values)
+        return out
+
+    # 이미 dict/list 등으로 들어오는 케이스도 방어
+    return block
+
+
+def c101_doc_to_dto(*, doc: NfsDoc, asof: datetime) -> C101DTO:
+    """
+    NfsDoc(domain) -> C101DTO(contracts)
+
+    규칙:
+      - C101은 labels를 비우는 것이 정상 (하지만 payload에는 항상 존재)
+      - blocks는 endpoint block_keys 순서대로 채우되, 각 블록은 BlockData를 언래핑해서 넣는다.
+    """
+    blocks: dict[str, Any] = {}
+    labels: dict[str, dict[str, str]] = {}
+
+    for bk in get_block_keys(endpoint_kind):
+        block = doc.blocks.get(bk)
+        blocks[str(bk)] = _unwrap_c101_block(block) if block is not None else {}
+
+        # C101은 labels 항상 empty
+        labels[str(bk)] = {}
+
+    payload: C101Payload = cast(C101Payload, {"blocks": cast(C101Blocks, blocks), "labels": labels})
+
+    return C101DTO(
+        code=doc.code,
+        asof=asof,
+        endpoint=endpoint,
+        payload=payload,
+    )
+
+
+class IngestC101:
+    def __init__(self, fetch: FetchC101, sink: NfsSinkPort[C101DTO]):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(
+        self, code: str, *, sleep_sec: float = 2.0, asof: datetime | None = None
+    ) -> C101DTO:
+        asof = asof or utcnow()
+        doc = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        logger.debug(f"doc:\n{doc}")
+        if doc is None:
+            raise RuntimeError(f"c101 fetch returned None: code={code}")
+
+        dto = c101_doc_to_dto(doc=doc, asof=asof)
+        logger.debug(f"dto:\n{dto}")
+
+        await self.sink.write(dto, endpoint=endpoint)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C101DTO]:
+        batch_asof = asof or utcnow()
+
+        docs = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        dtos = [c101_doc_to_dto(doc=d, asof=batch_asof) for d in docs]
+        logger.debug(f"dtos:\n{dtos}")
+        await self.sink.write_many(dtos, endpoint=endpoint)
+        return dtos

scraper2_hj3415/app/usecases/ingest/ingest_c103.py

@@ -0,0 +1,162 @@
+# scraper2_hj3415/app/usecases/ingest/ingest_c103.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional, cast
+
+from scraper2_hj3415.app.services.fetch.fetch_c103 import FetchC103
+from scraper2_hj3415.app.ports.sinks.nfs_sink_port import NfsSinkPort
+from common_hj3415.utils.time import utcnow
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import get_block_keys
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import MetricsBlock
+
+from contracts_hj3415.nfs.types import Endpoint
+
+from contracts_hj3415.nfs.c103_dto import (
+    C103DTO,
+    C103Payload,
+    C103Blocks,
+    C103Labels,
+    C103ValuesMap,
+)
+
+from logging_hj3415 import logger
+
+
+endpoint_kind = EndpointKind.C103
+endpoint: Endpoint = cast(Endpoint, endpoint_kind.value)
+
+
+def _metricsblock_to_c103_metric_map(block: MetricsBlock) -> dict[str, C103ValuesMap]:
+    """
+    MetricsBlock(domain) -> dict[MetricKey, dict[CodeKey, Num]]
+    - domain MetricSeries.values 가 이미 {code: num} 형태라면 그대로 dict로 고정시킴.
+    """
+    out: dict[str, C103ValuesMap] = {}
+    for mk, series in block.metrics.items():
+        out[mk] = dict(series.values)  # Mapping -> dict
+    return out
+
+
+def c103_doc_to_dto(*, doc: NfsDoc, asof: datetime) -> C103DTO:
+    """
+    NfsDoc(domain, endpoint=c103) -> C103DTO(contracts envelope)
+
+    C103Payload 구조:
+      {
+        "blocks": {"손익계산서y": {metric: {code: num}}, "손익계산서q": {...}},
+        "labels": {"손익계산서y": {metric: raw_label}, "손익계산서q": {...}}
+      }
+
+    정책:
+      - blocks/labels는 항상 y/q 키를 가진다. (없으면 빈 dict)
+      - doc.labels는 없을 수도 있으니 dict()로 안전 변환
+    """
+    # 1) 기본 골격은 항상 채운다 (규약 안정성)
+    blocks: C103Blocks = {
+        "손익계산서y": {},
+        "손익계산서q": {},
+        "재무상태표y": {},
+        "재무상태표q": {},
+        "현금흐름표y": {},
+        "현금흐름표q": {},
+    }
+    labels: C103Labels = {
+        "손익계산서y": {},
+        "손익계산서q": {},
+        "재무상태표y": {},
+        "재무상태표q": {},
+        "현금흐름표y": {},
+        "현금흐름표q": {},
+    }
+
+    # 2) 도메인 블록키 순서/목록 기준으로 채움
+    for bk in get_block_keys(endpoint_kind):
+        bd = doc.blocks.get(bk)
+        if bd is None:
+            continue
+
+        if not isinstance(bd, MetricsBlock):
+            raise TypeError(
+                f"c103 expects MetricsBlock, got {type(bd).__name__} | block_key={bk!r}"
+            )
+
+        metric_map = _metricsblock_to_c103_metric_map(bd)
+        label_map = dict(doc.labels.get(bk, {}))  # 없으면 {}
+
+        match bk:
+            case "손익계산서y":
+                blocks["손익계산서y"] = metric_map
+                labels["손익계산서y"] = label_map
+            case "손익계산서q":
+                blocks["손익계산서q"] = metric_map
+                labels["손익계산서q"] = label_map
+            case "재무상태표y":
+                blocks["재무상태표y"] = metric_map
+                labels["재무상태표y"] = label_map
+            case "재무상태표q":
+                blocks["재무상태표q"] = metric_map
+                labels["재무상태표q"] = label_map
+            case "현금흐름표y":
+                blocks["현금흐름표y"] = metric_map
+                labels["현금흐름표y"] = label_map
+            case "현금흐름표q":
+                blocks["현금흐름표q"] = metric_map
+                labels["현금흐름표q"] = label_map
+            case _:
+                raise ValueError(f"invalid c103 block key: {bk!r}")
+
+    payload: C103Payload = cast(C103Payload, {"blocks": blocks, "labels": labels})
+
+    # ⚠️ TypedDict는 런타임 검증이 아니라 타입체커용이므로,
+    # C103Payload(**payload) 같은 생성은 불가능(=TypedDict는 호출 불가)
+    return C103DTO(
+        code=doc.code,
+        asof=asof,
+        endpoint=endpoint,
+        payload=payload,  # 그대로 dict 주입
+    )
+
+
+class IngestC103:
+    def __init__(self, fetch: FetchC103, sink: NfsSinkPort[C103DTO]):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(
+        self,
+        code: str,
+        *,
+        sleep_sec: float = 2.0,
+        asof: datetime | None = None,
+    ) -> C103DTO:
+        asof = asof or utcnow()
+
+        doc = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        logger.debug(f"doc:\n{doc}")
+        if doc is None:
+            raise RuntimeError(f"c106 fetch returned None: code={code}")
+
+        dto = c103_doc_to_dto(doc=doc, asof=asof)
+        logger.debug(f"dto:\n{dto}")
+
+        await self.sink.write(dto, endpoint=endpoint)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C103DTO]:
+        batch_asof = asof or utcnow()
+
+        docs = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        dtos = [c103_doc_to_dto(doc=d, asof=batch_asof) for d in docs]
+        logger.debug(f"dtos:\n{dtos}")
+        await self.sink.write_many(dtos, endpoint=endpoint)
+        return dtos