scraper2-hj3415 2.4.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

scraper2_hj3415/app/usecases/ingest/ingest_c101.py

@@ -0,0 +1,111 @@
+# scraper2_hj3415/app/usecases/ingest/ingest_c101.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional, Any, cast
+
+from scraper2_hj3415.app.services.fetch.fetch_c101 import FetchC101
+from scraper2_hj3415.app.ports.sinks.nfs_sink_port import NfsSinkPort
+from common_hj3415.utils.time import utcnow
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import get_block_keys
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import KvBlock, RecordsBlock, MetricsBlock
+
+from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.nfs.c101_dto import C101DTO, C101Payload, C101Blocks
+
+from logging_hj3415 import logger
+
+endpoint_kind = EndpointKind.C101
+endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+
+
+def _unwrap_c101_block(block: Any) -> Any:
+    """
+    domain BlockData -> DTO로 들어갈 순수 python 구조(dict/list/...)
+    - C101은 KvBlock/RecordsBlock 위주
+    - (혹시 MetricsBlock이 섞여도 안전하게 처리)
+    """
+    if isinstance(block, KvBlock):
+        return dict(block.values)
+
+    if isinstance(block, RecordsBlock):
+        # rows: Sequence[Record] -> list[dict]
+        return [dict(r) for r in block.rows]
+
+    if isinstance(block, MetricsBlock):
+        # C101에서 MetricsBlock 쓸 일은 거의 없겠지만, 안전망
+        # metrics: Mapping[MetricKey, MetricSeries(values: Mapping[Period, Num])]
+        out: dict[str, dict[str, Any]] = {}
+        for mk, series in block.metrics.items():
+            out[str(mk)] = dict(series.values)
+        return out
+
+    # 이미 dict/list 등으로 들어오는 케이스도 방어
+    return block
+
+
+def c101_doc_to_dto(*, doc: NfsDoc, asof: datetime) -> C101DTO:
+    """
+    NfsDoc(domain) -> C101DTO(contracts)
+
+    규칙:
+      - C101은 labels를 비우는 것이 정상 (하지만 payload에는 항상 존재)
+      - blocks는 endpoint block_keys 순서대로 채우되, 각 블록은 BlockData를 언래핑해서 넣는다.
+    """
+    blocks: dict[str, Any] = {}
+    labels: dict[str, dict[str, str]] = {}
+
+    for bk in get_block_keys(endpoint_kind):
+        block = doc.blocks.get(bk)
+        blocks[str(bk)] = _unwrap_c101_block(block) if block is not None else {}
+
+        # C101은 labels 항상 empty
+        labels[str(bk)] = {}
+
+    payload: C101Payload = cast(C101Payload, {"blocks": cast(C101Blocks, blocks), "labels": labels})
+
+    return C101DTO(
+        code=doc.code,
+        asof=asof,
+        endpoint=endpoint,
+        payload=payload,
+    )
+
+
+class IngestC101:
+    def __init__(self, fetch: FetchC101, sink: NfsSinkPort[C101DTO]):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(
+        self, code: str, *, sleep_sec: float = 2.0, asof: datetime | None = None
+    ) -> C101DTO:
+        asof = asof or utcnow()
+        doc = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        logger.debug(f"doc:\n{doc}")
+        if doc is None:
+            raise RuntimeError(f"c101 fetch returned None: code={code}")
+
+        dto = c101_doc_to_dto(doc=doc, asof=asof)
+        logger.debug(f"dto:\n{dto}")
+
+        await self.sink.write(dto, endpoint=endpoint)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C101DTO]:
+        batch_asof = asof or utcnow()
+
+        docs = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        dtos = [c101_doc_to_dto(doc=d, asof=batch_asof) for d in docs]
+        logger.debug(f"dtos:\n{dtos}")
+        await self.sink.write_many(dtos, endpoint=endpoint)
+        return dtos

scraper2_hj3415/app/usecases/ingest/ingest_c103.py

@@ -0,0 +1,162 @@
+# scraper2_hj3415/app/usecases/ingest/ingest_c103.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional, cast
+
+from scraper2_hj3415.app.services.fetch.fetch_c103 import FetchC103
+from scraper2_hj3415.app.ports.sinks.nfs_sink_port import NfsSinkPort
+from common_hj3415.utils.time import utcnow
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import get_block_keys
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import MetricsBlock
+
+from contracts_hj3415.nfs.types import Endpoints
+
+from contracts_hj3415.nfs.c103_dto import (
+    C103DTO,
+    C103Payload,
+    C103Blocks,
+    C103Labels,
+    C103ValuesMap,
+)
+
+from logging_hj3415 import logger
+
+
+endpoint_kind = EndpointKind.C103
+endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+
+
+def _metricsblock_to_c103_metric_map(block: MetricsBlock) -> dict[str, C103ValuesMap]:
+    """
+    MetricsBlock(domain) -> dict[MetricKey, dict[CodeKey, Num]]
+    - domain MetricSeries.values 가 이미 {code: num} 형태라면 그대로 dict로 고정시킴.
+    """
+    out: dict[str, C103ValuesMap] = {}
+    for mk, series in block.metrics.items():
+        out[mk] = dict(series.values)  # Mapping -> dict
+    return out
+
+
+def c103_doc_to_dto(*, doc: NfsDoc, asof: datetime) -> C103DTO:
+    """
+    NfsDoc(domain, endpoint=c103) -> C103DTO(contracts envelope)
+
+    C103Payload 구조:
+      {
+        "blocks": {"손익계산서y": {metric: {code: num}}, "손익계산서q": {...}},
+        "labels": {"손익계산서y": {metric: raw_label}, "손익계산서q": {...}}
+      }
+
+    정책:
+      - blocks/labels는 항상 y/q 키를 가진다. (없으면 빈 dict)
+      - doc.labels는 없을 수도 있으니 dict()로 안전 변환
+    """
+    # 1) 기본 골격은 항상 채운다 (규약 안정성)
+    blocks: C103Blocks = {
+        "손익계산서y": {},
+        "손익계산서q": {},
+        "재무상태표y": {},
+        "재무상태표q": {},
+        "현금흐름표y": {},
+        "현금흐름표q": {},
+    }
+    labels: C103Labels = {
+        "손익계산서y": {},
+        "손익계산서q": {},
+        "재무상태표y": {},
+        "재무상태표q": {},
+        "현금흐름표y": {},
+        "현금흐름표q": {},
+    }
+
+    # 2) 도메인 블록키 순서/목록 기준으로 채움
+    for bk in get_block_keys(endpoint_kind):
+        bd = doc.blocks.get(bk)
+        if bd is None:
+            continue
+
+        if not isinstance(bd, MetricsBlock):
+            raise TypeError(
+                f"c103 expects MetricsBlock, got {type(bd).__name__} | block_key={bk!r}"
+            )
+
+        metric_map = _metricsblock_to_c103_metric_map(bd)
+        label_map = dict(doc.labels.get(bk, {}))  # 없으면 {}
+
+        match bk:
+            case "손익계산서y":
+                blocks["손익계산서y"] = metric_map
+                labels["손익계산서y"] = label_map
+            case "손익계산서q":
+                blocks["손익계산서q"] = metric_map
+                labels["손익계산서q"] = label_map
+            case "재무상태표y":
+                blocks["재무상태표y"] = metric_map
+                labels["재무상태표y"] = label_map
+            case "재무상태표q":
+                blocks["재무상태표q"] = metric_map
+                labels["재무상태표q"] = label_map
+            case "현금흐름표y":
+                blocks["현금흐름표y"] = metric_map
+                labels["현금흐름표y"] = label_map
+            case "현금흐름표q":
+                blocks["현금흐름표q"] = metric_map
+                labels["현금흐름표q"] = label_map
+            case _:
+                raise ValueError(f"invalid c103 block key: {bk!r}")
+
+    payload: C103Payload = cast(C103Payload, {"blocks": blocks, "labels": labels})
+
+    # ⚠️ TypedDict는 런타임 검증이 아니라 타입체커용이므로,
+    # C103Payload(**payload) 같은 생성은 불가능(=TypedDict는 호출 불가)
+    return C103DTO(
+        code=doc.code,
+        asof=asof,
+        endpoint=endpoint,
+        payload=payload,  # 그대로 dict 주입
+    )
+
+
+class IngestC103:
+    def __init__(self, fetch: FetchC103, sink: NfsSinkPort[C103DTO]):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(
+        self,
+        code: str,
+        *,
+        sleep_sec: float = 2.0,
+        asof: datetime | None = None,
+    ) -> C103DTO:
+        asof = asof or utcnow()
+
+        doc = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        logger.debug(f"doc:\n{doc}")
+        if doc is None:
+            raise RuntimeError(f"c106 fetch returned None: code={code}")
+
+        dto = c103_doc_to_dto(doc=doc, asof=asof)
+        logger.debug(f"dto:\n{dto}")
+
+        await self.sink.write(dto, endpoint=endpoint)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C103DTO]:
+        batch_asof = asof or utcnow()
+
+        docs = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        dtos = [c103_doc_to_dto(doc=d, asof=batch_asof) for d in docs]
+        logger.debug(f"dtos:\n{dtos}")
+        await self.sink.write_many(dtos, endpoint=endpoint)
+        return dtos

scraper2_hj3415/app/usecases/ingest/ingest_c104.py

@@ -0,0 +1,182 @@
+# scraper2_hj3415/app/usecases/ingest/ingest_c104.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional, cast
+
+from scraper2_hj3415.app.services.fetch.fetch_c104 import FetchC104
+from scraper2_hj3415.app.ports.sinks.nfs_sink_port import NfsSinkPort
+from common_hj3415.utils.time import utcnow
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import get_block_keys
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import MetricsBlock
+
+from contracts_hj3415.nfs.types import Endpoints
+
+from contracts_hj3415.nfs.c104_dto import (
+    C104DTO,
+    C104Payload,
+    C104Blocks,
+    C104Labels,
+    C104ValuesMap,
+)
+
+from logging_hj3415 import logger
+
+
+endpoint_kind = EndpointKind.C104
+endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+
+
+def _metricsblock_to_c104_metric_map(block: MetricsBlock) -> dict[str, C104ValuesMap]:
+    """
+    MetricsBlock(domain) -> dict[MetricKey, dict[CodeKey, Num]]
+    - domain MetricSeries.values 가 이미 {code: num} 형태라면 그대로 dict로 고정시킴.
+    """
+    out: dict[str, C104ValuesMap] = {}
+    for mk, series in block.metrics.items():
+        out[mk] = dict(series.values)  # Mapping -> dict
+    return out
+
+
+def c104_doc_to_dto(*, doc: NfsDoc, asof: datetime) -> C104DTO:
+    """
+    NfsDoc(domain, endpoint=c104) -> C104DTO(contracts envelope)
+
+    C104Payload 구조:
+      {
+        "blocks": {"수익성y": {metric: {code: num}}, "성장성y": {...}},
+        "labels": {"수익성y": {metric: raw_label}, "성장성y": {...}}
+      }
+
+    정책:
+      - blocks/labels는 항상 y/q 키를 가진다. (없으면 빈 dict)
+      - doc.labels는 없을 수도 있으니 dict()로 안전 변환
+    """
+    # 1) 기본 골격은 항상 채운다 (규약 안정성)
+    blocks: C104Blocks = {
+        "수익성y": {},
+        "성장성y": {},
+        "안정성y": {},
+        "활동성y": {},
+        "가치분석y": {},
+        "수익성q": {},
+        "성장성q": {},
+        "안정성q": {},
+        "활동성q": {},
+        "가치분석q": {},
+    }
+    labels: C104Labels = {
+        "수익성y": {},
+        "성장성y": {},
+        "안정성y": {},
+        "활동성y": {},
+        "가치분석y": {},
+        "수익성q": {},
+        "성장성q": {},
+        "안정성q": {},
+        "활동성q": {},
+        "가치분석q": {},
+    }
+
+    # 2) 도메인 블록키 순서/목록 기준으로 채움
+    for bk in get_block_keys(endpoint_kind):
+        bd = doc.blocks.get(bk)
+        if bd is None:
+            continue
+
+        if not isinstance(bd, MetricsBlock):
+            raise TypeError(
+                f"c104 expects MetricsBlock, got {type(bd).__name__} | block_key={bk!r}"
+            )
+
+        metric_map = _metricsblock_to_c104_metric_map(bd)
+        label_map = dict(doc.labels.get(bk, {}))  # 없으면 {}
+
+        match bk:
+            case "수익성y":
+                blocks["수익성y"] = metric_map
+                labels["수익성y"] = label_map
+            case "성장성y":
+                blocks["성장성y"] = metric_map
+                labels["성장성y"] = label_map
+            case "안정성y":
+                blocks["안정성y"] = metric_map
+                labels["안정성y"] = label_map
+            case "활동성y":
+                blocks["활동성y"] = metric_map
+                labels["활동성y"] = label_map
+            case "가치분석y":
+                blocks["가치분석y"] = metric_map
+                labels["가치분석y"] = label_map
+            case "수익성q":
+                blocks["수익성q"] = metric_map
+                labels["수익성q"] = label_map
+            case "성장성q":
+                blocks["성장성q"] = metric_map
+                labels["성장성q"] = label_map
+            case "안정성q":
+                blocks["안정성q"] = metric_map
+                labels["안정성q"] = label_map
+            case "활동성q":
+                blocks["활동성q"] = metric_map
+                labels["활동성q"] = label_map
+            case "가치분석q":
+                blocks["가치분석q"] = metric_map
+                labels["가치분석q"] = label_map
+            case _:
+                raise ValueError(f"invalid C104 block key: {bk!r}")
+
+    payload: C104Payload = cast(C104Payload, {"blocks": blocks, "labels": labels})
+
+    # ⚠️ TypedDict는 런타임 검증이 아니라 타입체커용이므로,
+    # C104Payload(**payload) 같은 생성은 불가능(=TypedDict는 호출 불가)
+    return C104DTO(
+        code=doc.code,
+        asof=asof,
+        endpoint=endpoint,
+        payload=payload,  # 그대로 dict 주입
+    )
+
+
+class IngestC104:
+    def __init__(self, fetch: FetchC104, sink: NfsSinkPort[C104DTO]):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(
+        self,
+        code: str,
+        *,
+        sleep_sec: float = 2.0,
+        asof: datetime | None = None,
+    ) -> C104DTO:
+        asof = asof or utcnow()
+
+        doc = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        logger.debug(f"doc:\n{doc}")
+        if doc is None:
+            raise RuntimeError(f"c106 fetch returned None: code={code}")
+
+        dto = c104_doc_to_dto(doc=doc, asof=asof)
+        logger.debug(f"dto:\n{dto}")
+
+        await self.sink.write(dto, endpoint=endpoint)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C104DTO]:
+        batch_asof = asof or utcnow()
+
+        docs = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        dtos = [c104_doc_to_dto(doc=d, asof=batch_asof) for d in docs]
+        logger.debug(f"dtos:\n{dtos}")
+        await self.sink.write_many(dtos, endpoint=endpoint)
+        return dtos

scraper2_hj3415/app/usecases/ingest/ingest_c106.py

@@ -0,0 +1,136 @@
+# scraper2_hj3415/app/usecases/ingest/ingest_c106.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional, cast
+
+from scraper2_hj3415.app.services.fetch.fetch_c106 import FetchC106
+from scraper2_hj3415.app.ports.sinks.nfs_sink_port import NfsSinkPort
+from common_hj3415.utils.time import utcnow
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import get_block_keys
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import MetricsBlock
+
+from contracts_hj3415.nfs.types import Endpoints
+
+from contracts_hj3415.nfs.c106_dto import (
+    C106DTO,
+    C106Payload,
+    C106Blocks,
+    C106Labels,
+    C106ValuesMap,
+)
+
+from logging_hj3415 import logger
+
+
+endpoint_kind = EndpointKind.C106
+endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+
+
+def _metricsblock_to_c106_metric_map(block: MetricsBlock) -> dict[str, C106ValuesMap]:
+    """
+    MetricsBlock(domain) -> dict[MetricKey, dict[CodeKey, Num]]
+    - domain MetricSeries.values 가 이미 {code: num} 형태라면 그대로 dict로 고정시킴.
+    """
+    out: dict[str, C106ValuesMap] = {}
+    for mk, series in block.metrics.items():
+        out[mk] = dict(series.values)  # Mapping -> dict
+    return out
+
+
+def c106_doc_to_dto(*, doc: NfsDoc, asof: datetime) -> C106DTO:
+    """
+    NfsDoc(domain, endpoint=c106) -> C106DTO(contracts envelope)
+
+    C106Payload 구조:
+      {
+        "blocks": {"y": {metric: {code: num}}, "q": {...}},
+        "labels": {"y": {metric: raw_label}, "q": {...}}
+      }
+
+    정책:
+      - blocks/labels는 항상 y/q 키를 가진다. (없으면 빈 dict)
+      - doc.labels는 없을 수도 있으니 dict()로 안전 변환
+    """
+    # 1) 기본 골격은 항상 채운다 (규약 안정성)
+    blocks: C106Blocks = {"y": {}, "q": {}}
+    labels: C106Labels = {"y": {}, "q": {}}
+
+    # 2) 도메인 블록키 순서/목록 기준으로 채움
+    for bk in get_block_keys(endpoint_kind):
+        bd = doc.blocks.get(bk)
+        if bd is None:
+            continue
+
+        if not isinstance(bd, MetricsBlock):
+            raise TypeError(
+                f"c106 expects MetricsBlock, got {type(bd).__name__} | block_key={bk!r}"
+            )
+
+        metric_map = _metricsblock_to_c106_metric_map(bd)
+        label_map = dict(doc.labels.get(bk, {}))  # 없으면 {}
+
+        match bk:
+            case "y":
+                blocks["y"] = metric_map
+                labels["y"] = label_map
+            case "q":
+                blocks["q"] = metric_map
+                labels["q"] = label_map
+            case _:
+                raise ValueError(f"invalid c106 block key: {bk!r}")
+
+    payload: C106Payload = cast(C106Payload, {"blocks": blocks, "labels": labels})
+
+    # ⚠️ TypedDict는 런타임 검증이 아니라 타입체커용이므로,
+    # C106Payload(**payload) 같은 생성은 불가능(=TypedDict는 호출 불가)
+    return C106DTO(
+        code=doc.code,
+        asof=asof,
+        endpoint=endpoint,
+        payload=payload,  # 그대로 dict 주입
+    )
+
+
+class IngestC106:
+    def __init__(self, fetch: FetchC106, sink: NfsSinkPort[C106DTO]):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(
+        self,
+        code: str,
+        *,
+        sleep_sec: float = 2.0,
+        asof: datetime | None = None,
+    ) -> C106DTO:
+        asof = asof or utcnow()
+
+        doc = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        logger.debug(f"doc:\n{doc}")
+        if doc is None:
+            raise RuntimeError(f"c106 fetch returned None: code={code}")
+
+        dto = c106_doc_to_dto(doc=doc, asof=asof)
+        logger.debug(f"dto:\n{dto}")
+
+        await self.sink.write(dto, endpoint=endpoint)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C106DTO]:
+        batch_asof = asof or utcnow()
+
+        docs = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        dtos = [c106_doc_to_dto(doc=d, asof=batch_asof) for d in docs]
+        logger.debug(f"dtos:\n{dtos}")
+        await self.sink.write_many(dtos, endpoint=endpoint)
+        return dtos