scraper2-hj3415 2.4.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

scraper2_hj3415/app/usecases/ingest/ingest_c108.py

@@ -0,0 +1,122 @@
+# scraper2_hj3415/app/usecases/ingest/ingest_c108.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional, cast
+
+from scraper2_hj3415.app.services.fetch.fetch_c108 import FetchC108
+from scraper2_hj3415.app.ports.sinks.nfs_sink_port import NfsSinkPort
+from common_hj3415.utils.time import utcnow
+
+from scraper2_hj3415.app.domain.endpoint import EndpointKind
+from scraper2_hj3415.app.domain.constants import get_block_keys
+from scraper2_hj3415.app.domain.doc import NfsDoc
+from scraper2_hj3415.app.domain.blocks import RecordsBlock
+
+from contracts_hj3415.nfs.types import Endpoints
+
+from contracts_hj3415.nfs.c108_dto import C108DTO, C108Payload, C108Blocks
+
+from logging_hj3415 import logger
+
+endpoint_kind = EndpointKind.C108
+endpoint: Endpoints = cast(Endpoints, endpoint_kind.value)
+
+
+def _to_list_of_dict(rows: object) -> list[dict]:
+    """
+    RecordsBlock.rows(Sequence[Mapping]) -> list[dict]
+    - sink/serialization 안전하게 dict로 강제
+    """
+    if not rows:
+        return []
+    out: list[dict] = []
+    if isinstance(rows, list):
+        for r in rows:
+            if isinstance(r, dict):
+                out.append(r)
+            else:
+                out.append(dict(r))  # Mapping이면 dict() 가능
+        return out
+
+    # Sequence[Mapping] 일반 케이스
+    try:
+        for r in rows:  # type: ignore[assignment]
+            out.append(dict(r))  # Mapping 가정
+    except Exception:
+        return []
+    return out
+
+
+def c108_doc_to_dto(*, doc: NfsDoc, asof: datetime) -> C108DTO:
+    """
+    NfsDoc(domain) -> C108DTO(contracts envelope)
+
+    규칙:
+      - labels는 항상 존재(빈 dict라도)
+      - c108은 labels를 비우는 것이 정상
+      - payload.blocks['리포트'] = list[dict]
+    """
+    if doc.endpoint_kind != EndpointKind.C108:
+        raise ValueError(f"c108_doc_to_dto expects C108 doc, got: {doc.endpoint_kind}")
+
+    # contracts payload 구조에 맞게: blocks/labels를 항상 구성
+    blocks: C108Blocks = {"리포트": []}
+
+    # block_keys를 따르되, 실질적으로는 '리포트' 하나만 있어도 충분
+    for bk in get_block_keys(EndpointKind.C108):
+        if bk != "리포트":
+            continue
+
+        block = doc.blocks.get(bk)
+        if isinstance(block, RecordsBlock):
+            blocks["리포트"] = _to_list_of_dict(block.rows)
+        else:
+            # 혹시 구조가 섞였으면 최대한 안전하게 빈 값
+            blocks["리포트"] = []
+
+    payload: C108Payload = {"blocks": blocks}
+
+    return C108DTO(
+        code=doc.code,
+        asof=asof,
+        endpoint=endpoint,
+        payload=payload,
+    )
+
+
+class IngestC108:
+    def __init__(self, fetch: FetchC108, sink: NfsSinkPort[C108DTO]):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(
+        self, code: str, *, sleep_sec: float = 2.0, asof: datetime | None = None
+    ) -> C108DTO:
+        asof = asof or utcnow()
+
+        doc = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        logger.debug(f"doc:\n{doc}")
+        if doc is None:
+            raise RuntimeError(f"c108 fetch returned None: code={code}")
+
+        dto = c108_doc_to_dto(doc=doc, asof=asof)
+        logger.debug(f"dto:\n{dto}")
+
+        await self.sink.write(dto, endpoint=endpoint)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C108DTO]:
+        batch_asof = asof or utcnow()
+
+        docs = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        dtos = [c108_doc_to_dto(doc=d, asof=batch_asof) for d in docs]
+        logger.debug(f"dtos:\n{dtos}")
+        await self.sink.write_many(dtos, endpoint=endpoint)
+        return dtos

scraper2/main.py → scraper2_hj3415/cli.py

@@ -1,21 +1,32 @@
-# scraper2/main.py
+# scraper2_hj3415/cli.py
 from __future__ import annotations
 
 import asyncio
-from typing import Any, Literal, cast
+from typing import Any, cast, get_args
 
+
+import time
 import typer
 from datetime import datetime, timezone
 
-from scraper2.app.composition import build_usecases
-from scraper2.app.ports.ingest_port import IngestPort
+from db2_hj3415.nfs.repo import ensure_indexes
+from db2_hj3415.settings import get_settings
+from db2_hj3415.universe.repo import list_universe_codes
+
+from scraper2_hj3415.app.composition import build_usecases
+from scraper2_hj3415.app.ports.ingest.nfs_ingest_port import NfsIngestPort
+from scraper2_hj3415.app.domain.types import Sink
 
-from logging_hj3415 import setup_logging
+from contracts_hj3415.nfs.types import Endpoints
+from contracts_hj3415.universe.types import UniverseNames
+
+from logging_hj3415 import setup_logging, current_log_level, reset_logging, to_pretty_json
 
 setup_logging()
+# 운영시에는 아래 항목 주석처리하고 환경변수로 제어할것
+reset_logging("DEBUG")
+print(f"Current log level - {current_log_level()}")
 
-Endpoint = Literal["c101", "c103", "c104", "c106", "c108", "all"]
-Sink = Literal["memory", "mongo"]
 
 app = typer.Typer(no_args_is_help=True)
 
@@ -30,64 +41,19 @@ app.add_typer(mi_app, name="mi")
 # small helpers
 # -------------------------
 
-def _endpoint_list(ep: Endpoint) -> list[str]:
-    return ["c101", "c103", "c104", "c106", "c108"] if ep == "all" else [ep]
-
-
-async def _maybe_await_close(obj: Any) -> None:
-    close = getattr(obj, "close", None)
-    if close is None:
-        return
-    out = close()
-    if asyncio.iscoroutine(out):
-        await out
-
+def _endpoint_list(endpoint: str) -> list[str]:
+    if endpoint == "all":
+        return list(get_args(Endpoints))  # -> ["c101", "c103", "c104", "c106", "c108"]
+    return [endpoint]
 
 async def _mongo_bootstrap(db) -> None:
-    from db2.nfs import ensure_indexes
-    from db2.settings import get_settings
-
     s = get_settings()
     await ensure_indexes(db, snapshot_ttl_days=s.SNAPSHOT_TTL_DAYS)
 
-
-async def _load_codes_from_universe(db, *, universe: str) -> list[str]:
-    """
-    db2.universe에 저장된 universe_latest에서 codes 로드.
-    (네 db2 API 명에 맞춰 조정하면 됨)
-    """
-    from db2.universe import get_universe_latest  # 네가 가진 API
-
-    doc = await get_universe_latest(db, universe=universe)
-    if not doc:
-        return []
-
-    # doc 형태가 {"items":[{code,name,...}, ...]} 혹은 {"payload":{"items":[...]}} 일 수 있어서 방어
-    data = doc
-    if isinstance(data, dict) and "payload" in data and isinstance(data["payload"], dict):
-        data = data["payload"]
-    if isinstance(data, dict) and "items" in data:
-        data = data["items"]
-
-    if not isinstance(data, list):
-        return []
-
-    codes: list[str] = []
-    for row in data:
-        if not isinstance(row, dict):
-            continue
-        code = str(row.get("code") or "").strip()
-        if code:
-            codes.append(code)
-    return codes
-
-
-import time
-
 async def _run_ingest_with_progress(
     *,
     ucs: Any,
-    endpoint: Endpoint,
+    endpoint: str,
     codes: list[str],
     sleep_sec: float,
     show: bool,
@@ -109,7 +75,7 @@ async def _run_ingest_with_progress(
             yield xs[i:i + n]
 
     async def _run_one_endpoint(ep: str) -> None:
-        ingest_uc = cast(IngestPort, getattr(ucs.ingest, ep))
+        ingest_uc = cast(NfsIngestPort, getattr(ucs.ingest, ep))
 
         ok = 0
         fail = 0
@@ -138,18 +104,6 @@ async def _run_ingest_with_progress(
     elapsed = time.perf_counter() - t0   # ✅ 종료 시각
     typer.echo(f"\n⏱ elapsed time: {_format_elapsed(elapsed)}")
 
-def _dto_to_pretty(obj: Any) -> str:
-    # pydantic v2 우선
-    if hasattr(obj, "model_dump_json"):
-        return obj.model_dump_json(indent=2, by_alias=False)
-    if hasattr(obj, "model_dump"):
-        import json
-        return json.dumps(obj.model_dump(), ensure_ascii=False, indent=2)
-    # dict fallback
-    if isinstance(obj, dict):
-        import json
-        return json.dumps(obj, ensure_ascii=False, indent=2, default=str)
-    return str(obj)
 
 def _format_elapsed(sec: float) -> str:
     if sec < 60:
@@ -199,11 +153,11 @@ def _parse_asof(asof: str | None) -> datetime:
 
 @nfs_app.command("one")
 def nfs_one(
-    endpoint: Endpoint = typer.Argument(..., help="c101|c103|c104|c106|c108|all"),
+    endpoint: str = typer.Argument(..., help="c101|c103|c104|c106|c108|all"),
     code: str = typer.Argument(..., help="종목코드 (예: 005930)"),
     sleep_sec: float = typer.Option(2.0, "--sleep"),
     sink: Sink = typer.Option("memory", "--sink"),
-    show: bool = typer.Option(True, "--show/--no-show", help="결과 DTO 출력"),
+    show: bool = typer.Option(False, "--show/--no-show", help="결과 DTO 출력"),
     asof: str | None = typer.Option(None, "--asof", help="배치 기준시각(ISO8601, UTC 권장). 예: 2026-01-09T05:00:00Z"),
 ):
     code = code.strip()
@@ -211,7 +165,7 @@ def nfs_one(
         raise typer.BadParameter("code는 비어있을 수 없습니다.")
 
     async def _run():
-        ucs = build_usecases(sink_kind=sink)
+        ucs = build_usecases(sink=sink)
 
         if sink == "mongo":
             if ucs.db is None:
@@ -221,17 +175,23 @@ def nfs_one(
         try:
             run_asof = _parse_asof(asof)
             for ep in _endpoint_list(endpoint):
-                ingest_uc = cast(IngestPort, getattr(ucs.ingest, ep))
+                ingest_uc = cast(NfsIngestPort, getattr(ucs.ingest, ep))
                 results = await ingest_uc.execute_many([code], sleep_sec=sleep_sec, asof=run_asof)
                 dto = results[0] if results else None
 
                 typer.echo(f"\n=== ONE DONE: {ep} {code} ===")
-                if not show:
+                is_memory_sink = sink == "memory"
+                should_show = show or is_memory_sink
+
+                if not should_show:
                     continue
+
                 if dto is None:
                     typer.echo("(no result)")
                 else:
-                    typer.echo(_dto_to_pretty(dto))
+                    if is_memory_sink:
+                        typer.echo("memory result:")
+                    typer.echo(to_pretty_json(dto))
         finally:
             await ucs.aclose()
 
@@ -240,7 +200,7 @@ def nfs_one(
 
 @nfs_app.command("all")
 def nfs_all(
-    endpoint: Endpoint = typer.Argument(..., help="c101|c103|c104|c106|c108|all"),
+    endpoint: str = typer.Argument(..., help="c101|c103|c104|c106|c108|all"),
     universe: str = typer.Option("krx300", "--universe"),
     limit: int = typer.Option(0, "--limit", help="0=전체"),
     sleep_sec: float = typer.Option(2.0, "--sleep"),
@@ -251,12 +211,12 @@ def nfs_all(
     asof: str | None = typer.Option(None, "--asof", help="배치 기준시각(ISO8601). 예: 2026-01-09T05:00:00Z"),
 ):
     async def _run():
-        ucs = build_usecases(sink_kind=sink)
+        ucs = build_usecases(sink=sink)
         if ucs.db is None:
             raise RuntimeError("all 모드는 DB가 필요합니다. mongo sink로 ucs.db를 노출하세요.")
         await _mongo_bootstrap(ucs.db)
 
-        codes = await _load_codes_from_universe(ucs.db, universe=universe)
+        codes = await list_universe_codes(ucs.db, universe=cast(UniverseNames, universe))
         if not codes:
             raise RuntimeError(f"universe='{universe}' codes가 비었습니다. 먼저 krx sync로 universe를 채우세요.")
 
@@ -291,4 +251,4 @@ def mi():
 
 
 if __name__ == "__main__":
-    app()
+    app()

{scraper2_hj3415-2.4.0.dist-info → scraper2_hj3415-2.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scraper2-hj3415
-Version: 2.4.0
+Version: 2.6.0
 Summary: Naver WiseReport scraper
 Keywords: example,demo
 Author-email: Hyungjin Kim <hj3415@gmail.com>
@@ -17,6 +17,8 @@ Requires-Dist: lxml>=6.0.2
 Requires-Dist: typer>=0.21.0
 Requires-Dist: db2-hj3415
 Requires-Dist: contracts-hj3415
+Requires-Dist: common-hj3415
+Requires-Dist: logging-hj3415
 
 # scraper2
 

scraper2_hj3415-2.6.0.dist-info/RECORD

@@ -0,0 +1,75 @@
+scraper2_hj3415/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/cli.py,sha256=idwTPbCBcqof2YVRsBzyBMnl92o2b5JRyMbzOxOXKZA,8068
+scraper2_hj3415/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/composition.py,sha256=t9NnNDLL6-VT28dT_kBtc_4Sd6qdR_cGKcY2_wmQHMI,6573
+scraper2_hj3415/app/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/adapters/out/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/adapters/out/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/adapters/out/playwright/browser.py,sha256=mbPNLynkTHkNHLqOG7UxVo7UclfNkHbNyN4M9xrDGOk,13152
+scraper2_hj3415/app/adapters/out/playwright/browser_factory.py,sha256=Tp30xdE4Z7cWHBCCgvaqD3tqY4qyg2ij_YmuLmj2WUg,3744
+scraper2_hj3415/app/adapters/out/playwright/session.py,sha256=GLlpO0rLQwXnxX1GyaRxy7P2UsYrceNczfhUvx3pepE,3734
+scraper2_hj3415/app/adapters/out/sinks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/adapters/out/sinks/memory_sink.py,sha256=VlywXRmWjfh33yuE7AuwKU4aPi_UJAZQMSWELlHJ-l8,722
+scraper2_hj3415/app/adapters/out/sinks/mongo_sink.py,sha256=GS61lV1gaxdxC9o7Z9APAPs9WnWL6-qa_V5Ls245yNk,1749
+scraper2_hj3415/app/adapters/out/sinks/store.py,sha256=yerl6NvaacVHDGnQ1Obc31aQFxxXUCWfGq2DKKV8aTc,2826
+scraper2_hj3415/app/adapters/site/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/adapters/site/wisereport_playwright.py,sha256=lpvhy6Rdy9j762KE0udhxgIVXi0n2pvDbmbs2y0WPHs,6970
+scraper2_hj3415/app/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/domain/blocks.py,sha256=ddrGYo12hRI3h3mLsGzrM9RUZuEabqdfBWsx5l8jVGs,1871
+scraper2_hj3415/app/domain/constants.py,sha256=-BxAKH6smxE5jEHJsXmf32erk0UlisRz8BkYHXYTSgA,1181
+scraper2_hj3415/app/domain/doc.py,sha256=G9Ik6f-HiNZjiT26Obtrx6HTyYsTbMg1vKaBQ536hOI,483
+scraper2_hj3415/app/domain/endpoint.py,sha256=8nwV0ybBYDKS8ULngNvc_dh-Pl4BqpMJrdKHDuThSTA,222
+scraper2_hj3415/app/domain/series.py,sha256=KZqHqavPkL8p14F58wuUrOS-N9k0pKfKBdhDL6kTYU4,300
+scraper2_hj3415/app/domain/types.py,sha256=_fPII4xFc_zTWuVm-V_SdaB092XR2OeS0sNdJVwE5t8,374
+scraper2_hj3415/app/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/parsing/c101_parser.py,sha256=fA7-EUlG3GhLlX8l7m9p-Bn5O2WQY0H9h7IAIDvlmGQ,2049
+scraper2_hj3415/app/parsing/c103_parser.py,sha256=Tn5tfpUh4lf6IMsKW9JAp2b12W32vKcF8FPfzEJYvtY,770
+scraper2_hj3415/app/parsing/c104_parser.py,sha256=NfwFcgNZb6EqCcDYRoWiMq281CzATELJsDO_Np64Clk,814
+scraper2_hj3415/app/parsing/c106_parser.py,sha256=DvXv_OndsWed4LOyv8D9bsCxbj8_6rYrfR6ICR-VBnM,4346
+scraper2_hj3415/app/parsing/c108_parser.py,sha256=Kopf3CAV4W66YR6at7isoNV-C8A7-eCOQcqPs85FgEE,7572
+scraper2_hj3415/app/parsing/_normalize/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/parsing/_normalize/label.py,sha256=yUtUalOlXuckzsQ7RXqdQ6F4Q2UmVBGr-zoEpZ6ryX0,2752
+scraper2_hj3415/app/parsing/_normalize/table.py,sha256=V6I79gOHeRsyFiuIXyEy0Whg-pxZeDdTaGh0cdR5mrE,1464
+scraper2_hj3415/app/parsing/_normalize/text.py,sha256=BnBZyaQiuydsQVUSDgIEn8JYYbxrM-3BZTmvNqiFK3g,683
+scraper2_hj3415/app/parsing/_normalize/values.py,sha256=X5H7xprg5y8pkXilXCg_br7UIPjErcLHGDkOrxjctbk,1824
+scraper2_hj3415/app/parsing/_tables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/parsing/_tables/html_table.py,sha256=m44eA0rVhhk1QL74fgR2dbldKuF1_K5mJr2xiyb-55U,2393
+scraper2_hj3415/app/parsing/c101/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/parsing/c101/_sise_normalizer.py,sha256=0wG9AZ6MYvWMf_UA9QK3_TbyDctdi3ldyKNKwmLJJic,3150
+scraper2_hj3415/app/parsing/c101/company_overview.py,sha256=R6K44Rlw9Iv740SR2XZDxBkZxsLi_SNB6n3xruO3xqk,1391
+scraper2_hj3415/app/parsing/c101/earning_surprise.py,sha256=QqiVVrdJuQXm25mvm0AsQB7gi461IiUbAoD8iCamUjg,7028
+scraper2_hj3415/app/parsing/c101/fundamentals.py,sha256=3Fy6ya53NF-VoXa16GDpqUTdoFo2PIEjt5rjlXNa8sI,2875
+scraper2_hj3415/app/parsing/c101/major_shareholders.py,sha256=sF1j1VNZSoIkQYHmuhMWSx52l00WDf6El2NkiRoXW0o,2047
+scraper2_hj3415/app/parsing/c101/sise.py,sha256=Mky6pLWZ_LZkeUMHIPcZfrv0icTNxWEc7uTYKU2uJ0M,1314
+scraper2_hj3415/app/parsing/c101/summary_cmp.py,sha256=hhBCtH7hgAKFUh4gr7J-mz-6c9NLT9KZODFY8LTG-Fc,2776
+scraper2_hj3415/app/parsing/c101/yearly_consensus.py,sha256=FCLA-pYCMbQffNYOV6YbZ8GnPJjyZHmCSIKdw9-EPuI,5572
+scraper2_hj3415/app/ports/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/ports/browser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/ports/browser/browser_factory_port.py,sha256=exG5XM3yen84lqsY0ggpZ58kcyCyoaMusDMooVxCGG0,353
+scraper2_hj3415/app/ports/browser/browser_port.py,sha256=VUxMDsrKWBAi1TVD8b-PbsHkCSjZ9ZMgsr3eVmhb_1I,3628
+scraper2_hj3415/app/ports/ingest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/ports/ingest/nfs_ingest_port.py,sha256=Ia8GByRLV9SPEU89I8A4V6tmUwsO8f7xM7-yVxnsA0o,658
+scraper2_hj3415/app/ports/sinks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/ports/sinks/nfs_sink_port.py,sha256=8EOdFH5-703yc8XP47PZ0mmdizg4d_kAzuR1-G5b4MY,522
+scraper2_hj3415/app/ports/site/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/ports/site/wisereport_port.py,sha256=ufbYJ1jyNkSmlIbV1CqI8BekxjgGgWvxj8yb73ZRUU0,663
+scraper2_hj3415/app/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/services/nfs_doc_builders.py,sha256=bz2Is3xXlM98gtd1QQUqgeoqWk2EwbuWNCJw5oJXkg8,8874
+scraper2_hj3415/app/services/fetch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/services/fetch/fetch_c101.py,sha256=nl96RKvahNS9YtusE4e9AFMb0Wf7UHaW2X6qAAzYuCY,2228
+scraper2_hj3415/app/services/fetch/fetch_c103.py,sha256=PKJwtZLJDfVNjj5l8nzx9EJOtqH1zR8OF-pOiUlMrRc,5148
+scraper2_hj3415/app/services/fetch/fetch_c104.py,sha256=djl7Z_CW58gpl8naMezumc_dbjc3l-EX6OQJeWU_ZAw,6549
+scraper2_hj3415/app/services/fetch/fetch_c106.py,sha256=UIAMaQB-FHXsvQ0ONI3fTO22M3npXrHILxO_4ayY2Lk,3462
+scraper2_hj3415/app/services/fetch/fetch_c108.py,sha256=o9GesH66jqCygZIEbrHlVwHGGK1_2Ilc2_1b24fSC54,2236
+scraper2_hj3415/app/usecases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/usecases/ingest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2_hj3415/app/usecases/ingest/ingest_c101.py,sha256=nLhdG27NsuZqxqQI6bWRcFI2T7BZV-QXmUldql6qCo8,3760
+scraper2_hj3415/app/usecases/ingest/ingest_c103.py,sha256=g36Kc1DK8v0AsyN_FCJq_A1Lv1EwrZ1vaS7qMCZTSEQ,5405
+scraper2_hj3415/app/usecases/ingest/ingest_c104.py,sha256=CfS1HsnnHt4N-N3YhtiZjahCE-SryHpWXoTT4-AyCac,5959
+scraper2_hj3415/app/usecases/ingest/ingest_c106.py,sha256=pmGTO-Obp5Lw-a27BVII0eyxyBAIiFVok7xiBqLEJXk,4301
+scraper2_hj3415/app/usecases/ingest/ingest_c108.py,sha256=6iDhJAjzLnqHiNBKDvNLsQxrnJcG5I68SdE-dojUMJY,3817
+scraper2_hj3415-2.6.0.dist-info/entry_points.txt,sha256=jNGmOvBmptIUr9_XUMQOH4s6jNKCU51jOhKd31gOe8c,52
+scraper2_hj3415-2.6.0.dist-info/licenses/LICENSE,sha256=QBiVGQuKAESeCfQE344Ik2ex6g2zfYdu9WqrRWydxIs,1068
+scraper2_hj3415-2.6.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+scraper2_hj3415-2.6.0.dist-info/METADATA,sha256=Xw3KsE6SxvOs_CJvsXF2-x4ebhkAssA7qVD0uApNeBw,3516
+scraper2_hj3415-2.6.0.dist-info/RECORD,,

scraper2_hj3415-2.6.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+scraper2=scraper2_hj3415.cli:app
+

scraper2/adapters/out/playwright/browser.py

@@ -1,102 +0,0 @@
-# src/scraper2/adapters/out/playwright/session.py
-from __future__ import annotations
-from typing import Any
-from io import StringIO
-import pandas as pd
-from playwright.async_api import Page
-
-class PlaywrightBrowser:
-    def __init__(self, page: Page):
-        self.page = page
-
-    async def goto(self, url: str, timeout_ms: int = 10_000) -> None:
-        await self.page.goto(url, timeout=timeout_ms)
-
-    async def title(self) -> str:
-        return await self.page.title()
-
-    async def current_url(self) -> str:
-        return self.page.url
-
-    async def wait(self, selector: str, timeout_ms: int = 10_000) -> None:
-        await self.page.wait_for_selector(selector, timeout=timeout_ms, state="attached")
-
-    async def text(self, selector: str) -> str:
-        await self.wait(selector)
-        return (await self.page.locator(selector).first.text_content()) or ""
-
-    async def texts(self, selector: str) -> list[str]:
-        await self.wait(selector)
-        loc = self.page.locator(selector)
-        items = await loc.all()
-        out: list[str] = []
-        for it in items:
-            out.append((await it.text_content()) or "")
-        return out
-
-    async def text_first_by_text(self, needle: str) -> str:
-        return (await self.page.get_by_text(needle).first.text_content()) or ""
-
-    async def inner_text(self, selector: str) -> str:
-        await self.wait(selector)
-        return await self.page.locator(selector).first.inner_text()
-
-    async def click(self, selector: str) -> None:
-        await self.wait(selector)
-        await self.page.locator(selector).click()
-
-    async def table_records(
-            self,
-            table_selector: str,
-            *,
-            header: int | list[int] = 0
-    ) -> list[dict[str, Any]]:
-        await self.wait(table_selector)
-
-        table = self.page.locator(table_selector).first
-        html = await table.evaluate("el => el.outerHTML")  # <table> 포함
-        #print(html)
-
-        try:
-            df = pd.read_html(StringIO(html), header=header)[0]
-            #print(df.head(3))
-        except Exception as e:
-            # ImportError(lxml 없음), ValueError 등 모두 여기서 잡아서 원인 노출
-            raise RuntimeError(f"pd.read_html failed: {type(e).__name__}: {e}") from e
-
-        if header == 0:
-            if "항목" in df.columns:
-                df["항목"] = df["항목"].astype(str).str.replace("펼치기", "").str.strip()
-
-            df.columns = (
-                df.columns.astype(str)
-                .str.replace("연간컨센서스보기", "", regex=False)
-                .str.replace("연간컨센서스닫기", "", regex=False)
-                .str.replace("(IFRS연결)", "", regex=False)
-                .str.replace("(IFRS별도)", "", regex=False)
-                .str.replace("(GAAP개별)", "", regex=False)
-                .str.replace("(YoY)", "", regex=False)
-                .str.replace("(QoQ)", "", regex=False)
-                .str.replace("(E)", "", regex=False)
-                .str.replace(".", "", regex=False)
-                .str.strip()
-            )
-
-        # NaN -> None 처리
-        records: list[dict[str, Any]] = df.where(pd.notnull(df), None).to_dict(orient="records")
-        return records
-
-    async def outer_html(self, selector: str) -> str:
-        loc = self.page.locator(selector).first
-        return await loc.evaluate("el => el.outerHTML")
-
-    async def all_texts(self, selector: str) -> list[str]:
-        # selector는 css도 되고, "xpath=..." 도 됨
-        loc = self.page.locator(selector)
-        return await loc.all_text_contents()
-
-    async def outer_html_nth(self, selector: str, index: int) -> str:
-        loc = self.page.locator(selector).nth(index)
-        # index가 범위를 벗어나면 playwright가 에러를 내는데,
-        # 필요하면 여기서 더 친절한 에러로 감싸도 됨.
-        return await loc.evaluate("el => el.outerHTML")

scraper2/adapters/out/sinks/memory/__init__.py

@@ -1,15 +0,0 @@
-# scraper2/adapters/out/sinks/memory/__init__.py
-from .c101_memory_sink import MemoryC101Sink
-from .c103_memory_sink import MemoryC103Sink
-from .c104_memory_sink import MemoryC104Sink
-from .c106_memory_sink import MemoryC106Sink
-from .c108_memory_sink import MemoryC108Sink
-
-__all__ = [
-    "MemoryC101Sink",
-    "MemoryC103Sink",
-    "MemoryC104Sink",
-    "MemoryC106Sink",
-    "MemoryC108Sink",
-]
-

scraper2/adapters/out/sinks/memory/c101_memory_sink.py

@@ -1,26 +0,0 @@
-#scraper2/adapters/out/sinks/memory/c101_memory_sink.py
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Iterable, Optional
-
-from contracts.nfs.c101 import C101DTO
-from scraper2.adapters.out.sinks.memory.store import InMemoryStore
-from scraper2.app.ports.sinks.c101_sink_port import C101SinkPort
-
-_ENDPOINT = "c101"
-
-class MemoryC101Sink(C101SinkPort):
-    def __init__(self, store: InMemoryStore[C101DTO]):
-        self._store = store
-
-    async def write(self, dto: C101DTO, *, asof: Optional[datetime] = None) -> None:
-        await self._store.put(_ENDPOINT, dto.코드, dto)
-
-    async def write_many(
-        self,
-        dtos: Iterable[C101DTO],
-        *,
-        asof: Optional[datetime] = None,
-    ) -> None:
-        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))

scraper2/adapters/out/sinks/memory/c103_memory_sink.py

@@ -1,26 +0,0 @@
-# scraper2/adapters/out/sinks/memory/c103_memory_sink.py
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Iterable, Optional
-
-from contracts.nfs.c103 import C103DTO
-from scraper2.adapters.out.sinks.memory.store import InMemoryStore
-from scraper2.app.ports.sinks.c103_sink_port import C103SinkPort
-
-_ENDPOINT = "c103"
-
-class MemoryC103Sink(C103SinkPort):
-    def __init__(self, store: InMemoryStore[C103DTO]):
-        self._store = store
-
-    async def write(self, dto: C103DTO, *, asof: Optional[datetime] = None) -> None:
-        await self._store.put(_ENDPOINT, dto.코드, dto)
-
-    async def write_many(
-            self,
-            dtos: Iterable[C103DTO],
-            *,
-            asof: Optional[datetime] = None,
-    ) -> None:
-        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))

scraper2/adapters/out/sinks/memory/c104_memory_sink.py

@@ -1,26 +0,0 @@
-# scraper2/adapters/out/sinks/memory/c104_memory_sink.py
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Iterable, Optional
-
-from contracts.nfs.c104 import C104DTO
-from scraper2.adapters.out.sinks.memory.store import InMemoryStore
-from scraper2.app.ports.sinks.c104_sink_port import C104SinkPort
-
-_ENDPOINT = "c104"
-
-class MemoryC104Sink(C104SinkPort):
-    def __init__(self, store: InMemoryStore[C104DTO]):
-        self._store = store
-
-    async def write(self, dto: C104DTO, *, asof: Optional[datetime] = None) -> None:
-        await self._store.put(_ENDPOINT, dto.코드, dto)
-
-    async def write_many(
-            self,
-            dtos: Iterable[C104DTO],
-            *,
-            asof: Optional[datetime] = None,
-    ) -> None:
-        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))

scraper2/adapters/out/sinks/memory/c106_memory_sink.py

@@ -1,26 +0,0 @@
-#scraper2/adapters/out/sinks/memory/c106_memory_sink.py
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Iterable, Optional
-
-from contracts.nfs.c106 import C106DTO
-from scraper2.adapters.out.sinks.memory.store import InMemoryStore
-from scraper2.app.ports.sinks.c106_sink_port import C106SinkPort
-
-_ENDPOINT = "c106"
-
-class MemoryC106Sink(C106SinkPort):
-    def __init__(self, store: InMemoryStore[C106DTO]):
-        self._store = store
-
-    async def write(self, dto: C106DTO, *, asof: Optional[datetime] = None) -> None:
-        await self._store.put(_ENDPOINT, dto.코드, dto)
-
-    async def write_many(
-            self,
-            dtos: Iterable[C106DTO],
-            *,
-            asof: Optional[datetime] = None,
-    ) -> None:
-        await self._store.put_many(_ENDPOINT, ((d.코드, d) for d in dtos))