scraper2-hj3415 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

scraper2/app/usecases/fetch/fetch_c103.py

@@ -0,0 +1,103 @@
+# scraper2/app/usecases/fetch/fetch_c103.py
+from __future__ import annotations
+
+import math
+import asyncio
+import random
+from typing import Iterable, Any
+from contracts.nfs.c103 import C103DTO, ItemsMap
+from scraper2.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2.app.parsing.c103_parser import parse_c103_to_dict
+from collections import defaultdict
+
+
+BLOCK_KEYS = (
+    "손익계산서y", "손익계산서q",
+    "재무상태표y", "재무상태표q",
+    "현금흐름표y", "현금흐름표q",
+)
+
+def _clean(v: Any) -> Any:
+    if isinstance(v, float) and math.isnan(v):
+        return None
+    return v
+
+def _is_all_none(row: dict[str, Any]) -> bool:
+    # row는 '항목'을 제외한 값들만 들어있다고 가정
+    return all(v is None for v in row.values())
+
+
+def records_to_items_map(records: list[dict[str, Any]]) -> ItemsMap:
+    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+
+    # 1) 항목별로 row를 먼저 모은다
+    for r in records:
+        item = r.get("항목")
+        if not item:
+            continue
+        item = str(item).strip()
+
+        row = {k: _clean(v) for k, v in r.items() if k != "항목"}
+        grouped[item].append(row)
+
+    # 2) 규칙 적용해서 out 구성
+    out: ItemsMap = {}
+
+    for item, rows in grouped.items():
+        if len(rows) == 1:
+            # ✅ 규칙 1: 중복이 아니면 전부 None이어도 추가
+            out[item] = rows[0]
+            continue
+
+        # ✅ 규칙 2: 중복이면 all-None row는 제거
+        kept = [row for row in rows if not _is_all_none(row)]
+        if not kept:
+            continue  # 중복 전부 None이면 그룹 자체 제거
+
+        # 남은 것들만 suffix 부여 (첫 번째는 suffix 없음)
+        for idx, row in enumerate(kept, start=1):
+            key = item if idx == 1 else f"{item}_{idx}"
+            out[key] = row
+
+    return out
+
+
+class FetchC103:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> C103DTO | None:
+        async with self.factory.lease() as browser:
+            url = f"https://navercomp.wisereport.co.kr/v2/company/c1030001.aspx?cn=&cmp_cd={code}"
+            await browser.goto(url, timeout_ms=10_000)
+
+            jitter_sec = 1.0
+            if sleep_sec > 0:
+                delay = sleep_sec + random.uniform(0, jitter_sec)
+                await asyncio.sleep(delay)
+
+            parsed = await parse_c103_to_dict(browser)
+
+            if not parsed or all(not parsed.get(k) for k in BLOCK_KEYS):
+                return None
+
+            data: dict[str, Any] = {"코드": code}
+            for k in BLOCK_KEYS:
+                data[k] = records_to_items_map(parsed.get(k, []))
+
+            return C103DTO(**data)
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C103DTO | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+    ) -> list[C103DTO]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
+            return_exceptions=False,
+        )
+        return [r for r in results if r is not None]

scraper2/app/usecases/fetch/fetch_c104.py

@@ -0,0 +1,76 @@
+# scraper2/app/usecases/fetch/fetch_c104.py
+from __future__ import annotations
+
+import math
+import asyncio
+import random
+from typing import Iterable, Any
+from collections import Counter
+
+from contracts.nfs.c104 import C104DTO, ItemsMap
+from scraper2.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2.app.parsing.c104_parser import parse_c104_to_dict
+
+
+BLOCK_KEYS = (
+    "수익성y", "성장성y", "안정성y", "활동성y", "가치분석y",
+    "수익성q", "성장성q", "안정성q", "활동성q", "가치분석q",
+)
+
+def _clean(v: Any) -> Any:
+    if isinstance(v, float) and math.isnan(v):
+        return None
+    return v
+
+def records_to_items_map(records: list[dict[str, Any]]) -> ItemsMap:
+    """
+    records(list[dict]) -> ItemsMap(dict[item -> row])
+    - item(항목) 중복이면 _2, _3...
+    """
+    out: ItemsMap = {}
+    seen: Counter[str] = Counter()
+
+    for r in records:
+        item = r.get("항목")
+        if not item:
+            continue
+
+        item = str(item).strip()
+        seen[item] += 1
+        key = item if seen[item] == 1 else f"{item}_{seen[item]}"
+
+        out[key] = {k: _clean(v) for k, v in r.items() if k != "항목"}
+
+    return out
+
+class FetchC104:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> C104DTO | None:
+        async with self.factory.lease() as browser:
+            url = f"https://navercomp.wisereport.co.kr/v2/company/c1040001.aspx?cn=&cmp_cd={code}"
+            await browser.goto(url, timeout_ms=10_000)
+
+            jitter_sec = 1.0
+            if sleep_sec > 0:
+                delay = sleep_sec + random.uniform(0, jitter_sec)
+                await asyncio.sleep(delay)
+
+            parsed = await parse_c104_to_dict(browser)
+
+            if not parsed or all(not parsed.get(k) for k in BLOCK_KEYS):
+                return None
+
+            data: dict[str, Any] = {"코드": code}
+            for k in BLOCK_KEYS:
+                data[k] = records_to_items_map(parsed.get(k, []))
+
+            return C104DTO(**data)
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C104DTO | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(self, codes: Iterable[str], *, sleep_sec: float = 2.0) -> list[C104DTO]:
+        results = await asyncio.gather(*(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes))
+        return [r for r in results if r is not None]

scraper2/app/usecases/fetch/fetch_c106.py

@@ -0,0 +1,90 @@
+# scraper2/app/usecases/fetch/fetch_c106.py
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable
+
+from contracts.nfs.c106 import C106DTO, C106Block
+from scraper2.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2.app.parsing.c106_parser import parse_c106_header, parse_c106_table_to_metrics, normalize_c106_metrics
+
+from logging_hj3415 import logger
+
+class FetchC106:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> C106DTO | None:
+        async with self.factory.lease() as browser:
+            url = f"https://navercomp.wisereport.co.kr/v2/company/c1060001.aspx?cn=&cmp_cd={code}"
+            await browser.goto(url, timeout_ms=10_000)
+
+            jitter_sec = 1.0
+            if sleep_sec > 0:
+                delay = sleep_sec + random.uniform(0, jitter_sec)
+                await asyncio.sleep(delay)
+
+            company_names = await parse_c106_header(browser)
+
+            table_url = (
+                f"https://navercomp.wisereport.co.kr/v2/company/cF6002.aspx"
+                f"?cmp_cd={code}&finGubun=MAIN&sec_cd=FG000&frq="
+            )
+
+            stage = "init"
+            try:
+                stage = "goto_q"
+                await browser.goto(table_url+'q', timeout_ms=10_000)
+                await asyncio.sleep(1)
+
+                stage = "parse_q"
+                q = await parse_c106_table_to_metrics(browser, company_names)
+
+                stage = "goto_y"
+                await browser.goto(table_url+'y', timeout_ms=10_000)
+                await asyncio.sleep(1)
+
+                stage = "parse_y"
+                y = await parse_c106_table_to_metrics(browser, company_names)
+
+                q_norm = normalize_c106_metrics(q)
+                y_norm = normalize_c106_metrics(y)
+
+                return C106DTO(
+                    코드=code,
+                    q=C106Block(**q_norm),
+                    y=C106Block(**y_norm),
+                )
+            except Exception as e:
+                title = ""
+                try:
+                    title = await browser.title()
+                except Exception:
+                    pass
+
+                logger.bind(
+                    endpoint="c106",
+                    code=code,
+                    stage=stage,
+                    page_title=title,
+                    url_q=table_url + "q",
+                ).warning("c106 skipped (unstable page/table)")
+                return None
+
+
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C106DTO | None:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+    ) -> list[C106DTO]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
+            return_exceptions=False,
+        )
+        return [r for r in results if r is not None]

scraper2/app/usecases/fetch/fetch_c108.py

@@ -0,0 +1,49 @@
+# scraper2/app/usecases/fetch/fetch_c108.py
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable
+
+from contracts.nfs.c108 import C108DTO
+from scraper2.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2.app.parsing.c108_parser import parse_c108_to_dicts
+
+
+class FetchC108:
+    def __init__(self, factory: BrowserFactoryPort):
+        self.factory = factory
+
+    async def _fetch_one(self, code: str, *, sleep_sec: float) -> list[C108DTO]:
+        async with self.factory.lease() as browser:
+            url = f"https://navercomp.wisereport.co.kr/v2/company/c1080001.aspx?cn=&cmp_cd={code}"
+            await browser.goto(url, timeout_ms=10_000)
+
+            # (C101과 동일한 형태로) sleep + jitter
+            jitter_sec = 1.0
+            if sleep_sec > 0:
+                delay = sleep_sec + random.uniform(0, jitter_sec)
+                await asyncio.sleep(delay)
+
+            c108_dicts = await parse_c108_to_dicts(browser)
+
+            return [C108DTO(**{**x, "코드": code}) for x in c108_dicts]
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> list[C108DTO]:
+        return await self._fetch_one(code, sleep_sec=sleep_sec)
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+    ) -> list[C108DTO]:
+        results = await asyncio.gather(
+            *(self._fetch_one(c, sleep_sec=sleep_sec) for c in codes),
+            return_exceptions=False,
+        )
+        # list[list[C108DTO]] -> flat list[C108DTO]
+        out: list[C108DTO] = []
+        for chunk in results:
+            out.extend(chunk)
+        return out

scraper2/app/usecases/ingest/ingest_c101.py

@@ -0,0 +1,36 @@
+# scraper2/app/usecases/ingest/ingest_c101.py
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional, Iterable
+
+from contracts.nfs.c101 import C101DTO
+from scraper2.app.usecases.fetch.fetch_c101 import FetchC101
+from scraper2.app.ports.sinks.c101_sink_port import C101SinkPort
+from scraper2.app.ports.ingest_port import IngestPort
+
+def _utcnow():
+    return datetime.now(timezone.utc)
+
+class IngestC101(IngestPort):
+    def __init__(self, fetch: FetchC101, sink: C101SinkPort):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C101DTO:
+        dto = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        await self.sink.write(dto)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C101DTO]:
+        batch_asof = asof or _utcnow()
+
+        dtos = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        await self.sink.write_many(dtos, asof=batch_asof)
+        return dtos

scraper2/app/usecases/ingest/ingest_c103.py

@@ -0,0 +1,37 @@
+# scraper2/app/usecases/ingest/ingest_c103.py
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional, Iterable
+
+from contracts.nfs.c103 import C103DTO
+from scraper2.app.usecases.fetch.fetch_c103 import FetchC103
+from scraper2.app.ports.sinks.c103_sink_port import C103SinkPort
+from scraper2.app.ports.ingest_port import IngestPort
+
+
+def _utcnow():
+    return datetime.now(timezone.utc)
+
+class IngestC103(IngestPort):
+    def __init__(self, fetch: FetchC103, sink: C103SinkPort):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C103DTO:
+        dto = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        await self.sink.write(dto)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C103DTO]:
+        batch_asof = asof or _utcnow()
+
+        dtos = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        await self.sink.write_many(dtos, asof=batch_asof)
+        return dtos

scraper2/app/usecases/ingest/ingest_c104.py

@@ -0,0 +1,37 @@
+# scraper2/app/usecases/ingest/ingest_c104.py
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional, Iterable
+
+from contracts.nfs.c104 import C104DTO
+from scraper2.app.usecases.fetch.fetch_c104 import FetchC104
+from scraper2.app.ports.sinks.c104_sink_port import C104SinkPort
+from scraper2.app.ports.ingest_port import IngestPort
+
+
+def _utcnow():
+    return datetime.now(timezone.utc)
+
+class IngestC104(IngestPort):
+    def __init__(self, fetch: FetchC104, sink: C104SinkPort):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C104DTO:
+        dto = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        await self.sink.write(dto)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C104DTO]:
+        batch_asof = asof or _utcnow()
+
+        dtos = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        await self.sink.write_many(dtos, asof=batch_asof)
+        return dtos

scraper2/app/usecases/ingest/ingest_c106.py

@@ -0,0 +1,38 @@
+# scraper2/app/usecases/ingest/ingest_c106.py
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional, Iterable
+
+from contracts.nfs.c106 import C106DTO
+from scraper2.app.usecases.fetch.fetch_c106 import FetchC106
+from scraper2.app.ports.sinks.c106_sink_port import C106SinkPort
+from scraper2.app.ports.ingest_port import IngestPort
+
+
+def _utcnow():
+    return datetime.now(timezone.utc)
+
+class IngestC106(IngestPort):
+    def __init__(self, fetch: FetchC106, sink: C106SinkPort):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C106DTO | None:
+        dto = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        if dto is not None:
+            await self.sink.write(dto)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C106DTO]:
+        batch_asof = asof or _utcnow()
+
+        dtos = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        await self.sink.write_many(dtos, asof=batch_asof)
+        return dtos

scraper2/app/usecases/ingest/ingest_c108.py

@@ -0,0 +1,39 @@
+# scraper2/app/usecases/ingest/ingest_c108.py
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional, Iterable
+
+from contracts.nfs.c108 import C108DTO
+from scraper2.app.usecases.fetch.fetch_c108 import FetchC108
+from scraper2.app.ports.sinks.c108_sink_port import C108SinkPort
+from scraper2.app.ports.ingest_port import IngestPort
+
+
+def _utcnow():
+    return datetime.now(timezone.utc)
+
+class IngestC108(IngestPort):
+    def __init__(self, fetch: FetchC108, sink: C108SinkPort):
+        self.fetch = fetch
+        self.sink = sink
+
+    async def execute(self, code: str, *, sleep_sec: float = 2.0) -> C108DTO | None:
+        dto = await self.fetch.execute(code, sleep_sec=sleep_sec)
+        if dto is not None:
+            await self.sink.write(dto)
+        return dto
+
+    async def execute_many(
+        self,
+        codes: Iterable[str],
+        *,
+        sleep_sec: float = 2.0,
+        asof: Optional[datetime] = None,
+    ) -> list[C108DTO]:
+        batch_asof = asof or _utcnow()
+
+        dtos = await self.fetch.execute_many(codes, sleep_sec=sleep_sec)
+        await self.sink.write_many(dtos, asof=batch_asof)
+        return dtos
+