scraper2-hj3415 1.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

scraper2/adapters/out/sinks/mongo/c106_mongo_sink.py

@@ -0,0 +1,41 @@
+# scraper2/adapters/out/sinks/mongo/c106_mongo_sink.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional
+
+from pymongo.asynchronous.database import AsyncDatabase
+
+from contracts.nfs.c106 import C106DTO
+from scraper2.app.ports.sinks.c106_sink_port import C106SinkPort
+
+from db2.nfs import (
+    upsert_latest,
+    upsert_latest_many,
+    insert_snapshot,
+    insert_snapshots_many,
+)
+
+_ENDPOINT = "c106"
+
+
+class MongoC106Sink(C106SinkPort):
+    def __init__(self, db: AsyncDatabase):
+        self._db = db
+
+    async def write(self, dto: C106DTO, *, asof: Optional[datetime] = None) -> None:
+        await upsert_latest(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+        await insert_snapshot(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+
+    async def write_many(
+        self,
+        dtos: Iterable[C106DTO],
+        *,
+        asof: Optional[datetime] = None,
+    ) -> None:
+        dtos_list = list(dtos)
+        if not dtos_list:
+            return
+
+        await upsert_latest_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)
+        await insert_snapshots_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)

scraper2/adapters/out/sinks/mongo/c108_mongo_sink.py

@@ -0,0 +1,41 @@
+# scraper2/adapters/out/sinks/mongo/c108_mongo_sink.py
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Iterable, Optional
+
+from pymongo.asynchronous.database import AsyncDatabase
+
+from contracts.nfs.c108 import C108DTO
+from scraper2.app.ports.sinks.c108_sink_port import C108SinkPort
+
+from db2.nfs import (
+    upsert_latest,
+    upsert_latest_many,
+    insert_snapshot,
+    insert_snapshots_many,
+)
+
+_ENDPOINT = "c108"
+
+
+class MongoC108Sink(C108SinkPort):
+    def __init__(self, db: AsyncDatabase):
+        self._db = db
+
+    async def write(self, dto: C108DTO, *, asof: Optional[datetime] = None) -> None:
+        await upsert_latest(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+        await insert_snapshot(self._db, endpoint=_ENDPOINT, dto=dto, asof=asof)
+
+    async def write_many(
+        self,
+        dtos: Iterable[C108DTO],
+        *,
+        asof: Optional[datetime] = None,
+    ) -> None:
+        dtos_list = list(dtos)
+        if not dtos_list:
+            return
+
+        await upsert_latest_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)
+        await insert_snapshots_many(self._db, endpoint=_ENDPOINT, dtos=dtos_list, asof=asof)

scraper2/app/composition.py

@@ -0,0 +1,195 @@
+# scraper2/app/composition.py
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Literal
+
+from pymongo.asynchronous.database import AsyncDatabase
+
+from scraper2.app.ports.browser.browser_factory_port import BrowserFactoryPort
+from scraper2.adapters.out.playwright.browser_factory import PlaywrightBrowserFactory
+
+from scraper2.app.usecases.fetch.fetch_c101 import FetchC101
+from scraper2.app.usecases.fetch.fetch_c103 import FetchC103
+from scraper2.app.usecases.fetch.fetch_c104 import FetchC104
+from scraper2.app.usecases.fetch.fetch_c106 import FetchC106
+from scraper2.app.usecases.fetch.fetch_c108 import FetchC108
+
+from scraper2.app.ports.ingest_port import IngestPort
+from scraper2.app.usecases.ingest.ingest_c101 import IngestC101
+from scraper2.app.usecases.ingest.ingest_c103 import IngestC103
+from scraper2.app.usecases.ingest.ingest_c104 import IngestC104
+from scraper2.app.usecases.ingest.ingest_c106 import IngestC106
+from scraper2.app.usecases.ingest.ingest_c108 import IngestC108
+
+from scraper2.adapters.out.sinks.memory.store import InMemoryStore
+from scraper2.adapters.out.sinks.memory.c101_memory_sink import MemoryC101Sink
+from scraper2.adapters.out.sinks.memory.c103_memory_sink import MemoryC103Sink
+from scraper2.adapters.out.sinks.memory.c104_memory_sink import MemoryC104Sink
+from scraper2.adapters.out.sinks.memory.c106_memory_sink import MemoryC106Sink
+from scraper2.adapters.out.sinks.memory.c108_memory_sink import MemoryC108Sink
+
+from scraper2.adapters.out.sinks.mongo.c101_mongo_sink import MongoC101Sink
+from scraper2.adapters.out.sinks.mongo.c103_mongo_sink import MongoC103Sink
+from scraper2.adapters.out.sinks.mongo.c104_mongo_sink import MongoC104Sink
+from scraper2.adapters.out.sinks.mongo.c106_mongo_sink import MongoC106Sink
+from scraper2.adapters.out.sinks.mongo.c108_mongo_sink import MongoC108Sink
+
+from scraper2.app.ports.sinks.c101_sink_port import C101SinkPort
+from scraper2.app.ports.sinks.c103_sink_port import C103SinkPort
+from scraper2.app.ports.sinks.c104_sink_port import C104SinkPort
+from scraper2.app.ports.sinks.c106_sink_port import C106SinkPort
+from scraper2.app.ports.sinks.c108_sink_port import C108SinkPort
+
+from db2.mongo import Mongo
+
+SinkKind = Literal["memory", "mongo"]
+
+
+def _env_bool(key: str, default: bool) -> bool:
+    v = os.getenv(key)
+    return default if v is None else v.strip().lower() in {"1", "true", "yes", "y", "on"}
+
+
+def _env_int(key: str, default: int) -> int:
+    v = os.getenv(key)
+    if v is None:
+        return default
+    try:
+        return int(v)
+    except ValueError:
+        return default
+
+
+def build_browser_factory() -> BrowserFactoryPort:
+    return PlaywrightBrowserFactory(
+        headless=_env_bool("SCRAPER_HEADLESS", True),
+        timeout_ms=_env_int("SCRAPER_TIMEOUT_MS", 20_000),
+        max_concurrency=_env_int("SCRAPER_MAX_CONCURRENCY", 2),
+    )
+
+
+# -------------------------
+# Bundles
+# -------------------------
+
+@dataclass(frozen=True)
+class FetchUsecases:
+    c101: FetchC101
+    c103: FetchC103
+    c104: FetchC104
+    c106: FetchC106
+    c108: FetchC108
+
+
+@dataclass(frozen=True)
+class Sinks:
+    c101: C101SinkPort
+    c103: C103SinkPort
+    c104: C104SinkPort
+    c106: C106SinkPort
+    c108: C108SinkPort
+
+
+@dataclass(frozen=True)
+class IngestUsecases:
+    c101: IngestPort
+    c103: IngestPort
+    c104: IngestPort
+    c106: IngestPort
+    c108: IngestPort
+
+
+@dataclass(frozen=True)
+class Usecases:
+    fetch: FetchUsecases
+    ingest: IngestUsecases
+    sinks: Sinks
+    store: InMemoryStore | None = None      # ✅ memory일 때만
+    mongo: Mongo | None = None              # ✅ mongo일 때만
+    db: AsyncDatabase | None = None         # ✅ mongo일 때만
+
+
+# -------------------------
+# builders
+# -------------------------
+
+def build_fetch_usecases(*, factory: BrowserFactoryPort) -> FetchUsecases:
+    return FetchUsecases(
+        c101=FetchC101(factory=factory),
+        c103=FetchC103(factory=factory),
+        c104=FetchC104(factory=factory),
+        c106=FetchC106(factory=factory),
+        c108=FetchC108(factory=factory),
+    )
+
+
+@dataclass(frozen=True)
+class MemoryBundle:
+    store: InMemoryStore
+    sinks: Sinks
+
+
+def build_memory_bundle() -> MemoryBundle:
+    store = InMemoryStore()
+    sinks = Sinks(
+        c101=MemoryC101Sink(store),
+        c103=MemoryC103Sink(store),
+        c104=MemoryC104Sink(store),
+        c106=MemoryC106Sink(store),
+        c108=MemoryC108Sink(store),
+    )
+    return MemoryBundle(store=store, sinks=sinks)
+
+# ---- mongo bundle ----
+
+@dataclass(frozen=True)
+class MongoBundle:
+    mongo: Mongo
+    db: AsyncDatabase
+    sinks: Sinks
+
+
+def build_mongo_bundle() -> MongoBundle:
+    mongo = Mongo()        # settings는 db2가 env로 읽음 (DB2_MONGO_URI 등)
+    db = mongo.get_db()
+    sinks = Sinks(
+        c101=MongoC101Sink(db),
+        c103=MongoC103Sink(db),
+        c104=MongoC104Sink(db),
+        c106=MongoC106Sink(db),
+        c108=MongoC108Sink(db),
+    )
+    return MongoBundle(mongo=mongo, db=db, sinks=sinks)
+
+
+def build_ingest_usecases(*, fetch: FetchUsecases, sinks: Sinks) -> IngestUsecases:
+    return IngestUsecases(
+        c101=IngestC101(fetch=fetch.c101, sink=sinks.c101),
+        c103=IngestC103(fetch=fetch.c103, sink=sinks.c103),
+        c104=IngestC104(fetch=fetch.c104, sink=sinks.c104),
+        c106=IngestC106(fetch=fetch.c106, sink=sinks.c106),
+        c108=IngestC108(fetch=fetch.c108, sink=sinks.c108),
+    )
+
+
+def build_usecases(
+    *,
+    factory: BrowserFactoryPort | None = None,
+    sink_kind: SinkKind = "memory",
+) -> Usecases:
+    factory = factory or build_browser_factory()
+    fetch = build_fetch_usecases(factory=factory)
+
+    if sink_kind == "memory":
+        bundle = build_memory_bundle()
+        ingest = build_ingest_usecases(fetch=fetch, sinks=bundle.sinks)
+        return Usecases(fetch=fetch, ingest=ingest, sinks=bundle.sinks, store=bundle.store)
+
+    if sink_kind == "mongo":
+        bundle = build_mongo_bundle()
+        ingest = build_ingest_usecases(fetch=fetch, sinks=bundle.sinks)
+        return Usecases(fetch=fetch, ingest=ingest, sinks=bundle.sinks, mongo=bundle.mongo, db=bundle.db)
+
+    raise ValueError(f"Unknown sink_kind: {sink_kind}")

scraper2/app/parsing/_converters.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+import re
+from typing import Iterable
+
+
+_EMPTY_VALUES = {"", "-", "N/A", "NA", "null", "None"}
+
+
+def normalize(s: str | None) -> str:
+    if s is None:
+        return ""
+    return s.strip()
+
+
+def _is_empty(s: str) -> bool:
+    return s in _EMPTY_VALUES
+
+
+def to_int(s: str | None) -> int | None:
+    """
+    C101 파서용 정수 변환기
+
+    처리 규칙:
+    - None / '' / '-' / 'N/A' → None
+    - ',', '주', '원', '주식' 제거
+    - '1,234' → 1234
+    """
+    s = normalize(s)
+    if _is_empty(s):
+        return None
+
+    for ch in (",", "원", "주", "주식"):
+        s = s.replace(ch, "")
+
+    try:
+        return int(s)
+    except ValueError:
+        return None
+
+
+def to_float(s: str | None) -> float | None:
+    """
+    C101 파서용 실수 변환기
+
+    처리 규칙:
+    - None / '' / '-' / 'N/A' → None
+    - ',', '%', '원' 제거
+    - '12.34%' → 12.34
+    """
+    s = normalize(s)
+    if _is_empty(s):
+        return None
+
+    for ch in (",", "%", "원"):
+        s = s.replace(ch, "")
+
+    try:
+        return float(s)
+    except ValueError:
+        return None
+
+
+def parse_won(text: str) -> int:
+    """
+    한국 화폐 표현 문자열을 숫자로 변환 (조원, 억원, 만원, 원, 억 등 처리)
+    """
+    units = {
+        "조원": 1_000_000_000_000,
+        "억원": 100_000_000,
+        "억":   100_000_000,
+        "만원": 10_000,
+        "원": 1,
+    }
+
+    text = text.replace(",", "").strip()
+    match = re.match(r"([-+]?[0-9]*\.?[0-9]+)([가-힣]+)", text)
+
+    if not match:
+        raise ValueError(f"형식이 잘못된 금액 문자열: {text}")
+
+    number, unit = match.groups()
+    if unit not in units:
+        raise ValueError(f"알 수 없는 단위: {unit}")
+
+    return int(float(number) * units[unit])

scraper2/app/parsing/_normalize.py

@@ -0,0 +1,134 @@
+# scraper2/app/parsing/_normalize.py
+from __future__ import annotations
+
+import re
+from collections import Counter
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+# -----------------------------
+# 1) 항목(행의 "항목" 값) 정규화
+# -----------------------------
+_IFRS_PATTERN = re.compile(r"\(IFRS[^)]*\)")
+_ETC_PAREN_PATTERN = re.compile(r"\((E|YoY|QoQ)[^)]*\)")
+_BRACKET_PATTERN = re.compile(r"\[[^\]]*\]")  # [구K-IFRS] 등
+_EXTRA_WORDS_PATTERN = re.compile(r"(펼치기|연간컨센서스보기|연간컨센서스닫기)")
+_ALL_PAREN_PATTERN = re.compile(r"\([^)]*\)")  # ★ 모든 괄호 제거
+
+def normalize_c1034_item(text: str | None) -> str:
+    """
+    C103 항목명(행 값) 정규화
+    - 펼치기/컨센서스 제거
+    - 모든 괄호 내용 제거 (발표기준, 연결, 개별 등 포함)
+    - [구K-IFRS] 제거
+    - '*' 제거
+    - 공백 정리
+    """
+    if not text:
+        return ""
+
+    s = str(text)
+
+    # 1) 불필요한 키워드
+    s = _EXTRA_WORDS_PATTERN.sub("", s)
+
+    # 2) 대괄호 제거
+    s = _BRACKET_PATTERN.sub("", s)
+
+    # 3) 모든 괄호 제거 (중요)
+    s = _ALL_PAREN_PATTERN.sub("", s)
+
+    # 4) 별표 제거
+    s = s.replace("*", "")
+
+    # 5) 공백 정리
+    s = re.sub(r"\s+", " ", s).strip()
+
+    return s
+
+
+# -----------------------------
+# 2) 컬럼명 정규화
+# -----------------------------
+_COL_PAREN_PATTERN = re.compile(r"\((IFRS[^)]*|E|YoY|QoQ)[^)]*\)")
+_COL_EXTRA_WORDS = re.compile(r"(연간컨센서스보기|연간컨센서스닫기)")
+_COL_DOTNUM = re.compile(r"\.\d+$")  # pandas 중복 컬럼 suffix 제거용 (.1, .2 ...)
+_COL_MULTI_SPACE = re.compile(r"\s+")
+
+def normalize_c1034_col(col: str | None) -> str:
+    """
+    C103 컬럼명 정규화
+    예)
+      "2024/12 (IFRS연결)  연간컨센서스보기" -> "2024/12"
+      "2025/12(E) (IFRS연결)  연간컨센서스닫기" -> "2025/12"
+      "전년대비 (YoY)" -> "전년대비"
+      "전년대비 (YoY).1" -> "전년대비"  (중복은 후처리에서 _2/_3로 자동 분리)
+    """
+    if col is None:
+        return ""
+
+    s = str(col)
+
+    # 1) pandas가 붙인 .1 같은 suffix 제거 (정규화 충돌은 후단에서 처리)
+    s = _COL_DOTNUM.sub("", s)
+
+    # 2) 컨센서스 문구 제거
+    s = _COL_EXTRA_WORDS.sub("", s)
+
+    # 3) 괄호 주석 제거: (IFRS...), (E), (YoY), (QoQ)
+    s = _COL_PAREN_PATTERN.sub("", s)
+
+    # 4) 공백 정리
+    s = _COL_MULTI_SPACE.sub(" ", s).strip()
+
+    return s
+
+
+def _dedupe_columns(cols: list[str]) -> list[str]:
+    """
+    정규화 후 중복 컬럼명이 생기면 자동으로 _2, _3 ... 붙여서 유니크하게 만든다.
+    예) ["전년대비", "전년대비"] -> ["전년대비", "전년대비_2"]
+    """
+    seen: Counter[str] = Counter()
+    out: list[str] = []
+    for c in cols:
+        c = c or ""
+        seen[c] += 1
+        if seen[c] == 1:
+            out.append(c)
+        else:
+            out.append(f"{c}_{seen[c]}")
+    return out
+
+
+# -----------------------------
+# 3) DataFrame 전체 정규화 + records 변환
+# -----------------------------
+def normalize_c1034_df(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    - 컬럼명 전체 정규화
+    - '항목' 값 정규화
+    - NaN -> None
+    - 중복 컬럼명 자동 분리(_2/_3)
+    """
+    if df is None or df.empty:
+        return df
+
+    df = df.copy()
+
+    # 컬럼명 정규화 + 중복 방지
+    norm_cols = [normalize_c1034_col(c) for c in df.columns.astype(str).tolist()]
+    df.columns = _dedupe_columns(norm_cols)
+
+    # 항목 값 정규화
+    if "항목" in df.columns:
+        df["항목"] = df["항목"].map(normalize_c1034_item)
+
+    # NaN -> None
+    df = df.replace({np.nan: None})
+    return df
+
+

scraper2/app/parsing/c101_parser.py

@@ -0,0 +1,143 @@
+# scraper2/app/parsing/c101_parser.py
+from __future__ import annotations
+
+from scraper2.app.parsing._converters import to_int, to_float, normalize, parse_won
+from scraper2.app.ports.browser.browser_port import BrowserPort
+from typing import Any
+
+
+class C101ParseError(RuntimeError):
+    pass
+
+
+def _after_colon(s: str) -> str:
+    # "업종: XXX" 같은 케이스
+    parts = s.split(":")
+    return parts[1].strip() if len(parts) > 1 else s.strip()
+
+
+async def parse_c101_to_dict(browser: BrowserPort) -> dict[str, Any] | None:
+    """
+    - BrowserPort만 사용
+    - dict만 반환
+    - 실패 시 None (기존 동작 유지)
+    """
+
+    # 날짜 파싱: 텍스트 검색 기반
+    raw_date_str = await browser.text_first_by_text("[기준:")
+    if not raw_date_str:
+        return None
+    날짜 = raw_date_str.replace("[기준:", "").replace("]", "").strip()
+
+    # 1) 재무정보 (1st table)
+    # 여기선 "tbody 존재"만 기다리면 됨
+    await browser.wait("#pArea > div.wrapper-table > div > table > tbody")
+
+    종목명 = normalize(await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(1) > td > dl > dt:nth-child(1) > span"
+    ))
+    코드 = normalize(await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(1) > td > dl > dt:nth-child(1) > b"
+    ))
+    업종_raw = await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(1) > td > dl > dt:nth-child(4)"
+    )
+    업종 = _after_colon(업종_raw)
+
+    eps = to_int(await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(3) > td > dl > dt:nth-child(1) > b"
+    ))
+    bps = to_int(await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(3) > td > dl > dt:nth-child(2) > b"
+    ))
+    per = to_float(await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(3) > td > dl > dt:nth-child(3) > b"
+    ))
+    업종per = to_float(await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(3) > td > dl > dt:nth-child(4) > b"
+    ))
+    pbr = to_float(await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(3) > td > dl > dt:nth-child(5) > b"
+    ))
+    배당수익률 = to_float(await browser.text(
+        "#pArea > div.wrapper-table > div > table > tbody "
+        "tr:nth-child(3) > td > dl > dt:nth-child(6) > b"
+    ))
+
+    # 2) 주가 정보 (2nd table)
+    await browser.wait("#cTB11 > tbody")
+
+    주가 = to_int(await browser.text("#cTB11 > tbody tr:nth-child(1) > td > strong"))
+
+    전일대비_raw = await browser.text("#cTB11 > tbody tr:nth-child(1) > td > span:nth-child(2)")
+    전일대비 = to_int(전일대비_raw.replace("원", ""))
+
+    수익률_raw = await browser.text("#cTB11 > tbody tr:nth-child(1) > td > span:nth-child(3)")
+    수익률 = to_float(수익률_raw.replace("%", ""))
+
+    최고최저52 = await browser.text("#cTB11 > tbody tr:nth-child(2) > td")
+    최고52, 최저52 = (to_int(x.strip().replace("원", "")) for x in 최고최저52.split("/"))
+
+    거래량거래대금 = await browser.text("#cTB11 > tbody tr:nth-child(4) > td")
+    거래량_str, 거래대금_str = (x.strip() for x in 거래량거래대금.split("/"))
+    거래량 = to_int(거래량_str.replace("주", ""))
+    거래대금 = parse_won(거래대금_str)
+
+    시가총액 = parse_won(await browser.text("#cTB11 > tbody tr:nth-child(5) > td"))
+    베타52주 = to_float(await browser.text("#cTB11 > tbody tr:nth-child(6) > td"))
+
+    발행주식유동비율 = await browser.text("#cTB11 > tbody tr:nth-child(7) > td")
+    발행주식_str, 유동비율_str = (x.strip() for x in 발행주식유동비율.split("/"))
+    발행주식 = to_int(발행주식_str.replace("주", ""))
+    유동비율 = to_float(유동비율_str.replace("%", ""))
+
+    외국인지분율 = to_float((await browser.text("#cTB11 > tbody tr:nth-child(8) > td")).replace("%", ""))
+
+    수익률1M3M6M1Y = await browser.text("#cTB11 > tbody tr:nth-child(9) > td")
+    수익률1M, 수익률3M, 수익률6M, 수익률1Y = (
+        to_float(x.strip().replace("%", "")) for x in 수익률1M3M6M1Y.split("/")
+    )
+
+    # 3) 개요
+    # ul 아래 li들을 전부 읽어서 합치기
+    await browser.wait("#wrapper > div:nth-child(6) > div.cmp_comment > ul")
+    li_texts = await browser.texts("#wrapper > div:nth-child(6) > div.cmp_comment > ul li")
+    개요 = "".join(t.strip() for t in li_texts if t and t.strip())
+
+    return {
+        "종목명": 종목명,
+        "코드": 코드,
+        "날짜": 날짜,
+        "업종": 업종,
+        "eps": eps,
+        "bps": bps,
+        "per": per,
+        "업종per": 업종per,
+        "pbr": pbr,
+        "배당수익률": 배당수익률,
+        "주가": 주가,
+        "전일대비": 전일대비,
+        "수익률": 수익률,
+        "최고52": 최고52,
+        "최저52": 최저52,
+        "거래량": 거래량,
+        "거래대금": 거래대금,
+        "시가총액": 시가총액,
+        "베타52주": 베타52주,
+        "발행주식": 발행주식,
+        "유동비율": 유동비율,
+        "외국인지분율": 외국인지분율,
+        "수익률1M": 수익률1M,
+        "수익률3M": 수익률3M,
+        "수익률6M": 수익률6M,
+        "수익률1Y": 수익률1Y,
+        "개요": 개요,
+    }