scraper2-hj3415 1.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

scraper2/main.py

@@ -0,0 +1,257 @@
+# scraper2/main.py
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Literal, cast
+
+import typer
+from datetime import datetime, timezone
+
+from scraper2.app.composition import build_usecases
+from scraper2.app.ports.ingest_port import IngestPort
+
+from logging_hj3415 import setup_logging
+
+setup_logging()
+
+Endpoint = Literal["c101", "c103", "c104", "c106", "c108", "all"]
+Sink = Literal["memory", "mongo"]
+
+app = typer.Typer(no_args_is_help=True)
+
+nfs_app = typer.Typer(no_args_is_help=True, help="NFS 수집/저장")
+mi_app = typer.Typer(no_args_is_help=True, help="(reserved) MI commands")
+
+app.add_typer(nfs_app, name="nfs")
+app.add_typer(mi_app, name="mi")
+
+
+# -------------------------
+# small helpers
+# -------------------------
+
+def _endpoint_list(ep: Endpoint) -> list[str]:
+    return ["c101", "c103", "c104", "c106", "c108"] if ep == "all" else [ep]
+
+
+async def _maybe_await_close(obj: Any) -> None:
+    close = getattr(obj, "close", None)
+    if close is None:
+        return
+    out = close()
+    if asyncio.iscoroutine(out):
+        await out
+
+
+async def _mongo_bootstrap(db) -> None:
+    from db2.nfs import ensure_indexes
+    from db2.settings import get_settings
+
+    s = get_settings()
+    await ensure_indexes(db, snapshot_ttl_days=s.SNAPSHOT_TTL_DAYS)
+
+
+async def _load_codes_from_universe(db, *, universe: str) -> list[str]:
+    """
+    db2.universe에 저장된 universe_latest에서 codes 로드.
+    (네 db2 API 명에 맞춰 조정하면 됨)
+    """
+    from db2.universe import get_universe_latest  # 네가 가진 API
+
+    doc = await get_universe_latest(db, universe=universe)
+    if not doc:
+        return []
+
+    # doc 형태가 {"items":[{code,name,...}, ...]} 혹은 {"payload":{"items":[...]}} 일 수 있어서 방어
+    data = doc
+    if isinstance(data, dict) and "payload" in data and isinstance(data["payload"], dict):
+        data = data["payload"]
+    if isinstance(data, dict) and "items" in data:
+        data = data["items"]
+
+    if not isinstance(data, list):
+        return []
+
+    codes: list[str] = []
+    for row in data:
+        if not isinstance(row, dict):
+            continue
+        code = str(row.get("code") or "").strip()
+        if code:
+            codes.append(code)
+    return codes
+
+
+import time
+
+async def _run_ingest_with_progress(
+    *,
+    ucs: Any,
+    endpoint: Endpoint,
+    codes: list[str],
+    sleep_sec: float,
+    show: bool,
+    show_n: int,
+    chunk_size: int = 10,
+    progress_every: int = 1,
+) -> None:
+    total = len(codes)
+    if total == 0:
+        typer.echo("(no codes)")
+        return
+
+    t0 = time.perf_counter()   # ✅ 시작 시각
+    run_asof = datetime.now(timezone.utc)
+
+    def _chunks(xs: list[str], n: int):
+        for i in range(0, len(xs), n):
+            yield xs[i:i + n]
+
+    async def _run_one_endpoint(ep: str) -> None:
+        ingest_uc = cast(IngestPort, getattr(ucs.ingest, ep))
+
+        ok = 0
+        fail = 0
+
+        typer.echo(f"\n=== START: {ep} === total={total}, chunk_size={chunk_size}")
+
+        for idx, batch in enumerate(_chunks(codes, chunk_size), start=1):
+            try:
+                results = await ingest_uc.execute_many(batch, sleep_sec=sleep_sec, asof=run_asof)
+                ok += sum(1 for r in results if r is not None)
+            except Exception as e:
+                fail += len(batch)
+                typer.echo(f"[WARN] batch failed: {e!r}")
+
+            done = min(idx * chunk_size, total)
+
+            if progress_every > 0 and (idx % progress_every == 0 or done == total):
+                typer.echo(f"progress: {done}/{total} (ok={ok}, fail={fail})")
+
+        typer.echo(f"=== DONE: {ep} === ok={ok}, fail={fail}, total={total}")
+
+    # --- 실제 실행 ---
+    for ep in _endpoint_list(endpoint):
+        await _run_one_endpoint(ep)
+
+    elapsed = time.perf_counter() - t0   # ✅ 종료 시각
+    typer.echo(f"\n⏱ elapsed time: {_format_elapsed(elapsed)}")
+
+def _dto_to_pretty(obj: Any) -> str:
+    # pydantic v2 우선
+    if hasattr(obj, "model_dump_json"):
+        return obj.model_dump_json(indent=2, by_alias=False)
+    if hasattr(obj, "model_dump"):
+        import json
+        return json.dumps(obj.model_dump(), ensure_ascii=False, indent=2)
+    # dict fallback
+    if isinstance(obj, dict):
+        import json
+        return json.dumps(obj, ensure_ascii=False, indent=2, default=str)
+    return str(obj)
+
+def _format_elapsed(sec: float) -> str:
+    if sec < 60:
+        return f"{sec:.1f}s"
+    m, s = divmod(int(sec), 60)
+    if m < 60:
+        return f"{m}m {s}s"
+    h, m = divmod(m, 60)
+    return f"{h}h {m}m {s}s"
+
+# -------------------------
+# nfs subcommands: one / all
+# -------------------------
+
+@nfs_app.command("one")
+def nfs_one(
+    endpoint: Endpoint = typer.Argument(..., help="c101|c103|c104|c106|c108|all"),
+    code: str = typer.Argument(..., help="종목코드 (예: 005930)"),
+    sleep_sec: float = typer.Option(2.0, "--sleep"),
+    sink: Sink = typer.Option("memory", "--sink"),
+    show: bool = typer.Option(True, "--show/--no-show", help="결과 DTO 출력"),
+):
+    code = code.strip()
+    if not code:
+        raise typer.BadParameter("code는 비어있을 수 없습니다.")
+
+    async def _run():
+        ucs = build_usecases(sink_kind=sink)
+
+        if sink == "mongo":
+            if ucs.db is None:
+                raise RuntimeError("mongo sink인데 ucs.db가 없습니다. composition에서 db를 노출하세요.")
+            await _mongo_bootstrap(ucs.db)
+
+        try:
+            run_asof = datetime.now(timezone.utc)
+            for ep in _endpoint_list(endpoint):
+                ingest_uc = cast(IngestPort, getattr(ucs.ingest, ep))
+                results = await ingest_uc.execute_many([code], sleep_sec=sleep_sec, asof=run_asof)
+                dto = results[0] if results else None
+
+                typer.echo(f"\n=== ONE DONE: {ep} {code} ===")
+                if not show:
+                    continue
+                if dto is None:
+                    typer.echo("(no result)")
+                else:
+                    typer.echo(_dto_to_pretty(dto))
+        finally:
+            if getattr(ucs, "mongo", None) is not None:
+                await _maybe_await_close(ucs.mongo)
+
+    asyncio.run(_run())
+
+
+@nfs_app.command("all")
+def nfs_all(
+    endpoint: Endpoint = typer.Argument(..., help="c101|c103|c104|c106|c108|all"),
+    universe: str = typer.Option("krx300", "--universe"),
+    limit: int = typer.Option(0, "--limit", help="0=전체"),
+    sleep_sec: float = typer.Option(2.0, "--sleep"),
+    sink: Sink = typer.Option("mongo", "--sink"),
+    chunk_size: int = typer.Option(5, "--chunk", help="진행률 표시용 배치 크기"),
+    show: bool = typer.Option(False, "--show/--no-show", help="일부 코드만 출력"),
+    show_n: int = typer.Option(3, "--show-n"),
+):
+    async def _run():
+        ucs = build_usecases(sink_kind=sink)
+        if ucs.db is None:
+            raise RuntimeError("all 모드는 DB가 필요합니다. mongo sink로 ucs.db를 노출하세요.")
+        await _mongo_bootstrap(ucs.db)
+
+        codes = await _load_codes_from_universe(ucs.db, universe=universe)
+        if not codes:
+            raise RuntimeError(f"universe='{universe}' codes가 비었습니다. 먼저 krx sync로 universe를 채우세요.")
+
+        if limit and limit > 0:
+            codes = codes[:limit]
+
+        typer.echo(f"\n=== NFS ALL === universe={universe}, endpoint={endpoint}, codes={len(codes)}, sink={sink}")
+
+        try:
+            await _run_ingest_with_progress(
+                ucs=ucs,
+                endpoint=endpoint,
+                codes=codes,
+                sleep_sec=sleep_sec,
+                show=show,
+                show_n=show_n,
+                chunk_size=chunk_size,
+                progress_every=1,   # chunk마다
+            )
+        finally:
+            if getattr(ucs, "mongo", None) is not None:
+                await _maybe_await_close(ucs.mongo)
+
+    asyncio.run(_run())
+
+
+@mi_app.callback(invoke_without_command=True)
+def mi():
+    pass
+
+
+if __name__ == "__main__":
+    app()

scraper2_hj3415-2.1.0.dist-info/METADATA

@@ -0,0 +1,164 @@
+Metadata-Version: 2.4
+Name: scraper2-hj3415
+Version: 2.1.0
+Summary: Naver WiseReport scraper
+Keywords: example,demo
+Author-email: Hyungjin Kim <hj3415@gmail.com>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Typing :: Typed
+License-File: LICENSE
+Requires-Dist: playwright>=1.55
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: pandas-stubs>=2.3.3
+Requires-Dist: lxml>=6.0.2
+Requires-Dist: typer>=0.21.0
+Requires-Dist: db2-hj3415
+
+# scraper2
+
+KRX 기반 국내 주식 재무(NFS) 데이터 스크래퍼 CLI 도구입니다.
+종목 단건 수집(one)과 유니버스 전체 수집(all)을 명확히 분리해 운영합니다.
+
+⸻
+
+설치
+
+pip install -e .
+
+설치 후 CLI 확인:
+
+scraper2 --help
+
+
+⸻
+
+CLI 구조
+
+scraper2
+ ├─ nfs
+ │   ├─ one   # 종목 1개 수집
+ │   └─ all   # 유니버스 전체 수집
+ └─ mi        # (reserved)
+
+
+⸻
+
+1️⃣ 단일 종목 수집 (one)
+
+사용법
+
+scraper2 nfs one <endpoint> <code> [options]
+
+예시
+
+# 삼성전자 c101 단건 수집 (메모리)
+scraper2 nfs one c101 005930
+
+# 여러 endpoint 동시 수집
+scraper2 nfs one all 005930
+
+# MongoDB 저장
+scraper2 nfs one c101 005930 --sink mongo
+
+# 결과 DTO 출력 비활성화
+scraper2 nfs one c101 005930 --no-show
+
+특징
+	•	항상 종목 1개만 처리
+	•	결과 DTO를 즉시 출력 (디버깅·검증용)
+	•	운영 자동화보다는 수동 테스트 / 검증용
+
+⸻
+
+2️⃣ 유니버스 전체 수집 (all)
+
+사용 전 필수
+
+# universe를 먼저 DB에 적재
+krx sync
+
+
+⸻
+
+사용법
+
+scraper2 nfs all <endpoint> [options]
+
+예시
+
+# krx300 전체 종목 c101 수집
+scraper2 nfs all c101
+
+# 모든 endpoint 수집
+scraper2 nfs all all
+
+# 특정 유니버스 지정
+scraper2 nfs all c103 --universe krx300
+
+# 처리 개수 제한 (테스트용)
+scraper2 nfs all c101 --limit 10
+
+# 진행률 배치 크기 조정
+scraper2 nfs all c101 --chunk 5
+
+실행 중 출력 예시
+
+=== NFS ALL === universe=krx300, endpoint=c101, codes=298, sink=mongo
+
+=== START: c101 === total=298, chunk_size=5
+progress: 25/298 (ok=25, fail=0)
+progress: 50/298 (ok=50, fail=0)
+...
+=== DONE: c101 === ok=297, fail=1, total=298
+
+⏱ elapsed time: 6m 42s
+
+특징
+	•	DB에 저장된 universe_latest 기준
+	•	진행률, 성공/실패 개수 실시간 출력
+	•	전체 작업 종료 시 총 소요 시간 표시
+	•	운영/스케줄러용 메인 모드
+
+⸻
+
+3️⃣ 작업 스케줄러 예시 (cron)
+
+매일 새벽 2시 전체 수집
+
+0 2 * * * /path/to/venv/bin/scraper2 nfs all all >> /var/log/scraper2_nfs.log 2>&1
+
+endpoint 분리 실행 (권장)
+
+0 2 * * * /path/to/venv/bin/scraper2 nfs all c101 >> /var/log/nfs_c101.log 2>&1
+10 2 * * * /path/to/venv/bin/scraper2 nfs all c103 >> /var/log/nfs_c103.log 2>&1
+
+테스트용 제한 실행
+
+0 1 * * * /path/to/venv/bin/scraper2 nfs all c101 --limit 20 >> /var/log/nfs_test.log 2>&1
+
+
+⸻
+
+운영 권장 사항
+	•	all 모드는 반드시 mongo sink 사용
+	•	universe는 krx sync로 관리
+	•	로그에는
+	•	진행률
+	•	ok / fail 개수
+	•	전체 소요 시간
+이 모두 남도록 설계됨
+	•	장애 발생 시 어디까지 진행됐는지 로그만 보고 판단 가능
+
+⸻
+
+요약
+
+목적	명령
+단건 검증	scraper2 nfs one c101 005930
+전체 수집	scraper2 nfs all all
+스케줄러	scraper2 nfs all <endpoint>
+
+

scraper2_hj3415-2.1.0.dist-info/RECORD

@@ -0,0 +1,63 @@
+scraper2/.DS_Store,sha256=miZHKI70yhXtniLCotYbxAAOTB3ML9T6Tb8xQYPeK8w,6148
+scraper2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/main.py,sha256=UTm9RcYpdBP8dMyvsJG-y0_9AINqOVgJfIkdFitMLfs,8081
+scraper2/adapters/out/.DS_Store,sha256=nUqwRB5F2DM82P8BALYvDI0YoD1UbmngfSi8ukKkY7E,6148
+scraper2/adapters/out/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/adapters/out/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/adapters/out/playwright/browser.py,sha256=XomDThsVhsd5MdwhatgXyaCocs1XCjYEKC_YHmo1jsk,3988
+scraper2/adapters/out/playwright/browser_factory.py,sha256=y008Dbu6VmzrEe7zn7CbODndQEVJtv_EBu04GsMbPGM,3697
+scraper2/adapters/out/playwright/session.py,sha256=hQDmYpi7pIVDjkymaTKQzJVWUsRRlvJg1V777V8q44M,3727
+scraper2/adapters/out/sinks/.DS_Store,sha256=c6VOGBl6bMmFzab335dcT09WinGd4BCZXZiPjrZjd7o,6148
+scraper2/adapters/out/sinks/memory/__init__.py,sha256=djvn50E0dBZr-H6Xmh9FMYalG2Zc0jL5kTXaBjnRaRo,400
+scraper2/adapters/out/sinks/memory/c101_memory_sink.py,sha256=klVgT_ra8EQYjJCk6ilWDkrNG3dj_ITP-eqdwX07nY4,682
+scraper2/adapters/out/sinks/memory/c103_memory_sink.py,sha256=hov3yH3uQqFie1BzxHa3GKsHjm_e5011ItZ4AiqaJIg,683
+scraper2/adapters/out/sinks/memory/c104_memory_sink.py,sha256=-cmNhK17OdsFfVklJjQFF4IWJvmH5ppGcjnpLf8VOns,683
+scraper2/adapters/out/sinks/memory/c106_memory_sink.py,sha256=xyrMDjIpPZJ2CHc8687weEF72wKcVl_flFp7PU6GNdo,682
+scraper2/adapters/out/sinks/memory/c108_memory_sink.py,sha256=HYxZ1YymdSlGzPpaNsdQGZhpzVr-ArgNgM0PyJdfubA,682
+scraper2/adapters/out/sinks/memory/store.py,sha256=h4dwiCF5gne5kloRdD78NWlqtcaailIqoId9xAjtBk4,2738
+scraper2/adapters/out/sinks/mongo/__init__.py,sha256=YmEZqNqh7S4PFTufxd5sCF2k24rTOsxY3ZFrFVyQzh8,382
+scraper2/adapters/out/sinks/mongo/c101_mongo_sink.py,sha256=CpcafwjBJ-Jo-sm02hRC1H214B4aKfvV5MNoNP1AfxQ,1270
+scraper2/adapters/out/sinks/mongo/c103_mongo_sink.py,sha256=gKoAOL3Dj4_JVjhdc5QAZObk49pFT-ERCyJCpUF9j2k,1203
+scraper2/adapters/out/sinks/mongo/c104_mongo_sink.py,sha256=IicVnc2fyeBXoBbgMasB7utzF7f1S6upgHV4g3sjs4g,1203
+scraper2/adapters/out/sinks/mongo/c106_mongo_sink.py,sha256=FMdCp8WVjPwidnh7tIPUoViQWr48O16xtB34O_iCtJI,1203
+scraper2/adapters/out/sinks/mongo/c108_mongo_sink.py,sha256=eSvIRtofWvNKVPchglwL1mOw5hsKDpUfNz5EOum-H3Y,1203
+scraper2/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/app/composition.py,sha256=HijMEE-2lMEMiYKlVN9ZWJD-E4vwgzIf9-plYzoSwGc,6229
+scraper2/app/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/app/parsing/_converters.py,sha256=z0kSL4nQnqq5w7QfJZBSbjZOLo1JhoqqDLpqlEAN4bo,1815
+scraper2/app/parsing/_normalize.py,sha256=2qqbXxTbzbuYlu7ttzQjyKgatFnwopme2b_fd0zahio,3738
+scraper2/app/parsing/c101_parser.py,sha256=QybZcd_7om4u3w5BWzbVNMcj50WrHtKr8hDOmkBMZGw,5537
+scraper2/app/parsing/c103_parser.py,sha256=BIHJ0OHUaGbVu3kyfgYQQIKf4O_lj4ZTXXPk6vl7Iok,3744
+scraper2/app/parsing/c104_parser.py,sha256=NGnzdVbhdqXFqJphwEDSlJPnM18RU759FCgry20a4ko,4193
+scraper2/app/parsing/c106_parser.py,sha256=JCml8HHnczgMUVnUkRI8AMEJ9mog1dOJfdd6hQKtv9I,4505
+scraper2/app/parsing/c108_parser.py,sha256=VEzzXliatoRdxR2_uSnHMHLNvV5h2irYiyoXAMQm8jc,1961
+scraper2/app/ports/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/app/ports/ingest_port.py,sha256=RHaRXJanq4VdDUXr42pEJr0vWru7zlIHNOTv7k940os,316
+scraper2/app/ports/browser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/app/ports/browser/browser_factory_port.py,sha256=dJ3JCc38MVF7wPmCH4MO3BdnHIXE5wSmfsV7cLysJ54,401
+scraper2/app/ports/browser/browser_port.py,sha256=tFYkdCeTCF-5XHvT1duioj31ytsc-ATQpmEgposi1X4,1133
+scraper2/app/ports/sinks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/app/ports/sinks/base_sink_port.py,sha256=1fJ0fCZ-uDOwfCy2MJLPOV5b_twsfOKGphlsxM2-uOw,414
+scraper2/app/ports/sinks/c101_sink_port.py,sha256=dO_A4AR-7lbPO_MlYz_CnumhO5SM_aJfC-09I2nTr2U,297
+scraper2/app/ports/sinks/c103_sink_port.py,sha256=xjRkgu_mxCMwnMHMyJy5dTHw8AxwrXmiWzT6cUCujXg,297
+scraper2/app/ports/sinks/c104_sink_port.py,sha256=0k_47ZZFTCt9jsFhWhDhhvtfdyZa3hVfF65bcLJX1AU,297
+scraper2/app/ports/sinks/c106_sink_port.py,sha256=cge47IiMoFGC_wmHAcHn2nTiH0h65df5N8HLwqzBuY4,297
+scraper2/app/ports/sinks/c108_sink_port.py,sha256=RLZRHJTvdZRsHcs18J0H98XQirW6xRmuMDx2XhiB3ac,297
+scraper2/app/usecases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/app/usecases/fetch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/app/usecases/fetch/fetch_c101.py,sha256=uqvnH_D8Jp2_BtoiEUcbwkPulv7M9qcq5WI77vsOzCc,1480
+scraper2/app/usecases/fetch/fetch_c103.py,sha256=NHXkUQxM2-Z7N7oW0uW88G40j57eGHX7RQtXCnZVWcY,3321
+scraper2/app/usecases/fetch/fetch_c104.py,sha256=ZAja-G3hCEqLCzVDba2iuu1EFN_wUiDnm9iMcG5nsO4,2518
+scraper2/app/usecases/fetch/fetch_c106.py,sha256=cIFNJ1-_MgyOCIGtVSkEKbiVRBFIhwtKiv0C9uKvrB0,3049
+scraper2/app/usecases/fetch/fetch_c108.py,sha256=okVbNmCcXcgy9-9GOhvgqrHd6ujXv_lL0uogPr9POEs,1685
+scraper2/app/usecases/ingest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scraper2/app/usecases/ingest/ingest_c101.py,sha256=feaasz-Tx_CCdcs_4Wh3KBQxj3oMefacbq3Ds6UDpmk,1149
+scraper2/app/usecases/ingest/ingest_c103.py,sha256=T_IKs5ikckin_05FnL-dxirAW1PPav867AuQsVsrZ5Y,1150
+scraper2/app/usecases/ingest/ingest_c104.py,sha256=2rGTcFbsATsn3d2KsSEkL5x4fmkGo7x9MHrysoxiICM,1150
+scraper2/app/usecases/ingest/ingest_c106.py,sha256=mQrbASbKVUQyVcIWiXzIczbX-1mNR5NGk760enVCfvo,1190
+scraper2/app/usecases/ingest/ingest_c108.py,sha256=49ULzdl0dN6z3istAKg29PcD5wHTxYholqAZiIEmUzU,1191
+scraper2_hj3415-2.1.0.dist-info/entry_points.txt,sha256=jUNx7ZJQedQ3QnsDN1ompQ0PjwdvVmnKdHHFMfQQPlI,46
+scraper2_hj3415-2.1.0.dist-info/licenses/LICENSE,sha256=QBiVGQuKAESeCfQE344Ik2ex6g2zfYdu9WqrRWydxIs,1068
+scraper2_hj3415-2.1.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+scraper2_hj3415-2.1.0.dist-info/METADATA,sha256=BM0PNscoPswXr_CaQCCas_f8VZ_QSLxiq1KL8oGTpZk,3425
+scraper2_hj3415-2.1.0.dist-info/RECORD,,

scraper2_hj3415-2.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+scraper2=scraper2.main:app
+

scraper2_hj3415/__main__.py

@@ -1,6 +0,0 @@
-# scraper2_hj3415/__main__.py
-
-from scraper2_hj3415.entrypoints.cli import app
-
-if __name__ == "__main__":
-    app()

scraper2_hj3415/adapters/_shared/utils.py

@@ -1,29 +0,0 @@
-# scraper2_hj3415/adapters/_shared/utils.py
-
-from loguru import logger
-import pandas as pd
-from typing import Iterable, Iterator, TypeVar, List
-
-T = TypeVar("T")
-
-def chunked(iterable: Iterable[T], size: int) -> Iterator[List[T]]:
-    # 큰 데이터 시퀀스를 size개씩 끊어서(List로 묶어서) 하나씩 yield 하는 제너레이터 함수.
-    buf: List[T] = []
-    for item in iterable:
-        buf.append(item)
-        if len(buf) >= size:
-            yield buf
-            buf = []
-    if buf:
-        yield buf
-
-def log_df(df: pd.DataFrame, name: str = "DataFrame", max_rows: int = 10):
-    """짧고 깔끔하게 DataFrame 로그 찍기"""
-    if df.empty:
-        logger.info(f"[{name}] DataFrame is empty.")
-        return
-
-    head = df.head(max_rows)
-    msg = head.to_markdown(index=False)
-    more = "" if len(df) <= max_rows else f"\n... ({len(df)} rows total)"
-    logger.debug(f"\n[{name}] shape={df.shape}\n{msg}{more}")

scraper2_hj3415/adapters/clients/browser.py

@@ -1,124 +0,0 @@
-# src/scraper2_hj3415/adapters/clients/browser.py
-
-from contextlib import asynccontextmanager
-from typing import AsyncGenerator
-import subprocess
-import sys
-import os
-
-from playwright.async_api import async_playwright, Browser, Error as PWError
-
-def _install_playwright_browsers(*names: str) -> None:
-    """
-    playwright install [names...] 를 코드에서 실행.
-    macOS/Windows에선 install만, Linux면 필요시 install-deps도 함께.
-    """
-    args = [sys.executable, "-m", "playwright", "install", *names]
-    subprocess.run(args, check=True)
-
-    if sys.platform.startswith("linux"):
-        try:
-            subprocess.run(
-                [sys.executable, "-m", "playwright", "install-deps"], check=True
-            )
-        except Exception:
-            pass
-
-class PlaywrightSession:
-    def __init__(self, browser, pw):
-        self.browser = browser
-        self.pw = pw
-
-    @classmethod
-    async def create(cls, headless=True):
-        browser, pw = await PlaywrightSession.create_browser(headless=headless)
-        return cls(browser, pw)
-
-    @staticmethod
-    async def create_browser(headless: bool = True) -> tuple[Browser, any]:
-        """
-        Playwright Browser 인스턴스를 생성하고 반환합니다.
-        (asynccontextmanager를 사용하지 않는 일반 버전)
-
-        사용 예시:
-            browser, pw = await create_browser()
-            page = await browser.new_page()
-            await page.goto("https://example.com")
-            html = await page.content()
-            await browser.close()
-            await pw.stop()
-
-        Args:
-            headless: 브라우저를 headless 모드로 실행할지 여부 (기본 True)
-        Returns:
-            (browser, pw): (Browser 객체, async_playwright 인스턴스)
-        """
-        pw = await async_playwright().start()
-        try:
-            try:
-                browser = await pw.chromium.launch(headless=headless)
-            except PWError as e:
-                msg = str(e)
-                need_install = (
-                    "Executable doesn't exist" in msg
-                    or "Please run the following command to download new browsers"
-                    in msg
-                )
-                if need_install and os.getenv("PW_SKIP_AUTO_INSTALL") != "1":
-                    await pw.stop()
-                    _install_playwright_browsers("chromium")
-                    pw = await async_playwright().start()
-                    browser = await pw.chromium.launch(headless=headless)
-                else:
-                    raise
-            return browser, pw
-        except Exception:
-            await pw.stop()
-            raise
-
-    async def close(self):
-        await self.browser.close()
-        await self.pw.stop()
-
-
-
-@asynccontextmanager
-async def browser_context(headless: bool = True) -> AsyncGenerator[Browser, None]:
-    """
-    Playwright Browser 인스턴스를 생성하고 반환합니다.
-    블록을 벗어나면 자동으로 종료됩니다.
-
-    Usage:
-        async with create_browser() as browser:
-            page = await browser.new_page()
-            await page.goto("https://example.com")
-            html = await page.content()
-
-    Args:
-        headless: 브라우저를 headless 모드로 실행할지 여부 (기본 True)
-    """
-    pw = await async_playwright().start()
-    try:
-        try:
-            browser = await pw.chromium.launch(headless=headless)
-        except PWError as e:
-            # 바이너리 미설치 상황 감지
-            msg = str(e)
-            need_install = "Executable doesn't exist" in msg or "Please run the following command to download new browsers" in msg
-            if need_install and os.getenv("PW_SKIP_AUTO_INSTALL") != "1":
-                # 일단 정리
-                await pw.stop()
-                # 브라우저 설치
-                _install_playwright_browsers("chromium")
-                # 재시작 후 재시도
-                pw = await async_playwright().start()
-                browser = await pw.chromium.launch(headless=headless)
-            else:
-                raise
-
-        try:
-            yield browser
-        finally:
-            await browser.close()
-    finally:
-        await pw.stop()