db2_hj3415 0.1.0__tar.gz

db2_hj3415-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: db2_hj3415
+Version: 0.1.0
+Summary: Gathering the stock data by playwright
+Author-email: Hyungjin Kim <hj3415@gmail.com>
+Description-Content-Type: text/markdown
+Classifier: License :: OSI Approved :: MIT License
+Requires-Dist: motor
+Requires-Dist: pandas
+Requires-Dist: deepdiff
+Requires-Dist: utils_hj3415>=3.2.3
+Project-URL: Home, https://www.hyungjin.kr
+
+### db2-hj3415
+
+#### Introduction 
+hj3415를 위한 mongo 데이터베이스 패키지
+
+#### Notice
+패키지내의 각 폴더는 db명, 파일은 컬렉션 단위로 구성한다.
+redi폴더는 레디스캐시관련 모듈
+common 공통 유틸함수

db2_hj3415-0.1.0/README.md

@@ -0,0 +1,9 @@
+### db2-hj3415
+
+#### Introduction 
+hj3415를 위한 mongo 데이터베이스 패키지
+
+#### Notice
+패키지내의 각 폴더는 db명, 파일은 컬렉션 단위로 구성한다.
+redi폴더는 레디스캐시관련 모듈
+common 공통 유틸함수

db2_hj3415-0.1.0/db2_hj3415/common/connection.py

@@ -0,0 +1,17 @@
+# MongoDB/Redis 연결
+import os
+from motor.motor_asyncio import AsyncIOMotorClient
+
+# 싱글톤 몽고 클라이언트 정의
+MONGO_URI = os.getenv("MONGO_ADDR", "mongodb://localhost:27017")
+client: AsyncIOMotorClient = None
+
+def get_mongo_client() -> AsyncIOMotorClient:
+    global client
+    if client is None:
+        client = AsyncIOMotorClient(MONGO_URI)
+    return client
+
+def close_mongo_client():
+    if client:
+        client.close()

db2_hj3415-0.1.0/db2_hj3415/common/db_ops.py

@@ -0,0 +1,5 @@
+# 공통 DB 연산 (insert, update, check 등)
+from motor.motor_asyncio import AsyncIOMotorClient
+
+def get_collection(client: AsyncIOMotorClient, db_name: str, col_name: str):
+    return client[db_name][col_name]

db2_hj3415-0.1.0/db2_hj3415/common/utils.py

@@ -0,0 +1 @@
+# 자주 쓰는 간단한 유틸 함수

db2_hj3415-0.1.0/db2_hj3415/mi/__init__.py

@@ -0,0 +1,2 @@
+DB_NAME = 'mi'
+DATE_FORMAT = "%Y.%m.%d"

db2_hj3415-0.1.0/db2_hj3415/mi/_ops.py

@@ -0,0 +1,117 @@
+from motor.motor_asyncio import AsyncIOMotorClient
+from datetime import datetime
+from pymongo import UpdateOne
+import pandas as pd
+
+from db2_hj3415.mi import DB_NAME, DATE_FORMAT
+from db2_hj3415.common.db_ops import get_collection
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+
+async def _save_one_collection(collection_name: str, doc: dict, client: AsyncIOMotorClient):
+    """단일 컬렉션에 대해 날짜 기준으로 문서 저장."""
+    try:
+        collection = get_collection(client, DB_NAME, collection_name)
+        date_str = doc.get("날짜")
+        if not date_str:
+            print(f"{collection_name}: 날짜 없음, 저장 건너뜀")
+            return
+
+        date_obj = datetime.strptime(date_str, DATE_FORMAT)
+        doc["날짜"] = date_obj
+        mylogger.debug(f"{collection_name} - 원본 날짜 문자열: {date_str}")
+
+        await collection.create_index("날짜", unique=True)
+
+        result = await collection.update_one(
+            {"날짜": date_obj},
+            {"$set": doc},
+            upsert=True
+        )
+
+        status = "삽입" if result.upserted_id else "업데이트"
+        print(f"{collection_name}: {status}")
+
+    except Exception as e:
+        print(f"{collection_name}: 오류 - {e}")
+
+
+async def save(data: dict[str, dict], client: AsyncIOMotorClient):
+    '''
+    전체 mi collection 데이터를 전달받아 저장하는 함수
+    '''
+    for collection_name, doc in data.items():
+        await _save_one_collection(collection_name, doc, client)
+
+
+async def find(col: str, date_str: str, client: AsyncIOMotorClient):
+    collection = get_collection(client, DB_NAME, col)
+    date_obj = datetime.strptime(date_str, DATE_FORMAT)
+    doc = await collection.find_one({"날짜": date_obj})
+    mylogger.debug(f"{col} 날짜 타입 확인:", doc["날짜"], repr(doc["날짜"]))
+    if doc:
+        print(f"조회 결과 ({col}): {doc}")
+    else:
+        print(f"{col} 컬렉션에 {date_str} 날짜 데이터 없음")
+
+
+async def delete(col: str, date_str: str, client: AsyncIOMotorClient):
+    collection = get_collection(client, DB_NAME, col)
+    date_obj = datetime.strptime(date_str, DATE_FORMAT)
+    result = await collection.delete_one({"날짜": date_obj})
+    if result.deleted_count > 0:
+        print(f"{col}: {date_str} 데이터 삭제 완료")
+    else:
+        print(f"{col}: 삭제할 데이터 없음")
+
+
+async def _save_market_history_type1(df: pd.DataFrame, market: str, numeric_columns: list | None, client: AsyncIOMotorClient):
+    if df.empty:
+        print("빈 데이터프레임입니다.")
+        return {"inserted": 0, "updated": 0}
+
+    db = client[DB_NAME]
+    collection = db[market]
+
+    # 컬럼 정리
+    df.columns = df.columns.str.strip()
+
+    # 날짜 파싱
+    try:
+        df["날짜"] = pd.to_datetime(df["날짜"], format=DATE_FORMAT, utc=True)
+    except Exception as e:
+        print(f"날짜 파싱 실패: {e}")
+        return {"inserted": 0, "updated": 0}
+
+    # 숫자형 변환
+    if numeric_columns:
+        for col in numeric_columns:
+            if col in df.columns:
+                df[col] = pd.to_numeric(df[col], errors="coerce")
+
+    # dict 변환
+    records = df.to_dict(orient="records")
+
+    # 인덱스 (1회만)
+    await collection.create_index("날짜", unique=True)
+
+    # upsert 준비
+    operations = []
+    for r in records:
+        if "날짜" not in r:
+            print("날짜 필드 없음 - 건너뜀:", r)
+            continue
+        operations.append(UpdateOne({"날짜": r["날짜"]}, {"$set": r}, upsert=True))
+
+    # 실행
+    if operations:
+        result = await collection.bulk_write(operations)
+        print(f"{market}: upsert 완료 - 삽입 {result.upserted_count}, 수정 {result.modified_count}")
+        return {"inserted": result.upserted_count, "updated": result.modified_count}
+    else:
+        print("실행할 작업이 없습니다.")
+        return {"inserted": 0, "updated": 0}
+

db2_hj3415-0.1.0/db2_hj3415/mi/aud.py

@@ -0,0 +1,23 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "aud"
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["종가", "전일대비"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)
+

db2_hj3415-0.1.0/db2_hj3415/mi/chf.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "chf"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["종가", "전일대비"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/mi/gbond3y.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "gbond3y"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["종가", "전일대비"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/mi/gold.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "gold"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["종가", "전일대비"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/mi/kosdaq.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "kosdaq"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["체결가", "전일비", "거래량(천주)", "거래대금(백만)"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/mi/kospi.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "kospi"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["체결가", "전일비", "거래량(천주)", "거래대금(백만)"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/mi/silver.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "silver"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["종가", "전일대비"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/mi/sp500.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "sp500"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["종가", "전일대비", "시가", "고가", "저가"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/mi/usdidx.py

@@ -0,0 +1,21 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "usdidx"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+

db2_hj3415-0.1.0/db2_hj3415/mi/usdkrw.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "usdkrw"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["매매기준율", "전일대비", "현찰로 사실 때", "현찰로 파실 때", "송금 보내실 때", "송금 받으실 때"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/mi/wti.py

@@ -0,0 +1,26 @@
+from db2_hj3415.mi import _ops
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "wti"
+
+
+async def save(data: dict, client: AsyncIOMotorClient):
+    return await _ops._save_one_collection(COL_NAME, data, client)
+
+
+async def find(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.find(COL_NAME, date_str, client)
+
+
+async def delete(date_str: str, client: AsyncIOMotorClient):
+    return await _ops.delete(COL_NAME, date_str, client)
+
+
+async def save_history(df: pd.DataFrame, client: AsyncIOMotorClient):
+    numeric_columns = ["종가", "전일대비"]
+    await _ops._save_market_history_type1(df, COL_NAME, numeric_columns, client)

db2_hj3415-0.1.0/db2_hj3415/nfs/__init__.py

@@ -0,0 +1,2 @@
+DB_NAME = "nfs"
+DATE_FORMAT = "%Y.%m.%d"

db2_hj3415-0.1.0/db2_hj3415/nfs/_c10346.py

@@ -0,0 +1,198 @@
+from deepdiff import DeepDiff
+from pymongo import ASCENDING, DESCENDING
+from motor.motor_asyncio import AsyncIOMotorClient
+import pprint
+import pandas as pd
+import json
+from db2_hj3415.nfs import DB_NAME
+from db2_hj3415.common.db_ops import get_collection
+from datetime import datetime, timezone
+
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+
+async def _compare_and_log_diff(code: str, new_doc: dict, latest_doc: dict | None, client: AsyncIOMotorClient) -> bool:
+    """
+    최신 문서와 새 문서를 비교하여 변경 사항이 있는 경우만 로그에 기록하고 True를 반환합니다.
+
+    - "_id"와 "날짜" 필드는 비교 대상에서 제외합니다.
+    - 변경된 항목이 있으면 MongoDB의 "change_log" 컬렉션에 차이(diff)를 기록합니다.
+    - 변경 사항이 없으면 저장하지 않고 False를 반환합니다.
+
+    Parameters:
+        code (str): 비교 대상이 되는 종목 코드
+        new_doc (dict): 새로 생성된 문서
+        latest_doc (dict | None): 기존의 최신 문서 (없을 수 있음)
+        client (AsyncIOMotorClient): 몽고 클라이언트
+
+    Returns:
+        bool: 변경 사항이 있으면 True, 없으면 False
+    """
+    if not latest_doc:
+        return True
+
+    latest_doc.pop("_id", None)
+    latest_doc.pop("날짜", None)
+    new_copy = dict(new_doc)
+    new_copy.pop("날짜", None)
+
+    diff = DeepDiff(latest_doc, new_copy, ignore_order=True)
+    if not diff:
+        print(f"{code} 기존 문서와 동일 - 저장하지 않음")
+        return False
+
+    print("변경된 항목:")
+    for change_type, changes in diff.items():
+        print(f"- {change_type}:")
+        for path, value in changes.items():
+            print(f"  {path}: {value}")
+
+    await client[DB_NAME]["change_log"].insert_one({
+        "코드": code,
+        "변경시각": datetime.now(timezone.utc),
+        "변경내용": json.loads(diff.to_json())
+    })
+
+    return True
+
+
+def _prepare_c10346_document(code: str, data: dict[str, pd.DataFrame]) -> dict:
+    """
+    종목 코드와 여러 페이지의 DataFrame 데이터를 바탕으로 MongoDB에 저장할 문서(dict)를 생성합니다.
+
+    각 DataFrame은 null 값을 None으로 변환한 후, 레코드(행) 단위의 딕셔너리 리스트로 변환됩니다.
+    생성된 문서에는 '코드', '날짜', 그리고 각 페이지 이름을 키로 하는 데이터가 포함됩니다.
+
+    Args:
+        code (str): 종목 코드 (6자리 문자열).
+        data (dict[str, pd.DataFrame]): 페이지 이름을 키로 하고, 해당 페이지의 데이터를 담은 DataFrame을 값으로 가지는 딕셔너리.
+
+    Returns:
+        dict: MongoDB에 저장 가능한 형식의 문서. 예: {
+            "코드": "005930",
+            "날짜": <datetime>,
+            "재무상태표y": <DataFrame>,
+            "손익계산서y": <DataFrame>,
+            ...
+        }
+    """
+    now = datetime.now(timezone.utc)
+    document = {"코드": code, "날짜": now}
+    for page, df in data.items():
+        if isinstance(df, pd.DataFrame):
+            document[page] = df.where(pd.notnull(df), None).to_dict(orient="records")
+    return document
+
+
+async def save(col: str, code: str, data: dict[str, pd.DataFrame], client: AsyncIOMotorClient) -> dict:
+    collection = get_collection(client, DB_NAME, col)
+
+    await collection.create_index([("코드", ASCENDING), ("날짜", ASCENDING)], unique=True)
+
+    document = _prepare_c10346_document(code, data)
+    latest_doc = await collection.find_one({"코드": code}, sort=[("날짜", DESCENDING)])
+
+    need_save = await _compare_and_log_diff(code, document, latest_doc, client)
+    if not need_save:
+        return {"status": "unchanged"}
+
+    result = await collection.insert_one(document)
+    print(f"삽입됨:  {DB_NAME}.{collection.name} / {code} (id={result.inserted_id})")
+    del_result = await collection.delete_many({
+        "_id": {"$in": [
+            doc["_id"] for doc in await collection.find({"코드": code}).sort("날짜", DESCENDING).skip(2).to_list(length=None)
+        ]}
+    })
+    print(f"삭제된 이전 문서 수: {del_result.deleted_count}")
+
+    return {"status": "inserted", "id": str(result.inserted_id)}
+
+
+async def save_many(col: str, many_data: dict[str, dict[str, pd.DataFrame]], client: AsyncIOMotorClient) -> list[dict]:
+    collection = get_collection(client, DB_NAME, col)
+    await collection.create_index([("코드", ASCENDING), ("날짜", ASCENDING)], unique=True)
+
+    results = []
+
+    for code, data in many_data.items():
+        document = _prepare_c10346_document(code, data)
+        latest_doc = await collection.find_one({"코드": code}, sort=[("날짜", DESCENDING)])
+        need_save = await _compare_and_log_diff(code, document, latest_doc, client)
+
+        if not need_save:
+            results.append({"code": code, "status": "unchanged"})
+            continue
+
+        result = await collection.insert_one(document)
+        await collection.delete_many({
+            "_id": {"$in": [
+                doc["_id"] for doc in await collection.find({"코드": code}).sort("날짜", DESCENDING).skip(2).to_list(length=None)
+            ]}
+        })
+
+        results.append({"code": code, "status": "inserted", "id": str(result.inserted_id)})
+
+    pprint.pprint(results)
+    return results
+
+
+async def get_latest(col: str, code: str, page: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
+    collection = get_collection(client, DB_NAME, col)
+
+    # 최신 날짜 기준으로 정렬하여 1건만 조회
+    latest_doc = await collection.find_one(
+        {"코드": code},
+        sort=[("날짜", DESCENDING)]
+    )
+
+    if not latest_doc or page not in latest_doc:
+        print(f"문서 없음 또는 '{page}' 항목 없음")
+        return None
+
+        # records → DataFrame
+    records = latest_doc[page]
+    df = pd.DataFrame(records)
+    return df
+
+
+async def has_doc_changed(col: str, code: str, client: AsyncIOMotorClient) -> bool:
+    """
+    MongoDB에서 특정 컬렉션과 종목 코드에 대해 최신 두 개의 문서를 비교하여 변경 여부를 확인합니다.
+
+    비교 대상 문서가 두 개 미만이면 True를 반환하여 새 문서로 간주합니다.
+    비교는 `_id`, `날짜` 필드를 제외하고 수행하며, 변경 내용이 있을 경우 change_log에 기록됩니다.
+
+    Args:
+        col (str): 컬렉션 이름 (예: 'c103' 'c104', 'c106'등).
+        code (str): 종목 코드 (6자리 문자열).
+        client (AsyncIOMotorClient): MongoDB 비동기 클라이언트 인스턴스.
+
+    Returns:
+        bool: 문서가 변경되었는지 여부. True면 변경됨 또는 비교 불가 상태.
+    """
+    collection = get_collection(client, DB_NAME, col)
+
+    # 최신 문서 2개 조회 (내림차순)
+    docs = await collection.find({"코드": code}).sort("날짜", DESCENDING).limit(2).to_list(length=2)
+
+    if len(docs) < 2:
+        print(f"{code} 문서가 1개 이하임 - 비교 불가")
+        return True  # 비교할 게 없으면 새로 저장해야 하므로 True
+
+    new_doc, latest_doc = docs[0], docs[1]
+
+    new_doc.pop("_id", None)
+    new_doc.pop("날짜", None)
+    latest_doc.pop("_id", None)
+    latest_doc.pop("날짜", None)
+
+    mylogger.debug(new_doc)
+    mylogger.debug(latest_doc)
+
+    # 비교 함수 호출
+    return await _compare_and_log_diff(code, new_doc, latest_doc, client)
+
+
+

db2_hj3415-0.1.0/db2_hj3415/nfs/_ops.py

@@ -0,0 +1,42 @@
+from motor.motor_asyncio import AsyncIOMotorClient
+from db2_hj3415.nfs import DB_NAME
+
+
+async def get_all_codes(client: AsyncIOMotorClient) -> list[str]:
+    """
+    c103, c104, c106 컬렉션에 모두 존재하는 코드의 리스트를 반환함.
+
+    Args:
+        client (AsyncIOMotorClient): MongoDB 비동기 클라이언트 객체
+
+    Returns:
+        list[str]: c103, c104, c106 컬렉션에 공통으로 존재하는 종목 코드 리스트
+    """
+    db = client[DB_NAME]
+
+    collections = ['c103', 'c104', 'c106']
+
+    # 첫 컬렉션으로 초기화
+    s = set(await db[collections[0]].distinct("코드"))
+
+    for col in collections[1:]:
+        codes = await db[col].distinct("코드")
+        s &= set(codes)
+
+    return list(s)
+
+
+async def delete_code_from_all_collections(code: str, client: AsyncIOMotorClient) -> dict[str, int]:
+    db = client[DB_NAME]
+
+    collections = ['c101', 'c103', 'c104', 'c106', 'c108']
+
+    deleted_counts = {}
+
+    for col in collections:
+        result = await db[col].delete_many({"코드": code})
+        deleted_counts[col] = result.deleted_count
+
+    print(f"삭제된 도큐먼트 갯수: {deleted_counts}")
+    return deleted_counts
+

db2_hj3415-0.1.0/db2_hj3415/nfs/c101.py

@@ -0,0 +1,124 @@
+from pymongo import ASCENDING, UpdateOne, DESCENDING
+from motor.motor_asyncio import AsyncIOMotorClient
+from datetime import datetime, timezone
+import pandas as pd
+
+from db2_hj3415.nfs import DATE_FORMAT, DB_NAME
+from db2_hj3415.common.db_ops import get_collection
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "c101"
+
+def _prepare_c101_document(doc: dict) -> dict | None:
+    """
+    C101 컬렉션에 저장할 문서를 사전 처리합니다.
+
+    - '코드'와 '날짜' 필드가 없으면 None을 반환합니다.
+    - '날짜' 필드는 UTC 타임존을 포함한 datetime 객체로 변환합니다.
+    - 날짜 형식이 잘못된 경우 None을 반환합니다.
+
+    Parameters:
+        doc (dict): 원시 입력 문서 (예: 스크래핑 또는 파싱 결과)
+
+    Returns:
+        dict | None: 정상적으로 처리된 문서 또는 오류 시 None
+    """
+    code = doc.get("코드")
+    date_str = doc.get("날짜")
+
+    if not code or not date_str:
+        print(f"코드 또는 날짜 누락: {code} / {date_str}")
+        return None
+
+    try:
+        doc["날짜"] = datetime.strptime(date_str, DATE_FORMAT).replace(tzinfo=timezone.utc)
+    except ValueError:
+        print(f"날짜 형식 오류 - 건너뜀: {code} / {date_str}")
+        return None
+
+    return doc
+
+
+async def save(data: dict | None, client: AsyncIOMotorClient) -> dict:
+    if not data:
+        print("데이터 없음 - 저장 생략")
+        return {"status": "unchanged"}
+
+    collection = get_collection(client, DB_NAME, COL_NAME)
+    await collection.create_index([("날짜", ASCENDING), ("코드", ASCENDING)], unique=True)
+
+    doc = _prepare_c101_document(data)
+    if not doc:
+        return {"status": "unchanged"}
+
+    filter_ = {"날짜": doc["날짜"], "코드": doc["코드"]}
+    result = await collection.update_one(filter_, {"$set": doc}, upsert=True)
+    if result.upserted_id:
+        return {"status": f"upserted {result.upserted_id}"}
+    elif result.modified_count:
+        return {"status": f"modified"}
+    else:
+        return {"status": "unchanged"}
+
+
+async def save_many(many_data: dict[str, dict | None], client: AsyncIOMotorClient) -> dict:
+    collection = get_collection(client, DB_NAME, COL_NAME)
+    await collection.create_index([("날짜", ASCENDING), ("코드", ASCENDING)], unique=True)
+
+    operations = []
+    inserted, updated, skipped = 0, 0, 0
+    for code, doc in many_data.items():
+        if not doc:
+            print(f"{code}: 데이터 없음 - 건너뜀")
+            continue
+
+        doc = _prepare_c101_document(doc)
+        if not doc:
+            continue
+
+        filter_ = {"날짜": doc["날짜"], "코드": doc["코드"]}
+        operations.append(UpdateOne(filter_, {"$set": doc}, upsert=True))
+
+    if operations:
+        result = await collection.bulk_write(operations)
+        inserted = result.upserted_count
+        updated = result.modified_count
+        print(f"저장 완료: inserted={inserted}, updated={updated}")
+    else:
+        print(f"저장할 작업 없음")
+    return {"inserted": inserted, "updated": updated}
+
+
+async def get_latest(code: str, client: AsyncIOMotorClient) -> dict | None:
+    collection = get_collection(client, DB_NAME, COL_NAME)
+    doc = await collection.find_one(
+        {"코드": code},
+        sort=[("날짜", DESCENDING)]
+    )
+
+    if doc:
+        doc.pop("_id", None)
+        return doc
+    else:
+        print(f"데이터 없음: {code}")
+        return None
+
+
+async def get_all_as_df(code: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
+    collection = get_collection(client, DB_NAME, COL_NAME)
+    cursor = collection.find({"코드": code}).sort("날짜", ASCENDING)
+    docs = await cursor.to_list(length=None)
+
+    if not docs:
+        print(f"[{code}] 관련 문서 없음")
+        return None
+
+    # _id 필드는 문자열로 변환하거나 제거
+    for doc in docs:
+        doc.pop("_id", None)
+
+    df = pd.DataFrame(docs)
+    return df
+

db2_hj3415-0.1.0/db2_hj3415/nfs/c103.py

@@ -0,0 +1,38 @@
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from db2_hj3415.nfs import _c10346
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "c103"
+
+
+async def save(code: str, data: dict[str, pd.DataFrame], client: AsyncIOMotorClient) -> dict:
+    return await _c10346.save(COL_NAME, code, data, client)
+
+
+async def save_many(many_data: dict[str, dict[str, pd.DataFrame]], client: AsyncIOMotorClient) -> list[dict]:
+    return await _c10346.save_many(COL_NAME, many_data, client)
+
+
+async def get_latest(code: str, page: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
+    return await _c10346.get_latest(COL_NAME, code, page, client)
+
+
+async def has_doc_changed(code: str, client: AsyncIOMotorClient) -> bool:
+    """
+    C103 컬렉션에서 종목 코드에 대해 최신 두 개의 문서를 비교하여 변경 여부를 확인합니다.
+
+    비교 대상 문서가 두 개 미만이면 True를 반환하여 새 문서로 간주합니다.
+    비교는 `_id`, `날짜` 필드를 제외하고 수행하며, 변경 내용이 있을 경우 change_log에 기록됩니다.
+
+    Args:
+        code (str): 종목 코드 (6자리 문자열).
+        client (AsyncIOMotorClient): MongoDB 비동기 클라이언트 인스턴스.
+
+    Returns:
+        bool: 문서가 변경되었는지 여부. True면 변경됨 또는 비교 불가 상태.
+    """
+    return await _c10346.has_doc_changed(COL_NAME, code, client)

db2_hj3415-0.1.0/db2_hj3415/nfs/c104.py

@@ -0,0 +1,38 @@
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from db2_hj3415.nfs import _c10346
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "c104"
+
+
+async def save(code: str, data: dict[str, pd.DataFrame], client: AsyncIOMotorClient) -> dict:
+    return await _c10346.save(COL_NAME, code, data, client)
+
+
+async def save_many(many_data: dict[str, dict[str, pd.DataFrame]], client: AsyncIOMotorClient) -> list[dict]:
+    return await _c10346.save_many(COL_NAME, many_data, client)
+
+
+async def get_latest(code: str, page: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
+    return await _c10346.get_latest(COL_NAME, code, page, client)
+
+
+async def has_doc_changed(code: str, client: AsyncIOMotorClient) -> bool:
+    """
+    C104 컬렉션에서 종목 코드에 대해 최신 두 개의 문서를 비교하여 변경 여부를 확인합니다.
+
+    비교 대상 문서가 두 개 미만이면 True를 반환하여 새 문서로 간주합니다.
+    비교는 `_id`, `날짜` 필드를 제외하고 수행하며, 변경 내용이 있을 경우 change_log에 기록됩니다.
+
+    Args:
+        code (str): 종목 코드 (6자리 문자열).
+        client (AsyncIOMotorClient): MongoDB 비동기 클라이언트 인스턴스.
+
+    Returns:
+        bool: 문서가 변경되었는지 여부. True면 변경됨 또는 비교 불가 상태.
+    """
+    return await _c10346.has_doc_changed(COL_NAME, code, client)

db2_hj3415-0.1.0/db2_hj3415/nfs/c106.py

@@ -0,0 +1,38 @@
+from motor.motor_asyncio import AsyncIOMotorClient
+import pandas as pd
+
+from db2_hj3415.nfs import _c10346
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = "c106"
+
+
+async def save(code: str, data: dict[str, pd.DataFrame], client: AsyncIOMotorClient) -> dict:
+    return await _c10346.save(COL_NAME, code, data, client)
+
+
+async def save_many(many_data: dict[str, dict[str, pd.DataFrame]], client: AsyncIOMotorClient) -> list[dict]:
+    return await _c10346.save_many(COL_NAME, many_data, client)
+
+
+async def get_latest(code: str, page: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
+    return await _c10346.get_latest(COL_NAME, code, page, client)
+
+
+async def has_doc_changed(code: str, client: AsyncIOMotorClient) -> bool:
+    """
+    C106 컬렉션에서 종목 코드에 대해 최신 두 개의 문서를 비교하여 변경 여부를 확인합니다.
+
+    비교 대상 문서가 두 개 미만이면 True를 반환하여 새 문서로 간주합니다.
+    비교는 `_id`, `날짜` 필드를 제외하고 수행하며, 변경 내용이 있을 경우 change_log에 기록됩니다.
+
+    Args:
+        code (str): 종목 코드 (6자리 문자열).
+        client (AsyncIOMotorClient): MongoDB 비동기 클라이언트 인스턴스.
+
+    Returns:
+        bool: 문서가 변경되었는지 여부. True면 변경됨 또는 비교 불가 상태.
+    """
+    return await _c10346.has_doc_changed(COL_NAME, code, client)

db2_hj3415-0.1.0/db2_hj3415/nfs/c108.py

@@ -0,0 +1,77 @@
+from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo import ASCENDING, UpdateOne
+import pandas as pd
+from datetime import datetime, timezone
+
+from db2_hj3415.nfs import DATE_FORMAT, DB_NAME
+from db2_hj3415.common.db_ops import get_collection
+from utils_hj3415 import setup_logger
+
+mylogger = setup_logger(__name__, 'WARNING')
+
+COL_NAME = 'c108'
+
+
+async def save(code: str, data: pd.DataFrame, client: AsyncIOMotorClient) -> dict:
+    if data is None or data.empty:
+        print("데이터 없음 - 저장 생략")
+        return {"status": "unchanged"}
+
+    collection = get_collection(client, DB_NAME, COL_NAME)
+
+    await collection.create_index(
+        [("코드", ASCENDING), ("날짜", ASCENDING), ("제목", ASCENDING)],
+        unique=True
+    )
+
+    # NaN -> None 변환
+    df = data.where(pd.notnull(data), None)
+    operations = []
+    inserted, updated, skipped = 0, 0, 0
+
+    for _, row in df.iterrows():
+        try:
+            date_str = str(row["날짜"])
+            date_obj = datetime.strptime(date_str, DATE_FORMAT).replace(tzinfo=timezone.utc)
+
+            doc = row.to_dict()
+            doc["코드"] = code
+            doc["날짜"] = date_obj
+
+            filter_ = {"코드": code, "날짜": date_obj, "제목": doc["제목"]}
+            operations.append(UpdateOne(filter_, {"$set": doc}, upsert=True))
+        except Exception as e:
+            print(f"변환 에러 - {row.get('제목', '제목 없음')}: {e}")
+            continue
+
+    if operations:
+        result = await collection.bulk_write(operations, ordered=False)
+        inserted = result.upserted_count
+        updated = result.modified_count
+        print(f"[{code}] 저장 완료: inserted={inserted}, updated={updated}")
+    else:
+        print(f"[{code}] 저장할 작업 없음")
+
+    return {"inserted": inserted, "updated": updated}
+
+
+async def save_many(many_data: dict[str, pd.DataFrame], client: AsyncIOMotorClient) -> dict:
+    total_result = {"inserted": 0, "updated": 0, "skipped": 0, "errors": []}
+
+    for code, df in many_data.items():
+        if df is None:
+            print(f"[{code}] 리포트 없음 - 건너뜀")
+            continue
+
+        try:
+            result = await save(code, df, client)
+            total_result["inserted"] += result.get("inserted", 0)
+            total_result["updated"] += result.get("updated", 0)
+            total_result["skipped"] += result.get("skipped", 0)
+        except Exception as e:
+            print(f"[{code}] 저장 실패: {e}")
+            total_result["errors"].append({"code": code, "error": str(e)})
+
+    return total_result
+
+

db2_hj3415-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "db2_hj3415"
+version = "0.1.0"
+authors = [{name = "Hyungjin Kim", email = "hj3415@gmail.com"}]
+description = "Gathering the stock data by playwright"
+readme = "README.md"
+classifiers = ["License :: OSI Approved :: MIT License"]
+dependencies = [
+    "motor",
+    "pandas",
+    "deepdiff",
+    "utils_hj3415>=3.2.3",
+]
+
+[project.scripts]
+db = "db2_hj3415.cli.db:main"
+
+[project.urls]
+Home = "https://www.hyungjin.kr"
+
+[tool.flit.sdist]
+exclude = ["tests/", ".gitignore",]