db2_hj3415 0.1.2__tar.gz → 0.1.4__tar.gz

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: db2_hj3415
-Version: 0.1.2
+Version: 0.1.4
 Summary: Gathering the stock data by playwright
 Author-email: Hyungjin Kim <hj3415@gmail.com>
 Description-Content-Type: text/markdown
@@ -10,6 +10,8 @@ Requires-Dist: pandas
 Requires-Dist: pandas-stubs
 Requires-Dist: deepdiff
 Requires-Dist: utils_hj3415>=3.2.3
+Requires-Dist: redis
+Requires-Dist: pydantic
 Project-URL: Home, https://www.hyungjin.kr
 
 ### db2-hj3415

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/db2_hj3415/common/connection.py

@@ -1,6 +1,8 @@
 # MongoDB/Redis 연결
 import os
 from motor.motor_asyncio import AsyncIOMotorClient
+import redis.asyncio as redis
+from redis.asyncio.client import Redis
 
 # 싱글톤 몽고 클라이언트 정의
 MONGO_URI = os.getenv("MONGO_ADDR", "mongodb://localhost:27017")
@@ -14,4 +16,14 @@ def get_mongo_client() -> AsyncIOMotorClient:
 
 def close_mongo_client():
     if client:
-        client.close()
+        client.close()
+
+
+async def get_redis() -> Redis:
+    client = redis.Redis(host="localhost", port=6379, decode_responses=True)
+    try:
+        await client.ping()
+        return client
+    except Exception as e:
+        print("Redis connection failed:", e)
+        return None

db2_hj3415-0.1.4/db2_hj3415/common/utils.py

@@ -0,0 +1,24 @@
+# 자주 쓰는 간단한 유틸 함수
+import pandas as pd
+import numpy as np
+
+def df_to_dict_replace_nan(df: pd.DataFrame) -> list[dict]:
+    # NaN → None으로 변환
+    return df.replace({np.nan: None}).to_dict(orient="records")
+
+import json
+from bson import json_util
+from pydantic import BaseModel
+
+def pretty_print(obj):
+    if isinstance(obj, BaseModel):
+        # Pydantic 모델이면 dict로 변환
+        data = obj.model_dump(by_alias=True)
+    elif isinstance(obj, list) and all(isinstance(o, BaseModel) for o in obj):
+        # 리스트 안에 BaseModel만 있다면 변환
+        data = [o.model_dump(by_alias=True) for o in obj]
+    else:
+        # 일반 dict나 기타 객체
+        data = obj
+
+    print(json.dumps(data, indent=2, ensure_ascii=False, default=json_util.default))

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/db2_hj3415/nfs/__init__.py

@@ -4,4 +4,5 @@ DATE_FORMAT = "%Y.%m.%d"
 from db2_hj3415.common import connection
 from db2_hj3415.common.utils import *
 from db2_hj3415.common.db_ops import *
-from db2_hj3415.nfs._ops import *
+from db2_hj3415.nfs._ops import *
+from db2_hj3415.nfs.models import *

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/db2_hj3415/nfs/_c10346.py

@@ -1,12 +1,16 @@
+from unittest import case
+
 from deepdiff import DeepDiff
 from pymongo import ASCENDING, DESCENDING
 from motor.motor_asyncio import AsyncIOMotorClient
 import pprint
 import pandas as pd
 import json
-from db2_hj3415.nfs import DB_NAME
+from db2_hj3415.nfs import DB_NAME, C103, C104, C106
 from db2_hj3415.common.db_ops import get_collection
 from datetime import datetime, timezone
+import numpy as np
+
 
 from utils_hj3415 import setup_logger
 
@@ -59,30 +63,15 @@ async def _compare_and_log_diff(code: str, new_doc: dict, latest_doc: dict | Non
 
 
 def _prepare_c10346_document(code: str, data: dict[str, pd.DataFrame]) -> dict:
-    """
-    종목 코드와 여러 페이지의 DataFrame 데이터를 바탕으로 MongoDB에 저장할 문서(dict)를 생성합니다.
-
-    각 DataFrame은 null 값을 None으로 변환한 후, 레코드(행) 단위의 딕셔너리 리스트로 변환됩니다.
-    생성된 문서에는 '코드', '날짜', 그리고 각 페이지 이름을 키로 하는 데이터가 포함됩니다.
-
-    Args:
-        code (str): 종목 코드 (6자리 문자열).
-        data (dict[str, pd.DataFrame]): 페이지 이름을 키로 하고, 해당 페이지의 데이터를 담은 DataFrame을 값으로 가지는 딕셔너리.
-
-    Returns:
-        dict: MongoDB에 저장 가능한 형식의 문서. 예: {
-            "코드": "005930",
-            "날짜": <datetime>,
-            "재무상태표y": <DataFrame>,
-            "손익계산서y": <DataFrame>,
-            ...
-        }
-    """
     now = datetime.now(timezone.utc)
     document = {"코드": code, "날짜": now}
+
     for page, df in data.items():
         if isinstance(df, pd.DataFrame):
-            document[page] = df.where(pd.notnull(df), None).to_dict(orient="records")
+            # NaN을 None으로 안전하게 변환
+            cleaned = df.replace({np.nan: None})
+            document[page] = cleaned.to_dict(orient="records")
+
     return document
 
 
@@ -138,7 +127,7 @@ async def save_many(col: str, many_data: dict[str, dict[str, pd.DataFrame]], cli
     return results
 
 
-async def get_latest(col: str, code: str, page: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
+async def get_latest(col: str, code: str, client: AsyncIOMotorClient) -> C103 | C104 | C106 | None:
     collection = get_collection(client, DB_NAME, col)
 
     # 최신 날짜 기준으로 정렬하여 1건만 조회
@@ -147,14 +136,24 @@ async def get_latest(col: str, code: str, page: str, client: AsyncIOMotorClient)
         sort=[("날짜", DESCENDING)]
     )
 
-    if not latest_doc or page not in latest_doc:
-        print(f"문서 없음 또는 '{page}' 항목 없음")
+    if not latest_doc:
+        print(f"문서 없음: {code}")
         return None
 
-        # records → DataFrame
-    records = latest_doc[page]
-    df = pd.DataFrame(records)
-    return df
+    latest_doc["_id"] = str(latest_doc["_id"])
+
+    try:
+        match col:
+            case 'c103':
+                return C103(**latest_doc)
+            case 'c104':
+                return C104(**latest_doc)
+            case 'c106':
+                return C106(**latest_doc)
+
+    except Exception as e:
+        print(f"[{code}] C103 파싱 실패: {e}")
+        return None
 
 
 async def has_doc_changed(col: str, code: str, client: AsyncIOMotorClient) -> bool:

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/db2_hj3415/nfs/c101.py

@@ -1,9 +1,10 @@
+from typing import Literal
+
 from pymongo import ASCENDING, UpdateOne, DESCENDING
 from motor.motor_asyncio import AsyncIOMotorClient
 from datetime import datetime, timezone
-import pandas as pd
 
-from db2_hj3415.nfs import DATE_FORMAT, DB_NAME
+from db2_hj3415.nfs import DATE_FORMAT, DB_NAME, C101
 from db2_hj3415.common.db_ops import get_collection
 from utils_hj3415 import setup_logger
 
@@ -91,7 +92,7 @@ async def save_many(many_data: dict[str, dict | None], client: AsyncIOMotorClien
     return {"inserted": inserted, "updated": updated}
 
 
-async def get_latest(code: str, client: AsyncIOMotorClient) -> dict | None:
+async def get_latest(code: str, client: AsyncIOMotorClient) -> C101 | None:
     collection = get_collection(client, DB_NAME, COL_NAME)
     doc = await collection.find_one(
         {"코드": code},
@@ -99,26 +100,45 @@ async def get_latest(code: str, client: AsyncIOMotorClient) -> dict | None:
     )
 
     if doc:
-        doc.pop("_id", None)
-        return doc
+        doc["_id"] = str(doc["_id"])
+        mylogger.debug(doc)
+        return C101(**doc)
     else:
         print(f"데이터 없음: {code}")
         return None
 
 
-async def get_all_as_df(code: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
+SortOrder = Literal["asc", "desc"]
+
+async def get_all_data(code: str, client: AsyncIOMotorClient, sort: SortOrder = 'asc') -> list[C101]:
+    """
+    지정한 종목 코드의 C101 도큐먼트 전체를 날짜 기준으로 정렬하여 반환합니다.
+
+    Args:
+        code (str): 조회할 종목 코드 (예: "005930").
+        client (AsyncIOMotorClient): 비동기 MongoDB 클라이언트 인스턴스.
+        sort (Literal["asc", "desc"], optional): 날짜 정렬 방식.
+            - "asc": 오름차순 (과거 → 최신)
+            - "desc": 내림차순 (최신 → 과거)
+            기본값은 "asc".
+
+    Returns:
+        list[C101]: 정렬된 C101 모델 리스트.
+                    문서가 없을 경우 빈 리스트를 반환합니다.
+    """
     collection = get_collection(client, DB_NAME, COL_NAME)
-    cursor = collection.find({"코드": code}).sort("날짜", ASCENDING)
+    sort_order = ASCENDING if sort == "asc" else DESCENDING
+    cursor = collection.find({"코드": code}).sort("날짜", sort_order)
     docs = await cursor.to_list(length=None)
 
     if not docs:
         print(f"[{code}] 관련 문서 없음")
-        return None
+        return []
 
-    # _id 필드는 문자열로 변환하거나 제거
+    result: list[C101] = []
     for doc in docs:
-        doc.pop("_id", None)
+        doc["_id"] = str(doc["_id"])  # ObjectId → str (C101에서 id: str)
+        result.append(C101(**doc))
 
-    df = pd.DataFrame(docs)
-    return df
+    return result
 

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/db2_hj3415/nfs/c103.py

@@ -1,7 +1,7 @@
 from motor.motor_asyncio import AsyncIOMotorClient
 import pandas as pd
 
-from db2_hj3415.nfs import _c10346
+from db2_hj3415.nfs import _c10346, C103
 from utils_hj3415 import setup_logger
 
 mylogger = setup_logger(__name__, 'WARNING')
@@ -17,8 +17,8 @@ async def save_many(many_data: dict[str, dict[str, pd.DataFrame]], client: Async
     return await _c10346.save_many(COL_NAME, many_data, client)
 
 
-async def get_latest(code: str, page: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
-    return await _c10346.get_latest(COL_NAME, code, page, client)
+async def get_latest(code: str, client: AsyncIOMotorClient) -> C103 | None:
+    return await _c10346.get_latest(COL_NAME, code, client)
 
 
 async def has_doc_changed(code: str, client: AsyncIOMotorClient) -> bool:

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/db2_hj3415/nfs/c104.py

@@ -1,7 +1,7 @@
 from motor.motor_asyncio import AsyncIOMotorClient
 import pandas as pd
 
-from db2_hj3415.nfs import _c10346
+from db2_hj3415.nfs import _c10346, C104
 from utils_hj3415 import setup_logger
 
 mylogger = setup_logger(__name__, 'WARNING')
@@ -17,8 +17,8 @@ async def save_many(many_data: dict[str, dict[str, pd.DataFrame]], client: Async
     return await _c10346.save_many(COL_NAME, many_data, client)
 
 
-async def get_latest(code: str, page: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
-    return await _c10346.get_latest(COL_NAME, code, page, client)
+async def get_latest(code: str, client: AsyncIOMotorClient) -> C104 | None:
+    return await _c10346.get_latest(COL_NAME, code, client)
 
 
 async def has_doc_changed(code: str, client: AsyncIOMotorClient) -> bool:

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/db2_hj3415/nfs/c106.py

@@ -1,7 +1,7 @@
 from motor.motor_asyncio import AsyncIOMotorClient
 import pandas as pd
 
-from db2_hj3415.nfs import _c10346
+from db2_hj3415.nfs import _c10346, C106
 from utils_hj3415 import setup_logger
 
 mylogger = setup_logger(__name__, 'WARNING')
@@ -17,8 +17,8 @@ async def save_many(many_data: dict[str, dict[str, pd.DataFrame]], client: Async
     return await _c10346.save_many(COL_NAME, many_data, client)
 
 
-async def get_latest(code: str, page: str, client: AsyncIOMotorClient) -> pd.DataFrame | None:
-    return await _c10346.get_latest(COL_NAME, code, page, client)
+async def get_latest(code: str, client: AsyncIOMotorClient) -> C106 | None:
+    return await _c10346.get_latest(COL_NAME, code, client)
 
 
 async def has_doc_changed(code: str, client: AsyncIOMotorClient) -> bool:

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/db2_hj3415/nfs/c108.py

@@ -1,9 +1,9 @@
 from motor.motor_asyncio import AsyncIOMotorClient
-from pymongo import ASCENDING, UpdateOne
+from pymongo import ASCENDING, UpdateOne, DESCENDING
 import pandas as pd
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 
-from db2_hj3415.nfs import DATE_FORMAT, DB_NAME
+from db2_hj3415.nfs import DATE_FORMAT, DB_NAME, C108
 from db2_hj3415.common.db_ops import get_collection
 from utils_hj3415 import setup_logger
 
@@ -75,3 +75,37 @@ async def save_many(many_data: dict[str, pd.DataFrame], client: AsyncIOMotorClie
     return total_result
 
 
+async def get_latest(code: str, client: AsyncIOMotorClient, days:int = 30) -> list[C108]:
+    """
+    최근 N일 이내의 C108 리포트 도큐먼트를 조회합니다.
+
+    Args:
+        code (str): 종목 코드 (예: "005930").
+        client (AsyncIOMotorClient): 비동기 MongoDB 클라이언트.
+        days (int, optional): 현재 시점에서 며칠 전까지의 데이터를 조회할지 설정합니다. 기본값은 30일.
+
+    Returns:
+        list[C108]: 조건에 해당하는 C108 도큐먼트 리스트. 일치하는 문서가 없을 경우 빈 리스트를 반환합니다.
+    """
+    collection = get_collection(client, DB_NAME, COL_NAME)
+    now = datetime.now(timezone.utc)
+    cutoff = now - timedelta(days=days)
+
+    cursor = collection.find(
+        {
+            "코드": code,
+            "날짜": {"$gte": cutoff}
+        }
+    ).sort("날짜", DESCENDING)
+
+    docs = await cursor.to_list(length=None)
+    if not docs:
+        return []
+    else:
+        c108s = []
+        for doc in docs:
+            doc["_id"] = str(doc["_id"])
+            c108s.append(C108(**doc))
+        return c108s
+
+

db2_hj3415-0.1.4/db2_hj3415/nfs/models.py

@@ -0,0 +1,147 @@
+from pydantic import BaseModel, Field, field_serializer, ConfigDict
+from datetime import datetime
+
+class C101(BaseModel):
+    id: str | None = Field(alias="_id")
+    날짜: datetime
+    코드: str
+    bps: int | None
+    eps: int | None
+    pbr: float | None
+    per: float | None
+    개요: str | None
+    거래대금: int | None
+    거래량: int | None
+    발행주식: int | None
+    배당수익률: float | None
+    베타52주: float | None
+    수익률: float | None
+    수익률1M: float | None
+    수익률1Y: float | None
+    수익률3M: float | None
+    수익률6M: float | None
+    시가총액: int | None
+    업종: str | None
+    업종per: float | None
+    외국인지분율: float | None
+    유동비율: float | None
+    전일대비: int | None
+    종목명: str | None
+    주가: int | None
+    최고52: int | None
+    최저52: int | None
+
+    @field_serializer("날짜")
+    def serialize_날짜(self, value: datetime) -> str:
+        return value.isoformat()
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        str_strip_whitespace=True,
+    )
+
+class 항목값y(BaseModel):
+    항목: str
+    전년대비: float | None
+    전년대비_1: float | None = Field(default=None, alias="전년대비 1")
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        extra="allow"
+    )
+
+class 항목값q(BaseModel):
+    항목: str
+    전분기대비: float | None
+
+    model_config = ConfigDict(extra="allow")
+
+
+class C103(BaseModel):
+    id: str = Field(alias="_id")
+    코드: str
+    날짜: datetime
+    손익계산서q: list[항목값q]
+    손익계산서y: list[항목값y]
+    재무상태표q: list[항목값q]
+    재무상태표y: list[항목값y]
+    현금흐름표q: list[항목값q]
+    현금흐름표y: list[항목값y]
+
+    @field_serializer("날짜")
+    def serialize_date(self, v: datetime) -> str:
+        return v.isoformat()
+
+    model_config = ConfigDict(populate_by_name=True)
+
+
+class C104(BaseModel):
+    id: str = Field(alias="_id")
+    코드: str
+    날짜: datetime
+
+    수익성y: list[항목값y] | None = None
+    성장성y: list[항목값y] | None = None
+    안정성y: list[항목값y] | None = None
+    활동성y: list[항목값y] | None = None
+    가치분석y: list[항목값y] | None = None
+
+    수익성q: list[항목값q] | None = None
+    성장성q: list[항목값q] | None = None
+    안정성q: list[항목값q] | None = None
+    활동성q: list[항목값q] | None = None
+    가치분석q: list[항목값q] | None = None
+
+    @field_serializer("날짜")
+    def serialize_date(self, v: datetime) -> str:
+        return v.isoformat()
+
+    model_config = ConfigDict(populate_by_name=True)
+
+
+class 기업데이터(BaseModel):
+    항목: str
+    항목2: str
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        extra="allow"
+    )
+
+class C106(BaseModel):
+    id: str = Field(alias="_id")
+    코드: str
+    날짜: datetime
+
+    q: list[기업데이터]
+    y: list[기업데이터]
+
+    @field_serializer("날짜")
+    def serialize_date(self, v: datetime) -> str:
+        return v.isoformat()
+
+    model_config = ConfigDict(populate_by_name=True)
+
+
+class C108(BaseModel):
+    id: str = Field(alias="_id")
+    코드: str
+    날짜: datetime
+
+    제목: str | None = None
+    내용: list[str] | None = None
+    목표가: int | None = None
+    분량: str | None = None
+    작성자: str | None = None
+    제공처: str | None = None
+    투자의견: str | None = None
+
+    # 날짜 필드 ISO 직렬화
+    @field_serializer("날짜")
+    def serialize_date(self, v: datetime) -> str:
+        return v.isoformat()
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        str_strip_whitespace=True
+    )

{db2_hj3415-0.1.2 → db2_hj3415-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "db2_hj3415"
-version = "0.1.2"
+version = "0.1.4"
 authors = [{name = "Hyungjin Kim", email = "hj3415@gmail.com"}]
 description = "Gathering the stock data by playwright"
 readme = "README.md"
@@ -15,6 +15,8 @@ dependencies = [
     "pandas-stubs",
     "deepdiff",
     "utils_hj3415>=3.2.3",
+    "redis",
+    "pydantic",
 ]
 
 [project.scripts]

db2_hj3415-0.1.2/db2_hj3415/common/utils.py

@@ -1 +0,0 @@
-# 자주 쓰는 간단한 유틸 함수