neurostats-API 0.0.21b0__py3-none-any.whl → 0.0.23b0__py3-none-any.whl

neurostats_API/__init__.py

@@ -1,4 +1,4 @@
-__version__='0.0.20'
+__version__='0.0.23b'
 
 from .fetchers import (
     BalanceSheetFetcher,

neurostats_API/fetchers/balance_sheet.py

@@ -1,9 +1,7 @@
 from .base import StatsFetcher, StatsDateTime
 import json
-import numpy as np
 import pandas as pd
 from ..utils import StatsDateTime, StatsProcessor
-import importlib.resources as pkg_resources
 import yaml
 
 
@@ -14,138 +12,167 @@ class BalanceSheetFetcher(StatsFetcher):
 
     def __init__(self, ticker, db_client):
         super().__init__(ticker, db_client)
-        self.table_settings = StatsProcessor.load_yaml("balance_sheet.yaml")
+        self.table_settings = StatsProcessor.load_yaml("twse/balance_sheet.yaml")
 
-    def prepare_query(self, target_year, target_season):
-        pipeline = super().prepare_query()
-
-        target_query = {
-            "year": "$$target_season_data.year",
-            "season": "$$target_season_data.season",
-            "balance_sheet": "$$$$target_season_data.balance_sheet"
+        self.process_function_map = {
+            "twse_stats": self.process_data_twse,
+            "us_stats": self.process_data_us
         }
 
-        pipeline.append({
-            "$project": {
-                "_id": 0,
-                "ticker": 1,
-                "company_name": 1,
-                "balance_sheets": {
-                    "$sortArray": {
-                        "input": {
-                            "$map": {
-                                "input": {
-                                    "$filter": {
-                                        "input": "$seasonal_data",
-                                        "as": "season",
-                                        "cond": {
-                                            "$eq":
-                                            ["$$season.season", target_season]
-                                        }
-                                    }
-                                },
-                                "as": "target_season_data",
-                                "in": {
-                                    "year":
-                                    "$$target_season_data.year",
-                                    "season":
-                                    "$$target_season_data.season",
-                                    "balance_sheet":
-                                    "$$target_season_data.balance_sheet"
-                                }
-                            }
-                        },
-                        "sortBy": {
-                            "year": -1
-                        }  # 按 year 降序排序
-                    }
+    def prepare_query(self):
+        pipeline = super().prepare_query()
+
+        pipeline = pipeline + [
+            {
+                "$unwind": "$seasonal_data"    # 展開 seasonal_data 陣列
+            },
+            {
+                "$project": {
+                    "_id": 0,
+                    "ticker": 1,
+                    "company_name": 1,
+                    "year": "$seasonal_data.year",
+                    "season": "$seasonal_data.season",
+                    "balance_sheet": {
+                        "$ifNull": ["$seasonal_data.balance_sheet", []]
+                    }    # 避免 null
+                }
+            },
+            {
+                "$sort": {
+                    "year": -1,
+                    "season": -1
                 }
             }
-        })
+        ]
 
         return pipeline
 
-    def collect_data(self, target_year, target_season):
-        pipeline = self.prepare_query(target_year, target_season)
-
-        fetched_data = self.collection.aggregate(pipeline)
-
-        fetched_data = list(fetched_data)
-
-        return fetched_data[-1]
+    def collect_data(self):
+        return super().collect_data()
 
     def query_data(self):
-        try:
-            latest_time = StatsDateTime.get_latest_time(
-                self.ticker, self.collection)['last_update_time']
-            year = latest_time['seasonal_data']['latest_year']
-            season = latest_time['seasonal_data']['latest_season']
-        except Exception as e:
-            today = StatsDateTime.get_today()
-            year = today.year - 1 if (today.season == 1) else today.year
-            season = 4 if (today.season == 1) else today.season - 1
-
-        fetched_data = self.collect_data(year, season)
+        fetched_data = self.collect_data()
+
+        process_fn = self.process_function_map[self.collection_name]
+        processed_data = process_fn(fetched_data)
+        return processed_data
+
+    def process_data_twse(self, fetched_data):
+        latest_time = StatsDateTime.get_latest_time(
+            self.ticker, self.collection
+        ).get('last_update_time', {})
+        # 取最新時間資料時間，沒取到就預設去年年底
+        target_year = latest_time.get('seasonal_data', {}).get(
+            'latest_target_year',
+            StatsDateTime.get_today().year - 1
+        )
+        target_season = latest_time.get('seasonal_data',
+                                        {}).get('latest_season', 4)
 
-        return self.process_data(season, fetched_data)
-
-    def process_data(self, target_season, fetched_data):
         return_dict = {
             "ticker": self.ticker,
-            "company_name": fetched_data['company_name']
+            "company_name": fetched_data[-1]['company_name']
         }
-
-        index_names = []
-
-        table_dict = dict()
-
-        balance_sheets = fetched_data['balance_sheets']
+        table_dict = {}
 
         # 將value與percentage跟著年分季度一筆筆取出
-        for data in balance_sheets:
-            year = data['year']
-
-            time_index = f"{year}Q{target_season}"
+        for data in fetched_data:
+            year, season, balance_sheet = data['year'], data['season'], data[
+                'balance_sheet']
+            time_index = f"{year}Q{season}"
 
+            new_balance_sheet = dict()
             # 蒐集整體的keys
-            index_names += list(data['balance_sheet'].keys())
-            balance_sheet = data['balance_sheet']
-
-            for index_name, value_dict in balance_sheet.items():
-                for item_name, item in value_dict.items():
-                    try:  # table_dict[項目][(2020Q1, '%')]
-                        if (item_name == 'percentage'):
-                            if (isinstance(item, (float, int))):
-                                item = StatsProcessor.cal_non_percentage(item, to_str=True, postfix="%")
-                        elif ("YoY" in item_name):
-                            if (isinstance(item, (float, int))):
-                                item = StatsProcessor.cal_percentage(item)
-                        else:
-                            if (isinstance(item, (float, int))):
-                                item = StatsProcessor.cal_non_percentage(item,  postfix="千元")
-                        table_dict[index_name][(time_index, item_name)] = item
-
-                    except KeyError:
-                        if (index_name not in table_dict.keys()):
-                            table_dict[index_name] = dict()
-
-                        table_dict[index_name][(time_index, item_name)] = item
-
-        total_table = pd.DataFrame.from_dict(table_dict, orient='index')
-        total_table.columns = pd.MultiIndex.from_tuples(total_table.columns)
+            index_names = list(balance_sheet.keys())
+
+            table_dict[time_index] = balance_sheet
+            # flatten dict:
+            # {<key>: {"value": <value>, "percentage": <value>}}
+            # -> {<key>_value: <value>, <key>_percentage:<value>}
+        
+        old_balance_sheet = pd.DataFrame(table_dict)
+        target_season_col = old_balance_sheet.columns.str.endswith(f"Q{target_season}")
+        old_balance_sheet = old_balance_sheet.loc[:, target_season_col]
+        old_balance_sheet = StatsProcessor.expand_value_percentage(old_balance_sheet)
+
+        for time_index, data_dict in table_dict.items():
+            new_balance_sheet = self.flatten_dict(
+                balance_sheet, 
+                indexes = index_names,
+                target_keys=["value", "percentage"]
+            )
+
+            table_dict[time_index] = new_balance_sheet
+
+        total_table = pd.DataFrame.from_dict(table_dict)
+        value_index = total_table.index.str.endswith("_value")
+        total_table.loc[value_index, :] = (
+            total_table.loc[value_index, :].apply(
+                lambda x: StatsProcessor.cal_non_percentage(x, postfix="元"),
+                axis=0
+            )
+        )
+
+        percenrage_index = total_table.index.str.endswith(
+            "_percentage"
+        )
+        total_table.loc[percenrage_index, :] = (
+            total_table.loc[percenrage_index, :].apply(
+                lambda x: StatsProcessor.
+                cal_non_percentage(x, to_str=True, postfix="%"),
+                axis=0
+            )
+        )
+
+        target_season_columns = total_table.columns.str.endswith(
+            f"Q{target_season}"
+        )
+        total_table_YoY = total_table.loc[:, target_season_columns]
 
         for name, setting in self.table_settings.items():
-            if ('target_index' in setting.keys()):
-                target_indexes = [target_index.strip() for target_index in setting['target_index']]
-            else:
-                target_indexes = [None]
+            target_indexes = setting.get('target_index', [None])
             for target_index in target_indexes:
                 try:
-                    return_dict[name] = StatsProcessor.slice_multi_col_table(
-                        total_table=total_table,
-                        mode=setting['mode'],
-                        target_index=target_index)
+                    return_dict[name] = StatsProcessor.slice_old_table(
+                        total_table=old_balance_sheet,
+                        target_index=target_index
+                    )
                     break
                 except Exception as e:
+                    print(f"error : {str(e)}")
                     continue
+        
+        return_dict.update(
+            {
+                "balance_sheet": old_balance_sheet,
+                "balance_sheet_all": total_table.copy(),
+                "balance_sheet_YoY": total_table_YoY
+            }
+        )
+        return return_dict
+
+    def process_data_us(self, fetched_data):
+        return_dict = {
+            "ticker": self.ticker,
+            "company_name": fetched_data[-1]['company_name']
+        }
+
+        table_dict = dict()
+
+        for data in fetched_data:
+            year, season, balance_sheet = data['year'], data['season'], data[
+                'balance_sheet']
+            table_dict[f"{year}Q{season}"] = balance_sheet
+
+        table_dict = pd.DataFrame.from_dict(table_dict)
+
+        return_dict["balance_sheet"] = table_dict
+
+        latest_season = fetched_data[0]['season']
+        target_season_columns = table_dict.columns.str.endswith(
+            f"Q{latest_season}"
+        )
+        table_dict_YoY = table_dict.loc[:, target_season_columns]
+        return_dict["balance_sheet_YoY"] = table_dict_YoY
         return return_dict

neurostats_API/fetchers/base.py

@@ -1,22 +1,31 @@
 import abc
-from typing import Union
-from pymongo import MongoClient
-import pandas as pd
+from datetime import datetime, timedelta, date
 import json
+import pandas as pd
+from pymongo import MongoClient
 import pytz
-from datetime import datetime, timedelta, date
+from typing import Union
 from ..utils import StatsDateTime, StatsProcessor, YoY_Calculator
-import yaml
 
 
-class StatsFetcher:
+class StatsFetcher(abc.ABC):
 
-    def __init__(self, ticker, db_client):
+    def __init__(self, ticker: str, db_client: MongoClient):
         self.ticker = ticker
-        self.db = db_client["company"]    # Replace with your database name
-        self.collection = self.db["twse_stats"]
-
         self.timezone = pytz.timezone("Asia/Taipei")
+        self.tw_company_list = StatsProcessor.load_json("company_list/tw.json")
+        db_mapping = {
+            "company": "twse_stats",
+            "company_us": "us_stats",
+        }
+
+        name_mapping = {"company": "台股", "company_us": "美股"}
+
+        db_name = "company" if self.ticker in self.tw_company_list else "company_us"
+        self.db = db_client[db_name]
+        self.collection_name = db_mapping.get(db_name, "unknown")
+        assert self.collection_name != "unknown", f"請確認 {ticker} 是否是 {','.join(list(name_mapping.values()))}"
+        self.collection = db_client[db_name][self.collection_name]
 
         self.target_metric_dict = {
             'value': ['value'],
@@ -37,40 +46,41 @@ class StatsFetcher:
                 }
             },
         ]
+    
+    def query_data(self):
+        return NotImplementedError()
 
-    def collect_data(self, start_date, end_date):
+    def collect_data(self):
         pipeline = self.prepare_query()
-
         fetched_data = list(self.collection.aggregate(pipeline))
+        return fetched_data if fetched_data else None
 
-        return fetched_data[0]
-
-    def str_to_datetime(self, date_str):
-        year, month, day = [int(num) for num in date_str.split("-")]
-
-        date = datetime.strptime(date_str, "%Y-%m-%d")
-        date = self.timezone.localize(date)
-
+    def str_to_datetime(self, date_str: str) -> StatsDateTime:
+        date = self.timezone.localize(datetime.strptime(date_str, "%Y-%m-%d"))
+        year, month, day = date.year, date.month, date.day
         season = (month - 1) // 3 + 1
-
         return StatsDateTime(date, year, month, day, season)
 
-    def has_required_columns(self, df: pd.DataFrame, required_cols=None):
-        """
-        Check if the required columns are present in the DataFrame.
-
-        Args:
-            df (pd.DataFrame): The DataFrame to check.
-            required_cols (list, optional): List of required column names. 
-                                            Defaults to ['date', 'open', 'high', 'low', 'close', 'volume'].
-
-        Returns:
-            bool: True if all required columns are present, False otherwise.
-        """
+    def has_required_columns(
+        self, df: pd.DataFrame, required_cols=None
+    ) -> bool:
         if required_cols is None:
             required_cols = ['date', 'open', 'high', 'low', 'close', 'volume']
-
         return all(col in df.columns for col in required_cols)
+    
+    @staticmethod
+    def flatten_dict(value_dict, indexes, target_keys):
+        indexes = value_dict.keys()
+        new_dict = {}
+
+        for key in indexes:
+            new_dict.update(
+                {
+                    f"{key}_{sub_key}": value_dict[key].get(sub_key, None) 
+                    for sub_key in target_keys
+                }
+            )
+        return new_dict
 
 
 class BaseTEJFetcher(abc.ABC):
@@ -81,21 +91,14 @@ class BaseTEJFetcher(abc.ABC):
 
     def get_latest_data_time(self, ticker):
         latest_data = self.collection.find_one(
-            {
-                "ticker": ticker
-            }, 
-            {
+            {"ticker": ticker}, {
                 "last_update": 1,
                 "_id": 0
             }
         )
 
-        try:
-            latest_date = latest_data['last_update']["latest_data_date"]
-        except Exception as e:
-            latest_date = None
-
-        return latest_date
+        # return 得到最新日期或None
+        return latest_data.get('last_update', {}).get("latest_data_date", None)
 
     def process_value(self, value):
         if isinstance(value, str) and "%" in value:
@@ -107,77 +110,89 @@ class BaseTEJFetcher(abc.ABC):
 
     def calculate_growth(self, this_value, last_value, delta):
         try:
-            return YoY_Calculator.cal_growth(this_value, last_value, delta) * 100
+            return YoY_Calculator.cal_growth(
+                this_value, last_value, delta
+            ) * 100
         except Exception:
             return None
 
-    def cal_YoY(self, data_dict: dict, start_year: int, end_year: int, season: int):
+    def cal_YoY(
+        self, data_dict: dict, start_year: int, end_year: int, season: int
+    ):
         year_shifts = [1, 3, 5, 10]
         return_dict = {}
-        
+
         for year in range(start_year, end_year + 1):
             year_data = data_dict.get(f"{year}Q{season}", {}).copy()
             if not year_data:
                 continue
-            
-            for key in list(year_data.keys()):
+
+            for key, value in list(year_data.items()):
                 if key == "season":
                     continue
-                
-                this_value = self.process_value(year_data[key])
+
+                this_value = self.process_value(value)
                 if this_value is None:
                     year_data.pop(key)
                     continue
-                
-                temp_dict = {"value": year_data[key]}
+
+                temp_dict = {"value": value}
                 for shift in year_shifts:
-                    past_year = year - shift
-                    last_value = data_dict.get(f"{past_year}Q{season}", {}).get(key)
-                    last_value = self.process_value(last_value)
-                    growth = self.calculate_growth(this_value, last_value, shift) if last_value is not None else None
+                    past_value = self.process_value(
+                        data_dict.get(f"{year - shift}Q{season}", {}).get(key)
+                    )
+                    growth = self.calculate_growth(this_value, past_value, shift) if past_value else None
+
+                    temp_dict[
+                        f"YoY_{shift}"
+                    ] = f"{growth:.2f}%" if growth else None
 
-                    temp_dict[f"YoY_{shift}"] = (f"{growth:.2f}%" if growth else None)
                 year_data[key] = temp_dict
-            
             return_dict[f"{year}Q{season}"] = year_data
-        
         return return_dict
 
     def cal_QoQ(self, data_dict):
         return_dict = {}
-        
+
         for time_index, this_data in data_dict.items():
             year, season = map(int, time_index.split("Q"))
-            last_year, last_season = (year - 1, 4) if season == 1 else (year, season - 1)
-            
+            last_year, last_season = (
+                year - 1, 4
+            ) if season == 1 else (year, season - 1)
+
             for key in list(this_data.keys()):
                 if key == "season":
                     continue
-                
+
                 this_value = self.process_value(this_data[key])
                 if this_value is None:
                     this_data.pop(key)
                     continue
-                
+
                 temp_dict = {"value": this_data[key]}
-                last_value = data_dict.get(f"{last_year}Q{last_season}", {}).get(key, {}).get('value')
+                last_value = data_dict.get(
+                    f"{last_year}Q{last_season}",{}
+                ).get(key, {}).get('value')
+
                 last_value = self.process_value(last_value)
-                growth = self.calculate_growth(this_value, last_value, 1) if last_value is not None else None
+                growth = self.calculate_growth(
+                    this_value, last_value, 1
+                ) if last_value is not None else None
                 temp_dict['growth'] = (f"{growth:.2f}%" if growth else None)
-                
+
                 this_data[key] = temp_dict
-            
+
             return_dict[time_index] = this_data
-        
+
         return return_dict
 
     def get_dict_of_df(self, data_dict):
         """
         dict[dict] -> dict[df]
         """
-        for key in data_dict.keys():
-            data_dict[key] = pd.DataFrame.from_dict(data_dict[key])
-        return data_dict
+        return {
+            key: pd.DataFrame.from_dict(data) for key, data in data_dict.items()
+        }
 
     def set_time_shift(self, date: Union[str, datetime], period: str):
         if isinstance(date, str):
@@ -197,4 +212,4 @@ class BaseTEJFetcher(abc.ABC):
         if period == "all":
             return datetime.strptime("1991-01-01", "%Y-%m-%d")
 
-        return date - period_mapping.get(period, timedelta(days=0))  # 預設為不變"
+        return date - period_mapping.get(period, timedelta(days=0))    # 預設為不變"