neurostats-API 0.0.22__py3-none-any.whl → 0.0.23__py3-none-any.whl

neurostats_API/fetchers/profit_lose.py

@@ -22,6 +22,16 @@ class ProfitLoseFetcher(StatsFetcher):
             "us_stats": self.process_data_us
         }
 
+        self.return_keys = [
+            'profit_lose', 'grand_total_profit_lose', 'revenue', 'grand_total_revenue', 
+            'gross_profit', 'grand_total_gross_profit', 'gross_profit_percentage', 
+            'grand_total_gross_profit_percentage', 'operating_income', 'grand_total_operating_income', 'operating_income_percentage', 
+            'grand_total_operating_income_percentage', 'net_income_before_tax', 'grand_total_net_income_before_tax', 'net_income_before_tax_percentage', 
+            'grand_total_net_income_before_tax_percentage', 'net_income', 'grand_total_net_income', 'net_income_percentage', 
+            'grand_total_income_percentage', 'EPS', 'EPS_growth', 'grand_total_EPS', 
+            'grand_total_EPS_growth', 'profit_lose_all', 'profit_lose_YoY'
+        ]
+
     def prepare_query(self):
         pipeline = super().prepare_query()
 
@@ -30,28 +40,25 @@ class ProfitLoseFetcher(StatsFetcher):
         chart_name = name_map.get(self.collection_name, "income_statement")
 
         append_pipeline = [
-            {
-                "$unwind": "$seasonal_data"    # 展開 seasonal_data 陣列
-            },
-            {
-                "$project": {
-                    "_id": 0,
-                    "ticker": 1,
-                    "company_name": 1,
-                    "year": "$seasonal_data.year",
-                    "season": "$seasonal_data.season",
-                    "profit_lose": {
-                        "$ifNull": [f"$seasonal_data.{chart_name}", []]
-                    }    # 避免 null
-                }
-            },
-            {
-                "$sort": {
-                    "year": -1,
-                    "season": -1
+        {
+            "$project": {
+                "_id": 0,
+                "ticker": 1,
+                "company_name": 1,
+                "seasonal_data": {
+                    "$map": {
+                        "input": {"$ifNull": ["$seasonal_data", []]},
+                        "as": "season",
+                        "in": {
+                            "year": "$$season.year",
+                            "season": "$$season.season",
+                            "data": {"$ifNull": [f"$$season.{chart_name}", []]}
+                        }
+                    }
                 }
             }
-        ]
+        }
+    ]
 
         pipeline = pipeline + append_pipeline
 
@@ -63,6 +70,8 @@ class ProfitLoseFetcher(StatsFetcher):
     def query_data(self):
         fetched_data = self.collect_data()
 
+        fetched_data = fetched_data[0]
+
         process_fn = self.process_function_map.get(
             self.collection_name, self.process_data_us
         )
@@ -79,101 +88,108 @@ class ProfitLoseFetcher(StatsFetcher):
             'latest_target_year',
             StatsDateTime.get_today().year - 1
         )
-        target_season = latest_time.get(
-            'seasonal_data',{}
-        ).get('latest_season', 4)
+        target_season = latest_time.get('seasonal_data',
+                                        {}).get('latest_season', 4)
 
         return_dict = {
             "ticker": self.ticker,
-            "company_name": fetched_data[-1]['company_name'],
+            "company_name": fetched_data['company_name'],
         }
 
+        seasonal_data = fetched_data.get('seasonal_data', [])
+
+        if (not seasonal_data):
+            return_dict.update(self._get_empty_structure())
+            return return_dict
+
         profit_lose_dict = {
-            f"{data['year']}Q{data['season']}": data['profit_lose'] 
-            for data in fetched_data
+            f"{data['year']}Q{data['season']}": data['data']
+            for data in seasonal_data
         }
 
+        profit_lose_dict = YoY_Calculator.cal_QoQ(profit_lose_dict)
         profit_lose_df = pd.DataFrame.from_dict(profit_lose_dict)
         target_season_col = profit_lose_df.columns.str.endswith(
             f"Q{target_season}"
         )
         profit_lose_df = profit_lose_df.loc[:, target_season_col]
-        profit_lose_df = StatsProcessor.expand_value_percentage(
+
+        old_profit_lose_df = StatsProcessor.expand_value_percentage(
             profit_lose_df
         )
-
-        value_col = profit_lose_df.columns.str.endswith(f"_value")
-        percentage_col = profit_lose_df.columns.str.endswith(f"_percentage")
-
-        grand_total_value_col = profit_lose_df.columns.str.endswith(
+        # OLD: 回傳包含value & percentage
+        value_col = old_profit_lose_df.columns.str.endswith(f"_value")
+        percentage_col = old_profit_lose_df.columns.str.endswith(f"_percentage")
+        # OLD: 回傳剔除grand_total
+        grand_total_value_col = old_profit_lose_df.columns.str.endswith(
             f"grand_total_value"
         )
-        grand_total_percentage_col = profit_lose_df.columns.str.endswith(
+        grand_total_percentage_col = old_profit_lose_df.columns.str.endswith(
             f"grand_total_percentage"
         )
 
-        profit_lose_stats_df = profit_lose_df.loc[:, (
+        old_profit_lose_df = old_profit_lose_df.loc[:, (
             (value_col & ~grand_total_value_col) |
             (percentage_col & ~grand_total_percentage_col)
         )]
 
-        for time_index, profit_lose in profit_lose_dict.items():
-            # 蒐集整體的keys
-            index_names = list(profit_lose.keys())
-            target_keys = [
-                "value",
-                "percentage",
-                "grand_total",
-                "grand_total_percentage",
-                "YoY_1",
-                "YoY_3",
-                "YoY_5",
-                "YoY_10",
-                "grand_total_YoY_1",
-                "grand_total_YoY_3",
-                "grand_total_YoY_5",
-                "grand_total_YoY_10",
-            ]
-            # flatten dict
-            new_profit_lose = self.flatten_dict(
-                profit_lose, index_names, target_keys
+        for time_index, data_dict in profit_lose_dict.items():
+            profit_lose_dict[time_index] = self.flatten_dict(
+                value_dict=data_dict,
+                indexes=list(data_dict.keys()),
+                target_keys=[
+                    "value", "growth", "percentage", "grand_total",
+                    "grand_total_percentage"
+                ] + [f"YoY_{i}" for i in [1, 3, 5, 10]] +
+                [f"grand_total_YoY_{i}" for i in [1, 3, 5, 10]]
             )
-            profit_lose_dict[time_index] = new_profit_lose
-
-        profit_lose_df = pd.DataFrame.from_dict(profit_lose_dict)
 
+        profit_lose_df = pd.DataFrame.from_dict(profit_lose_dict).T
         # EPS的value用元計算
-        eps_index = profit_lose_df.index.str.endswith(
-            "_value"
-        ) & profit_lose_df.index.str.contains("每股盈餘")
-        profit_lose_df.loc[eps_index] = profit_lose_df.loc[
-            eps_index].apply(
-                lambda x: StatsProcessor.cal_non_percentage(x, postfix="元")
-            )
+        eps_index = (
+            profit_lose_df.columns.str.endswith("_value")
+            & profit_lose_df.columns.str.contains("每股盈餘")
+        )
+        eps_copy = profit_lose_df.loc[:, eps_index].copy()
+        eps_mask_index = eps_copy.columns
+        profit_lose_df[eps_mask_index] = profit_lose_df[eps_mask_index].map(
+            lambda x: StatsProcessor.cal_non_percentage(x, postfix="元")
+        )
 
         # percentage處理
-        percentage_index = profit_lose_df.index.str.endswith("percentage")
-        profit_lose_df.loc[percentage_index] = profit_lose_df.loc[
-            percentage_index].apply(
+        percentage_index = profit_lose_df.columns.str.endswith("percentage")
+        growth_index = profit_lose_df.columns.str.endswith("growth")
+        percentage_mask = (percentage_index | growth_index)
+        percentage_copy = profit_lose_df.loc[:, percentage_mask]
+        percentage_mask_index = percentage_copy.columns
+
+        profit_lose_df[percentage_mask_index] = profit_lose_df[
+            percentage_mask_index].map(
                 lambda x: StatsProcessor.
                 cal_non_percentage(x, to_str=True, postfix="%")
             )
 
         # YoY處理: 乘以100
-        YoY_index = profit_lose_df.index.str.contains("YoY")
-        profit_lose_df.loc[YoY_index] = profit_lose_df.loc[
-            YoY_index].apply(lambda x: StatsProcessor.cal_percentage(x))
+        YoY_index = profit_lose_df.columns.str.contains("YoY")
+        YoY_mask = YoY_index
+        YoY_copy = profit_lose_df.loc[:, YoY_mask]
+        YoY_mask_cols = YoY_copy.columns
+
+        profit_lose_df[YoY_mask_cols] = profit_lose_df[YoY_mask_cols].map(
+            lambda x: StatsProcessor.cal_percentage(x)
+        )
 
         # 剩下的處理: 乘以千元
         value_index = ~(
-            percentage_index | YoY_index | profit_lose_df.index.isin(eps_index)
+            percentage_index | growth_index | YoY_index | eps_index
         )    # 除了上述以外的 index
-        profit_lose_df.loc[value_index] = profit_lose_df.loc[
-            value_index].apply(
-                lambda x: StatsProcessor.cal_non_percentage(x, postfix="千元")
-            )
 
-        total_table = profit_lose_df.replace("N/A", None)
+        value_col = profit_lose_df.loc[:, value_index].columns
+        profit_lose_df[value_col] = profit_lose_df[value_col].map(
+            lambda x: StatsProcessor.cal_non_percentage(x, postfix="千元")
+        )
+
+        total_table = profit_lose_df.replace("N/A", None).T
 
         # 取特定季度
         target_season_columns = total_table.columns.str.endswith(
@@ -192,12 +208,11 @@ class ProfitLoseFetcher(StatsFetcher):
                     )
                     break
                 except Exception as e:
-                    print(str(e))
                     continue
 
         return_dict.update(
             {
-                "profit_lose": profit_lose_stats_df,
+                "profit_lose": old_profit_lose_df,
                 "profit_lose_all": total_table.copy(),
                 "profit_lose_YoY": total_table_YoY
             }
@@ -218,8 +233,8 @@ class ProfitLoseFetcher(StatsFetcher):
             table_dict[time_index] = self.flatten_dict(
                 value_dict=data_dict,
                 indexes=list(data_dict.keys()),
-                target_keys=["value", "growth"] + 
-                            [f"YoY_{i}" for i in [1, 3, 5, 10]]
+                target_keys=["value", "growth"] +
+                [f"YoY_{i}" for i in [1, 3, 5, 10]]
             )
 
         # 計算QoQ
@@ -231,3 +246,8 @@ class ProfitLoseFetcher(StatsFetcher):
         }
 
         return return_dict
+
+    def _get_empty_structure(self):
+        return {
+            key: pd.DataFrame(columns= pd.Index([], name = 'date')) for key in self.return_keys
+        }

neurostats_API/fetchers/tech.py

@@ -13,7 +13,8 @@ class TechFetcher(StatsFetcher):
 
         super().__init__(ticker, db_client)
         if (ticker in self.tw_company_list.keys()):
-            self.collection = self.db["TWN/APIPRCD"]
+            self.twse_collection = self.db['twse_stats']
+            self.tej_collection = self.db["TWN/APIPRCD"]
 
         self.full_ohlcv = self._get_ohlcv()
         self.basic_indexes = [
@@ -53,35 +54,47 @@ class TechFetcher(StatsFetcher):
 
         required_cols = ['date', 'open', 'high', 'low', 'close', 'volume']
 
-        try:
-            # 先對yf search
-            if self.ticker in ['GSPC', 'IXIC', 'DJI', 'TWII']:
-                full_tick = f'^{self.ticker}'
-            elif(self.ticker in self.tw_company_list.keys()):
-                full_tick = f'{self.ticker}.tw'
-            else:
-                full_tick = f"{self.ticker}"
-
+        if self.ticker in ['GSPC', 'IXIC', 'DJI', 'TWII']:
+            full_tick = f'^{self.ticker}'
             df = self.conduct_yf_search(full_tick)
 
-            if (
-                self.ticker in self.tw_company_list.keys() and
-                not self.has_required_columns(df, required_cols)
-            ):
-                full_tick = f'{self.ticker}.two'
+            return df[required_cols]
 
-                df = self.conduct_yf_search(full_tick)
+        elif(self.ticker in self.tw_company_list.keys()):
+            search_fns = [
+                self.conduct_db_search_twse, 
+                self.conduct_db_search_tej, 
+                lambda: self.conduct_yf_search(f'{self.ticker}.tw'),
+                lambda: self.conduct_yf_search(f'{self.ticker}.two')
+            ]
+
+            for search_method in search_fns:
+                try:
+                    df = search_method()
+                    break
+                except (KeyError, ValueError, TypeError):
+                    continue
+            else:
+                return pd.DataFrame(columns=required_cols)
 
-                if (df.empty):
-                    raise ValueError(f"No data found for ticker: {self.ticker}")\
-            
+            # break跳出後
             return df[required_cols]
+        else:  # 美股
+            search_fns = [
+                self.conduct_db_search_us,
+                lambda : self.conduct_yf_search(f"{self.ticker}")
+            ]
+            for search_method in search_fns:
+                try:
+                    df = search_method()
+                    break
+                except (KeyError, ValueError, TypeError):
+                    continue
+            else:
+                df = pd.DataFrame()
 
-        except (KeyError, ValueError, TypeError) as e:
-            if (self.collection_name in ['TWN/APIPRCD']):
-                return self.conduct_db_search_tej()
-            elif (self.collection_name == 'us_stats'):
-                return self.conduct_db_search_us()
+            return df
+        
 
     def get_daily(self):
 
@@ -175,13 +188,35 @@ class TechFetcher(StatsFetcher):
         if not self.has_required_columns(df, required_cols):
             missing_cols = [col for col in required_cols if col not in df.columns]
             missing_cols = ",".join(missing_cols)
-            print(Warning(f"{missing_cols} not in columns"))
             for col in missing_cols:
                 df[col] = pd.NA
-            # raise KeyError(f"Missing required columns")
 
         return df[required_cols]
 
+    
+    def conduct_db_search_twse(self):
+        required_cols = ['date', 'open', 'high', 'low', 'close', 'volume']
+        match_query = {"ticker" : self.ticker}
+        proj_query = {"_id": 0, "daily_data": 1}
+
+        full_data = self.twse_collection.find_one(match_query, proj_query)
+
+        if (not full_data):
+            raise ValueError("No ticker found in database twse_stats")
+
+        daily_data = full_data.get("daily_data", [])
+
+        if (not isinstance(daily_data, list)):
+            raise ValueError("No ticker found in database twse_stats")
+
+        df = pd.DataFrame(daily_data)
+        if not self.has_required_columns(df, required_cols):
+            raise KeyError(f"Missing required columns")
+        
+        df = df[required_cols]
+        df = df.sort_values(by = 'date').drop_duplicates(subset=['date'])
+
+        return df
 
 class TechProcessor:
 

neurostats_API/fetchers/tej_finance_report.py

@@ -1,6 +1,7 @@
 from .base import BaseTEJFetcher
 from datetime import datetime
 from enum import Enum
+import numpy as np
 import pandas as pd
 from pymongo import MongoClient
 from .tech import TechProcessor
@@ -176,7 +177,10 @@ class FinanceReportFetcher(BaseTEJFetcher):
         fetched_data = self.collection.aggregate(pipeline).to_list()
         data_dict = self.transform_value(
             StatsProcessor.list_of_dict_to_dict(
-                fetched_data, ["year", "season"], "Q", report_type
+                data_list=fetched_data,
+                keys=["year", "season"],
+                delimeter="Q",
+                data_key=report_type
             )
         )
 
@@ -195,9 +199,9 @@ class FinanceReportFetcher(BaseTEJFetcher):
                                   (self.percent_index_list, "%")]:
             process_list = list(set(data_df.index) & set(category))
             if postfix == "%":
-                data_df.loc[process_list] = data_df.loc[process_list].astype(
-                    str
-                ).map(lambda x: f"{x}%")
+                data_df = data_df.T
+                data_df[process_list] = data_df[process_list].map(lambda x: f"{x}%") # if (not np.isnan(x)) else None)
+                data_df = data_df.T
             else:
                 data_df.loc[process_list] = data_df.loc[process_list].map(
                     lambda x: StatsProcessor.
@@ -241,7 +245,7 @@ class FinanceReportFetcher(BaseTEJFetcher):
                     "$gt": start_year,
                     "$lt": end_year
                 },
-                "data.season": end_season
+                "data.season": start_season
             }
         else:
             match_stage = {
@@ -327,7 +331,10 @@ class TEJStockPriceFetcher(BaseTEJFetcher):
 
         if (period is not None):
             latest_date = self.get_latest_data_time(ticker)
-            start_date = self.set_time_shift(date=latest_date, period=period)
+            if (latest_date):
+                start_date = self.set_time_shift(date=latest_date, period=period)
+            else:
+                start_date = datetime.strptime(start_date, "%Y-%m-%d")
         else:
             start_date = datetime.strptime(start_date, "%Y-%m-%d")
 
@@ -355,7 +362,15 @@ class TEJStockPriceFetcher(BaseTEJFetcher):
         datas = self.collection.aggregate(pipeline).to_list()
 
         elements = [element['data'] for element in datas]
-
-        data_df = pd.DataFrame(elements).set_index('mdate')
+        try:
+            data_df = pd.DataFrame(elements).set_index('mdate')
+        except:
+            column_names = [
+                "coid", "mdate", "mkt", "open_d", "high_d", "low_d", "close_d",
+                "adjfac", "vol", "amt", "trn", "bid", "offer", "avgprc", "roi",
+                "hmlpct", "turnover", "shares", "mktcap", "mktcap_pct",
+                "amt_pct", "per", "pbr", "div_yid", "cdiv_yid"
+            ]
+            data_df = pd.DataFrame(columns = column_names)
 
         return data_df

neurostats_API/fetchers/value_invest.py

@@ -8,6 +8,11 @@ class ValueFetcher(StatsFetcher):
 
     def __init__(self, ticker: str, db_client):
         super().__init__(ticker, db_client)
+        self.value_keys = [
+            "P_E", "P_FCF", "P_B", "P_S", 
+            "EV_OPI", "EV_EBIT", "EV_EBITDA", "EV_S", 
+            "Yield"
+        ]
 
     def prepare_query(self, start_date, end_date):
         pipeline = super().prepare_query()
@@ -54,7 +59,8 @@ class ValueFetcher(StatsFetcher):
                                                 "EV_OPI": "$$daily_item.EV_OPI",
                                                 "EV_EBIT": "$$daily_item.EV_EBIT",
                                                 "EV_EBITDA": "$$daily_item.EV_EBITDA",
-                                                "EV_S": "$$daily_item.EV_S"
+                                                "EV_S": "$$daily_item.EV_S",
+                                                "Yield": "$$daily_item.Yield"
                                             }
                                     }
                             },
@@ -89,10 +95,13 @@ class ValueFetcher(StatsFetcher):
         fetched_data = self.collect_data(start_date, end_date)
 
         fetched_data['daily_data'] = fetched_data['daily_data'][-1]
+        for key in self.value_keys:
+            fetched_data['daily_data'][key] = fetched_data['daily_data'].get(key, None)
 
         fetched_data['yearly_data'] = ValueProcessor.transform_to_df(
             fetched_data['daily_data'],
             fetched_data['yearly_data'],
+            required_keys=self.value_keys
         )
 
         return fetched_data
@@ -110,7 +119,8 @@ class ValueFetcher(StatsFetcher):
                 EV_OPI,
                 EV_EBIT,
                 EV_EBITDA,
-                EV_S
+                EV_S,
+                Yield
             }
         """
 
@@ -142,7 +152,8 @@ class ValueFetcher(StatsFetcher):
                                                 "EV_OPI": "$$daily.EV_OPI",
                                                 "EV_EBIT": "$$daily.EV_EBIT",
                                                 "EV_EBITDA": "$$daily.EV_EBITDA",
-                                                "EV_S": "$$daily.EV_S"
+                                                "EV_S": "$$daily.EV_S",
+                                                "Yield": "$$daily.Yield"
                                             }
                                     }
                             }
@@ -153,15 +164,12 @@ class ValueFetcher(StatsFetcher):
         fetched_data = self.collection.aggregate(pipeline).to_list()
         fetched_data = fetched_data[0]
 
-        value_keys = ["P_E", "P_FCF", "P_B", "P_S", "EV_OPI", "EV_EBIT", "EV_EVITDA", "EV_S"]
-        return_dict = {value_key: dict() for value_key in value_keys}
+        return_dict = {value_key: dict() for value_key in self.value_keys}
+
+        for data in fetched_data['daily_data']:
+            for value_key in self.value_keys:
+                return_dict[value_key].update({data['date']: data.get(value_key, None)})
 
-        for value_key in value_keys:
-            for data in fetched_data['daily_data']:
-                if (value_key not in data.keys()):
-                    continue
-                else:
-                    return_dict[value_key].update({data['date']: data[value_key]})
 
         return_dict = {
             value_key: pd.DataFrame.from_dict(value_dict, orient='index', columns=[value_key])
@@ -173,7 +181,7 @@ class ValueFetcher(StatsFetcher):
 class ValueProcessor(StatsProcessor):
 
     @staticmethod
-    def transform_to_df(daily_dict, yearly_dict):
+    def transform_to_df(daily_dict: dict, yearly_dict: dict, required_keys: list):
         latest_data = {"year": f"過去4季"}
 
         latest_data.update(daily_dict)
@@ -184,4 +192,16 @@ class ValueProcessor(StatsProcessor):
 
         yearly_dict = pd.DataFrame.from_dict(yearly_dict)
 
+        total_keys = [
+            # 年度評價
+            'year', 'P_E', 'P_FCF', 'P_B', 'P_S', 
+            'EV_OPI', 'EV_EBIT', 'EV_EBITDA','EV_S', 
+            
+            # 最新一日交易
+            'open', 'high', 'low', 'close', 'volume', 'dividends',
+            'stock splits', 'Yield', 'dividend_year', 'finance_report_time'
+       ]
+
+        total_keys_list = list(total_keys)
+        yearly_dict = yearly_dict.reindex(columns=total_keys, fill_value=None)
         return yearly_dict

neurostats_API/tools/tej_db/tej_db_percent_index.yaml

@@ -29,9 +29,6 @@ TWN/AINVFQ1:
 - r607
 - r613
 - r612
-- r609
-- r614
-- r611
 TWN/AFESTM1:
 - r105
 - r106

neurostats_API/tools/tej_db/tej_db_skip_index.yaml

@@ -9,6 +9,13 @@ TWN/AINVFQ1:
 - annd
 - fin_ind
 - eps
+- r307
+- r305
+- r306
+- r316
+- r609
+- r614
+- r611
 TWN/AFESTM1:
 - coid
 - mdate
@@ -19,4 +26,8 @@ TWN/AFESTM1:
 - curr
 - annd
 - fin_ind
-- eps
+- eps
+- r307
+- r305
+- r306
+- r316

neurostats_API/tools/tej_db/tej_db_thousand_index.yaml

@@ -56,10 +56,6 @@ TWN/AINVFQ1:
 - person
 - shares
 - wavg
-- r304
-- r305
-- r306
-- r316
 - r834
 TWN/AFESTM1:
 - ip12

neurostats_API/utils/calculate_value.py

@@ -1,4 +1,5 @@
 from .data_process import StatsProcessor
+import numpy as np
 class YoY_Calculator:
     def __init__(self):
         pass
@@ -23,6 +24,9 @@ class YoY_Calculator:
 
         if (isinstance(YoY, complex)): # 年化成長率有複數問題
             return None
+        
+        if YoY != YoY:  # 確認是否為nan (Python 中 nan不等於自己)
+            return None
 
         return YoY
     @classmethod
@@ -91,7 +95,6 @@ class YoY_Calculator:
             for key in list(this_data.keys()):
                 if key == "season":
                     continue
-                
 
                 value = this_data.get(key, None)
 
@@ -102,7 +105,7 @@ class YoY_Calculator:
                     temp_dict = {"value": value}
                     this_value = value
                 
-                this_value = StatsProcessor.process_str_to_float(this_data[key])
+                this_value = StatsProcessor.process_str_to_float(this_value)
                 
                 last_value = data_dict.get(
                     f"{last_year}Q{last_season}",{}